diff --git "a/checkpoint-50000/trainer_state.json" "b/checkpoint-50000/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-50000/trainer_state.json" @@ -0,0 +1,361086 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9929697740000795, + "global_step": 50000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 1.5873015873015873e-06, + "loss": 10.5641, + "theoretical_loss": 20.84128112621979, + "tokens_seen": 65536 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031746e-06, + "loss": 10.5513, + "theoretical_loss": 17.594664429001284, + "tokens_seen": 131072 + }, + { + "epoch": 0.0, + "learning_rate": 4.7619047619047615e-06, + "loss": 10.4631, + "theoretical_loss": 15.967940417509208, + "tokens_seen": 196608 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-06, + "loss": 10.2495, + "theoretical_loss": 14.92078008066121, + "tokens_seen": 262144 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507936e-06, + "loss": 10.0077, + "theoretical_loss": 14.164680262435617, + "tokens_seen": 327680 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809523e-06, + "loss": 9.7256, + "theoretical_loss": 13.581024797222863, + "tokens_seen": 393216 + }, + { + "epoch": 0.0, + "learning_rate": 1.1111111111111112e-05, + "loss": 9.5124, + "theoretical_loss": 13.11027232607383, + "tokens_seen": 458752 + }, + { + "epoch": 0.0, + "learning_rate": 1.2698412698412699e-05, + "loss": 9.3381, + "theoretical_loss": 12.718592950155966, + "tokens_seen": 524288 + }, + { + "epoch": 0.0, + "learning_rate": 1.4285714285714285e-05, + "loss": 9.1669, + "theoretical_loss": 12.385055788546264, + "tokens_seen": 589824 + }, + { + "epoch": 0.0, + "learning_rate": 1.5873015873015872e-05, + "loss": 8.9763, + "theoretical_loss": 12.09587593170772, + "tokens_seen": 655360 + }, + { + "epoch": 0.0, + "learning_rate": 1.746031746031746e-05, + "loss": 8.8295, + "theoretical_loss": 11.841529555774319, + "tokens_seen": 720896 + }, + { + "epoch": 0.0, + "learning_rate": 1.9047619047619046e-05, + "loss": 8.8699, + "theoretical_loss": 11.615182533379372, + "tokens_seen": 786432 + }, + { + "epoch": 0.0, + "learning_rate": 2.0634920634920636e-05, + "loss": 8.7295, + "theoretical_loss": 11.41177746485943, + "tokens_seen": 851968 + }, + { + "epoch": 0.0, + "learning_rate": 2.2222222222222223e-05, + "loss": 8.6618, + "theoretical_loss": 11.227475026784514, + "tokens_seen": 917504 + }, + { + "epoch": 0.0, + "learning_rate": 2.380952380952381e-05, + "loss": 8.6244, + "theoretical_loss": 11.059297242650555, + "tokens_seen": 983040 + }, + { + "epoch": 0.0, + "learning_rate": 2.5396825396825397e-05, + "loss": 8.5977, + "theoretical_loss": 10.90489141112959, + "tokens_seen": 1048576 + }, + { + "epoch": 0.0, + "learning_rate": 2.6984126984126984e-05, + "loss": 8.4756, + "theoretical_loss": 10.762369064735367, + "tokens_seen": 1114112 + }, + { + "epoch": 0.0, + "learning_rate": 2.857142857142857e-05, + "loss": 8.4981, + "theoretical_loss": 10.63019320090292, + "tokens_seen": 1179648 + }, + { + "epoch": 0.0, + "learning_rate": 3.0158730158730158e-05, + "loss": 8.3172, + "theoretical_loss": 10.507097497029282, + "tokens_seen": 1245184 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031745e-05, + "loss": 8.176, + "theoretical_loss": 10.392027268435971, + "tokens_seen": 1310720 + }, + { + "epoch": 0.0, + "learning_rate": 3.3333333333333335e-05, + "loss": 8.1964, + "theoretical_loss": 10.284095545569386, + "tokens_seen": 1376256 + }, + { + "epoch": 0.0, + "learning_rate": 3.492063492063492e-05, + "loss": 8.1379, + "theoretical_loss": 10.182549877942659, + "tokens_seen": 1441792 + }, + { + "epoch": 0.0, + "learning_rate": 3.650793650793651e-05, + "loss": 7.9969, + "theoretical_loss": 10.08674688640524, + "tokens_seen": 1507328 + }, + { + "epoch": 0.0, + "learning_rate": 3.809523809523809e-05, + "loss": 8.1059, + "theoretical_loss": 9.99613250351292, + "tokens_seen": 1572864 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-05, + "loss": 7.9094, + "theoretical_loss": 9.91022645106575, + "tokens_seen": 1638400 + }, + { + "epoch": 0.0, + "learning_rate": 4.126984126984127e-05, + "loss": 7.7211, + "theoretical_loss": 9.828609916213201, + "tokens_seen": 1703936 + }, + { + "epoch": 0.0, + "learning_rate": 4.2857142857142856e-05, + "loss": 7.7885, + "theoretical_loss": 9.750915671580087, + "tokens_seen": 1769472 + }, + { + "epoch": 0.0, + "learning_rate": 4.4444444444444447e-05, + "loss": 7.7711, + "theoretical_loss": 9.676820083754187, + "tokens_seen": 1835008 + }, + { + "epoch": 0.0, + "learning_rate": 4.603174603174603e-05, + "loss": 7.6633, + "theoretical_loss": 9.606036595818281, + "tokens_seen": 1900544 + }, + { + "epoch": 0.0, + "learning_rate": 4.761904761904762e-05, + "loss": 7.5596, + "theoretical_loss": 9.538310371437493, + "tokens_seen": 1966080 + }, + { + "epoch": 0.0, + "learning_rate": 4.9206349206349204e-05, + "loss": 7.5956, + "theoretical_loss": 9.473413862304575, + "tokens_seen": 2031616 + }, + { + "epoch": 0.0, + "learning_rate": 5.0793650793650794e-05, + "loss": 7.4655, + "theoretical_loss": 9.411143115583098, + "tokens_seen": 2097152 + }, + { + "epoch": 0.0, + "learning_rate": 5.2380952380952384e-05, + "loss": 7.5066, + "theoretical_loss": 9.351314678906622, + "tokens_seen": 2162688 + }, + { + "epoch": 0.0, + "learning_rate": 5.396825396825397e-05, + "loss": 7.3266, + "theoretical_loss": 9.293762991332915, + "tokens_seen": 2228224 + }, + { + "epoch": 0.0, + "learning_rate": 5.555555555555556e-05, + "loss": 7.2918, + "theoretical_loss": 9.238338172117398, + "tokens_seen": 2293760 + }, + { + "epoch": 0.0, + "learning_rate": 5.714285714285714e-05, + "loss": 7.3421, + "theoretical_loss": 9.184904137180933, + "tokens_seen": 2359296 + }, + { + "epoch": 0.0, + "learning_rate": 5.873015873015873e-05, + "loss": 7.2943, + "theoretical_loss": 9.133336987085992, + "tokens_seen": 2424832 + }, + { + "epoch": 0.0, + "learning_rate": 6.0317460317460316e-05, + "loss": 7.277, + "theoretical_loss": 9.083523621206375, + "tokens_seen": 2490368 + }, + { + "epoch": 0.0, + "learning_rate": 6.19047619047619e-05, + "loss": 7.1754, + "theoretical_loss": 9.03536054131647, + "tokens_seen": 2555904 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-05, + "loss": 7.1903, + "theoretical_loss": 8.988752814581996, + "tokens_seen": 2621440 + }, + { + "epoch": 0.0, + "learning_rate": 6.507936507936509e-05, + "loss": 7.1207, + "theoretical_loss": 8.943613171313253, + "tokens_seen": 2686976 + }, + { + "epoch": 0.0, + "learning_rate": 6.666666666666667e-05, + "loss": 7.0332, + "theoretical_loss": 8.899861217150864, + "tokens_seen": 2752512 + }, + { + "epoch": 0.0, + "learning_rate": 6.825396825396825e-05, + "loss": 6.8774, + "theoretical_loss": 8.85742274282614, + "tokens_seen": 2818048 + }, + { + "epoch": 0.0, + "learning_rate": 6.984126984126984e-05, + "loss": 6.9871, + "theoretical_loss": 8.816229117451055, + "tokens_seen": 2883584 + }, + { + "epoch": 0.0, + "learning_rate": 7.142857142857143e-05, + "loss": 6.7777, + "theoretical_loss": 8.776216753584098, + "tokens_seen": 2949120 + }, + { + "epoch": 0.0, + "learning_rate": 7.301587301587302e-05, + "loss": 6.8747, + "theoretical_loss": 8.737326634193474, + "tokens_seen": 3014656 + }, + { + "epoch": 0.0, + "learning_rate": 7.460317460317461e-05, + "loss": 6.6371, + "theoretical_loss": 8.699503893181488, + "tokens_seen": 3080192 + }, + { + "epoch": 0.0, + "learning_rate": 7.619047619047618e-05, + "loss": 6.5967, + "theoretical_loss": 8.662697442408115, + "tokens_seen": 3145728 + }, + { + "epoch": 0.0, + "learning_rate": 7.777777777777778e-05, + "loss": 6.7174, + "theoretical_loss": 8.626859639208966, + "tokens_seen": 3211264 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 7703, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 6.697691917419434, + "objective/train/theoretical_loss": 8.591945989283708, + "objective/train/tokens_used": 23736800, + "theoretical_loss": 8.591945989283708, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507937e-05, + "loss": 6.7105, + "theoretical_loss": 8.591945989283708, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 8.095238095238096e-05, + "loss": 6.599, + "theoretical_loss": 8.557914880567758, + "tokens_seen": 3342336 + }, + { + "epoch": 0.0, + "learning_rate": 8.253968253968255e-05, + "loss": 6.4888, + "theoretical_loss": 8.524727344318643, + "tokens_seen": 3407872 + }, + { + "epoch": 0.0, + "learning_rate": 8.412698412698413e-05, + "loss": 6.4092, + "theoretical_loss": 8.492346840169828, + "tokens_seen": 3473408 + }, + { + "epoch": 0.0, + "learning_rate": 8.571428571428571e-05, + "loss": 6.5305, + "theoretical_loss": 8.460739062345422, + "tokens_seen": 3538944 + }, + { + "epoch": 0.0, + "learning_rate": 8.730158730158731e-05, + "loss": 6.3927, + "theoretical_loss": 8.429871764603359, + "tokens_seen": 3604480 + }, + { + "epoch": 0.0, + "learning_rate": 8.888888888888889e-05, + "loss": 6.2062, + "theoretical_loss": 8.399714601792851, + "tokens_seen": 3670016 + }, + { + "epoch": 0.0, + "learning_rate": 9.047619047619048e-05, + "loss": 6.2861, + "theoretical_loss": 8.370238986183718, + "tokens_seen": 3735552 + }, + { + "epoch": 0.0, + "learning_rate": 9.206349206349206e-05, + "loss": 6.2594, + "theoretical_loss": 8.34141795695797, + "tokens_seen": 3801088 + }, + { + "epoch": 0.0, + "learning_rate": 9.365079365079366e-05, + "loss": 6.0057, + "theoretical_loss": 8.313226061453822, + "tokens_seen": 3866624 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809524e-05, + "loss": 6.1933, + "theoretical_loss": 8.285639246924504, + "tokens_seen": 3932160 + }, + { + "epoch": 0.0, + "learning_rate": 9.682539682539682e-05, + "loss": 6.1101, + "theoretical_loss": 8.258634761722755, + "tokens_seen": 3997696 + }, + { + "epoch": 0.0, + "learning_rate": 9.841269841269841e-05, + "loss": 6.0988, + "theoretical_loss": 8.232191064950612, + "tokens_seen": 4063232 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001, + "loss": 6.0056, + "theoretical_loss": 8.206287743725687, + "tokens_seen": 4128768 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010158730158730159, + "loss": 6.0406, + "theoretical_loss": 8.180905437312258, + "tokens_seen": 4194304 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010317460317460319, + "loss": 5.9078, + "theoretical_loss": 8.15602576745014, + "tokens_seen": 4259840 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010476190476190477, + "loss": 5.7249, + "theoretical_loss": 8.13163127428835, + "tokens_seen": 4325376 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010634920634920635, + "loss": 5.847, + "theoretical_loss": 8.107705357395282, + "tokens_seen": 4390912 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010793650793650794, + "loss": 5.7916, + "theoretical_loss": 8.084232221374057, + "tokens_seen": 4456448 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010952380952380953, + "loss": 5.7336, + "theoretical_loss": 8.06119682566165, + "tokens_seen": 4521984 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011111111111111112, + "loss": 5.7012, + "theoretical_loss": 8.038584838134476, + "tokens_seen": 4587520 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001126984126984127, + "loss": 5.6223, + "theoretical_loss": 8.016382592182019, + "tokens_seen": 4653056 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011428571428571428, + "loss": 5.6187, + "theoretical_loss": 7.994577046944442, + "tokens_seen": 4718592 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001158730158730159, + "loss": 5.5594, + "theoretical_loss": 7.973155750440629, + "tokens_seen": 4784128 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011746031746031746, + "loss": 5.5114, + "theoretical_loss": 7.95210680534016, + "tokens_seen": 4849664 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011904761904761905, + "loss": 5.7208, + "theoretical_loss": 7.931418837156708, + "tokens_seen": 4915200 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012063492063492063, + "loss": 5.6176, + "theoretical_loss": 7.911080964661844, + "tokens_seen": 4980736 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012222222222222224, + "loss": 5.4612, + "theoretical_loss": 7.891082772337269, + "tokens_seen": 5046272 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001238095238095238, + "loss": 5.4758, + "theoretical_loss": 7.871414284700578, + "tokens_seen": 5111808 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001253968253968254, + "loss": 5.4206, + "theoretical_loss": 7.852065942354954, + "tokens_seen": 5177344 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012698412698412698, + "loss": 5.4236, + "theoretical_loss": 7.833028579626806, + "tokens_seen": 5242880 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012857142857142858, + "loss": 5.526, + "theoretical_loss": 7.814293403667714, + "tokens_seen": 5308416 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013015873015873017, + "loss": 5.4883, + "theoretical_loss": 7.795851974908005, + "tokens_seen": 5373952 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013174603174603174, + "loss": 5.3321, + "theoretical_loss": 7.77769618875926, + "tokens_seen": 5439488 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013333333333333334, + "loss": 5.3773, + "theoretical_loss": 7.7598182584719595, + "tokens_seen": 5505024 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013492063492063494, + "loss": 5.3975, + "theoretical_loss": 7.742210699062561, + "tokens_seen": 5570560 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001365079365079365, + "loss": 5.301, + "theoretical_loss": 7.724866312231575, + "tokens_seen": 5636096 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001380952380952381, + "loss": 5.3225, + "theoretical_loss": 7.707778172200822, + "tokens_seen": 5701632 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013968253968253967, + "loss": 5.4532, + "theoretical_loss": 7.6909396124039695, + "tokens_seen": 5767168 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001412698412698413, + "loss": 5.3501, + "theoretical_loss": 7.67434421296992, + "tokens_seen": 5832704 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014285714285714287, + "loss": 5.3752, + "theoretical_loss": 7.657985788943502, + "tokens_seen": 5898240 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014444444444444444, + "loss": 5.1601, + "theoretical_loss": 7.641858379192337, + "tokens_seen": 5963776 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014603174603174603, + "loss": 5.1855, + "theoretical_loss": 7.625956235952921, + "tokens_seen": 6029312 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014761904761904763, + "loss": 5.2364, + "theoretical_loss": 7.610273814972529, + "tokens_seen": 6094848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014920634920634923, + "loss": 5.1903, + "theoretical_loss": 7.594805766207012, + "tokens_seen": 6160384 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001507936507936508, + "loss": 5.2898, + "theoretical_loss": 7.57954692503759, + "tokens_seen": 6225920 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015238095238095237, + "loss": 5.1998, + "theoretical_loss": 7.5644923039725604, + "tokens_seen": 6291456 + }, + { + "epoch": 0.0, + "learning_rate": 0.000153968253968254, + "loss": 4.9998, + "theoretical_loss": 7.549637084802403, + "tokens_seen": 6356992 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015555555555555556, + "loss": 5.2391, + "theoretical_loss": 7.534976611179138, + "tokens_seen": 6422528 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015714285714285716, + "loss": 5.1035, + "theoretical_loss": 7.52050638159289, + "tokens_seen": 6488064 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 8901, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 4.883451461791992, + "objective/train/theoretical_loss": 7.506222042720672, + "objective/train/tokens_used": 27013600, + "theoretical_loss": 7.506222042720672, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015873015873015873, + "loss": 5.0335, + "theoretical_loss": 7.506222042720672, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016031746031746033, + "loss": 5.1059, + "theoretical_loss": 7.492119383124103, + "tokens_seen": 6619136 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016190476190476192, + "loss": 5.2313, + "theoretical_loss": 7.478194327274557, + "tokens_seen": 6684672 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001634920634920635, + "loss": 5.0888, + "theoretical_loss": 7.46444292988566, + "tokens_seen": 6750208 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001650793650793651, + "loss": 4.9131, + "theoretical_loss": 7.450861370534546, + "tokens_seen": 6815744 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001666666666666667, + "loss": 5.1414, + "theoretical_loss": 7.437445948554516, + "tokens_seen": 6881280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016825396825396826, + "loss": 5.0188, + "theoretical_loss": 7.424193078182986, + "tokens_seen": 6946816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016984126984126986, + "loss": 5.0327, + "theoretical_loss": 7.411099283949674, + "tokens_seen": 7012352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017142857142857143, + "loss": 5.1783, + "theoretical_loss": 7.3981611962910145, + "tokens_seen": 7077888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017301587301587302, + "loss": 5.1053, + "theoretical_loss": 7.38537554737773, + "tokens_seen": 7143424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017460317460317462, + "loss": 4.9697, + "theoretical_loss": 7.372739167143329, + "tokens_seen": 7208960 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001761904761904762, + "loss": 5.009, + "theoretical_loss": 7.36024897950216, + "tokens_seen": 7274496 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017777777777777779, + "loss": 5.0185, + "theoretical_loss": 7.347901998746318, + "tokens_seen": 7340032 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017936507936507938, + "loss": 5.0807, + "theoretical_loss": 7.335695326111465, + "tokens_seen": 7405568 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018095238095238095, + "loss": 4.9311, + "theoretical_loss": 7.323626146502205, + "tokens_seen": 7471104 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018253968253968255, + "loss": 5.051, + "theoretical_loss": 7.311691725368293, + "tokens_seen": 7536640 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018412698412698412, + "loss": 5.0423, + "theoretical_loss": 7.299889405723473, + "tokens_seen": 7602176 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018571428571428572, + "loss": 4.9575, + "theoretical_loss": 7.288216605299264, + "tokens_seen": 7667712 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018730158730158731, + "loss": 4.9026, + "theoretical_loss": 7.276670813826511, + "tokens_seen": 7733248 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018888888888888888, + "loss": 4.9363, + "theoretical_loss": 7.265249590437903, + "tokens_seen": 7798784 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019047619047619048, + "loss": 4.9141, + "theoretical_loss": 7.253950561185138, + "tokens_seen": 7864320 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019206349206349208, + "loss": 4.8576, + "theoretical_loss": 7.242771416664761, + "tokens_seen": 7929856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019365079365079365, + "loss": 4.8336, + "theoretical_loss": 7.231709909747042, + "tokens_seen": 7995392 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019523809523809525, + "loss": 4.9158, + "theoretical_loss": 7.220763853402656, + "tokens_seen": 8060928 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019682539682539682, + "loss": 4.8969, + "theoretical_loss": 7.209931118622157, + "tokens_seen": 8126464 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019841269841269844, + "loss": 5.0328, + "theoretical_loss": 7.199209632423606, + "tokens_seen": 8192000 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002, + "loss": 4.7254, + "theoretical_loss": 7.188597375943913, + "tokens_seen": 8257536 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020158730158730158, + "loss": 4.8566, + "theoretical_loss": 7.178092382609779, + "tokens_seen": 8323072 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020317460317460318, + "loss": 4.9601, + "theoretical_loss": 7.1676927363842795, + "tokens_seen": 8388608 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020476190476190475, + "loss": 4.8234, + "theoretical_loss": 7.157396570085435, + "tokens_seen": 8454144 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020634920634920637, + "loss": 4.9181, + "theoretical_loss": 7.147202063773257, + "tokens_seen": 8519680 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020793650793650797, + "loss": 4.8418, + "theoretical_loss": 7.137107443201986, + "tokens_seen": 8585216 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020952380952380954, + "loss": 4.819, + "theoretical_loss": 7.127110978334411, + "tokens_seen": 8650752 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021111111111111113, + "loss": 4.9044, + "theoretical_loss": 7.117210981915334, + "tokens_seen": 8716288 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002126984126984127, + "loss": 4.8086, + "theoretical_loss": 7.10740580810139, + "tokens_seen": 8781824 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002142857142857143, + "loss": 4.7956, + "theoretical_loss": 7.097693851144607, + "tokens_seen": 8847360 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021587301587301587, + "loss": 4.8153, + "theoretical_loss": 7.088073544127209, + "tokens_seen": 8912896 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021746031746031744, + "loss": 4.8548, + "theoretical_loss": 7.07854335774531, + "tokens_seen": 8978432 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021904761904761907, + "loss": 4.6907, + "theoretical_loss": 7.069101799139266, + "tokens_seen": 9043968 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022063492063492066, + "loss": 4.7805, + "theoretical_loss": 7.0597474107685745, + "tokens_seen": 9109504 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022222222222222223, + "loss": 4.7233, + "theoretical_loss": 7.050478769329319, + "tokens_seen": 9175040 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022380952380952383, + "loss": 4.8346, + "theoretical_loss": 7.0412944847122505, + "tokens_seen": 9240576 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002253968253968254, + "loss": 4.7503, + "theoretical_loss": 7.032193198999712, + "tokens_seen": 9306112 + }, + { + "epoch": 0.0, + "learning_rate": 0.000226984126984127, + "loss": 4.6828, + "theoretical_loss": 7.023173585499698, + "tokens_seen": 9371648 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022857142857142857, + "loss": 4.6974, + "theoretical_loss": 7.014234347815421, + "tokens_seen": 9437184 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023015873015873016, + "loss": 4.8072, + "theoretical_loss": 7.00537421894883, + "tokens_seen": 9502720 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002317460317460318, + "loss": 4.692, + "theoretical_loss": 6.996591960436656, + "tokens_seen": 9568256 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023333333333333336, + "loss": 4.7647, + "theoretical_loss": 6.9878863615175355, + "tokens_seen": 9633792 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023492063492063493, + "loss": 4.7072, + "theoretical_loss": 6.97925623832895, + "tokens_seen": 9699328 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023650793650793653, + "loss": 4.8046, + "theoretical_loss": 6.97070043313267, + "tokens_seen": 9764864 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 9988, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 4.5934672355651855, + "objective/train/theoretical_loss": 6.962217813567541, + "objective/train/tokens_used": 30290400, + "theoretical_loss": 6.962217813567541, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002380952380952381, + "loss": 4.7083, + "theoretical_loss": 6.962217813567541, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002396825396825397, + "loss": 4.6527, + "theoretical_loss": 6.953807271928458, + "tokens_seen": 9895936 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024126984126984126, + "loss": 4.4851, + "theoretical_loss": 6.945467724470439, + "tokens_seen": 9961472 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024285714285714286, + "loss": 4.4885, + "theoretical_loss": 6.937198110736788, + "tokens_seen": 10027008 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002444444444444445, + "loss": 4.5112, + "theoretical_loss": 6.928997392910329, + "tokens_seen": 10092544 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024603174603174605, + "loss": 4.7377, + "theoretical_loss": 6.920864555186801, + "tokens_seen": 10158080 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002476190476190476, + "loss": 4.6674, + "theoretical_loss": 6.912798603169512, + "tokens_seen": 10223616 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024920634920634925, + "loss": 4.5936, + "theoretical_loss": 6.904798563284391, + "tokens_seen": 10289152 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002507936507936508, + "loss": 4.8009, + "theoretical_loss": 6.896863482214638, + "tokens_seen": 10354688 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002523809523809524, + "loss": 4.7057, + "theoretical_loss": 6.888992426354192, + "tokens_seen": 10420224 + }, + { + "epoch": 0.0, + "learning_rate": 0.00025396825396825396, + "loss": 4.7217, + "theoretical_loss": 6.881184481279271, + "tokens_seen": 10485760 + }, + { + "epoch": 0.0, + "learning_rate": 0.00025555555555555553, + "loss": 4.5575, + "theoretical_loss": 6.873438751237283, + "tokens_seen": 10551296 + }, + { + "epoch": 0.0, + "learning_rate": 0.00025714285714285715, + "loss": 4.6313, + "theoretical_loss": 6.865754358652436, + "tokens_seen": 10616832 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002587301587301588, + "loss": 4.5677, + "theoretical_loss": 6.85813044364739, + "tokens_seen": 10682368 + }, + { + "epoch": 0.0, + "learning_rate": 0.00026031746031746035, + "loss": 4.5783, + "theoretical_loss": 6.85056616358036, + "tokens_seen": 10747904 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002619047619047619, + "loss": 4.7549, + "theoretical_loss": 6.843060692597036, + "tokens_seen": 10813440 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002634920634920635, + "loss": 4.5659, + "theoretical_loss": 6.835613221196822, + "tokens_seen": 10878976 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002650793650793651, + "loss": 4.6755, + "theoretical_loss": 6.8282229558127865, + "tokens_seen": 10944512 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002666666666666667, + "loss": 4.5563, + "theoretical_loss": 6.820889118404866, + "tokens_seen": 11010048 + }, + { + "epoch": 0.0, + "learning_rate": 0.00026825396825396825, + "loss": 4.5433, + "theoretical_loss": 6.813610946065789, + "tokens_seen": 11075584 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002698412698412699, + "loss": 4.5981, + "theoretical_loss": 6.806387690639279, + "tokens_seen": 11141120 + }, + { + "epoch": 0.0, + "learning_rate": 0.00027142857142857144, + "loss": 4.5952, + "theoretical_loss": 6.799218618350064, + "tokens_seen": 11206656 + }, + { + "epoch": 0.0, + "learning_rate": 0.000273015873015873, + "loss": 4.6503, + "theoretical_loss": 6.792103009445266, + "tokens_seen": 11272192 + }, + { + "epoch": 0.0, + "learning_rate": 0.00027460317460317464, + "loss": 4.4218, + "theoretical_loss": 6.7850401578467645, + "tokens_seen": 11337728 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002761904761904762, + "loss": 4.5615, + "theoretical_loss": 6.778029370814127, + "tokens_seen": 11403264 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002777777777777778, + "loss": 4.535, + "theoretical_loss": 6.771069968617732, + "tokens_seen": 11468800 + }, + { + "epoch": 0.0, + "learning_rate": 0.00027936507936507935, + "loss": 4.6074, + "theoretical_loss": 6.764161284221716, + "tokens_seen": 11534336 + }, + { + "epoch": 0.0, + "learning_rate": 0.00028095238095238097, + "loss": 4.7694, + "theoretical_loss": 6.757302662976402, + "tokens_seen": 11599872 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002825396825396826, + "loss": 4.5586, + "theoretical_loss": 6.750493462319867, + "tokens_seen": 11665408 + }, + { + "epoch": 0.0, + "learning_rate": 0.00028412698412698417, + "loss": 4.4784, + "theoretical_loss": 6.74373305148833, + "tokens_seen": 11730944 + }, + { + "epoch": 0.0, + "learning_rate": 0.00028571428571428574, + "loss": 4.4651, + "theoretical_loss": 6.737020811235054, + "tokens_seen": 11796480 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002873015873015873, + "loss": 4.3919, + "theoretical_loss": 6.730356133557465, + "tokens_seen": 11862016 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002888888888888889, + "loss": 4.5578, + "theoretical_loss": 6.7237384214322, + "tokens_seen": 11927552 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002904761904761905, + "loss": 4.4157, + "theoretical_loss": 6.717167088557821, + "tokens_seen": 11993088 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029206349206349207, + "loss": 4.534, + "theoretical_loss": 6.710641559104915, + "tokens_seen": 12058624 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029365079365079364, + "loss": 4.5115, + "theoretical_loss": 6.704161267473356, + "tokens_seen": 12124160 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029523809523809526, + "loss": 4.5263, + "theoretical_loss": 6.697725658056456, + "tokens_seen": 12189696 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029682539682539683, + "loss": 4.623, + "theoretical_loss": 6.691334185011799, + "tokens_seen": 12255232 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029841269841269846, + "loss": 4.4268, + "theoretical_loss": 6.68498631203852, + "tokens_seen": 12320768 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030000000000000003, + "loss": 4.4161, + "theoretical_loss": 6.678681512160812, + "tokens_seen": 12386304 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003015873015873016, + "loss": 4.3853, + "theoretical_loss": 6.672419267517472, + "tokens_seen": 12451840 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030317460317460317, + "loss": 4.4433, + "theoretical_loss": 6.666199069157267, + "tokens_seen": 12517376 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030476190476190474, + "loss": 4.5607, + "theoretical_loss": 6.660020416839945, + "tokens_seen": 12582912 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030634920634920636, + "loss": 4.4346, + "theoretical_loss": 6.65388281884269, + "tokens_seen": 12648448 + }, + { + "epoch": 0.0, + "learning_rate": 0.000307936507936508, + "loss": 4.4253, + "theoretical_loss": 6.647785791771864, + "tokens_seen": 12713984 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030952380952380956, + "loss": 4.423, + "theoretical_loss": 6.641728860379852, + "tokens_seen": 12779520 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003111111111111111, + "loss": 4.5214, + "theoretical_loss": 6.635711557386854, + "tokens_seen": 12845056 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003126984126984127, + "loss": 4.5018, + "theoretical_loss": 6.629733423307451, + "tokens_seen": 12910592 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003142857142857143, + "loss": 4.5074, + "theoretical_loss": 6.6237940062818215, + "tokens_seen": 12976128 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003158730158730159, + "loss": 4.5088, + "theoretical_loss": 6.617892861911426, + "tokens_seen": 13041664 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 11319, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 4.34834098815918, + "objective/train/theoretical_loss": 6.612029553099055, + "objective/train/tokens_used": 33567200, + "theoretical_loss": 6.612029553099055, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.00031746031746031746, + "loss": 4.4598, + "theoretical_loss": 6.612029553099055, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003190476190476191, + "loss": 4.5264, + "theoretical_loss": 6.606203649893072, + "tokens_seen": 13172736 + }, + { + "epoch": 0.0, + "learning_rate": 0.00032063492063492065, + "loss": 4.4953, + "theoretical_loss": 6.600414729335739, + "tokens_seen": 13238272 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003222222222222222, + "loss": 4.5654, + "theoretical_loss": 6.5946623753155045, + "tokens_seen": 13303808 + }, + { + "epoch": 0.0, + "learning_rate": 0.00032380952380952385, + "loss": 4.4299, + "theoretical_loss": 6.588946178423104, + "tokens_seen": 13369344 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003253968253968254, + "loss": 4.5707, + "theoretical_loss": 6.583265735811386, + "tokens_seen": 13434880 + }, + { + "epoch": 0.0, + "learning_rate": 0.000326984126984127, + "loss": 4.4346, + "theoretical_loss": 6.577620651058732, + "tokens_seen": 13500416 + }, + { + "epoch": 0.0, + "learning_rate": 0.00032857142857142856, + "loss": 4.4908, + "theoretical_loss": 6.572010534035971, + "tokens_seen": 13565952 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003301587301587302, + "loss": 4.5556, + "theoretical_loss": 6.566435000776667, + "tokens_seen": 13631488 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003317460317460318, + "loss": 4.5845, + "theoretical_loss": 6.5608936733507095, + "tokens_seen": 13697024 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003333333333333334, + "loss": 4.5322, + "theoretical_loss": 6.555386179741061, + "tokens_seen": 13762560 + }, + { + "epoch": 0.0, + "learning_rate": 0.00033492063492063495, + "loss": 4.4791, + "theoretical_loss": 6.549912153723619, + "tokens_seen": 13828096 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003365079365079365, + "loss": 4.4018, + "theoretical_loss": 6.544471234750057, + "tokens_seen": 13893632 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003380952380952381, + "loss": 4.451, + "theoretical_loss": 6.539063067833591, + "tokens_seen": 13959168 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003396825396825397, + "loss": 4.2116, + "theoretical_loss": 6.533687303437551, + "tokens_seen": 14024704 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003412698412698413, + "loss": 4.462, + "theoretical_loss": 6.528343597366709, + "tokens_seen": 14090240 + }, + { + "epoch": 0.0, + "learning_rate": 0.00034285714285714285, + "loss": 4.405, + "theoretical_loss": 6.523031610661259, + "tokens_seen": 14155776 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003444444444444445, + "loss": 4.5102, + "theoretical_loss": 6.517751009493386, + "tokens_seen": 14221312 + }, + { + "epoch": 0.0, + "learning_rate": 0.00034603174603174604, + "loss": 4.3705, + "theoretical_loss": 6.51250146506634, + "tokens_seen": 14286848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00034761904761904767, + "loss": 4.4277, + "theoretical_loss": 6.507282653515952, + "tokens_seen": 14352384 + }, + { + "epoch": 0.0, + "learning_rate": 0.00034920634920634924, + "loss": 4.384, + "theoretical_loss": 6.5020942558145105, + "tokens_seen": 14417920 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003507936507936508, + "loss": 4.4511, + "theoretical_loss": 6.496935957676949, + "tokens_seen": 14483456 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003523809523809524, + "loss": 4.4827, + "theoretical_loss": 6.491807449469257, + "tokens_seen": 14548992 + }, + { + "epoch": 0.0, + "learning_rate": 0.00035396825396825395, + "loss": 4.4038, + "theoretical_loss": 6.486708426119071, + "tokens_seen": 14614528 + }, + { + "epoch": 0.0, + "learning_rate": 0.00035555555555555557, + "loss": 4.4651, + "theoretical_loss": 6.481638587028371, + "tokens_seen": 14680064 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003571428571428572, + "loss": 4.2654, + "theoretical_loss": 6.4765976359882345, + "tokens_seen": 14745600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00035873015873015877, + "loss": 4.3412, + "theoretical_loss": 6.47158528109558, + "tokens_seen": 14811136 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036031746031746034, + "loss": 4.4479, + "theoretical_loss": 6.466601234671855, + "tokens_seen": 14876672 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003619047619047619, + "loss": 4.2494, + "theoretical_loss": 6.461645213183605, + "tokens_seen": 14942208 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036349206349206353, + "loss": 4.3764, + "theoretical_loss": 6.45671693716488, + "tokens_seen": 15007744 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003650793650793651, + "loss": 4.2567, + "theoretical_loss": 6.451816131141427, + "tokens_seen": 15073280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036666666666666667, + "loss": 4.3653, + "theoretical_loss": 6.4469425235566105, + "tokens_seen": 15138816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036825396825396824, + "loss": 4.462, + "theoretical_loss": 6.442095846699032, + "tokens_seen": 15204352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036984126984126986, + "loss": 4.4221, + "theoretical_loss": 6.437275836631781, + "tokens_seen": 15269888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00037142857142857143, + "loss": 4.2723, + "theoretical_loss": 6.432482233123297, + "tokens_seen": 15335424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00037301587301587306, + "loss": 4.2973, + "theoretical_loss": 6.427714779579773, + "tokens_seen": 15400960 + }, + { + "epoch": 0.0, + "learning_rate": 0.00037460317460317463, + "loss": 4.2562, + "theoretical_loss": 6.422973222979093, + "tokens_seen": 15466496 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003761904761904762, + "loss": 4.3652, + "theoretical_loss": 6.418257313806215, + "tokens_seen": 15532032 + }, + { + "epoch": 0.0, + "learning_rate": 0.00037777777777777777, + "loss": 4.3504, + "theoretical_loss": 6.413566805990028, + "tokens_seen": 15597568 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003793650793650794, + "loss": 4.2707, + "theoretical_loss": 6.408901456841564, + "tokens_seen": 15663104 + }, + { + "epoch": 0.0, + "learning_rate": 0.00038095238095238096, + "loss": 4.2359, + "theoretical_loss": 6.404261026993607, + "tokens_seen": 15728640 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003825396825396826, + "loss": 4.3195, + "theoretical_loss": 6.399645280341607, + "tokens_seen": 15794176 + }, + { + "epoch": 0.0, + "learning_rate": 0.00038412698412698416, + "loss": 4.3046, + "theoretical_loss": 6.395053983985889, + "tokens_seen": 15859712 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003857142857142857, + "loss": 4.1676, + "theoretical_loss": 6.390486908175121, + "tokens_seen": 15925248 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003873015873015873, + "loss": 4.3473, + "theoretical_loss": 6.385943826251012, + "tokens_seen": 15990784 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003888888888888889, + "loss": 4.3593, + "theoretical_loss": 6.381424514594194, + "tokens_seen": 16056320 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003904761904761905, + "loss": 4.2621, + "theoretical_loss": 6.376928752571271, + "tokens_seen": 16121856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00039206349206349206, + "loss": 4.2769, + "theoretical_loss": 6.372456322483007, + "tokens_seen": 16187392 + }, + { + "epoch": 0.0, + "learning_rate": 0.00039365079365079363, + "loss": 4.3993, + "theoretical_loss": 6.368007009513611, + "tokens_seen": 16252928 + }, + { + "epoch": 0.0, + "learning_rate": 0.00039523809523809526, + "loss": 4.3462, + "theoretical_loss": 6.363580601681101, + "tokens_seen": 16318464 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 12210, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 4.425621032714844, + "objective/train/theoretical_loss": 6.359176889788724, + "objective/train/tokens_used": 36844000, + "theoretical_loss": 6.359176889788724, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003968253968253969, + "loss": 4.3752, + "theoretical_loss": 6.359176889788724, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.00039841269841269845, + "loss": 4.2801, + "theoretical_loss": 6.354795667377392, + "tokens_seen": 16449536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004, + "loss": 4.282, + "theoretical_loss": 6.350436730679111, + "tokens_seen": 16515072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004015873015873016, + "loss": 4.3637, + "theoretical_loss": 6.346099878571405, + "tokens_seen": 16580608 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040317460317460316, + "loss": 4.2939, + "theoretical_loss": 6.341784912532662, + "tokens_seen": 16646144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004047619047619048, + "loss": 4.3405, + "theoretical_loss": 6.337491636598426, + "tokens_seen": 16711680 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040634920634920635, + "loss": 4.1746, + "theoretical_loss": 6.33321985731858, + "tokens_seen": 16777216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004079365079365079, + "loss": 4.2457, + "theoretical_loss": 6.328969383715416, + "tokens_seen": 16842752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004095238095238095, + "loss": 4.254, + "theoretical_loss": 6.324740027242559, + "tokens_seen": 16908288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004111111111111111, + "loss": 4.1043, + "theoretical_loss": 6.32053160174474, + "tokens_seen": 16973824 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041269841269841274, + "loss": 4.1842, + "theoretical_loss": 6.316343923418371, + "tokens_seen": 17039360 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041428571428571437, + "loss": 4.0588, + "theoretical_loss": 6.312176810772938, + "tokens_seen": 17104896 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041587301587301594, + "loss": 4.1163, + "theoretical_loss": 6.308030084593148, + "tokens_seen": 17170432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004174603174603175, + "loss": 4.2286, + "theoretical_loss": 6.30390356790187, + "tokens_seen": 17235968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004190476190476191, + "loss": 4.2368, + "theoretical_loss": 6.2997970859237835, + "tokens_seen": 17301504 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042063492063492065, + "loss": 4.2891, + "theoretical_loss": 6.295710466049778, + "tokens_seen": 17367040 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042222222222222227, + "loss": 4.3256, + "theoretical_loss": 6.29164353780205, + "tokens_seen": 17432576 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042380952380952384, + "loss": 4.2051, + "theoretical_loss": 6.287596132799887, + "tokens_seen": 17498112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004253968253968254, + "loss": 4.2521, + "theoretical_loss": 6.283568084726138, + "tokens_seen": 17563648 + }, + { + "epoch": 0.01, + "learning_rate": 0.000426984126984127, + "loss": 4.214, + "theoretical_loss": 6.27955922929433, + "tokens_seen": 17629184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004285714285714286, + "loss": 4.1344, + "theoretical_loss": 6.2755694042164425, + "tokens_seen": 17694720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004301587301587302, + "loss": 4.2665, + "theoretical_loss": 6.271598449171293, + "tokens_seen": 17760256 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043174603174603174, + "loss": 4.1745, + "theoretical_loss": 6.267646205773557, + "tokens_seen": 17825792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004333333333333333, + "loss": 4.1578, + "theoretical_loss": 6.263712517543373, + "tokens_seen": 17891328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004349206349206349, + "loss": 4.0887, + "theoretical_loss": 6.25979722987654, + "tokens_seen": 17956864 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004365079365079365, + "loss": 4.1966, + "theoretical_loss": 6.255900190015284, + "tokens_seen": 18022400 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043809523809523813, + "loss": 4.2064, + "theoretical_loss": 6.252021247019596, + "tokens_seen": 18087936 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043968253968253976, + "loss": 4.153, + "theoretical_loss": 6.2481602517390975, + "tokens_seen": 18153472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004412698412698413, + "loss": 4.1275, + "theoretical_loss": 6.244317056785462, + "tokens_seen": 18219008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004428571428571429, + "loss": 4.1322, + "theoretical_loss": 6.240491516505343, + "tokens_seen": 18284544 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044444444444444447, + "loss": 4.1679, + "theoretical_loss": 6.236683486953819, + "tokens_seen": 18350080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004460317460317461, + "loss": 4.0966, + "theoretical_loss": 6.232892825868342, + "tokens_seen": 18415616 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044761904761904766, + "loss": 4.1138, + "theoretical_loss": 6.229119392643171, + "tokens_seen": 18481152 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044920634920634923, + "loss": 4.1271, + "theoretical_loss": 6.225363048304281, + "tokens_seen": 18546688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004507936507936508, + "loss": 4.2432, + "theoretical_loss": 6.221623655484739, + "tokens_seen": 18612224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045238095238095237, + "loss": 4.2264, + "theoretical_loss": 6.217901078400542, + "tokens_seen": 18677760 + }, + { + "epoch": 0.01, + "learning_rate": 0.000453968253968254, + "loss": 4.2129, + "theoretical_loss": 6.214195182826902, + "tokens_seen": 18743296 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045555555555555556, + "loss": 4.105, + "theoretical_loss": 6.2105058360749545, + "tokens_seen": 18808832 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045714285714285713, + "loss": 4.0347, + "theoretical_loss": 6.206832906968911, + "tokens_seen": 18874368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004587301587301587, + "loss": 4.0975, + "theoretical_loss": 6.203176265823624, + "tokens_seen": 18939904 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046031746031746033, + "loss": 4.2128, + "theoretical_loss": 6.1995357844225545, + "tokens_seen": 19005440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004619047619047619, + "loss": 4.0438, + "theoretical_loss": 6.195911335996151, + "tokens_seen": 19070976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004634920634920636, + "loss": 4.2097, + "theoretical_loss": 6.192302795200606, + "tokens_seen": 19136512 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046507936507936515, + "loss": 4.1158, + "theoretical_loss": 6.188710038097013, + "tokens_seen": 19202048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004666666666666667, + "loss": 4.1666, + "theoretical_loss": 6.185132942130884, + "tokens_seen": 19267584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004682539682539683, + "loss": 4.1723, + "theoretical_loss": 6.181571386112038, + "tokens_seen": 19333120 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046984126984126986, + "loss": 4.1159, + "theoretical_loss": 6.178025250194852, + "tokens_seen": 19398656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004714285714285715, + "loss": 4.2241, + "theoretical_loss": 6.174494415858863, + "tokens_seen": 19464192 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047301587301587305, + "loss": 4.0897, + "theoretical_loss": 6.170978765889705, + "tokens_seen": 19529728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004746031746031746, + "loss": 4.1264, + "theoretical_loss": 6.167478184360402, + "tokens_seen": 19595264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 13501, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 4.086112976074219, + "objective/train/theoretical_loss": 6.163992556612967, + "objective/train/tokens_used": 40120800, + "theoretical_loss": 6.163992556612967, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004761904761904762, + "loss": 4.1434, + "theoretical_loss": 6.163992556612967, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004777777777777778, + "loss": 4.0815, + "theoretical_loss": 6.160521769240346, + "tokens_seen": 19726336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004793650793650794, + "loss": 4.1549, + "theoretical_loss": 6.157065710068663, + "tokens_seen": 19791872 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048095238095238095, + "loss": 4.0909, + "theoretical_loss": 6.1536242681397795, + "tokens_seen": 19857408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004825396825396825, + "loss": 3.918, + "theoretical_loss": 6.150197333694162, + "tokens_seen": 19922944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004841269841269841, + "loss": 4.0747, + "theoretical_loss": 6.146784798154039, + "tokens_seen": 19988480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004857142857142857, + "loss": 4.1139, + "theoretical_loss": 6.143386554106854, + "tokens_seen": 20054016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004873015873015873, + "loss": 3.9063, + "theoretical_loss": 6.140002495289, + "tokens_seen": 20119552 + }, + { + "epoch": 0.01, + "learning_rate": 0.000488888888888889, + "loss": 4.0389, + "theoretical_loss": 6.136632516569836, + "tokens_seen": 20185088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004904761904761905, + "loss": 4.0393, + "theoretical_loss": 6.133276513935977, + "tokens_seen": 20250624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004920634920634921, + "loss": 4.0666, + "theoretical_loss": 6.129934384475845, + "tokens_seen": 20316160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004936507936507937, + "loss": 4.0097, + "theoretical_loss": 6.126606026364497, + "tokens_seen": 20381696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004952380952380952, + "loss": 4.0515, + "theoretical_loss": 6.123291338848694, + "tokens_seen": 20447232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004968253968253969, + "loss": 4.0653, + "theoretical_loss": 6.119990222232234, + "tokens_seen": 20512768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984126984126985, + "loss": 4.0558, + "theoretical_loss": 6.116702577861526, + "tokens_seen": 20578304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 4.1177, + "theoretical_loss": 6.113428308111404, + "tokens_seen": 20643840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005015873015873016, + "loss": 4.1428, + "theoretical_loss": 6.110167316371188, + "tokens_seen": 20709376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005031746031746032, + "loss": 3.9055, + "theoretical_loss": 6.106919507030966, + "tokens_seen": 20774912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005047619047619048, + "loss": 4.1163, + "theoretical_loss": 6.103684785468113, + "tokens_seen": 20840448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005063492063492064, + "loss": 3.929, + "theoretical_loss": 6.100463058034027, + "tokens_seen": 20905984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005079365079365079, + "loss": 4.0137, + "theoretical_loss": 6.097254232041089, + "tokens_seen": 20971520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005095238095238095, + "loss": 3.9822, + "theoretical_loss": 6.094058215749833, + "tokens_seen": 21037056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005111111111111111, + "loss": 4.1239, + "theoretical_loss": 6.090874918356328, + "tokens_seen": 21102592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005126984126984128, + "loss": 3.888, + "theoretical_loss": 6.087704249979775, + "tokens_seen": 21168128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005142857142857143, + "loss": 3.9305, + "theoretical_loss": 6.084546121650289, + "tokens_seen": 21233664 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005158730158730159, + "loss": 4.0623, + "theoretical_loss": 6.081400445296899, + "tokens_seen": 21299200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005174603174603176, + "loss": 4.0524, + "theoretical_loss": 6.078267133735724, + "tokens_seen": 21364736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005190476190476191, + "loss": 4.0179, + "theoretical_loss": 6.075146100658356, + "tokens_seen": 21430272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005206349206349207, + "loss": 3.8274, + "theoretical_loss": 6.072037260620421, + "tokens_seen": 21495808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005222222222222222, + "loss": 3.9728, + "theoretical_loss": 6.0689405290303275, + "tokens_seen": 21561344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005238095238095238, + "loss": 4.0168, + "theoretical_loss": 6.065855822138193, + "tokens_seen": 21626880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005253968253968255, + "loss": 3.9993, + "theoretical_loss": 6.06278305702495, + "tokens_seen": 21692416 + }, + { + "epoch": 0.01, + "learning_rate": 0.000526984126984127, + "loss": 3.9977, + "theoretical_loss": 6.05972215159162, + "tokens_seen": 21757952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005285714285714286, + "loss": 4.0765, + "theoretical_loss": 6.056673024548766, + "tokens_seen": 21823488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005301587301587302, + "loss": 3.9494, + "theoretical_loss": 6.053635595406105, + "tokens_seen": 21889024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005317460317460317, + "loss": 3.8884, + "theoretical_loss": 6.050609784462289, + "tokens_seen": 21954560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005333333333333334, + "loss": 3.9952, + "theoretical_loss": 6.047595512794841, + "tokens_seen": 22020096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005349206349206349, + "loss": 3.8465, + "theoretical_loss": 6.044592702250255, + "tokens_seen": 22085632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005365079365079365, + "loss": 3.8355, + "theoretical_loss": 6.041601275434251, + "tokens_seen": 22151168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005380952380952381, + "loss": 3.9425, + "theoretical_loss": 6.038621155702174, + "tokens_seen": 22216704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005396825396825397, + "loss": 3.9988, + "theoretical_loss": 6.0356522671495485, + "tokens_seen": 22282240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005412698412698414, + "loss": 3.9923, + "theoretical_loss": 6.032694534602784, + "tokens_seen": 22347776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005428571428571429, + "loss": 3.8449, + "theoretical_loss": 6.029747883610009, + "tokens_seen": 22413312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005444444444444445, + "loss": 3.8843, + "theoretical_loss": 6.026812240432067, + "tokens_seen": 22478848 + }, + { + "epoch": 0.01, + "learning_rate": 0.000546031746031746, + "loss": 3.9525, + "theoretical_loss": 6.023887532033628, + "tokens_seen": 22544384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005476190476190477, + "loss": 3.9851, + "theoretical_loss": 6.020973686074459, + "tokens_seen": 22609920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005492063492063493, + "loss": 3.8682, + "theoretical_loss": 6.018070630900809, + "tokens_seen": 22675456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005507936507936508, + "loss": 3.9546, + "theoretical_loss": 6.015178295536933, + "tokens_seen": 22740992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005523809523809524, + "loss": 3.8417, + "theoretical_loss": 6.012296609676753, + "tokens_seen": 22806528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005539682539682539, + "loss": 3.9796, + "theoretical_loss": 6.009425503675624, + "tokens_seen": 22872064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 14491, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.740633487701416, + "objective/train/theoretical_loss": 6.006564908542249, + "objective/train/tokens_used": 43397600, + "theoretical_loss": 6.006564908542249, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005555555555555556, + "loss": 3.8125, + "theoretical_loss": 6.006564908542249, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005571428571428572, + "loss": 4.0247, + "theoretical_loss": 6.003714755930698, + "tokens_seen": 23003136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005587301587301587, + "loss": 3.8499, + "theoretical_loss": 6.000874978132554, + "tokens_seen": 23068672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005603174603174603, + "loss": 3.7935, + "theoretical_loss": 5.998045508069175, + "tokens_seen": 23134208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005619047619047619, + "loss": 3.6955, + "theoretical_loss": 5.995226279284073, + "tokens_seen": 23199744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005634920634920636, + "loss": 3.848, + "theoretical_loss": 5.992417225935405, + "tokens_seen": 23265280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005650793650793652, + "loss": 4.0086, + "theoretical_loss": 5.989618282788578, + "tokens_seen": 23330816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005666666666666667, + "loss": 4.0299, + "theoretical_loss": 5.9868293852089565, + "tokens_seen": 23396352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005682539682539683, + "loss": 3.7135, + "theoretical_loss": 5.984050469154685, + "tokens_seen": 23461888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005698412698412698, + "loss": 3.9052, + "theoretical_loss": 5.981281471169619, + "tokens_seen": 23527424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005714285714285715, + "loss": 3.916, + "theoretical_loss": 5.978522328376346, + "tokens_seen": 23592960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005730158730158731, + "loss": 3.9566, + "theoretical_loss": 5.975772978469321, + "tokens_seen": 23658496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005746031746031746, + "loss": 3.861, + "theoretical_loss": 5.973033359708099, + "tokens_seen": 23724032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005761904761904762, + "loss": 4.0453, + "theoretical_loss": 5.9703034109106685, + "tokens_seen": 23789568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005777777777777778, + "loss": 3.8675, + "theoretical_loss": 5.967583071446869, + "tokens_seen": 23855104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005793650793650794, + "loss": 3.8817, + "theoretical_loss": 5.9648722812319255, + "tokens_seen": 23920640 + }, + { + "epoch": 0.01, + "learning_rate": 0.000580952380952381, + "loss": 3.9538, + "theoretical_loss": 5.962170980720055, + "tokens_seen": 23986176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005825396825396825, + "loss": 3.8526, + "theoretical_loss": 5.959479110898181, + "tokens_seen": 24051712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005841269841269841, + "loss": 3.8755, + "theoretical_loss": 5.956796613279727, + "tokens_seen": 24117248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005857142857142857, + "loss": 3.9102, + "theoretical_loss": 5.954123429898508, + "tokens_seen": 24182784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005873015873015873, + "loss": 3.721, + "theoretical_loss": 5.951459503302701, + "tokens_seen": 24248320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005888888888888889, + "loss": 3.8645, + "theoretical_loss": 5.948804776548908, + "tokens_seen": 24313856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005904761904761905, + "loss": 4.0393, + "theoretical_loss": 5.9461591931963, + "tokens_seen": 24379392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005920634920634922, + "loss": 3.9327, + "theoretical_loss": 5.943522697300843, + "tokens_seen": 24444928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005936507936507937, + "loss": 3.6637, + "theoretical_loss": 5.940895233409613, + "tokens_seen": 24510464 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005952380952380953, + "loss": 3.8462, + "theoretical_loss": 5.938276746555175, + "tokens_seen": 24576000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005968253968253969, + "loss": 3.8563, + "theoretical_loss": 5.9356671822500635, + "tokens_seen": 24641536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005984126984126984, + "loss": 3.6734, + "theoretical_loss": 5.933066486481318, + "tokens_seen": 24707072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006000000000000001, + "loss": 3.9587, + "theoretical_loss": 5.930474605705113, + "tokens_seen": 24772608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006015873015873016, + "loss": 3.8129, + "theoretical_loss": 5.927891486841449, + "tokens_seen": 24838144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006031746031746032, + "loss": 3.8233, + "theoretical_loss": 5.925317077268927, + "tokens_seen": 24903680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006047619047619048, + "loss": 3.8697, + "theoretical_loss": 5.92275132481959, + "tokens_seen": 24969216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006063492063492063, + "loss": 3.7977, + "theoretical_loss": 5.9201941777738405, + "tokens_seen": 25034752 + }, + { + "epoch": 0.01, + "learning_rate": 0.000607936507936508, + "loss": 3.8119, + "theoretical_loss": 5.917645584855421, + "tokens_seen": 25100288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006095238095238095, + "loss": 3.8489, + "theoretical_loss": 5.915105495226474, + "tokens_seen": 25165824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006111111111111111, + "loss": 3.7896, + "theoretical_loss": 5.912573858482657, + "tokens_seen": 25231360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006126984126984127, + "loss": 3.7893, + "theoretical_loss": 5.910050624648335, + "tokens_seen": 25296896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006142857142857143, + "loss": 3.6833, + "theoretical_loss": 5.907535744171835, + "tokens_seen": 25362432 + }, + { + "epoch": 0.01, + "learning_rate": 0.000615873015873016, + "loss": 3.7628, + "theoretical_loss": 5.905029167920765, + "tokens_seen": 25427968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006174603174603175, + "loss": 3.7026, + "theoretical_loss": 5.902530847177394, + "tokens_seen": 25493504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006190476190476191, + "loss": 3.8071, + "theoretical_loss": 5.900040733634098, + "tokens_seen": 25559040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006206349206349206, + "loss": 3.8134, + "theoretical_loss": 5.897558779388872, + "tokens_seen": 25624576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006222222222222223, + "loss": 3.6805, + "theoretical_loss": 5.895084936940887, + "tokens_seen": 25690112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006238095238095239, + "loss": 3.7538, + "theoretical_loss": 5.892619159186127, + "tokens_seen": 25755648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006253968253968254, + "loss": 3.7874, + "theoretical_loss": 5.890161399413071, + "tokens_seen": 25821184 + }, + { + "epoch": 0.01, + "learning_rate": 0.000626984126984127, + "loss": 3.7056, + "theoretical_loss": 5.887711611298428, + "tokens_seen": 25886720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006285714285714286, + "loss": 3.7162, + "theoretical_loss": 5.885269748902955, + "tokens_seen": 25952256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006301587301587302, + "loss": 3.6735, + "theoretical_loss": 5.882835766667295, + "tokens_seen": 26017792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006317460317460318, + "loss": 3.721, + "theoretical_loss": 5.880409619407899, + "tokens_seen": 26083328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006333333333333333, + "loss": 3.618, + "theoretical_loss": 5.877991262312986, + "tokens_seen": 26148864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 15706, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.5033531188964844, + "objective/train/theoretical_loss": 5.875580650938565, + "objective/train/tokens_used": 46674400, + "theoretical_loss": 5.875580650938565, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006349206349206349, + "loss": 3.6113, + "theoretical_loss": 5.875580650938565, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006365079365079365, + "loss": 3.7019, + "theoretical_loss": 5.873177741204501, + "tokens_seen": 26279936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006380952380952382, + "loss": 3.644, + "theoretical_loss": 5.8707824893906455, + "tokens_seen": 26345472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006396825396825398, + "loss": 3.6657, + "theoretical_loss": 5.868394852133001, + "tokens_seen": 26411008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006412698412698413, + "loss": 3.7231, + "theoretical_loss": 5.866014786419951, + "tokens_seen": 26476544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006428571428571429, + "loss": 3.8293, + "theoretical_loss": 5.86364224958853, + "tokens_seen": 26542080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006444444444444444, + "loss": 3.7392, + "theoretical_loss": 5.861277199320743, + "tokens_seen": 26607616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006460317460317461, + "loss": 3.7438, + "theoretical_loss": 5.858919593639941, + "tokens_seen": 26673152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006476190476190477, + "loss": 3.7997, + "theoretical_loss": 5.856569390907229, + "tokens_seen": 26738688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006492063492063492, + "loss": 3.8363, + "theoretical_loss": 5.854226549817936, + "tokens_seen": 26804224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006507936507936508, + "loss": 3.6657, + "theoretical_loss": 5.851891029398114, + "tokens_seen": 26869760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006523809523809524, + "loss": 3.7011, + "theoretical_loss": 5.8495627890011015, + "tokens_seen": 26935296 + }, + { + "epoch": 0.01, + "learning_rate": 0.000653968253968254, + "loss": 3.7339, + "theoretical_loss": 5.847241788304114, + "tokens_seen": 27000832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006555555555555556, + "loss": 3.77, + "theoretical_loss": 5.844927987304884, + "tokens_seen": 27066368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006571428571428571, + "loss": 3.5868, + "theoretical_loss": 5.842621346318348, + "tokens_seen": 27131904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006587301587301587, + "loss": 3.709, + "theoretical_loss": 5.84032182597337, + "tokens_seen": 27197440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006603174603174604, + "loss": 3.6199, + "theoretical_loss": 5.8380293872095095, + "tokens_seen": 27262976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006619047619047619, + "loss": 3.6197, + "theoretical_loss": 5.835743991273832, + "tokens_seen": 27328512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006634920634920636, + "loss": 3.507, + "theoretical_loss": 5.833465599717752, + "tokens_seen": 27394048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006650793650793651, + "loss": 3.7224, + "theoretical_loss": 5.831194174393929, + "tokens_seen": 27459584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006666666666666668, + "loss": 3.5851, + "theoretical_loss": 5.828929677453187, + "tokens_seen": 27525120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006682539682539683, + "loss": 3.5854, + "theoretical_loss": 5.826672071341488, + "tokens_seen": 27590656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006698412698412699, + "loss": 3.6313, + "theoretical_loss": 5.824421318796932, + "tokens_seen": 27656192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006714285714285715, + "loss": 3.6213, + "theoretical_loss": 5.822177382846799, + "tokens_seen": 27721728 + }, + { + "epoch": 0.01, + "learning_rate": 0.000673015873015873, + "loss": 3.6204, + "theoretical_loss": 5.8199402268046265, + "tokens_seen": 27787264 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006746031746031747, + "loss": 3.6112, + "theoretical_loss": 5.817709814267329, + "tokens_seen": 27852800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006761904761904762, + "loss": 3.7146, + "theoretical_loss": 5.815486109112342, + "tokens_seen": 27918336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006777777777777778, + "loss": 3.5925, + "theoretical_loss": 5.813269075494812, + "tokens_seen": 27983872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006793650793650794, + "loss": 3.5826, + "theoretical_loss": 5.811058677844817, + "tokens_seen": 28049408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006809523809523809, + "loss": 3.6308, + "theoretical_loss": 5.808854880864618, + "tokens_seen": 28114944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006825396825396826, + "loss": 3.5636, + "theoretical_loss": 5.8066576495259525, + "tokens_seen": 28180480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006841269841269841, + "loss": 3.5684, + "theoretical_loss": 5.804466949067352, + "tokens_seen": 28246016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006857142857142857, + "loss": 3.6715, + "theoretical_loss": 5.8022827449914995, + "tokens_seen": 28311552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006873015873015873, + "loss": 3.6888, + "theoretical_loss": 5.800105003062612, + "tokens_seen": 28377088 + }, + { + "epoch": 0.01, + "learning_rate": 0.000688888888888889, + "loss": 3.5941, + "theoretical_loss": 5.797933689303866, + "tokens_seen": 28442624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006904761904761906, + "loss": 3.6386, + "theoretical_loss": 5.795768769994842, + "tokens_seen": 28508160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006920634920634921, + "loss": 3.5843, + "theoretical_loss": 5.793610211669004, + "tokens_seen": 28573696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006936507936507937, + "loss": 3.5697, + "theoretical_loss": 5.79145798111122, + "tokens_seen": 28639232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006952380952380953, + "loss": 3.6252, + "theoretical_loss": 5.789312045355292, + "tokens_seen": 28704768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006968253968253969, + "loss": 3.6311, + "theoretical_loss": 5.787172371681535, + "tokens_seen": 28770304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006984126984126985, + "loss": 3.7606, + "theoretical_loss": 5.785038927614373, + "tokens_seen": 28835840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007, + "loss": 3.3781, + "theoretical_loss": 5.782911680919971, + "tokens_seen": 28901376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007015873015873016, + "loss": 3.6269, + "theoretical_loss": 5.78079059960389, + "tokens_seen": 28966912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007031746031746032, + "loss": 3.5331, + "theoretical_loss": 5.77867565190877, + "tokens_seen": 29032448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007047619047619048, + "loss": 3.5922, + "theoretical_loss": 5.776566806312052, + "tokens_seen": 29097984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007063492063492064, + "loss": 3.572, + "theoretical_loss": 5.774464031523705, + "tokens_seen": 29163520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007079365079365079, + "loss": 3.5933, + "theoretical_loss": 5.772367296484001, + "tokens_seen": 29229056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007095238095238095, + "loss": 3.624, + "theoretical_loss": 5.770276570361309, + "tokens_seen": 29294592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007111111111111111, + "loss": 3.5538, + "theoretical_loss": 5.768191822549908, + "tokens_seen": 29360128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007126984126984128, + "loss": 3.5156, + "theoretical_loss": 5.766113022667838, + "tokens_seen": 29425664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 16666, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.571195363998413, + "objective/train/theoretical_loss": 5.764040140554766, + "objective/train/tokens_used": 49951200, + "theoretical_loss": 5.764040140554766, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007142857142857144, + "loss": 3.5886, + "theoretical_loss": 5.764040140554766, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007158730158730159, + "loss": 3.6281, + "theoretical_loss": 5.761973146269881, + "tokens_seen": 29556736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007174603174603175, + "loss": 3.3317, + "theoretical_loss": 5.759912010089819, + "tokens_seen": 29622272 + }, + { + "epoch": 0.01, + "learning_rate": 0.000719047619047619, + "loss": 3.5542, + "theoretical_loss": 5.757856702506597, + "tokens_seen": 29687808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007206349206349207, + "loss": 3.584, + "theoretical_loss": 5.755807194225595, + "tokens_seen": 29753344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007222222222222223, + "loss": 3.4624, + "theoretical_loss": 5.753763456163529, + "tokens_seen": 29818880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007238095238095238, + "loss": 3.5575, + "theoretical_loss": 5.751725459446487, + "tokens_seen": 29884416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007253968253968254, + "loss": 3.5754, + "theoretical_loss": 5.7496931754079466, + "tokens_seen": 29949952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007269841269841271, + "loss": 3.6041, + "theoretical_loss": 5.747666575586849, + "tokens_seen": 30015488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007285714285714286, + "loss": 3.5614, + "theoretical_loss": 5.7456456317256785, + "tokens_seen": 30081024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007301587301587302, + "loss": 3.68, + "theoretical_loss": 5.743630315768563, + "tokens_seen": 30146560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007317460317460317, + "loss": 3.5685, + "theoretical_loss": 5.741620599859402, + "tokens_seen": 30212096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007333333333333333, + "loss": 3.646, + "theoretical_loss": 5.739616456340021, + "tokens_seen": 30277632 + }, + { + "epoch": 0.01, + "learning_rate": 0.000734920634920635, + "loss": 3.5125, + "theoretical_loss": 5.737617857748329, + "tokens_seen": 30343168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007365079365079365, + "loss": 3.5059, + "theoretical_loss": 5.73562477681652, + "tokens_seen": 30408704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007380952380952382, + "loss": 3.6117, + "theoretical_loss": 5.733637186469274, + "tokens_seen": 30474240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007396825396825397, + "loss": 3.5285, + "theoretical_loss": 5.731655059821993, + "tokens_seen": 30539776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007412698412698414, + "loss": 3.5112, + "theoretical_loss": 5.729678370179052, + "tokens_seen": 30605312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007428571428571429, + "loss": 3.3423, + "theoretical_loss": 5.727707091032062, + "tokens_seen": 30670848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007444444444444445, + "loss": 3.5549, + "theoretical_loss": 5.725741196058175, + "tokens_seen": 30736384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007460317460317461, + "loss": 3.5069, + "theoretical_loss": 5.723780659118377, + "tokens_seen": 30801920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007476190476190476, + "loss": 3.5296, + "theoretical_loss": 5.721825454255825, + "tokens_seen": 30867456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007492063492063493, + "loss": 3.6073, + "theoretical_loss": 5.71987555569419, + "tokens_seen": 30932992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007507936507936508, + "loss": 3.4977, + "theoretical_loss": 5.717930937836029, + "tokens_seen": 30998528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007523809523809524, + "loss": 3.5041, + "theoretical_loss": 5.715991575261159, + "tokens_seen": 31064064 + }, + { + "epoch": 0.01, + "learning_rate": 0.000753968253968254, + "loss": 3.4418, + "theoretical_loss": 5.714057442725068, + "tokens_seen": 31129600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007555555555555555, + "loss": 3.5898, + "theoretical_loss": 5.712128515157324, + "tokens_seen": 31195136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007571428571428572, + "loss": 3.5238, + "theoretical_loss": 5.710204767660022, + "tokens_seen": 31260672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007587301587301588, + "loss": 3.4737, + "theoretical_loss": 5.708286175506232, + "tokens_seen": 31326208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007603174603174603, + "loss": 3.5078, + "theoretical_loss": 5.706372714138472, + "tokens_seen": 31391744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007619047619047619, + "loss": 3.3246, + "theoretical_loss": 5.704464359167195, + "tokens_seen": 31457280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007634920634920635, + "loss": 3.5155, + "theoretical_loss": 5.702561086369294, + "tokens_seen": 31522816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007650793650793652, + "loss": 3.6084, + "theoretical_loss": 5.7006628716866246, + "tokens_seen": 31588352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007666666666666667, + "loss": 3.54, + "theoretical_loss": 5.698769691224539, + "tokens_seen": 31653888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007682539682539683, + "loss": 3.5811, + "theoretical_loss": 5.696881521250441, + "tokens_seen": 31719424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007698412698412699, + "loss": 3.4774, + "theoretical_loss": 5.694998338192356, + "tokens_seen": 31784960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007714285714285715, + "loss": 3.3984, + "theoretical_loss": 5.693120118637513, + "tokens_seen": 31850496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007730158730158731, + "loss": 3.495, + "theoretical_loss": 5.691246839330946, + "tokens_seen": 31916032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007746031746031746, + "loss": 3.3938, + "theoretical_loss": 5.689378477174106, + "tokens_seen": 31981568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007761904761904762, + "loss": 3.4542, + "theoretical_loss": 5.687515009223498, + "tokens_seen": 32047104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007777777777777778, + "loss": 3.5173, + "theoretical_loss": 5.685656412689318, + "tokens_seen": 32112640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007793650793650794, + "loss": 3.4633, + "theoretical_loss": 5.683802664934115, + "tokens_seen": 32178176 + }, + { + "epoch": 0.01, + "learning_rate": 0.000780952380952381, + "loss": 3.3623, + "theoretical_loss": 5.681953743471466, + "tokens_seen": 32243712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007825396825396825, + "loss": 3.5352, + "theoretical_loss": 5.680109625964663, + "tokens_seen": 32309248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007841269841269841, + "loss": 3.5366, + "theoretical_loss": 5.678270290225415, + "tokens_seen": 32374784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007857142857142857, + "loss": 3.4648, + "theoretical_loss": 5.676435714212562, + "tokens_seen": 32440320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007873015873015873, + "loss": 3.3286, + "theoretical_loss": 5.674605876030808, + "tokens_seen": 32505856 + }, + { + "epoch": 0.01, + "learning_rate": 0.000788888888888889, + "loss": 3.4303, + "theoretical_loss": 5.672780753929461, + "tokens_seen": 32571392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007904761904761905, + "loss": 3.4569, + "theoretical_loss": 5.67096032630119, + "tokens_seen": 32636928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007920634920634921, + "loss": 3.4918, + "theoretical_loss": 5.669144571680797, + "tokens_seen": 32702464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 17795, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.261939525604248, + "objective/train/theoretical_loss": 5.667333468743994, + "objective/train/tokens_used": 53228000, + "theoretical_loss": 5.667333468743994, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007936507936507938, + "loss": 3.3689, + "theoretical_loss": 5.667333468743994, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007952380952380953, + "loss": 3.3711, + "theoretical_loss": 5.665526996306205, + "tokens_seen": 32833536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968253968253969, + "loss": 3.5329, + "theoretical_loss": 5.663725133321369, + "tokens_seen": 32899072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984126984126984, + "loss": 3.5592, + "theoretical_loss": 5.661927858880761, + "tokens_seen": 32964608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0008, + "loss": 3.3173, + "theoretical_loss": 5.6601351522118275, + "tokens_seen": 33030144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999839518555668, + "loss": 3.5531, + "theoretical_loss": 5.65834699267703, + "tokens_seen": 33095680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999679037111334, + "loss": 3.2348, + "theoretical_loss": 5.6565633597727025, + "tokens_seen": 33161216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999518555667002, + "loss": 3.4544, + "theoretical_loss": 5.65478423312792, + "tokens_seen": 33226752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999358074222668, + "loss": 3.4522, + "theoretical_loss": 5.65300959250338, + "tokens_seen": 33292288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999197592778336, + "loss": 3.4293, + "theoretical_loss": 5.651239417790293, + "tokens_seen": 33357824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999037111334002, + "loss": 3.3407, + "theoretical_loss": 5.649473689009292, + "tokens_seen": 33423360 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799887662988967, + "loss": 3.3672, + "theoretical_loss": 5.6477123863093395, + "tokens_seen": 33488896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007998716148445336, + "loss": 3.403, + "theoretical_loss": 5.645955489966661, + "tokens_seen": 33554432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007998555667001004, + "loss": 3.3867, + "theoretical_loss": 5.644202980383678, + "tokens_seen": 33619968 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799839518555667, + "loss": 3.351, + "theoretical_loss": 5.6424548380879616, + "tokens_seen": 33685504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007998234704112338, + "loss": 3.4848, + "theoretical_loss": 5.640711043731184, + "tokens_seen": 33751040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007998074222668004, + "loss": 3.4619, + "theoretical_loss": 5.6389715780880945, + "tokens_seen": 33816576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997913741223672, + "loss": 3.4355, + "theoretical_loss": 5.6372364220555, + "tokens_seen": 33882112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997753259779338, + "loss": 3.4325, + "theoretical_loss": 5.635505556651251, + "tokens_seen": 33947648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997592778335006, + "loss": 3.4346, + "theoretical_loss": 5.633778963013247, + "tokens_seen": 34013184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997432296890672, + "loss": 3.3384, + "theoretical_loss": 5.632056622398448, + "tokens_seen": 34078720 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799727181544634, + "loss": 3.4516, + "theoretical_loss": 5.63033851618189, + "tokens_seen": 34144256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997111334002006, + "loss": 3.2575, + "theoretical_loss": 5.628624625855725, + "tokens_seen": 34209792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996950852557674, + "loss": 3.3118, + "theoretical_loss": 5.626914933028255, + "tokens_seen": 34275328 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799679037111334, + "loss": 3.3996, + "theoretical_loss": 5.625209419422983, + "tokens_seen": 34340864 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996629889669008, + "loss": 3.4081, + "theoretical_loss": 5.623508066877678, + "tokens_seen": 34406400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996469408224675, + "loss": 3.3567, + "theoretical_loss": 5.621810857343441, + "tokens_seen": 34471936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996308926780342, + "loss": 3.2694, + "theoretical_loss": 5.620117772883784, + "tokens_seen": 34537472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996148445336009, + "loss": 3.4846, + "theoretical_loss": 5.618428795673718, + "tokens_seen": 34603008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995987963891676, + "loss": 3.4082, + "theoretical_loss": 5.616743907998853, + "tokens_seen": 34668544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995827482447343, + "loss": 3.3935, + "theoretical_loss": 5.6150630922545, + "tokens_seen": 34734080 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799566700100301, + "loss": 3.3234, + "theoretical_loss": 5.613386330944792, + "tokens_seen": 34799616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995506519558677, + "loss": 3.3006, + "theoretical_loss": 5.6117136066818, + "tokens_seen": 34865152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995346038114344, + "loss": 3.3978, + "theoretical_loss": 5.610044902184672, + "tokens_seen": 34930688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995185556670011, + "loss": 3.3917, + "theoretical_loss": 5.608380200278775, + "tokens_seen": 34996224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995025075225678, + "loss": 3.2776, + "theoretical_loss": 5.606719483894837, + "tokens_seen": 35061760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994864593781345, + "loss": 3.429, + "theoretical_loss": 5.605062736068113, + "tokens_seen": 35127296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994704112337012, + "loss": 3.3658, + "theoretical_loss": 5.603409939937547, + "tokens_seen": 35192832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994543630892679, + "loss": 3.3409, + "theoretical_loss": 5.601761078744952, + "tokens_seen": 35258368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994383149448346, + "loss": 3.3312, + "theoretical_loss": 5.600116135834183, + "tokens_seen": 35323904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994222668004013, + "loss": 3.1932, + "theoretical_loss": 5.598475094650334, + "tokens_seen": 35389440 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799406218655968, + "loss": 3.2632, + "theoretical_loss": 5.59683793873894, + "tokens_seen": 35454976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993901705115347, + "loss": 3.3286, + "theoretical_loss": 5.595204651745176, + "tokens_seen": 35520512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993741223671014, + "loss": 3.1714, + "theoretical_loss": 5.593575217413074, + "tokens_seen": 35586048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993580742226681, + "loss": 3.2249, + "theoretical_loss": 5.591949619584746, + "tokens_seen": 35651584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993420260782347, + "loss": 3.1943, + "theoretical_loss": 5.590327842199611, + "tokens_seen": 35717120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993259779338015, + "loss": 3.261, + "theoretical_loss": 5.588709869293634, + "tokens_seen": 35782656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993099297893681, + "loss": 3.3474, + "theoretical_loss": 5.5870956849985705, + "tokens_seen": 35848192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992938816449349, + "loss": 3.4015, + "theoretical_loss": 5.585485273541212, + "tokens_seen": 35913728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992778335005015, + "loss": 3.3678, + "theoretical_loss": 5.583878619242652, + "tokens_seen": 35979264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 18676, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.2103042602539062, + "objective/train/theoretical_loss": 5.582275706517549, + "objective/train/tokens_used": 56504800, + "theoretical_loss": 5.582275706517549, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992617853560683, + "loss": 3.3616, + "theoretical_loss": 5.582275706517549, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992457372116349, + "loss": 3.3332, + "theoretical_loss": 5.580676519873396, + "tokens_seen": 36110336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992296890672017, + "loss": 3.5113, + "theoretical_loss": 5.579081043909808, + "tokens_seen": 36175872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992136409227683, + "loss": 3.4334, + "theoretical_loss": 5.577489263317796, + "tokens_seen": 36241408 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799197592778335, + "loss": 3.3125, + "theoretical_loss": 5.5759011628790764, + "tokens_seen": 36306944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991815446339017, + "loss": 3.4013, + "theoretical_loss": 5.574316727465358, + "tokens_seen": 36372480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991654964894684, + "loss": 3.2732, + "theoretical_loss": 5.572735942037655, + "tokens_seen": 36438016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991494483450351, + "loss": 3.1625, + "theoretical_loss": 5.571158791645597, + "tokens_seen": 36503552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991334002006018, + "loss": 3.284, + "theoretical_loss": 5.569585261426752, + "tokens_seen": 36569088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991173520561685, + "loss": 3.4507, + "theoretical_loss": 5.568015336605953, + "tokens_seen": 36634624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991013039117352, + "loss": 3.3153, + "theoretical_loss": 5.566449002494627, + "tokens_seen": 36700160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990852557673019, + "loss": 3.135, + "theoretical_loss": 5.564886244490136, + "tokens_seen": 36765696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990692076228686, + "loss": 3.3133, + "theoretical_loss": 5.563327048075124, + "tokens_seen": 36831232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990531594784353, + "loss": 3.3034, + "theoretical_loss": 5.5617713988168616, + "tokens_seen": 36896768 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799037111334002, + "loss": 3.2984, + "theoretical_loss": 5.560219282366614, + "tokens_seen": 36962304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990210631895687, + "loss": 3.3318, + "theoretical_loss": 5.558670684458994, + "tokens_seen": 37027840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990050150451354, + "loss": 3.3465, + "theoretical_loss": 5.557125590911339, + "tokens_seen": 37093376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989889669007021, + "loss": 3.3319, + "theoretical_loss": 5.5555839876230815, + "tokens_seen": 37158912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989729187562688, + "loss": 3.2583, + "theoretical_loss": 5.55404586057513, + "tokens_seen": 37224448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989568706118355, + "loss": 3.1394, + "theoretical_loss": 5.5525111958292594, + "tokens_seen": 37289984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989408224674022, + "loss": 3.3485, + "theoretical_loss": 5.550979979527501, + "tokens_seen": 37355520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989247743229689, + "loss": 3.3191, + "theoretical_loss": 5.549452197891538, + "tokens_seen": 37421056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989087261785356, + "loss": 3.243, + "theoretical_loss": 5.547927837222119, + "tokens_seen": 37486592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988926780341023, + "loss": 3.128, + "theoretical_loss": 5.546406883898451, + "tokens_seen": 37552128 + }, + { + "epoch": 0.01, + "learning_rate": 0.000798876629889669, + "loss": 3.2572, + "theoretical_loss": 5.544889324377628, + "tokens_seen": 37617664 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988605817452357, + "loss": 3.1857, + "theoretical_loss": 5.543375145194048, + "tokens_seen": 37683200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988445336008024, + "loss": 3.1529, + "theoretical_loss": 5.541864332958831, + "tokens_seen": 37748736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988284854563691, + "loss": 3.2037, + "theoretical_loss": 5.540356874359257, + "tokens_seen": 37814272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988124373119358, + "loss": 3.2591, + "theoretical_loss": 5.538852756158201, + "tokens_seen": 37879808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987963891675025, + "loss": 3.1802, + "theoretical_loss": 5.537351965193569, + "tokens_seen": 37945344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987803410230693, + "loss": 3.1964, + "theoretical_loss": 5.535854488377751, + "tokens_seen": 38010880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987642928786359, + "loss": 3.346, + "theoretical_loss": 5.534360312697064, + "tokens_seen": 38076416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987482447342027, + "loss": 3.1631, + "theoretical_loss": 5.532869425211214, + "tokens_seen": 38141952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987321965897693, + "loss": 3.2718, + "theoretical_loss": 5.531381813052757, + "tokens_seen": 38207488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987161484453361, + "loss": 3.17, + "theoretical_loss": 5.529897463426559, + "tokens_seen": 38273024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987001003009027, + "loss": 3.2378, + "theoretical_loss": 5.528416363609276, + "tokens_seen": 38338560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986840521564695, + "loss": 3.1696, + "theoretical_loss": 5.526938500948817, + "tokens_seen": 38404096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986680040120361, + "loss": 3.2103, + "theoretical_loss": 5.525463862863836, + "tokens_seen": 38469632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986519558676029, + "loss": 3.0849, + "theoretical_loss": 5.523992436843209, + "tokens_seen": 38535168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986359077231695, + "loss": 3.2743, + "theoretical_loss": 5.522524210445528, + "tokens_seen": 38600704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986198595787363, + "loss": 3.272, + "theoretical_loss": 5.521059171298592, + "tokens_seen": 38666240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986038114343029, + "loss": 3.3046, + "theoretical_loss": 5.519597307098907, + "tokens_seen": 38731776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985877632898697, + "loss": 3.2986, + "theoretical_loss": 5.518138605611188, + "tokens_seen": 38797312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985717151454363, + "loss": 3.2102, + "theoretical_loss": 5.516683054667868, + "tokens_seen": 38862848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985556670010031, + "loss": 3.3232, + "theoretical_loss": 5.515230642168607, + "tokens_seen": 38928384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985396188565697, + "loss": 3.2207, + "theoretical_loss": 5.513781356079811, + "tokens_seen": 38993920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985235707121365, + "loss": 3.1524, + "theoretical_loss": 5.512335184434152, + "tokens_seen": 39059456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985075225677031, + "loss": 3.3457, + "theoretical_loss": 5.5108921153300905, + "tokens_seen": 39124992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984914744232699, + "loss": 3.1989, + "theoretical_loss": 5.5094521369314045, + "tokens_seen": 39190528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984754262788365, + "loss": 3.0462, + "theoretical_loss": 5.5080152374667275, + "tokens_seen": 39256064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 19810, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.971580982208252, + "objective/train/theoretical_loss": 5.506581405229079, + "objective/train/tokens_used": 59781600, + "theoretical_loss": 5.506581405229079, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984593781344033, + "loss": 3.0874, + "theoretical_loss": 5.506581405229079, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984433299899699, + "loss": 3.2052, + "theoretical_loss": 5.505150628575409, + "tokens_seen": 39387136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984272818455367, + "loss": 3.164, + "theoretical_loss": 5.503722895926143, + "tokens_seen": 39452672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984112337011033, + "loss": 3.0422, + "theoretical_loss": 5.50229819576473, + "tokens_seen": 39518208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983951855566701, + "loss": 3.0343, + "theoretical_loss": 5.500876516637199, + "tokens_seen": 39583744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983791374122367, + "loss": 3.1188, + "theoretical_loss": 5.49945784715171, + "tokens_seen": 39649280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983630892678035, + "loss": 3.0288, + "theoretical_loss": 5.498042175978123, + "tokens_seen": 39714816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983470411233701, + "loss": 3.2065, + "theoretical_loss": 5.496629491847553, + "tokens_seen": 39780352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983309929789369, + "loss": 3.1265, + "theoretical_loss": 5.495219783551947, + "tokens_seen": 39845888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983149448345036, + "loss": 3.1179, + "theoretical_loss": 5.493813039943654, + "tokens_seen": 39911424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982988966900703, + "loss": 2.9897, + "theoretical_loss": 5.492409249934996, + "tokens_seen": 39976960 + }, + { + "epoch": 0.01, + "learning_rate": 0.000798282848545637, + "loss": 3.1839, + "theoretical_loss": 5.491008402497855, + "tokens_seen": 40042496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982668004012037, + "loss": 3.1042, + "theoretical_loss": 5.489610486663252, + "tokens_seen": 40108032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982507522567704, + "loss": 3.0255, + "theoretical_loss": 5.488215491520933, + "tokens_seen": 40173568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982347041123371, + "loss": 3.0889, + "theoretical_loss": 5.486823406218962, + "tokens_seen": 40239104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982186559679038, + "loss": 3.2075, + "theoretical_loss": 5.485434219963315, + "tokens_seen": 40304640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982026078234705, + "loss": 3.0804, + "theoretical_loss": 5.484047922017476, + "tokens_seen": 40370176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981865596790372, + "loss": 3.199, + "theoretical_loss": 5.482664501702036, + "tokens_seen": 40435712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981705115346039, + "loss": 3.0804, + "theoretical_loss": 5.481283948394303, + "tokens_seen": 40501248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981544633901706, + "loss": 3.1914, + "theoretical_loss": 5.479906251527901, + "tokens_seen": 40566784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981384152457373, + "loss": 3.1122, + "theoretical_loss": 5.478531400592393, + "tokens_seen": 40632320 + }, + { + "epoch": 0.01, + "learning_rate": 0.000798122367101304, + "loss": 3.0153, + "theoretical_loss": 5.4771593851328815, + "tokens_seen": 40697856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981063189568707, + "loss": 3.0745, + "theoretical_loss": 5.475790194749637, + "tokens_seen": 40763392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980902708124374, + "loss": 3.2291, + "theoretical_loss": 5.474423819097713, + "tokens_seen": 40828928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980742226680041, + "loss": 3.072, + "theoretical_loss": 5.473060247886572, + "tokens_seen": 40894464 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980581745235708, + "loss": 3.0078, + "theoretical_loss": 5.471699470879715, + "tokens_seen": 40960000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980421263791375, + "loss": 3.0983, + "theoretical_loss": 5.470341477894303, + "tokens_seen": 41025536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980260782347042, + "loss": 3.1138, + "theoretical_loss": 5.468986258800805, + "tokens_seen": 41091072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980100300902709, + "loss": 3.0204, + "theoretical_loss": 5.4676338035226175, + "tokens_seen": 41156608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007979939819458376, + "loss": 3.2892, + "theoretical_loss": 5.466284102035719, + "tokens_seen": 41222144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007979779338014043, + "loss": 3.1621, + "theoretical_loss": 5.464937144368307, + "tokens_seen": 41287680 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797961885656971, + "loss": 3.0173, + "theoretical_loss": 5.463592920600435, + "tokens_seen": 41353216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007979458375125376, + "loss": 3.0774, + "theoretical_loss": 5.46225142086368, + "tokens_seen": 41418752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007979297893681044, + "loss": 3.1238, + "theoretical_loss": 5.460912635340775, + "tokens_seen": 41484288 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797913741223671, + "loss": 3.0618, + "theoretical_loss": 5.459576554265276, + "tokens_seen": 41549824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978976930792378, + "loss": 3.1313, + "theoretical_loss": 5.458243167921212, + "tokens_seen": 41615360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978816449348044, + "loss": 3.2165, + "theoretical_loss": 5.456912466642748, + "tokens_seen": 41680896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978655967903712, + "loss": 2.9085, + "theoretical_loss": 5.455584440813853, + "tokens_seen": 41746432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978495486459378, + "loss": 2.9859, + "theoretical_loss": 5.454259080867951, + "tokens_seen": 41811968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978335005015046, + "loss": 2.9411, + "theoretical_loss": 5.4529363772876085, + "tokens_seen": 41877504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978174523570712, + "loss": 3.0094, + "theoretical_loss": 5.451616320604193, + "tokens_seen": 41943040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978014042126379, + "loss": 2.9767, + "theoretical_loss": 5.450298901397547, + "tokens_seen": 42008576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977853560682046, + "loss": 3.1042, + "theoretical_loss": 5.448984110295672, + "tokens_seen": 42074112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977693079237713, + "loss": 3.0762, + "theoretical_loss": 5.447671937974404, + "tokens_seen": 42139648 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797753259779338, + "loss": 2.9742, + "theoretical_loss": 5.446362375157092, + "tokens_seen": 42205184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977372116349047, + "loss": 3.2779, + "theoretical_loss": 5.445055412614287, + "tokens_seen": 42270720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977211634904714, + "loss": 2.9849, + "theoretical_loss": 5.443751041163428, + "tokens_seen": 42336256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977051153460381, + "loss": 3.0884, + "theoretical_loss": 5.442449251668533, + "tokens_seen": 42401792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976890672016048, + "loss": 3.0079, + "theoretical_loss": 5.441150035039886, + "tokens_seen": 42467328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976730190571715, + "loss": 3.0901, + "theoretical_loss": 5.439853382233738, + "tokens_seen": 42532864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 20768, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.992009401321411, + "objective/train/theoretical_loss": 5.438559284252003, + "objective/train/tokens_used": 63058400, + "theoretical_loss": 5.438559284252003, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976569709127382, + "loss": 2.9284, + "theoretical_loss": 5.438559284252003, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976409227683049, + "loss": 2.9027, + "theoretical_loss": 5.437267732141949, + "tokens_seen": 42663936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976248746238716, + "loss": 2.9383, + "theoretical_loss": 5.435978716995917, + "tokens_seen": 42729472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976088264794383, + "loss": 3.0424, + "theoretical_loss": 5.434692229951009, + "tokens_seen": 42795008 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797592778335005, + "loss": 3.0061, + "theoretical_loss": 5.433408262188802, + "tokens_seen": 42860544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975767301905718, + "loss": 3.0298, + "theoretical_loss": 5.432126804935059, + "tokens_seen": 42926080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975606820461384, + "loss": 3.1409, + "theoretical_loss": 5.430847849459438, + "tokens_seen": 42991616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975446339017052, + "loss": 3.0722, + "theoretical_loss": 5.429571387075203, + "tokens_seen": 43057152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975285857572718, + "loss": 3.055, + "theoretical_loss": 5.428297409138947, + "tokens_seen": 43122688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975125376128386, + "loss": 2.936, + "theoretical_loss": 5.427025907050307, + "tokens_seen": 43188224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974964894684052, + "loss": 3.0608, + "theoretical_loss": 5.425756872251682, + "tokens_seen": 43253760 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797480441323972, + "loss": 2.969, + "theoretical_loss": 5.42449029622796, + "tokens_seen": 43319296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974643931795386, + "loss": 2.982, + "theoretical_loss": 5.423226170506243, + "tokens_seen": 43384832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974483450351054, + "loss": 2.959, + "theoretical_loss": 5.421964486655567, + "tokens_seen": 43450368 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797432296890672, + "loss": 3.1053, + "theoretical_loss": 5.420705236286646, + "tokens_seen": 43515904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974162487462388, + "loss": 3.1943, + "theoretical_loss": 5.41944841105159, + "tokens_seen": 43581440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974002006018054, + "loss": 2.9255, + "theoretical_loss": 5.418194002643643, + "tokens_seen": 43646976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973841524573722, + "loss": 3.0381, + "theoretical_loss": 5.416942002796926, + "tokens_seen": 43712512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973681043129388, + "loss": 2.9989, + "theoretical_loss": 5.415692403286162, + "tokens_seen": 43778048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973520561685056, + "loss": 3.0323, + "theoretical_loss": 5.414445195926426, + "tokens_seen": 43843584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973360080240722, + "loss": 2.9815, + "theoretical_loss": 5.413200372572884, + "tokens_seen": 43909120 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797319959879639, + "loss": 3.0141, + "theoretical_loss": 5.411957925120538, + "tokens_seen": 43974656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973039117352056, + "loss": 3.0708, + "theoretical_loss": 5.410717845503969, + "tokens_seen": 44040192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972878635907724, + "loss": 2.8975, + "theoretical_loss": 5.40948012569709, + "tokens_seen": 44105728 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797271815446339, + "loss": 3.0037, + "theoretical_loss": 5.408244757712893, + "tokens_seen": 44171264 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972557673019058, + "loss": 2.9777, + "theoretical_loss": 5.407011733603199, + "tokens_seen": 44236800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972397191574724, + "loss": 2.885, + "theoretical_loss": 5.4057810454584185, + "tokens_seen": 44302336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972236710130392, + "loss": 2.9414, + "theoretical_loss": 5.404552685407301, + "tokens_seen": 44367872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972076228686058, + "loss": 3.097, + "theoretical_loss": 5.403326645616698, + "tokens_seen": 44433408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007971915747241726, + "loss": 2.9061, + "theoretical_loss": 5.402102918291316, + "tokens_seen": 44498944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007971755265797392, + "loss": 3.0093, + "theoretical_loss": 5.400881495673488, + "tokens_seen": 44564480 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797159478435306, + "loss": 3.103, + "theoretical_loss": 5.3996623700429245, + "tokens_seen": 44630016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007971434302908726, + "loss": 2.9274, + "theoretical_loss": 5.398445533716492, + "tokens_seen": 44695552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007971273821464394, + "loss": 3.0244, + "theoretical_loss": 5.3972309790479684, + "tokens_seen": 44761088 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797111334002006, + "loss": 2.9874, + "theoretical_loss": 5.39601869842782, + "tokens_seen": 44826624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970952858575728, + "loss": 2.893, + "theoretical_loss": 5.394808684282968, + "tokens_seen": 44892160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970792377131394, + "loss": 2.9119, + "theoretical_loss": 5.393600929076564, + "tokens_seen": 44957696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970631895687062, + "loss": 2.845, + "theoretical_loss": 5.392395425307758, + "tokens_seen": 45023232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970471414242728, + "loss": 2.8557, + "theoretical_loss": 5.391192165511484, + "tokens_seen": 45088768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970310932798396, + "loss": 2.9916, + "theoretical_loss": 5.389991142258227, + "tokens_seen": 45154304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970150451354062, + "loss": 3.0289, + "theoretical_loss": 5.38879234815381, + "tokens_seen": 45219840 + }, + { + "epoch": 0.01, + "learning_rate": 0.000796998996990973, + "loss": 2.857, + "theoretical_loss": 5.387595775839175, + "tokens_seen": 45285376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969829488465396, + "loss": 2.9597, + "theoretical_loss": 5.386401417990159, + "tokens_seen": 45350912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969669007021064, + "loss": 2.9621, + "theoretical_loss": 5.385209267317286, + "tokens_seen": 45416448 + }, + { + "epoch": 0.01, + "learning_rate": 0.000796950852557673, + "loss": 2.9259, + "theoretical_loss": 5.3840193165655466, + "tokens_seen": 45481984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969348044132398, + "loss": 2.8076, + "theoretical_loss": 5.382831558514191, + "tokens_seen": 45547520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969187562688065, + "loss": 2.9423, + "theoretical_loss": 5.3816459859765144, + "tokens_seen": 45613056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969027081243732, + "loss": 2.7761, + "theoretical_loss": 5.380462591799651, + "tokens_seen": 45678592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968866599799399, + "loss": 2.8484, + "theoretical_loss": 5.379281368864362, + "tokens_seen": 45744128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968706118355066, + "loss": 2.8477, + "theoretical_loss": 5.378102310084834, + "tokens_seen": 45809664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 22096, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.943756580352783, + "objective/train/theoretical_loss": 5.376925408408475, + "objective/train/tokens_used": 66335200, + "theoretical_loss": 5.376925408408475, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968545636910733, + "loss": 2.9656, + "theoretical_loss": 5.376925408408475, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00079683851554664, + "loss": 2.8387, + "theoretical_loss": 5.375750656815705, + "tokens_seen": 45940736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968224674022067, + "loss": 2.9168, + "theoretical_loss": 5.37457804831976, + "tokens_seen": 46006272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968064192577734, + "loss": 2.8562, + "theoretical_loss": 5.373407575966493, + "tokens_seen": 46071808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967903711133401, + "loss": 2.8091, + "theoretical_loss": 5.372239232834173, + "tokens_seen": 46137344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967743229689068, + "loss": 2.9054, + "theoretical_loss": 5.3710730120332855, + "tokens_seen": 46202880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967582748244735, + "loss": 2.9022, + "theoretical_loss": 5.369908906706346, + "tokens_seen": 46268416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967422266800402, + "loss": 2.9656, + "theoretical_loss": 5.368746910027696, + "tokens_seen": 46333952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967261785356069, + "loss": 2.7845, + "theoretical_loss": 5.3675870152033145, + "tokens_seen": 46399488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967101303911736, + "loss": 2.8521, + "theoretical_loss": 5.36642921547063, + "tokens_seen": 46465024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966940822467403, + "loss": 2.8249, + "theoretical_loss": 5.365273504098326, + "tokens_seen": 46530560 + }, + { + "epoch": 0.01, + "learning_rate": 0.000796678034102307, + "loss": 2.7694, + "theoretical_loss": 5.364119874386157, + "tokens_seen": 46596096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966619859578737, + "loss": 2.9288, + "theoretical_loss": 5.362968319664756, + "tokens_seen": 46661632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966459378134404, + "loss": 2.9025, + "theoretical_loss": 5.361818833295455, + "tokens_seen": 46727168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966298896690071, + "loss": 2.9219, + "theoretical_loss": 5.3606714086701, + "tokens_seen": 46792704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966138415245738, + "loss": 2.9099, + "theoretical_loss": 5.359526039210861, + "tokens_seen": 46858240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965977933801405, + "loss": 2.88, + "theoretical_loss": 5.358382718370063, + "tokens_seen": 46923776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965817452357072, + "loss": 2.8903, + "theoretical_loss": 5.357241439629992, + "tokens_seen": 46989312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965656970912739, + "loss": 2.7691, + "theoretical_loss": 5.3561021965027305, + "tokens_seen": 47054848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965496489468405, + "loss": 2.8294, + "theoretical_loss": 5.354964982529967, + "tokens_seen": 47120384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965336008024073, + "loss": 2.8102, + "theoretical_loss": 5.353829791282831, + "tokens_seen": 47185920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965175526579739, + "loss": 2.7822, + "theoretical_loss": 5.35269661636171, + "tokens_seen": 47251456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965015045135407, + "loss": 2.8876, + "theoretical_loss": 5.35156545139608, + "tokens_seen": 47316992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964854563691073, + "loss": 2.7635, + "theoretical_loss": 5.350436290044332, + "tokens_seen": 47382528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964694082246741, + "loss": 2.8377, + "theoretical_loss": 5.3493091259936, + "tokens_seen": 47448064 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964533600802407, + "loss": 2.8324, + "theoretical_loss": 5.348183952959596, + "tokens_seen": 47513600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964373119358075, + "loss": 2.7908, + "theoretical_loss": 5.347060764686436, + "tokens_seen": 47579136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964212637913741, + "loss": 2.8496, + "theoretical_loss": 5.345939554946472, + "tokens_seen": 47644672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964052156469408, + "loss": 2.7488, + "theoretical_loss": 5.344820317540132, + "tokens_seen": 47710208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963891675025075, + "loss": 2.8114, + "theoretical_loss": 5.343703046295751, + "tokens_seen": 47775744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963731193580743, + "loss": 2.8671, + "theoretical_loss": 5.342587735069408, + "tokens_seen": 47841280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963570712136409, + "loss": 2.9205, + "theoretical_loss": 5.3414743777447615, + "tokens_seen": 47906816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963410230692077, + "loss": 2.7384, + "theoretical_loss": 5.340362968232892, + "tokens_seen": 47972352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963249749247743, + "loss": 2.9322, + "theoretical_loss": 5.339253500472138, + "tokens_seen": 48037888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963089267803411, + "loss": 2.6983, + "theoretical_loss": 5.338145968427941, + "tokens_seen": 48103424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962928786359077, + "loss": 2.8289, + "theoretical_loss": 5.337040366092687, + "tokens_seen": 48168960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962768304914745, + "loss": 2.9222, + "theoretical_loss": 5.335936687485542, + "tokens_seen": 48234496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962607823470411, + "loss": 2.7881, + "theoretical_loss": 5.334834926652308, + "tokens_seen": 48300032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962447342026079, + "loss": 2.7958, + "theoretical_loss": 5.33373507766526, + "tokens_seen": 48365568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962286860581745, + "loss": 2.8646, + "theoretical_loss": 5.332637134622999, + "tokens_seen": 48431104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962126379137413, + "loss": 2.7199, + "theoretical_loss": 5.331541091650294, + "tokens_seen": 48496640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961965897693079, + "loss": 2.8311, + "theoretical_loss": 5.330446942897931, + "tokens_seen": 48562176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961805416248747, + "loss": 2.8684, + "theoretical_loss": 5.32935468254257, + "tokens_seen": 48627712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961644934804413, + "loss": 2.822, + "theoretical_loss": 5.328264304786586, + "tokens_seen": 48693248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961484453360081, + "loss": 2.7969, + "theoretical_loss": 5.327175803857929, + "tokens_seen": 48758784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961323971915747, + "loss": 2.7726, + "theoretical_loss": 5.326089174009972, + "tokens_seen": 48824320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961163490471415, + "loss": 2.7235, + "theoretical_loss": 5.325004409521368, + "tokens_seen": 48889856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961003009027081, + "loss": 2.7985, + "theoretical_loss": 5.323921504695903, + "tokens_seen": 48955392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960842527582749, + "loss": 2.9327, + "theoretical_loss": 5.322840453862356, + "tokens_seen": 49020928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960682046138415, + "loss": 2.7259, + "theoretical_loss": 5.3217612513743475, + "tokens_seen": 49086464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 22959, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.8861923217773438, + "objective/train/theoretical_loss": 5.320683891610208, + "objective/train/tokens_used": 69612000, + "theoretical_loss": 5.320683891610208, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960521564694083, + "loss": 2.8363, + "theoretical_loss": 5.320683891610208, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960361083249749, + "loss": 2.8106, + "theoretical_loss": 5.3196083689728315, + "tokens_seen": 49217536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960200601805417, + "loss": 2.8124, + "theoretical_loss": 5.318534677889536, + "tokens_seen": 49283072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960040120361083, + "loss": 2.6993, + "theoretical_loss": 5.317462812811925, + "tokens_seen": 49348608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007959879638916751, + "loss": 2.7378, + "theoretical_loss": 5.316392768215751, + "tokens_seen": 49414144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007959719157472417, + "loss": 2.7691, + "theoretical_loss": 5.315324538600778, + "tokens_seen": 49479680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007959558676028085, + "loss": 2.9151, + "theoretical_loss": 5.31425811849064, + "tokens_seen": 49545216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007959398194583751, + "loss": 2.616, + "theoretical_loss": 5.313193502432718, + "tokens_seen": 49610752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007959237713139419, + "loss": 2.7306, + "theoretical_loss": 5.312130684997992, + "tokens_seen": 49676288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007959077231695085, + "loss": 2.801, + "theoretical_loss": 5.31106966078092, + "tokens_seen": 49741824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958916750250753, + "loss": 2.736, + "theoretical_loss": 5.310010424399295, + "tokens_seen": 49807360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958756268806419, + "loss": 2.7149, + "theoretical_loss": 5.308952970494124, + "tokens_seen": 49872896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958595787362087, + "loss": 2.7643, + "theoretical_loss": 5.307897293729489, + "tokens_seen": 49938432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958435305917753, + "loss": 2.7688, + "theoretical_loss": 5.306843388792423, + "tokens_seen": 50003968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958274824473421, + "loss": 2.6344, + "theoretical_loss": 5.305791250392778, + "tokens_seen": 50069504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958114343029087, + "loss": 2.7253, + "theoretical_loss": 5.304740873263098, + "tokens_seen": 50135040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957953861584755, + "loss": 2.7176, + "theoretical_loss": 5.303692252158496, + "tokens_seen": 50200576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957793380140421, + "loss": 2.7815, + "theoretical_loss": 5.302645381856522, + "tokens_seen": 50266112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957632898696089, + "loss": 2.7198, + "theoretical_loss": 5.30160025715704, + "tokens_seen": 50331648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957472417251755, + "loss": 2.5469, + "theoretical_loss": 5.300556872882108, + "tokens_seen": 50397184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957311935807423, + "loss": 2.6499, + "theoretical_loss": 5.299515223875848, + "tokens_seen": 50462720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957151454363089, + "loss": 2.7165, + "theoretical_loss": 5.2984753050043265, + "tokens_seen": 50528256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956990972918757, + "loss": 2.6517, + "theoretical_loss": 5.2974371111554355, + "tokens_seen": 50593792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956830491474423, + "loss": 2.6449, + "theoretical_loss": 5.296400637238764, + "tokens_seen": 50659328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956670010030091, + "loss": 2.5609, + "theoretical_loss": 5.295365878185488, + "tokens_seen": 50724864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956509528585757, + "loss": 2.4886, + "theoretical_loss": 5.294332828948243, + "tokens_seen": 50790400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956349047141425, + "loss": 2.6212, + "theoretical_loss": 5.2933014845010105, + "tokens_seen": 50855936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956188565697091, + "loss": 2.6007, + "theoretical_loss": 5.292271839838996, + "tokens_seen": 50921472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956028084252759, + "loss": 2.7466, + "theoretical_loss": 5.291243889978516, + "tokens_seen": 50987008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955867602808426, + "loss": 2.7354, + "theoretical_loss": 5.290217629956882, + "tokens_seen": 51052544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955707121364093, + "loss": 2.8293, + "theoretical_loss": 5.289193054832282, + "tokens_seen": 51118080 + }, + { + "epoch": 0.02, + "learning_rate": 0.000795554663991976, + "loss": 2.7158, + "theoretical_loss": 5.288170159683668, + "tokens_seen": 51183616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955386158475427, + "loss": 2.6579, + "theoretical_loss": 5.287148939610645, + "tokens_seen": 51249152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955225677031094, + "loss": 2.5181, + "theoretical_loss": 5.28612938973335, + "tokens_seen": 51314688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955065195586761, + "loss": 2.7695, + "theoretical_loss": 5.285111505192349, + "tokens_seen": 51380224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954904714142428, + "loss": 2.6868, + "theoretical_loss": 5.284095281148522, + "tokens_seen": 51445760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954744232698095, + "loss": 2.747, + "theoretical_loss": 5.283080712782951, + "tokens_seen": 51511296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954583751253762, + "loss": 2.6446, + "theoretical_loss": 5.282067795296811, + "tokens_seen": 51576832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954423269809429, + "loss": 2.6414, + "theoretical_loss": 5.28105652391126, + "tokens_seen": 51642368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954262788365096, + "loss": 2.6543, + "theoretical_loss": 5.280046893867333, + "tokens_seen": 51707904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954102306920763, + "loss": 2.5674, + "theoretical_loss": 5.279038900425832, + "tokens_seen": 51773440 + }, + { + "epoch": 0.02, + "learning_rate": 0.000795394182547643, + "loss": 2.6305, + "theoretical_loss": 5.278032538867221, + "tokens_seen": 51838976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953781344032097, + "loss": 2.5573, + "theoretical_loss": 5.277027804491517, + "tokens_seen": 51904512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953620862587764, + "loss": 2.6, + "theoretical_loss": 5.2760246926181855, + "tokens_seen": 51970048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953460381143431, + "loss": 2.6884, + "theoretical_loss": 5.275023198586037, + "tokens_seen": 52035584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953299899699098, + "loss": 2.6203, + "theoretical_loss": 5.274023317753125, + "tokens_seen": 52101120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953139418254765, + "loss": 2.6784, + "theoretical_loss": 5.273025045496631, + "tokens_seen": 52166656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952978936810432, + "loss": 2.5281, + "theoretical_loss": 5.272028377212777, + "tokens_seen": 52232192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952818455366099, + "loss": 2.5807, + "theoretical_loss": 5.271033308316715, + "tokens_seen": 52297728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952657973921766, + "loss": 2.6348, + "theoretical_loss": 5.270039834242425, + "tokens_seen": 52363264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 24243, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.5289902687072754, + "objective/train/theoretical_loss": 5.269047950442619, + "objective/train/tokens_used": 72888800, + "theoretical_loss": 5.269047950442619, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952497492477433, + "loss": 2.485, + "theoretical_loss": 5.269047950442619, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.00079523370110331, + "loss": 2.6376, + "theoretical_loss": 5.2680576523886335, + "tokens_seen": 52494336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952176529588767, + "loss": 2.7194, + "theoretical_loss": 5.2670689355703395, + "tokens_seen": 52559872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952016048144434, + "loss": 2.6341, + "theoretical_loss": 5.266081795496035, + "tokens_seen": 52625408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951855566700101, + "loss": 2.5662, + "theoretical_loss": 5.265096227692354, + "tokens_seen": 52690944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951695085255768, + "loss": 2.6624, + "theoretical_loss": 5.264112227704162, + "tokens_seen": 52756480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951534603811434, + "loss": 2.7557, + "theoretical_loss": 5.263129791094466, + "tokens_seen": 52822016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951374122367102, + "loss": 2.6062, + "theoretical_loss": 5.26214891344431, + "tokens_seen": 52887552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951213640922768, + "loss": 2.5699, + "theoretical_loss": 5.26116959035269, + "tokens_seen": 52953088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951053159478436, + "loss": 2.506, + "theoretical_loss": 5.260191817436452, + "tokens_seen": 53018624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007950892678034102, + "loss": 2.6012, + "theoretical_loss": 5.259215590330195, + "tokens_seen": 53084160 + }, + { + "epoch": 0.02, + "learning_rate": 0.000795073219658977, + "loss": 2.5899, + "theoretical_loss": 5.258240904686186, + "tokens_seen": 53149696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007950571715145436, + "loss": 2.5064, + "theoretical_loss": 5.25726775617426, + "tokens_seen": 53215232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007950411233701104, + "loss": 2.4435, + "theoretical_loss": 5.256296140481728, + "tokens_seen": 53280768 + }, + { + "epoch": 0.02, + "learning_rate": 0.000795025075225677, + "loss": 2.5262, + "theoretical_loss": 5.255326053313292, + "tokens_seen": 53346304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007950090270812438, + "loss": 2.5646, + "theoretical_loss": 5.254357490390941, + "tokens_seen": 53411840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949929789368104, + "loss": 2.4133, + "theoretical_loss": 5.253390447453873, + "tokens_seen": 53477376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949769307923772, + "loss": 2.6187, + "theoretical_loss": 5.252424920258396, + "tokens_seen": 53542912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949608826479438, + "loss": 2.5912, + "theoretical_loss": 5.251460904577845, + "tokens_seen": 53608448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949448345035106, + "loss": 2.5508, + "theoretical_loss": 5.250498396202486, + "tokens_seen": 53673984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949287863590772, + "loss": 2.5112, + "theoretical_loss": 5.249537390939436, + "tokens_seen": 53739520 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794912738214644, + "loss": 2.4452, + "theoretical_loss": 5.248577884612564, + "tokens_seen": 53805056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948966900702106, + "loss": 2.4712, + "theoretical_loss": 5.247619873062416, + "tokens_seen": 53870592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948806419257774, + "loss": 2.5985, + "theoretical_loss": 5.246663352146122, + "tokens_seen": 53936128 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794864593781344, + "loss": 2.6442, + "theoretical_loss": 5.245708317737305, + "tokens_seen": 54001664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948485456369108, + "loss": 2.4876, + "theoretical_loss": 5.244754765726007, + "tokens_seen": 54067200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948324974924774, + "loss": 2.5376, + "theoretical_loss": 5.243802692018596, + "tokens_seen": 54132736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948164493480442, + "loss": 2.4553, + "theoretical_loss": 5.242852092537677, + "tokens_seen": 54198272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948004012036108, + "loss": 2.6929, + "theoretical_loss": 5.241902963222023, + "tokens_seen": 54263808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007947843530591776, + "loss": 2.5035, + "theoretical_loss": 5.240955300026476, + "tokens_seen": 54329344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007947683049147442, + "loss": 2.5704, + "theoretical_loss": 5.240009098921874, + "tokens_seen": 54394880 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794752256770311, + "loss": 2.4209, + "theoretical_loss": 5.239064355894965, + "tokens_seen": 54460416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007947362086258776, + "loss": 2.5877, + "theoretical_loss": 5.238121066948324, + "tokens_seen": 54525952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007947201604814444, + "loss": 2.4356, + "theoretical_loss": 5.237179228100272, + "tokens_seen": 54591488 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794704112337011, + "loss": 2.498, + "theoretical_loss": 5.2362388353848, + "tokens_seen": 54657024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946880641925778, + "loss": 2.426, + "theoretical_loss": 5.23529988485148, + "tokens_seen": 54722560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946720160481444, + "loss": 2.527, + "theoretical_loss": 5.2343623725653945, + "tokens_seen": 54788096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946559679037112, + "loss": 2.5915, + "theoretical_loss": 5.233426294607049, + "tokens_seen": 54853632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946399197592778, + "loss": 2.5706, + "theoretical_loss": 5.232491647072299, + "tokens_seen": 54919168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946238716148446, + "loss": 2.5656, + "theoretical_loss": 5.231558426072269, + "tokens_seen": 54984704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946078234704112, + "loss": 2.4501, + "theoretical_loss": 5.230626627733274, + "tokens_seen": 55050240 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794591775325978, + "loss": 2.5487, + "theoretical_loss": 5.2296962481967455, + "tokens_seen": 55115776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007945757271815446, + "loss": 2.3679, + "theoretical_loss": 5.228767283619151, + "tokens_seen": 55181312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007945596790371114, + "loss": 2.6326, + "theoretical_loss": 5.227839730171918, + "tokens_seen": 55246848 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794543630892678, + "loss": 2.4476, + "theoretical_loss": 5.2269135840413625, + "tokens_seen": 55312384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007945275827482448, + "loss": 2.309, + "theoretical_loss": 5.225988841428606, + "tokens_seen": 55377920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007945115346038114, + "loss": 2.2936, + "theoretical_loss": 5.225065498549508, + "tokens_seen": 55443456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944954864593782, + "loss": 2.4656, + "theoretical_loss": 5.224143551634588, + "tokens_seen": 55508992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944794383149448, + "loss": 2.428, + "theoretical_loss": 5.22322299692895, + "tokens_seen": 55574528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944633901705116, + "loss": 2.4442, + "theoretical_loss": 5.222303830692211, + "tokens_seen": 55640064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 25141, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.6088738441467285, + "objective/train/theoretical_loss": 5.22138604919843, + "objective/train/tokens_used": 76165600, + "theoretical_loss": 5.22138604919843, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944473420260782, + "loss": 2.5252, + "theoretical_loss": 5.22138604919843, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794431293881645, + "loss": 2.6352, + "theoretical_loss": 5.220469648736031, + "tokens_seen": 55771136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944152457372116, + "loss": 2.4382, + "theoretical_loss": 5.219554625607731, + "tokens_seen": 55836672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943991975927784, + "loss": 2.5073, + "theoretical_loss": 5.218640976130474, + "tokens_seen": 55902208 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794383149448345, + "loss": 2.502, + "theoretical_loss": 5.217728696635353, + "tokens_seen": 55967744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943671013039118, + "loss": 2.3945, + "theoretical_loss": 5.216817783467543, + "tokens_seen": 56033280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943510531594784, + "loss": 2.6394, + "theoretical_loss": 5.215908232986227, + "tokens_seen": 56098816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943350050150452, + "loss": 2.4114, + "theoretical_loss": 5.215000041564533, + "tokens_seen": 56164352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943189568706118, + "loss": 2.47, + "theoretical_loss": 5.214093205589455, + "tokens_seen": 56229888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943029087261786, + "loss": 2.4864, + "theoretical_loss": 5.213187721461791, + "tokens_seen": 56295424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007942868605817452, + "loss": 2.5074, + "theoretical_loss": 5.212283585596071, + "tokens_seen": 56360960 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794270812437312, + "loss": 2.3196, + "theoretical_loss": 5.211380794420492, + "tokens_seen": 56426496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007942547642928786, + "loss": 2.3877, + "theoretical_loss": 5.210479344376845, + "tokens_seen": 56492032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007942387161484454, + "loss": 2.5349, + "theoretical_loss": 5.209579231920451, + "tokens_seen": 56557568 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794222668004012, + "loss": 2.4561, + "theoretical_loss": 5.208680453520092, + "tokens_seen": 56623104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007942066198595788, + "loss": 2.4743, + "theoretical_loss": 5.2077830056579515, + "tokens_seen": 56688640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941905717151455, + "loss": 2.5124, + "theoretical_loss": 5.206886884829535, + "tokens_seen": 56754176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941745235707122, + "loss": 2.3724, + "theoretical_loss": 5.205992087543614, + "tokens_seen": 56819712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941584754262789, + "loss": 2.5328, + "theoretical_loss": 5.205098610322162, + "tokens_seen": 56885248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941424272818456, + "loss": 2.6109, + "theoretical_loss": 5.204206449700279, + "tokens_seen": 56950784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941263791374123, + "loss": 2.4425, + "theoretical_loss": 5.203315602226139, + "tokens_seen": 57016320 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794110330992979, + "loss": 2.353, + "theoretical_loss": 5.2024260644609175, + "tokens_seen": 57081856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940942828485457, + "loss": 2.4324, + "theoretical_loss": 5.201537832978731, + "tokens_seen": 57147392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940782347041124, + "loss": 2.3614, + "theoretical_loss": 5.200650904366574, + "tokens_seen": 57212928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940621865596791, + "loss": 2.4856, + "theoretical_loss": 5.199765275224252, + "tokens_seen": 57278464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940461384152458, + "loss": 2.4181, + "theoretical_loss": 5.198880942164323, + "tokens_seen": 57344000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940300902708125, + "loss": 2.351, + "theoretical_loss": 5.197997901812036, + "tokens_seen": 57409536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940140421263792, + "loss": 2.5241, + "theoretical_loss": 5.197116150805264, + "tokens_seen": 57475072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939979939819459, + "loss": 2.3707, + "theoretical_loss": 5.196235685794446, + "tokens_seen": 57540608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939819458375126, + "loss": 2.4242, + "theoretical_loss": 5.195356503442525, + "tokens_seen": 57606144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939658976930794, + "loss": 2.4264, + "theoretical_loss": 5.194478600424889, + "tokens_seen": 57671680 + }, + { + "epoch": 0.02, + "learning_rate": 0.000793949849548646, + "loss": 2.4905, + "theoretical_loss": 5.193601973429306, + "tokens_seen": 57737216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939338014042128, + "loss": 2.3264, + "theoretical_loss": 5.192726619155867, + "tokens_seen": 57802752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939177532597794, + "loss": 2.2947, + "theoretical_loss": 5.191852534316928, + "tokens_seen": 57868288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939017051153462, + "loss": 2.5367, + "theoretical_loss": 5.190979715637049, + "tokens_seen": 57933824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938856569709128, + "loss": 2.4345, + "theoretical_loss": 5.190108159852929, + "tokens_seen": 57999360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938696088264795, + "loss": 2.48, + "theoretical_loss": 5.189237863713357, + "tokens_seen": 58064896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938535606820462, + "loss": 2.5036, + "theoretical_loss": 5.188368823979152, + "tokens_seen": 58130432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938375125376129, + "loss": 2.2739, + "theoretical_loss": 5.187501037423096, + "tokens_seen": 58195968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938214643931796, + "loss": 2.3866, + "theoretical_loss": 5.186634500829886, + "tokens_seen": 58261504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938054162487463, + "loss": 2.4664, + "theoretical_loss": 5.185769210996076, + "tokens_seen": 58327040 + }, + { + "epoch": 0.02, + "learning_rate": 0.000793789368104313, + "loss": 2.2739, + "theoretical_loss": 5.184905164730012, + "tokens_seen": 58392576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937733199598797, + "loss": 2.4244, + "theoretical_loss": 5.184042358851785, + "tokens_seen": 58458112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937572718154463, + "loss": 2.3718, + "theoretical_loss": 5.1831807901931715, + "tokens_seen": 58523648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937412236710131, + "loss": 2.2701, + "theoretical_loss": 5.18232045559757, + "tokens_seen": 58589184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937251755265797, + "loss": 2.3716, + "theoretical_loss": 5.181461351919959, + "tokens_seen": 58654720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937091273821465, + "loss": 2.5091, + "theoretical_loss": 5.180603476026832, + "tokens_seen": 58720256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936930792377131, + "loss": 2.3163, + "theoretical_loss": 5.179746824796146, + "tokens_seen": 58785792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936770310932799, + "loss": 2.2918, + "theoretical_loss": 5.178891395117262, + "tokens_seen": 58851328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936609829488465, + "loss": 2.4097, + "theoretical_loss": 5.178037183890902, + "tokens_seen": 58916864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 26367, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.4226865768432617, + "objective/train/theoretical_loss": 5.177184188029082, + "objective/train/tokens_used": 79442400, + "theoretical_loss": 5.177184188029082, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936449348044133, + "loss": 2.3572, + "theoretical_loss": 5.177184188029082, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936288866599799, + "loss": 2.452, + "theoretical_loss": 5.1763324044550645, + "tokens_seen": 59047936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936128385155467, + "loss": 2.3822, + "theoretical_loss": 5.175481830103307, + "tokens_seen": 59113472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935967903711133, + "loss": 2.376, + "theoretical_loss": 5.174632461919406, + "tokens_seen": 59179008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935807422266801, + "loss": 2.3096, + "theoretical_loss": 5.173784296860043, + "tokens_seen": 59244544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935646940822467, + "loss": 2.4012, + "theoretical_loss": 5.172937331892934, + "tokens_seen": 59310080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935486459378135, + "loss": 2.2779, + "theoretical_loss": 5.17209156399678, + "tokens_seen": 59375616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935325977933801, + "loss": 2.4276, + "theoretical_loss": 5.171246990161208, + "tokens_seen": 59441152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935165496489469, + "loss": 2.4266, + "theoretical_loss": 5.17040360738673, + "tokens_seen": 59506688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935005015045135, + "loss": 2.2993, + "theoretical_loss": 5.169561412684677, + "tokens_seen": 59572224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934844533600803, + "loss": 2.4694, + "theoretical_loss": 5.168720403077165, + "tokens_seen": 59637760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934684052156469, + "loss": 2.2188, + "theoretical_loss": 5.167880575597031, + "tokens_seen": 59703296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934523570712137, + "loss": 2.2279, + "theoretical_loss": 5.167041927287787, + "tokens_seen": 59768832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934363089267803, + "loss": 2.4385, + "theoretical_loss": 5.166204455203575, + "tokens_seen": 59834368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934202607823471, + "loss": 2.2816, + "theoretical_loss": 5.16536815640911, + "tokens_seen": 59899904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934042126379137, + "loss": 2.4746, + "theoretical_loss": 5.1645330279796315, + "tokens_seen": 59965440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933881644934805, + "loss": 2.3473, + "theoretical_loss": 5.1636990670008585, + "tokens_seen": 60030976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933721163490471, + "loss": 2.4463, + "theoretical_loss": 5.162866270568938, + "tokens_seen": 60096512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933560682046139, + "loss": 2.3849, + "theoretical_loss": 5.162034635790397, + "tokens_seen": 60162048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933400200601805, + "loss": 2.384, + "theoretical_loss": 5.1612041597820895, + "tokens_seen": 60227584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933239719157473, + "loss": 2.4535, + "theoretical_loss": 5.160374839671159, + "tokens_seen": 60293120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933079237713139, + "loss": 2.3911, + "theoretical_loss": 5.1595466725949795, + "tokens_seen": 60358656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932918756268807, + "loss": 2.3502, + "theoretical_loss": 5.158719655701115, + "tokens_seen": 60424192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932758274824473, + "loss": 2.4282, + "theoretical_loss": 5.157893786147269, + "tokens_seen": 60489728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932597793380141, + "loss": 2.3445, + "theoretical_loss": 5.157069061101238, + "tokens_seen": 60555264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932437311935807, + "loss": 2.3545, + "theoretical_loss": 5.156245477740866, + "tokens_seen": 60620800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932276830491475, + "loss": 2.2794, + "theoretical_loss": 5.155423033253997, + "tokens_seen": 60686336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932116349047141, + "loss": 2.2316, + "theoretical_loss": 5.154601724838431, + "tokens_seen": 60751872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931955867602809, + "loss": 2.3536, + "theoretical_loss": 5.153781549701872, + "tokens_seen": 60817408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931795386158475, + "loss": 2.4524, + "theoretical_loss": 5.15296250506189, + "tokens_seen": 60882944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931634904714143, + "loss": 2.3219, + "theoretical_loss": 5.152144588145871, + "tokens_seen": 60948480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931474423269809, + "loss": 2.2977, + "theoretical_loss": 5.151327796190975, + "tokens_seen": 61014016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931313941825477, + "loss": 2.2728, + "theoretical_loss": 5.1505121264440845, + "tokens_seen": 61079552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931153460381143, + "loss": 2.4291, + "theoretical_loss": 5.149697576161769, + "tokens_seen": 61145088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930992978936811, + "loss": 2.258, + "theoretical_loss": 5.148884142610232, + "tokens_seen": 61210624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930832497492477, + "loss": 2.3553, + "theoretical_loss": 5.148071823065274, + "tokens_seen": 61276160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930672016048145, + "loss": 2.3682, + "theoretical_loss": 5.147260614812246, + "tokens_seen": 61341696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930511534603811, + "loss": 2.3699, + "theoretical_loss": 5.146450515146002, + "tokens_seen": 61407232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930351053159479, + "loss": 2.2222, + "theoretical_loss": 5.145641521370861, + "tokens_seen": 61472768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930190571715145, + "loss": 2.3013, + "theoretical_loss": 5.144833630800562, + "tokens_seen": 61538304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930030090270813, + "loss": 2.3248, + "theoretical_loss": 5.1440268407582215, + "tokens_seen": 61603840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929869608826479, + "loss": 2.3713, + "theoretical_loss": 5.143221148576288, + "tokens_seen": 61669376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929709127382147, + "loss": 2.1181, + "theoretical_loss": 5.142416551596505, + "tokens_seen": 61734912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929548645937813, + "loss": 2.1712, + "theoretical_loss": 5.141613047169862, + "tokens_seen": 61800448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929388164493481, + "loss": 2.3425, + "theoretical_loss": 5.14081063265656, + "tokens_seen": 61865984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929227683049147, + "loss": 2.2708, + "theoretical_loss": 5.140009305425963, + "tokens_seen": 61931520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929067201604815, + "loss": 2.205, + "theoretical_loss": 5.13920906285656, + "tokens_seen": 61997056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928906720160481, + "loss": 2.3632, + "theoretical_loss": 5.138409902335925, + "tokens_seen": 62062592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928746238716149, + "loss": 2.2693, + "theoretical_loss": 5.137611821260673, + "tokens_seen": 62128128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928585757271815, + "loss": 2.2251, + "theoretical_loss": 5.13681481703642, + "tokens_seen": 62193664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 27626, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.3975930213928223, + "objective/train/theoretical_loss": 5.136018887077743, + "objective/train/tokens_used": 82719200, + "theoretical_loss": 5.136018887077743, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928425275827483, + "loss": 2.3101, + "theoretical_loss": 5.136018887077743, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792826479438315, + "loss": 2.446, + "theoretical_loss": 5.13522402880814, + "tokens_seen": 62324736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928104312938817, + "loss": 2.4389, + "theoretical_loss": 5.13443023965999, + "tokens_seen": 62390272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927943831494484, + "loss": 2.3297, + "theoretical_loss": 5.13363751707451, + "tokens_seen": 62455808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927783350050151, + "loss": 2.2738, + "theoretical_loss": 5.132845858501721, + "tokens_seen": 62521344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927622868605819, + "loss": 2.2318, + "theoretical_loss": 5.132055261400403, + "tokens_seen": 62586880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927462387161485, + "loss": 2.193, + "theoretical_loss": 5.131265723238059, + "tokens_seen": 62652416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927301905717153, + "loss": 2.2944, + "theoretical_loss": 5.130477241490876, + "tokens_seen": 62717952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927141424272819, + "loss": 2.2733, + "theoretical_loss": 5.129689813643683, + "tokens_seen": 62783488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926980942828487, + "loss": 2.2248, + "theoretical_loss": 5.128903437189917, + "tokens_seen": 62849024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926820461384153, + "loss": 2.2887, + "theoretical_loss": 5.128118109631581, + "tokens_seen": 62914560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926659979939821, + "loss": 2.31, + "theoretical_loss": 5.127333828479208, + "tokens_seen": 62980096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926499498495487, + "loss": 2.2063, + "theoretical_loss": 5.126550591251821, + "tokens_seen": 63045632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926339017051155, + "loss": 2.3662, + "theoretical_loss": 5.125768395476896, + "tokens_seen": 63111168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926178535606821, + "loss": 2.2167, + "theoretical_loss": 5.124987238690329, + "tokens_seen": 63176704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926018054162489, + "loss": 2.3044, + "theoretical_loss": 5.124207118436388, + "tokens_seen": 63242240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925857572718155, + "loss": 2.3506, + "theoretical_loss": 5.123428032267688, + "tokens_seen": 63307776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925697091273823, + "loss": 2.2806, + "theoretical_loss": 5.122649977745146, + "tokens_seen": 63373312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925536609829489, + "loss": 2.3118, + "theoretical_loss": 5.121872952437947, + "tokens_seen": 63438848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925376128385157, + "loss": 2.3615, + "theoretical_loss": 5.121096953923507, + "tokens_seen": 63504384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925215646940823, + "loss": 2.3037, + "theoretical_loss": 5.1203219797874375, + "tokens_seen": 63569920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925055165496491, + "loss": 2.3262, + "theoretical_loss": 5.119548027623509, + "tokens_seen": 63635456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924894684052157, + "loss": 2.3319, + "theoretical_loss": 5.118775095033612, + "tokens_seen": 63700992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924734202607824, + "loss": 2.2909, + "theoretical_loss": 5.118003179627729, + "tokens_seen": 63766528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924573721163491, + "loss": 2.24, + "theoretical_loss": 5.11723227902389, + "tokens_seen": 63832064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924413239719158, + "loss": 2.3249, + "theoretical_loss": 5.1164623908481435, + "tokens_seen": 63897600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924252758274825, + "loss": 2.2101, + "theoretical_loss": 5.115693512734515, + "tokens_seen": 63963136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924092276830492, + "loss": 2.1202, + "theoretical_loss": 5.114925642324982, + "tokens_seen": 64028672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923931795386159, + "loss": 2.267, + "theoretical_loss": 5.114158777269429, + "tokens_seen": 64094208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923771313941826, + "loss": 2.2321, + "theoretical_loss": 5.113392915225617, + "tokens_seen": 64159744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923610832497493, + "loss": 2.35, + "theoretical_loss": 5.112628053859153, + "tokens_seen": 64225280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792345035105316, + "loss": 2.0774, + "theoretical_loss": 5.1118641908434475, + "tokens_seen": 64290816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923289869608826, + "loss": 2.2412, + "theoretical_loss": 5.111101323859687, + "tokens_seen": 64356352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923129388164494, + "loss": 2.3277, + "theoretical_loss": 5.1103394505968005, + "tokens_seen": 64421888 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792296890672016, + "loss": 2.315, + "theoretical_loss": 5.109578568751417, + "tokens_seen": 64487424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922808425275828, + "loss": 2.413, + "theoretical_loss": 5.108818676027845, + "tokens_seen": 64552960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922647943831494, + "loss": 2.1771, + "theoretical_loss": 5.108059770138027, + "tokens_seen": 64618496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922487462387162, + "loss": 2.3026, + "theoretical_loss": 5.107301848801519, + "tokens_seen": 64684032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922326980942828, + "loss": 2.253, + "theoretical_loss": 5.106544909745443, + "tokens_seen": 64749568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922166499498496, + "loss": 2.1783, + "theoretical_loss": 5.105788950704467, + "tokens_seen": 64815104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922006018054162, + "loss": 2.2324, + "theoretical_loss": 5.105033969420764, + "tokens_seen": 64880640 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792184553660983, + "loss": 2.2459, + "theoretical_loss": 5.104279963643982, + "tokens_seen": 64946176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007921685055165496, + "loss": 2.1426, + "theoretical_loss": 5.103526931131217, + "tokens_seen": 65011712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007921524573721164, + "loss": 2.2505, + "theoretical_loss": 5.102774869646972, + "tokens_seen": 65077248 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792136409227683, + "loss": 2.0972, + "theoretical_loss": 5.102023776963131, + "tokens_seen": 65142784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007921203610832498, + "loss": 2.1683, + "theoretical_loss": 5.101273650858926, + "tokens_seen": 65208320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007921043129388164, + "loss": 2.2065, + "theoretical_loss": 5.100524489120902, + "tokens_seen": 65273856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920882647943832, + "loss": 2.2527, + "theoretical_loss": 5.099776289542893, + "tokens_seen": 65339392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920722166499498, + "loss": 2.1725, + "theoretical_loss": 5.099029049925985, + "tokens_seen": 65404928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920561685055166, + "loss": 2.2228, + "theoretical_loss": 5.098282768078483, + "tokens_seen": 65470464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 28581, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.2630820274353027, + "objective/train/theoretical_loss": 5.097537441815887, + "objective/train/tokens_used": 85996000, + "theoretical_loss": 5.097537441815887, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920401203610832, + "loss": 2.244, + "theoretical_loss": 5.097537441815887, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.00079202407221665, + "loss": 2.2649, + "theoretical_loss": 5.096793068960857, + "tokens_seen": 65601536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920080240722166, + "loss": 2.2591, + "theoretical_loss": 5.096049647343182, + "tokens_seen": 65667072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919919759277834, + "loss": 2.256, + "theoretical_loss": 5.095307174799752, + "tokens_seen": 65732608 + }, + { + "epoch": 0.02, + "learning_rate": 0.00079197592778335, + "loss": 2.2502, + "theoretical_loss": 5.094565649174524, + "tokens_seen": 65798144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919598796389168, + "loss": 2.1983, + "theoretical_loss": 5.0938250683184965, + "tokens_seen": 65863680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919438314944834, + "loss": 2.261, + "theoretical_loss": 5.093085430089674, + "tokens_seen": 65929216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919277833500502, + "loss": 2.3129, + "theoretical_loss": 5.092346732353046, + "tokens_seen": 65994752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919117352056168, + "loss": 2.2523, + "theoretical_loss": 5.091608972980545, + "tokens_seen": 66060288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007918956870611836, + "loss": 2.2646, + "theoretical_loss": 5.090872149851029, + "tokens_seen": 66125824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007918796389167502, + "loss": 2.2916, + "theoretical_loss": 5.090136260850245, + "tokens_seen": 66191360 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791863590772317, + "loss": 2.1602, + "theoretical_loss": 5.089401303870801, + "tokens_seen": 66256896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007918475426278836, + "loss": 2.2429, + "theoretical_loss": 5.088667276812138, + "tokens_seen": 66322432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007918314944834504, + "loss": 2.1725, + "theoretical_loss": 5.087934177580502, + "tokens_seen": 66387968 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791815446339017, + "loss": 2.2774, + "theoretical_loss": 5.087202004088914, + "tokens_seen": 66453504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917993981945838, + "loss": 2.1667, + "theoretical_loss": 5.086470754257139, + "tokens_seen": 66519040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917833500501504, + "loss": 2.2032, + "theoretical_loss": 5.0857404260116645, + "tokens_seen": 66584576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917673019057172, + "loss": 2.148, + "theoretical_loss": 5.085011017285662, + "tokens_seen": 66650112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917512537612838, + "loss": 2.2979, + "theoretical_loss": 5.084282526018972, + "tokens_seen": 66715648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917352056168506, + "loss": 2.2236, + "theoretical_loss": 5.083554950158062, + "tokens_seen": 66781184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917191574724172, + "loss": 2.1649, + "theoretical_loss": 5.0828282876560085, + "tokens_seen": 66846720 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791703109327984, + "loss": 2.2443, + "theoretical_loss": 5.0821025364724655, + "tokens_seen": 66912256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916870611835506, + "loss": 2.3334, + "theoretical_loss": 5.081377694573638, + "tokens_seen": 66977792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916710130391174, + "loss": 2.2181, + "theoretical_loss": 5.080653759932255, + "tokens_seen": 67043328 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791654964894684, + "loss": 2.1852, + "theoretical_loss": 5.079930730527538, + "tokens_seen": 67108864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916389167502508, + "loss": 2.1556, + "theoretical_loss": 5.079208604345181, + "tokens_seen": 67174400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916228686058174, + "loss": 2.2345, + "theoretical_loss": 5.078487379377318, + "tokens_seen": 67239936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916068204613842, + "loss": 2.1317, + "theoretical_loss": 5.0777670536225, + "tokens_seen": 67305472 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791590772316951, + "loss": 2.1145, + "theoretical_loss": 5.07704762508566, + "tokens_seen": 67371008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007915747241725176, + "loss": 2.1057, + "theoretical_loss": 5.076329091778101, + "tokens_seen": 67436544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007915586760280844, + "loss": 2.1766, + "theoretical_loss": 5.075611451717457, + "tokens_seen": 67502080 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791542627883651, + "loss": 2.1238, + "theoretical_loss": 5.0748947029276685, + "tokens_seen": 67567616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007915265797392178, + "loss": 2.2196, + "theoretical_loss": 5.074178843438963, + "tokens_seen": 67633152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007915105315947844, + "loss": 2.2516, + "theoretical_loss": 5.073463871287823, + "tokens_seen": 67698688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914944834503512, + "loss": 2.2305, + "theoretical_loss": 5.07274978451696, + "tokens_seen": 67764224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914784353059178, + "loss": 2.2993, + "theoretical_loss": 5.072036581175293, + "tokens_seen": 67829760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914623871614846, + "loss": 2.2447, + "theoretical_loss": 5.071324259317921, + "tokens_seen": 67895296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914463390170512, + "loss": 2.1236, + "theoretical_loss": 5.070612817006094, + "tokens_seen": 67960832 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791430290872618, + "loss": 2.3058, + "theoretical_loss": 5.06990225230719, + "tokens_seen": 68026368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914142427281846, + "loss": 2.1259, + "theoretical_loss": 5.069192563294694, + "tokens_seen": 68091904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913981945837514, + "loss": 2.3627, + "theoretical_loss": 5.068483748048166, + "tokens_seen": 68157440 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791382146439318, + "loss": 2.3496, + "theoretical_loss": 5.06777580465322, + "tokens_seen": 68222976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913660982948848, + "loss": 2.315, + "theoretical_loss": 5.067068731201498, + "tokens_seen": 68288512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913500501504514, + "loss": 2.1493, + "theoretical_loss": 5.066362525790646, + "tokens_seen": 68354048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913340020060182, + "loss": 2.113, + "theoretical_loss": 5.065657186524287, + "tokens_seen": 68419584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913179538615848, + "loss": 2.2397, + "theoretical_loss": 5.064952711512001, + "tokens_seen": 68485120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913019057171516, + "loss": 2.2762, + "theoretical_loss": 5.064249098869295, + "tokens_seen": 68550656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007912858575727182, + "loss": 2.2793, + "theoretical_loss": 5.063546346717585, + "tokens_seen": 68616192 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791269809428285, + "loss": 2.2232, + "theoretical_loss": 5.062844453184166, + "tokens_seen": 68681728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007912537612838516, + "loss": 2.2233, + "theoretical_loss": 5.062143416402192, + "tokens_seen": 68747264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 29892, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.302198648452759, + "objective/train/theoretical_loss": 5.061443234510648, + "objective/train/tokens_used": 89272800, + "theoretical_loss": 5.061443234510648, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007912377131394184, + "loss": 2.2377, + "theoretical_loss": 5.061443234510648, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791221664994985, + "loss": 2.2414, + "theoretical_loss": 5.060743905654332, + "tokens_seen": 68878336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007912056168505518, + "loss": 2.2892, + "theoretical_loss": 5.0600454279838285, + "tokens_seen": 68943872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911895687061184, + "loss": 2.1666, + "theoretical_loss": 5.059347799655483, + "tokens_seen": 69009408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911735205616852, + "loss": 2.1416, + "theoretical_loss": 5.05865101883138, + "tokens_seen": 69074944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911574724172518, + "loss": 2.1452, + "theoretical_loss": 5.057955083679321, + "tokens_seen": 69140480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911414242728186, + "loss": 2.2087, + "theoretical_loss": 5.0572599923728, + "tokens_seen": 69206016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911253761283852, + "loss": 2.2024, + "theoretical_loss": 5.056565743090982, + "tokens_seen": 69271552 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791109327983952, + "loss": 2.0612, + "theoretical_loss": 5.055872334018677, + "tokens_seen": 69337088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910932798395186, + "loss": 2.2113, + "theoretical_loss": 5.0551797633463185, + "tokens_seen": 69402624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910772316950853, + "loss": 2.1887, + "theoretical_loss": 5.054488029269946, + "tokens_seen": 69468160 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791061183550652, + "loss": 2.2918, + "theoretical_loss": 5.053797129991171, + "tokens_seen": 69533696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910451354062187, + "loss": 2.2494, + "theoretical_loss": 5.053107063717167, + "tokens_seen": 69599232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910290872617854, + "loss": 2.2314, + "theoretical_loss": 5.052417828660639, + "tokens_seen": 69664768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910130391173521, + "loss": 2.2445, + "theoretical_loss": 5.051729423039802, + "tokens_seen": 69730304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909969909729188, + "loss": 2.2076, + "theoretical_loss": 5.0510418450783625, + "tokens_seen": 69795840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909809428284855, + "loss": 2.2421, + "theoretical_loss": 5.050355093005495, + "tokens_seen": 69861376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909648946840522, + "loss": 2.0975, + "theoretical_loss": 5.049669165055816, + "tokens_seen": 69926912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909488465396189, + "loss": 2.1393, + "theoretical_loss": 5.048984059469369, + "tokens_seen": 69992448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909327983951855, + "loss": 2.1042, + "theoretical_loss": 5.048299774491598, + "tokens_seen": 70057984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909167502507523, + "loss": 2.1535, + "theoretical_loss": 5.047616308373328, + "tokens_seen": 70123520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909007021063189, + "loss": 2.2246, + "theoretical_loss": 5.046933659370742, + "tokens_seen": 70189056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908846539618857, + "loss": 2.1902, + "theoretical_loss": 5.046251825745361, + "tokens_seen": 70254592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908686058174523, + "loss": 2.247, + "theoretical_loss": 5.0455708057640205, + "tokens_seen": 70320128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908525576730191, + "loss": 2.0948, + "theoretical_loss": 5.044890597698853, + "tokens_seen": 70385664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908365095285857, + "loss": 2.2894, + "theoretical_loss": 5.044211199827263, + "tokens_seen": 70451200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908204613841525, + "loss": 2.1606, + "theoretical_loss": 5.04353261043191, + "tokens_seen": 70516736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908044132397191, + "loss": 2.2046, + "theoretical_loss": 5.0428548278006815, + "tokens_seen": 70582272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907883650952859, + "loss": 2.0641, + "theoretical_loss": 5.042177850226682, + "tokens_seen": 70647808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907723169508525, + "loss": 2.1669, + "theoretical_loss": 5.0415016760082, + "tokens_seen": 70713344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907562688064193, + "loss": 2.1651, + "theoretical_loss": 5.040826303448698, + "tokens_seen": 70778880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907402206619859, + "loss": 2.1432, + "theoretical_loss": 5.040151730856788, + "tokens_seen": 70844416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907241725175527, + "loss": 2.1211, + "theoretical_loss": 5.039477956546207, + "tokens_seen": 70909952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907081243731193, + "loss": 2.2094, + "theoretical_loss": 5.038804978835808, + "tokens_seen": 70975488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906920762286861, + "loss": 2.2419, + "theoretical_loss": 5.038132796049523, + "tokens_seen": 71041024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906760280842527, + "loss": 2.1796, + "theoretical_loss": 5.037461406516362, + "tokens_seen": 71106560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906599799398195, + "loss": 2.1457, + "theoretical_loss": 5.036790808570377, + "tokens_seen": 71172096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906439317953861, + "loss": 2.1222, + "theoretical_loss": 5.036121000550651, + "tokens_seen": 71237632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906278836509529, + "loss": 2.2514, + "theoretical_loss": 5.035451980801276, + "tokens_seen": 71303168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906118355065195, + "loss": 2.1864, + "theoretical_loss": 5.034783747671332, + "tokens_seen": 71368704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905957873620863, + "loss": 2.148, + "theoretical_loss": 5.034116299514871, + "tokens_seen": 71434240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905797392176529, + "loss": 2.1176, + "theoretical_loss": 5.033449634690893, + "tokens_seen": 71499776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905636910732197, + "loss": 2.0967, + "theoretical_loss": 5.032783751563327, + "tokens_seen": 71565312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905476429287863, + "loss": 2.1888, + "theoretical_loss": 5.032118648501017, + "tokens_seen": 71630848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905315947843531, + "loss": 2.3203, + "theoretical_loss": 5.031454323877698, + "tokens_seen": 71696384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905155466399197, + "loss": 2.1714, + "theoretical_loss": 5.030790776071976, + "tokens_seen": 71761920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904994984954865, + "loss": 2.0794, + "theoretical_loss": 5.030128003467312, + "tokens_seen": 71827456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904834503510531, + "loss": 2.1015, + "theoretical_loss": 5.029466004452004, + "tokens_seen": 71892992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904674022066199, + "loss": 2.0396, + "theoretical_loss": 5.028804777419165, + "tokens_seen": 71958528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904513540621865, + "loss": 2.1914, + "theoretical_loss": 5.028144320766701, + "tokens_seen": 72024064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 30813, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.0049593448638916, + "objective/train/theoretical_loss": 5.0274846328973055, + "objective/train/tokens_used": 92549600, + "theoretical_loss": 5.0274846328973055, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904353059177533, + "loss": 2.1007, + "theoretical_loss": 5.0274846328973055, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904192577733199, + "loss": 2.2275, + "theoretical_loss": 5.0268257122184234, + "tokens_seen": 72155136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904032096288867, + "loss": 2.1329, + "theoretical_loss": 5.026167557142247, + "tokens_seen": 72220672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903871614844534, + "loss": 2.098, + "theoretical_loss": 5.02551016608569, + "tokens_seen": 72286208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903711133400201, + "loss": 2.2127, + "theoretical_loss": 5.024853537470369, + "tokens_seen": 72351744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903550651955868, + "loss": 2.0479, + "theoretical_loss": 5.02419766972259, + "tokens_seen": 72417280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903390170511535, + "loss": 2.2583, + "theoretical_loss": 5.023542561273327, + "tokens_seen": 72482816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903229689067202, + "loss": 2.2263, + "theoretical_loss": 5.022888210558206, + "tokens_seen": 72548352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903069207622869, + "loss": 2.2046, + "theoretical_loss": 5.02223461601748, + "tokens_seen": 72613888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902908726178537, + "loss": 2.3318, + "theoretical_loss": 5.021581776096022, + "tokens_seen": 72679424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902748244734203, + "loss": 2.131, + "theoretical_loss": 5.0209296892433, + "tokens_seen": 72744960 + }, + { + "epoch": 0.02, + "learning_rate": 0.000790258776328987, + "loss": 2.2132, + "theoretical_loss": 5.020278353913362, + "tokens_seen": 72810496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902427281845537, + "loss": 2.0379, + "theoretical_loss": 5.019627768564816, + "tokens_seen": 72876032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902266800401205, + "loss": 2.1585, + "theoretical_loss": 5.018977931660814, + "tokens_seen": 72941568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902106318956871, + "loss": 2.1721, + "theoretical_loss": 5.018328841669037, + "tokens_seen": 73007104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901945837512539, + "loss": 2.3544, + "theoretical_loss": 5.017680497061671, + "tokens_seen": 73072640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901785356068205, + "loss": 2.175, + "theoretical_loss": 5.017032896315397, + "tokens_seen": 73138176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901624874623873, + "loss": 2.0713, + "theoretical_loss": 5.016386037911371, + "tokens_seen": 73203712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901464393179539, + "loss": 2.2291, + "theoretical_loss": 5.015739920335203, + "tokens_seen": 73269248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901303911735207, + "loss": 2.185, + "theoretical_loss": 5.015094542076945, + "tokens_seen": 73334784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901143430290873, + "loss": 2.09, + "theoretical_loss": 5.014449901631075, + "tokens_seen": 73400320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900982948846541, + "loss": 2.1841, + "theoretical_loss": 5.013805997496473, + "tokens_seen": 73465856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900822467402207, + "loss": 2.1307, + "theoretical_loss": 5.013162828176412, + "tokens_seen": 73531392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900661985957875, + "loss": 2.2505, + "theoretical_loss": 5.012520392178539, + "tokens_seen": 73596928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900501504513541, + "loss": 2.1488, + "theoretical_loss": 5.011878688014852, + "tokens_seen": 73662464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900341023069209, + "loss": 2.1883, + "theoretical_loss": 5.011237714201694, + "tokens_seen": 73728000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900180541624875, + "loss": 2.0638, + "theoretical_loss": 5.010597469259729, + "tokens_seen": 73793536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900020060180543, + "loss": 2.1461, + "theoretical_loss": 5.009957951713927, + "tokens_seen": 73859072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899859578736209, + "loss": 2.1418, + "theoretical_loss": 5.009319160093551, + "tokens_seen": 73924608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899699097291877, + "loss": 2.1104, + "theoretical_loss": 5.008681092932137, + "tokens_seen": 73990144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899538615847543, + "loss": 2.2886, + "theoretical_loss": 5.008043748767477, + "tokens_seen": 74055680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899378134403211, + "loss": 2.1436, + "theoretical_loss": 5.007407126141608, + "tokens_seen": 74121216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899217652958877, + "loss": 2.171, + "theoretical_loss": 5.00677122360079, + "tokens_seen": 74186752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899057171514545, + "loss": 2.189, + "theoretical_loss": 5.006136039695496, + "tokens_seen": 74252288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898896690070211, + "loss": 2.1575, + "theoretical_loss": 5.005501572980391, + "tokens_seen": 74317824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898736208625879, + "loss": 2.0729, + "theoretical_loss": 5.004867822014317, + "tokens_seen": 74383360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898575727181545, + "loss": 2.144, + "theoretical_loss": 5.004234785360282, + "tokens_seen": 74448896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898415245737213, + "loss": 2.1052, + "theoretical_loss": 5.0036024615854355, + "tokens_seen": 74514432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898254764292879, + "loss": 2.113, + "theoretical_loss": 5.002970849261065, + "tokens_seen": 74579968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898094282848547, + "loss": 2.1806, + "theoretical_loss": 5.002339946962568, + "tokens_seen": 74645504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897933801404213, + "loss": 2.1274, + "theoretical_loss": 5.001709753269443, + "tokens_seen": 74711040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897773319959881, + "loss": 2.1426, + "theoretical_loss": 5.001080266765275, + "tokens_seen": 74776576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897612838515547, + "loss": 2.2028, + "theoretical_loss": 5.0004514860377185, + "tokens_seen": 74842112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897452357071215, + "loss": 2.1624, + "theoretical_loss": 4.999823409678481, + "tokens_seen": 74907648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897291875626881, + "loss": 2.1102, + "theoretical_loss": 4.999196036283308, + "tokens_seen": 74973184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897131394182549, + "loss": 2.196, + "theoretical_loss": 4.9985693644519715, + "tokens_seen": 75038720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896970912738215, + "loss": 2.1068, + "theoretical_loss": 4.997943392788251, + "tokens_seen": 75104256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896810431293882, + "loss": 2.0444, + "theoretical_loss": 4.997318119899922, + "tokens_seen": 75169792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896649949849549, + "loss": 1.96, + "theoretical_loss": 4.996693544398734, + "tokens_seen": 75235328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896489468405216, + "loss": 2.133, + "theoretical_loss": 4.996069664900406, + "tokens_seen": 75300864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 32032, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.0214736461639404, + "objective/train/theoretical_loss": 4.9954464800246035, + "objective/train/tokens_used": 95826400, + "theoretical_loss": 4.9954464800246035, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896328986960883, + "loss": 2.0347, + "theoretical_loss": 4.9954464800246035, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789616850551655, + "loss": 2.1474, + "theoretical_loss": 4.994823988394928, + "tokens_seen": 75431936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896008024072217, + "loss": 2.1111, + "theoretical_loss": 4.994202188638901, + "tokens_seen": 75497472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895847542627884, + "loss": 2.1961, + "theoretical_loss": 4.9935810793879485, + "tokens_seen": 75563008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895687061183551, + "loss": 2.1599, + "theoretical_loss": 4.992960659277389, + "tokens_seen": 75628544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895526579739218, + "loss": 2.1578, + "theoretical_loss": 4.992340926946417, + "tokens_seen": 75694080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895366098294884, + "loss": 2.041, + "theoretical_loss": 4.991721881038091, + "tokens_seen": 75759616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895205616850552, + "loss": 2.1556, + "theoretical_loss": 4.991103520199314, + "tokens_seen": 75825152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895045135406218, + "loss": 2.0935, + "theoretical_loss": 4.990485843080824, + "tokens_seen": 75890688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007894884653961886, + "loss": 2.153, + "theoretical_loss": 4.989868848337181, + "tokens_seen": 75956224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007894724172517552, + "loss": 2.1098, + "theoretical_loss": 4.989252534626749, + "tokens_seen": 76021760 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789456369107322, + "loss": 2.2057, + "theoretical_loss": 4.988636900611683, + "tokens_seen": 76087296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007894403209628886, + "loss": 2.1796, + "theoretical_loss": 4.988021944957915, + "tokens_seen": 76152832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007894242728184554, + "loss": 2.1348, + "theoretical_loss": 4.987407666335143, + "tokens_seen": 76218368 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789408224674022, + "loss": 2.003, + "theoretical_loss": 4.986794063416813, + "tokens_seen": 76283904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893921765295888, + "loss": 2.2363, + "theoretical_loss": 4.986181134880107, + "tokens_seen": 76349440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893761283851554, + "loss": 2.1553, + "theoretical_loss": 4.98556887940593, + "tokens_seen": 76414976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893600802407222, + "loss": 2.0457, + "theoretical_loss": 4.984957295678894, + "tokens_seen": 76480512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893440320962888, + "loss": 2.0834, + "theoretical_loss": 4.98434638238731, + "tokens_seen": 76546048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893279839518556, + "loss": 2.1583, + "theoretical_loss": 4.983736138223165, + "tokens_seen": 76611584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893119358074222, + "loss": 2.1141, + "theoretical_loss": 4.983126561882118, + "tokens_seen": 76677120 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789295887662989, + "loss": 2.1146, + "theoretical_loss": 4.982517652063482, + "tokens_seen": 76742656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007892798395185556, + "loss": 2.0532, + "theoretical_loss": 4.981909407470209, + "tokens_seen": 76808192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007892637913741224, + "loss": 2.1857, + "theoretical_loss": 4.981301826808882, + "tokens_seen": 76873728 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789247743229689, + "loss": 2.1357, + "theoretical_loss": 4.9806949087896975, + "tokens_seen": 76939264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007892316950852558, + "loss": 1.9885, + "theoretical_loss": 4.980088652126453, + "tokens_seen": 77004800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007892156469408224, + "loss": 2.1321, + "theoretical_loss": 4.9794830555365355, + "tokens_seen": 77070336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891995987963892, + "loss": 2.0872, + "theoretical_loss": 4.978878117740907, + "tokens_seen": 77135872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891835506519559, + "loss": 2.1593, + "theoretical_loss": 4.978273837464091, + "tokens_seen": 77201408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891675025075226, + "loss": 2.0579, + "theoretical_loss": 4.977670213434161, + "tokens_seen": 77266944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891514543630893, + "loss": 2.2943, + "theoretical_loss": 4.977067244382729, + "tokens_seen": 77332480 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789135406218656, + "loss": 1.9335, + "theoretical_loss": 4.976464929044928, + "tokens_seen": 77398016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891193580742227, + "loss": 2.0717, + "theoretical_loss": 4.975863266159403, + "tokens_seen": 77463552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891033099297894, + "loss": 2.0471, + "theoretical_loss": 4.975262254468299, + "tokens_seen": 77529088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890872617853561, + "loss": 2.07, + "theoretical_loss": 4.9746618927172435, + "tokens_seen": 77594624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890712136409228, + "loss": 2.1022, + "theoretical_loss": 4.9740621796553395, + "tokens_seen": 77660160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890551654964895, + "loss": 2.191, + "theoretical_loss": 4.973463114035149, + "tokens_seen": 77725696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890391173520562, + "loss": 2.0829, + "theoretical_loss": 4.972864694612684, + "tokens_seen": 77791232 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789023069207623, + "loss": 2.0328, + "theoretical_loss": 4.972266920147391, + "tokens_seen": 77856768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890070210631896, + "loss": 2.1021, + "theoretical_loss": 4.971669789402139, + "tokens_seen": 77922304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889909729187563, + "loss": 2.045, + "theoretical_loss": 4.971073301143207, + "tokens_seen": 77987840 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788974924774323, + "loss": 2.1521, + "theoretical_loss": 4.970477454140278, + "tokens_seen": 78053376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889588766298897, + "loss": 2.1709, + "theoretical_loss": 4.9698822471664155, + "tokens_seen": 78118912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889428284854564, + "loss": 2.0816, + "theoretical_loss": 4.96928767899806, + "tokens_seen": 78184448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889267803410232, + "loss": 2.1294, + "theoretical_loss": 4.9686937484150135, + "tokens_seen": 78249984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889107321965898, + "loss": 2.0653, + "theoretical_loss": 4.96810045420043, + "tokens_seen": 78315520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007888946840521566, + "loss": 2.0877, + "theoretical_loss": 4.967507795140797, + "tokens_seen": 78381056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007888786359077232, + "loss": 2.17, + "theoretical_loss": 4.966915770025935, + "tokens_seen": 78446592 + }, + { + "epoch": 0.02, + "learning_rate": 0.00078886258776329, + "loss": 2.1473, + "theoretical_loss": 4.966324377648973, + "tokens_seen": 78512128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007888465396188566, + "loss": 2.1472, + "theoretical_loss": 4.965733616806345, + "tokens_seen": 78577664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 32979, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.1013200283050537, + "objective/train/theoretical_loss": 4.965143486297777, + "objective/train/tokens_used": 99103200, + "theoretical_loss": 4.965143486297777, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007888304914744234, + "loss": 2.0807, + "theoretical_loss": 4.965143486297777, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.00078881444332999, + "loss": 2.1694, + "theoretical_loss": 4.964553984926271, + "tokens_seen": 78708736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887983951855568, + "loss": 2.0942, + "theoretical_loss": 4.963965111498098, + "tokens_seen": 78774272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887823470411234, + "loss": 2.1027, + "theoretical_loss": 4.963376864822784, + "tokens_seen": 78839808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887662988966902, + "loss": 2.1018, + "theoretical_loss": 4.962789243713102, + "tokens_seen": 78905344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887502507522568, + "loss": 2.1184, + "theoretical_loss": 4.962202246985054, + "tokens_seen": 78970880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887342026078236, + "loss": 2.0848, + "theoretical_loss": 4.9616158734578635, + "tokens_seen": 79036416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887181544633902, + "loss": 2.0948, + "theoretical_loss": 4.961030121953965, + "tokens_seen": 79101952 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788702106318957, + "loss": 2.1181, + "theoretical_loss": 4.960444991298992, + "tokens_seen": 79167488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886860581745236, + "loss": 2.1234, + "theoretical_loss": 4.959860480321762, + "tokens_seen": 79233024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886700100300904, + "loss": 2.112, + "theoretical_loss": 4.959276587854272, + "tokens_seen": 79298560 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788653961885657, + "loss": 2.1383, + "theoretical_loss": 4.958693312731681, + "tokens_seen": 79364096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886379137412238, + "loss": 2.1764, + "theoretical_loss": 4.9581106537923, + "tokens_seen": 79429632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886218655967904, + "loss": 2.0779, + "theoretical_loss": 4.957528609877587, + "tokens_seen": 79495168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886058174523572, + "loss": 2.0655, + "theoretical_loss": 4.9569471798321265, + "tokens_seen": 79560704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885897693079238, + "loss": 2.013, + "theoretical_loss": 4.9563663625036245, + "tokens_seen": 79626240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885737211634906, + "loss": 2.0405, + "theoretical_loss": 4.955786156742898, + "tokens_seen": 79691776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885576730190572, + "loss": 2.0635, + "theoretical_loss": 4.955206561403859, + "tokens_seen": 79757312 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788541624874624, + "loss": 2.1037, + "theoretical_loss": 4.954627575343509, + "tokens_seen": 79822848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885255767301906, + "loss": 1.9574, + "theoretical_loss": 4.954049197421925, + "tokens_seen": 79888384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885095285857574, + "loss": 2.1531, + "theoretical_loss": 4.953471426502249, + "tokens_seen": 79953920 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788493480441324, + "loss": 2.157, + "theoretical_loss": 4.952894261450679, + "tokens_seen": 80019456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884774322968908, + "loss": 2.044, + "theoretical_loss": 4.952317701136457, + "tokens_seen": 80084992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884613841524574, + "loss": 2.1789, + "theoretical_loss": 4.9517417444318585, + "tokens_seen": 80150528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884453360080242, + "loss": 2.1454, + "theoretical_loss": 4.951166390212181, + "tokens_seen": 80216064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884292878635908, + "loss": 2.0481, + "theoretical_loss": 4.950591637355737, + "tokens_seen": 80281600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884132397191576, + "loss": 2.0555, + "theoretical_loss": 4.950017484743835, + "tokens_seen": 80347136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883971915747242, + "loss": 2.0603, + "theoretical_loss": 4.949443931260783, + "tokens_seen": 80412672 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788381143430291, + "loss": 2.0551, + "theoretical_loss": 4.948870975793863, + "tokens_seen": 80478208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883650952858576, + "loss": 2.1272, + "theoretical_loss": 4.9482986172333305, + "tokens_seen": 80543744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883490471414244, + "loss": 2.035, + "theoretical_loss": 4.9477268544724, + "tokens_seen": 80609280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788332998996991, + "loss": 2.0542, + "theoretical_loss": 4.947155686407239, + "tokens_seen": 80674816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883169508525578, + "loss": 2.1393, + "theoretical_loss": 4.946585111936951, + "tokens_seen": 80740352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883009027081244, + "loss": 2.0737, + "theoretical_loss": 4.94601512996357, + "tokens_seen": 80805888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882848545636911, + "loss": 1.9408, + "theoretical_loss": 4.945445739392049, + "tokens_seen": 80871424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882688064192578, + "loss": 1.8858, + "theoretical_loss": 4.944876939130252, + "tokens_seen": 80936960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882527582748245, + "loss": 2.0729, + "theoretical_loss": 4.94430872808894, + "tokens_seen": 81002496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882367101303912, + "loss": 2.04, + "theoretical_loss": 4.943741105181765, + "tokens_seen": 81068032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882206619859579, + "loss": 2.0131, + "theoretical_loss": 4.943174069325255, + "tokens_seen": 81133568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882046138415246, + "loss": 1.9603, + "theoretical_loss": 4.94260761943881, + "tokens_seen": 81199104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881885656970913, + "loss": 1.985, + "theoretical_loss": 4.942041754444688, + "tokens_seen": 81264640 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788172517552658, + "loss": 2.039, + "theoretical_loss": 4.941476473267996, + "tokens_seen": 81330176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881564694082247, + "loss": 2.1917, + "theoretical_loss": 4.940911774836682, + "tokens_seen": 81395712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881404212637913, + "loss": 2.155, + "theoretical_loss": 4.940347658081521, + "tokens_seen": 81461248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881243731193581, + "loss": 2.0186, + "theoretical_loss": 4.939784121936112, + "tokens_seen": 81526784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881083249749247, + "loss": 2.0964, + "theoretical_loss": 4.9392211653368605, + "tokens_seen": 81592320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880922768304915, + "loss": 1.9986, + "theoretical_loss": 4.938658787222975, + "tokens_seen": 81657856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880762286860581, + "loss": 2.078, + "theoretical_loss": 4.938096986536452, + "tokens_seen": 81723392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880601805416249, + "loss": 2.0657, + "theoretical_loss": 4.937535762222075, + "tokens_seen": 81788928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880441323971915, + "loss": 2.111, + "theoretical_loss": 4.936975113227394, + "tokens_seen": 81854464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 34316, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.1313977241516113, + "objective/train/theoretical_loss": 4.936415038502723, + "objective/train/tokens_used": 102380000, + "theoretical_loss": 4.936415038502723, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880280842527583, + "loss": 2.1647, + "theoretical_loss": 4.936415038502723, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880120361083249, + "loss": 2.0975, + "theoretical_loss": 4.93585553700113, + "tokens_seen": 81985536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879959879638917, + "loss": 2.0226, + "theoretical_loss": 4.935296607678426, + "tokens_seen": 82051072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879799398194584, + "loss": 2.0912, + "theoretical_loss": 4.934738249493156, + "tokens_seen": 82116608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879638916750251, + "loss": 2.0794, + "theoretical_loss": 4.934180461406591, + "tokens_seen": 82182144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879478435305918, + "loss": 2.0681, + "theoretical_loss": 4.933623242382714, + "tokens_seen": 82247680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879317953861585, + "loss": 2.0694, + "theoretical_loss": 4.9330665913882195, + "tokens_seen": 82313216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879157472417252, + "loss": 2.0263, + "theoretical_loss": 4.932510507392495, + "tokens_seen": 82378752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007878996990972919, + "loss": 2.0941, + "theoretical_loss": 4.931954989367618, + "tokens_seen": 82444288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878836509528586, + "loss": 2.1159, + "theoretical_loss": 4.931400036288343, + "tokens_seen": 82509824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878676028084253, + "loss": 1.965, + "theoretical_loss": 4.930845647132097, + "tokens_seen": 82575360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000787851554663992, + "loss": 2.0173, + "theoretical_loss": 4.9302918208789634, + "tokens_seen": 82640896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878355065195587, + "loss": 2.0566, + "theoretical_loss": 4.929738556511681, + "tokens_seen": 82706432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878194583751254, + "loss": 2.0425, + "theoretical_loss": 4.929185853015629, + "tokens_seen": 82771968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878034102306921, + "loss": 2.0891, + "theoretical_loss": 4.928633709378822, + "tokens_seen": 82837504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877873620862588, + "loss": 2.1192, + "theoretical_loss": 4.9280821245918975, + "tokens_seen": 82903040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877713139418255, + "loss": 1.8945, + "theoretical_loss": 4.92753109764811, + "tokens_seen": 82968576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877552657973922, + "loss": 2.1443, + "theoretical_loss": 4.92698062754332, + "tokens_seen": 83034112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877392176529589, + "loss": 2.263, + "theoretical_loss": 4.9264307132759875, + "tokens_seen": 83099648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877231695085256, + "loss": 2.0305, + "theoretical_loss": 4.92588135384716, + "tokens_seen": 83165184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877071213640923, + "loss": 2.0397, + "theoretical_loss": 4.925332548260469, + "tokens_seen": 83230720 + }, + { + "epoch": 0.03, + "learning_rate": 0.000787691073219659, + "loss": 1.9951, + "theoretical_loss": 4.924784295522113, + "tokens_seen": 83296256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876750250752257, + "loss": 2.014, + "theoretical_loss": 4.92423659464086, + "tokens_seen": 83361792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876589769307924, + "loss": 2.0954, + "theoretical_loss": 4.923689444628027, + "tokens_seen": 83427328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876429287863591, + "loss": 1.9732, + "theoretical_loss": 4.923142844497482, + "tokens_seen": 83492864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876268806419258, + "loss": 2.0625, + "theoretical_loss": 4.922596793265625, + "tokens_seen": 83558400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876108324974925, + "loss": 2.0099, + "theoretical_loss": 4.922051289951391, + "tokens_seen": 83623936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875947843530592, + "loss": 1.9741, + "theoretical_loss": 4.921506333576232, + "tokens_seen": 83689472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875787362086259, + "loss": 2.0757, + "theoretical_loss": 4.920961923164114, + "tokens_seen": 83755008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875626880641927, + "loss": 2.0747, + "theoretical_loss": 4.920418057741504, + "tokens_seen": 83820544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875466399197593, + "loss": 2.1795, + "theoretical_loss": 4.919874736337368, + "tokens_seen": 83886080 + }, + { + "epoch": 0.03, + "learning_rate": 0.000787530591775326, + "loss": 2.0879, + "theoretical_loss": 4.9193319579831565, + "tokens_seen": 83951616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875145436308927, + "loss": 2.0928, + "theoretical_loss": 4.918789721712799, + "tokens_seen": 84017152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874984954864595, + "loss": 2.1112, + "theoretical_loss": 4.918248026562697, + "tokens_seen": 84082688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874824473420261, + "loss": 2.0547, + "theoretical_loss": 4.917706871571712, + "tokens_seen": 84148224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874663991975929, + "loss": 2.1856, + "theoretical_loss": 4.9171662557811615, + "tokens_seen": 84213760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874503510531595, + "loss": 2.1379, + "theoretical_loss": 4.916626178234807, + "tokens_seen": 84279296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874343029087263, + "loss": 2.0925, + "theoretical_loss": 4.916086637978851, + "tokens_seen": 84344832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874182547642929, + "loss": 2.1199, + "theoretical_loss": 4.915547634061921, + "tokens_seen": 84410368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874022066198597, + "loss": 2.206, + "theoretical_loss": 4.91500916553507, + "tokens_seen": 84475904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873861584754263, + "loss": 2.1473, + "theoretical_loss": 4.914471231451762, + "tokens_seen": 84541440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873701103309931, + "loss": 2.1729, + "theoretical_loss": 4.913933830867868, + "tokens_seen": 84606976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873540621865597, + "loss": 2.0473, + "theoretical_loss": 4.913396962841655, + "tokens_seen": 84672512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873380140421265, + "loss": 2.1081, + "theoretical_loss": 4.9128606264337815, + "tokens_seen": 84738048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873219658976931, + "loss": 2.2149, + "theoretical_loss": 4.912324820707285, + "tokens_seen": 84803584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873059177532599, + "loss": 2.1657, + "theoretical_loss": 4.911789544727581, + "tokens_seen": 84869120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872898696088265, + "loss": 2.0486, + "theoretical_loss": 4.911254797562448, + "tokens_seen": 84934656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872738214643933, + "loss": 2.1763, + "theoretical_loss": 4.910720578282021, + "tokens_seen": 85000192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872577733199599, + "loss": 2.025, + "theoretical_loss": 4.91018688595879, + "tokens_seen": 85065728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872417251755267, + "loss": 2.0928, + "theoretical_loss": 4.909653719667584, + "tokens_seen": 85131264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 35098, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.9277470111846924, + "objective/train/theoretical_loss": 4.909121078485567, + "objective/train/tokens_used": 105656800, + "theoretical_loss": 4.909121078485567, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872256770310933, + "loss": 2.1105, + "theoretical_loss": 4.909121078485567, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872096288866601, + "loss": 2.0636, + "theoretical_loss": 4.908588961492235, + "tokens_seen": 85262336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871935807422267, + "loss": 2.1548, + "theoretical_loss": 4.908057367769396, + "tokens_seen": 85327872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871775325977935, + "loss": 2.146, + "theoretical_loss": 4.907526296401175, + "tokens_seen": 85393408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871614844533601, + "loss": 2.0305, + "theoretical_loss": 4.906995746474001, + "tokens_seen": 85458944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871454363089269, + "loss": 2.1158, + "theoretical_loss": 4.9064657170766, + "tokens_seen": 85524480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871293881644935, + "loss": 2.1002, + "theoretical_loss": 4.905936207299984, + "tokens_seen": 85590016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871133400200603, + "loss": 2.0993, + "theoretical_loss": 4.90540721623745, + "tokens_seen": 85655552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870972918756269, + "loss": 1.978, + "theoretical_loss": 4.904878742984569, + "tokens_seen": 85721088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870812437311937, + "loss": 2.0845, + "theoretical_loss": 4.904350786639178, + "tokens_seen": 85786624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870651955867603, + "loss": 2.1174, + "theoretical_loss": 4.903823346301374, + "tokens_seen": 85852160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870491474423271, + "loss": 1.9148, + "theoretical_loss": 4.903296421073506, + "tokens_seen": 85917696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870330992978937, + "loss": 2.203, + "theoretical_loss": 4.902770010060166, + "tokens_seen": 85983232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870170511534605, + "loss": 2.1191, + "theoretical_loss": 4.902244112368188, + "tokens_seen": 86048768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870010030090271, + "loss": 2.1491, + "theoretical_loss": 4.901718727106631, + "tokens_seen": 86114304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869849548645939, + "loss": 2.1635, + "theoretical_loss": 4.90119385338678, + "tokens_seen": 86179840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869689067201605, + "loss": 1.9647, + "theoretical_loss": 4.900669490322134, + "tokens_seen": 86245376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869528585757273, + "loss": 2.0817, + "theoretical_loss": 4.900145637028402, + "tokens_seen": 86310912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869368104312939, + "loss": 2.1156, + "theoretical_loss": 4.899622292623493, + "tokens_seen": 86376448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869207622868607, + "loss": 1.9936, + "theoretical_loss": 4.8990994562275105, + "tokens_seen": 86441984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869047141424273, + "loss": 2.1465, + "theoretical_loss": 4.898577126962746, + "tokens_seen": 86507520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868886659979941, + "loss": 2.1501, + "theoretical_loss": 4.898055303953669, + "tokens_seen": 86573056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868726178535607, + "loss": 2.0958, + "theoretical_loss": 4.897533986326922, + "tokens_seen": 86638592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868565697091274, + "loss": 2.0493, + "theoretical_loss": 4.897013173211316, + "tokens_seen": 86704128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868405215646941, + "loss": 2.061, + "theoretical_loss": 4.896492863737818, + "tokens_seen": 86769664 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868244734202608, + "loss": 2.1131, + "theoretical_loss": 4.895973057039548, + "tokens_seen": 86835200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868084252758275, + "loss": 2.0576, + "theoretical_loss": 4.89545375225177, + "tokens_seen": 86900736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867923771313942, + "loss": 2.0952, + "theoretical_loss": 4.894934948511889, + "tokens_seen": 86966272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867763289869609, + "loss": 2.0713, + "theoretical_loss": 4.894416644959437, + "tokens_seen": 87031808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867602808425276, + "loss": 2.1069, + "theoretical_loss": 4.893898840736071, + "tokens_seen": 87097344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867442326980943, + "loss": 2.0183, + "theoretical_loss": 4.8933815349855685, + "tokens_seen": 87162880 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786728184553661, + "loss": 2.1052, + "theoretical_loss": 4.892864726853814, + "tokens_seen": 87228416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867121364092277, + "loss": 2.0933, + "theoretical_loss": 4.892348415488799, + "tokens_seen": 87293952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866960882647944, + "loss": 2.0157, + "theoretical_loss": 4.891832600040609, + "tokens_seen": 87359488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866800401203611, + "loss": 2.1374, + "theoretical_loss": 4.891317279661422, + "tokens_seen": 87425024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866639919759278, + "loss": 2.0133, + "theoretical_loss": 4.890802453505498, + "tokens_seen": 87490560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866479438314945, + "loss": 1.901, + "theoretical_loss": 4.890288120729174, + "tokens_seen": 87556096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866318956870612, + "loss": 2.1402, + "theoretical_loss": 4.88977428049086, + "tokens_seen": 87621632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866158475426279, + "loss": 1.9311, + "theoretical_loss": 4.889260931951026, + "tokens_seen": 87687168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865997993981946, + "loss": 2.1391, + "theoretical_loss": 4.888748074272201, + "tokens_seen": 87752704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865837512537613, + "loss": 2.1003, + "theoretical_loss": 4.888235706618964, + "tokens_seen": 87818240 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786567703109328, + "loss": 2.0694, + "theoretical_loss": 4.887723828157939, + "tokens_seen": 87883776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865516549648947, + "loss": 2.0203, + "theoretical_loss": 4.8872124380577855, + "tokens_seen": 87949312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865356068204614, + "loss": 2.0587, + "theoretical_loss": 4.886701535489195, + "tokens_seen": 88014848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865195586760281, + "loss": 2.1354, + "theoretical_loss": 4.886191119624883, + "tokens_seen": 88080384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865035105315948, + "loss": 2.034, + "theoretical_loss": 4.885681189639584, + "tokens_seen": 88145920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864874623871615, + "loss": 2.1003, + "theoretical_loss": 4.885171744710044, + "tokens_seen": 88211456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864714142427282, + "loss": 2.1345, + "theoretical_loss": 4.884662784015012, + "tokens_seen": 88276992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864553660982949, + "loss": 2.0893, + "theoretical_loss": 4.884154306735239, + "tokens_seen": 88342528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864393179538616, + "loss": 2.1243, + "theoretical_loss": 4.8836463120534646, + "tokens_seen": 88408064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 36417, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8406845331192017, + "objective/train/theoretical_loss": 4.88313879915442, + "objective/train/tokens_used": 108933600, + "theoretical_loss": 4.88313879915442, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864232698094283, + "loss": 1.8913, + "theoretical_loss": 4.88313879915442, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786407221664995, + "loss": 2.1626, + "theoretical_loss": 4.882631767224812, + "tokens_seen": 88539136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863911735205617, + "loss": 2.0407, + "theoretical_loss": 4.882125215453321, + "tokens_seen": 88604672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863751253761284, + "loss": 2.0725, + "theoretical_loss": 4.881619143030598, + "tokens_seen": 88670208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863590772316951, + "loss": 2.007, + "theoretical_loss": 4.8811135491492506, + "tokens_seen": 88735744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863430290872618, + "loss": 2.0586, + "theoretical_loss": 4.880608433003846, + "tokens_seen": 88801280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863269809428285, + "loss": 2.0653, + "theoretical_loss": 4.880103793790896, + "tokens_seen": 88866816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863109327983952, + "loss": 2.0433, + "theoretical_loss": 4.87959963070886, + "tokens_seen": 88932352 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786294884653962, + "loss": 2.0326, + "theoretical_loss": 4.879095942958127, + "tokens_seen": 88997888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007862788365095286, + "loss": 2.0379, + "theoretical_loss": 4.878592729741024, + "tokens_seen": 89063424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007862627883650953, + "loss": 2.1523, + "theoretical_loss": 4.878089990261797, + "tokens_seen": 89128960 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786246740220662, + "loss": 1.9848, + "theoretical_loss": 4.87758772372661, + "tokens_seen": 89194496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007862306920762287, + "loss": 1.9817, + "theoretical_loss": 4.877085929343544, + "tokens_seen": 89260032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007862146439317954, + "loss": 2.1303, + "theoretical_loss": 4.876584606322583, + "tokens_seen": 89325568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861985957873622, + "loss": 2.0567, + "theoretical_loss": 4.8760837538756086, + "tokens_seen": 89391104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861825476429288, + "loss": 1.9851, + "theoretical_loss": 4.875583371216404, + "tokens_seen": 89456640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861664994984956, + "loss": 2.0458, + "theoretical_loss": 4.875083457560633, + "tokens_seen": 89522176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861504513540622, + "loss": 1.9532, + "theoretical_loss": 4.874584012125847, + "tokens_seen": 89587712 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786134403209629, + "loss": 2.0171, + "theoretical_loss": 4.874085034131472, + "tokens_seen": 89653248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861183550651956, + "loss": 2.016, + "theoretical_loss": 4.873586522798803, + "tokens_seen": 89718784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861023069207624, + "loss": 2.0034, + "theoretical_loss": 4.873088477351005, + "tokens_seen": 89784320 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786086258776329, + "loss": 2.0347, + "theoretical_loss": 4.8725908970130964, + "tokens_seen": 89849856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860702106318958, + "loss": 2.0416, + "theoretical_loss": 4.872093781011952, + "tokens_seen": 89915392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860541624874624, + "loss": 1.9762, + "theoretical_loss": 4.871597128576292, + "tokens_seen": 89980928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860381143430292, + "loss": 1.9752, + "theoretical_loss": 4.8711009389366815, + "tokens_seen": 90046464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860220661985958, + "loss": 2.0416, + "theoretical_loss": 4.870605211325517, + "tokens_seen": 90112000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860060180541626, + "loss": 1.9874, + "theoretical_loss": 4.870109944977029, + "tokens_seen": 90177536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007859899699097292, + "loss": 2.0183, + "theoretical_loss": 4.869615139127273, + "tokens_seen": 90243072 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785973921765296, + "loss": 2.079, + "theoretical_loss": 4.869120793014117, + "tokens_seen": 90308608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007859578736208626, + "loss": 2.039, + "theoretical_loss": 4.868626905877251, + "tokens_seen": 90374144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007859418254764294, + "loss": 2.0082, + "theoretical_loss": 4.8681334769581674, + "tokens_seen": 90439680 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785925777331996, + "loss": 1.88, + "theoretical_loss": 4.867640505500161, + "tokens_seen": 90505216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007859097291875628, + "loss": 2.0596, + "theoretical_loss": 4.8671479907483235, + "tokens_seen": 90570752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858936810431294, + "loss": 2.0777, + "theoretical_loss": 4.866655931949537, + "tokens_seen": 90636288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858776328986962, + "loss": 2.0637, + "theoretical_loss": 4.86616432835247, + "tokens_seen": 90701824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858615847542628, + "loss": 1.9995, + "theoretical_loss": 4.865673179207572, + "tokens_seen": 90767360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858455366098296, + "loss": 2.1029, + "theoretical_loss": 4.865182483767063, + "tokens_seen": 90832896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858294884653962, + "loss": 2.0009, + "theoretical_loss": 4.864692241284933, + "tokens_seen": 90898432 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785813440320963, + "loss": 2.0887, + "theoretical_loss": 4.864202451016939, + "tokens_seen": 90963968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857973921765296, + "loss": 2.0239, + "theoretical_loss": 4.863713112220592, + "tokens_seen": 91029504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857813440320964, + "loss": 2.0429, + "theoretical_loss": 4.8632242241551555, + "tokens_seen": 91095040 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785765295887663, + "loss": 2.0759, + "theoretical_loss": 4.862735786081642, + "tokens_seen": 91160576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857492477432298, + "loss": 2.014, + "theoretical_loss": 4.862247797262806, + "tokens_seen": 91226112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857331995987964, + "loss": 2.0947, + "theoretical_loss": 4.861760256963136, + "tokens_seen": 91291648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857171514543632, + "loss": 2.0521, + "theoretical_loss": 4.861273164448854, + "tokens_seen": 91357184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857011033099298, + "loss": 1.9563, + "theoretical_loss": 4.860786518987906, + "tokens_seen": 91422720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007856850551654966, + "loss": 2.0038, + "theoretical_loss": 4.86030031984996, + "tokens_seen": 91488256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007856690070210632, + "loss": 1.9585, + "theoretical_loss": 4.859814566306397, + "tokens_seen": 91553792 + }, + { + "epoch": 0.03, + "learning_rate": 0.00078565295887663, + "loss": 1.9885, + "theoretical_loss": 4.859329257630311, + "tokens_seen": 91619328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007856369107321966, + "loss": 2.0648, + "theoretical_loss": 4.858844393096497, + "tokens_seen": 91684864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 37720, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.0330617427825928, + "objective/train/theoretical_loss": 4.858359971981454, + "objective/train/tokens_used": 112210400, + "theoretical_loss": 4.858359971981454, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007856208625877634, + "loss": 2.0313, + "theoretical_loss": 4.858359971981454, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.00078560481444333, + "loss": 2.1479, + "theoretical_loss": 4.85787599356337, + "tokens_seen": 91815936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855887662988968, + "loss": 2.0328, + "theoretical_loss": 4.857392457122128, + "tokens_seen": 91881472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855727181544634, + "loss": 2.1289, + "theoretical_loss": 4.856909361939288, + "tokens_seen": 91947008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855566700100302, + "loss": 1.9571, + "theoretical_loss": 4.856426707298096, + "tokens_seen": 92012544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855406218655968, + "loss": 1.9227, + "theoretical_loss": 4.8559444924834665, + "tokens_seen": 92078080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855245737211636, + "loss": 1.9564, + "theoretical_loss": 4.855462716781985, + "tokens_seen": 92143616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855085255767302, + "loss": 2.0934, + "theoretical_loss": 4.854981379481901, + "tokens_seen": 92209152 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785492477432297, + "loss": 2.0322, + "theoretical_loss": 4.854500479873119, + "tokens_seen": 92274688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007854764292878636, + "loss": 1.9538, + "theoretical_loss": 4.854020017247203, + "tokens_seen": 92340224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007854603811434303, + "loss": 1.9776, + "theoretical_loss": 4.853539990897358, + "tokens_seen": 92405760 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785444332998997, + "loss": 2.0646, + "theoretical_loss": 4.85306040011844, + "tokens_seen": 92471296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007854282848545637, + "loss": 1.9781, + "theoretical_loss": 4.852581244206938, + "tokens_seen": 92536832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007854122367101304, + "loss": 2.0651, + "theoretical_loss": 4.852102522460975, + "tokens_seen": 92602368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853961885656971, + "loss": 1.9221, + "theoretical_loss": 4.851624234180306, + "tokens_seen": 92667904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853801404212638, + "loss": 2.1273, + "theoretical_loss": 4.851146378666305, + "tokens_seen": 92733440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853640922768305, + "loss": 1.9866, + "theoretical_loss": 4.85066895522197, + "tokens_seen": 92798976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853480441323972, + "loss": 2.0728, + "theoretical_loss": 4.850191963151908, + "tokens_seen": 92864512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853319959879639, + "loss": 2.1431, + "theoretical_loss": 4.849715401762337, + "tokens_seen": 92930048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853159478435306, + "loss": 2.0067, + "theoretical_loss": 4.849239270361078, + "tokens_seen": 92995584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852998996990973, + "loss": 1.9941, + "theoretical_loss": 4.848763568257554, + "tokens_seen": 93061120 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785283851554664, + "loss": 2.0881, + "theoretical_loss": 4.848288294762779, + "tokens_seen": 93126656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852678034102307, + "loss": 2.0293, + "theoretical_loss": 4.847813449189358, + "tokens_seen": 93192192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852517552657974, + "loss": 1.958, + "theoretical_loss": 4.847339030851482, + "tokens_seen": 93257728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852357071213641, + "loss": 1.9812, + "theoretical_loss": 4.84686503906492, + "tokens_seen": 93323264 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852196589769308, + "loss": 2.1079, + "theoretical_loss": 4.846391473147019, + "tokens_seen": 93388800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852036108324975, + "loss": 1.9527, + "theoretical_loss": 4.8459183324166935, + "tokens_seen": 93454336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851875626880642, + "loss": 1.9455, + "theoretical_loss": 4.845445616194426, + "tokens_seen": 93519872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851715145436309, + "loss": 2.1299, + "theoretical_loss": 4.844973323802259, + "tokens_seen": 93585408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851554663991976, + "loss": 2.0971, + "theoretical_loss": 4.844501454563792, + "tokens_seen": 93650944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851394182547643, + "loss": 1.9184, + "theoretical_loss": 4.844030007804177, + "tokens_seen": 93716480 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785123370110331, + "loss": 1.9424, + "theoretical_loss": 4.843558982850113, + "tokens_seen": 93782016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851073219658977, + "loss": 2.0786, + "theoretical_loss": 4.84308837902984, + "tokens_seen": 93847552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850912738214644, + "loss": 1.9767, + "theoretical_loss": 4.842618195673138, + "tokens_seen": 93913088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850752256770311, + "loss": 2.0386, + "theoretical_loss": 4.84214843211132, + "tokens_seen": 93978624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850591775325978, + "loss": 1.9345, + "theoretical_loss": 4.841679087677226, + "tokens_seen": 94044160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850431293881645, + "loss": 2.1009, + "theoretical_loss": 4.841210161705223, + "tokens_seen": 94109696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850270812437312, + "loss": 2.0208, + "theoretical_loss": 4.840741653531195, + "tokens_seen": 94175232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850110330992979, + "loss": 1.8831, + "theoretical_loss": 4.840273562492545, + "tokens_seen": 94240768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849949849548646, + "loss": 2.0515, + "theoretical_loss": 4.839805887928181, + "tokens_seen": 94306304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849789368104313, + "loss": 1.9214, + "theoretical_loss": 4.839338629178522, + "tokens_seen": 94371840 + }, + { + "epoch": 0.03, + "learning_rate": 0.000784962888665998, + "loss": 1.9477, + "theoretical_loss": 4.8388717855854875, + "tokens_seen": 94437376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849468405215647, + "loss": 1.927, + "theoretical_loss": 4.838405356492494, + "tokens_seen": 94502912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849307923771314, + "loss": 1.9606, + "theoretical_loss": 4.83793934124445, + "tokens_seen": 94568448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849147442326981, + "loss": 1.9475, + "theoretical_loss": 4.837473739187754, + "tokens_seen": 94633984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848986960882648, + "loss": 2.0574, + "theoretical_loss": 4.837008549670285, + "tokens_seen": 94699520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848826479438315, + "loss": 2.0748, + "theoretical_loss": 4.8365437720414075, + "tokens_seen": 94765056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848665997993982, + "loss": 2.0284, + "theoretical_loss": 4.836079405651956, + "tokens_seen": 94830592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848505516549649, + "loss": 2.039, + "theoretical_loss": 4.835615449854238, + "tokens_seen": 94896128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848345035105317, + "loss": 1.9871, + "theoretical_loss": 4.8351519040020285, + "tokens_seen": 94961664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 38548, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.0551602840423584, + "objective/train/theoretical_loss": 4.834688767450562, + "objective/train/tokens_used": 115487200, + "theoretical_loss": 4.834688767450562, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848184553660983, + "loss": 2.0169, + "theoretical_loss": 4.834688767450562, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.000784802407221665, + "loss": 1.9129, + "theoretical_loss": 4.8342260395565315, + "tokens_seen": 95092736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847863590772317, + "loss": 1.9872, + "theoretical_loss": 4.833763719678085, + "tokens_seen": 95158272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847703109327985, + "loss": 1.954, + "theoretical_loss": 4.833301807174822, + "tokens_seen": 95223808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847542627883651, + "loss": 2.0831, + "theoretical_loss": 4.83284030140778, + "tokens_seen": 95289344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847382146439319, + "loss": 1.894, + "theoretical_loss": 4.832379201739442, + "tokens_seen": 95354880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847221664994985, + "loss": 1.9736, + "theoretical_loss": 4.831918507533728, + "tokens_seen": 95420416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847061183550653, + "loss": 2.0363, + "theoretical_loss": 4.831458218155989, + "tokens_seen": 95485952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846900702106319, + "loss": 2.0462, + "theoretical_loss": 4.830998332973005, + "tokens_seen": 95551488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846740220661987, + "loss": 1.8864, + "theoretical_loss": 4.830538851352976, + "tokens_seen": 95617024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846579739217653, + "loss": 1.9409, + "theoretical_loss": 4.830079772665529, + "tokens_seen": 95682560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846419257773321, + "loss": 1.9883, + "theoretical_loss": 4.829621096281702, + "tokens_seen": 95748096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846258776328987, + "loss": 1.8937, + "theoretical_loss": 4.8291628215739415, + "tokens_seen": 95813632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846098294884655, + "loss": 1.9911, + "theoretical_loss": 4.828704947916108, + "tokens_seen": 95879168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845937813440321, + "loss": 1.9991, + "theoretical_loss": 4.82824747468346, + "tokens_seen": 95944704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845777331995989, + "loss": 1.9115, + "theoretical_loss": 4.827790401252658, + "tokens_seen": 96010240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845616850551655, + "loss": 1.9385, + "theoretical_loss": 4.827333727001756, + "tokens_seen": 96075776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845456369107323, + "loss": 1.9522, + "theoretical_loss": 4.8268774513101995, + "tokens_seen": 96141312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845295887662989, + "loss": 1.8942, + "theoretical_loss": 4.82642157355882, + "tokens_seen": 96206848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845135406218657, + "loss": 1.9496, + "theoretical_loss": 4.825966093129834, + "tokens_seen": 96272384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844974924774323, + "loss": 1.9805, + "theoretical_loss": 4.825511009406833, + "tokens_seen": 96337920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844814443329991, + "loss": 1.9984, + "theoretical_loss": 4.825056321774788, + "tokens_seen": 96403456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844653961885657, + "loss": 1.9623, + "theoretical_loss": 4.8246020296200385, + "tokens_seen": 96468992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844493480441325, + "loss": 1.9433, + "theoretical_loss": 4.82414813233029, + "tokens_seen": 96534528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844332998996991, + "loss": 2.0465, + "theoretical_loss": 4.823694629294609, + "tokens_seen": 96600064 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844172517552659, + "loss": 2.0518, + "theoretical_loss": 4.823241519903428, + "tokens_seen": 96665600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844012036108325, + "loss": 1.9394, + "theoretical_loss": 4.8227888035485265, + "tokens_seen": 96731136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843851554663993, + "loss": 1.8898, + "theoretical_loss": 4.82233647962304, + "tokens_seen": 96796672 + }, + { + "epoch": 0.03, + "learning_rate": 0.000784369107321966, + "loss": 1.9072, + "theoretical_loss": 4.821884547521449, + "tokens_seen": 96862208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843530591775327, + "loss": 2.0767, + "theoretical_loss": 4.821433006639578, + "tokens_seen": 96927744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843370110330994, + "loss": 1.9428, + "theoretical_loss": 4.820981856374589, + "tokens_seen": 96993280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843209628886661, + "loss": 1.9907, + "theoretical_loss": 4.8205310961249825, + "tokens_seen": 97058816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843049147442328, + "loss": 1.9943, + "theoretical_loss": 4.820080725290589, + "tokens_seen": 97124352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842888665997995, + "loss": 1.9588, + "theoretical_loss": 4.819630743272566, + "tokens_seen": 97189888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842728184553662, + "loss": 2.0534, + "theoretical_loss": 4.819181149473396, + "tokens_seen": 97255424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842567703109329, + "loss": 2.0432, + "theoretical_loss": 4.81873194329688, + "tokens_seen": 97320960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842407221664996, + "loss": 2.0024, + "theoretical_loss": 4.81828312414814, + "tokens_seen": 97386496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842246740220663, + "loss": 2.0566, + "theoretical_loss": 4.817834691433602, + "tokens_seen": 97452032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842086258776329, + "loss": 1.9204, + "theoretical_loss": 4.81738664456101, + "tokens_seen": 97517568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841925777331997, + "loss": 1.8443, + "theoretical_loss": 4.816938982939407, + "tokens_seen": 97583104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841765295887663, + "loss": 2.0024, + "theoretical_loss": 4.816491705979138, + "tokens_seen": 97648640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841604814443331, + "loss": 2.0781, + "theoretical_loss": 4.816044813091848, + "tokens_seen": 97714176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841444332998997, + "loss": 2.0516, + "theoretical_loss": 4.815598303690473, + "tokens_seen": 97779712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841283851554665, + "loss": 1.8457, + "theoretical_loss": 4.8151521771892405, + "tokens_seen": 97845248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841123370110331, + "loss": 1.9794, + "theoretical_loss": 4.814706433003665, + "tokens_seen": 97910784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840962888665999, + "loss": 1.988, + "theoretical_loss": 4.8142610705505415, + "tokens_seen": 97976320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840802407221665, + "loss": 1.9506, + "theoretical_loss": 4.813816089247945, + "tokens_seen": 98041856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840641925777332, + "loss": 2.0677, + "theoretical_loss": 4.813371488515227, + "tokens_seen": 98107392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840481444332999, + "loss": 1.8749, + "theoretical_loss": 4.812927267773008, + "tokens_seen": 98172928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840320962888666, + "loss": 2.0423, + "theoretical_loss": 4.812483426443181, + "tokens_seen": 98238464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 39774, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.065314292907715, + "objective/train/theoretical_loss": 4.812039963948898, + "objective/train/tokens_used": 118764000, + "theoretical_loss": 4.812039963948898, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840160481444333, + "loss": 1.9389, + "theoretical_loss": 4.812039963948898, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.000784, + "loss": 2.0512, + "theoretical_loss": 4.811596879714575, + "tokens_seen": 98369536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839839518555667, + "loss": 2.0692, + "theoretical_loss": 4.811154173165886, + "tokens_seen": 98435072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839679037111334, + "loss": 2.0087, + "theoretical_loss": 4.810711843729758, + "tokens_seen": 98500608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839518555667001, + "loss": 1.9157, + "theoretical_loss": 4.810269890834364, + "tokens_seen": 98566144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839358074222668, + "loss": 1.8269, + "theoretical_loss": 4.809828313909129, + "tokens_seen": 98631680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839197592778335, + "loss": 1.9956, + "theoretical_loss": 4.809387112384721, + "tokens_seen": 98697216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839037111334002, + "loss": 1.9715, + "theoretical_loss": 4.808946285693043, + "tokens_seen": 98762752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838876629889669, + "loss": 1.8838, + "theoretical_loss": 4.808505833267237, + "tokens_seen": 98828288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838716148445336, + "loss": 1.8904, + "theoretical_loss": 4.808065754541676, + "tokens_seen": 98893824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838555667001003, + "loss": 1.89, + "theoretical_loss": 4.807626048951965, + "tokens_seen": 98959360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783839518555667, + "loss": 1.9809, + "theoretical_loss": 4.807186715934931, + "tokens_seen": 99024896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838234704112337, + "loss": 1.8864, + "theoretical_loss": 4.806747754928622, + "tokens_seen": 99090432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838074222668004, + "loss": 1.8412, + "theoretical_loss": 4.806309165372311, + "tokens_seen": 99155968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837913741223671, + "loss": 1.8538, + "theoretical_loss": 4.805870946706479, + "tokens_seen": 99221504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837753259779338, + "loss": 2.0107, + "theoretical_loss": 4.805433098372822, + "tokens_seen": 99287040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837592778335005, + "loss": 2.0084, + "theoretical_loss": 4.8049956198142425, + "tokens_seen": 99352576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837432296890672, + "loss": 1.9323, + "theoretical_loss": 4.804558510474852, + "tokens_seen": 99418112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837271815446339, + "loss": 1.9652, + "theoretical_loss": 4.804121769799959, + "tokens_seen": 99483648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837111334002006, + "loss": 1.8399, + "theoretical_loss": 4.8036853972360705, + "tokens_seen": 99549184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836950852557673, + "loss": 1.9356, + "theoretical_loss": 4.80324939223089, + "tokens_seen": 99614720 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783679037111334, + "loss": 2.0503, + "theoretical_loss": 4.802813754233311, + "tokens_seen": 99680256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836629889669007, + "loss": 1.9646, + "theoretical_loss": 4.802378482693417, + "tokens_seen": 99745792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836469408224674, + "loss": 1.9134, + "theoretical_loss": 4.801943577062472, + "tokens_seen": 99811328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836308926780341, + "loss": 1.9831, + "theoretical_loss": 4.801509036792925, + "tokens_seen": 99876864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836148445336008, + "loss": 2.1101, + "theoretical_loss": 4.801074861338401, + "tokens_seen": 99942400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835987963891675, + "loss": 1.9731, + "theoretical_loss": 4.800641050153703, + "tokens_seen": 100007936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835827482447342, + "loss": 2.0708, + "theoretical_loss": 4.800207602694801, + "tokens_seen": 100073472 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783566700100301, + "loss": 1.9291, + "theoretical_loss": 4.7997745184188325, + "tokens_seen": 100139008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835506519558676, + "loss": 2.0162, + "theoretical_loss": 4.799341796784106, + "tokens_seen": 100204544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835346038114343, + "loss": 2.0161, + "theoretical_loss": 4.798909437250086, + "tokens_seen": 100270080 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783518555667001, + "loss": 1.872, + "theoretical_loss": 4.798477439277397, + "tokens_seen": 100335616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835025075225677, + "loss": 1.9359, + "theoretical_loss": 4.7980458023278185, + "tokens_seen": 100401152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834864593781344, + "loss": 2.046, + "theoretical_loss": 4.797614525864282, + "tokens_seen": 100466688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834704112337011, + "loss": 2.0471, + "theoretical_loss": 4.797183609350869, + "tokens_seen": 100532224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834543630892678, + "loss": 1.9752, + "theoretical_loss": 4.796753052252802, + "tokens_seen": 100597760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834383149448346, + "loss": 1.8555, + "theoretical_loss": 4.79632285403645, + "tokens_seen": 100663296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834222668004012, + "loss": 1.9888, + "theoretical_loss": 4.795893014169323, + "tokens_seen": 100728832 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783406218655968, + "loss": 1.9027, + "theoretical_loss": 4.795463532120058, + "tokens_seen": 100794368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833901705115346, + "loss": 1.9115, + "theoretical_loss": 4.795034407358435, + "tokens_seen": 100859904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833741223671014, + "loss": 1.9345, + "theoretical_loss": 4.794605639355357, + "tokens_seen": 100925440 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783358074222668, + "loss": 1.9462, + "theoretical_loss": 4.794177227582855, + "tokens_seen": 100990976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833420260782348, + "loss": 1.9011, + "theoretical_loss": 4.793749171514085, + "tokens_seen": 101056512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833259779338014, + "loss": 1.8762, + "theoretical_loss": 4.793321470623321, + "tokens_seen": 101122048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833099297893682, + "loss": 1.9244, + "theoretical_loss": 4.792894124385955, + "tokens_seen": 101187584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832938816449348, + "loss": 1.9741, + "theoretical_loss": 4.792467132278494, + "tokens_seen": 101253120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832778335005016, + "loss": 1.9348, + "theoretical_loss": 4.792040493778553, + "tokens_seen": 101318656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832617853560682, + "loss": 2.0157, + "theoretical_loss": 4.791614208364859, + "tokens_seen": 101384192 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783245737211635, + "loss": 1.9452, + "theoretical_loss": 4.791188275517239, + "tokens_seen": 101449728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832296890672016, + "loss": 1.9476, + "theoretical_loss": 4.7907626947166255, + "tokens_seen": 101515264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 40787, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.9380860328674316, + "objective/train/theoretical_loss": 4.790337465445049, + "objective/train/tokens_used": 122040800, + "theoretical_loss": 4.790337465445049, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832136409227684, + "loss": 1.9874, + "theoretical_loss": 4.790337465445049, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783197592778335, + "loss": 2.06, + "theoretical_loss": 4.7899125871856345, + "tokens_seen": 101646336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831815446339018, + "loss": 1.8105, + "theoretical_loss": 4.789488059422599, + "tokens_seen": 101711872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831654964894685, + "loss": 2.0042, + "theoretical_loss": 4.789063881641251, + "tokens_seen": 101777408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831494483450352, + "loss": 2.0481, + "theoretical_loss": 4.7886400533279865, + "tokens_seen": 101842944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831334002006019, + "loss": 1.9965, + "theoretical_loss": 4.788216573970281, + "tokens_seen": 101908480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831173520561686, + "loss": 1.9324, + "theoretical_loss": 4.787793443056694, + "tokens_seen": 101974016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831013039117353, + "loss": 1.8745, + "theoretical_loss": 4.787370660076862, + "tokens_seen": 102039552 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783085255767302, + "loss": 2.0481, + "theoretical_loss": 4.786948224521495, + "tokens_seen": 102105088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830692076228687, + "loss": 1.9234, + "theoretical_loss": 4.786526135882378, + "tokens_seen": 102170624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830531594784354, + "loss": 2.0611, + "theoretical_loss": 4.786104393652359, + "tokens_seen": 102236160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830371113340021, + "loss": 1.9584, + "theoretical_loss": 4.785682997325358, + "tokens_seen": 102301696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830210631895688, + "loss": 1.9192, + "theoretical_loss": 4.7852619463963535, + "tokens_seen": 102367232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830050150451355, + "loss": 1.8914, + "theoretical_loss": 4.784841240361389, + "tokens_seen": 102432768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829889669007022, + "loss": 1.9203, + "theoretical_loss": 4.784420878717558, + "tokens_seen": 102498304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829729187562689, + "loss": 1.9652, + "theoretical_loss": 4.7840008609630145, + "tokens_seen": 102563840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829568706118356, + "loss": 1.9889, + "theoretical_loss": 4.783581186596962, + "tokens_seen": 102629376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829408224674023, + "loss": 1.8846, + "theoretical_loss": 4.78316185511965, + "tokens_seen": 102694912 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782924774322969, + "loss": 2.0007, + "theoretical_loss": 4.782742866032379, + "tokens_seen": 102760448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829087261785357, + "loss": 1.8957, + "theoretical_loss": 4.782324218837486, + "tokens_seen": 102825984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828926780341024, + "loss": 1.9462, + "theoretical_loss": 4.781905913038351, + "tokens_seen": 102891520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828766298896691, + "loss": 1.9478, + "theoretical_loss": 4.781487948139393, + "tokens_seen": 102957056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828605817452358, + "loss": 1.9648, + "theoretical_loss": 4.7810703236460625, + "tokens_seen": 103022592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828445336008025, + "loss": 1.9738, + "theoretical_loss": 4.780653039064843, + "tokens_seen": 103088128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828284854563692, + "loss": 1.9539, + "theoretical_loss": 4.780236093903245, + "tokens_seen": 103153664 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828124373119358, + "loss": 1.8898, + "theoretical_loss": 4.779819487669805, + "tokens_seen": 103219200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827963891675026, + "loss": 1.8978, + "theoretical_loss": 4.779403219874085, + "tokens_seen": 103284736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827803410230692, + "loss": 1.97, + "theoretical_loss": 4.778987290026665, + "tokens_seen": 103350272 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782764292878636, + "loss": 1.9551, + "theoretical_loss": 4.778571697639142, + "tokens_seen": 103415808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827482447342026, + "loss": 1.8563, + "theoretical_loss": 4.778156442224132, + "tokens_seen": 103481344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827321965897694, + "loss": 2.0819, + "theoretical_loss": 4.7777415232952585, + "tokens_seen": 103546880 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782716148445336, + "loss": 1.9497, + "theoretical_loss": 4.777326940367155, + "tokens_seen": 103612416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827001003009028, + "loss": 2.0545, + "theoretical_loss": 4.776912692955463, + "tokens_seen": 103677952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826840521564694, + "loss": 1.9213, + "theoretical_loss": 4.776498780576826, + "tokens_seen": 103743488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826680040120361, + "loss": 1.9726, + "theoretical_loss": 4.776085202748893, + "tokens_seen": 103809024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826519558676028, + "loss": 1.8898, + "theoretical_loss": 4.775671958990305, + "tokens_seen": 103874560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826359077231695, + "loss": 2.004, + "theoretical_loss": 4.775259048820702, + "tokens_seen": 103940096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826198595787362, + "loss": 1.9216, + "theoretical_loss": 4.7748464717607195, + "tokens_seen": 104005632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826038114343029, + "loss": 1.8736, + "theoretical_loss": 4.774434227331979, + "tokens_seen": 104071168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825877632898696, + "loss": 1.9748, + "theoretical_loss": 4.774022315057092, + "tokens_seen": 104136704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825717151454363, + "loss": 1.9511, + "theoretical_loss": 4.773610734459654, + "tokens_seen": 104202240 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782555667001003, + "loss": 1.8595, + "theoretical_loss": 4.773199485064243, + "tokens_seen": 104267776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825396188565697, + "loss": 1.8144, + "theoretical_loss": 4.772788566396418, + "tokens_seen": 104333312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825235707121364, + "loss": 1.9711, + "theoretical_loss": 4.772377977982714, + "tokens_seen": 104398848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825075225677031, + "loss": 1.8884, + "theoretical_loss": 4.771967719350641, + "tokens_seen": 104464384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824914744232698, + "loss": 1.9437, + "theoretical_loss": 4.77155779002868, + "tokens_seen": 104529920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824754262788365, + "loss": 1.8921, + "theoretical_loss": 4.771148189546282, + "tokens_seen": 104595456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824593781344032, + "loss": 2.0112, + "theoretical_loss": 4.770738917433864, + "tokens_seen": 104660992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824433299899699, + "loss": 1.9376, + "theoretical_loss": 4.770329973222809, + "tokens_seen": 104726528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824272818455366, + "loss": 1.966, + "theoretical_loss": 4.769921356445458, + "tokens_seen": 104792064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 42078, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.965983510017395, + "objective/train/theoretical_loss": 4.769513066635114, + "objective/train/tokens_used": 125317600, + "theoretical_loss": 4.769513066635114, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824112337011033, + "loss": 1.9, + "theoretical_loss": 4.769513066635114, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.00078239518555667, + "loss": 1.9222, + "theoretical_loss": 4.769105103326036, + "tokens_seen": 104923136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823791374122367, + "loss": 2.0369, + "theoretical_loss": 4.768697466053435, + "tokens_seen": 104988672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823630892678034, + "loss": 1.9214, + "theoretical_loss": 4.768290154353474, + "tokens_seen": 105054208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823470411233701, + "loss": 1.9252, + "theoretical_loss": 4.767883167763267, + "tokens_seen": 105119744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823309929789368, + "loss": 1.9184, + "theoretical_loss": 4.76747650582087, + "tokens_seen": 105185280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823149448345035, + "loss": 1.8375, + "theoretical_loss": 4.767070168065285, + "tokens_seen": 105250816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822988966900702, + "loss": 1.8323, + "theoretical_loss": 4.766664154036456, + "tokens_seen": 105316352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822828485456369, + "loss": 1.7604, + "theoretical_loss": 4.766258463275265, + "tokens_seen": 105381888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822668004012036, + "loss": 1.9705, + "theoretical_loss": 4.7658530953235285, + "tokens_seen": 105447424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822507522567703, + "loss": 1.9077, + "theoretical_loss": 4.765448049723998, + "tokens_seen": 105512960 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782234704112337, + "loss": 1.7903, + "theoretical_loss": 4.765043326020358, + "tokens_seen": 105578496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822186559679037, + "loss": 1.9913, + "theoretical_loss": 4.764638923757217, + "tokens_seen": 105644032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822026078234704, + "loss": 1.8003, + "theoretical_loss": 4.764234842480114, + "tokens_seen": 105709568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821865596790371, + "loss": 1.8746, + "theoretical_loss": 4.7638310817355105, + "tokens_seen": 105775104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821705115346038, + "loss": 1.9363, + "theoretical_loss": 4.763427641070789, + "tokens_seen": 105840640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821544633901705, + "loss": 1.8989, + "theoretical_loss": 4.76302452003425, + "tokens_seen": 105906176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821384152457372, + "loss": 1.9954, + "theoretical_loss": 4.7626217181751125, + "tokens_seen": 105971712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821223671013039, + "loss": 1.9583, + "theoretical_loss": 4.762219235043508, + "tokens_seen": 106037248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821063189568706, + "loss": 1.9582, + "theoretical_loss": 4.761817070190481, + "tokens_seen": 106102784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820902708124373, + "loss": 2.018, + "theoretical_loss": 4.761415223167982, + "tokens_seen": 106168320 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782074222668004, + "loss": 2.0643, + "theoretical_loss": 4.761013693528874, + "tokens_seen": 106233856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820581745235707, + "loss": 1.9534, + "theoretical_loss": 4.760612480826917, + "tokens_seen": 106299392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820421263791375, + "loss": 1.9656, + "theoretical_loss": 4.76021158461678, + "tokens_seen": 106364928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820260782347041, + "loss": 1.9398, + "theoretical_loss": 4.759811004454027, + "tokens_seen": 106430464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820100300902709, + "loss": 1.9891, + "theoretical_loss": 4.759410739895122, + "tokens_seen": 106496000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819939819458375, + "loss": 1.9332, + "theoretical_loss": 4.759010790497422, + "tokens_seen": 106561536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819779338014043, + "loss": 1.9756, + "theoretical_loss": 4.758611155819178, + "tokens_seen": 106627072 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781961885656971, + "loss": 1.9532, + "theoretical_loss": 4.758211835419531, + "tokens_seen": 106692608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819458375125377, + "loss": 1.8169, + "theoretical_loss": 4.757812828858508, + "tokens_seen": 106758144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819297893681044, + "loss": 1.8261, + "theoretical_loss": 4.757414135697024, + "tokens_seen": 106823680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819137412236711, + "loss": 1.9218, + "theoretical_loss": 4.757015755496877, + "tokens_seen": 106889216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818976930792378, + "loss": 1.7847, + "theoretical_loss": 4.756617687820743, + "tokens_seen": 106954752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818816449348045, + "loss": 1.9165, + "theoretical_loss": 4.756219932232181, + "tokens_seen": 107020288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818655967903712, + "loss": 1.9031, + "theoretical_loss": 4.755822488295621, + "tokens_seen": 107085824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818495486459379, + "loss": 2.0325, + "theoretical_loss": 4.755425355576373, + "tokens_seen": 107151360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818335005015046, + "loss": 1.9744, + "theoretical_loss": 4.755028533640614, + "tokens_seen": 107216896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818174523570713, + "loss": 1.9273, + "theoretical_loss": 4.754632022055392, + "tokens_seen": 107282432 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781801404212638, + "loss": 1.9457, + "theoretical_loss": 4.754235820388624, + "tokens_seen": 107347968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817853560682047, + "loss": 1.7709, + "theoretical_loss": 4.7538399282090875, + "tokens_seen": 107413504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817693079237714, + "loss": 1.9709, + "theoretical_loss": 4.753444345086428, + "tokens_seen": 107479040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817532597793381, + "loss": 1.8569, + "theoretical_loss": 4.753049070591146, + "tokens_seen": 107544576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817372116349048, + "loss": 1.8751, + "theoretical_loss": 4.752654104294605, + "tokens_seen": 107610112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817211634904715, + "loss": 1.9358, + "theoretical_loss": 4.752259445769022, + "tokens_seen": 107675648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817051153460382, + "loss": 1.8664, + "theoretical_loss": 4.751865094587465, + "tokens_seen": 107741184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816890672016049, + "loss": 1.8228, + "theoretical_loss": 4.75147105032386, + "tokens_seen": 107806720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816730190571716, + "loss": 1.8293, + "theoretical_loss": 4.751077312552976, + "tokens_seen": 107872256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816569709127383, + "loss": 1.8754, + "theoretical_loss": 4.750683880850433, + "tokens_seen": 107937792 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781640922768305, + "loss": 1.9043, + "theoretical_loss": 4.750290754792694, + "tokens_seen": 108003328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816248746238717, + "loss": 1.8329, + "theoretical_loss": 4.749897933957065, + "tokens_seen": 108068864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 42949, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.9812370538711548, + "objective/train/theoretical_loss": 4.749505417921692, + "objective/train/tokens_used": 128594400, + "theoretical_loss": 4.749505417921692, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816088264794384, + "loss": 1.989, + "theoretical_loss": 4.749505417921692, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815927783350051, + "loss": 1.9188, + "theoretical_loss": 4.7491132062655605, + "tokens_seen": 108199936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815767301905718, + "loss": 1.8283, + "theoretical_loss": 4.748721298568491, + "tokens_seen": 108265472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815606820461385, + "loss": 2.0035, + "theoretical_loss": 4.748329694411137, + "tokens_seen": 108331008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815446339017052, + "loss": 1.9027, + "theoretical_loss": 4.747938393374987, + "tokens_seen": 108396544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815285857572719, + "loss": 1.8085, + "theoretical_loss": 4.747547395042355, + "tokens_seen": 108462080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815125376128386, + "loss": 1.8566, + "theoretical_loss": 4.747156698996388, + "tokens_seen": 108527616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814964894684053, + "loss": 1.8354, + "theoretical_loss": 4.746766304821053, + "tokens_seen": 108593152 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781480441323972, + "loss": 1.98, + "theoretical_loss": 4.746376212101142, + "tokens_seen": 108658688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814643931795387, + "loss": 1.98, + "theoretical_loss": 4.745986420422267, + "tokens_seen": 108724224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814483450351054, + "loss": 1.8918, + "theoretical_loss": 4.745596929370863, + "tokens_seen": 108789760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814322968906721, + "loss": 1.9303, + "theoretical_loss": 4.745207738534177, + "tokens_seen": 108855296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814162487462388, + "loss": 1.8991, + "theoretical_loss": 4.744818847500274, + "tokens_seen": 108920832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814002006018055, + "loss": 1.9216, + "theoretical_loss": 4.744430255858029, + "tokens_seen": 108986368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813841524573721, + "loss": 1.8368, + "theoretical_loss": 4.744041963197132, + "tokens_seen": 109051904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813681043129389, + "loss": 1.9503, + "theoretical_loss": 4.743653969108076, + "tokens_seen": 109117440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813520561685055, + "loss": 1.8052, + "theoretical_loss": 4.743266273182163, + "tokens_seen": 109182976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813360080240723, + "loss": 1.8667, + "theoretical_loss": 4.742878875011501, + "tokens_seen": 109248512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813199598796389, + "loss": 1.8948, + "theoretical_loss": 4.742491774188997, + "tokens_seen": 109314048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813039117352057, + "loss": 2.0147, + "theoretical_loss": 4.742104970308359, + "tokens_seen": 109379584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812878635907723, + "loss": 1.8349, + "theoretical_loss": 4.741718462964095, + "tokens_seen": 109445120 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781271815446339, + "loss": 2.0293, + "theoretical_loss": 4.741332251751509, + "tokens_seen": 109510656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812557673019057, + "loss": 1.9101, + "theoretical_loss": 4.740946336266697, + "tokens_seen": 109576192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812397191574724, + "loss": 1.9097, + "theoretical_loss": 4.740560716106549, + "tokens_seen": 109641728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812236710130391, + "loss": 1.9329, + "theoretical_loss": 4.740175390868742, + "tokens_seen": 109707264 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812076228686058, + "loss": 1.8663, + "theoretical_loss": 4.739790360151744, + "tokens_seen": 109772800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811915747241725, + "loss": 1.9846, + "theoretical_loss": 4.73940562355481, + "tokens_seen": 109838336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811755265797392, + "loss": 1.8642, + "theoretical_loss": 4.739021180677976, + "tokens_seen": 109903872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811594784353059, + "loss": 1.8875, + "theoretical_loss": 4.73863703112206, + "tokens_seen": 109969408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811434302908726, + "loss": 1.8295, + "theoretical_loss": 4.738253174488662, + "tokens_seen": 110034944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811273821464393, + "loss": 1.8563, + "theoretical_loss": 4.73786961038016, + "tokens_seen": 110100480 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781111334002006, + "loss": 1.9474, + "theoretical_loss": 4.737486338399704, + "tokens_seen": 110166016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810952858575727, + "loss": 2.0176, + "theoretical_loss": 4.737103358151225, + "tokens_seen": 110231552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810792377131394, + "loss": 1.8906, + "theoretical_loss": 4.7367206692394195, + "tokens_seen": 110297088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810631895687061, + "loss": 1.8809, + "theoretical_loss": 4.736338271269759, + "tokens_seen": 110362624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810471414242728, + "loss": 1.931, + "theoretical_loss": 4.735956163848478, + "tokens_seen": 110428160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810310932798395, + "loss": 1.968, + "theoretical_loss": 4.735574346582582, + "tokens_seen": 110493696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810150451354062, + "loss": 1.9077, + "theoretical_loss": 4.735192819079838, + "tokens_seen": 110559232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809989969909729, + "loss": 1.8817, + "theoretical_loss": 4.734811580948779, + "tokens_seen": 110624768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809829488465396, + "loss": 1.8448, + "theoretical_loss": 4.734430631798692, + "tokens_seen": 110690304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809669007021063, + "loss": 1.9445, + "theoretical_loss": 4.734049971239628, + "tokens_seen": 110755840 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780950852557673, + "loss": 1.9162, + "theoretical_loss": 4.7336695988823925, + "tokens_seen": 110821376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809348044132397, + "loss": 1.9709, + "theoretical_loss": 4.733289514338546, + "tokens_seen": 110886912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809187562688064, + "loss": 1.8657, + "theoretical_loss": 4.7329097172204, + "tokens_seen": 110952448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809027081243731, + "loss": 1.7885, + "theoretical_loss": 4.73253020714102, + "tokens_seen": 111017984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808866599799398, + "loss": 1.9659, + "theoretical_loss": 4.732150983714217, + "tokens_seen": 111083520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808706118355065, + "loss": 1.8298, + "theoretical_loss": 4.731772046554552, + "tokens_seen": 111149056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808545636910732, + "loss": 1.9417, + "theoretical_loss": 4.731393395277331, + "tokens_seen": 111214592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808385155466399, + "loss": 1.8618, + "theoretical_loss": 4.731015029498598, + "tokens_seen": 111280128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808224674022066, + "loss": 1.9002, + "theoretical_loss": 4.730636948835148, + "tokens_seen": 111345664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 44130, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.9857747554779053, + "objective/train/theoretical_loss": 4.730259152904507, + "objective/train/tokens_used": 131871200, + "theoretical_loss": 4.730259152904507, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808064192577733, + "loss": 1.9456, + "theoretical_loss": 4.730259152904507, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.00078079037111334, + "loss": 1.7301, + "theoretical_loss": 4.7298816413249405, + "tokens_seen": 111476736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807743229689067, + "loss": 1.7904, + "theoretical_loss": 4.729504413715454, + "tokens_seen": 111542272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807582748244735, + "loss": 1.8232, + "theoretical_loss": 4.729127469695783, + "tokens_seen": 111607808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807422266800401, + "loss": 1.8824, + "theoretical_loss": 4.728750808886394, + "tokens_seen": 111673344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807261785356069, + "loss": 1.9541, + "theoretical_loss": 4.72837443090849, + "tokens_seen": 111738880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807101303911736, + "loss": 1.9042, + "theoretical_loss": 4.727998335383996, + "tokens_seen": 111804416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806940822467403, + "loss": 1.8814, + "theoretical_loss": 4.727622521935563, + "tokens_seen": 111869952 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780678034102307, + "loss": 1.8453, + "theoretical_loss": 4.7272469901865755, + "tokens_seen": 111935488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806619859578737, + "loss": 1.8953, + "theoretical_loss": 4.726871739761132, + "tokens_seen": 112001024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806459378134404, + "loss": 1.8651, + "theoretical_loss": 4.726496770284056, + "tokens_seen": 112066560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806298896690071, + "loss": 1.9383, + "theoretical_loss": 4.726122081380891, + "tokens_seen": 112132096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806138415245738, + "loss": 1.8434, + "theoretical_loss": 4.725747672677894, + "tokens_seen": 112197632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805977933801405, + "loss": 1.9197, + "theoretical_loss": 4.725373543802043, + "tokens_seen": 112263168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805817452357072, + "loss": 1.9461, + "theoretical_loss": 4.724999694381027, + "tokens_seen": 112328704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805656970912739, + "loss": 1.858, + "theoretical_loss": 4.724626124043247, + "tokens_seen": 112394240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805496489468406, + "loss": 1.9208, + "theoretical_loss": 4.724252832417817, + "tokens_seen": 112459776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805336008024073, + "loss": 1.7769, + "theoretical_loss": 4.723879819134558, + "tokens_seen": 112525312 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780517552657974, + "loss": 1.807, + "theoretical_loss": 4.7235070838239945, + "tokens_seen": 112590848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805015045135407, + "loss": 1.9201, + "theoretical_loss": 4.723134626117363, + "tokens_seen": 112656384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804854563691074, + "loss": 1.8909, + "theoretical_loss": 4.722762445646598, + "tokens_seen": 112721920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804694082246741, + "loss": 1.9641, + "theoretical_loss": 4.722390542044339, + "tokens_seen": 112787456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804533600802408, + "loss": 1.8542, + "theoretical_loss": 4.722018914943924, + "tokens_seen": 112852992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804373119358075, + "loss": 1.7947, + "theoretical_loss": 4.721647563979385, + "tokens_seen": 112918528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804212637913742, + "loss": 1.7628, + "theoretical_loss": 4.72127648878546, + "tokens_seen": 112984064 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804052156469409, + "loss": 2.0437, + "theoretical_loss": 4.720905688997572, + "tokens_seen": 113049600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803891675025076, + "loss": 1.8696, + "theoretical_loss": 4.720535164251842, + "tokens_seen": 113115136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803731193580743, + "loss": 2.0169, + "theoretical_loss": 4.720164914185084, + "tokens_seen": 113180672 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780357071213641, + "loss": 1.9562, + "theoretical_loss": 4.719794938434794, + "tokens_seen": 113246208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803410230692077, + "loss": 1.8761, + "theoretical_loss": 4.7194252366391645, + "tokens_seen": 113311744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803249749247744, + "loss": 1.8482, + "theoretical_loss": 4.719055808437068, + "tokens_seen": 113377280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803089267803411, + "loss": 1.8214, + "theoretical_loss": 4.718686653468065, + "tokens_seen": 113442816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802928786359078, + "loss": 1.89, + "theoretical_loss": 4.718317771372398, + "tokens_seen": 113508352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802768304914745, + "loss": 2.0138, + "theoretical_loss": 4.7179491617909886, + "tokens_seen": 113573888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802607823470412, + "loss": 1.8962, + "theoretical_loss": 4.717580824365439, + "tokens_seen": 113639424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802447342026079, + "loss": 1.9767, + "theoretical_loss": 4.71721275873803, + "tokens_seen": 113704960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802286860581746, + "loss": 1.8079, + "theoretical_loss": 4.716844964551717, + "tokens_seen": 113770496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802126379137413, + "loss": 1.9601, + "theoretical_loss": 4.716477441450131, + "tokens_seen": 113836032 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780196589769308, + "loss": 1.8804, + "theoretical_loss": 4.716110189077575, + "tokens_seen": 113901568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801805416248747, + "loss": 1.9204, + "theoretical_loss": 4.715743207079022, + "tokens_seen": 113967104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801644934804414, + "loss": 1.8374, + "theoretical_loss": 4.715376495100118, + "tokens_seen": 114032640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801484453360081, + "loss": 1.9237, + "theoretical_loss": 4.7150100527871714, + "tokens_seen": 114098176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801323971915748, + "loss": 1.8361, + "theoretical_loss": 4.714643879787161, + "tokens_seen": 114163712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801163490471415, + "loss": 1.9508, + "theoretical_loss": 4.714277975747729, + "tokens_seen": 114229248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801003009027082, + "loss": 1.8377, + "theoretical_loss": 4.7139123403171785, + "tokens_seen": 114294784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800842527582749, + "loss": 1.771, + "theoretical_loss": 4.713546973144476, + "tokens_seen": 114360320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800682046138416, + "loss": 1.7296, + "theoretical_loss": 4.713181873879248, + "tokens_seen": 114425856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800521564694083, + "loss": 1.9301, + "theoretical_loss": 4.712817042171776, + "tokens_seen": 114491392 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780036108324975, + "loss": 1.804, + "theoretical_loss": 4.712452477673001, + "tokens_seen": 114556928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800200601805417, + "loss": 1.7899, + "theoretical_loss": 4.712088180034517, + "tokens_seen": 114622464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 45100, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7656515836715698, + "objective/train/theoretical_loss": 4.711724148908571, + "objective/train/tokens_used": 135148000, + "theoretical_loss": 4.711724148908571, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800040120361084, + "loss": 1.7773, + "theoretical_loss": 4.711724148908571, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.000779987963891675, + "loss": 1.8698, + "theoretical_loss": 4.711360383948064, + "tokens_seen": 114753536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799719157472418, + "loss": 1.81, + "theoretical_loss": 4.710996884806542, + "tokens_seen": 114819072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799558676028084, + "loss": 1.8963, + "theoretical_loss": 4.710633651138206, + "tokens_seen": 114884608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799398194583752, + "loss": 2.0256, + "theoretical_loss": 4.710270682597898, + "tokens_seen": 114950144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799237713139418, + "loss": 1.9627, + "theoretical_loss": 4.709907978841109, + "tokens_seen": 115015680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799077231695086, + "loss": 1.9976, + "theoretical_loss": 4.709545539523971, + "tokens_seen": 115081216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007798916750250752, + "loss": 1.7934, + "theoretical_loss": 4.709183364303258, + "tokens_seen": 115146752 + }, + { + "epoch": 0.03, + "learning_rate": 0.000779875626880642, + "loss": 1.7876, + "theoretical_loss": 4.708821452836388, + "tokens_seen": 115212288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007798595787362086, + "loss": 1.8766, + "theoretical_loss": 4.708459804781414, + "tokens_seen": 115277824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007798435305917753, + "loss": 1.8795, + "theoretical_loss": 4.708098419797031, + "tokens_seen": 115343360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000779827482447342, + "loss": 1.8992, + "theoretical_loss": 4.707737297542563, + "tokens_seen": 115408896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007798114343029087, + "loss": 1.7714, + "theoretical_loss": 4.707376437677974, + "tokens_seen": 115474432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797953861584754, + "loss": 1.8703, + "theoretical_loss": 4.707015839863859, + "tokens_seen": 115539968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797793380140421, + "loss": 1.8995, + "theoretical_loss": 4.706655503761443, + "tokens_seen": 115605504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797632898696088, + "loss": 1.8673, + "theoretical_loss": 4.706295429032584, + "tokens_seen": 115671040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797472417251755, + "loss": 1.8975, + "theoretical_loss": 4.705935615339763, + "tokens_seen": 115736576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797311935807422, + "loss": 1.964, + "theoretical_loss": 4.705576062346094, + "tokens_seen": 115802112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797151454363089, + "loss": 1.7302, + "theoretical_loss": 4.70521676971531, + "tokens_seen": 115867648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796990972918756, + "loss": 1.8957, + "theoretical_loss": 4.704857737111773, + "tokens_seen": 115933184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796830491474423, + "loss": 1.8279, + "theoretical_loss": 4.704498964200463, + "tokens_seen": 115998720 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779667001003009, + "loss": 1.9176, + "theoretical_loss": 4.70414045064698, + "tokens_seen": 116064256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796509528585757, + "loss": 1.9368, + "theoretical_loss": 4.703782196117549, + "tokens_seen": 116129792 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796349047141424, + "loss": 1.9024, + "theoretical_loss": 4.703424200279007, + "tokens_seen": 116195328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796188565697091, + "loss": 1.7334, + "theoretical_loss": 4.703066462798809, + "tokens_seen": 116260864 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796028084252758, + "loss": 2.0384, + "theoretical_loss": 4.7027089833450235, + "tokens_seen": 116326400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007795867602808425, + "loss": 1.8372, + "theoretical_loss": 4.7023517615863355, + "tokens_seen": 116391936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007795707121364092, + "loss": 1.7742, + "theoretical_loss": 4.701994797192038, + "tokens_seen": 116457472 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779554663991976, + "loss": 1.977, + "theoretical_loss": 4.701638089832034, + "tokens_seen": 116523008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007795386158475426, + "loss": 1.9297, + "theoretical_loss": 4.701281639176839, + "tokens_seen": 116588544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007795225677031094, + "loss": 1.8724, + "theoretical_loss": 4.700925444897575, + "tokens_seen": 116654080 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779506519558676, + "loss": 1.7618, + "theoretical_loss": 4.700569506665965, + "tokens_seen": 116719616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794904714142428, + "loss": 1.8056, + "theoretical_loss": 4.700213824154341, + "tokens_seen": 116785152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794744232698094, + "loss": 1.8923, + "theoretical_loss": 4.699858397035637, + "tokens_seen": 116850688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794583751253762, + "loss": 1.9056, + "theoretical_loss": 4.699503224983388, + "tokens_seen": 116916224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794423269809428, + "loss": 1.9701, + "theoretical_loss": 4.699148307671729, + "tokens_seen": 116981760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794262788365096, + "loss": 1.8863, + "theoretical_loss": 4.698793644775394, + "tokens_seen": 117047296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794102306920762, + "loss": 1.9355, + "theoretical_loss": 4.698439235969713, + "tokens_seen": 117112832 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779394182547643, + "loss": 1.9582, + "theoretical_loss": 4.698085080930612, + "tokens_seen": 117178368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007793781344032096, + "loss": 1.8616, + "theoretical_loss": 4.697731179334614, + "tokens_seen": 117243904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007793620862587764, + "loss": 1.7266, + "theoretical_loss": 4.69737753085883, + "tokens_seen": 117309440 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779346038114343, + "loss": 1.9112, + "theoretical_loss": 4.697024135180968, + "tokens_seen": 117374976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007793299899699098, + "loss": 1.8236, + "theoretical_loss": 4.696670991979321, + "tokens_seen": 117440512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007793139418254765, + "loss": 1.8411, + "theoretical_loss": 4.696318100932773, + "tokens_seen": 117506048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792978936810432, + "loss": 1.7059, + "theoretical_loss": 4.695965461720796, + "tokens_seen": 117571584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792818455366099, + "loss": 1.822, + "theoretical_loss": 4.695613074023446, + "tokens_seen": 117637120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792657973921766, + "loss": 1.894, + "theoretical_loss": 4.695260937521364, + "tokens_seen": 117702656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792497492477433, + "loss": 1.7513, + "theoretical_loss": 4.694909051895775, + "tokens_seen": 117768192 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077923370110331, + "loss": 1.8627, + "theoretical_loss": 4.694557416828484, + "tokens_seen": 117833728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792176529588767, + "loss": 1.7953, + "theoretical_loss": 4.694206032001879, + "tokens_seen": 117899264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 46257, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.9019111394882202, + "objective/train/theoretical_loss": 4.693854897098923, + "objective/train/tokens_used": 138424800, + "theoretical_loss": 4.693854897098923, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792016048144434, + "loss": 1.9417, + "theoretical_loss": 4.693854897098923, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791855566700101, + "loss": 1.8928, + "theoretical_loss": 4.693504011803159, + "tokens_seen": 118030336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791695085255768, + "loss": 1.8751, + "theoretical_loss": 4.6931533757987065, + "tokens_seen": 118095872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791534603811435, + "loss": 1.9013, + "theoretical_loss": 4.692802988770258, + "tokens_seen": 118161408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791374122367102, + "loss": 1.8634, + "theoretical_loss": 4.692452850403081, + "tokens_seen": 118226944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791213640922769, + "loss": 2.0152, + "theoretical_loss": 4.692102960383014, + "tokens_seen": 118292480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791053159478436, + "loss": 1.7348, + "theoretical_loss": 4.691753318396467, + "tokens_seen": 118358016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790892678034103, + "loss": 1.8863, + "theoretical_loss": 4.691403924130418, + "tokens_seen": 118423552 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779073219658977, + "loss": 1.9038, + "theoretical_loss": 4.691054777272415, + "tokens_seen": 118489088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790571715145437, + "loss": 1.8282, + "theoretical_loss": 4.69070587751057, + "tokens_seen": 118554624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790411233701104, + "loss": 1.9085, + "theoretical_loss": 4.690357224533564, + "tokens_seen": 118620160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790250752256771, + "loss": 1.8251, + "theoretical_loss": 4.690008818030638, + "tokens_seen": 118685696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790090270812438, + "loss": 1.6978, + "theoretical_loss": 4.689660657691598, + "tokens_seen": 118751232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789929789368105, + "loss": 1.9915, + "theoretical_loss": 4.689312743206811, + "tokens_seen": 118816768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789769307923772, + "loss": 1.9805, + "theoretical_loss": 4.688965074267203, + "tokens_seen": 118882304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789608826479439, + "loss": 1.8472, + "theoretical_loss": 4.688617650564259, + "tokens_seen": 118947840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789448345035106, + "loss": 1.9364, + "theoretical_loss": 4.688270471790023, + "tokens_seen": 119013376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789287863590773, + "loss": 1.8973, + "theoretical_loss": 4.687923537637094, + "tokens_seen": 119078912 + }, + { + "epoch": 0.04, + "learning_rate": 0.000778912738214644, + "loss": 1.8099, + "theoretical_loss": 4.687576847798622, + "tokens_seen": 119144448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788966900702107, + "loss": 1.8415, + "theoretical_loss": 4.687230401968319, + "tokens_seen": 119209984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788806419257774, + "loss": 1.9467, + "theoretical_loss": 4.68688419984044, + "tokens_seen": 119275520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788645937813441, + "loss": 1.8803, + "theoretical_loss": 4.686538241109796, + "tokens_seen": 119341056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788485456369108, + "loss": 1.8097, + "theoretical_loss": 4.686192525471746, + "tokens_seen": 119406592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788324974924775, + "loss": 1.8324, + "theoretical_loss": 4.685847052622199, + "tokens_seen": 119472128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788164493480442, + "loss": 1.8055, + "theoretical_loss": 4.685501822257608, + "tokens_seen": 119537664 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788004012036109, + "loss": 1.7789, + "theoretical_loss": 4.685156834074972, + "tokens_seen": 119603200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787843530591776, + "loss": 1.763, + "theoretical_loss": 4.684812087771839, + "tokens_seen": 119668736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787683049147443, + "loss": 1.8904, + "theoretical_loss": 4.684467583046292, + "tokens_seen": 119734272 + }, + { + "epoch": 0.04, + "learning_rate": 0.000778752256770311, + "loss": 1.84, + "theoretical_loss": 4.6841233195969645, + "tokens_seen": 119799808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787362086258777, + "loss": 1.9018, + "theoretical_loss": 4.683779297123024, + "tokens_seen": 119865344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787201604814444, + "loss": 1.804, + "theoretical_loss": 4.683435515324181, + "tokens_seen": 119930880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787041123370111, + "loss": 1.9663, + "theoretical_loss": 4.683091973900682, + "tokens_seen": 119996416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786880641925778, + "loss": 1.849, + "theoretical_loss": 4.682748672553313, + "tokens_seen": 120061952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786720160481445, + "loss": 1.8176, + "theoretical_loss": 4.68240561098339, + "tokens_seen": 120127488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786559679037112, + "loss": 1.7518, + "theoretical_loss": 4.68206278889277, + "tokens_seen": 120193024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786399197592779, + "loss": 1.8349, + "theoretical_loss": 4.681720205983839, + "tokens_seen": 120258560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786238716148446, + "loss": 1.8489, + "theoretical_loss": 4.681377861959516, + "tokens_seen": 120324096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786078234704113, + "loss": 1.8931, + "theoretical_loss": 4.681035756523249, + "tokens_seen": 120389632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785917753259779, + "loss": 1.9067, + "theoretical_loss": 4.680693889379018, + "tokens_seen": 120455168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785757271815447, + "loss": 1.855, + "theoretical_loss": 4.680352260231329, + "tokens_seen": 120520704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785596790371113, + "loss": 1.8698, + "theoretical_loss": 4.680010868785216, + "tokens_seen": 120586240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785436308926781, + "loss": 1.83, + "theoretical_loss": 4.679669714746238, + "tokens_seen": 120651776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785275827482447, + "loss": 1.9867, + "theoretical_loss": 4.679328797820478, + "tokens_seen": 120717312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785115346038115, + "loss": 1.944, + "theoretical_loss": 4.678988117714544, + "tokens_seen": 120782848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784954864593781, + "loss": 1.7626, + "theoretical_loss": 4.678647674135563, + "tokens_seen": 120848384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784794383149449, + "loss": 1.8657, + "theoretical_loss": 4.6783074667911855, + "tokens_seen": 120913920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784633901705115, + "loss": 1.8642, + "theoretical_loss": 4.677967495389581, + "tokens_seen": 120979456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784473420260782, + "loss": 1.7655, + "theoretical_loss": 4.677627759639435, + "tokens_seen": 121044992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784312938816449, + "loss": 1.8806, + "theoretical_loss": 4.677288259249954, + "tokens_seen": 121110528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784152457372116, + "loss": 1.8441, + "theoretical_loss": 4.676948993930857, + "tokens_seen": 121176064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 47440, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.9147517681121826, + "objective/train/theoretical_loss": 4.67660996339238, + "objective/train/tokens_used": 141701600, + "theoretical_loss": 4.67660996339238, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783991975927783, + "loss": 1.9058, + "theoretical_loss": 4.67660996339238, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.000778383149448345, + "loss": 1.9118, + "theoretical_loss": 4.676271167345272, + "tokens_seen": 121307136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783671013039117, + "loss": 1.9903, + "theoretical_loss": 4.675932605500794, + "tokens_seen": 121372672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783510531594785, + "loss": 1.846, + "theoretical_loss": 4.675594277570718, + "tokens_seen": 121438208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783350050150451, + "loss": 1.7874, + "theoretical_loss": 4.675256183267327, + "tokens_seen": 121503744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783189568706119, + "loss": 1.8911, + "theoretical_loss": 4.674918322303411, + "tokens_seen": 121569280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783029087261785, + "loss": 1.7944, + "theoretical_loss": 4.6745806943922705, + "tokens_seen": 121634816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782868605817453, + "loss": 1.7367, + "theoretical_loss": 4.67424329924771, + "tokens_seen": 121700352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782708124373119, + "loss": 1.8909, + "theoretical_loss": 4.673906136584041, + "tokens_seen": 121765888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782547642928787, + "loss": 1.9456, + "theoretical_loss": 4.673569206116078, + "tokens_seen": 121831424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782387161484453, + "loss": 1.8463, + "theoretical_loss": 4.6732325075591366, + "tokens_seen": 121896960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782226680040121, + "loss": 1.9333, + "theoretical_loss": 4.67289604062904, + "tokens_seen": 121962496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782066198595787, + "loss": 1.7753, + "theoretical_loss": 4.672559805042107, + "tokens_seen": 122028032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781905717151455, + "loss": 1.8157, + "theoretical_loss": 4.672223800515159, + "tokens_seen": 122093568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781745235707121, + "loss": 1.8541, + "theoretical_loss": 4.671888026765512, + "tokens_seen": 122159104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781584754262789, + "loss": 1.7951, + "theoretical_loss": 4.6715524835109825, + "tokens_seen": 122224640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781424272818455, + "loss": 1.8092, + "theoretical_loss": 4.671217170469884, + "tokens_seen": 122290176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781263791374123, + "loss": 1.7207, + "theoretical_loss": 4.67088208736102, + "tokens_seen": 122355712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781103309929789, + "loss": 1.9078, + "theoretical_loss": 4.670547233903694, + "tokens_seen": 122421248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780942828485457, + "loss": 1.8607, + "theoretical_loss": 4.670212609817698, + "tokens_seen": 122486784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780782347041123, + "loss": 1.8305, + "theoretical_loss": 4.669878214823316, + "tokens_seen": 122552320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780621865596791, + "loss": 1.9033, + "theoretical_loss": 4.669544048641325, + "tokens_seen": 122617856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780461384152457, + "loss": 1.9115, + "theoretical_loss": 4.669210110992989, + "tokens_seen": 122683392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780300902708125, + "loss": 1.9634, + "theoretical_loss": 4.668876401600059, + "tokens_seen": 122748928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780140421263791, + "loss": 1.7661, + "theoretical_loss": 4.668542920184779, + "tokens_seen": 122814464 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779979939819459, + "loss": 1.8726, + "theoretical_loss": 4.6682096664698705, + "tokens_seen": 122880000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779819458375126, + "loss": 1.9475, + "theoretical_loss": 4.667876640178546, + "tokens_seen": 122945536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779658976930793, + "loss": 1.9562, + "theoretical_loss": 4.667543841034502, + "tokens_seen": 123011072 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777949849548646, + "loss": 1.9686, + "theoretical_loss": 4.667211268761914, + "tokens_seen": 123076608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779338014042127, + "loss": 1.8776, + "theoretical_loss": 4.66687892308544, + "tokens_seen": 123142144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779177532597794, + "loss": 1.9679, + "theoretical_loss": 4.6665468037302205, + "tokens_seen": 123207680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779017051153461, + "loss": 1.804, + "theoretical_loss": 4.666214910421875, + "tokens_seen": 123273216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778856569709128, + "loss": 1.8628, + "theoretical_loss": 4.665883242886498, + "tokens_seen": 123338752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778696088264795, + "loss": 1.8482, + "theoretical_loss": 4.6655518008506665, + "tokens_seen": 123404288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778535606820462, + "loss": 1.8696, + "theoretical_loss": 4.665220584041428, + "tokens_seen": 123469824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778375125376129, + "loss": 1.9316, + "theoretical_loss": 4.664889592186309, + "tokens_seen": 123535360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778214643931796, + "loss": 1.9424, + "theoretical_loss": 4.6645588250133105, + "tokens_seen": 123600896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778054162487463, + "loss": 1.9075, + "theoretical_loss": 4.664228282250902, + "tokens_seen": 123666432 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777789368104313, + "loss": 1.8398, + "theoretical_loss": 4.663897963628029, + "tokens_seen": 123731968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777733199598797, + "loss": 1.8985, + "theoretical_loss": 4.6635678688741065, + "tokens_seen": 123797504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777572718154464, + "loss": 1.9671, + "theoretical_loss": 4.663237997719017, + "tokens_seen": 123863040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777412236710131, + "loss": 1.8071, + "theoretical_loss": 4.662908349893116, + "tokens_seen": 123928576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777251755265798, + "loss": 1.9107, + "theoretical_loss": 4.662578925127223, + "tokens_seen": 123994112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777091273821465, + "loss": 1.7927, + "theoretical_loss": 4.662249723152624, + "tokens_seen": 124059648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776930792377132, + "loss": 1.9306, + "theoretical_loss": 4.661920743701073, + "tokens_seen": 124125184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776770310932799, + "loss": 2.021, + "theoretical_loss": 4.661591986504786, + "tokens_seen": 124190720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776609829488466, + "loss": 1.9438, + "theoretical_loss": 4.661263451296444, + "tokens_seen": 124256256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776449348044133, + "loss": 1.8106, + "theoretical_loss": 4.660935137809188, + "tokens_seen": 124321792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077762888665998, + "loss": 1.8744, + "theoretical_loss": 4.660607045776624, + "tokens_seen": 124387328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776128385155467, + "loss": 1.7867, + "theoretical_loss": 4.660279174932814, + "tokens_seen": 124452864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 48338, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7055823802947998, + "objective/train/theoretical_loss": 4.659951525012284, + "objective/train/tokens_used": 144978400, + "theoretical_loss": 4.659951525012284, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775967903711134, + "loss": 1.8029, + "theoretical_loss": 4.659951525012284, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775807422266801, + "loss": 1.8085, + "theoretical_loss": 4.6596240957500115, + "tokens_seen": 124583936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775646940822468, + "loss": 1.8204, + "theoretical_loss": 4.659296886881438, + "tokens_seen": 124649472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775486459378135, + "loss": 1.8372, + "theoretical_loss": 4.658969898142456, + "tokens_seen": 124715008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775325977933802, + "loss": 1.7415, + "theoretical_loss": 4.658643129269416, + "tokens_seen": 124780544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775165496489469, + "loss": 1.7994, + "theoretical_loss": 4.658316579999122, + "tokens_seen": 124846080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775005015045136, + "loss": 1.8632, + "theoretical_loss": 4.65799025006883, + "tokens_seen": 124911616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774844533600803, + "loss": 1.8766, + "theoretical_loss": 4.65766413921625, + "tokens_seen": 124977152 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777468405215647, + "loss": 1.8277, + "theoretical_loss": 4.657338247179538, + "tokens_seen": 125042688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774523570712137, + "loss": 1.8551, + "theoretical_loss": 4.657012573697307, + "tokens_seen": 125108224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774363089267804, + "loss": 1.8749, + "theoretical_loss": 4.656687118508616, + "tokens_seen": 125173760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774202607823471, + "loss": 1.7936, + "theoretical_loss": 4.656361881352968, + "tokens_seen": 125239296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774042126379138, + "loss": 1.7392, + "theoretical_loss": 4.656036861970319, + "tokens_seen": 125304832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773881644934805, + "loss": 1.8527, + "theoretical_loss": 4.655712060101068, + "tokens_seen": 125370368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773721163490472, + "loss": 1.854, + "theoretical_loss": 4.65538747548606, + "tokens_seen": 125435904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773560682046139, + "loss": 1.8442, + "theoretical_loss": 4.655063107866582, + "tokens_seen": 125501440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773400200601806, + "loss": 2.0244, + "theoretical_loss": 4.654738956984366, + "tokens_seen": 125566976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773239719157473, + "loss": 1.7517, + "theoretical_loss": 4.654415022581585, + "tokens_seen": 125632512 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777307923771314, + "loss": 1.8149, + "theoretical_loss": 4.654091304400853, + "tokens_seen": 125698048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772918756268807, + "loss": 1.9215, + "theoretical_loss": 4.653767802185225, + "tokens_seen": 125763584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772758274824474, + "loss": 1.8353, + "theoretical_loss": 4.653444515678195, + "tokens_seen": 125829120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772597793380141, + "loss": 1.8363, + "theoretical_loss": 4.653121444623693, + "tokens_seen": 125894656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772437311935808, + "loss": 1.7447, + "theoretical_loss": 4.652798588766088, + "tokens_seen": 125960192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772276830491475, + "loss": 1.7671, + "theoretical_loss": 4.652475947850185, + "tokens_seen": 126025728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772116349047142, + "loss": 1.753, + "theoretical_loss": 4.652153521621223, + "tokens_seen": 126091264 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771955867602808, + "loss": 1.9541, + "theoretical_loss": 4.651831309824875, + "tokens_seen": 126156800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771795386158476, + "loss": 1.7825, + "theoretical_loss": 4.65150931220725, + "tokens_seen": 126222336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771634904714142, + "loss": 1.9339, + "theoretical_loss": 4.651187528514887, + "tokens_seen": 126287872 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777147442326981, + "loss": 1.8252, + "theoretical_loss": 4.650865958494755, + "tokens_seen": 126353408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771313941825476, + "loss": 1.8909, + "theoretical_loss": 4.650544601894257, + "tokens_seen": 126418944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771153460381144, + "loss": 1.9024, + "theoretical_loss": 4.650223458461221, + "tokens_seen": 126484480 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777099297893681, + "loss": 1.862, + "theoretical_loss": 4.649902527943908, + "tokens_seen": 126550016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770832497492478, + "loss": 1.8003, + "theoretical_loss": 4.649581810091003, + "tokens_seen": 126615552 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770672016048144, + "loss": 2.0104, + "theoretical_loss": 4.649261304651619, + "tokens_seen": 126681088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770511534603812, + "loss": 1.7893, + "theoretical_loss": 4.6489410113752925, + "tokens_seen": 126746624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770351053159478, + "loss": 1.814, + "theoretical_loss": 4.6486209300119885, + "tokens_seen": 126812160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770190571715146, + "loss": 1.9046, + "theoretical_loss": 4.648301060312093, + "tokens_seen": 126877696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770030090270812, + "loss": 1.8817, + "theoretical_loss": 4.647981402026417, + "tokens_seen": 126943232 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776986960882648, + "loss": 1.8367, + "theoretical_loss": 4.647661954906188, + "tokens_seen": 127008768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007769709127382146, + "loss": 1.7926, + "theoretical_loss": 4.647342718703062, + "tokens_seen": 127074304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007769548645937814, + "loss": 1.9298, + "theoretical_loss": 4.647023693169107, + "tokens_seen": 127139840 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776938816449348, + "loss": 1.7827, + "theoretical_loss": 4.646704878056816, + "tokens_seen": 127205376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007769227683049148, + "loss": 1.7387, + "theoretical_loss": 4.6463862731191, + "tokens_seen": 127270912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007769067201604814, + "loss": 1.91, + "theoretical_loss": 4.6460678781092835, + "tokens_seen": 127336448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768906720160482, + "loss": 1.993, + "theoretical_loss": 4.6457496927811075, + "tokens_seen": 127401984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768746238716148, + "loss": 1.6914, + "theoretical_loss": 4.645431716888734, + "tokens_seen": 127467520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768585757271816, + "loss": 1.8668, + "theoretical_loss": 4.645113950186732, + "tokens_seen": 127533056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768425275827482, + "loss": 1.7944, + "theoretical_loss": 4.64479639243009, + "tokens_seen": 127598592 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776826479438315, + "loss": 1.7889, + "theoretical_loss": 4.644479043374204, + "tokens_seen": 127664128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768104312938816, + "loss": 1.8859, + "theoretical_loss": 4.644161902774886, + "tokens_seen": 127729664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 49581, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.94822096824646, + "objective/train/theoretical_loss": 4.643844970388358, + "objective/train/tokens_used": 148255200, + "theoretical_loss": 4.643844970388358, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767943831494484, + "loss": 1.9133, + "theoretical_loss": 4.643844970388358, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776778335005015, + "loss": 1.8609, + "theoretical_loss": 4.6435282459712495, + "tokens_seen": 127860736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767622868605818, + "loss": 1.8257, + "theoretical_loss": 4.643211729280601, + "tokens_seen": 127926272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767462387161484, + "loss": 1.7747, + "theoretical_loss": 4.64289542007386, + "tokens_seen": 127991808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767301905717152, + "loss": 1.8617, + "theoretical_loss": 4.642579318108883, + "tokens_seen": 128057344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767141424272818, + "loss": 1.8137, + "theoretical_loss": 4.642263423143932, + "tokens_seen": 128122880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766980942828486, + "loss": 1.8099, + "theoretical_loss": 4.641947734937673, + "tokens_seen": 128188416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766820461384152, + "loss": 1.8716, + "theoretical_loss": 4.641632253249179, + "tokens_seen": 128253952 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776665997993982, + "loss": 2.0769, + "theoretical_loss": 4.641316977837924, + "tokens_seen": 128319488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766499498495486, + "loss": 1.89, + "theoretical_loss": 4.641001908463787, + "tokens_seen": 128385024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766339017051154, + "loss": 1.8734, + "theoretical_loss": 4.6406870448870485, + "tokens_seen": 128450560 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776617853560682, + "loss": 1.8353, + "theoretical_loss": 4.64037238686839, + "tokens_seen": 128516096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766018054162488, + "loss": 1.8572, + "theoretical_loss": 4.640057934168891, + "tokens_seen": 128581632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765857572718155, + "loss": 1.7848, + "theoretical_loss": 4.639743686550034, + "tokens_seen": 128647168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765697091273822, + "loss": 1.8396, + "theoretical_loss": 4.639429643773697, + "tokens_seen": 128712704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765536609829489, + "loss": 1.8095, + "theoretical_loss": 4.639115805602156, + "tokens_seen": 128778240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765376128385156, + "loss": 1.8191, + "theoretical_loss": 4.638802171798086, + "tokens_seen": 128843776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765215646940823, + "loss": 1.9008, + "theoretical_loss": 4.638488742124554, + "tokens_seen": 128909312 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776505516549649, + "loss": 1.8786, + "theoretical_loss": 4.638175516345027, + "tokens_seen": 128974848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764894684052157, + "loss": 1.8323, + "theoretical_loss": 4.637862494223359, + "tokens_seen": 129040384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764734202607824, + "loss": 1.7773, + "theoretical_loss": 4.637549675523806, + "tokens_seen": 129105920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764573721163491, + "loss": 1.8804, + "theoretical_loss": 4.637237060011007, + "tokens_seen": 129171456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764413239719158, + "loss": 1.847, + "theoretical_loss": 4.636924647450002, + "tokens_seen": 129236992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764252758274825, + "loss": 1.8726, + "theoretical_loss": 4.636612437606212, + "tokens_seen": 129302528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764092276830492, + "loss": 1.8728, + "theoretical_loss": 4.636300430245457, + "tokens_seen": 129368064 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763931795386159, + "loss": 1.7916, + "theoretical_loss": 4.63598862513394, + "tokens_seen": 129433600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763771313941826, + "loss": 1.8813, + "theoretical_loss": 4.635677022038254, + "tokens_seen": 129499136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763610832497493, + "loss": 1.8075, + "theoretical_loss": 4.63536562072538, + "tokens_seen": 129564672 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776345035105316, + "loss": 1.8598, + "theoretical_loss": 4.635054420962684, + "tokens_seen": 129630208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763289869608827, + "loss": 1.8148, + "theoretical_loss": 4.634743422517918, + "tokens_seen": 129695744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763129388164494, + "loss": 1.7992, + "theoretical_loss": 4.634432625159221, + "tokens_seen": 129761280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762968906720161, + "loss": 1.8888, + "theoretical_loss": 4.634122028655112, + "tokens_seen": 129826816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762808425275828, + "loss": 1.7781, + "theoretical_loss": 4.633811632774496, + "tokens_seen": 129892352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762647943831495, + "loss": 1.8929, + "theoretical_loss": 4.633501437286659, + "tokens_seen": 129957888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762487462387162, + "loss": 1.9065, + "theoretical_loss": 4.633191441961272, + "tokens_seen": 130023424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762326980942829, + "loss": 1.7713, + "theoretical_loss": 4.63288164656838, + "tokens_seen": 130088960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762166499498496, + "loss": 1.7753, + "theoretical_loss": 4.632572050878415, + "tokens_seen": 130154496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762006018054163, + "loss": 1.8997, + "theoretical_loss": 4.632262654662181, + "tokens_seen": 130220032 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776184553660983, + "loss": 1.8561, + "theoretical_loss": 4.631953457690868, + "tokens_seen": 130285568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761685055165497, + "loss": 1.9013, + "theoretical_loss": 4.631644459736036, + "tokens_seen": 130351104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761524573721164, + "loss": 1.9393, + "theoretical_loss": 4.631335660569626, + "tokens_seen": 130416640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761364092276831, + "loss": 1.9441, + "theoretical_loss": 4.631027059963953, + "tokens_seen": 130482176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761203610832498, + "loss": 1.8176, + "theoretical_loss": 4.630718657691709, + "tokens_seen": 130547712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761043129388165, + "loss": 1.7216, + "theoretical_loss": 4.630410453525958, + "tokens_seen": 130613248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760882647943832, + "loss": 1.766, + "theoretical_loss": 4.630102447240138, + "tokens_seen": 130678784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760722166499499, + "loss": 1.8036, + "theoretical_loss": 4.629794638608059, + "tokens_seen": 130744320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760561685055166, + "loss": 1.851, + "theoretical_loss": 4.629487027403905, + "tokens_seen": 130809856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760401203610833, + "loss": 1.8026, + "theoretical_loss": 4.629179613402228, + "tokens_seen": 130875392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077602407221665, + "loss": 1.8168, + "theoretical_loss": 4.628872396377953, + "tokens_seen": 130940928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760080240722167, + "loss": 1.872, + "theoretical_loss": 4.628565376106371, + "tokens_seen": 131006464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 50603, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7769169807434082, + "objective/train/theoretical_loss": 4.628258552363146, + "objective/train/tokens_used": 151532000, + "theoretical_loss": 4.628258552363146, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759919759277834, + "loss": 1.8162, + "theoretical_loss": 4.628258552363146, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759759277833502, + "loss": 1.8492, + "theoretical_loss": 4.627951924924305, + "tokens_seen": 131137536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759598796389168, + "loss": 1.8724, + "theoretical_loss": 4.627645493566245, + "tokens_seen": 131203072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759438314944836, + "loss": 1.8344, + "theoretical_loss": 4.62733925806573, + "tokens_seen": 131268608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759277833500502, + "loss": 1.8899, + "theoretical_loss": 4.627033218199886, + "tokens_seen": 131334144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759117352056169, + "loss": 1.8562, + "theoretical_loss": 4.626727373746206, + "tokens_seen": 131399680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758956870611836, + "loss": 1.8066, + "theoretical_loss": 4.626421724482549, + "tokens_seen": 131465216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758796389167503, + "loss": 1.8461, + "theoretical_loss": 4.626116270187131, + "tokens_seen": 131530752 + }, + { + "epoch": 0.04, + "learning_rate": 0.000775863590772317, + "loss": 1.736, + "theoretical_loss": 4.625811010638538, + "tokens_seen": 131596288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758475426278837, + "loss": 1.8001, + "theoretical_loss": 4.62550594561571, + "tokens_seen": 131661824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758314944834504, + "loss": 1.823, + "theoretical_loss": 4.625201074897952, + "tokens_seen": 131727360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758154463390171, + "loss": 1.9525, + "theoretical_loss": 4.62489639826493, + "tokens_seen": 131792896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757993981945837, + "loss": 1.8137, + "theoretical_loss": 4.6245919154966675, + "tokens_seen": 131858432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757833500501505, + "loss": 1.8273, + "theoretical_loss": 4.624287626373545, + "tokens_seen": 131923968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757673019057171, + "loss": 1.7471, + "theoretical_loss": 4.623983530676304, + "tokens_seen": 131989504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757512537612839, + "loss": 1.8519, + "theoretical_loss": 4.62367962818604, + "tokens_seen": 132055040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757352056168505, + "loss": 1.944, + "theoretical_loss": 4.623375918684207, + "tokens_seen": 132120576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757191574724173, + "loss": 1.8725, + "theoretical_loss": 4.623072401952614, + "tokens_seen": 132186112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757031093279839, + "loss": 1.8953, + "theoretical_loss": 4.622769077773423, + "tokens_seen": 132251648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756870611835507, + "loss": 1.879, + "theoretical_loss": 4.622465945929152, + "tokens_seen": 132317184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756710130391173, + "loss": 1.7999, + "theoretical_loss": 4.622163006202671, + "tokens_seen": 132382720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756549648946841, + "loss": 1.7466, + "theoretical_loss": 4.621860258377203, + "tokens_seen": 132448256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756389167502507, + "loss": 1.8657, + "theoretical_loss": 4.621557702236323, + "tokens_seen": 132513792 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756228686058175, + "loss": 1.806, + "theoretical_loss": 4.621255337563955, + "tokens_seen": 132579328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756068204613841, + "loss": 1.8109, + "theoretical_loss": 4.620953164144376, + "tokens_seen": 132644864 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755907723169509, + "loss": 1.8321, + "theoretical_loss": 4.620651181762212, + "tokens_seen": 132710400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755747241725175, + "loss": 1.8282, + "theoretical_loss": 4.620349390202436, + "tokens_seen": 132775936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755586760280843, + "loss": 1.8138, + "theoretical_loss": 4.6200477892503695, + "tokens_seen": 132841472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755426278836509, + "loss": 1.8533, + "theoretical_loss": 4.619746378691682, + "tokens_seen": 132907008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755265797392177, + "loss": 1.7597, + "theoretical_loss": 4.61944515831239, + "tokens_seen": 132972544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755105315947843, + "loss": 1.8646, + "theoretical_loss": 4.6191441278988545, + "tokens_seen": 133038080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754944834503511, + "loss": 1.7799, + "theoretical_loss": 4.618843287237782, + "tokens_seen": 133103616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754784353059177, + "loss": 1.8689, + "theoretical_loss": 4.618542636116224, + "tokens_seen": 133169152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754623871614845, + "loss": 1.833, + "theoretical_loss": 4.618242174321575, + "tokens_seen": 133234688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754463390170511, + "loss": 1.8088, + "theoretical_loss": 4.617941901641573, + "tokens_seen": 133300224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754302908726179, + "loss": 1.8624, + "theoretical_loss": 4.617641817864296, + "tokens_seen": 133365760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754142427281845, + "loss": 1.9235, + "theoretical_loss": 4.617341922778168, + "tokens_seen": 133431296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753981945837513, + "loss": 1.701, + "theoretical_loss": 4.617042216171949, + "tokens_seen": 133496832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753821464393179, + "loss": 1.7201, + "theoretical_loss": 4.616742697834742, + "tokens_seen": 133562368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753660982948847, + "loss": 1.8286, + "theoretical_loss": 4.616443367555988, + "tokens_seen": 133627904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753500501504513, + "loss": 1.8104, + "theoretical_loss": 4.616144225125465, + "tokens_seen": 133693440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753340020060181, + "loss": 1.8212, + "theoretical_loss": 4.615845270333294, + "tokens_seen": 133758976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753179538615847, + "loss": 1.7838, + "theoretical_loss": 4.615546502969929, + "tokens_seen": 133824512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753019057171515, + "loss": 1.8179, + "theoretical_loss": 4.61524792282616, + "tokens_seen": 133890048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752858575727181, + "loss": 1.8488, + "theoretical_loss": 4.614949529693115, + "tokens_seen": 133955584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752698094282849, + "loss": 1.7209, + "theoretical_loss": 4.614651323362257, + "tokens_seen": 134021120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752537612838515, + "loss": 1.6942, + "theoretical_loss": 4.614353303625382, + "tokens_seen": 134086656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752377131394183, + "loss": 1.819, + "theoretical_loss": 4.6140554702746215, + "tokens_seen": 134152192 + }, + { + "epoch": 0.04, + "learning_rate": 0.000775221664994985, + "loss": 1.7107, + "theoretical_loss": 4.613757823102437, + "tokens_seen": 134217728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752056168505517, + "loss": 1.782, + "theoretical_loss": 4.613460361901627, + "tokens_seen": 134283264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 51744, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.0655648708343506, + "objective/train/theoretical_loss": 4.613163086465315, + "objective/train/tokens_used": 154808800, + "theoretical_loss": 4.613163086465315, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751895687061184, + "loss": 1.9222, + "theoretical_loss": 4.613163086465315, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751735205616851, + "loss": 1.7955, + "theoretical_loss": 4.612865996586962, + "tokens_seen": 134414336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751574724172518, + "loss": 1.7857, + "theoretical_loss": 4.612569092060355, + "tokens_seen": 134479872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751414242728185, + "loss": 1.8141, + "theoretical_loss": 4.612272372679611, + "tokens_seen": 134545408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751253761283852, + "loss": 1.9289, + "theoretical_loss": 4.611975838239179, + "tokens_seen": 134610944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751093279839519, + "loss": 1.6775, + "theoretical_loss": 4.611679488533832, + "tokens_seen": 134676480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750932798395186, + "loss": 1.8499, + "theoretical_loss": 4.611383323358671, + "tokens_seen": 134742016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750772316950853, + "loss": 1.8098, + "theoretical_loss": 4.61108734250913, + "tokens_seen": 134807552 + }, + { + "epoch": 0.04, + "learning_rate": 0.000775061183550652, + "loss": 1.8215, + "theoretical_loss": 4.610791545780959, + "tokens_seen": 134873088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750451354062187, + "loss": 1.6756, + "theoretical_loss": 4.610495932970241, + "tokens_seen": 134938624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750290872617854, + "loss": 1.7515, + "theoretical_loss": 4.61020050387338, + "tokens_seen": 135004160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750130391173521, + "loss": 1.7896, + "theoretical_loss": 4.609905258287107, + "tokens_seen": 135069696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749969909729188, + "loss": 1.8716, + "theoretical_loss": 4.609610196008473, + "tokens_seen": 135135232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749809428284855, + "loss": 1.7909, + "theoretical_loss": 4.609315316834855, + "tokens_seen": 135200768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749648946840522, + "loss": 1.7184, + "theoretical_loss": 4.60902062056395, + "tokens_seen": 135266304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749488465396189, + "loss": 1.8287, + "theoretical_loss": 4.608726106993776, + "tokens_seen": 135331840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749327983951856, + "loss": 1.9142, + "theoretical_loss": 4.608431775922673, + "tokens_seen": 135397376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749167502507523, + "loss": 1.7733, + "theoretical_loss": 4.608137627149302, + "tokens_seen": 135462912 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774900702106319, + "loss": 1.8085, + "theoretical_loss": 4.607843660472642, + "tokens_seen": 135528448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748846539618857, + "loss": 1.8051, + "theoretical_loss": 4.607549875691989, + "tokens_seen": 135593984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748686058174524, + "loss": 1.8313, + "theoretical_loss": 4.607256272606961, + "tokens_seen": 135659520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748525576730191, + "loss": 1.8189, + "theoretical_loss": 4.606962851017489, + "tokens_seen": 135725056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748365095285858, + "loss": 1.905, + "theoretical_loss": 4.606669610723825, + "tokens_seen": 135790592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748204613841525, + "loss": 1.8924, + "theoretical_loss": 4.606376551526536, + "tokens_seen": 135856128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748044132397192, + "loss": 1.7682, + "theoretical_loss": 4.6060836732265, + "tokens_seen": 135921664 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747883650952859, + "loss": 1.7632, + "theoretical_loss": 4.605790975624917, + "tokens_seen": 135987200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747723169508527, + "loss": 1.8681, + "theoretical_loss": 4.605498458523296, + "tokens_seen": 136052736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747562688064193, + "loss": 1.7845, + "theoretical_loss": 4.605206121723461, + "tokens_seen": 136118272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747402206619861, + "loss": 1.7568, + "theoretical_loss": 4.604913965027551, + "tokens_seen": 136183808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747241725175527, + "loss": 1.8752, + "theoretical_loss": 4.604621988238012, + "tokens_seen": 136249344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747081243731195, + "loss": 1.9572, + "theoretical_loss": 4.604330191157608, + "tokens_seen": 136314880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746920762286861, + "loss": 1.7565, + "theoretical_loss": 4.604038573589408, + "tokens_seen": 136380416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746760280842529, + "loss": 1.783, + "theoretical_loss": 4.603747135336796, + "tokens_seen": 136445952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746599799398195, + "loss": 1.8388, + "theoretical_loss": 4.603455876203463, + "tokens_seen": 136511488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746439317953863, + "loss": 1.7053, + "theoretical_loss": 4.603164795993409, + "tokens_seen": 136577024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746278836509529, + "loss": 1.7839, + "theoretical_loss": 4.602873894510945, + "tokens_seen": 136642560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746118355065197, + "loss": 1.7043, + "theoretical_loss": 4.602583171560686, + "tokens_seen": 136708096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745957873620863, + "loss": 1.7794, + "theoretical_loss": 4.602292626947556, + "tokens_seen": 136773632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745797392176531, + "loss": 1.8616, + "theoretical_loss": 4.602002260476787, + "tokens_seen": 136839168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745636910732197, + "loss": 1.783, + "theoretical_loss": 4.601712071953914, + "tokens_seen": 136904704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745476429287865, + "loss": 1.7862, + "theoretical_loss": 4.601422061184778, + "tokens_seen": 136970240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745315947843531, + "loss": 1.6995, + "theoretical_loss": 4.601132227975527, + "tokens_seen": 137035776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745155466399198, + "loss": 2.0007, + "theoretical_loss": 4.60084257213261, + "tokens_seen": 137101312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744994984954865, + "loss": 1.7742, + "theoretical_loss": 4.600553093462781, + "tokens_seen": 137166848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744834503510532, + "loss": 1.9237, + "theoretical_loss": 4.600263791773097, + "tokens_seen": 137232384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744674022066199, + "loss": 1.8399, + "theoretical_loss": 4.599974666870914, + "tokens_seen": 137297920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744513540621866, + "loss": 1.7832, + "theoretical_loss": 4.599685718563894, + "tokens_seen": 137363456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744353059177533, + "loss": 1.801, + "theoretical_loss": 4.599396946659997, + "tokens_seen": 137428992 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077441925777332, + "loss": 1.7621, + "theoretical_loss": 4.5991083509674855, + "tokens_seen": 137494528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744032096288867, + "loss": 1.7486, + "theoretical_loss": 4.598819931294919, + "tokens_seen": 137560064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 52717, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7819393873214722, + "objective/train/theoretical_loss": 4.598531687451157, + "objective/train/tokens_used": 158085600, + "theoretical_loss": 4.598531687451157, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743871614844534, + "loss": 1.8069, + "theoretical_loss": 4.598531687451157, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077437111334002, + "loss": 1.8632, + "theoretical_loss": 4.598243619245359, + "tokens_seen": 137691136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743550651955868, + "loss": 1.8544, + "theoretical_loss": 4.59795572648698, + "tokens_seen": 137756672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743390170511534, + "loss": 1.8115, + "theoretical_loss": 4.597668008985774, + "tokens_seen": 137822208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743229689067202, + "loss": 1.7225, + "theoretical_loss": 4.59738046655179, + "tokens_seen": 137887744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743069207622868, + "loss": 1.7712, + "theoretical_loss": 4.597093098995377, + "tokens_seen": 137953280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007742908726178536, + "loss": 1.735, + "theoretical_loss": 4.596805906127173, + "tokens_seen": 138018816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007742748244734202, + "loss": 1.7213, + "theoretical_loss": 4.596518887758117, + "tokens_seen": 138084352 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774258776328987, + "loss": 1.867, + "theoretical_loss": 4.596232043699438, + "tokens_seen": 138149888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007742427281845536, + "loss": 1.842, + "theoretical_loss": 4.59594537376266, + "tokens_seen": 138215424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007742266800401204, + "loss": 1.8787, + "theoretical_loss": 4.595658877759602, + "tokens_seen": 138280960 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774210631895687, + "loss": 1.9223, + "theoretical_loss": 4.595372555502371, + "tokens_seen": 138346496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741945837512538, + "loss": 1.9055, + "theoretical_loss": 4.59508640680337, + "tokens_seen": 138412032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741785356068204, + "loss": 1.8217, + "theoretical_loss": 4.594800431475292, + "tokens_seen": 138477568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741624874623872, + "loss": 1.7421, + "theoretical_loss": 4.594514629331121, + "tokens_seen": 138543104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741464393179538, + "loss": 1.7933, + "theoretical_loss": 4.594229000184128, + "tokens_seen": 138608640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741303911735206, + "loss": 1.8861, + "theoretical_loss": 4.593943543847878, + "tokens_seen": 138674176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741143430290872, + "loss": 1.8354, + "theoretical_loss": 4.5936582601362215, + "tokens_seen": 138739712 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774098294884654, + "loss": 1.8875, + "theoretical_loss": 4.593373148863301, + "tokens_seen": 138805248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740822467402206, + "loss": 1.8701, + "theoretical_loss": 4.593088209843543, + "tokens_seen": 138870784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740661985957874, + "loss": 1.8651, + "theoretical_loss": 4.592803442891663, + "tokens_seen": 138936320 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774050150451354, + "loss": 1.8731, + "theoretical_loss": 4.592518847822661, + "tokens_seen": 139001856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740341023069208, + "loss": 1.7787, + "theoretical_loss": 4.592234424451828, + "tokens_seen": 139067392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740180541624874, + "loss": 1.8654, + "theoretical_loss": 4.591950172594734, + "tokens_seen": 139132928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740020060180542, + "loss": 1.728, + "theoretical_loss": 4.591666092067238, + "tokens_seen": 139198464 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739859578736208, + "loss": 1.8389, + "theoretical_loss": 4.591382182685484, + "tokens_seen": 139264000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739699097291876, + "loss": 1.7957, + "theoretical_loss": 4.591098444265896, + "tokens_seen": 139329536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739538615847542, + "loss": 1.7924, + "theoretical_loss": 4.590814876625183, + "tokens_seen": 139395072 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773937813440321, + "loss": 1.8632, + "theoretical_loss": 4.590531479580338, + "tokens_seen": 139460608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739217652958876, + "loss": 1.747, + "theoretical_loss": 4.5902482529486335, + "tokens_seen": 139526144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739057171514544, + "loss": 1.8658, + "theoretical_loss": 4.589965196547625, + "tokens_seen": 139591680 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773889669007021, + "loss": 1.904, + "theoretical_loss": 4.5896823101951485, + "tokens_seen": 139657216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738736208625878, + "loss": 1.8037, + "theoretical_loss": 4.58939959370932, + "tokens_seen": 139722752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738575727181545, + "loss": 1.7745, + "theoretical_loss": 4.5891170469085365, + "tokens_seen": 139788288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738415245737212, + "loss": 1.7752, + "theoretical_loss": 4.588834669611472, + "tokens_seen": 139853824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738254764292879, + "loss": 1.8438, + "theoretical_loss": 4.5885524616370805, + "tokens_seen": 139919360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738094282848546, + "loss": 1.8438, + "theoretical_loss": 4.5882704228045945, + "tokens_seen": 139984896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737933801404213, + "loss": 1.916, + "theoretical_loss": 4.587988552933523, + "tokens_seen": 140050432 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773777331995988, + "loss": 1.7058, + "theoretical_loss": 4.587706851843652, + "tokens_seen": 140115968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737612838515547, + "loss": 1.8099, + "theoretical_loss": 4.587425319355044, + "tokens_seen": 140181504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737452357071214, + "loss": 1.7592, + "theoretical_loss": 4.587143955288038, + "tokens_seen": 140247040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737291875626881, + "loss": 1.8013, + "theoretical_loss": 4.586862759463248, + "tokens_seen": 140312576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737131394182548, + "loss": 1.8653, + "theoretical_loss": 4.586581731701562, + "tokens_seen": 140378112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736970912738215, + "loss": 1.8956, + "theoretical_loss": 4.586300871824143, + "tokens_seen": 140443648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736810431293882, + "loss": 1.8632, + "theoretical_loss": 4.586020179652427, + "tokens_seen": 140509184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736649949849549, + "loss": 1.7402, + "theoretical_loss": 4.585739655008123, + "tokens_seen": 140574720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736489468405216, + "loss": 1.8236, + "theoretical_loss": 4.585459297713215, + "tokens_seen": 140640256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736328986960883, + "loss": 1.8042, + "theoretical_loss": 4.585179107589955, + "tokens_seen": 140705792 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773616850551655, + "loss": 1.8166, + "theoretical_loss": 4.58489908446087, + "tokens_seen": 140771328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736008024072217, + "loss": 1.7575, + "theoretical_loss": 4.584619228148755, + "tokens_seen": 140836864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 53917, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8665790557861328, + "objective/train/theoretical_loss": 4.584339538476678, + "objective/train/tokens_used": 161362400, + "theoretical_loss": 4.584339538476678, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735847542627884, + "loss": 1.8941, + "theoretical_loss": 4.584339538476678, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735687061183552, + "loss": 1.6903, + "theoretical_loss": 4.5840600152679745, + "tokens_seen": 140967936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735526579739218, + "loss": 1.8307, + "theoretical_loss": 4.583780658346252, + "tokens_seen": 141033472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735366098294886, + "loss": 1.7627, + "theoretical_loss": 4.583501467535383, + "tokens_seen": 141099008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735205616850552, + "loss": 1.6755, + "theoretical_loss": 4.583222442659514, + "tokens_seen": 141164544 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773504513540622, + "loss": 1.8388, + "theoretical_loss": 4.5829435835430505, + "tokens_seen": 141230080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734884653961886, + "loss": 1.833, + "theoretical_loss": 4.582664890010674, + "tokens_seen": 141295616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734724172517554, + "loss": 1.9159, + "theoretical_loss": 4.582386361887329, + "tokens_seen": 141361152 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773456369107322, + "loss": 1.7725, + "theoretical_loss": 4.582107998998225, + "tokens_seen": 141426688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734403209628888, + "loss": 1.8281, + "theoretical_loss": 4.581829801168839, + "tokens_seen": 141492224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734242728184554, + "loss": 1.7452, + "theoretical_loss": 4.5815517682249105, + "tokens_seen": 141557760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734082246740222, + "loss": 1.8201, + "theoretical_loss": 4.581273899992447, + "tokens_seen": 141623296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733921765295888, + "loss": 1.8788, + "theoretical_loss": 4.580996196297718, + "tokens_seen": 141688832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733761283851556, + "loss": 1.8429, + "theoretical_loss": 4.580718656967257, + "tokens_seen": 141754368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733600802407222, + "loss": 1.8612, + "theoretical_loss": 4.580441281827858, + "tokens_seen": 141819904 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773344032096289, + "loss": 1.7208, + "theoretical_loss": 4.580164070706584, + "tokens_seen": 141885440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733279839518556, + "loss": 1.7845, + "theoretical_loss": 4.579887023430752, + "tokens_seen": 141950976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733119358074224, + "loss": 1.7119, + "theoretical_loss": 4.579610139827945, + "tokens_seen": 142016512 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773295887662989, + "loss": 1.7464, + "theoretical_loss": 4.579333419726007, + "tokens_seen": 142082048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732798395185558, + "loss": 1.8061, + "theoretical_loss": 4.579056862953039, + "tokens_seen": 142147584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732637913741224, + "loss": 1.8128, + "theoretical_loss": 4.578780469337407, + "tokens_seen": 142213120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732477432296892, + "loss": 1.77, + "theoretical_loss": 4.5785042387077315, + "tokens_seen": 142278656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732316950852558, + "loss": 1.8202, + "theoretical_loss": 4.578228170892896, + "tokens_seen": 142344192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732156469408226, + "loss": 1.9073, + "theoretical_loss": 4.577952265722038, + "tokens_seen": 142409728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731995987963892, + "loss": 1.8577, + "theoretical_loss": 4.5776765230245555, + "tokens_seen": 142475264 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773183550651956, + "loss": 1.8928, + "theoretical_loss": 4.5774009426301046, + "tokens_seen": 142540800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731675025075226, + "loss": 1.8294, + "theoretical_loss": 4.577125524368595, + "tokens_seen": 142606336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731514543630894, + "loss": 1.7549, + "theoretical_loss": 4.5768502680701975, + "tokens_seen": 142671872 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773135406218656, + "loss": 1.8273, + "theoretical_loss": 4.576575173565334, + "tokens_seen": 142737408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731193580742227, + "loss": 1.8096, + "theoretical_loss": 4.5763002406846836, + "tokens_seen": 142802944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731033099297894, + "loss": 1.8756, + "theoretical_loss": 4.57602546925918, + "tokens_seen": 142868480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730872617853561, + "loss": 1.6733, + "theoretical_loss": 4.575750859120014, + "tokens_seen": 142934016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730712136409228, + "loss": 1.8921, + "theoretical_loss": 4.575476410098624, + "tokens_seen": 142999552 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730551654964895, + "loss": 1.7704, + "theoretical_loss": 4.575202122026706, + "tokens_seen": 143065088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730391173520562, + "loss": 1.773, + "theoretical_loss": 4.5749279947362105, + "tokens_seen": 143130624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730230692076229, + "loss": 1.8325, + "theoretical_loss": 4.574654028059336, + "tokens_seen": 143196160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730070210631896, + "loss": 1.8279, + "theoretical_loss": 4.574380221828535, + "tokens_seen": 143261696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729909729187563, + "loss": 1.7535, + "theoretical_loss": 4.574106575876511, + "tokens_seen": 143327232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729749247743229, + "loss": 1.8514, + "theoretical_loss": 4.573833090036219, + "tokens_seen": 143392768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729588766298897, + "loss": 1.8637, + "theoretical_loss": 4.573559764140864, + "tokens_seen": 143458304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729428284854563, + "loss": 1.6962, + "theoretical_loss": 4.573286598023898, + "tokens_seen": 143523840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729267803410231, + "loss": 1.8597, + "theoretical_loss": 4.573013591519029, + "tokens_seen": 143589376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729107321965897, + "loss": 1.9061, + "theoretical_loss": 4.572740744460207, + "tokens_seen": 143654912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728946840521565, + "loss": 1.9639, + "theoretical_loss": 4.572468056681634, + "tokens_seen": 143720448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728786359077231, + "loss": 1.8062, + "theoretical_loss": 4.57219552801776, + "tokens_seen": 143785984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728625877632899, + "loss": 1.791, + "theoretical_loss": 4.57192315830328, + "tokens_seen": 143851520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728465396188565, + "loss": 1.7563, + "theoretical_loss": 4.571650947373138, + "tokens_seen": 143917056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728304914744233, + "loss": 1.8727, + "theoretical_loss": 4.571378895062525, + "tokens_seen": 143982592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728144433299899, + "loss": 1.7582, + "theoretical_loss": 4.571107001206875, + "tokens_seen": 144048128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727983951855567, + "loss": 1.885, + "theoretical_loss": 4.570835265641872, + "tokens_seen": 144113664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 54793, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8981997966766357, + "objective/train/theoretical_loss": 4.57056368820344, + "objective/train/tokens_used": 164639200, + "theoretical_loss": 4.57056368820344, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727823470411233, + "loss": 1.7713, + "theoretical_loss": 4.57056368820344, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727662988966901, + "loss": 1.722, + "theoretical_loss": 4.570292268727751, + "tokens_seen": 144244736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727502507522567, + "loss": 1.7943, + "theoretical_loss": 4.570021007051221, + "tokens_seen": 144310272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727342026078235, + "loss": 1.7481, + "theoretical_loss": 4.569749903010507, + "tokens_seen": 144375808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727181544633901, + "loss": 1.7384, + "theoretical_loss": 4.569478956442513, + "tokens_seen": 144441344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727021063189569, + "loss": 2.0361, + "theoretical_loss": 4.569208167184382, + "tokens_seen": 144506880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726860581745235, + "loss": 1.7303, + "theoretical_loss": 4.5689375350735, + "tokens_seen": 144572416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726700100300903, + "loss": 1.7676, + "theoretical_loss": 4.568667059947496, + "tokens_seen": 144637952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726539618856569, + "loss": 1.8165, + "theoretical_loss": 4.568396741644241, + "tokens_seen": 144703488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726379137412237, + "loss": 1.8475, + "theoretical_loss": 4.568126580001843, + "tokens_seen": 144769024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726218655967903, + "loss": 1.6271, + "theoretical_loss": 4.567856574858656, + "tokens_seen": 144834560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726058174523571, + "loss": 1.9387, + "theoretical_loss": 4.567586726053266, + "tokens_seen": 144900096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725897693079237, + "loss": 1.7635, + "theoretical_loss": 4.567317033424507, + "tokens_seen": 144965632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725737211634905, + "loss": 1.7297, + "theoretical_loss": 4.567047496811445, + "tokens_seen": 145031168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725576730190571, + "loss": 1.8948, + "theoretical_loss": 4.566778116053389, + "tokens_seen": 145096704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725416248746239, + "loss": 1.9421, + "theoretical_loss": 4.566508890989883, + "tokens_seen": 145162240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725255767301905, + "loss": 1.8752, + "theoretical_loss": 4.566239821460712, + "tokens_seen": 145227776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725095285857573, + "loss": 1.7984, + "theoretical_loss": 4.5659709073058945, + "tokens_seen": 145293312 + }, + { + "epoch": 0.04, + "learning_rate": 0.000772493480441324, + "loss": 1.8638, + "theoretical_loss": 4.565702148365686, + "tokens_seen": 145358848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724774322968907, + "loss": 1.7664, + "theoretical_loss": 4.56543354448058, + "tokens_seen": 145424384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724613841524574, + "loss": 1.842, + "theoretical_loss": 4.565165095491306, + "tokens_seen": 145489920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724453360080241, + "loss": 1.7626, + "theoretical_loss": 4.5648968012388265, + "tokens_seen": 145555456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724292878635908, + "loss": 1.7205, + "theoretical_loss": 4.56462866156434, + "tokens_seen": 145620992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724132397191575, + "loss": 1.7857, + "theoretical_loss": 4.564360676309279, + "tokens_seen": 145686528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723971915747242, + "loss": 1.8482, + "theoretical_loss": 4.564092845315312, + "tokens_seen": 145752064 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723811434302909, + "loss": 1.7422, + "theoretical_loss": 4.563825168424337, + "tokens_seen": 145817600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723650952858577, + "loss": 1.7175, + "theoretical_loss": 4.563557645478488, + "tokens_seen": 145883136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723490471414243, + "loss": 1.8653, + "theoretical_loss": 4.56329027632013, + "tokens_seen": 145948672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723329989969911, + "loss": 1.7809, + "theoretical_loss": 4.563023060791862, + "tokens_seen": 146014208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723169508525577, + "loss": 1.729, + "theoretical_loss": 4.562755998736511, + "tokens_seen": 146079744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723009027081245, + "loss": 1.7853, + "theoretical_loss": 4.562489089997141, + "tokens_seen": 146145280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722848545636911, + "loss": 1.7846, + "theoretical_loss": 4.562222334417041, + "tokens_seen": 146210816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722688064192579, + "loss": 1.7191, + "theoretical_loss": 4.561955731839734, + "tokens_seen": 146276352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722527582748245, + "loss": 1.8433, + "theoretical_loss": 4.561689282108971, + "tokens_seen": 146341888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722367101303913, + "loss": 1.818, + "theoretical_loss": 4.561422985068733, + "tokens_seen": 146407424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722206619859579, + "loss": 1.7675, + "theoretical_loss": 4.561156840563231, + "tokens_seen": 146472960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722046138415247, + "loss": 1.836, + "theoretical_loss": 4.560890848436902, + "tokens_seen": 146538496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721885656970913, + "loss": 1.8614, + "theoretical_loss": 4.560625008534414, + "tokens_seen": 146604032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721725175526581, + "loss": 1.824, + "theoretical_loss": 4.560359320700662, + "tokens_seen": 146669568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721564694082247, + "loss": 1.7436, + "theoretical_loss": 4.560093784780766, + "tokens_seen": 146735104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721404212637915, + "loss": 1.8323, + "theoretical_loss": 4.559828400620075, + "tokens_seen": 146800640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721243731193581, + "loss": 1.89, + "theoretical_loss": 4.559563168064165, + "tokens_seen": 146866176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721083249749249, + "loss": 1.9487, + "theoretical_loss": 4.559298086958837, + "tokens_seen": 146931712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720922768304915, + "loss": 1.6885, + "theoretical_loss": 4.559033157150115, + "tokens_seen": 146997248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720762286860583, + "loss": 1.8212, + "theoretical_loss": 4.5587683784842525, + "tokens_seen": 147062784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720601805416249, + "loss": 1.7993, + "theoretical_loss": 4.558503750807725, + "tokens_seen": 147128320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720441323971917, + "loss": 1.7828, + "theoretical_loss": 4.558239273967234, + "tokens_seen": 147193856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720280842527583, + "loss": 1.8936, + "theoretical_loss": 4.5579749478097025, + "tokens_seen": 147259392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720120361083251, + "loss": 1.8044, + "theoretical_loss": 4.557710772182278, + "tokens_seen": 147324928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719959879638917, + "loss": 1.8211, + "theoretical_loss": 4.5574467469323325, + "tokens_seen": 147390464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 56056, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8968627452850342, + "objective/train/theoretical_loss": 4.557182871907457, + "objective/train/tokens_used": 167916000, + "theoretical_loss": 4.557182871907457, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719799398194585, + "loss": 1.8249, + "theoretical_loss": 4.557182871907457, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719638916750251, + "loss": 1.7435, + "theoretical_loss": 4.556919146955469, + "tokens_seen": 147521536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719478435305919, + "loss": 1.7664, + "theoretical_loss": 4.556655571924404, + "tokens_seen": 147587072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719317953861585, + "loss": 1.8627, + "theoretical_loss": 4.556392146662521, + "tokens_seen": 147652608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719157472417253, + "loss": 1.8259, + "theoretical_loss": 4.556128871018297, + "tokens_seen": 147718144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718996990972919, + "loss": 2.0033, + "theoretical_loss": 4.555865744840434, + "tokens_seen": 147783680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718836509528587, + "loss": 1.698, + "theoretical_loss": 4.55560276797785, + "tokens_seen": 147849216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718676028084253, + "loss": 1.7972, + "theoretical_loss": 4.5553399402796835, + "tokens_seen": 147914752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718515546639921, + "loss": 1.7489, + "theoretical_loss": 4.5550772615952955, + "tokens_seen": 147980288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718355065195587, + "loss": 1.8099, + "theoretical_loss": 4.554814731774259, + "tokens_seen": 148045824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718194583751255, + "loss": 1.8885, + "theoretical_loss": 4.554552350666373, + "tokens_seen": 148111360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718034102306921, + "loss": 1.8405, + "theoretical_loss": 4.554290118121649, + "tokens_seen": 148176896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007717873620862589, + "loss": 1.8405, + "theoretical_loss": 4.554028033990319, + "tokens_seen": 148242432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007717713139418255, + "loss": 1.81, + "theoretical_loss": 4.5537660981228285, + "tokens_seen": 148307968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007717552657973923, + "loss": 1.8125, + "theoretical_loss": 4.553504310369844, + "tokens_seen": 148373504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007717392176529589, + "loss": 1.8753, + "theoretical_loss": 4.553242670582245, + "tokens_seen": 148439040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007717231695085256, + "loss": 1.9341, + "theoretical_loss": 4.55298117861113, + "tokens_seen": 148504576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007717071213640923, + "loss": 1.7652, + "theoretical_loss": 4.55271983430781, + "tokens_seen": 148570112 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771691073219659, + "loss": 1.9011, + "theoretical_loss": 4.552458637523815, + "tokens_seen": 148635648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716750250752257, + "loss": 1.7245, + "theoretical_loss": 4.552197588110882, + "tokens_seen": 148701184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716589769307924, + "loss": 1.736, + "theoretical_loss": 4.551936685920971, + "tokens_seen": 148766720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716429287863591, + "loss": 1.6701, + "theoretical_loss": 4.551675930806251, + "tokens_seen": 148832256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716268806419258, + "loss": 1.8453, + "theoretical_loss": 4.551415322619107, + "tokens_seen": 148897792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716108324974925, + "loss": 1.8299, + "theoretical_loss": 4.551154861212133, + "tokens_seen": 148963328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715947843530592, + "loss": 1.97, + "theoretical_loss": 4.550894546438139, + "tokens_seen": 149028864 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715787362086258, + "loss": 1.7303, + "theoretical_loss": 4.550634378150148, + "tokens_seen": 149094400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715626880641926, + "loss": 1.8074, + "theoretical_loss": 4.550374356201392, + "tokens_seen": 149159936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715466399197592, + "loss": 1.8377, + "theoretical_loss": 4.550114480445316, + "tokens_seen": 149225472 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771530591775326, + "loss": 1.8604, + "theoretical_loss": 4.549854750735575, + "tokens_seen": 149291008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715145436308926, + "loss": 1.9376, + "theoretical_loss": 4.5495951669260375, + "tokens_seen": 149356544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714984954864594, + "loss": 1.8481, + "theoretical_loss": 4.54933572887078, + "tokens_seen": 149422080 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771482447342026, + "loss": 1.8748, + "theoretical_loss": 4.5490764364240865, + "tokens_seen": 149487616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714663991975928, + "loss": 1.7571, + "theoretical_loss": 4.548817289440455, + "tokens_seen": 149553152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714503510531594, + "loss": 1.7867, + "theoretical_loss": 4.548558287774592, + "tokens_seen": 149618688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714343029087262, + "loss": 1.857, + "theoretical_loss": 4.548299431281409, + "tokens_seen": 149684224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714182547642928, + "loss": 1.773, + "theoretical_loss": 4.548040719816029, + "tokens_seen": 149749760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714022066198596, + "loss": 1.8777, + "theoretical_loss": 4.547782153233783, + "tokens_seen": 149815296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007713861584754262, + "loss": 1.7381, + "theoretical_loss": 4.547523731390206, + "tokens_seen": 149880832 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771370110330993, + "loss": 1.7556, + "theoretical_loss": 4.547265454141046, + "tokens_seen": 149946368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007713540621865596, + "loss": 1.7892, + "theoretical_loss": 4.547007321342251, + "tokens_seen": 150011904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007713380140421264, + "loss": 1.8294, + "theoretical_loss": 4.546749332849982, + "tokens_seen": 150077440 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771321965897693, + "loss": 1.7976, + "theoretical_loss": 4.546491488520601, + "tokens_seen": 150142976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007713059177532598, + "loss": 1.7639, + "theoretical_loss": 4.546233788210679, + "tokens_seen": 150208512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712898696088264, + "loss": 1.7777, + "theoretical_loss": 4.545976231776988, + "tokens_seen": 150274048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712738214643932, + "loss": 1.8345, + "theoretical_loss": 4.545718819076509, + "tokens_seen": 150339584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712577733199598, + "loss": 1.8569, + "theoretical_loss": 4.545461549966426, + "tokens_seen": 150405120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712417251755266, + "loss": 1.7723, + "theoretical_loss": 4.545204424304126, + "tokens_seen": 150470656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712256770310932, + "loss": 1.7852, + "theoretical_loss": 4.544947441947201, + "tokens_seen": 150536192 + }, + { + "epoch": 0.05, + "learning_rate": 0.00077120962888666, + "loss": 1.7586, + "theoretical_loss": 4.544690602753445, + "tokens_seen": 150601728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711935807422266, + "loss": 1.8189, + "theoretical_loss": 4.544433906580857, + "tokens_seen": 150667264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 56915, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7465159893035889, + "objective/train/theoretical_loss": 4.544177353287637, + "objective/train/tokens_used": 171192800, + "theoretical_loss": 4.544177353287637, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711775325977934, + "loss": 1.7764, + "theoretical_loss": 4.544177353287637, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711614844533602, + "loss": 1.8765, + "theoretical_loss": 4.543920942732187, + "tokens_seen": 150798336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711454363089268, + "loss": 1.7625, + "theoretical_loss": 4.543664674773112, + "tokens_seen": 150863872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711293881644936, + "loss": 1.8125, + "theoretical_loss": 4.543408549269218, + "tokens_seen": 150929408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711133400200602, + "loss": 1.7914, + "theoretical_loss": 4.54315256607951, + "tokens_seen": 150994944 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771097291875627, + "loss": 1.8573, + "theoretical_loss": 4.542896725063195, + "tokens_seen": 151060480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710812437311936, + "loss": 1.6647, + "theoretical_loss": 4.542641026079683, + "tokens_seen": 151126016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710651955867604, + "loss": 1.8338, + "theoretical_loss": 4.542385468988581, + "tokens_seen": 151191552 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771049147442327, + "loss": 1.8271, + "theoretical_loss": 4.542130053649695, + "tokens_seen": 151257088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710330992978938, + "loss": 1.6977, + "theoretical_loss": 4.541874779923031, + "tokens_seen": 151322624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710170511534604, + "loss": 1.7345, + "theoretical_loss": 4.541619647668796, + "tokens_seen": 151388160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710010030090272, + "loss": 1.9003, + "theoretical_loss": 4.541364656747392, + "tokens_seen": 151453696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709849548645938, + "loss": 1.7489, + "theoretical_loss": 4.541109807019422, + "tokens_seen": 151519232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709689067201606, + "loss": 1.7527, + "theoretical_loss": 4.540855098345683, + "tokens_seen": 151584768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709528585757272, + "loss": 1.784, + "theoretical_loss": 4.540600530587175, + "tokens_seen": 151650304 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770936810431294, + "loss": 1.7235, + "theoretical_loss": 4.5403461036050885, + "tokens_seen": 151715840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709207622868606, + "loss": 1.838, + "theoretical_loss": 4.540091817260815, + "tokens_seen": 151781376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709047141424274, + "loss": 1.8029, + "theoretical_loss": 4.539837671415941, + "tokens_seen": 151846912 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770888665997994, + "loss": 1.7071, + "theoretical_loss": 4.539583665932249, + "tokens_seen": 151912448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708726178535608, + "loss": 1.7128, + "theoretical_loss": 4.539329800671718, + "tokens_seen": 151977984 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708565697091274, + "loss": 1.7997, + "theoretical_loss": 4.539076075496519, + "tokens_seen": 152043520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708405215646942, + "loss": 1.9835, + "theoretical_loss": 4.538822490269021, + "tokens_seen": 152109056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708244734202608, + "loss": 1.7025, + "theoretical_loss": 4.538569044851785, + "tokens_seen": 152174592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708084252758276, + "loss": 1.7776, + "theoretical_loss": 4.538315739107571, + "tokens_seen": 152240128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007707923771313942, + "loss": 1.63, + "theoretical_loss": 4.5380625728993245, + "tokens_seen": 152305664 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770776328986961, + "loss": 1.736, + "theoretical_loss": 4.537809546090193, + "tokens_seen": 152371200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007707602808425276, + "loss": 1.8733, + "theoretical_loss": 4.53755665854351, + "tokens_seen": 152436736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007707442326980944, + "loss": 1.73, + "theoretical_loss": 4.537303910122807, + "tokens_seen": 152502272 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770728184553661, + "loss": 1.753, + "theoretical_loss": 4.537051300691805, + "tokens_seen": 152567808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007707121364092278, + "loss": 1.7725, + "theoretical_loss": 4.5367988301144155, + "tokens_seen": 152633344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706960882647944, + "loss": 1.7438, + "theoretical_loss": 4.536546498254746, + "tokens_seen": 152698880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706800401203612, + "loss": 1.8448, + "theoretical_loss": 4.536294304977092, + "tokens_seen": 152764416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706639919759278, + "loss": 1.76, + "theoretical_loss": 4.536042250145941, + "tokens_seen": 152829952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706479438314946, + "loss": 1.5725, + "theoretical_loss": 4.53579033362597, + "tokens_seen": 152895488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706318956870612, + "loss": 1.8505, + "theoretical_loss": 4.535538555282048, + "tokens_seen": 152961024 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770615847542628, + "loss": 1.9077, + "theoretical_loss": 4.535286914979232, + "tokens_seen": 153026560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705997993981946, + "loss": 1.7547, + "theoretical_loss": 4.5350354125827685, + "tokens_seen": 153092096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705837512537614, + "loss": 1.747, + "theoretical_loss": 4.534784047958096, + "tokens_seen": 153157632 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770567703109328, + "loss": 1.7225, + "theoretical_loss": 4.534532820970839, + "tokens_seen": 153223168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705516549648948, + "loss": 1.7695, + "theoretical_loss": 4.534281731486812, + "tokens_seen": 153288704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705356068204614, + "loss": 1.7913, + "theoretical_loss": 4.5340307793720145, + "tokens_seen": 153354240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705195586760282, + "loss": 1.703, + "theoretical_loss": 4.533779964492638, + "tokens_seen": 153419776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705035105315948, + "loss": 1.695, + "theoretical_loss": 4.533529286715059, + "tokens_seen": 153485312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007704874623871616, + "loss": 1.7663, + "theoretical_loss": 4.533278745905843, + "tokens_seen": 153550848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007704714142427282, + "loss": 1.6136, + "theoretical_loss": 4.533028341931738, + "tokens_seen": 153616384 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770455366098295, + "loss": 1.9402, + "theoretical_loss": 4.532778074659685, + "tokens_seen": 153681920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007704393179538616, + "loss": 1.8058, + "theoretical_loss": 4.532527943956804, + "tokens_seen": 153747456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007704232698094284, + "loss": 1.8964, + "theoretical_loss": 4.532277949690407, + "tokens_seen": 153812992 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770407221664995, + "loss": 1.6986, + "theoretical_loss": 4.532028091727986, + "tokens_seen": 153878528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703911735205618, + "loss": 1.6689, + "theoretical_loss": 4.531778369937222, + "tokens_seen": 153944064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 58041, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8966615200042725, + "objective/train/theoretical_loss": 4.53152878418598, + "objective/train/tokens_used": 174469600, + "theoretical_loss": 4.53152878418598, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703751253761284, + "loss": 1.7864, + "theoretical_loss": 4.53152878418598, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703590772316952, + "loss": 1.724, + "theoretical_loss": 4.531279334342306, + "tokens_seen": 154075136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703430290872618, + "loss": 1.7493, + "theoretical_loss": 4.531030020274436, + "tokens_seen": 154140672 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703269809428285, + "loss": 1.7452, + "theoretical_loss": 4.530780841850783, + "tokens_seen": 154206208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703109327983952, + "loss": 1.7928, + "theoretical_loss": 4.5305317989399505, + "tokens_seen": 154271744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702948846539619, + "loss": 1.7797, + "theoretical_loss": 4.530282891410717, + "tokens_seen": 154337280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702788365095286, + "loss": 1.7927, + "theoretical_loss": 4.530034119132052, + "tokens_seen": 154402816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702627883650953, + "loss": 1.7336, + "theoretical_loss": 4.529785481973102, + "tokens_seen": 154468352 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770246740220662, + "loss": 1.7642, + "theoretical_loss": 4.529536979803195, + "tokens_seen": 154533888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702306920762287, + "loss": 1.9063, + "theoretical_loss": 4.529288612491846, + "tokens_seen": 154599424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702146439317954, + "loss": 1.6956, + "theoretical_loss": 4.5290403799087455, + "tokens_seen": 154664960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701985957873621, + "loss": 1.9776, + "theoretical_loss": 4.528792281923769, + "tokens_seen": 154730496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701825476429287, + "loss": 1.8414, + "theoretical_loss": 4.52854431840697, + "tokens_seen": 154796032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701664994984955, + "loss": 1.8142, + "theoretical_loss": 4.528296489228585, + "tokens_seen": 154861568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701504513540621, + "loss": 1.6601, + "theoretical_loss": 4.528048794259028, + "tokens_seen": 154927104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701344032096289, + "loss": 1.8983, + "theoretical_loss": 4.527801233368896, + "tokens_seen": 154992640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701183550651955, + "loss": 1.7475, + "theoretical_loss": 4.527553806428962, + "tokens_seen": 155058176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701023069207623, + "loss": 1.8392, + "theoretical_loss": 4.5273065133101795, + "tokens_seen": 155123712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700862587763289, + "loss": 1.7844, + "theoretical_loss": 4.527059353883682, + "tokens_seen": 155189248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700702106318957, + "loss": 1.8887, + "theoretical_loss": 4.526812328020778, + "tokens_seen": 155254784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700541624874623, + "loss": 1.736, + "theoretical_loss": 4.52656543559296, + "tokens_seen": 155320320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700381143430291, + "loss": 1.7517, + "theoretical_loss": 4.526318676471891, + "tokens_seen": 155385856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700220661985957, + "loss": 1.7906, + "theoretical_loss": 4.526072050529417, + "tokens_seen": 155451392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700060180541625, + "loss": 1.8339, + "theoretical_loss": 4.52582555763756, + "tokens_seen": 155516928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699899699097291, + "loss": 1.8402, + "theoretical_loss": 4.525579197668517, + "tokens_seen": 155582464 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699739217652959, + "loss": 1.8406, + "theoretical_loss": 4.525332970494663, + "tokens_seen": 155648000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699578736208627, + "loss": 1.793, + "theoretical_loss": 4.525086875988549, + "tokens_seen": 155713536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699418254764293, + "loss": 1.8599, + "theoretical_loss": 4.524840914022902, + "tokens_seen": 155779072 + }, + { + "epoch": 0.05, + "learning_rate": 0.000769925777331996, + "loss": 1.7589, + "theoretical_loss": 4.524595084470625, + "tokens_seen": 155844608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699097291875627, + "loss": 1.8455, + "theoretical_loss": 4.524349387204794, + "tokens_seen": 155910144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698936810431295, + "loss": 1.6923, + "theoretical_loss": 4.524103822098664, + "tokens_seen": 155975680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698776328986961, + "loss": 1.7622, + "theoretical_loss": 4.52385838902566, + "tokens_seen": 156041216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698615847542629, + "loss": 1.6998, + "theoretical_loss": 4.523613087859386, + "tokens_seen": 156106752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698455366098295, + "loss": 1.7923, + "theoretical_loss": 4.523367918473614, + "tokens_seen": 156172288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698294884653963, + "loss": 1.814, + "theoretical_loss": 4.523122880742298, + "tokens_seen": 156237824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698134403209629, + "loss": 1.7696, + "theoretical_loss": 4.522877974539556, + "tokens_seen": 156303360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697973921765297, + "loss": 1.8467, + "theoretical_loss": 4.522633199739687, + "tokens_seen": 156368896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697813440320963, + "loss": 1.7256, + "theoretical_loss": 4.522388556217157, + "tokens_seen": 156434432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697652958876631, + "loss": 1.7527, + "theoretical_loss": 4.522144043846609, + "tokens_seen": 156499968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697492477432297, + "loss": 1.784, + "theoretical_loss": 4.521899662502856, + "tokens_seen": 156565504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697331995987965, + "loss": 1.7029, + "theoretical_loss": 4.5216554120608805, + "tokens_seen": 156631040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697171514543631, + "loss": 1.7115, + "theoretical_loss": 4.521411292395841, + "tokens_seen": 156696576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697011033099299, + "loss": 1.7872, + "theoretical_loss": 4.521167303383065, + "tokens_seen": 156762112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696850551654965, + "loss": 1.6923, + "theoretical_loss": 4.520923444898051, + "tokens_seen": 156827648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696690070210633, + "loss": 1.7266, + "theoretical_loss": 4.520679716816467, + "tokens_seen": 156893184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696529588766299, + "loss": 1.7961, + "theoretical_loss": 4.520436119014155, + "tokens_seen": 156958720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696369107321967, + "loss": 1.7192, + "theoretical_loss": 4.520192651367121, + "tokens_seen": 157024256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696208625877633, + "loss": 1.5909, + "theoretical_loss": 4.519949313751547, + "tokens_seen": 157089792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696048144433301, + "loss": 1.8818, + "theoretical_loss": 4.519706106043779, + "tokens_seen": 157155328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695887662988967, + "loss": 1.7198, + "theoretical_loss": 4.519463028120338, + "tokens_seen": 157220864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 59100, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.76185941696167, + "objective/train/theoretical_loss": 4.519220079857908, + "objective/train/tokens_used": 177746400, + "theoretical_loss": 4.519220079857908, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695727181544635, + "loss": 1.7576, + "theoretical_loss": 4.519220079857908, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695566700100301, + "loss": 1.7981, + "theoretical_loss": 4.518977261133343, + "tokens_seen": 157351936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695406218655969, + "loss": 1.6934, + "theoretical_loss": 4.518734571823668, + "tokens_seen": 157417472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695245737211635, + "loss": 1.7905, + "theoretical_loss": 4.518492011806073, + "tokens_seen": 157483008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695085255767303, + "loss": 1.7677, + "theoretical_loss": 4.518249580957917, + "tokens_seen": 157548544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694924774322969, + "loss": 1.8178, + "theoretical_loss": 4.518007279156725, + "tokens_seen": 157614080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694764292878637, + "loss": 1.8327, + "theoretical_loss": 4.5177651062801925, + "tokens_seen": 157679616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694603811434303, + "loss": 1.8089, + "theoretical_loss": 4.517523062206175, + "tokens_seen": 157745152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694443329989971, + "loss": 1.7748, + "theoretical_loss": 4.517281146812701, + "tokens_seen": 157810688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694282848545637, + "loss": 1.8415, + "theoretical_loss": 4.517039359977961, + "tokens_seen": 157876224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694122367101305, + "loss": 1.8614, + "theoretical_loss": 4.516797701580314, + "tokens_seen": 157941760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693961885656971, + "loss": 1.6887, + "theoretical_loss": 4.516556171498284, + "tokens_seen": 158007296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693801404212639, + "loss": 1.6983, + "theoretical_loss": 4.516314769610557, + "tokens_seen": 158072832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693640922768305, + "loss": 1.693, + "theoretical_loss": 4.516073495795988, + "tokens_seen": 158138368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693480441323973, + "loss": 1.8146, + "theoretical_loss": 4.5158323499335955, + "tokens_seen": 158203904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693319959879639, + "loss": 1.7686, + "theoretical_loss": 4.515591331902561, + "tokens_seen": 158269440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693159478435307, + "loss": 1.6949, + "theoretical_loss": 4.515350441582231, + "tokens_seen": 158334976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692998996990973, + "loss": 1.8541, + "theoretical_loss": 4.515109678852117, + "tokens_seen": 158400512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692838515546641, + "loss": 1.9401, + "theoretical_loss": 4.514869043591891, + "tokens_seen": 158466048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692678034102307, + "loss": 1.7639, + "theoretical_loss": 4.514628535681391, + "tokens_seen": 158531584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692517552657975, + "loss": 1.7224, + "theoretical_loss": 4.514388155000615, + "tokens_seen": 158597120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692357071213641, + "loss": 1.76, + "theoretical_loss": 4.5141479014297285, + "tokens_seen": 158662656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692196589769309, + "loss": 1.8276, + "theoretical_loss": 4.513907774849054, + "tokens_seen": 158728192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692036108324975, + "loss": 1.8055, + "theoretical_loss": 4.513667775139078, + "tokens_seen": 158793728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691875626880643, + "loss": 1.7416, + "theoretical_loss": 4.5134279021804495, + "tokens_seen": 158859264 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691715145436309, + "loss": 1.675, + "theoretical_loss": 4.513188155853978, + "tokens_seen": 158924800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691554663991977, + "loss": 1.8391, + "theoretical_loss": 4.512948536040636, + "tokens_seen": 158990336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691394182547643, + "loss": 1.8972, + "theoretical_loss": 4.512709042621554, + "tokens_seen": 159055872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691233701103311, + "loss": 1.7549, + "theoretical_loss": 4.512469675478026, + "tokens_seen": 159121408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691073219658977, + "loss": 1.7728, + "theoretical_loss": 4.512230434491503, + "tokens_seen": 159186944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690912738214645, + "loss": 1.8985, + "theoretical_loss": 4.511991319543599, + "tokens_seen": 159252480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690752256770311, + "loss": 1.7964, + "theoretical_loss": 4.511752330516086, + "tokens_seen": 159318016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690591775325979, + "loss": 1.658, + "theoretical_loss": 4.5115134672908965, + "tokens_seen": 159383552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690431293881645, + "loss": 1.9224, + "theoretical_loss": 4.511274729750123, + "tokens_seen": 159449088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690270812437313, + "loss": 1.7319, + "theoretical_loss": 4.5110361177760145, + "tokens_seen": 159514624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690110330992979, + "loss": 1.7229, + "theoretical_loss": 4.5107976312509805, + "tokens_seen": 159580160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689949849548647, + "loss": 1.7498, + "theoretical_loss": 4.510559270057587, + "tokens_seen": 159645696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689789368104313, + "loss": 1.7295, + "theoretical_loss": 4.51032103407856, + "tokens_seen": 159711232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689628886659981, + "loss": 1.7314, + "theoretical_loss": 4.510082923196784, + "tokens_seen": 159776768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689468405215647, + "loss": 1.7616, + "theoretical_loss": 4.5098449372952985, + "tokens_seen": 159842304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689307923771315, + "loss": 1.7165, + "theoretical_loss": 4.5096070762573, + "tokens_seen": 159907840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689147442326981, + "loss": 1.7341, + "theoretical_loss": 4.509369339966146, + "tokens_seen": 159973376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688986960882648, + "loss": 1.7801, + "theoretical_loss": 4.509131728305347, + "tokens_seen": 160038912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688826479438315, + "loss": 1.9064, + "theoretical_loss": 4.50889424115857, + "tokens_seen": 160104448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688665997993982, + "loss": 1.6784, + "theoretical_loss": 4.50865687840964, + "tokens_seen": 160169984 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688505516549649, + "loss": 1.7526, + "theoretical_loss": 4.508419639942538, + "tokens_seen": 160235520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688345035105316, + "loss": 1.7459, + "theoretical_loss": 4.5081825256413985, + "tokens_seen": 160301056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688184553660983, + "loss": 1.8808, + "theoretical_loss": 4.507945535390512, + "tokens_seen": 160366592 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768802407221665, + "loss": 1.816, + "theoretical_loss": 4.507708669074326, + "tokens_seen": 160432128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687863590772316, + "loss": 1.7083, + "theoretical_loss": 4.5074719265774394, + "tokens_seen": 160497664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 60314, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.1015968322753906, + "objective/train/theoretical_loss": 4.507235307784608, + "objective/train/tokens_used": 181023200, + "theoretical_loss": 4.507235307784608, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687703109327984, + "loss": 1.9357, + "theoretical_loss": 4.507235307784608, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687542627883651, + "loss": 1.6547, + "theoretical_loss": 4.5069988125807425, + "tokens_seen": 160628736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687382146439318, + "loss": 1.7709, + "theoretical_loss": 4.506762440850906, + "tokens_seen": 160694272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687221664994985, + "loss": 1.7327, + "theoretical_loss": 4.506526192480315, + "tokens_seen": 160759808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687061183550652, + "loss": 1.7753, + "theoretical_loss": 4.506290067354341, + "tokens_seen": 160825344 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768690070210632, + "loss": 1.7335, + "theoretical_loss": 4.506054065358508, + "tokens_seen": 160890880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007686740220661986, + "loss": 1.7166, + "theoretical_loss": 4.505818186378491, + "tokens_seen": 160956416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007686579739217653, + "loss": 1.6904, + "theoretical_loss": 4.505582430300121, + "tokens_seen": 161021952 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768641925777332, + "loss": 1.8413, + "theoretical_loss": 4.505346797009381, + "tokens_seen": 161087488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007686258776328987, + "loss": 1.7868, + "theoretical_loss": 4.505111286392402, + "tokens_seen": 161153024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007686098294884654, + "loss": 1.6634, + "theoretical_loss": 4.50487589833547, + "tokens_seen": 161218560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685937813440322, + "loss": 1.6353, + "theoretical_loss": 4.504640632725025, + "tokens_seen": 161284096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685777331995988, + "loss": 1.7892, + "theoretical_loss": 4.504405489447654, + "tokens_seen": 161349632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685616850551656, + "loss": 1.7234, + "theoretical_loss": 4.504170468390096, + "tokens_seen": 161415168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685456369107322, + "loss": 1.6762, + "theoretical_loss": 4.503935569439242, + "tokens_seen": 161480704 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768529588766299, + "loss": 1.8172, + "theoretical_loss": 4.5037007924821335, + "tokens_seen": 161546240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685135406218656, + "loss": 1.7786, + "theoretical_loss": 4.503466137405961, + "tokens_seen": 161611776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684974924774324, + "loss": 1.732, + "theoretical_loss": 4.503231604098065, + "tokens_seen": 161677312 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768481444332999, + "loss": 1.8209, + "theoretical_loss": 4.50299719244594, + "tokens_seen": 161742848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684653961885658, + "loss": 1.8043, + "theoretical_loss": 4.502762902337222, + "tokens_seen": 161808384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684493480441324, + "loss": 1.8455, + "theoretical_loss": 4.502528733659704, + "tokens_seen": 161873920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684332998996992, + "loss": 1.7825, + "theoretical_loss": 4.502294686301323, + "tokens_seen": 161939456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684172517552658, + "loss": 1.6838, + "theoretical_loss": 4.502060760150167, + "tokens_seen": 162004992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684012036108326, + "loss": 1.8061, + "theoretical_loss": 4.501826955094472, + "tokens_seen": 162070528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007683851554663992, + "loss": 1.6355, + "theoretical_loss": 4.501593271022623, + "tokens_seen": 162136064 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768369107321966, + "loss": 1.7908, + "theoretical_loss": 4.501359707823149, + "tokens_seen": 162201600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007683530591775326, + "loss": 1.7182, + "theoretical_loss": 4.501126265384732, + "tokens_seen": 162267136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007683370110330994, + "loss": 1.9053, + "theoretical_loss": 4.500892943596199, + "tokens_seen": 162332672 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768320962888666, + "loss": 1.8499, + "theoretical_loss": 4.500659742346524, + "tokens_seen": 162398208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007683049147442328, + "loss": 1.7459, + "theoretical_loss": 4.500426661524828, + "tokens_seen": 162463744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682888665997994, + "loss": 1.774, + "theoretical_loss": 4.500193701020379, + "tokens_seen": 162529280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682728184553662, + "loss": 1.8263, + "theoretical_loss": 4.499960860722592, + "tokens_seen": 162594816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682567703109328, + "loss": 1.7356, + "theoretical_loss": 4.499728140521025, + "tokens_seen": 162660352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682407221664996, + "loss": 1.8387, + "theoretical_loss": 4.499495540305388, + "tokens_seen": 162725888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682246740220662, + "loss": 1.6331, + "theoretical_loss": 4.49926305996553, + "tokens_seen": 162791424 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768208625877633, + "loss": 1.7426, + "theoretical_loss": 4.499030699391449, + "tokens_seen": 162856960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681925777331996, + "loss": 1.617, + "theoretical_loss": 4.49879845847329, + "tokens_seen": 162922496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681765295887664, + "loss": 1.7124, + "theoretical_loss": 4.498566337101337, + "tokens_seen": 162988032 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768160481444333, + "loss": 1.7712, + "theoretical_loss": 4.4983343351660245, + "tokens_seen": 163053568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681444332998998, + "loss": 1.9108, + "theoretical_loss": 4.498102452557928, + "tokens_seen": 163119104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681283851554664, + "loss": 1.6287, + "theoretical_loss": 4.497870689167771, + "tokens_seen": 163184640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681123370110332, + "loss": 1.8195, + "theoretical_loss": 4.497639044886412, + "tokens_seen": 163250176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680962888665998, + "loss": 1.7055, + "theoretical_loss": 4.497407519604866, + "tokens_seen": 163315712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680802407221666, + "loss": 1.8599, + "theoretical_loss": 4.49717611321428, + "tokens_seen": 163381248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680641925777332, + "loss": 1.6922, + "theoretical_loss": 4.496944825605951, + "tokens_seen": 163446784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680481444333, + "loss": 1.6333, + "theoretical_loss": 4.496713656671317, + "tokens_seen": 163512320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680320962888666, + "loss": 1.8145, + "theoretical_loss": 4.496482606301957, + "tokens_seen": 163577856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680160481444334, + "loss": 1.7066, + "theoretical_loss": 4.496251674389596, + "tokens_seen": 163643392 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768, + "loss": 1.7063, + "theoretical_loss": 4.496020860826098, + "tokens_seen": 163708928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679839518555668, + "loss": 1.7411, + "theoretical_loss": 4.49579016550347, + "tokens_seen": 163774464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 61072, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7547411918640137, + "objective/train/theoretical_loss": 4.49555958831386, + "objective/train/tokens_used": 184300000, + "theoretical_loss": 4.49555958831386, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679679037111334, + "loss": 1.7758, + "theoretical_loss": 4.49555958831386, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679518555667002, + "loss": 1.7571, + "theoretical_loss": 4.49532912914956, + "tokens_seen": 163905536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679358074222668, + "loss": 1.7834, + "theoretical_loss": 4.495098787903, + "tokens_seen": 163971072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679197592778336, + "loss": 1.7879, + "theoretical_loss": 4.4948685644667545, + "tokens_seen": 164036608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679037111334002, + "loss": 1.6408, + "theoretical_loss": 4.494638458733535, + "tokens_seen": 164102144 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767887662988967, + "loss": 1.6375, + "theoretical_loss": 4.494408470596195, + "tokens_seen": 164167680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007678716148445336, + "loss": 1.7134, + "theoretical_loss": 4.494178599947729, + "tokens_seen": 164233216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007678555667001004, + "loss": 1.8468, + "theoretical_loss": 4.4939488466812705, + "tokens_seen": 164298752 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767839518555667, + "loss": 1.8571, + "theoretical_loss": 4.493719210690093, + "tokens_seen": 164364288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007678234704112338, + "loss": 1.711, + "theoretical_loss": 4.493489691867612, + "tokens_seen": 164429824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007678074222668004, + "loss": 1.6744, + "theoretical_loss": 4.493260290107379, + "tokens_seen": 164495360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677913741223672, + "loss": 1.6627, + "theoretical_loss": 4.493031005303084, + "tokens_seen": 164560896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677753259779338, + "loss": 1.7317, + "theoretical_loss": 4.49280183734856, + "tokens_seen": 164626432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677592778335006, + "loss": 1.8705, + "theoretical_loss": 4.492572786137774, + "tokens_seen": 164691968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677432296890672, + "loss": 1.7851, + "theoretical_loss": 4.492343851564835, + "tokens_seen": 164757504 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767727181544634, + "loss": 1.8575, + "theoretical_loss": 4.492115033523987, + "tokens_seen": 164823040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677111334002006, + "loss": 1.7289, + "theoretical_loss": 4.491886331909615, + "tokens_seen": 164888576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676950852557674, + "loss": 1.7442, + "theoretical_loss": 4.491657746616241, + "tokens_seen": 164954112 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767679037111334, + "loss": 1.6633, + "theoretical_loss": 4.49142927753852, + "tokens_seen": 165019648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676629889669008, + "loss": 1.6377, + "theoretical_loss": 4.491200924571251, + "tokens_seen": 165085184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676469408224674, + "loss": 1.5454, + "theoretical_loss": 4.490972687609364, + "tokens_seen": 165150720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676308926780342, + "loss": 1.7009, + "theoretical_loss": 4.4907445665479315, + "tokens_seen": 165216256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676148445336008, + "loss": 1.7974, + "theoretical_loss": 4.490516561282157, + "tokens_seen": 165281792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675987963891676, + "loss": 1.6644, + "theoretical_loss": 4.4902886717073835, + "tokens_seen": 165347328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675827482447342, + "loss": 1.7853, + "theoretical_loss": 4.49006089771909, + "tokens_seen": 165412864 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767566700100301, + "loss": 1.5877, + "theoretical_loss": 4.489833239212888, + "tokens_seen": 165478400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675506519558676, + "loss": 1.6106, + "theoretical_loss": 4.48960569608453, + "tokens_seen": 165543936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675346038114344, + "loss": 1.7594, + "theoretical_loss": 4.489378268229897, + "tokens_seen": 165609472 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767518555667001, + "loss": 1.8262, + "theoretical_loss": 4.489150955545012, + "tokens_seen": 165675008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675025075225677, + "loss": 1.7828, + "theoretical_loss": 4.488923757926031, + "tokens_seen": 165740544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674864593781344, + "loss": 1.6994, + "theoretical_loss": 4.4886966752692405, + "tokens_seen": 165806080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674704112337011, + "loss": 1.6913, + "theoretical_loss": 4.488469707471066, + "tokens_seen": 165871616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674543630892678, + "loss": 1.6736, + "theoretical_loss": 4.488242854428064, + "tokens_seen": 165937152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674383149448345, + "loss": 1.6912, + "theoretical_loss": 4.488016116036929, + "tokens_seen": 166002688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674222668004012, + "loss": 1.9221, + "theoretical_loss": 4.487789492194485, + "tokens_seen": 166068224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674062186559679, + "loss": 1.7546, + "theoretical_loss": 4.487562982797693, + "tokens_seen": 166133760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673901705115346, + "loss": 1.7376, + "theoretical_loss": 4.487336587743643, + "tokens_seen": 166199296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673741223671013, + "loss": 1.7413, + "theoretical_loss": 4.487110306929562, + "tokens_seen": 166264832 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767358074222668, + "loss": 1.7142, + "theoretical_loss": 4.48688414025281, + "tokens_seen": 166330368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673420260782347, + "loss": 1.8665, + "theoretical_loss": 4.486658087610875, + "tokens_seen": 166395904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673259779338014, + "loss": 1.6341, + "theoretical_loss": 4.486432148901384, + "tokens_seen": 166461440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673099297893681, + "loss": 1.7621, + "theoretical_loss": 4.48620632402209, + "tokens_seen": 166526976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672938816449348, + "loss": 1.7723, + "theoretical_loss": 4.485980612870883, + "tokens_seen": 166592512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672778335005015, + "loss": 1.7573, + "theoretical_loss": 4.48575501534578, + "tokens_seen": 166658048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672617853560682, + "loss": 1.7641, + "theoretical_loss": 4.485529531344932, + "tokens_seen": 166723584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672457372116349, + "loss": 1.8136, + "theoretical_loss": 4.485304160766624, + "tokens_seen": 166789120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672296890672016, + "loss": 1.7361, + "theoretical_loss": 4.485078903509266, + "tokens_seen": 166854656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672136409227683, + "loss": 1.8344, + "theoretical_loss": 4.484853759471403, + "tokens_seen": 166920192 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767197592778335, + "loss": 1.833, + "theoretical_loss": 4.484628728551711, + "tokens_seen": 166985728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671815446339017, + "loss": 1.7914, + "theoretical_loss": 4.484403810648994, + "tokens_seen": 167051264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 62380, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6307178735733032, + "objective/train/theoretical_loss": 4.484179005662186, + "objective/train/tokens_used": 187576800, + "theoretical_loss": 4.484179005662186, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671654964894685, + "loss": 1.7346, + "theoretical_loss": 4.484179005662186, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671494483450351, + "loss": 1.651, + "theoretical_loss": 4.483954313490354, + "tokens_seen": 167182336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671334002006019, + "loss": 1.7858, + "theoretical_loss": 4.483729734032694, + "tokens_seen": 167247872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671173520561685, + "loss": 1.7515, + "theoretical_loss": 4.483505267188528, + "tokens_seen": 167313408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671013039117353, + "loss": 1.7517, + "theoretical_loss": 4.48328091285731, + "tokens_seen": 167378944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670852557673019, + "loss": 1.8486, + "theoretical_loss": 4.483056670938625, + "tokens_seen": 167444480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670692076228687, + "loss": 1.796, + "theoretical_loss": 4.482832541332183, + "tokens_seen": 167510016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670531594784353, + "loss": 1.7272, + "theoretical_loss": 4.482608523937824, + "tokens_seen": 167575552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670371113340021, + "loss": 1.6783, + "theoretical_loss": 4.4823846186555185, + "tokens_seen": 167641088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670210631895687, + "loss": 1.658, + "theoretical_loss": 4.482160825385363, + "tokens_seen": 167706624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670050150451355, + "loss": 1.6597, + "theoretical_loss": 4.481937144027583, + "tokens_seen": 167772160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669889669007021, + "loss": 1.7993, + "theoretical_loss": 4.48171357448253, + "tokens_seen": 167837696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669729187562689, + "loss": 1.5537, + "theoretical_loss": 4.481490116650686, + "tokens_seen": 167903232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669568706118355, + "loss": 1.6828, + "theoretical_loss": 4.48126677043266, + "tokens_seen": 167968768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669408224674023, + "loss": 1.7082, + "theoretical_loss": 4.481043535729183, + "tokens_seen": 168034304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669247743229689, + "loss": 1.75, + "theoretical_loss": 4.480820412441122, + "tokens_seen": 168099840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669087261785357, + "loss": 1.8383, + "theoretical_loss": 4.4805974004694615, + "tokens_seen": 168165376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668926780341023, + "loss": 1.7588, + "theoretical_loss": 4.48037449971532, + "tokens_seen": 168230912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668766298896691, + "loss": 1.7177, + "theoretical_loss": 4.480151710079937, + "tokens_seen": 168296448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668605817452357, + "loss": 1.722, + "theoretical_loss": 4.479929031464682, + "tokens_seen": 168361984 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668445336008025, + "loss": 1.7248, + "theoretical_loss": 4.479706463771047, + "tokens_seen": 168427520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668284854563691, + "loss": 1.6552, + "theoretical_loss": 4.479484006900652, + "tokens_seen": 168493056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668124373119359, + "loss": 1.6428, + "theoretical_loss": 4.479261660755242, + "tokens_seen": 168558592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667963891675025, + "loss": 1.8342, + "theoretical_loss": 4.479039425236687, + "tokens_seen": 168624128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667803410230693, + "loss": 1.7043, + "theoretical_loss": 4.478817300246982, + "tokens_seen": 168689664 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667642928786359, + "loss": 1.7463, + "theoretical_loss": 4.478595285688247, + "tokens_seen": 168755200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667482447342027, + "loss": 1.6112, + "theoretical_loss": 4.478373381462728, + "tokens_seen": 168820736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667321965897693, + "loss": 1.689, + "theoretical_loss": 4.478151587472791, + "tokens_seen": 168886272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667161484453361, + "loss": 1.7633, + "theoretical_loss": 4.477929903620931, + "tokens_seen": 168951808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667001003009027, + "loss": 1.7455, + "theoretical_loss": 4.477708329809766, + "tokens_seen": 169017344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666840521564695, + "loss": 1.7587, + "theoretical_loss": 4.477486865942036, + "tokens_seen": 169082880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666680040120361, + "loss": 1.8086, + "theoretical_loss": 4.477265511920607, + "tokens_seen": 169148416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666519558676029, + "loss": 1.7175, + "theoretical_loss": 4.477044267648466, + "tokens_seen": 169213952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666359077231695, + "loss": 1.8273, + "theoretical_loss": 4.476823133028725, + "tokens_seen": 169279488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666198595787363, + "loss": 1.7291, + "theoretical_loss": 4.47660210796462, + "tokens_seen": 169345024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666038114343029, + "loss": 1.7855, + "theoretical_loss": 4.476381192359504, + "tokens_seen": 169410560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665877632898697, + "loss": 1.8167, + "theoretical_loss": 4.476160386116861, + "tokens_seen": 169476096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665717151454363, + "loss": 1.8135, + "theoretical_loss": 4.475939689140291, + "tokens_seen": 169541632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665556670010031, + "loss": 1.6252, + "theoretical_loss": 4.475719101333519, + "tokens_seen": 169607168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665396188565697, + "loss": 1.851, + "theoretical_loss": 4.475498622600391, + "tokens_seen": 169672704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665235707121365, + "loss": 1.642, + "theoretical_loss": 4.475278252844875, + "tokens_seen": 169738240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665075225677031, + "loss": 1.7247, + "theoretical_loss": 4.475057991971062, + "tokens_seen": 169803776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664914744232699, + "loss": 1.729, + "theoretical_loss": 4.474837839883162, + "tokens_seen": 169869312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664754262788365, + "loss": 1.7534, + "theoretical_loss": 4.474617796485507, + "tokens_seen": 169934848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664593781344033, + "loss": 1.7936, + "theoretical_loss": 4.474397861682552, + "tokens_seen": 170000384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664433299899699, + "loss": 1.747, + "theoretical_loss": 4.474178035378869, + "tokens_seen": 170065920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664272818455367, + "loss": 1.8412, + "theoretical_loss": 4.473958317479154, + "tokens_seen": 170131456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664112337011033, + "loss": 1.5907, + "theoretical_loss": 4.473738707888221, + "tokens_seen": 170196992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663951855566701, + "loss": 1.778, + "theoretical_loss": 4.473519206511006, + "tokens_seen": 170262528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663791374122367, + "loss": 1.8488, + "theoretical_loss": 4.473299813252565, + "tokens_seen": 170328064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 63171, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6936179399490356, + "objective/train/theoretical_loss": 4.473080528018071, + "objective/train/tokens_used": 190853600, + "theoretical_loss": 4.473080528018071, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663630892678035, + "loss": 1.7195, + "theoretical_loss": 4.473080528018071, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663470411233702, + "loss": 1.7942, + "theoretical_loss": 4.4728613507128205, + "tokens_seen": 170459136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663309929789369, + "loss": 1.673, + "theoretical_loss": 4.472642281242226, + "tokens_seen": 170524672 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663149448345036, + "loss": 1.6945, + "theoretical_loss": 4.472423319511822, + "tokens_seen": 170590208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662988966900703, + "loss": 1.7537, + "theoretical_loss": 4.472204465427261, + "tokens_seen": 170655744 + }, + { + "epoch": 0.05, + "learning_rate": 0.000766282848545637, + "loss": 1.7052, + "theoretical_loss": 4.471985718894312, + "tokens_seen": 170721280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662668004012037, + "loss": 1.7989, + "theoretical_loss": 4.471767079818866, + "tokens_seen": 170786816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662507522567703, + "loss": 1.7626, + "theoretical_loss": 4.471548548106932, + "tokens_seen": 170852352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662347041123371, + "loss": 1.7287, + "theoretical_loss": 4.471330123664634, + "tokens_seen": 170917888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662186559679037, + "loss": 1.5974, + "theoretical_loss": 4.471111806398218, + "tokens_seen": 170983424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662026078234705, + "loss": 1.7094, + "theoretical_loss": 4.4708935962140455, + "tokens_seen": 171048960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661865596790371, + "loss": 1.7946, + "theoretical_loss": 4.4706754930185975, + "tokens_seen": 171114496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661705115346039, + "loss": 1.677, + "theoretical_loss": 4.47045749671847, + "tokens_seen": 171180032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661544633901705, + "loss": 1.6607, + "theoretical_loss": 4.470239607220377, + "tokens_seen": 171245568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661384152457373, + "loss": 1.7783, + "theoretical_loss": 4.470021824431152, + "tokens_seen": 171311104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661223671013039, + "loss": 1.7274, + "theoretical_loss": 4.469804148257742, + "tokens_seen": 171376640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661063189568706, + "loss": 1.8638, + "theoretical_loss": 4.469586578607213, + "tokens_seen": 171442176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660902708124373, + "loss": 1.7533, + "theoretical_loss": 4.469369115386746, + "tokens_seen": 171507712 + }, + { + "epoch": 0.05, + "learning_rate": 0.000766074222668004, + "loss": 1.7417, + "theoretical_loss": 4.46915175850364, + "tokens_seen": 171573248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660581745235707, + "loss": 1.6826, + "theoretical_loss": 4.468934507865308, + "tokens_seen": 171638784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660421263791374, + "loss": 1.7231, + "theoretical_loss": 4.46871736337928, + "tokens_seen": 171704320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660260782347041, + "loss": 1.7589, + "theoretical_loss": 4.468500324953202, + "tokens_seen": 171769856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660100300902708, + "loss": 1.8372, + "theoretical_loss": 4.468283392494836, + "tokens_seen": 171835392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007659939819458375, + "loss": 1.8463, + "theoretical_loss": 4.468066565912057, + "tokens_seen": 171900928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007659779338014042, + "loss": 1.7143, + "theoretical_loss": 4.467849845112859, + "tokens_seen": 171966464 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765961885656971, + "loss": 1.7111, + "theoretical_loss": 4.467633230005347, + "tokens_seen": 172032000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007659458375125376, + "loss": 1.7905, + "theoretical_loss": 4.467416720497743, + "tokens_seen": 172097536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007659297893681043, + "loss": 1.7305, + "theoretical_loss": 4.467200316498385, + "tokens_seen": 172163072 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765913741223671, + "loss": 1.6837, + "theoretical_loss": 4.4669840179157205, + "tokens_seen": 172228608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658976930792377, + "loss": 1.6952, + "theoretical_loss": 4.466767824658317, + "tokens_seen": 172294144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658816449348044, + "loss": 1.6608, + "theoretical_loss": 4.466551736634852, + "tokens_seen": 172359680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658655967903711, + "loss": 1.7433, + "theoretical_loss": 4.466335753754118, + "tokens_seen": 172425216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658495486459378, + "loss": 1.7105, + "theoretical_loss": 4.466119875925022, + "tokens_seen": 172490752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658335005015046, + "loss": 1.6972, + "theoretical_loss": 4.465904103056586, + "tokens_seen": 172556288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658174523570712, + "loss": 1.7239, + "theoretical_loss": 4.465688435057938, + "tokens_seen": 172621824 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765801404212638, + "loss": 1.6552, + "theoretical_loss": 4.46547287183833, + "tokens_seen": 172687360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657853560682046, + "loss": 1.5746, + "theoretical_loss": 4.465257413307119, + "tokens_seen": 172752896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657693079237714, + "loss": 1.7696, + "theoretical_loss": 4.465042059373776, + "tokens_seen": 172818432 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765753259779338, + "loss": 1.7059, + "theoretical_loss": 4.464826809947887, + "tokens_seen": 172883968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657372116349048, + "loss": 1.7048, + "theoretical_loss": 4.464611664939149, + "tokens_seen": 172949504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657211634904714, + "loss": 1.6989, + "theoretical_loss": 4.46439662425737, + "tokens_seen": 173015040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657051153460382, + "loss": 1.6436, + "theoretical_loss": 4.464181687812474, + "tokens_seen": 173080576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656890672016048, + "loss": 1.7972, + "theoretical_loss": 4.4639668555144905, + "tokens_seen": 173146112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656730190571716, + "loss": 1.7853, + "theoretical_loss": 4.463752127273567, + "tokens_seen": 173211648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656569709127382, + "loss": 1.795, + "theoretical_loss": 4.463537502999959, + "tokens_seen": 173277184 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765640922768305, + "loss": 1.6484, + "theoretical_loss": 4.4633229826040335, + "tokens_seen": 173342720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656248746238716, + "loss": 1.6078, + "theoretical_loss": 4.46310856599627, + "tokens_seen": 173408256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656088264794384, + "loss": 1.7552, + "theoretical_loss": 4.462894253087258, + "tokens_seen": 173473792 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765592778335005, + "loss": 1.8263, + "theoretical_loss": 4.462680043787699, + "tokens_seen": 173539328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655767301905718, + "loss": 1.794, + "theoretical_loss": 4.4624659380084015, + "tokens_seen": 173604864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 64355, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7965178489685059, + "objective/train/theoretical_loss": 4.462251935660287, + "objective/train/tokens_used": 194130400, + "theoretical_loss": 4.462251935660287, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655606820461384, + "loss": 1.6888, + "theoretical_loss": 4.462251935660287, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655446339017052, + "loss": 1.722, + "theoretical_loss": 4.46203803665439, + "tokens_seen": 173735936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655285857572718, + "loss": 1.8336, + "theoretical_loss": 4.4618242409018505, + "tokens_seen": 173801472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655125376128386, + "loss": 1.7571, + "theoretical_loss": 4.461610548313919, + "tokens_seen": 173867008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654964894684052, + "loss": 1.7694, + "theoretical_loss": 4.461396958801959, + "tokens_seen": 173932544 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765480441323972, + "loss": 1.7376, + "theoretical_loss": 4.461183472277439, + "tokens_seen": 173998080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654643931795386, + "loss": 1.6581, + "theoretical_loss": 4.460970088651941, + "tokens_seen": 174063616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654483450351054, + "loss": 1.6625, + "theoretical_loss": 4.460756807837154, + "tokens_seen": 174129152 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765432296890672, + "loss": 1.6933, + "theoretical_loss": 4.460543629744874, + "tokens_seen": 174194688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654162487462388, + "loss": 1.4361, + "theoretical_loss": 4.460330554287011, + "tokens_seen": 174260224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654002006018054, + "loss": 1.7742, + "theoretical_loss": 4.460117581375579, + "tokens_seen": 174325760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653841524573722, + "loss": 1.6683, + "theoretical_loss": 4.459904710922704, + "tokens_seen": 174391296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653681043129388, + "loss": 1.7873, + "theoretical_loss": 4.459691942840616, + "tokens_seen": 174456832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653520561685056, + "loss": 1.7836, + "theoretical_loss": 4.459479277041657, + "tokens_seen": 174522368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653360080240722, + "loss": 1.616, + "theoretical_loss": 4.459266713438277, + "tokens_seen": 174587904 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765319959879639, + "loss": 1.7366, + "theoretical_loss": 4.459054251943029, + "tokens_seen": 174653440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653039117352056, + "loss": 1.7021, + "theoretical_loss": 4.4588418924685795, + "tokens_seen": 174718976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652878635907724, + "loss": 1.9371, + "theoretical_loss": 4.458629634927698, + "tokens_seen": 174784512 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765271815446339, + "loss": 1.6323, + "theoretical_loss": 4.458417479233265, + "tokens_seen": 174850048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652557673019058, + "loss": 1.7393, + "theoretical_loss": 4.4582054252982655, + "tokens_seen": 174915584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652397191574724, + "loss": 1.7396, + "theoretical_loss": 4.457993473035792, + "tokens_seen": 174981120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652236710130392, + "loss": 1.6274, + "theoretical_loss": 4.457781622359043, + "tokens_seen": 175046656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652076228686058, + "loss": 1.6795, + "theoretical_loss": 4.457569873181326, + "tokens_seen": 175112192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651915747241726, + "loss": 1.7442, + "theoretical_loss": 4.457358225416052, + "tokens_seen": 175177728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651755265797392, + "loss": 1.6656, + "theoretical_loss": 4.4571466789767396, + "tokens_seen": 175243264 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765159478435306, + "loss": 1.6225, + "theoretical_loss": 4.456935233777013, + "tokens_seen": 175308800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651434302908727, + "loss": 1.7749, + "theoretical_loss": 4.456723889730605, + "tokens_seen": 175374336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651273821464394, + "loss": 1.7267, + "theoretical_loss": 4.456512646751349, + "tokens_seen": 175439872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651113340020061, + "loss": 1.7734, + "theoretical_loss": 4.456301504753188, + "tokens_seen": 175505408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650952858575728, + "loss": 1.7111, + "theoretical_loss": 4.456090463650169, + "tokens_seen": 175570944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650792377131395, + "loss": 1.6561, + "theoretical_loss": 4.455879523356444, + "tokens_seen": 175636480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650631895687062, + "loss": 1.77, + "theoretical_loss": 4.4556686837862705, + "tokens_seen": 175702016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650471414242729, + "loss": 1.6699, + "theoretical_loss": 4.455457944854011, + "tokens_seen": 175767552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650310932798396, + "loss": 1.7107, + "theoretical_loss": 4.455247306474131, + "tokens_seen": 175833088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650150451354063, + "loss": 1.8197, + "theoretical_loss": 4.455036768561204, + "tokens_seen": 175898624 + }, + { + "epoch": 0.05, + "learning_rate": 0.000764998996990973, + "loss": 1.7647, + "theoretical_loss": 4.4548263310299046, + "tokens_seen": 175964160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649829488465397, + "loss": 1.7439, + "theoretical_loss": 4.454615993795015, + "tokens_seen": 176029696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649669007021064, + "loss": 1.6774, + "theoretical_loss": 4.454405756771415, + "tokens_seen": 176095232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649508525576731, + "loss": 1.822, + "theoretical_loss": 4.454195619874096, + "tokens_seen": 176160768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649348044132398, + "loss": 1.7231, + "theoretical_loss": 4.453985583018149, + "tokens_seen": 176226304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649187562688065, + "loss": 1.7635, + "theoretical_loss": 4.4537756461187685, + "tokens_seen": 176291840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649027081243732, + "loss": 1.7327, + "theoretical_loss": 4.453565809091254, + "tokens_seen": 176357376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648866599799399, + "loss": 1.7449, + "theoretical_loss": 4.453356071851006, + "tokens_seen": 176422912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648706118355066, + "loss": 1.7211, + "theoretical_loss": 4.453146434313531, + "tokens_seen": 176488448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648545636910732, + "loss": 1.851, + "theoretical_loss": 4.452936896394435, + "tokens_seen": 176553984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00076483851554664, + "loss": 1.7872, + "theoretical_loss": 4.452727458009431, + "tokens_seen": 176619520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648224674022066, + "loss": 1.7427, + "theoretical_loss": 4.45251811907433, + "tokens_seen": 176685056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648064192577734, + "loss": 1.6532, + "theoretical_loss": 4.452308879505047, + "tokens_seen": 176750592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00076479037111334, + "loss": 1.8032, + "theoretical_loss": 4.452099739217601, + "tokens_seen": 176816128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647743229689068, + "loss": 1.6829, + "theoretical_loss": 4.451890698128112, + "tokens_seen": 176881664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 65665, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5994759798049927, + "objective/train/theoretical_loss": 4.4516817561528015, + "objective/train/tokens_used": 197407200, + "theoretical_loss": 4.4516817561528015, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647582748244734, + "loss": 1.6761, + "theoretical_loss": 4.4516817561528015, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647422266800402, + "loss": 1.6801, + "theoretical_loss": 4.451472913207992, + "tokens_seen": 177012736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647261785356068, + "loss": 1.7034, + "theoretical_loss": 4.451264169210107, + "tokens_seen": 177078272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647101303911735, + "loss": 1.651, + "theoretical_loss": 4.451055524075677, + "tokens_seen": 177143808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646940822467402, + "loss": 1.6878, + "theoretical_loss": 4.450846977721326, + "tokens_seen": 177209344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646780341023069, + "loss": 1.7719, + "theoretical_loss": 4.450638530063786, + "tokens_seen": 177274880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646619859578736, + "loss": 1.7237, + "theoretical_loss": 4.450430181019884, + "tokens_seen": 177340416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646459378134403, + "loss": 1.7421, + "theoretical_loss": 4.450221930506553, + "tokens_seen": 177405952 + }, + { + "epoch": 0.05, + "learning_rate": 0.000764629889669007, + "loss": 1.6159, + "theoretical_loss": 4.450013778440822, + "tokens_seen": 177471488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646138415245737, + "loss": 1.6349, + "theoretical_loss": 4.449805724739824, + "tokens_seen": 177537024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645977933801404, + "loss": 1.6987, + "theoretical_loss": 4.449597769320791, + "tokens_seen": 177602560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645817452357071, + "loss": 1.7311, + "theoretical_loss": 4.4493899121010525, + "tokens_seen": 177668096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645656970912738, + "loss": 1.7738, + "theoretical_loss": 4.449182152998044, + "tokens_seen": 177733632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645496489468405, + "loss": 1.6177, + "theoretical_loss": 4.448974491929297, + "tokens_seen": 177799168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645336008024072, + "loss": 1.8036, + "theoretical_loss": 4.448766928812441, + "tokens_seen": 177864704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645175526579739, + "loss": 1.7037, + "theoretical_loss": 4.448559463565209, + "tokens_seen": 177930240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645015045135406, + "loss": 1.6341, + "theoretical_loss": 4.44835209610543, + "tokens_seen": 177995776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644854563691073, + "loss": 1.7024, + "theoretical_loss": 4.448144826351036, + "tokens_seen": 178061312 + }, + { + "epoch": 0.05, + "learning_rate": 0.000764469408224674, + "loss": 1.6771, + "theoretical_loss": 4.447937654220054, + "tokens_seen": 178126848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644533600802407, + "loss": 1.7449, + "theoretical_loss": 4.447730579630612, + "tokens_seen": 178192384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644373119358075, + "loss": 1.5651, + "theoretical_loss": 4.447523602500936, + "tokens_seen": 178257920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644212637913741, + "loss": 1.6465, + "theoretical_loss": 4.447316722749351, + "tokens_seen": 178323456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644052156469409, + "loss": 1.6778, + "theoretical_loss": 4.447109940294283, + "tokens_seen": 178388992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643891675025075, + "loss": 1.7214, + "theoretical_loss": 4.44690325505425, + "tokens_seen": 178454528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643731193580743, + "loss": 1.6484, + "theoretical_loss": 4.446696666947874, + "tokens_seen": 178520064 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643570712136409, + "loss": 1.733, + "theoretical_loss": 4.446490175893872, + "tokens_seen": 178585600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643410230692077, + "loss": 1.6846, + "theoretical_loss": 4.44628378181106, + "tokens_seen": 178651136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643249749247743, + "loss": 1.7986, + "theoretical_loss": 4.4460774846183515, + "tokens_seen": 178716672 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643089267803411, + "loss": 1.6765, + "theoretical_loss": 4.445871284234757, + "tokens_seen": 178782208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642928786359077, + "loss": 1.8968, + "theoretical_loss": 4.445665180579383, + "tokens_seen": 178847744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642768304914745, + "loss": 1.6697, + "theoretical_loss": 4.445459173571438, + "tokens_seen": 178913280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642607823470411, + "loss": 1.731, + "theoretical_loss": 4.445253263130223, + "tokens_seen": 178978816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642447342026079, + "loss": 1.668, + "theoretical_loss": 4.445047449175136, + "tokens_seen": 179044352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642286860581745, + "loss": 1.6233, + "theoretical_loss": 4.444841731625674, + "tokens_seen": 179109888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642126379137413, + "loss": 1.8112, + "theoretical_loss": 4.444636110401431, + "tokens_seen": 179175424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641965897693079, + "loss": 1.7487, + "theoretical_loss": 4.444430585422094, + "tokens_seen": 179240960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641805416248747, + "loss": 1.5881, + "theoretical_loss": 4.444225156607449, + "tokens_seen": 179306496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641644934804413, + "loss": 1.7401, + "theoretical_loss": 4.444019823877378, + "tokens_seen": 179372032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641484453360081, + "loss": 1.6739, + "theoretical_loss": 4.4438145871518575, + "tokens_seen": 179437568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641323971915747, + "loss": 1.7123, + "theoretical_loss": 4.443609446350962, + "tokens_seen": 179503104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641163490471415, + "loss": 1.697, + "theoretical_loss": 4.44340440139486, + "tokens_seen": 179568640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641003009027081, + "loss": 1.7086, + "theoretical_loss": 4.443199452203816, + "tokens_seen": 179634176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640842527582749, + "loss": 1.6203, + "theoretical_loss": 4.442994598698189, + "tokens_seen": 179699712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640682046138415, + "loss": 1.7007, + "theoretical_loss": 4.4427898407984365, + "tokens_seen": 179765248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640521564694083, + "loss": 1.6982, + "theoretical_loss": 4.442585178425107, + "tokens_seen": 179830784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640361083249749, + "loss": 1.6712, + "theoretical_loss": 4.442380611498846, + "tokens_seen": 179896320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640200601805417, + "loss": 1.7155, + "theoretical_loss": 4.442176139940393, + "tokens_seen": 179961856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640040120361083, + "loss": 1.8174, + "theoretical_loss": 4.441971763670585, + "tokens_seen": 180027392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639879638916751, + "loss": 1.8372, + "theoretical_loss": 4.441767482610349, + "tokens_seen": 180092928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639719157472417, + "loss": 1.7136, + "theoretical_loss": 4.441563296680709, + "tokens_seen": 180158464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 66635, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7017289400100708, + "objective/train/theoretical_loss": 4.441359205802783, + "objective/train/tokens_used": 200684000, + "theoretical_loss": 4.441359205802783, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639558676028085, + "loss": 1.6671, + "theoretical_loss": 4.441359205802783, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639398194583752, + "loss": 1.74, + "theoretical_loss": 4.441155209897782, + "tokens_seen": 180289536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639237713139419, + "loss": 1.7344, + "theoretical_loss": 4.440951308887014, + "tokens_seen": 180355072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639077231695086, + "loss": 1.7618, + "theoretical_loss": 4.440747502691876, + "tokens_seen": 180420608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638916750250753, + "loss": 1.7888, + "theoretical_loss": 4.440543791233862, + "tokens_seen": 180486144 + }, + { + "epoch": 0.05, + "learning_rate": 0.000763875626880642, + "loss": 1.6206, + "theoretical_loss": 4.440340174434558, + "tokens_seen": 180551680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638595787362087, + "loss": 1.7611, + "theoretical_loss": 4.440136652215646, + "tokens_seen": 180617216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638435305917754, + "loss": 1.7102, + "theoretical_loss": 4.439933224498898, + "tokens_seen": 180682752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638274824473421, + "loss": 1.7108, + "theoretical_loss": 4.43972989120618, + "tokens_seen": 180748288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638114343029088, + "loss": 1.8015, + "theoretical_loss": 4.439526652259451, + "tokens_seen": 180813824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637953861584755, + "loss": 1.6381, + "theoretical_loss": 4.439323507580761, + "tokens_seen": 180879360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637793380140422, + "loss": 1.633, + "theoretical_loss": 4.439120457092258, + "tokens_seen": 180944896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637632898696089, + "loss": 1.6394, + "theoretical_loss": 4.438917500716178, + "tokens_seen": 181010432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637472417251756, + "loss": 1.658, + "theoretical_loss": 4.438714638374849, + "tokens_seen": 181075968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637311935807423, + "loss": 1.7647, + "theoretical_loss": 4.438511869990693, + "tokens_seen": 181141504 + }, + { + "epoch": 0.05, + "learning_rate": 0.000763715145436309, + "loss": 1.7654, + "theoretical_loss": 4.438309195486223, + "tokens_seen": 181207040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007636990972918757, + "loss": 1.7345, + "theoretical_loss": 4.438106614784045, + "tokens_seen": 181272576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007636830491474424, + "loss": 1.7755, + "theoretical_loss": 4.437904127806856, + "tokens_seen": 181338112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007636670010030091, + "loss": 1.7887, + "theoretical_loss": 4.437701734477445, + "tokens_seen": 181403648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007636509528585758, + "loss": 1.7479, + "theoretical_loss": 4.437499434718692, + "tokens_seen": 181469184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007636349047141425, + "loss": 1.8136, + "theoretical_loss": 4.437297228453567, + "tokens_seen": 181534720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007636188565697092, + "loss": 1.6026, + "theoretical_loss": 4.4370951156051355, + "tokens_seen": 181600256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007636028084252759, + "loss": 1.7126, + "theoretical_loss": 4.43689309609655, + "tokens_seen": 181665792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635867602808426, + "loss": 1.7019, + "theoretical_loss": 4.436691169851055, + "tokens_seen": 181731328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635707121364093, + "loss": 1.5814, + "theoretical_loss": 4.436489336791985, + "tokens_seen": 181796864 + }, + { + "epoch": 0.06, + "learning_rate": 0.000763554663991976, + "loss": 1.7454, + "theoretical_loss": 4.436287596842769, + "tokens_seen": 181862400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635386158475427, + "loss": 1.6315, + "theoretical_loss": 4.4360859499269205, + "tokens_seen": 181927936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635225677031094, + "loss": 1.7988, + "theoretical_loss": 4.435884395968047, + "tokens_seen": 181993472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635065195586761, + "loss": 1.763, + "theoretical_loss": 4.435682934889847, + "tokens_seen": 182059008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634904714142428, + "loss": 1.7406, + "theoretical_loss": 4.435481566616108, + "tokens_seen": 182124544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634744232698095, + "loss": 1.605, + "theoretical_loss": 4.435280291070705, + "tokens_seen": 182190080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634583751253762, + "loss": 1.7259, + "theoretical_loss": 4.435079108177606, + "tokens_seen": 182255616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634423269809429, + "loss": 1.7148, + "theoretical_loss": 4.434878017860869, + "tokens_seen": 182321152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634262788365095, + "loss": 1.752, + "theoretical_loss": 4.434677020044637, + "tokens_seen": 182386688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634102306920763, + "loss": 1.7357, + "theoretical_loss": 4.4344761146531475, + "tokens_seen": 182452224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633941825476429, + "loss": 1.712, + "theoretical_loss": 4.434275301610725, + "tokens_seen": 182517760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633781344032097, + "loss": 1.7025, + "theoretical_loss": 4.434074580841784, + "tokens_seen": 182583296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633620862587763, + "loss": 1.6831, + "theoretical_loss": 4.433873952270826, + "tokens_seen": 182648832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633460381143431, + "loss": 1.6822, + "theoretical_loss": 4.433673415822443, + "tokens_seen": 182714368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633299899699097, + "loss": 1.6375, + "theoretical_loss": 4.433472971421315, + "tokens_seen": 182779904 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633139418254764, + "loss": 1.7446, + "theoretical_loss": 4.433272618992213, + "tokens_seen": 182845440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632978936810431, + "loss": 1.6624, + "theoretical_loss": 4.433072358459991, + "tokens_seen": 182910976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632818455366098, + "loss": 1.667, + "theoretical_loss": 4.432872189749596, + "tokens_seen": 182976512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632657973921765, + "loss": 1.6365, + "theoretical_loss": 4.432672112786063, + "tokens_seen": 183042048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632497492477432, + "loss": 1.7046, + "theoretical_loss": 4.4324721274945125, + "tokens_seen": 183107584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632337011033099, + "loss": 1.6919, + "theoretical_loss": 4.432272233800154, + "tokens_seen": 183173120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632176529588766, + "loss": 1.6129, + "theoretical_loss": 4.432072431628286, + "tokens_seen": 183238656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632016048144433, + "loss": 1.6262, + "theoretical_loss": 4.431872720904292, + "tokens_seen": 183304192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00076318555667001, + "loss": 1.6003, + "theoretical_loss": 4.4316731015536455, + "tokens_seen": 183369728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631695085255767, + "loss": 1.6435, + "theoretical_loss": 4.431473573501906, + "tokens_seen": 183435264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 67570, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8887503147125244, + "objective/train/theoretical_loss": 4.431274136674721, + "objective/train/tokens_used": 203960800, + "theoretical_loss": 4.431274136674721, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631534603811434, + "loss": 1.8266, + "theoretical_loss": 4.431274136674721, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631374122367101, + "loss": 1.6985, + "theoretical_loss": 4.431074790997823, + "tokens_seen": 183566336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631213640922768, + "loss": 1.7259, + "theoretical_loss": 4.430875536397036, + "tokens_seen": 183631872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631053159478436, + "loss": 1.7227, + "theoretical_loss": 4.430676372798265, + "tokens_seen": 183697408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007630892678034102, + "loss": 1.8484, + "theoretical_loss": 4.430477300127507, + "tokens_seen": 183762944 + }, + { + "epoch": 0.06, + "learning_rate": 0.000763073219658977, + "loss": 1.7766, + "theoretical_loss": 4.43027831831084, + "tokens_seen": 183828480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007630571715145436, + "loss": 1.7096, + "theoretical_loss": 4.430079427274435, + "tokens_seen": 183894016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007630411233701104, + "loss": 1.6482, + "theoretical_loss": 4.429880626944543, + "tokens_seen": 183959552 + }, + { + "epoch": 0.06, + "learning_rate": 0.000763025075225677, + "loss": 1.7298, + "theoretical_loss": 4.429681917247506, + "tokens_seen": 184025088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007630090270812438, + "loss": 1.7268, + "theoretical_loss": 4.429483298109749, + "tokens_seen": 184090624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629929789368104, + "loss": 1.6922, + "theoretical_loss": 4.429284769457784, + "tokens_seen": 184156160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629769307923772, + "loss": 1.5606, + "theoretical_loss": 4.429086331218208, + "tokens_seen": 184221696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629608826479438, + "loss": 1.6067, + "theoretical_loss": 4.428887983317706, + "tokens_seen": 184287232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629448345035106, + "loss": 1.7639, + "theoretical_loss": 4.4286897256830455, + "tokens_seen": 184352768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629287863590772, + "loss": 1.7258, + "theoretical_loss": 4.428491558241081, + "tokens_seen": 184418304 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762912738214644, + "loss": 1.6622, + "theoretical_loss": 4.428293480918751, + "tokens_seen": 184483840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628966900702106, + "loss": 1.7358, + "theoretical_loss": 4.428095493643081, + "tokens_seen": 184549376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628806419257774, + "loss": 1.6846, + "theoretical_loss": 4.427897596341181, + "tokens_seen": 184614912 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762864593781344, + "loss": 1.6989, + "theoretical_loss": 4.427699788940243, + "tokens_seen": 184680448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628485456369108, + "loss": 1.6336, + "theoretical_loss": 4.427502071367549, + "tokens_seen": 184745984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628324974924774, + "loss": 1.684, + "theoretical_loss": 4.42730444355046, + "tokens_seen": 184811520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628164493480442, + "loss": 1.7098, + "theoretical_loss": 4.4271069054164265, + "tokens_seen": 184877056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628004012036108, + "loss": 1.7049, + "theoretical_loss": 4.426909456892979, + "tokens_seen": 184942592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627843530591776, + "loss": 1.6428, + "theoretical_loss": 4.426712097907735, + "tokens_seen": 185008128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627683049147442, + "loss": 1.6611, + "theoretical_loss": 4.426514828388394, + "tokens_seen": 185073664 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762752256770311, + "loss": 1.5817, + "theoretical_loss": 4.426317648262743, + "tokens_seen": 185139200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627362086258777, + "loss": 1.6968, + "theoretical_loss": 4.426120557458649, + "tokens_seen": 185204736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627201604814444, + "loss": 1.6527, + "theoretical_loss": 4.425923555904065, + "tokens_seen": 185270272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627041123370111, + "loss": 1.6712, + "theoretical_loss": 4.4257266435270255, + "tokens_seen": 185335808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626880641925778, + "loss": 1.622, + "theoretical_loss": 4.425529820255651, + "tokens_seen": 185401344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626720160481445, + "loss": 1.7908, + "theoretical_loss": 4.425333086018144, + "tokens_seen": 185466880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626559679037112, + "loss": 1.7091, + "theoretical_loss": 4.425136440742788, + "tokens_seen": 185532416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626399197592779, + "loss": 1.6781, + "theoretical_loss": 4.424939884357956, + "tokens_seen": 185597952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626238716148446, + "loss": 1.697, + "theoretical_loss": 4.424743416792096, + "tokens_seen": 185663488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626078234704113, + "loss": 1.6791, + "theoretical_loss": 4.424547037973745, + "tokens_seen": 185729024 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762591775325978, + "loss": 1.699, + "theoretical_loss": 4.424350747831518, + "tokens_seen": 185794560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625757271815447, + "loss": 1.7917, + "theoretical_loss": 4.424154546294117, + "tokens_seen": 185860096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625596790371114, + "loss": 1.707, + "theoretical_loss": 4.4239584332903235, + "tokens_seen": 185925632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625436308926781, + "loss": 1.6794, + "theoretical_loss": 4.423762408749001, + "tokens_seen": 185991168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625275827482448, + "loss": 1.6936, + "theoretical_loss": 4.423566472599099, + "tokens_seen": 186056704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625115346038115, + "loss": 1.8014, + "theoretical_loss": 4.423370624769643, + "tokens_seen": 186122240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624954864593782, + "loss": 1.7071, + "theoretical_loss": 4.423174865189747, + "tokens_seen": 186187776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624794383149449, + "loss": 1.6442, + "theoretical_loss": 4.422979193788601, + "tokens_seen": 186253312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624633901705116, + "loss": 1.5889, + "theoretical_loss": 4.422783610495481, + "tokens_seen": 186318848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624473420260783, + "loss": 1.6439, + "theoretical_loss": 4.422588115239742, + "tokens_seen": 186384384 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762431293881645, + "loss": 1.7129, + "theoretical_loss": 4.422392707950824, + "tokens_seen": 186449920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624152457372117, + "loss": 1.7146, + "theoretical_loss": 4.42219738855824, + "tokens_seen": 186515456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623991975927784, + "loss": 1.663, + "theoretical_loss": 4.422002156991597, + "tokens_seen": 186580992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623831494483451, + "loss": 1.6474, + "theoretical_loss": 4.421807013180571, + "tokens_seen": 186646528 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623671013039118, + "loss": 1.7353, + "theoretical_loss": 4.421611957054925, + "tokens_seen": 186712064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 68808, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6501104831695557, + "objective/train/theoretical_loss": 4.421416988544503, + "objective/train/tokens_used": 207237600, + "theoretical_loss": 4.421416988544503, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623510531594785, + "loss": 1.7163, + "theoretical_loss": 4.421416988544503, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623350050150452, + "loss": 1.7814, + "theoretical_loss": 4.4212221075792275, + "tokens_seen": 186843136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623189568706119, + "loss": 1.7138, + "theoretical_loss": 4.421027314089103, + "tokens_seen": 186908672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623029087261786, + "loss": 1.6188, + "theoretical_loss": 4.420832608004215, + "tokens_seen": 186974208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622868605817453, + "loss": 1.6911, + "theoretical_loss": 4.420637989254726, + "tokens_seen": 187039744 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762270812437312, + "loss": 1.6176, + "theoretical_loss": 4.420443457770885, + "tokens_seen": 187105280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622547642928787, + "loss": 1.7182, + "theoretical_loss": 4.420249013483014, + "tokens_seen": 187170816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622387161484454, + "loss": 1.7003, + "theoretical_loss": 4.42005465632152, + "tokens_seen": 187236352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622226680040121, + "loss": 1.7183, + "theoretical_loss": 4.419860386216888, + "tokens_seen": 187301888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622066198595788, + "loss": 1.737, + "theoretical_loss": 4.419666203099683, + "tokens_seen": 187367424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621905717151455, + "loss": 1.7014, + "theoretical_loss": 4.41947210690055, + "tokens_seen": 187432960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621745235707122, + "loss": 1.7964, + "theoretical_loss": 4.419278097550212, + "tokens_seen": 187498496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621584754262789, + "loss": 1.6903, + "theoretical_loss": 4.419084174979473, + "tokens_seen": 187564032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621424272818456, + "loss": 1.7771, + "theoretical_loss": 4.4188903391192165, + "tokens_seen": 187629568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621263791374123, + "loss": 1.7877, + "theoretical_loss": 4.418696589900405, + "tokens_seen": 187695104 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762110330992979, + "loss": 1.7553, + "theoretical_loss": 4.418502927254078, + "tokens_seen": 187760640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620942828485457, + "loss": 1.7333, + "theoretical_loss": 4.418309351111356, + "tokens_seen": 187826176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620782347041124, + "loss": 1.6975, + "theoretical_loss": 4.418115861403439, + "tokens_seen": 187891712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620621865596791, + "loss": 1.5968, + "theoretical_loss": 4.417922458061603, + "tokens_seen": 187957248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620461384152458, + "loss": 1.6263, + "theoretical_loss": 4.417729141017205, + "tokens_seen": 188022784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620300902708124, + "loss": 1.7959, + "theoretical_loss": 4.4175359102016785, + "tokens_seen": 188088320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620140421263792, + "loss": 1.7892, + "theoretical_loss": 4.417342765546538, + "tokens_seen": 188153856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619979939819458, + "loss": 1.75, + "theoretical_loss": 4.417149706983373, + "tokens_seen": 188219392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619819458375126, + "loss": 1.6113, + "theoretical_loss": 4.416956734443854, + "tokens_seen": 188284928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619658976930792, + "loss": 1.5337, + "theoretical_loss": 4.416763847859728, + "tokens_seen": 188350464 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761949849548646, + "loss": 1.6184, + "theoretical_loss": 4.41657104716282, + "tokens_seen": 188416000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619338014042126, + "loss": 1.7843, + "theoretical_loss": 4.416378332285031, + "tokens_seen": 188481536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619177532597794, + "loss": 1.6608, + "theoretical_loss": 4.4161857031583445, + "tokens_seen": 188547072 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761901705115346, + "loss": 1.7039, + "theoretical_loss": 4.415993159714818, + "tokens_seen": 188612608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618856569709127, + "loss": 1.8617, + "theoretical_loss": 4.415800701886584, + "tokens_seen": 188678144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618696088264794, + "loss": 1.813, + "theoretical_loss": 4.415608329605859, + "tokens_seen": 188743680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618535606820461, + "loss": 1.5579, + "theoretical_loss": 4.415416042804931, + "tokens_seen": 188809216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618375125376128, + "loss": 1.6717, + "theoretical_loss": 4.415223841416167, + "tokens_seen": 188874752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618214643931795, + "loss": 1.7537, + "theoretical_loss": 4.415031725372011, + "tokens_seen": 188940288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618054162487462, + "loss": 1.6978, + "theoretical_loss": 4.414839694604985, + "tokens_seen": 189005824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617893681043129, + "loss": 1.6423, + "theoretical_loss": 4.414647749047685, + "tokens_seen": 189071360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617733199598796, + "loss": 1.6646, + "theoretical_loss": 4.414455888632786, + "tokens_seen": 189136896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617572718154463, + "loss": 1.7303, + "theoretical_loss": 4.414264113293038, + "tokens_seen": 189202432 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761741223671013, + "loss": 1.7068, + "theoretical_loss": 4.41407242296127, + "tokens_seen": 189267968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617251755265797, + "loss": 1.6995, + "theoretical_loss": 4.413880817570382, + "tokens_seen": 189333504 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617091273821465, + "loss": 1.6645, + "theoretical_loss": 4.413689297053356, + "tokens_seen": 189399040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616930792377131, + "loss": 1.6716, + "theoretical_loss": 4.413497861343247, + "tokens_seen": 189464576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616770310932799, + "loss": 1.6816, + "theoretical_loss": 4.413306510373185, + "tokens_seen": 189530112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616609829488465, + "loss": 1.62, + "theoretical_loss": 4.41311524407638, + "tokens_seen": 189595648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616449348044133, + "loss": 1.715, + "theoretical_loss": 4.412924062386112, + "tokens_seen": 189661184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616288866599799, + "loss": 1.8238, + "theoretical_loss": 4.412732965235742, + "tokens_seen": 189726720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616128385155467, + "loss": 1.8357, + "theoretical_loss": 4.412541952558703, + "tokens_seen": 189792256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615967903711133, + "loss": 1.6682, + "theoretical_loss": 4.412351024288504, + "tokens_seen": 189857792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615807422266801, + "loss": 1.6815, + "theoretical_loss": 4.412160180358731, + "tokens_seen": 189923328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615646940822467, + "loss": 1.6328, + "theoretical_loss": 4.411969420703042, + "tokens_seen": 189988864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 70105, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.612324595451355, + "objective/train/theoretical_loss": 4.411778745255174, + "objective/train/tokens_used": 210514400, + "theoretical_loss": 4.411778745255174, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615486459378135, + "loss": 1.6841, + "theoretical_loss": 4.411778745255174, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615325977933802, + "loss": 1.7375, + "theoretical_loss": 4.411588153948935, + "tokens_seen": 190119936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615165496489469, + "loss": 1.8672, + "theoretical_loss": 4.411397646718211, + "tokens_seen": 190185472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615005015045136, + "loss": 1.6434, + "theoretical_loss": 4.411207223496962, + "tokens_seen": 190251008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614844533600803, + "loss": 1.6167, + "theoretical_loss": 4.41101688421922, + "tokens_seen": 190316544 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761468405215647, + "loss": 1.7425, + "theoretical_loss": 4.410826628819094, + "tokens_seen": 190382080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614523570712137, + "loss": 1.6488, + "theoretical_loss": 4.410636457230767, + "tokens_seen": 190447616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614363089267804, + "loss": 1.6163, + "theoretical_loss": 4.410446369388497, + "tokens_seen": 190513152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614202607823471, + "loss": 1.7284, + "theoretical_loss": 4.410256365226614, + "tokens_seen": 190578688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614042126379138, + "loss": 1.7268, + "theoretical_loss": 4.410066444679524, + "tokens_seen": 190644224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613881644934805, + "loss": 1.7091, + "theoretical_loss": 4.409876607681706, + "tokens_seen": 190709760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613721163490472, + "loss": 1.5607, + "theoretical_loss": 4.409686854167714, + "tokens_seen": 190775296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613560682046139, + "loss": 1.6649, + "theoretical_loss": 4.4094971840721735, + "tokens_seen": 190840832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613400200601806, + "loss": 1.7259, + "theoretical_loss": 4.409307597329786, + "tokens_seen": 190906368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613239719157473, + "loss": 1.8349, + "theoretical_loss": 4.409118093875324, + "tokens_seen": 190971904 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761307923771314, + "loss": 1.7556, + "theoretical_loss": 4.408928673643636, + "tokens_seen": 191037440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612918756268807, + "loss": 1.8219, + "theoretical_loss": 4.408739336569644, + "tokens_seen": 191102976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612758274824474, + "loss": 1.5849, + "theoretical_loss": 4.408550082588339, + "tokens_seen": 191168512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612597793380141, + "loss": 1.716, + "theoretical_loss": 4.408360911634789, + "tokens_seen": 191234048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612437311935808, + "loss": 1.6816, + "theoretical_loss": 4.408171823644135, + "tokens_seen": 191299584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612276830491475, + "loss": 1.7805, + "theoretical_loss": 4.407982818551589, + "tokens_seen": 191365120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612116349047142, + "loss": 1.822, + "theoretical_loss": 4.407793896292437, + "tokens_seen": 191430656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611955867602809, + "loss": 1.7945, + "theoretical_loss": 4.407605056802035, + "tokens_seen": 191496192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611795386158476, + "loss": 1.7431, + "theoretical_loss": 4.407416300015816, + "tokens_seen": 191561728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611634904714143, + "loss": 1.562, + "theoretical_loss": 4.407227625869283, + "tokens_seen": 191627264 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761147442326981, + "loss": 1.7065, + "theoretical_loss": 4.407039034298011, + "tokens_seen": 191692800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611313941825477, + "loss": 1.55, + "theoretical_loss": 4.4068505252376475, + "tokens_seen": 191758336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611153460381144, + "loss": 1.8253, + "theoretical_loss": 4.406662098623913, + "tokens_seen": 191823872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610992978936811, + "loss": 1.5841, + "theoretical_loss": 4.406473754392598, + "tokens_seen": 191889408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610832497492478, + "loss": 1.7049, + "theoretical_loss": 4.406285492479569, + "tokens_seen": 191954944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610672016048145, + "loss": 1.7289, + "theoretical_loss": 4.406097312820759, + "tokens_seen": 192020480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610511534603812, + "loss": 1.5932, + "theoretical_loss": 4.4059092153521755, + "tokens_seen": 192086016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610351053159479, + "loss": 1.7163, + "theoretical_loss": 4.405721200009898, + "tokens_seen": 192151552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610190571715146, + "loss": 1.7145, + "theoretical_loss": 4.405533266730077, + "tokens_seen": 192217088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610030090270813, + "loss": 1.7585, + "theoretical_loss": 4.405345415448934, + "tokens_seen": 192282624 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760986960882648, + "loss": 1.809, + "theoretical_loss": 4.405157646102762, + "tokens_seen": 192348160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609709127382147, + "loss": 1.6969, + "theoretical_loss": 4.404969958627927, + "tokens_seen": 192413696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609548645937814, + "loss": 1.6836, + "theoretical_loss": 4.4047823529608605, + "tokens_seen": 192479232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609388164493481, + "loss": 1.6517, + "theoretical_loss": 4.404594829038071, + "tokens_seen": 192544768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609227683049148, + "loss": 1.8073, + "theoretical_loss": 4.404407386796137, + "tokens_seen": 192610304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609067201604815, + "loss": 1.6976, + "theoretical_loss": 4.404220026171704, + "tokens_seen": 192675840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608906720160482, + "loss": 1.8556, + "theoretical_loss": 4.404032747101493, + "tokens_seen": 192741376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608746238716149, + "loss": 1.6249, + "theoretical_loss": 4.4038455495222895, + "tokens_seen": 192806912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608585757271816, + "loss": 1.7009, + "theoretical_loss": 4.403658433370957, + "tokens_seen": 192872448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608425275827483, + "loss": 1.6373, + "theoretical_loss": 4.403471398584422, + "tokens_seen": 192937984 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760826479438315, + "loss": 1.6357, + "theoretical_loss": 4.403284445099688, + "tokens_seen": 193003520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608104312938817, + "loss": 1.7354, + "theoretical_loss": 4.403097572853823, + "tokens_seen": 193069056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607943831494484, + "loss": 1.7761, + "theoretical_loss": 4.402910781783968, + "tokens_seen": 193134592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607783350050151, + "loss": 1.7687, + "theoretical_loss": 4.4027240718273335, + "tokens_seen": 193200128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607622868605818, + "loss": 1.6265, + "theoretical_loss": 4.4025374429212, + "tokens_seen": 193265664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 71032, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6292566061019897, + "objective/train/theoretical_loss": 4.402350895002916, + "objective/train/tokens_used": 213791200, + "theoretical_loss": 4.402350895002916, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607462387161485, + "loss": 1.6732, + "theoretical_loss": 4.402350895002916, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607301905717152, + "loss": 1.7352, + "theoretical_loss": 4.402164428009904, + "tokens_seen": 193396736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607141424272819, + "loss": 1.7918, + "theoretical_loss": 4.40197804187965, + "tokens_seen": 193462272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606980942828486, + "loss": 1.6467, + "theoretical_loss": 4.401791736549714, + "tokens_seen": 193527808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606820461384153, + "loss": 1.7642, + "theoretical_loss": 4.401605511957724, + "tokens_seen": 193593344 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760665997993982, + "loss": 1.6585, + "theoretical_loss": 4.401419368041377, + "tokens_seen": 193658880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606499498495487, + "loss": 1.6439, + "theoretical_loss": 4.401233304738438, + "tokens_seen": 193724416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606339017051153, + "loss": 1.7171, + "theoretical_loss": 4.401047321986745, + "tokens_seen": 193789952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606178535606821, + "loss": 1.7524, + "theoretical_loss": 4.4008614197242, + "tokens_seen": 193855488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606018054162487, + "loss": 1.7401, + "theoretical_loss": 4.400675597888777, + "tokens_seen": 193921024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605857572718155, + "loss": 1.6179, + "theoretical_loss": 4.400489856418518, + "tokens_seen": 193986560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605697091273821, + "loss": 1.6743, + "theoretical_loss": 4.4003041952515325, + "tokens_seen": 194052096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605536609829489, + "loss": 1.7264, + "theoretical_loss": 4.400118614326001, + "tokens_seen": 194117632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605376128385155, + "loss": 1.6531, + "theoretical_loss": 4.39993311358017, + "tokens_seen": 194183168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605215646940823, + "loss": 1.5623, + "theoretical_loss": 4.399747692952355, + "tokens_seen": 194248704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605055165496489, + "loss": 1.7761, + "theoretical_loss": 4.399562352380941, + "tokens_seen": 194314240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604894684052156, + "loss": 1.7038, + "theoretical_loss": 4.399377091804379, + "tokens_seen": 194379776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604734202607823, + "loss": 1.7686, + "theoretical_loss": 4.399191911161191, + "tokens_seen": 194445312 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760457372116349, + "loss": 1.6672, + "theoretical_loss": 4.399006810389963, + "tokens_seen": 194510848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604413239719157, + "loss": 1.6675, + "theoretical_loss": 4.398821789429354, + "tokens_seen": 194576384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604252758274824, + "loss": 1.844, + "theoretical_loss": 4.398636848218084, + "tokens_seen": 194641920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604092276830491, + "loss": 1.7601, + "theoretical_loss": 4.398451986694948, + "tokens_seen": 194707456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603931795386158, + "loss": 1.7763, + "theoretical_loss": 4.398267204798803, + "tokens_seen": 194772992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603771313941825, + "loss": 1.6735, + "theoretical_loss": 4.398082502468577, + "tokens_seen": 194838528 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603610832497492, + "loss": 1.5872, + "theoretical_loss": 4.397897879643262, + "tokens_seen": 194904064 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760345035105316, + "loss": 1.6719, + "theoretical_loss": 4.39771333626192, + "tokens_seen": 194969600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603289869608827, + "loss": 1.7731, + "theoretical_loss": 4.397528872263679, + "tokens_seen": 195035136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603129388164494, + "loss": 1.6565, + "theoretical_loss": 4.397344487587736, + "tokens_seen": 195100672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602968906720161, + "loss": 1.7334, + "theoretical_loss": 4.3971601821733515, + "tokens_seen": 195166208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602808425275828, + "loss": 1.6964, + "theoretical_loss": 4.396975955959856, + "tokens_seen": 195231744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602647943831495, + "loss": 1.7469, + "theoretical_loss": 4.396791808886643, + "tokens_seen": 195297280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602487462387162, + "loss": 1.8126, + "theoretical_loss": 4.396607740893179, + "tokens_seen": 195362816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602326980942829, + "loss": 1.6569, + "theoretical_loss": 4.396423751918991, + "tokens_seen": 195428352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602166499498496, + "loss": 1.5221, + "theoretical_loss": 4.396239841903674, + "tokens_seen": 195493888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602006018054163, + "loss": 1.6202, + "theoretical_loss": 4.396056010786893, + "tokens_seen": 195559424 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760184553660983, + "loss": 1.8545, + "theoretical_loss": 4.3958722585083745, + "tokens_seen": 195624960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601685055165497, + "loss": 1.7108, + "theoretical_loss": 4.395688585007913, + "tokens_seen": 195690496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601524573721164, + "loss": 1.7769, + "theoretical_loss": 4.395504990225371, + "tokens_seen": 195756032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601364092276831, + "loss": 1.8607, + "theoretical_loss": 4.395321474100673, + "tokens_seen": 195821568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601203610832498, + "loss": 1.7631, + "theoretical_loss": 4.395138036573814, + "tokens_seen": 195887104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601043129388165, + "loss": 1.7287, + "theoretical_loss": 4.3949546775848525, + "tokens_seen": 195952640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600882647943832, + "loss": 1.6529, + "theoretical_loss": 4.394771397073912, + "tokens_seen": 196018176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600722166499499, + "loss": 1.7395, + "theoretical_loss": 4.394588194981182, + "tokens_seen": 196083712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600561685055166, + "loss": 1.8479, + "theoretical_loss": 4.394405071246919, + "tokens_seen": 196149248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600401203610833, + "loss": 1.733, + "theoretical_loss": 4.394222025811445, + "tokens_seen": 196214784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00076002407221665, + "loss": 1.6186, + "theoretical_loss": 4.394039058615144, + "tokens_seen": 196280320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600080240722167, + "loss": 1.6977, + "theoretical_loss": 4.39385616959847, + "tokens_seen": 196345856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599919759277834, + "loss": 1.7501, + "theoretical_loss": 4.393673358701939, + "tokens_seen": 196411392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599759277833501, + "loss": 1.5986, + "theoretical_loss": 4.393490625866132, + "tokens_seen": 196476928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599598796389168, + "loss": 1.777, + "theoretical_loss": 4.393307971031697, + "tokens_seen": 196542464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 71932, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4793413877487183, + "objective/train/theoretical_loss": 4.3931253941393456, + "objective/train/tokens_used": 217068000, + "theoretical_loss": 4.3931253941393456, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599438314944835, + "loss": 1.5309, + "theoretical_loss": 4.3931253941393456, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599277833500502, + "loss": 1.7435, + "theoretical_loss": 4.392942895129854, + "tokens_seen": 196673536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599117352056169, + "loss": 1.6521, + "theoretical_loss": 4.392760473944064, + "tokens_seen": 196739072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598956870611836, + "loss": 1.708, + "theoretical_loss": 4.392578130522883, + "tokens_seen": 196804608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598796389167503, + "loss": 1.7905, + "theoretical_loss": 4.392395864807279, + "tokens_seen": 196870144 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759863590772317, + "loss": 1.7152, + "theoretical_loss": 4.392213676738288, + "tokens_seen": 196935680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598475426278837, + "loss": 1.8214, + "theoretical_loss": 4.3920315662570095, + "tokens_seen": 197001216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598314944834504, + "loss": 1.7028, + "theoretical_loss": 4.391849533304607, + "tokens_seen": 197066752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598154463390171, + "loss": 1.6381, + "theoretical_loss": 4.391667577822308, + "tokens_seen": 197132288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597993981945838, + "loss": 1.7244, + "theoretical_loss": 4.391485699751406, + "tokens_seen": 197197824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597833500501505, + "loss": 1.7536, + "theoretical_loss": 4.391303899033253, + "tokens_seen": 197263360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597673019057172, + "loss": 1.6795, + "theoretical_loss": 4.391122175609272, + "tokens_seen": 197328896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597512537612839, + "loss": 1.6457, + "theoretical_loss": 4.390940529420946, + "tokens_seen": 197394432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597352056168506, + "loss": 1.6911, + "theoretical_loss": 4.39075896040982, + "tokens_seen": 197459968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597191574724173, + "loss": 1.5381, + "theoretical_loss": 4.390577468517508, + "tokens_seen": 197525504 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759703109327984, + "loss": 1.7558, + "theoretical_loss": 4.390396053685682, + "tokens_seen": 197591040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596870611835507, + "loss": 1.7281, + "theoretical_loss": 4.39021471585608, + "tokens_seen": 197656576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596710130391174, + "loss": 1.6057, + "theoretical_loss": 4.390033454970504, + "tokens_seen": 197722112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596549648946841, + "loss": 1.6983, + "theoretical_loss": 4.389852270970818, + "tokens_seen": 197787648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596389167502508, + "loss": 1.7131, + "theoretical_loss": 4.389671163798949, + "tokens_seen": 197853184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596228686058175, + "loss": 1.7804, + "theoretical_loss": 4.3894901333968885, + "tokens_seen": 197918720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596068204613842, + "loss": 1.7999, + "theoretical_loss": 4.38930917970669, + "tokens_seen": 197984256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595907723169509, + "loss": 1.7063, + "theoretical_loss": 4.3891283026704695, + "tokens_seen": 198049792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595747241725176, + "loss": 1.7568, + "theoretical_loss": 4.388947502230407, + "tokens_seen": 198115328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595586760280843, + "loss": 1.6771, + "theoretical_loss": 4.388766778328744, + "tokens_seen": 198180864 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759542627883651, + "loss": 1.6316, + "theoretical_loss": 4.388586130907786, + "tokens_seen": 198246400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595265797392177, + "loss": 1.7721, + "theoretical_loss": 4.388405559909899, + "tokens_seen": 198311936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595105315947844, + "loss": 1.672, + "theoretical_loss": 4.3882250652775125, + "tokens_seen": 198377472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594944834503511, + "loss": 1.6478, + "theoretical_loss": 4.38804464695312, + "tokens_seen": 198443008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594784353059178, + "loss": 1.648, + "theoretical_loss": 4.387864304879275, + "tokens_seen": 198508544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594623871614845, + "loss": 1.7571, + "theoretical_loss": 4.387684038998595, + "tokens_seen": 198574080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594463390170512, + "loss": 1.7198, + "theoretical_loss": 4.387503849253757, + "tokens_seen": 198639616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594302908726179, + "loss": 1.7731, + "theoretical_loss": 4.387323735587502, + "tokens_seen": 198705152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594142427281846, + "loss": 1.7331, + "theoretical_loss": 4.387143697942634, + "tokens_seen": 198770688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593981945837513, + "loss": 1.8219, + "theoretical_loss": 4.386963736262016, + "tokens_seen": 198836224 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759382146439318, + "loss": 1.685, + "theoretical_loss": 4.386783850488575, + "tokens_seen": 198901760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593660982948847, + "loss": 1.7198, + "theoretical_loss": 4.386604040565299, + "tokens_seen": 198967296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593500501504514, + "loss": 1.7957, + "theoretical_loss": 4.386424306435236, + "tokens_seen": 199032832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593340020060181, + "loss": 1.7063, + "theoretical_loss": 4.386244648041498, + "tokens_seen": 199098368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593179538615848, + "loss": 1.6459, + "theoretical_loss": 4.386065065327258, + "tokens_seen": 199163904 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593019057171515, + "loss": 1.6441, + "theoretical_loss": 4.385885558235747, + "tokens_seen": 199229440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007592858575727182, + "loss": 1.7275, + "theoretical_loss": 4.385706126710264, + "tokens_seen": 199294976 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759269809428285, + "loss": 1.7563, + "theoretical_loss": 4.385526770694161, + "tokens_seen": 199360512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007592537612838516, + "loss": 1.6168, + "theoretical_loss": 4.385347490130858, + "tokens_seen": 199426048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007592377131394182, + "loss": 1.678, + "theoretical_loss": 4.385168284963832, + "tokens_seen": 199491584 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759221664994985, + "loss": 1.7211, + "theoretical_loss": 4.384989155136621, + "tokens_seen": 199557120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007592056168505516, + "loss": 1.6404, + "theoretical_loss": 4.384810100592826, + "tokens_seen": 199622656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591895687061184, + "loss": 1.7634, + "theoretical_loss": 4.384631121276108, + "tokens_seen": 199688192 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759173520561685, + "loss": 1.649, + "theoretical_loss": 4.384452217130187, + "tokens_seen": 199753728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591574724172518, + "loss": 1.6701, + "theoretical_loss": 4.384273388098845, + "tokens_seen": 199819264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 73160, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7228140830993652, + "objective/train/theoretical_loss": 4.384094634125924, + "objective/train/tokens_used": 220344800, + "theoretical_loss": 4.384094634125924, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591414242728184, + "loss": 1.6614, + "theoretical_loss": 4.384094634125924, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591253761283852, + "loss": 1.6742, + "theoretical_loss": 4.383915955155326, + "tokens_seen": 199950336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591093279839518, + "loss": 1.6602, + "theoretical_loss": 4.383737351131016, + "tokens_seen": 200015872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007590932798395186, + "loss": 1.7915, + "theoretical_loss": 4.383558821997014, + "tokens_seen": 200081408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007590772316950852, + "loss": 1.7205, + "theoretical_loss": 4.383380367697406, + "tokens_seen": 200146944 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759061183550652, + "loss": 1.7392, + "theoretical_loss": 4.383201988176333, + "tokens_seen": 200212480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007590451354062186, + "loss": 1.735, + "theoretical_loss": 4.383023683377999, + "tokens_seen": 200278016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007590290872617854, + "loss": 1.796, + "theoretical_loss": 4.382845453246668, + "tokens_seen": 200343552 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759013039117352, + "loss": 1.705, + "theoretical_loss": 4.382667297726661, + "tokens_seen": 200409088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589969909729188, + "loss": 1.6234, + "theoretical_loss": 4.3824892167623615, + "tokens_seen": 200474624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589809428284855, + "loss": 1.725, + "theoretical_loss": 4.382311210298212, + "tokens_seen": 200540160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589648946840522, + "loss": 1.7048, + "theoretical_loss": 4.3821332782787135, + "tokens_seen": 200605696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589488465396189, + "loss": 1.718, + "theoretical_loss": 4.381955420648428, + "tokens_seen": 200671232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589327983951856, + "loss": 1.7411, + "theoretical_loss": 4.381777637351974, + "tokens_seen": 200736768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589167502507523, + "loss": 1.8566, + "theoretical_loss": 4.381599928334033, + "tokens_seen": 200802304 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758900702106319, + "loss": 1.7141, + "theoretical_loss": 4.381422293539345, + "tokens_seen": 200867840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588846539618857, + "loss": 1.695, + "theoretical_loss": 4.381244732912706, + "tokens_seen": 200933376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588686058174524, + "loss": 1.6169, + "theoretical_loss": 4.381067246398975, + "tokens_seen": 200998912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588525576730191, + "loss": 1.6962, + "theoretical_loss": 4.380889833943066, + "tokens_seen": 201064448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588365095285858, + "loss": 1.6986, + "theoretical_loss": 4.380712495489957, + "tokens_seen": 201129984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588204613841525, + "loss": 1.7617, + "theoretical_loss": 4.380535230984679, + "tokens_seen": 201195520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588044132397192, + "loss": 1.6525, + "theoretical_loss": 4.380358040372325, + "tokens_seen": 201261056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587883650952859, + "loss": 1.7725, + "theoretical_loss": 4.380180923598047, + "tokens_seen": 201326592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587723169508526, + "loss": 1.7453, + "theoretical_loss": 4.380003880607055, + "tokens_seen": 201392128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587562688064193, + "loss": 1.787, + "theoretical_loss": 4.379826911344617, + "tokens_seen": 201457664 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758740220661986, + "loss": 1.7013, + "theoretical_loss": 4.3796500157560585, + "tokens_seen": 201523200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587241725175527, + "loss": 1.7672, + "theoretical_loss": 4.379473193786765, + "tokens_seen": 201588736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587081243731194, + "loss": 1.7227, + "theoretical_loss": 4.37929644538218, + "tokens_seen": 201654272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586920762286861, + "loss": 1.7301, + "theoretical_loss": 4.379119770487805, + "tokens_seen": 201719808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586760280842528, + "loss": 1.7752, + "theoretical_loss": 4.378943169049198, + "tokens_seen": 201785344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586599799398195, + "loss": 1.7993, + "theoretical_loss": 4.378766641011978, + "tokens_seen": 201850880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586439317953862, + "loss": 1.6976, + "theoretical_loss": 4.378590186321819, + "tokens_seen": 201916416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586278836509529, + "loss": 1.73, + "theoretical_loss": 4.378413804924454, + "tokens_seen": 201981952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586118355065196, + "loss": 1.6686, + "theoretical_loss": 4.378237496765674, + "tokens_seen": 202047488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585957873620863, + "loss": 1.6747, + "theoretical_loss": 4.378061261791328, + "tokens_seen": 202113024 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758579739217653, + "loss": 1.7438, + "theoretical_loss": 4.377885099947321, + "tokens_seen": 202178560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585636910732197, + "loss": 1.6094, + "theoretical_loss": 4.3777090111796175, + "tokens_seen": 202244096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585476429287864, + "loss": 1.7414, + "theoretical_loss": 4.377532995434238, + "tokens_seen": 202309632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585315947843531, + "loss": 1.7977, + "theoretical_loss": 4.37735705265726, + "tokens_seen": 202375168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585155466399198, + "loss": 1.6395, + "theoretical_loss": 4.3771811827948195, + "tokens_seen": 202440704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584994984954865, + "loss": 1.7897, + "theoretical_loss": 4.37700538579311, + "tokens_seen": 202506240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584834503510532, + "loss": 1.6584, + "theoretical_loss": 4.37682966159838, + "tokens_seen": 202571776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584674022066199, + "loss": 1.6479, + "theoretical_loss": 4.376654010156939, + "tokens_seen": 202637312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584513540621866, + "loss": 1.6911, + "theoretical_loss": 4.376478431415148, + "tokens_seen": 202702848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584353059177533, + "loss": 1.6819, + "theoretical_loss": 4.376302925319427, + "tokens_seen": 202768384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00075841925777332, + "loss": 1.7279, + "theoretical_loss": 4.3761274918162565, + "tokens_seen": 202833920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584032096288867, + "loss": 1.5278, + "theoretical_loss": 4.375952130852169, + "tokens_seen": 202899456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583871614844534, + "loss": 1.5301, + "theoretical_loss": 4.375776842373755, + "tokens_seen": 202964992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583711133400201, + "loss": 1.7144, + "theoretical_loss": 4.375601626327662, + "tokens_seen": 203030528 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583550651955868, + "loss": 1.6743, + "theoretical_loss": 4.375426482660595, + "tokens_seen": 203096064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 74031, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6785008907318115, + "objective/train/theoretical_loss": 4.375251411319312, + "objective/train/tokens_used": 223621600, + "theoretical_loss": 4.375251411319312, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583390170511535, + "loss": 1.7139, + "theoretical_loss": 4.375251411319312, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583229689067202, + "loss": 1.6417, + "theoretical_loss": 4.375076412250632, + "tokens_seen": 203227136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583069207622869, + "loss": 1.6601, + "theoretical_loss": 4.374901485401426, + "tokens_seen": 203292672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582908726178536, + "loss": 1.6632, + "theoretical_loss": 4.374726630718624, + "tokens_seen": 203358208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582748244734203, + "loss": 1.6369, + "theoretical_loss": 4.37455184814921, + "tokens_seen": 203423744 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758258776328987, + "loss": 1.5944, + "theoretical_loss": 4.374377137640225, + "tokens_seen": 203489280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582427281845537, + "loss": 1.633, + "theoretical_loss": 4.374202499138768, + "tokens_seen": 203554816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582266800401204, + "loss": 1.6212, + "theoretical_loss": 4.374027932591989, + "tokens_seen": 203620352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582106318956871, + "loss": 1.6938, + "theoretical_loss": 4.373853437947099, + "tokens_seen": 203685888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581945837512538, + "loss": 1.7528, + "theoretical_loss": 4.37367901515136, + "tokens_seen": 203751424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581785356068205, + "loss": 1.7577, + "theoretical_loss": 4.373504664152094, + "tokens_seen": 203816960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581624874623872, + "loss": 1.5288, + "theoretical_loss": 4.373330384896676, + "tokens_seen": 203882496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581464393179539, + "loss": 1.6905, + "theoretical_loss": 4.373156177332536, + "tokens_seen": 203948032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581303911735206, + "loss": 1.7408, + "theoretical_loss": 4.372982041407161, + "tokens_seen": 204013568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581143430290873, + "loss": 1.64, + "theoretical_loss": 4.372807977068092, + "tokens_seen": 204079104 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758098294884654, + "loss": 1.6473, + "theoretical_loss": 4.372633984262928, + "tokens_seen": 204144640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580822467402207, + "loss": 1.6681, + "theoretical_loss": 4.372460062939318, + "tokens_seen": 204210176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580661985957874, + "loss": 1.6786, + "theoretical_loss": 4.372286213044972, + "tokens_seen": 204275712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580501504513541, + "loss": 1.7793, + "theoretical_loss": 4.37211243452765, + "tokens_seen": 204341248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580341023069208, + "loss": 1.5185, + "theoretical_loss": 4.37193872733517, + "tokens_seen": 204406784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580180541624875, + "loss": 1.7019, + "theoretical_loss": 4.371765091415404, + "tokens_seen": 204472320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580020060180542, + "loss": 1.7595, + "theoretical_loss": 4.371591526716279, + "tokens_seen": 204537856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579859578736209, + "loss": 1.7805, + "theoretical_loss": 4.371418033185777, + "tokens_seen": 204603392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579699097291876, + "loss": 1.5919, + "theoretical_loss": 4.3712446107719325, + "tokens_seen": 204668928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579538615847543, + "loss": 1.7362, + "theoretical_loss": 4.371071259422838, + "tokens_seen": 204734464 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757937813440321, + "loss": 1.6875, + "theoretical_loss": 4.370897979086636, + "tokens_seen": 204800000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579217652958877, + "loss": 1.8177, + "theoretical_loss": 4.370724769711529, + "tokens_seen": 204865536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579057171514544, + "loss": 1.7856, + "theoretical_loss": 4.370551631245769, + "tokens_seen": 204931072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578896690070211, + "loss": 1.6602, + "theoretical_loss": 4.370378563637667, + "tokens_seen": 204996608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578736208625878, + "loss": 1.7391, + "theoretical_loss": 4.3702055668355815, + "tokens_seen": 205062144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578575727181545, + "loss": 1.7766, + "theoretical_loss": 4.370032640787931, + "tokens_seen": 205127680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578415245737211, + "loss": 1.7982, + "theoretical_loss": 4.369859785443188, + "tokens_seen": 205193216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578254764292879, + "loss": 1.6197, + "theoretical_loss": 4.369687000749873, + "tokens_seen": 205258752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578094282848545, + "loss": 1.7441, + "theoretical_loss": 4.3695142866565675, + "tokens_seen": 205324288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577933801404213, + "loss": 1.6629, + "theoretical_loss": 4.369341643111902, + "tokens_seen": 205389824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577773319959879, + "loss": 1.7171, + "theoretical_loss": 4.369169070064563, + "tokens_seen": 205455360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577612838515547, + "loss": 1.6443, + "theoretical_loss": 4.368996567463292, + "tokens_seen": 205520896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577452357071213, + "loss": 1.6624, + "theoretical_loss": 4.3688241352568795, + "tokens_seen": 205586432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577291875626881, + "loss": 1.8175, + "theoretical_loss": 4.368651773394173, + "tokens_seen": 205651968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577131394182547, + "loss": 1.5977, + "theoretical_loss": 4.368479481824075, + "tokens_seen": 205717504 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576970912738215, + "loss": 1.6515, + "theoretical_loss": 4.368307260495536, + "tokens_seen": 205783040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576810431293881, + "loss": 1.6509, + "theoretical_loss": 4.368135109357564, + "tokens_seen": 205848576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576649949849549, + "loss": 1.7278, + "theoretical_loss": 4.36796302835922, + "tokens_seen": 205914112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576489468405215, + "loss": 1.5491, + "theoretical_loss": 4.367791017449618, + "tokens_seen": 205979648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576328986960883, + "loss": 1.555, + "theoretical_loss": 4.367619076577923, + "tokens_seen": 206045184 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757616850551655, + "loss": 1.6893, + "theoretical_loss": 4.367447205693354, + "tokens_seen": 206110720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576008024072217, + "loss": 1.7705, + "theoretical_loss": 4.367275404745186, + "tokens_seen": 206176256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575847542627884, + "loss": 1.7363, + "theoretical_loss": 4.367103673682742, + "tokens_seen": 206241792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575687061183551, + "loss": 1.7842, + "theoretical_loss": 4.3669320124554, + "tokens_seen": 206307328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575526579739218, + "loss": 1.6683, + "theoretical_loss": 4.366760421012592, + "tokens_seen": 206372864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 75241, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8373074531555176, + "objective/train/theoretical_loss": 4.366588899303801, + "objective/train/tokens_used": 226898400, + "theoretical_loss": 4.366588899303801, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575366098294885, + "loss": 1.7905, + "theoretical_loss": 4.366588899303801, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575205616850552, + "loss": 1.7056, + "theoretical_loss": 4.366417447278565, + "tokens_seen": 206503936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575045135406219, + "loss": 1.6869, + "theoretical_loss": 4.366246064886468, + "tokens_seen": 206569472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574884653961886, + "loss": 1.7188, + "theoretical_loss": 4.3660747520771555, + "tokens_seen": 206635008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574724172517553, + "loss": 1.6279, + "theoretical_loss": 4.36590350880032, + "tokens_seen": 206700544 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757456369107322, + "loss": 1.6978, + "theoretical_loss": 4.365732335005706, + "tokens_seen": 206766080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574403209628887, + "loss": 1.647, + "theoretical_loss": 4.365561230643112, + "tokens_seen": 206831616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574242728184554, + "loss": 1.691, + "theoretical_loss": 4.365390195662389, + "tokens_seen": 206897152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574082246740221, + "loss": 1.7532, + "theoretical_loss": 4.365219230013438, + "tokens_seen": 206962688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573921765295888, + "loss": 1.7129, + "theoretical_loss": 4.3650483336462145, + "tokens_seen": 207028224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573761283851555, + "loss": 1.7066, + "theoretical_loss": 4.364877506510723, + "tokens_seen": 207093760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573600802407222, + "loss": 1.6376, + "theoretical_loss": 4.3647067485570235, + "tokens_seen": 207159296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573440320962889, + "loss": 1.5943, + "theoretical_loss": 4.364536059735224, + "tokens_seen": 207224832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573279839518556, + "loss": 1.7438, + "theoretical_loss": 4.364365439995488, + "tokens_seen": 207290368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573119358074223, + "loss": 1.7198, + "theoretical_loss": 4.3641948892880285, + "tokens_seen": 207355904 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757295887662989, + "loss": 1.7217, + "theoretical_loss": 4.364024407563109, + "tokens_seen": 207421440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572798395185557, + "loss": 1.6305, + "theoretical_loss": 4.363853994771048, + "tokens_seen": 207486976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572637913741224, + "loss": 1.713, + "theoretical_loss": 4.363683650862212, + "tokens_seen": 207552512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572477432296891, + "loss": 1.6457, + "theoretical_loss": 4.36351337578702, + "tokens_seen": 207618048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572316950852558, + "loss": 1.6328, + "theoretical_loss": 4.363343169495945, + "tokens_seen": 207683584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572156469408225, + "loss": 1.717, + "theoretical_loss": 4.363173031939506, + "tokens_seen": 207749120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571995987963892, + "loss": 1.6263, + "theoretical_loss": 4.363002963068277, + "tokens_seen": 207814656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571835506519559, + "loss": 1.7193, + "theoretical_loss": 4.362832962832883, + "tokens_seen": 207880192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571675025075226, + "loss": 1.7143, + "theoretical_loss": 4.362663031183999, + "tokens_seen": 207945728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571514543630893, + "loss": 1.6003, + "theoretical_loss": 4.36249316807235, + "tokens_seen": 208011264 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757135406218656, + "loss": 1.7562, + "theoretical_loss": 4.3623233734487155, + "tokens_seen": 208076800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571193580742227, + "loss": 1.7425, + "theoretical_loss": 4.362153647263921, + "tokens_seen": 208142336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571033099297894, + "loss": 1.6691, + "theoretical_loss": 4.361983989468847, + "tokens_seen": 208207872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570872617853561, + "loss": 1.5959, + "theoretical_loss": 4.3618144000144214, + "tokens_seen": 208273408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570712136409228, + "loss": 1.6334, + "theoretical_loss": 4.361644878851626, + "tokens_seen": 208338944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570551654964895, + "loss": 1.832, + "theoretical_loss": 4.36147542593149, + "tokens_seen": 208404480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570391173520562, + "loss": 1.7518, + "theoretical_loss": 4.361306041205095, + "tokens_seen": 208470016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570230692076229, + "loss": 1.7394, + "theoretical_loss": 4.361136724623573, + "tokens_seen": 208535552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570070210631896, + "loss": 1.6212, + "theoretical_loss": 4.360967476138105, + "tokens_seen": 208601088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569909729187563, + "loss": 1.732, + "theoretical_loss": 4.360798295699925, + "tokens_seen": 208666624 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756974924774323, + "loss": 1.7454, + "theoretical_loss": 4.360629183260313, + "tokens_seen": 208732160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569588766298897, + "loss": 1.6911, + "theoretical_loss": 4.360460138770604, + "tokens_seen": 208797696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569428284854564, + "loss": 1.5386, + "theoretical_loss": 4.360291162182179, + "tokens_seen": 208863232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569267803410231, + "loss": 1.6819, + "theoretical_loss": 4.360122253446472, + "tokens_seen": 208928768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569107321965898, + "loss": 1.7132, + "theoretical_loss": 4.359953412514965, + "tokens_seen": 208994304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568946840521565, + "loss": 1.6954, + "theoretical_loss": 4.359784639339191, + "tokens_seen": 209059840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568786359077232, + "loss": 1.5804, + "theoretical_loss": 4.359615933870732, + "tokens_seen": 209125376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568625877632899, + "loss": 1.826, + "theoretical_loss": 4.359447296061219, + "tokens_seen": 209190912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568465396188566, + "loss": 1.6614, + "theoretical_loss": 4.359278725862336, + "tokens_seen": 209256448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568304914744233, + "loss": 1.7834, + "theoretical_loss": 4.359110223225814, + "tokens_seen": 209321984 + }, + { + "epoch": 0.06, + "learning_rate": 0.00075681444332999, + "loss": 1.7298, + "theoretical_loss": 4.358941788103433, + "tokens_seen": 209387520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567983951855567, + "loss": 1.6115, + "theoretical_loss": 4.358773420447024, + "tokens_seen": 209453056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567823470411234, + "loss": 1.7582, + "theoretical_loss": 4.358605120208469, + "tokens_seen": 209518592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567662988966901, + "loss": 1.6202, + "theoretical_loss": 4.358436887339694, + "tokens_seen": 209584128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567502507522569, + "loss": 1.6705, + "theoretical_loss": 4.35826872179268, + "tokens_seen": 209649664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 76142, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.782550573348999, + "objective/train/theoretical_loss": 4.358100623519453, + "objective/train/tokens_used": 230175200, + "theoretical_loss": 4.358100623519453, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567342026078235, + "loss": 1.6927, + "theoretical_loss": 4.358100623519453, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567181544633903, + "loss": 1.6178, + "theoretical_loss": 4.357932592472092, + "tokens_seen": 209780736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567021063189569, + "loss": 1.6931, + "theoretical_loss": 4.357764628602721, + "tokens_seen": 209846272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566860581745237, + "loss": 1.6771, + "theoretical_loss": 4.357596731863517, + "tokens_seen": 209911808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566700100300903, + "loss": 1.7497, + "theoretical_loss": 4.357428902206704, + "tokens_seen": 209977344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566539618856571, + "loss": 1.792, + "theoretical_loss": 4.357261139584553, + "tokens_seen": 210042880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566379137412237, + "loss": 1.7182, + "theoretical_loss": 4.357093443949388, + "tokens_seen": 210108416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566218655967905, + "loss": 1.7378, + "theoretical_loss": 4.356925815253579, + "tokens_seen": 210173952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566058174523571, + "loss": 1.623, + "theoretical_loss": 4.356758253449544, + "tokens_seen": 210239488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565897693079239, + "loss": 1.6864, + "theoretical_loss": 4.356590758489753, + "tokens_seen": 210305024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565737211634905, + "loss": 1.6433, + "theoretical_loss": 4.356423330326722, + "tokens_seen": 210370560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565576730190572, + "loss": 1.6874, + "theoretical_loss": 4.356255968913015, + "tokens_seen": 210436096 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756541624874624, + "loss": 1.6989, + "theoretical_loss": 4.356088674201246, + "tokens_seen": 210501632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565255767301906, + "loss": 1.6619, + "theoretical_loss": 4.355921446144077, + "tokens_seen": 210567168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565095285857573, + "loss": 1.6612, + "theoretical_loss": 4.355754284694218, + "tokens_seen": 210632704 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756493480441324, + "loss": 1.8118, + "theoretical_loss": 4.355587189804427, + "tokens_seen": 210698240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564774322968907, + "loss": 1.6183, + "theoretical_loss": 4.355420161427512, + "tokens_seen": 210763776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564613841524574, + "loss": 1.6552, + "theoretical_loss": 4.355253199516326, + "tokens_seen": 210829312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564453360080242, + "loss": 1.6956, + "theoretical_loss": 4.355086304023773, + "tokens_seen": 210894848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564292878635908, + "loss": 1.6164, + "theoretical_loss": 4.354919474902804, + "tokens_seen": 210960384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564132397191574, + "loss": 1.6407, + "theoretical_loss": 4.354752712106417, + "tokens_seen": 211025920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563971915747242, + "loss": 1.6663, + "theoretical_loss": 4.354586015587659, + "tokens_seen": 211091456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563811434302908, + "loss": 1.7245, + "theoretical_loss": 4.354419385299623, + "tokens_seen": 211156992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563650952858576, + "loss": 1.6848, + "theoretical_loss": 4.354252821195453, + "tokens_seen": 211222528 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563490471414242, + "loss": 1.7278, + "theoretical_loss": 4.354086323228338, + "tokens_seen": 211288064 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756332998996991, + "loss": 1.7467, + "theoretical_loss": 4.353919891351516, + "tokens_seen": 211353600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563169508525576, + "loss": 1.6958, + "theoretical_loss": 4.35375352551827, + "tokens_seen": 211419136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563009027081244, + "loss": 1.7661, + "theoretical_loss": 4.353587225681935, + "tokens_seen": 211484672 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756284854563691, + "loss": 1.7008, + "theoretical_loss": 4.35342099179589, + "tokens_seen": 211550208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562688064192578, + "loss": 1.6198, + "theoretical_loss": 4.353254823813561, + "tokens_seen": 211615744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562527582748245, + "loss": 1.6464, + "theoretical_loss": 4.353088721688424, + "tokens_seen": 211681280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562367101303912, + "loss": 1.6186, + "theoretical_loss": 4.352922685374001, + "tokens_seen": 211746816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562206619859579, + "loss": 1.7021, + "theoretical_loss": 4.352756714823859, + "tokens_seen": 211812352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562046138415246, + "loss": 1.6447, + "theoretical_loss": 4.352590809991616, + "tokens_seen": 211877888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561885656970913, + "loss": 1.7092, + "theoretical_loss": 4.352424970830935, + "tokens_seen": 211943424 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756172517552658, + "loss": 1.6352, + "theoretical_loss": 4.352259197295525, + "tokens_seen": 212008960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561564694082247, + "loss": 1.6442, + "theoretical_loss": 4.352093489339143, + "tokens_seen": 212074496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561404212637914, + "loss": 1.6247, + "theoretical_loss": 4.351927846915595, + "tokens_seen": 212140032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561243731193581, + "loss": 1.7138, + "theoretical_loss": 4.35176226997873, + "tokens_seen": 212205568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561083249749248, + "loss": 1.7305, + "theoretical_loss": 4.351596758482445, + "tokens_seen": 212271104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560922768304915, + "loss": 1.6121, + "theoretical_loss": 4.351431312380684, + "tokens_seen": 212336640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560762286860582, + "loss": 1.6804, + "theoretical_loss": 4.351265931627439, + "tokens_seen": 212402176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560601805416249, + "loss": 1.664, + "theoretical_loss": 4.351100616176747, + "tokens_seen": 212467712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560441323971916, + "loss": 1.6836, + "theoretical_loss": 4.350935365982691, + "tokens_seen": 212533248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560280842527583, + "loss": 1.7276, + "theoretical_loss": 4.350770180999402, + "tokens_seen": 212598784 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756012036108325, + "loss": 1.6969, + "theoretical_loss": 4.3506050611810565, + "tokens_seen": 212664320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559959879638917, + "loss": 1.5556, + "theoretical_loss": 4.350440006481877, + "tokens_seen": 212729856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559799398194584, + "loss": 1.7229, + "theoretical_loss": 4.350275016856132, + "tokens_seen": 212795392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559638916750251, + "loss": 1.6752, + "theoretical_loss": 4.350110092258138, + "tokens_seen": 212860928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559478435305918, + "loss": 1.6807, + "theoretical_loss": 4.349945232642256, + "tokens_seen": 212926464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 77362, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6674554347991943, + "objective/train/theoretical_loss": 4.3497804379628935, + "objective/train/tokens_used": 233452000, + "theoretical_loss": 4.3497804379628935, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559317953861585, + "loss": 1.7286, + "theoretical_loss": 4.3497804379628935, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559157472417252, + "loss": 1.6309, + "theoretical_loss": 4.349615708174504, + "tokens_seen": 213057536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558996990972919, + "loss": 1.589, + "theoretical_loss": 4.3494510432315865, + "tokens_seen": 213123072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558836509528586, + "loss": 1.6323, + "theoretical_loss": 4.349286443088687, + "tokens_seen": 213188608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558676028084253, + "loss": 1.6095, + "theoretical_loss": 4.349121907700396, + "tokens_seen": 213254144 + }, + { + "epoch": 0.06, + "learning_rate": 0.000755851554663992, + "loss": 1.7698, + "theoretical_loss": 4.348957437021351, + "tokens_seen": 213319680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558355065195587, + "loss": 1.7637, + "theoretical_loss": 4.348793031006235, + "tokens_seen": 213385216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558194583751254, + "loss": 1.636, + "theoretical_loss": 4.348628689609775, + "tokens_seen": 213450752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558034102306921, + "loss": 1.6531, + "theoretical_loss": 4.348464412786746, + "tokens_seen": 213516288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557873620862588, + "loss": 1.7114, + "theoretical_loss": 4.348300200491966, + "tokens_seen": 213581824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557713139418255, + "loss": 1.6954, + "theoretical_loss": 4.348136052680301, + "tokens_seen": 213647360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557552657973922, + "loss": 1.7568, + "theoretical_loss": 4.347971969306662, + "tokens_seen": 213712896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557392176529589, + "loss": 1.5466, + "theoretical_loss": 4.347807950326002, + "tokens_seen": 213778432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557231695085256, + "loss": 1.7092, + "theoretical_loss": 4.347643995693325, + "tokens_seen": 213843968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557071213640923, + "loss": 1.6906, + "theoretical_loss": 4.347480105363674, + "tokens_seen": 213909504 + }, + { + "epoch": 0.06, + "learning_rate": 0.000755691073219659, + "loss": 1.767, + "theoretical_loss": 4.3473162792921425, + "tokens_seen": 213975040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556750250752257, + "loss": 1.5911, + "theoretical_loss": 4.347152517433866, + "tokens_seen": 214040576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556589769307924, + "loss": 1.5156, + "theoretical_loss": 4.346988819744026, + "tokens_seen": 214106112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556429287863591, + "loss": 1.6893, + "theoretical_loss": 4.346825186177848, + "tokens_seen": 214171648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556268806419258, + "loss": 1.6864, + "theoretical_loss": 4.346661616690605, + "tokens_seen": 214237184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556108324974925, + "loss": 1.7354, + "theoretical_loss": 4.346498111237612, + "tokens_seen": 214302720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007555947843530592, + "loss": 1.6528, + "theoretical_loss": 4.346334669774231, + "tokens_seen": 214368256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007555787362086259, + "loss": 1.5685, + "theoretical_loss": 4.346171292255867, + "tokens_seen": 214433792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007555626880641926, + "loss": 1.6147, + "theoretical_loss": 4.34600797863797, + "tokens_seen": 214499328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007555466399197594, + "loss": 1.6934, + "theoretical_loss": 4.345844728876036, + "tokens_seen": 214564864 + }, + { + "epoch": 0.07, + "learning_rate": 0.000755530591775326, + "loss": 1.7624, + "theoretical_loss": 4.345681542925604, + "tokens_seen": 214630400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007555145436308928, + "loss": 1.5814, + "theoretical_loss": 4.345518420742259, + "tokens_seen": 214695936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554984954864594, + "loss": 1.6765, + "theoretical_loss": 4.345355362281628, + "tokens_seen": 214761472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554824473420262, + "loss": 1.6003, + "theoretical_loss": 4.345192367499386, + "tokens_seen": 214827008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554663991975928, + "loss": 1.7416, + "theoretical_loss": 4.3450294363512505, + "tokens_seen": 214892544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554503510531596, + "loss": 1.7229, + "theoretical_loss": 4.34486656879298, + "tokens_seen": 214958080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554343029087262, + "loss": 1.557, + "theoretical_loss": 4.344703764780386, + "tokens_seen": 215023616 + }, + { + "epoch": 0.07, + "learning_rate": 0.000755418254764293, + "loss": 1.7031, + "theoretical_loss": 4.344541024269312, + "tokens_seen": 215089152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554022066198596, + "loss": 1.6627, + "theoretical_loss": 4.344378347215657, + "tokens_seen": 215154688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553861584754264, + "loss": 1.6637, + "theoretical_loss": 4.344215733575357, + "tokens_seen": 215220224 + }, + { + "epoch": 0.07, + "learning_rate": 0.000755370110330993, + "loss": 1.5809, + "theoretical_loss": 4.344053183304396, + "tokens_seen": 215285760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553540621865598, + "loss": 1.7869, + "theoretical_loss": 4.343890696358798, + "tokens_seen": 215351296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553380140421264, + "loss": 1.4857, + "theoretical_loss": 4.343728272694635, + "tokens_seen": 215416832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553219658976932, + "loss": 1.4274, + "theoretical_loss": 4.34356591226802, + "tokens_seen": 215482368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553059177532598, + "loss": 1.7362, + "theoretical_loss": 4.343403615035111, + "tokens_seen": 215547904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007552898696088266, + "loss": 1.6923, + "theoretical_loss": 4.343241380952109, + "tokens_seen": 215613440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007552738214643932, + "loss": 1.7153, + "theoretical_loss": 4.34307920997526, + "tokens_seen": 215678976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075525777331996, + "loss": 1.7254, + "theoretical_loss": 4.34291710206085, + "tokens_seen": 215744512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007552417251755266, + "loss": 1.769, + "theoretical_loss": 4.342755057165214, + "tokens_seen": 215810048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007552256770310934, + "loss": 1.6395, + "theoretical_loss": 4.342593075244728, + "tokens_seen": 215875584 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075520962888666, + "loss": 1.6217, + "theoretical_loss": 4.342431156255809, + "tokens_seen": 215941120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551935807422268, + "loss": 1.5778, + "theoretical_loss": 4.3422693001549195, + "tokens_seen": 216006656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551775325977934, + "loss": 1.654, + "theoretical_loss": 4.342107506898567, + "tokens_seen": 216072192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551614844533601, + "loss": 1.6215, + "theoretical_loss": 4.3419457764433, + "tokens_seen": 216137728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551454363089268, + "loss": 1.5418, + "theoretical_loss": 4.34178410874571, + "tokens_seen": 216203264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 78526, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7171001434326172, + "objective/train/theoretical_loss": 4.341622503762434, + "objective/train/tokens_used": 236728800, + "theoretical_loss": 4.341622503762434, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551293881644935, + "loss": 1.6569, + "theoretical_loss": 4.341622503762434, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551133400200602, + "loss": 1.6461, + "theoretical_loss": 4.341460961450148, + "tokens_seen": 216334336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550972918756269, + "loss": 1.5453, + "theoretical_loss": 4.341299481765575, + "tokens_seen": 216399872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550812437311937, + "loss": 1.6849, + "theoretical_loss": 4.34113806466548, + "tokens_seen": 216465408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550651955867603, + "loss": 1.7779, + "theoretical_loss": 4.340976710106671, + "tokens_seen": 216530944 + }, + { + "epoch": 0.07, + "learning_rate": 0.000755049147442327, + "loss": 1.6228, + "theoretical_loss": 4.340815418045995, + "tokens_seen": 216596480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550330992978937, + "loss": 1.7109, + "theoretical_loss": 4.340654188440349, + "tokens_seen": 216662016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550170511534603, + "loss": 1.6088, + "theoretical_loss": 4.340493021246668, + "tokens_seen": 216727552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550010030090271, + "loss": 1.63, + "theoretical_loss": 4.3403319164219285, + "tokens_seen": 216793088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549849548645937, + "loss": 1.7283, + "theoretical_loss": 4.340170873923155, + "tokens_seen": 216858624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549689067201605, + "loss": 1.6471, + "theoretical_loss": 4.340009893707408, + "tokens_seen": 216924160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549528585757271, + "loss": 1.6254, + "theoretical_loss": 4.339848975731796, + "tokens_seen": 216989696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549368104312939, + "loss": 1.6024, + "theoretical_loss": 4.3396881199534665, + "tokens_seen": 217055232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549207622868605, + "loss": 1.6612, + "theoretical_loss": 4.339527326329613, + "tokens_seen": 217120768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549047141424273, + "loss": 1.6202, + "theoretical_loss": 4.3393665948174664, + "tokens_seen": 217186304 + }, + { + "epoch": 0.07, + "learning_rate": 0.000754888665997994, + "loss": 1.6312, + "theoretical_loss": 4.339205925374305, + "tokens_seen": 217251840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548726178535607, + "loss": 1.5936, + "theoretical_loss": 4.339045317957446, + "tokens_seen": 217317376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548565697091274, + "loss": 1.6884, + "theoretical_loss": 4.33888477252425, + "tokens_seen": 217382912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548405215646941, + "loss": 1.6815, + "theoretical_loss": 4.33872428903212, + "tokens_seen": 217448448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548244734202608, + "loss": 1.6695, + "theoretical_loss": 4.338563867438499, + "tokens_seen": 217513984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548084252758275, + "loss": 1.6838, + "theoretical_loss": 4.338403507700877, + "tokens_seen": 217579520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547923771313942, + "loss": 1.4434, + "theoretical_loss": 4.33824320977678, + "tokens_seen": 217645056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547763289869609, + "loss": 1.6037, + "theoretical_loss": 4.33808297362378, + "tokens_seen": 217710592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547602808425276, + "loss": 1.6591, + "theoretical_loss": 4.337922799199489, + "tokens_seen": 217776128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547442326980943, + "loss": 1.6701, + "theoretical_loss": 4.337762686461561, + "tokens_seen": 217841664 + }, + { + "epoch": 0.07, + "learning_rate": 0.000754728184553661, + "loss": 1.6801, + "theoretical_loss": 4.337602635367694, + "tokens_seen": 217907200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547121364092277, + "loss": 1.642, + "theoretical_loss": 4.337442645875624, + "tokens_seen": 217972736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546960882647944, + "loss": 1.695, + "theoretical_loss": 4.337282717943132, + "tokens_seen": 218038272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546800401203611, + "loss": 1.7297, + "theoretical_loss": 4.337122851528037, + "tokens_seen": 218103808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546639919759278, + "loss": 1.5796, + "theoretical_loss": 4.336963046588204, + "tokens_seen": 218169344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546479438314945, + "loss": 1.6636, + "theoretical_loss": 4.336803303081538, + "tokens_seen": 218234880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546318956870612, + "loss": 1.608, + "theoretical_loss": 4.336643620965982, + "tokens_seen": 218300416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546158475426279, + "loss": 1.6229, + "theoretical_loss": 4.336484000199524, + "tokens_seen": 218365952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545997993981946, + "loss": 1.5969, + "theoretical_loss": 4.336324440740195, + "tokens_seen": 218431488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545837512537613, + "loss": 1.6826, + "theoretical_loss": 4.336164942546061, + "tokens_seen": 218497024 + }, + { + "epoch": 0.07, + "learning_rate": 0.000754567703109328, + "loss": 1.5707, + "theoretical_loss": 4.336005505575235, + "tokens_seen": 218562560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545516549648947, + "loss": 1.7505, + "theoretical_loss": 4.335846129785869, + "tokens_seen": 218628096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545356068204614, + "loss": 1.554, + "theoretical_loss": 4.335686815136157, + "tokens_seen": 218693632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545195586760281, + "loss": 1.6179, + "theoretical_loss": 4.335527561584333, + "tokens_seen": 218759168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545035105315948, + "loss": 1.6906, + "theoretical_loss": 4.3353683690886715, + "tokens_seen": 218824704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544874623871615, + "loss": 1.7096, + "theoretical_loss": 4.33520923760749, + "tokens_seen": 218890240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544714142427282, + "loss": 1.6438, + "theoretical_loss": 4.335050167099145, + "tokens_seen": 218955776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544553660982949, + "loss": 1.7231, + "theoretical_loss": 4.334891157522035, + "tokens_seen": 219021312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544393179538616, + "loss": 1.6563, + "theoretical_loss": 4.3347322088346, + "tokens_seen": 219086848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544232698094283, + "loss": 1.5714, + "theoretical_loss": 4.33457332099532, + "tokens_seen": 219152384 + }, + { + "epoch": 0.07, + "learning_rate": 0.000754407221664995, + "loss": 1.5984, + "theoretical_loss": 4.334414493962714, + "tokens_seen": 219217920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543911735205617, + "loss": 1.6528, + "theoretical_loss": 4.334255727695344, + "tokens_seen": 219283456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543751253761284, + "loss": 1.6906, + "theoretical_loss": 4.334097022151812, + "tokens_seen": 219348992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543590772316951, + "loss": 1.71, + "theoretical_loss": 4.33393837729076, + "tokens_seen": 219414528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543430290872619, + "loss": 1.7512, + "theoretical_loss": 4.333779793070871, + "tokens_seen": 219480064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 79445, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6968114376068115, + "objective/train/theoretical_loss": 4.33362126945087, + "objective/train/tokens_used": 240005600, + "theoretical_loss": 4.33362126945087, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543269809428285, + "loss": 1.7406, + "theoretical_loss": 4.33362126945087, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543109327983953, + "loss": 1.6159, + "theoretical_loss": 4.333462806389517, + "tokens_seen": 219611136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542948846539619, + "loss": 1.6682, + "theoretical_loss": 4.33330440384562, + "tokens_seen": 219676672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542788365095287, + "loss": 1.6814, + "theoretical_loss": 4.333146061778021, + "tokens_seen": 219742208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542627883650953, + "loss": 1.6632, + "theoretical_loss": 4.332987780145606, + "tokens_seen": 219807744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542467402206621, + "loss": 1.5992, + "theoretical_loss": 4.3328295589073, + "tokens_seen": 219873280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542306920762287, + "loss": 1.6456, + "theoretical_loss": 4.332671398022067, + "tokens_seen": 219938816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542146439317955, + "loss": 1.632, + "theoretical_loss": 4.332513297448912, + "tokens_seen": 220004352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541985957873621, + "loss": 1.7385, + "theoretical_loss": 4.332355257146881, + "tokens_seen": 220069888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541825476429289, + "loss": 1.5406, + "theoretical_loss": 4.3321972770750605, + "tokens_seen": 220135424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541664994984955, + "loss": 1.6117, + "theoretical_loss": 4.332039357192572, + "tokens_seen": 220200960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541504513540623, + "loss": 1.6715, + "theoretical_loss": 4.331881497458584, + "tokens_seen": 220266496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541344032096289, + "loss": 1.6171, + "theoretical_loss": 4.3317236978323, + "tokens_seen": 220332032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541183550651957, + "loss": 1.6819, + "theoretical_loss": 4.3315659582729635, + "tokens_seen": 220397568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541023069207623, + "loss": 1.5839, + "theoretical_loss": 4.331408278739861, + "tokens_seen": 220463104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540862587763291, + "loss": 1.6511, + "theoretical_loss": 4.331250659192316, + "tokens_seen": 220528640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540702106318957, + "loss": 1.7196, + "theoretical_loss": 4.33109309958969, + "tokens_seen": 220594176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540541624874625, + "loss": 1.6633, + "theoretical_loss": 4.330935599891389, + "tokens_seen": 220659712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540381143430291, + "loss": 1.6971, + "theoretical_loss": 4.330778160056855, + "tokens_seen": 220725248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540220661985959, + "loss": 1.7276, + "theoretical_loss": 4.330620780045571, + "tokens_seen": 220790784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540060180541625, + "loss": 1.7717, + "theoretical_loss": 4.330463459817057, + "tokens_seen": 220856320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539899699097293, + "loss": 1.6917, + "theoretical_loss": 4.3303061993308765, + "tokens_seen": 220921856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539739217652959, + "loss": 1.6614, + "theoretical_loss": 4.330148998546628, + "tokens_seen": 220987392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539578736208627, + "loss": 1.6307, + "theoretical_loss": 4.329991857423953, + "tokens_seen": 221052928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539418254764293, + "loss": 1.5909, + "theoretical_loss": 4.329834775922529, + "tokens_seen": 221118464 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539257773319961, + "loss": 1.7151, + "theoretical_loss": 4.329677754002074, + "tokens_seen": 221184000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539097291875627, + "loss": 1.6996, + "theoretical_loss": 4.329520791622348, + "tokens_seen": 221249536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538936810431295, + "loss": 1.6523, + "theoretical_loss": 4.329363888743145, + "tokens_seen": 221315072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538776328986961, + "loss": 1.6329, + "theoretical_loss": 4.329207045324301, + "tokens_seen": 221380608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538615847542629, + "loss": 1.6521, + "theoretical_loss": 4.3290502613256905, + "tokens_seen": 221446144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538455366098295, + "loss": 1.8236, + "theoretical_loss": 4.328893536707228, + "tokens_seen": 221511680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538294884653963, + "loss": 1.7811, + "theoretical_loss": 4.328736871428864, + "tokens_seen": 221577216 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753813440320963, + "loss": 1.6121, + "theoretical_loss": 4.328580265450592, + "tokens_seen": 221642752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537973921765297, + "loss": 1.6411, + "theoretical_loss": 4.32842371873244, + "tokens_seen": 221708288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537813440320963, + "loss": 1.7128, + "theoretical_loss": 4.328267231234476, + "tokens_seen": 221773824 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753765295887663, + "loss": 1.7062, + "theoretical_loss": 4.328110802916811, + "tokens_seen": 221839360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537492477432297, + "loss": 1.8148, + "theoretical_loss": 4.327954433739588, + "tokens_seen": 221904896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537331995987964, + "loss": 1.6588, + "theoretical_loss": 4.327798123662992, + "tokens_seen": 221970432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537171514543632, + "loss": 1.5768, + "theoretical_loss": 4.327641872647248, + "tokens_seen": 222035968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537011033099298, + "loss": 1.7152, + "theoretical_loss": 4.327485680652615, + "tokens_seen": 222101504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007536850551654966, + "loss": 1.6536, + "theoretical_loss": 4.327329547639396, + "tokens_seen": 222167040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007536690070210632, + "loss": 1.6875, + "theoretical_loss": 4.327173473567927, + "tokens_seen": 222232576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075365295887663, + "loss": 1.7102, + "theoretical_loss": 4.327017458398587, + "tokens_seen": 222298112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007536369107321966, + "loss": 1.6412, + "theoretical_loss": 4.32686150209179, + "tokens_seen": 222363648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007536208625877632, + "loss": 1.6459, + "theoretical_loss": 4.32670560460799, + "tokens_seen": 222429184 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075360481444333, + "loss": 1.5749, + "theoretical_loss": 4.326549765907679, + "tokens_seen": 222494720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535887662988966, + "loss": 1.7364, + "theoretical_loss": 4.326393985951386, + "tokens_seen": 222560256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535727181544634, + "loss": 1.5982, + "theoretical_loss": 4.326238264699679, + "tokens_seen": 222625792 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075355667001003, + "loss": 1.7705, + "theoretical_loss": 4.326082602113164, + "tokens_seen": 222691328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535406218655968, + "loss": 1.6681, + "theoretical_loss": 4.325926998152487, + "tokens_seen": 222756864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 80592, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5793734788894653, + "objective/train/theoretical_loss": 4.325771452778328, + "objective/train/tokens_used": 243282400, + "theoretical_loss": 4.325771452778328, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535245737211634, + "loss": 1.6079, + "theoretical_loss": 4.325771452778328, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535085255767302, + "loss": 1.731, + "theoretical_loss": 4.325615965951408, + "tokens_seen": 222887936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534924774322969, + "loss": 1.6734, + "theoretical_loss": 4.325460537632483, + "tokens_seen": 222953472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534764292878636, + "loss": 1.6681, + "theoretical_loss": 4.325305167782352, + "tokens_seen": 223019008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534603811434303, + "loss": 1.6635, + "theoretical_loss": 4.325149856361845, + "tokens_seen": 223084544 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753444332998997, + "loss": 1.6284, + "theoretical_loss": 4.3249946033318345, + "tokens_seen": 223150080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534282848545637, + "loss": 1.7733, + "theoretical_loss": 4.32483940865323, + "tokens_seen": 223215616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534122367101304, + "loss": 1.6484, + "theoretical_loss": 4.324684272286978, + "tokens_seen": 223281152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533961885656971, + "loss": 1.6395, + "theoretical_loss": 4.324529194194062, + "tokens_seen": 223346688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533801404212638, + "loss": 1.6507, + "theoretical_loss": 4.324374174335503, + "tokens_seen": 223412224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533640922768305, + "loss": 1.7281, + "theoretical_loss": 4.324219212672363, + "tokens_seen": 223477760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533480441323972, + "loss": 1.7564, + "theoretical_loss": 4.324064309165734, + "tokens_seen": 223543296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533319959879639, + "loss": 1.5957, + "theoretical_loss": 4.3239094637767534, + "tokens_seen": 223608832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533159478435306, + "loss": 1.7318, + "theoretical_loss": 4.323754676466592, + "tokens_seen": 223674368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532998996990973, + "loss": 1.7811, + "theoretical_loss": 4.323599947196458, + "tokens_seen": 223739904 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753283851554664, + "loss": 1.7292, + "theoretical_loss": 4.323445275927597, + "tokens_seen": 223805440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532678034102307, + "loss": 1.712, + "theoretical_loss": 4.323290662621292, + "tokens_seen": 223870976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532517552657974, + "loss": 1.7663, + "theoretical_loss": 4.323136107238865, + "tokens_seen": 223936512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532357071213641, + "loss": 1.7012, + "theoretical_loss": 4.322981609741671, + "tokens_seen": 224002048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532196589769308, + "loss": 1.6102, + "theoretical_loss": 4.322827170091106, + "tokens_seen": 224067584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532036108324975, + "loss": 1.6167, + "theoretical_loss": 4.322672788248601, + "tokens_seen": 224133120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531875626880642, + "loss": 1.5478, + "theoretical_loss": 4.3225184641756265, + "tokens_seen": 224198656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531715145436309, + "loss": 1.6447, + "theoretical_loss": 4.322364197833684, + "tokens_seen": 224264192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531554663991976, + "loss": 1.618, + "theoretical_loss": 4.322209989184319, + "tokens_seen": 224329728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531394182547644, + "loss": 1.6174, + "theoretical_loss": 4.32205583818911, + "tokens_seen": 224395264 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753123370110331, + "loss": 1.6718, + "theoretical_loss": 4.321901744809672, + "tokens_seen": 224460800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531073219658978, + "loss": 1.6626, + "theoretical_loss": 4.321747709007658, + "tokens_seen": 224526336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530912738214644, + "loss": 1.6943, + "theoretical_loss": 4.321593730744759, + "tokens_seen": 224591872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530752256770312, + "loss": 1.7384, + "theoretical_loss": 4.3214398099826985, + "tokens_seen": 224657408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530591775325978, + "loss": 1.6295, + "theoretical_loss": 4.321285946683242, + "tokens_seen": 224722944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530431293881646, + "loss": 1.6245, + "theoretical_loss": 4.321132140808186, + "tokens_seen": 224788480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530270812437312, + "loss": 1.6358, + "theoretical_loss": 4.320978392319368, + "tokens_seen": 224854016 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753011033099298, + "loss": 1.6467, + "theoretical_loss": 4.320824701178661, + "tokens_seen": 224919552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529949849548646, + "loss": 1.6154, + "theoretical_loss": 4.320671067347972, + "tokens_seen": 224985088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529789368104314, + "loss": 1.7091, + "theoretical_loss": 4.320517490789246, + "tokens_seen": 225050624 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752962888665998, + "loss": 1.5441, + "theoretical_loss": 4.320363971464466, + "tokens_seen": 225116160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529468405215648, + "loss": 1.913, + "theoretical_loss": 4.320210509335649, + "tokens_seen": 225181696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529307923771314, + "loss": 1.7173, + "theoretical_loss": 4.320057104364848, + "tokens_seen": 225247232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529147442326982, + "loss": 1.5435, + "theoretical_loss": 4.3199037565141545, + "tokens_seen": 225312768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528986960882648, + "loss": 1.5532, + "theoretical_loss": 4.319750465745694, + "tokens_seen": 225378304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528826479438316, + "loss": 1.5809, + "theoretical_loss": 4.31959723202163, + "tokens_seen": 225443840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528665997993982, + "loss": 1.6615, + "theoretical_loss": 4.319444055304161, + "tokens_seen": 225509376 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752850551654965, + "loss": 1.6242, + "theoretical_loss": 4.319290935555521, + "tokens_seen": 225574912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528345035105316, + "loss": 1.6218, + "theoretical_loss": 4.319137872737979, + "tokens_seen": 225640448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528184553660984, + "loss": 1.7047, + "theoretical_loss": 4.318984866813844, + "tokens_seen": 225705984 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752802407221665, + "loss": 1.678, + "theoretical_loss": 4.318831917745458, + "tokens_seen": 225771520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527863590772318, + "loss": 1.6995, + "theoretical_loss": 4.318679025495197, + "tokens_seen": 225837056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527703109327984, + "loss": 1.6975, + "theoretical_loss": 4.318526190025478, + "tokens_seen": 225902592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527542627883652, + "loss": 1.7384, + "theoretical_loss": 4.318373411298749, + "tokens_seen": 225968128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527382146439318, + "loss": 1.6878, + "theoretical_loss": 4.318220689277496, + "tokens_seen": 226033664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 81462, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.65253746509552, + "objective/train/theoretical_loss": 4.318068023924241, + "objective/train/tokens_used": 246559200, + "theoretical_loss": 4.318068023924241, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527221664994986, + "loss": 1.7266, + "theoretical_loss": 4.318068023924241, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527061183550652, + "loss": 1.536, + "theoretical_loss": 4.317915415201538, + "tokens_seen": 226164736 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752690070210632, + "loss": 1.512, + "theoretical_loss": 4.317762863071982, + "tokens_seen": 226230272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007526740220661986, + "loss": 1.5732, + "theoretical_loss": 4.317610367498201, + "tokens_seen": 226295808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007526579739217654, + "loss": 1.7787, + "theoretical_loss": 4.317457928442856, + "tokens_seen": 226361344 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752641925777332, + "loss": 1.6781, + "theoretical_loss": 4.317305545868648, + "tokens_seen": 226426880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007526258776328988, + "loss": 1.7043, + "theoretical_loss": 4.317153219738311, + "tokens_seen": 226492416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007526098294884654, + "loss": 1.6454, + "theoretical_loss": 4.317000950014615, + "tokens_seen": 226557952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525937813440322, + "loss": 1.5971, + "theoretical_loss": 4.316848736660363, + "tokens_seen": 226623488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525777331995988, + "loss": 1.6237, + "theoretical_loss": 4.316696579638396, + "tokens_seen": 226689024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525616850551656, + "loss": 1.6166, + "theoretical_loss": 4.316544478911592, + "tokens_seen": 226754560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525456369107322, + "loss": 1.7207, + "theoretical_loss": 4.316392434442858, + "tokens_seen": 226820096 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752529588766299, + "loss": 1.6948, + "theoretical_loss": 4.316240446195142, + "tokens_seen": 226885632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525135406218656, + "loss": 1.6371, + "theoretical_loss": 4.3160885141314225, + "tokens_seen": 226951168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524974924774324, + "loss": 1.6582, + "theoretical_loss": 4.315936638214718, + "tokens_seen": 227016704 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752481444332999, + "loss": 1.6293, + "theoretical_loss": 4.315784818408079, + "tokens_seen": 227082240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524653961885658, + "loss": 1.6876, + "theoretical_loss": 4.31563305467459, + "tokens_seen": 227147776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524493480441324, + "loss": 1.6117, + "theoretical_loss": 4.315481346977374, + "tokens_seen": 227213312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524332998996992, + "loss": 1.5356, + "theoretical_loss": 4.315329695279584, + "tokens_seen": 227278848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524172517552658, + "loss": 1.7156, + "theoretical_loss": 4.315178099544413, + "tokens_seen": 227344384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524012036108326, + "loss": 1.5758, + "theoretical_loss": 4.315026559735085, + "tokens_seen": 227409920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523851554663992, + "loss": 1.7364, + "theoretical_loss": 4.314875075814861, + "tokens_seen": 227475456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523691073219659, + "loss": 1.716, + "theoretical_loss": 4.314723647747035, + "tokens_seen": 227540992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523530591775327, + "loss": 1.5991, + "theoretical_loss": 4.314572275494937, + "tokens_seen": 227606528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523370110330993, + "loss": 1.4546, + "theoretical_loss": 4.3144209590219305, + "tokens_seen": 227672064 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752320962888666, + "loss": 1.6459, + "theoretical_loss": 4.314269698291415, + "tokens_seen": 227737600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523049147442327, + "loss": 1.7276, + "theoretical_loss": 4.3141184932668235, + "tokens_seen": 227803136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522888665997995, + "loss": 1.6812, + "theoretical_loss": 4.3139673439116235, + "tokens_seen": 227868672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522728184553661, + "loss": 1.7196, + "theoretical_loss": 4.313816250189317, + "tokens_seen": 227934208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522567703109329, + "loss": 1.6224, + "theoretical_loss": 4.313665212063441, + "tokens_seen": 227999744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522407221664995, + "loss": 1.6276, + "theoretical_loss": 4.313514229497566, + "tokens_seen": 228065280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522246740220661, + "loss": 1.5987, + "theoretical_loss": 4.313363302455299, + "tokens_seen": 228130816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522086258776329, + "loss": 1.7328, + "theoretical_loss": 4.313212430900277, + "tokens_seen": 228196352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521925777331995, + "loss": 1.5894, + "theoretical_loss": 4.313061614796176, + "tokens_seen": 228261888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521765295887663, + "loss": 1.766, + "theoretical_loss": 4.312910854106702, + "tokens_seen": 228327424 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752160481444333, + "loss": 1.6051, + "theoretical_loss": 4.3127601487956, + "tokens_seen": 228392960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521444332998997, + "loss": 1.7288, + "theoretical_loss": 4.312609498826644, + "tokens_seen": 228458496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521283851554664, + "loss": 1.5024, + "theoretical_loss": 4.312458904163645, + "tokens_seen": 228524032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521123370110331, + "loss": 1.7327, + "theoretical_loss": 4.312308364770448, + "tokens_seen": 228589568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520962888665998, + "loss": 1.5368, + "theoretical_loss": 4.312157880610931, + "tokens_seen": 228655104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520802407221665, + "loss": 1.5935, + "theoretical_loss": 4.312007451649006, + "tokens_seen": 228720640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520641925777332, + "loss": 1.6157, + "theoretical_loss": 4.31185707784862, + "tokens_seen": 228786176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520481444332999, + "loss": 1.5835, + "theoretical_loss": 4.311706759173753, + "tokens_seen": 228851712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520320962888666, + "loss": 1.5946, + "theoretical_loss": 4.311556495588419, + "tokens_seen": 228917248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520160481444333, + "loss": 1.6795, + "theoretical_loss": 4.3114062870566645, + "tokens_seen": 228982784 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752, + "loss": 1.5991, + "theoretical_loss": 4.311256133542573, + "tokens_seen": 229048320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519839518555667, + "loss": 1.6723, + "theoretical_loss": 4.311106035010259, + "tokens_seen": 229113856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519679037111334, + "loss": 1.5913, + "theoretical_loss": 4.310955991423871, + "tokens_seen": 229179392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519518555667001, + "loss": 1.6719, + "theoretical_loss": 4.310806002747592, + "tokens_seen": 229244928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519358074222669, + "loss": 1.5921, + "theoretical_loss": 4.310656068945637, + "tokens_seen": 229310464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 82681, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5853383541107178, + "objective/train/theoretical_loss": 4.310506189982257, + "objective/train/tokens_used": 249836000, + "theoretical_loss": 4.310506189982257, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519197592778335, + "loss": 1.601, + "theoretical_loss": 4.310506189982257, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519037111334003, + "loss": 1.7332, + "theoretical_loss": 4.310356365821734, + "tokens_seen": 229441536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518876629889669, + "loss": 1.7276, + "theoretical_loss": 4.310206596428387, + "tokens_seen": 229507072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518716148445337, + "loss": 1.6907, + "theoretical_loss": 4.310056881766561, + "tokens_seen": 229572608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518555667001003, + "loss": 1.6095, + "theoretical_loss": 4.309907221800645, + "tokens_seen": 229638144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518395185556671, + "loss": 1.7901, + "theoretical_loss": 4.309757616495053, + "tokens_seen": 229703680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518234704112337, + "loss": 1.6816, + "theoretical_loss": 4.309608065814234, + "tokens_seen": 229769216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518074222668005, + "loss": 1.5572, + "theoretical_loss": 4.309458569722673, + "tokens_seen": 229834752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517913741223671, + "loss": 1.7167, + "theoretical_loss": 4.309309128184886, + "tokens_seen": 229900288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517753259779339, + "loss": 1.6913, + "theoretical_loss": 4.309159741165422, + "tokens_seen": 229965824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517592778335005, + "loss": 1.6231, + "theoretical_loss": 4.309010408628865, + "tokens_seen": 230031360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517432296890673, + "loss": 1.5985, + "theoretical_loss": 4.308861130539829, + "tokens_seen": 230096896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517271815446339, + "loss": 1.6563, + "theoretical_loss": 4.308711906862965, + "tokens_seen": 230162432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517111334002007, + "loss": 1.6262, + "theoretical_loss": 4.308562737562953, + "tokens_seen": 230227968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516950852557673, + "loss": 1.7089, + "theoretical_loss": 4.30841362260451, + "tokens_seen": 230293504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516790371113341, + "loss": 1.612, + "theoretical_loss": 4.308264561952381, + "tokens_seen": 230359040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516629889669007, + "loss": 1.6092, + "theoretical_loss": 4.30811555557135, + "tokens_seen": 230424576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516469408224675, + "loss": 1.6265, + "theoretical_loss": 4.307966603426227, + "tokens_seen": 230490112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516308926780341, + "loss": 1.7801, + "theoretical_loss": 4.307817705481861, + "tokens_seen": 230555648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516148445336009, + "loss": 1.7435, + "theoretical_loss": 4.307668861703131, + "tokens_seen": 230621184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515987963891675, + "loss": 1.6081, + "theoretical_loss": 4.307520072054947, + "tokens_seen": 230686720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515827482447343, + "loss": 1.5812, + "theoretical_loss": 4.307371336502254, + "tokens_seen": 230752256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515667001003009, + "loss": 1.6561, + "theoretical_loss": 4.307222655010031, + "tokens_seen": 230817792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515506519558677, + "loss": 1.6483, + "theoretical_loss": 4.307074027543287, + "tokens_seen": 230883328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515346038114343, + "loss": 1.7229, + "theoretical_loss": 4.3069254540670645, + "tokens_seen": 230948864 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515185556670011, + "loss": 1.7828, + "theoretical_loss": 4.306776934546436, + "tokens_seen": 231014400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515025075225677, + "loss": 1.5098, + "theoretical_loss": 4.306628468946512, + "tokens_seen": 231079936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514864593781345, + "loss": 1.55, + "theoretical_loss": 4.306480057232431, + "tokens_seen": 231145472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514704112337011, + "loss": 1.6725, + "theoretical_loss": 4.306331699369366, + "tokens_seen": 231211008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514543630892679, + "loss": 1.6515, + "theoretical_loss": 4.30618339532252, + "tokens_seen": 231276544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514383149448345, + "loss": 1.7387, + "theoretical_loss": 4.306035145057132, + "tokens_seen": 231342080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514222668004013, + "loss": 1.6858, + "theoretical_loss": 4.30588694853847, + "tokens_seen": 231407616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514062186559679, + "loss": 1.6754, + "theoretical_loss": 4.305738805731836, + "tokens_seen": 231473152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513901705115347, + "loss": 1.6655, + "theoretical_loss": 4.305590716602563, + "tokens_seen": 231538688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513741223671013, + "loss": 1.6448, + "theoretical_loss": 4.305442681116018, + "tokens_seen": 231604224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513580742226681, + "loss": 1.6165, + "theoretical_loss": 4.305294699237598, + "tokens_seen": 231669760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513420260782347, + "loss": 1.5662, + "theoretical_loss": 4.305146770932734, + "tokens_seen": 231735296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513259779338015, + "loss": 1.6911, + "theoretical_loss": 4.304998896166888, + "tokens_seen": 231800832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513099297893681, + "loss": 1.709, + "theoretical_loss": 4.304851074905554, + "tokens_seen": 231866368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512938816449349, + "loss": 1.6598, + "theoretical_loss": 4.304703307114258, + "tokens_seen": 231931904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512778335005015, + "loss": 1.6776, + "theoretical_loss": 4.304555592758559, + "tokens_seen": 231997440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512617853560683, + "loss": 1.5082, + "theoretical_loss": 4.304407931804047, + "tokens_seen": 232062976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512457372116349, + "loss": 1.6469, + "theoretical_loss": 4.304260324216344, + "tokens_seen": 232128512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512296890672017, + "loss": 1.6334, + "theoretical_loss": 4.304112769961102, + "tokens_seen": 232194048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512136409227683, + "loss": 1.532, + "theoretical_loss": 4.303965269004008, + "tokens_seen": 232259584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511975927783351, + "loss": 1.7242, + "theoretical_loss": 4.30381782131078, + "tokens_seen": 232325120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511815446339017, + "loss": 1.6597, + "theoretical_loss": 4.303670426847166, + "tokens_seen": 232390656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511654964894685, + "loss": 1.5863, + "theoretical_loss": 4.303523085578946, + "tokens_seen": 232456192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511494483450351, + "loss": 1.6315, + "theoretical_loss": 4.303375797471935, + "tokens_seen": 232521728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511334002006019, + "loss": 1.6744, + "theoretical_loss": 4.303228562491974, + "tokens_seen": 232587264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 83903, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6346615552902222, + "objective/train/theoretical_loss": 4.303081380604939, + "objective/train/tokens_used": 253112800, + "theoretical_loss": 4.303081380604939, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511173520561685, + "loss": 1.6698, + "theoretical_loss": 4.303081380604939, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511013039117353, + "loss": 1.5696, + "theoretical_loss": 4.302934251776739, + "tokens_seen": 232718336 + }, + { + "epoch": 0.07, + "learning_rate": 0.000751085255767302, + "loss": 1.7224, + "theoretical_loss": 4.302787175973311, + "tokens_seen": 232783872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510692076228687, + "loss": 1.7526, + "theoretical_loss": 4.302640153160625, + "tokens_seen": 232849408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510531594784353, + "loss": 1.6315, + "theoretical_loss": 4.302493183304683, + "tokens_seen": 232914944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510371113340021, + "loss": 1.6435, + "theoretical_loss": 4.302346266371517, + "tokens_seen": 232980480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510210631895687, + "loss": 1.6499, + "theoretical_loss": 4.302199402327191, + "tokens_seen": 233046016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510050150451355, + "loss": 1.6708, + "theoretical_loss": 4.302052591137802, + "tokens_seen": 233111552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509889669007021, + "loss": 1.6101, + "theoretical_loss": 4.301905832769474, + "tokens_seen": 233177088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509729187562688, + "loss": 1.6334, + "theoretical_loss": 4.301759127188367, + "tokens_seen": 233242624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509568706118356, + "loss": 1.7197, + "theoretical_loss": 4.301612474360669, + "tokens_seen": 233308160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509408224674022, + "loss": 1.7176, + "theoretical_loss": 4.301465874252601, + "tokens_seen": 233373696 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750924774322969, + "loss": 1.7187, + "theoretical_loss": 4.301319326830413, + "tokens_seen": 233439232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509087261785356, + "loss": 1.7268, + "theoretical_loss": 4.301172832060389, + "tokens_seen": 233504768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007508926780341024, + "loss": 1.6295, + "theoretical_loss": 4.3010263899088415, + "tokens_seen": 233570304 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750876629889669, + "loss": 1.6408, + "theoretical_loss": 4.300880000342114, + "tokens_seen": 233635840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007508605817452358, + "loss": 1.6239, + "theoretical_loss": 4.300733663326583, + "tokens_seen": 233701376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007508445336008024, + "loss": 1.5342, + "theoretical_loss": 4.300587378828655, + "tokens_seen": 233766912 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750828485456369, + "loss": 1.7235, + "theoretical_loss": 4.300441146814766, + "tokens_seen": 233832448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007508124373119358, + "loss": 1.619, + "theoretical_loss": 4.300294967251386, + "tokens_seen": 233897984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507963891675024, + "loss": 1.6433, + "theoretical_loss": 4.300148840105011, + "tokens_seen": 233963520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507803410230692, + "loss": 1.5639, + "theoretical_loss": 4.300002765342173, + "tokens_seen": 234029056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507642928786359, + "loss": 1.672, + "theoretical_loss": 4.299856742929431, + "tokens_seen": 234094592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507482447342026, + "loss": 1.6359, + "theoretical_loss": 4.299710772833377, + "tokens_seen": 234160128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507321965897694, + "loss": 1.7174, + "theoretical_loss": 4.299564855020631, + "tokens_seen": 234225664 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750716148445336, + "loss": 1.5812, + "theoretical_loss": 4.299418989457848, + "tokens_seen": 234291200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507001003009028, + "loss": 1.6371, + "theoretical_loss": 4.299273176111709, + "tokens_seen": 234356736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506840521564694, + "loss": 1.6428, + "theoretical_loss": 4.299127414948927, + "tokens_seen": 234422272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506680040120362, + "loss": 1.654, + "theoretical_loss": 4.298981705936248, + "tokens_seen": 234487808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506519558676028, + "loss": 1.7671, + "theoretical_loss": 4.298836049040444, + "tokens_seen": 234553344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506359077231696, + "loss": 1.7702, + "theoretical_loss": 4.298690444228322, + "tokens_seen": 234618880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506198595787362, + "loss": 1.5735, + "theoretical_loss": 4.298544891466717, + "tokens_seen": 234684416 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750603811434303, + "loss": 1.6077, + "theoretical_loss": 4.298399390722493, + "tokens_seen": 234749952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505877632898696, + "loss": 1.725, + "theoretical_loss": 4.298253941962548, + "tokens_seen": 234815488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505717151454364, + "loss": 1.5374, + "theoretical_loss": 4.2981085451538075, + "tokens_seen": 234881024 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750555667001003, + "loss": 1.6508, + "theoretical_loss": 4.297963200263228, + "tokens_seen": 234946560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505396188565698, + "loss": 1.7077, + "theoretical_loss": 4.297817907257797, + "tokens_seen": 235012096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505235707121364, + "loss": 1.6419, + "theoretical_loss": 4.29767266610453, + "tokens_seen": 235077632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505075225677032, + "loss": 1.6664, + "theoretical_loss": 4.297527476770476, + "tokens_seen": 235143168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504914744232698, + "loss": 1.6296, + "theoretical_loss": 4.297382339222711, + "tokens_seen": 235208704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504754262788366, + "loss": 1.693, + "theoretical_loss": 4.297237253428342, + "tokens_seen": 235274240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504593781344032, + "loss": 1.8014, + "theoretical_loss": 4.297092219354509, + "tokens_seen": 235339776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075044332998997, + "loss": 1.7015, + "theoretical_loss": 4.296947236968376, + "tokens_seen": 235405312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504272818455366, + "loss": 1.6181, + "theoretical_loss": 4.296802306237143, + "tokens_seen": 235470848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504112337011034, + "loss": 1.7072, + "theoretical_loss": 4.296657427128036, + "tokens_seen": 235536384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075039518555667, + "loss": 1.629, + "theoretical_loss": 4.296512599608311, + "tokens_seen": 235601920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503791374122368, + "loss": 1.6502, + "theoretical_loss": 4.296367823645259, + "tokens_seen": 235667456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503630892678034, + "loss": 1.671, + "theoretical_loss": 4.296223099206193, + "tokens_seen": 235732992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503470411233702, + "loss": 1.7477, + "theoretical_loss": 4.296078426258462, + "tokens_seen": 235798528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503309929789368, + "loss": 1.521, + "theoretical_loss": 4.295933804769442, + "tokens_seen": 235864064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 84743, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6421600580215454, + "objective/train/theoretical_loss": 4.295789234706538, + "objective/train/tokens_used": 256389600, + "theoretical_loss": 4.295789234706538, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503149448345036, + "loss": 1.7047, + "theoretical_loss": 4.295789234706538, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502988966900702, + "loss": 1.7024, + "theoretical_loss": 4.295644716037188, + "tokens_seen": 235995136 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750282848545637, + "loss": 1.7471, + "theoretical_loss": 4.295500248728856, + "tokens_seen": 236060672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502668004012036, + "loss": 1.7045, + "theoretical_loss": 4.295355832749038, + "tokens_seen": 236126208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502507522567704, + "loss": 1.7213, + "theoretical_loss": 4.295211468065258, + "tokens_seen": 236191744 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750234704112337, + "loss": 1.715, + "theoretical_loss": 4.295067154645071, + "tokens_seen": 236257280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502186559679038, + "loss": 1.6261, + "theoretical_loss": 4.294922892456061, + "tokens_seen": 236322816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502026078234704, + "loss": 1.601, + "theoretical_loss": 4.294778681465841, + "tokens_seen": 236388352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501865596790372, + "loss": 1.4909, + "theoretical_loss": 4.294634521642053, + "tokens_seen": 236453888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501705115346038, + "loss": 1.7144, + "theoretical_loss": 4.294490412952371, + "tokens_seen": 236519424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501544633901706, + "loss": 1.7742, + "theoretical_loss": 4.294346355364494, + "tokens_seen": 236584960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501384152457372, + "loss": 1.678, + "theoretical_loss": 4.294202348846156, + "tokens_seen": 236650496 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750122367101304, + "loss": 1.6133, + "theoretical_loss": 4.294058393365114, + "tokens_seen": 236716032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501063189568706, + "loss": 1.685, + "theoretical_loss": 4.293914488889161, + "tokens_seen": 236781568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500902708124374, + "loss": 1.6133, + "theoretical_loss": 4.293770635386112, + "tokens_seen": 236847104 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750074222668004, + "loss": 1.716, + "theoretical_loss": 4.293626832823818, + "tokens_seen": 236912640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500581745235708, + "loss": 1.6319, + "theoretical_loss": 4.293483081170153, + "tokens_seen": 236978176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500421263791374, + "loss": 1.5912, + "theoretical_loss": 4.293339380393027, + "tokens_seen": 237043712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500260782347042, + "loss": 1.6264, + "theoretical_loss": 4.293195730460372, + "tokens_seen": 237109248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500100300902708, + "loss": 1.5781, + "theoretical_loss": 4.293052131340154, + "tokens_seen": 237174784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007499939819458376, + "loss": 1.7007, + "theoretical_loss": 4.292908583000365, + "tokens_seen": 237240320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007499779338014042, + "loss": 1.663, + "theoretical_loss": 4.29276508540903, + "tokens_seen": 237305856 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749961885656971, + "loss": 1.5671, + "theoretical_loss": 4.292621638534198, + "tokens_seen": 237371392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007499458375125376, + "loss": 1.7502, + "theoretical_loss": 4.29247824234395, + "tokens_seen": 237436928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007499297893681044, + "loss": 1.715, + "theoretical_loss": 4.292334896806397, + "tokens_seen": 237502464 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749913741223671, + "loss": 1.6047, + "theoretical_loss": 4.292191601889673, + "tokens_seen": 237568000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498976930792378, + "loss": 1.6171, + "theoretical_loss": 4.2920483575619475, + "tokens_seen": 237633536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498816449348044, + "loss": 1.7442, + "theoretical_loss": 4.291905163791417, + "tokens_seen": 237699072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498655967903712, + "loss": 1.6901, + "theoretical_loss": 4.291762020546304, + "tokens_seen": 237764608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498495486459378, + "loss": 1.6801, + "theoretical_loss": 4.291618927794862, + "tokens_seen": 237830144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498335005015046, + "loss": 1.669, + "theoretical_loss": 4.291475885505374, + "tokens_seen": 237895680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498174523570712, + "loss": 1.6468, + "theoretical_loss": 4.291332893646149, + "tokens_seen": 237961216 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749801404212638, + "loss": 1.5978, + "theoretical_loss": 4.291189952185527, + "tokens_seen": 238026752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497853560682046, + "loss": 1.5644, + "theoretical_loss": 4.291047061091875, + "tokens_seen": 238092288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497693079237714, + "loss": 1.5605, + "theoretical_loss": 4.290904220333591, + "tokens_seen": 238157824 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749753259779338, + "loss": 1.6864, + "theoretical_loss": 4.2907614298790975, + "tokens_seen": 238223360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497372116349048, + "loss": 1.685, + "theoretical_loss": 4.290618689696848, + "tokens_seen": 238288896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497211634904714, + "loss": 1.6605, + "theoretical_loss": 4.290475999755326, + "tokens_seen": 238354432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497051153460382, + "loss": 1.7451, + "theoretical_loss": 4.29033336002304, + "tokens_seen": 238419968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496890672016048, + "loss": 1.5776, + "theoretical_loss": 4.290190770468529, + "tokens_seen": 238485504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496730190571716, + "loss": 1.6556, + "theoretical_loss": 4.290048231060361, + "tokens_seen": 238551040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496569709127382, + "loss": 1.5922, + "theoretical_loss": 4.289905741767129, + "tokens_seen": 238616576 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749640922768305, + "loss": 1.6411, + "theoretical_loss": 4.289763302557457, + "tokens_seen": 238682112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496248746238716, + "loss": 1.7339, + "theoretical_loss": 4.289620913399998, + "tokens_seen": 238747648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496088264794384, + "loss": 1.61, + "theoretical_loss": 4.2894785742634305, + "tokens_seen": 238813184 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749592778335005, + "loss": 1.6489, + "theoretical_loss": 4.289336285116463, + "tokens_seen": 238878720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495767301905718, + "loss": 1.6366, + "theoretical_loss": 4.289194045927832, + "tokens_seen": 238944256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495606820461385, + "loss": 1.57, + "theoretical_loss": 4.289051856666303, + "tokens_seen": 239009792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495446339017051, + "loss": 1.659, + "theoretical_loss": 4.288909717300667, + "tokens_seen": 239075328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495285857572719, + "loss": 1.6939, + "theoretical_loss": 4.288767627799745, + "tokens_seen": 239140864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 85669, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6066745519638062, + "objective/train/theoretical_loss": 4.288625588132385, + "objective/train/tokens_used": 259666400, + "theoretical_loss": 4.288625588132385, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495125376128385, + "loss": 1.6276, + "theoretical_loss": 4.288625588132385, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494964894684053, + "loss": 1.715, + "theoretical_loss": 4.288483598267465, + "tokens_seen": 239271936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494804413239719, + "loss": 1.5889, + "theoretical_loss": 4.288341658173888, + "tokens_seen": 239337472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494643931795387, + "loss": 1.7141, + "theoretical_loss": 4.288199767820587, + "tokens_seen": 239403008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494483450351053, + "loss": 1.682, + "theoretical_loss": 4.2880579271765225, + "tokens_seen": 239468544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494322968906721, + "loss": 1.5611, + "theoretical_loss": 4.287916136210682, + "tokens_seen": 239534080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494162487462387, + "loss": 1.6503, + "theoretical_loss": 4.287774394892082, + "tokens_seen": 239599616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494002006018055, + "loss": 1.6604, + "theoretical_loss": 4.287632703189766, + "tokens_seen": 239665152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493841524573721, + "loss": 1.5737, + "theoretical_loss": 4.2874910610728065, + "tokens_seen": 239730688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493681043129389, + "loss": 1.6437, + "theoretical_loss": 4.287349468510302, + "tokens_seen": 239796224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493520561685055, + "loss": 1.4903, + "theoretical_loss": 4.28720792547138, + "tokens_seen": 239861760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493360080240723, + "loss": 1.6124, + "theoretical_loss": 4.287066431925194, + "tokens_seen": 239927296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493199598796389, + "loss": 1.5895, + "theoretical_loss": 4.286924987840926, + "tokens_seen": 239992832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493039117352057, + "loss": 1.7164, + "theoretical_loss": 4.2867835931877885, + "tokens_seen": 240058368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492878635907723, + "loss": 1.6419, + "theoretical_loss": 4.286642247935017, + "tokens_seen": 240123904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492718154463391, + "loss": 1.6765, + "theoretical_loss": 4.286500952051876, + "tokens_seen": 240189440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492557673019057, + "loss": 1.5405, + "theoretical_loss": 4.286359705507659, + "tokens_seen": 240254976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492397191574725, + "loss": 1.558, + "theoretical_loss": 4.286218508271686, + "tokens_seen": 240320512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492236710130391, + "loss": 1.5783, + "theoretical_loss": 4.286077360313303, + "tokens_seen": 240386048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492076228686059, + "loss": 1.5213, + "theoretical_loss": 4.285936261601887, + "tokens_seen": 240451584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491915747241725, + "loss": 1.5629, + "theoretical_loss": 4.285795212106839, + "tokens_seen": 240517120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491755265797393, + "loss": 1.7373, + "theoretical_loss": 4.2856542117975875, + "tokens_seen": 240582656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491594784353059, + "loss": 1.7176, + "theoretical_loss": 4.285513260643591, + "tokens_seen": 240648192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491434302908727, + "loss": 1.6888, + "theoretical_loss": 4.285372358614332, + "tokens_seen": 240713728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491273821464393, + "loss": 1.6917, + "theoretical_loss": 4.285231505679324, + "tokens_seen": 240779264 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491113340020061, + "loss": 1.6539, + "theoretical_loss": 4.285090701808103, + "tokens_seen": 240844800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490952858575727, + "loss": 1.612, + "theoretical_loss": 4.284949946970236, + "tokens_seen": 240910336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490792377131395, + "loss": 1.7414, + "theoretical_loss": 4.284809241135315, + "tokens_seen": 240975872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490631895687061, + "loss": 1.6637, + "theoretical_loss": 4.284668584272962, + "tokens_seen": 241041408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490471414242729, + "loss": 1.6183, + "theoretical_loss": 4.284527976352823, + "tokens_seen": 241106944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490310932798395, + "loss": 1.6879, + "theoretical_loss": 4.284387417344572, + "tokens_seen": 241172480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490150451354063, + "loss": 1.6791, + "theoretical_loss": 4.28424690721791, + "tokens_seen": 241238016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489989969909729, + "loss": 1.7164, + "theoretical_loss": 4.284106445942565, + "tokens_seen": 241303552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489829488465397, + "loss": 1.6794, + "theoretical_loss": 4.2839660334882925, + "tokens_seen": 241369088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489669007021063, + "loss": 1.4591, + "theoretical_loss": 4.283825669824875, + "tokens_seen": 241434624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489508525576731, + "loss": 1.6535, + "theoretical_loss": 4.28368535492212, + "tokens_seen": 241500160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489348044132397, + "loss": 1.6211, + "theoretical_loss": 4.283545088749865, + "tokens_seen": 241565696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489187562688065, + "loss": 1.6274, + "theoretical_loss": 4.2834048712779715, + "tokens_seen": 241631232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489027081243731, + "loss": 1.5905, + "theoretical_loss": 4.28326470247633, + "tokens_seen": 241696768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488866599799399, + "loss": 1.6188, + "theoretical_loss": 4.283124582314856, + "tokens_seen": 241762304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488706118355065, + "loss": 1.5419, + "theoretical_loss": 4.282984510763493, + "tokens_seen": 241827840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488545636910733, + "loss": 1.6195, + "theoretical_loss": 4.282844487792209, + "tokens_seen": 241893376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488385155466399, + "loss": 1.6174, + "theoretical_loss": 4.282704513371002, + "tokens_seen": 241958912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488224674022067, + "loss": 1.5593, + "theoretical_loss": 4.282564587469896, + "tokens_seen": 242024448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488064192577733, + "loss": 1.5889, + "theoretical_loss": 4.282424710058938, + "tokens_seen": 242089984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487903711133401, + "loss": 1.5381, + "theoretical_loss": 4.282284881108206, + "tokens_seen": 242155520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487743229689067, + "loss": 1.6208, + "theoretical_loss": 4.282145100587802, + "tokens_seen": 242221056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487582748244735, + "loss": 1.6344, + "theoretical_loss": 4.282005368467856, + "tokens_seen": 242286592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487422266800401, + "loss": 1.6803, + "theoretical_loss": 4.2818656847185235, + "tokens_seen": 242352128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487261785356069, + "loss": 1.523, + "theoretical_loss": 4.281726049309987, + "tokens_seen": 242417664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 86947, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.533555507659912, + "objective/train/theoretical_loss": 4.281586462212455, + "objective/train/tokens_used": 262943200, + "theoretical_loss": 4.281586462212455, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487101303911735, + "loss": 1.5863, + "theoretical_loss": 4.281586462212455, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486940822467403, + "loss": 1.6372, + "theoretical_loss": 4.281446923396162, + "tokens_seen": 242548736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486780341023069, + "loss": 1.6006, + "theoretical_loss": 4.281307432831371, + "tokens_seen": 242614272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486619859578737, + "loss": 1.5637, + "theoretical_loss": 4.281167990488369, + "tokens_seen": 242679808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486459378134403, + "loss": 1.6963, + "theoretical_loss": 4.281028596337469, + "tokens_seen": 242745344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486298896690071, + "loss": 1.5761, + "theoretical_loss": 4.280889250349014, + "tokens_seen": 242810880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486138415245737, + "loss": 1.5397, + "theoretical_loss": 4.280749952493368, + "tokens_seen": 242876416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485977933801405, + "loss": 1.6975, + "theoretical_loss": 4.280610702740926, + "tokens_seen": 242941952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485817452357071, + "loss": 1.5826, + "theoretical_loss": 4.280471501062106, + "tokens_seen": 243007488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485656970912739, + "loss": 1.7064, + "theoretical_loss": 4.280332347427353, + "tokens_seen": 243073024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485496489468405, + "loss": 1.6329, + "theoretical_loss": 4.280193241807139, + "tokens_seen": 243138560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485336008024073, + "loss": 1.5267, + "theoretical_loss": 4.280054184171963, + "tokens_seen": 243204096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485175526579739, + "loss": 1.6792, + "theoretical_loss": 4.279915174492345, + "tokens_seen": 243269632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485015045135407, + "loss": 1.638, + "theoretical_loss": 4.279776212738838, + "tokens_seen": 243335168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484854563691073, + "loss": 1.6731, + "theoretical_loss": 4.279637298882016, + "tokens_seen": 243400704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484694082246741, + "loss": 1.5625, + "theoretical_loss": 4.27949843289248, + "tokens_seen": 243466240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484533600802407, + "loss": 1.6964, + "theoretical_loss": 4.2793596147408595, + "tokens_seen": 243531776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484373119358075, + "loss": 1.6091, + "theoretical_loss": 4.279220844397806, + "tokens_seen": 243597312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484212637913741, + "loss": 1.6271, + "theoretical_loss": 4.279082121834001, + "tokens_seen": 243662848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484052156469409, + "loss": 1.6291, + "theoretical_loss": 4.278943447020148, + "tokens_seen": 243728384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483891675025075, + "loss": 1.7249, + "theoretical_loss": 4.278804819926978, + "tokens_seen": 243793920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483731193580743, + "loss": 1.656, + "theoretical_loss": 4.278666240525249, + "tokens_seen": 243859456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483570712136409, + "loss": 1.7174, + "theoretical_loss": 4.278527708785743, + "tokens_seen": 243924992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483410230692077, + "loss": 1.6751, + "theoretical_loss": 4.278389224679268, + "tokens_seen": 243990528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483249749247745, + "loss": 1.4917, + "theoretical_loss": 4.27825078817666, + "tokens_seen": 244056064 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483089267803411, + "loss": 1.5895, + "theoretical_loss": 4.2781123992487755, + "tokens_seen": 244121600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482928786359077, + "loss": 1.6703, + "theoretical_loss": 4.277974057866503, + "tokens_seen": 244187136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482768304914745, + "loss": 1.5676, + "theoretical_loss": 4.277835764000752, + "tokens_seen": 244252672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482607823470411, + "loss": 1.6488, + "theoretical_loss": 4.277697517622459, + "tokens_seen": 244318208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482447342026079, + "loss": 1.6331, + "theoretical_loss": 4.277559318702588, + "tokens_seen": 244383744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482286860581746, + "loss": 1.5206, + "theoretical_loss": 4.277421167212125, + "tokens_seen": 244449280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482126379137413, + "loss": 1.6105, + "theoretical_loss": 4.277283063122083, + "tokens_seen": 244514816 + }, + { + "epoch": 0.07, + "learning_rate": 0.000748196589769308, + "loss": 1.6427, + "theoretical_loss": 4.277145006403503, + "tokens_seen": 244580352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481805416248747, + "loss": 1.7534, + "theoretical_loss": 4.2770069970274465, + "tokens_seen": 244645888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481644934804414, + "loss": 1.6926, + "theoretical_loss": 4.276869034965005, + "tokens_seen": 244711424 + }, + { + "epoch": 0.07, + "learning_rate": 0.000748148445336008, + "loss": 1.7342, + "theoretical_loss": 4.276731120187293, + "tokens_seen": 244776960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481323971915748, + "loss": 1.6409, + "theoretical_loss": 4.2765932526654495, + "tokens_seen": 244842496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481163490471414, + "loss": 1.6483, + "theoretical_loss": 4.276455432370643, + "tokens_seen": 244908032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481003009027082, + "loss": 1.5419, + "theoretical_loss": 4.276317659274062, + "tokens_seen": 244973568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480842527582748, + "loss": 1.5604, + "theoretical_loss": 4.276179933346924, + "tokens_seen": 245039104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480682046138416, + "loss": 1.4418, + "theoretical_loss": 4.27604225456047, + "tokens_seen": 245104640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480521564694082, + "loss": 1.6223, + "theoretical_loss": 4.275904622885967, + "tokens_seen": 245170176 + }, + { + "epoch": 0.07, + "learning_rate": 0.000748036108324975, + "loss": 1.6681, + "theoretical_loss": 4.275767038294708, + "tokens_seen": 245235712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480200601805416, + "loss": 1.4692, + "theoretical_loss": 4.275629500758008, + "tokens_seen": 245301248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480040120361084, + "loss": 1.5969, + "theoretical_loss": 4.27549201024721, + "tokens_seen": 245366784 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747987963891675, + "loss": 1.6522, + "theoretical_loss": 4.275354566733682, + "tokens_seen": 245432320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479719157472418, + "loss": 1.6379, + "theoretical_loss": 4.2752171701888155, + "tokens_seen": 245497856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479558676028084, + "loss": 1.5753, + "theoretical_loss": 4.275079820584029, + "tokens_seen": 245563392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479398194583752, + "loss": 1.6753, + "theoretical_loss": 4.274942517890764, + "tokens_seen": 245628928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479237713139418, + "loss": 1.6301, + "theoretical_loss": 4.2748052620804895, + "tokens_seen": 245694464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 87859, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.655349850654602, + "objective/train/theoretical_loss": 4.274668053124696, + "objective/train/tokens_used": 266220000, + "theoretical_loss": 4.274668053124696, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479077231695086, + "loss": 1.6396, + "theoretical_loss": 4.274668053124696, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007478916750250752, + "loss": 1.5806, + "theoretical_loss": 4.2745308909949005, + "tokens_seen": 245825536 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747875626880642, + "loss": 1.7082, + "theoretical_loss": 4.274393775662648, + "tokens_seen": 245891072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007478595787362086, + "loss": 1.6241, + "theoretical_loss": 4.274256707099502, + "tokens_seen": 245956608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007478435305917754, + "loss": 1.5844, + "theoretical_loss": 4.274119685277058, + "tokens_seen": 246022144 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747827482447342, + "loss": 1.5921, + "theoretical_loss": 4.273982710166931, + "tokens_seen": 246087680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007478114343029088, + "loss": 1.6682, + "theoretical_loss": 4.273845781740762, + "tokens_seen": 246153216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477953861584754, + "loss": 1.5675, + "theoretical_loss": 4.273708899970218, + "tokens_seen": 246218752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477793380140422, + "loss": 1.7248, + "theoretical_loss": 4.273572064826991, + "tokens_seen": 246284288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477632898696088, + "loss": 1.6493, + "theoretical_loss": 4.273435276282795, + "tokens_seen": 246349824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477472417251756, + "loss": 1.6431, + "theoretical_loss": 4.2732985343093715, + "tokens_seen": 246415360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477311935807422, + "loss": 1.5834, + "theoretical_loss": 4.273161838878485, + "tokens_seen": 246480896 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747715145436309, + "loss": 1.649, + "theoretical_loss": 4.273025189961926, + "tokens_seen": 246546432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476990972918756, + "loss": 1.5257, + "theoretical_loss": 4.272888587531508, + "tokens_seen": 246611968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476830491474424, + "loss": 1.7432, + "theoretical_loss": 4.272752031559071, + "tokens_seen": 246677504 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747667001003009, + "loss": 1.5807, + "theoretical_loss": 4.272615522016476, + "tokens_seen": 246743040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476509528585758, + "loss": 1.5903, + "theoretical_loss": 4.272479058875614, + "tokens_seen": 246808576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476349047141424, + "loss": 1.5591, + "theoretical_loss": 4.272342642108396, + "tokens_seen": 246874112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476188565697092, + "loss": 1.5049, + "theoretical_loss": 4.272206271686758, + "tokens_seen": 246939648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476028084252758, + "loss": 1.6674, + "theoretical_loss": 4.272069947582662, + "tokens_seen": 247005184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007475867602808426, + "loss": 1.5916, + "theoretical_loss": 4.271933669768094, + "tokens_seen": 247070720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007475707121364092, + "loss": 1.6944, + "theoretical_loss": 4.271797438215064, + "tokens_seen": 247136256 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747554663991976, + "loss": 1.5855, + "theoretical_loss": 4.271661252895605, + "tokens_seen": 247201792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007475386158475426, + "loss": 1.7893, + "theoretical_loss": 4.271525113781777, + "tokens_seen": 247267328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007475225677031094, + "loss": 1.6591, + "theoretical_loss": 4.271389020845662, + "tokens_seen": 247332864 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747506519558676, + "loss": 1.6706, + "theoretical_loss": 4.27125297405937, + "tokens_seen": 247398400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007474904714142428, + "loss": 1.6419, + "theoretical_loss": 4.271116973395028, + "tokens_seen": 247463936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474744232698094, + "loss": 1.7433, + "theoretical_loss": 4.270981018824795, + "tokens_seen": 247529472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474583751253762, + "loss": 1.6313, + "theoretical_loss": 4.270845110320851, + "tokens_seen": 247595008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474423269809428, + "loss": 1.6649, + "theoretical_loss": 4.270709247855397, + "tokens_seen": 247660544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474262788365096, + "loss": 1.5666, + "theoretical_loss": 4.270573431400663, + "tokens_seen": 247726080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474102306920762, + "loss": 1.729, + "theoretical_loss": 4.270437660928902, + "tokens_seen": 247791616 + }, + { + "epoch": 0.08, + "learning_rate": 0.000747394182547643, + "loss": 1.6638, + "theoretical_loss": 4.270301936412389, + "tokens_seen": 247857152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007473781344032096, + "loss": 1.6416, + "theoretical_loss": 4.2701662578234245, + "tokens_seen": 247922688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007473620862587764, + "loss": 1.5788, + "theoretical_loss": 4.270030625134334, + "tokens_seen": 247988224 + }, + { + "epoch": 0.08, + "learning_rate": 0.000747346038114343, + "loss": 1.5882, + "theoretical_loss": 4.269895038317464, + "tokens_seen": 248053760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007473299899699098, + "loss": 1.5104, + "theoretical_loss": 4.269759497345188, + "tokens_seen": 248119296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007473139418254764, + "loss": 1.6717, + "theoretical_loss": 4.269624002189901, + "tokens_seen": 248184832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472978936810432, + "loss": 1.7941, + "theoretical_loss": 4.269488552824024, + "tokens_seen": 248250368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472818455366098, + "loss": 1.7017, + "theoretical_loss": 4.269353149220001, + "tokens_seen": 248315904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472657973921766, + "loss": 1.5691, + "theoretical_loss": 4.2692177913503, + "tokens_seen": 248381440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472497492477432, + "loss": 1.5977, + "theoretical_loss": 4.2690824791874125, + "tokens_seen": 248446976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074723370110331, + "loss": 1.6137, + "theoretical_loss": 4.2689472127038535, + "tokens_seen": 248512512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472176529588766, + "loss": 1.5904, + "theoretical_loss": 4.2688119918721625, + "tokens_seen": 248578048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472016048144434, + "loss": 1.6154, + "theoretical_loss": 4.268676816664902, + "tokens_seen": 248643584 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074718555667001, + "loss": 1.6636, + "theoretical_loss": 4.26854168705466, + "tokens_seen": 248709120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007471695085255768, + "loss": 1.6526, + "theoretical_loss": 4.2684066030140455, + "tokens_seen": 248774656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007471534603811434, + "loss": 1.5885, + "theoretical_loss": 4.268271564515693, + "tokens_seen": 248840192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007471374122367102, + "loss": 1.6694, + "theoretical_loss": 4.26813657153226, + "tokens_seen": 248905728 + }, + { + "epoch": 0.08, + "learning_rate": 0.000747121364092277, + "loss": 1.6456, + "theoretical_loss": 4.268001624036429, + "tokens_seen": 248971264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 89113, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5617129802703857, + "objective/train/theoretical_loss": 4.267866722000902, + "objective/train/tokens_used": 269496800, + "theoretical_loss": 4.267866722000902, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007471053159478436, + "loss": 1.6179, + "theoretical_loss": 4.267866722000902, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470892678034103, + "loss": 1.8281, + "theoretical_loss": 4.26773186539841, + "tokens_seen": 249102336 + }, + { + "epoch": 0.08, + "learning_rate": 0.000747073219658977, + "loss": 1.6578, + "theoretical_loss": 4.267597054201705, + "tokens_seen": 249167872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470571715145438, + "loss": 1.7296, + "theoretical_loss": 4.267462288383561, + "tokens_seen": 249233408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470411233701104, + "loss": 1.5899, + "theoretical_loss": 4.267327567916777, + "tokens_seen": 249298944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470250752256772, + "loss": 1.6601, + "theoretical_loss": 4.267192892774176, + "tokens_seen": 249364480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470090270812438, + "loss": 1.5475, + "theoretical_loss": 4.267058262928604, + "tokens_seen": 249430016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007469929789368106, + "loss": 1.6116, + "theoretical_loss": 4.26692367835293, + "tokens_seen": 249495552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007469769307923772, + "loss": 1.4896, + "theoretical_loss": 4.266789139020045, + "tokens_seen": 249561088 + }, + { + "epoch": 0.08, + "learning_rate": 0.000746960882647944, + "loss": 1.734, + "theoretical_loss": 4.266654644902868, + "tokens_seen": 249626624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007469448345035106, + "loss": 1.6688, + "theoretical_loss": 4.266520195974336, + "tokens_seen": 249692160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007469287863590774, + "loss": 1.6633, + "theoretical_loss": 4.266385792207412, + "tokens_seen": 249757696 + }, + { + "epoch": 0.08, + "learning_rate": 0.000746912738214644, + "loss": 1.7313, + "theoretical_loss": 4.266251433575082, + "tokens_seen": 249823232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468966900702106, + "loss": 1.6623, + "theoretical_loss": 4.266117120050355, + "tokens_seen": 249888768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468806419257774, + "loss": 1.659, + "theoretical_loss": 4.265982851606263, + "tokens_seen": 249954304 + }, + { + "epoch": 0.08, + "learning_rate": 0.000746864593781344, + "loss": 1.7441, + "theoretical_loss": 4.265848628215862, + "tokens_seen": 250019840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468485456369108, + "loss": 1.7166, + "theoretical_loss": 4.265714449852229, + "tokens_seen": 250085376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468324974924775, + "loss": 1.7004, + "theoretical_loss": 4.265580316488467, + "tokens_seen": 250150912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468164493480442, + "loss": 1.6103, + "theoretical_loss": 4.2654462280977015, + "tokens_seen": 250216448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468004012036109, + "loss": 1.5756, + "theoretical_loss": 4.265312184653078, + "tokens_seen": 250281984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467843530591776, + "loss": 1.6858, + "theoretical_loss": 4.2651781861277716, + "tokens_seen": 250347520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467683049147443, + "loss": 1.7365, + "theoretical_loss": 4.265044232494972, + "tokens_seen": 250413056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467522567703109, + "loss": 1.7981, + "theoretical_loss": 4.264910323727898, + "tokens_seen": 250478592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467362086258777, + "loss": 1.6383, + "theoretical_loss": 4.264776459799791, + "tokens_seen": 250544128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467201604814443, + "loss": 1.6194, + "theoretical_loss": 4.264642640683911, + "tokens_seen": 250609664 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467041123370111, + "loss": 1.6848, + "theoretical_loss": 4.264508866353546, + "tokens_seen": 250675200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466880641925777, + "loss": 1.7286, + "theoretical_loss": 4.264375136782004, + "tokens_seen": 250740736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466720160481445, + "loss": 1.7089, + "theoretical_loss": 4.264241451942617, + "tokens_seen": 250806272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466559679037111, + "loss": 1.5735, + "theoretical_loss": 4.264107811808739, + "tokens_seen": 250871808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466399197592779, + "loss": 1.6118, + "theoretical_loss": 4.263974216353749, + "tokens_seen": 250937344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466238716148445, + "loss": 1.5047, + "theoretical_loss": 4.2638406655510455, + "tokens_seen": 251002880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466078234704113, + "loss": 1.6449, + "theoretical_loss": 4.263707159374051, + "tokens_seen": 251068416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465917753259779, + "loss": 1.5712, + "theoretical_loss": 4.263573697796213, + "tokens_seen": 251133952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465757271815447, + "loss": 1.6243, + "theoretical_loss": 4.263440280790999, + "tokens_seen": 251199488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465596790371113, + "loss": 1.6167, + "theoretical_loss": 4.263306908331899, + "tokens_seen": 251265024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465436308926781, + "loss": 1.6755, + "theoretical_loss": 4.2631735803924276, + "tokens_seen": 251330560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465275827482447, + "loss": 1.7118, + "theoretical_loss": 4.263040296946122, + "tokens_seen": 251396096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465115346038115, + "loss": 1.7401, + "theoretical_loss": 4.26290705796654, + "tokens_seen": 251461632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464954864593781, + "loss": 1.6834, + "theoretical_loss": 4.262773863427265, + "tokens_seen": 251527168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464794383149449, + "loss": 1.7491, + "theoretical_loss": 4.262640713301899, + "tokens_seen": 251592704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464633901705115, + "loss": 1.5683, + "theoretical_loss": 4.26250760756407, + "tokens_seen": 251658240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464473420260783, + "loss": 1.6373, + "theoretical_loss": 4.262374546187428, + "tokens_seen": 251723776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464312938816449, + "loss": 1.4961, + "theoretical_loss": 4.262241529145643, + "tokens_seen": 251789312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464152457372117, + "loss": 1.6324, + "theoretical_loss": 4.262108556412411, + "tokens_seen": 251854848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463991975927783, + "loss": 1.8033, + "theoretical_loss": 4.261975627961448, + "tokens_seen": 251920384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463831494483451, + "loss": 1.7477, + "theoretical_loss": 4.2618427437664925, + "tokens_seen": 251985920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463671013039117, + "loss": 1.6968, + "theoretical_loss": 4.261709903801307, + "tokens_seen": 252051456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463510531594785, + "loss": 1.6687, + "theoretical_loss": 4.261577108039674, + "tokens_seen": 252116992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463350050150451, + "loss": 1.7375, + "theoretical_loss": 4.261444356455402, + "tokens_seen": 252182528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463189568706119, + "loss": 1.6986, + "theoretical_loss": 4.261311649022318, + "tokens_seen": 252248064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 90341, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.549142599105835, + "objective/train/theoretical_loss": 4.261178985714273, + "objective/train/tokens_used": 272773600, + "theoretical_loss": 4.261178985714273, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463029087261785, + "loss": 1.6768, + "theoretical_loss": 4.261178985714273, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462868605817453, + "loss": 1.5756, + "theoretical_loss": 4.26104636650514, + "tokens_seen": 252379136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462708124373119, + "loss": 1.4787, + "theoretical_loss": 4.260913791368815, + "tokens_seen": 252444672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462547642928787, + "loss": 1.5706, + "theoretical_loss": 4.260781260279216, + "tokens_seen": 252510208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462387161484453, + "loss": 1.5784, + "theoretical_loss": 4.260648773210281, + "tokens_seen": 252575744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462226680040121, + "loss": 1.6752, + "theoretical_loss": 4.2605163301359745, + "tokens_seen": 252641280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462066198595787, + "loss": 1.5707, + "theoretical_loss": 4.260383931030279, + "tokens_seen": 252706816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461905717151455, + "loss": 1.6033, + "theoretical_loss": 4.2602515758672, + "tokens_seen": 252772352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461745235707121, + "loss": 1.7105, + "theoretical_loss": 4.260119264620768, + "tokens_seen": 252837888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461584754262789, + "loss": 1.7694, + "theoretical_loss": 4.259986997265033, + "tokens_seen": 252903424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461424272818455, + "loss": 1.5492, + "theoretical_loss": 4.259854773774066, + "tokens_seen": 252968960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461263791374123, + "loss": 1.5911, + "theoretical_loss": 4.259722594121963, + "tokens_seen": 253034496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461103309929789, + "loss": 1.6801, + "theoretical_loss": 4.259590458282839, + "tokens_seen": 253100032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460942828485457, + "loss": 1.6821, + "theoretical_loss": 4.259458366230835, + "tokens_seen": 253165568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460782347041123, + "loss": 1.6812, + "theoretical_loss": 4.2593263179401095, + "tokens_seen": 253231104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460621865596791, + "loss": 1.6885, + "theoretical_loss": 4.259194313384846, + "tokens_seen": 253296640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460461384152457, + "loss": 1.6347, + "theoretical_loss": 4.259062352539247, + "tokens_seen": 253362176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460300902708125, + "loss": 1.6389, + "theoretical_loss": 4.258930435377541, + "tokens_seen": 253427712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460140421263791, + "loss": 1.6788, + "theoretical_loss": 4.258798561873974, + "tokens_seen": 253493248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459979939819459, + "loss": 1.6139, + "theoretical_loss": 4.258666732002817, + "tokens_seen": 253558784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459819458375125, + "loss": 1.6973, + "theoretical_loss": 4.258534945738362, + "tokens_seen": 253624320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459658976930793, + "loss": 1.6564, + "theoretical_loss": 4.258403203054923, + "tokens_seen": 253689856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459498495486459, + "loss": 1.6421, + "theoretical_loss": 4.258271503926833, + "tokens_seen": 253755392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459338014042127, + "loss": 1.6334, + "theoretical_loss": 4.258139848328451, + "tokens_seen": 253820928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459177532597794, + "loss": 1.6213, + "theoretical_loss": 4.258008236234154, + "tokens_seen": 253886464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459017051153461, + "loss": 1.6186, + "theoretical_loss": 4.257876667618344, + "tokens_seen": 253952000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458856569709128, + "loss": 1.6503, + "theoretical_loss": 4.257745142455442, + "tokens_seen": 254017536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458696088264795, + "loss": 1.7115, + "theoretical_loss": 4.257613660719892, + "tokens_seen": 254083072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458535606820462, + "loss": 1.6601, + "theoretical_loss": 4.25748222238616, + "tokens_seen": 254148608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458375125376129, + "loss": 1.5442, + "theoretical_loss": 4.2573508274287315, + "tokens_seen": 254214144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458214643931796, + "loss": 1.4927, + "theoretical_loss": 4.257219475822116, + "tokens_seen": 254279680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458054162487463, + "loss": 1.5835, + "theoretical_loss": 4.257088167540843, + "tokens_seen": 254345216 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745789368104313, + "loss": 1.5372, + "theoretical_loss": 4.256956902559465, + "tokens_seen": 254410752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457733199598797, + "loss": 1.6738, + "theoretical_loss": 4.256825680852554, + "tokens_seen": 254476288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457572718154464, + "loss": 1.646, + "theoretical_loss": 4.256694502394704, + "tokens_seen": 254541824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457412236710131, + "loss": 1.6983, + "theoretical_loss": 4.256563367160533, + "tokens_seen": 254607360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457251755265798, + "loss": 1.6854, + "theoretical_loss": 4.256432275124676, + "tokens_seen": 254672896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457091273821465, + "loss": 1.6195, + "theoretical_loss": 4.256301226261794, + "tokens_seen": 254738432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456930792377133, + "loss": 1.5589, + "theoretical_loss": 4.256170220546565, + "tokens_seen": 254803968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456770310932799, + "loss": 1.4944, + "theoretical_loss": 4.2560392579536925, + "tokens_seen": 254869504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456609829488467, + "loss": 1.5076, + "theoretical_loss": 4.255908338457898, + "tokens_seen": 254935040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456449348044133, + "loss": 1.5108, + "theoretical_loss": 4.255777462033926, + "tokens_seen": 255000576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074562888665998, + "loss": 1.5947, + "theoretical_loss": 4.255646628656542, + "tokens_seen": 255066112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456128385155467, + "loss": 1.5601, + "theoretical_loss": 4.255515838300534, + "tokens_seen": 255131648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455967903711135, + "loss": 1.6591, + "theoretical_loss": 4.255385090940708, + "tokens_seen": 255197184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455807422266801, + "loss": 1.6293, + "theoretical_loss": 4.255254386551894, + "tokens_seen": 255262720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455646940822469, + "loss": 1.5927, + "theoretical_loss": 4.255123725108943, + "tokens_seen": 255328256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455486459378135, + "loss": 1.584, + "theoretical_loss": 4.254993106586725, + "tokens_seen": 255393792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455325977933803, + "loss": 1.6033, + "theoretical_loss": 4.254862530960134, + "tokens_seen": 255459328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455165496489469, + "loss": 1.628, + "theoretical_loss": 4.254731998204083, + "tokens_seen": 255524864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 91319, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5541565418243408, + "objective/train/theoretical_loss": 4.254601508293508, + "objective/train/tokens_used": 276050400, + "theoretical_loss": 4.254601508293508, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455005015045136, + "loss": 1.5715, + "theoretical_loss": 4.254601508293508, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454844533600803, + "loss": 1.6553, + "theoretical_loss": 4.254471061203364, + "tokens_seen": 255655936 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745468405215647, + "loss": 1.7168, + "theoretical_loss": 4.254340656908628, + "tokens_seen": 255721472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454523570712137, + "loss": 1.636, + "theoretical_loss": 4.254210295384298, + "tokens_seen": 255787008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454363089267804, + "loss": 1.6813, + "theoretical_loss": 4.254079976605394, + "tokens_seen": 255852544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454202607823471, + "loss": 1.5497, + "theoretical_loss": 4.253949700546956, + "tokens_seen": 255918080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454042126379138, + "loss": 1.5719, + "theoretical_loss": 4.2538194671840435, + "tokens_seen": 255983616 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453881644934805, + "loss": 1.7041, + "theoretical_loss": 4.25368927649174, + "tokens_seen": 256049152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453721163490472, + "loss": 1.5223, + "theoretical_loss": 4.253559128445148, + "tokens_seen": 256114688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453560682046138, + "loss": 1.6966, + "theoretical_loss": 4.253429023019391, + "tokens_seen": 256180224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453400200601806, + "loss": 1.6931, + "theoretical_loss": 4.253298960189614, + "tokens_seen": 256245760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453239719157472, + "loss": 1.5582, + "theoretical_loss": 4.253168939930982, + "tokens_seen": 256311296 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745307923771314, + "loss": 1.5669, + "theoretical_loss": 4.253038962218682, + "tokens_seen": 256376832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452918756268806, + "loss": 1.7125, + "theoretical_loss": 4.25290902702792, + "tokens_seen": 256442368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452758274824474, + "loss": 1.6, + "theoretical_loss": 4.252779134333926, + "tokens_seen": 256507904 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745259779338014, + "loss": 1.7343, + "theoretical_loss": 4.252649284111946, + "tokens_seen": 256573440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452437311935808, + "loss": 1.6937, + "theoretical_loss": 4.252519476337251, + "tokens_seen": 256638976 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452276830491474, + "loss": 1.5897, + "theoretical_loss": 4.252389710985131, + "tokens_seen": 256704512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452116349047142, + "loss": 1.7181, + "theoretical_loss": 4.252259988030896, + "tokens_seen": 256770048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451955867602808, + "loss": 1.6677, + "theoretical_loss": 4.252130307449878, + "tokens_seen": 256835584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451795386158476, + "loss": 1.6119, + "theoretical_loss": 4.252000669217429, + "tokens_seen": 256901120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451634904714142, + "loss": 1.722, + "theoretical_loss": 4.251871073308923, + "tokens_seen": 256966656 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745147442326981, + "loss": 1.6881, + "theoretical_loss": 4.251741519699751, + "tokens_seen": 257032192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451313941825476, + "loss": 1.6454, + "theoretical_loss": 4.251612008365328, + "tokens_seen": 257097728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451153460381144, + "loss": 1.527, + "theoretical_loss": 4.2514825392810875, + "tokens_seen": 257163264 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745099297893681, + "loss": 1.6586, + "theoretical_loss": 4.251353112422486, + "tokens_seen": 257228800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450832497492478, + "loss": 1.6275, + "theoretical_loss": 4.251223727764999, + "tokens_seen": 257294336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450672016048144, + "loss": 1.716, + "theoretical_loss": 4.251094385284121, + "tokens_seen": 257359872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450511534603812, + "loss": 1.6393, + "theoretical_loss": 4.25096508495537, + "tokens_seen": 257425408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450351053159478, + "loss": 1.6811, + "theoretical_loss": 4.250835826754281, + "tokens_seen": 257490944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450190571715146, + "loss": 1.6071, + "theoretical_loss": 4.2507066106564135, + "tokens_seen": 257556480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450030090270812, + "loss": 1.7105, + "theoretical_loss": 4.250577436637344, + "tokens_seen": 257622016 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744986960882648, + "loss": 1.6548, + "theoretical_loss": 4.25044830467267, + "tokens_seen": 257687552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007449709127382146, + "loss": 1.7264, + "theoretical_loss": 4.250319214738011, + "tokens_seen": 257753088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007449548645937814, + "loss": 1.6103, + "theoretical_loss": 4.250190166809005, + "tokens_seen": 257818624 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744938816449348, + "loss": 1.6217, + "theoretical_loss": 4.250061160861311, + "tokens_seen": 257884160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007449227683049148, + "loss": 1.5565, + "theoretical_loss": 4.249932196870608, + "tokens_seen": 257949696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007449067201604814, + "loss": 1.6584, + "theoretical_loss": 4.249803274812598, + "tokens_seen": 258015232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448906720160482, + "loss": 1.7406, + "theoretical_loss": 4.249674394662998, + "tokens_seen": 258080768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448746238716148, + "loss": 1.5355, + "theoretical_loss": 4.249545556397549, + "tokens_seen": 258146304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448585757271816, + "loss": 1.6406, + "theoretical_loss": 4.249416759992011, + "tokens_seen": 258211840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448425275827482, + "loss": 1.5055, + "theoretical_loss": 4.249288005422166, + "tokens_seen": 258277376 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744826479438315, + "loss": 1.5312, + "theoretical_loss": 4.249159292663812, + "tokens_seen": 258342912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448104312938816, + "loss": 1.6441, + "theoretical_loss": 4.249030621692772, + "tokens_seen": 258408448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447943831494484, + "loss": 1.6091, + "theoretical_loss": 4.248901992484885, + "tokens_seen": 258473984 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744778335005015, + "loss": 1.6485, + "theoretical_loss": 4.248773405016014, + "tokens_seen": 258539520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447622868605818, + "loss": 1.6405, + "theoretical_loss": 4.248644859262039, + "tokens_seen": 258605056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447462387161484, + "loss": 1.5547, + "theoretical_loss": 4.248516355198861, + "tokens_seen": 258670592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447301905717152, + "loss": 1.6182, + "theoretical_loss": 4.2483878928024, + "tokens_seen": 258736128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447141424272819, + "loss": 1.592, + "theoretical_loss": 4.248259472048598, + "tokens_seen": 258801664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 92500, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5000085830688477, + "objective/train/theoretical_loss": 4.248131092913416, + "objective/train/tokens_used": 279327200, + "theoretical_loss": 4.248131092913416, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446980942828486, + "loss": 1.5452, + "theoretical_loss": 4.248131092913416, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446820461384153, + "loss": 1.542, + "theoretical_loss": 4.248002755372836, + "tokens_seen": 258932736 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744665997993982, + "loss": 1.6977, + "theoretical_loss": 4.247874459402857, + "tokens_seen": 258998272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446499498495487, + "loss": 1.5523, + "theoretical_loss": 4.2477462049795, + "tokens_seen": 259063808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446339017051154, + "loss": 1.6554, + "theoretical_loss": 4.2476179920788075, + "tokens_seen": 259129344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446178535606821, + "loss": 1.557, + "theoretical_loss": 4.247489820676839, + "tokens_seen": 259194880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446018054162488, + "loss": 1.6801, + "theoretical_loss": 4.247361690749676, + "tokens_seen": 259260416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445857572718155, + "loss": 1.604, + "theoretical_loss": 4.2472336022734165, + "tokens_seen": 259325952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445697091273822, + "loss": 1.6637, + "theoretical_loss": 4.2471055552241825, + "tokens_seen": 259391488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445536609829489, + "loss": 1.6168, + "theoretical_loss": 4.246977549578113, + "tokens_seen": 259457024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445376128385156, + "loss": 1.6913, + "theoretical_loss": 4.246849585311368, + "tokens_seen": 259522560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445215646940823, + "loss": 1.6747, + "theoretical_loss": 4.246721662400128, + "tokens_seen": 259588096 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744505516549649, + "loss": 1.5241, + "theoretical_loss": 4.246593780820589, + "tokens_seen": 259653632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444894684052157, + "loss": 1.5544, + "theoretical_loss": 4.2464659405489735, + "tokens_seen": 259719168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444734202607824, + "loss": 1.4963, + "theoretical_loss": 4.246338141561518, + "tokens_seen": 259784704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444573721163491, + "loss": 1.5923, + "theoretical_loss": 4.24621038383448, + "tokens_seen": 259850240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444413239719158, + "loss": 1.643, + "theoretical_loss": 4.246082667344139, + "tokens_seen": 259915776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444252758274825, + "loss": 1.5979, + "theoretical_loss": 4.245954992066792, + "tokens_seen": 259981312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444092276830492, + "loss": 1.7024, + "theoretical_loss": 4.245827357978754, + "tokens_seen": 260046848 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744393179538616, + "loss": 1.5572, + "theoretical_loss": 4.245699765056365, + "tokens_seen": 260112384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007443771313941826, + "loss": 1.5896, + "theoretical_loss": 4.245572213275978, + "tokens_seen": 260177920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007443610832497493, + "loss": 1.8514, + "theoretical_loss": 4.245444702613971, + "tokens_seen": 260243456 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744345035105316, + "loss": 1.586, + "theoretical_loss": 4.245317233046738, + "tokens_seen": 260308992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007443289869608828, + "loss": 1.7146, + "theoretical_loss": 4.2451898045506935, + "tokens_seen": 260374528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007443129388164494, + "loss": 1.6225, + "theoretical_loss": 4.245062417102272, + "tokens_seen": 260440064 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442968906720162, + "loss": 1.592, + "theoretical_loss": 4.244935070677927, + "tokens_seen": 260505600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442808425275828, + "loss": 1.6766, + "theoretical_loss": 4.244807765254132, + "tokens_seen": 260571136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442647943831496, + "loss": 1.6643, + "theoretical_loss": 4.244680500807378, + "tokens_seen": 260636672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442487462387162, + "loss": 1.5448, + "theoretical_loss": 4.244553277314178, + "tokens_seen": 260702208 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744232698094283, + "loss": 1.6036, + "theoretical_loss": 4.244426094751063, + "tokens_seen": 260767744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442166499498496, + "loss": 1.6473, + "theoretical_loss": 4.244298953094583, + "tokens_seen": 260833280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442006018054164, + "loss": 1.6384, + "theoretical_loss": 4.244171852321308, + "tokens_seen": 260898816 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744184553660983, + "loss": 1.6358, + "theoretical_loss": 4.244044792407828, + "tokens_seen": 260964352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441685055165498, + "loss": 1.6318, + "theoretical_loss": 4.24391777333075, + "tokens_seen": 261029888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441524573721164, + "loss": 1.6431, + "theoretical_loss": 4.243790795066703, + "tokens_seen": 261095424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441364092276832, + "loss": 1.5781, + "theoretical_loss": 4.243663857592332, + "tokens_seen": 261160960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441203610832498, + "loss": 1.5393, + "theoretical_loss": 4.243536960884305, + "tokens_seen": 261226496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441043129388166, + "loss": 1.6555, + "theoretical_loss": 4.243410104919307, + "tokens_seen": 261292032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440882647943832, + "loss": 1.6852, + "theoretical_loss": 4.243283289674042, + "tokens_seen": 261357568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440722166499499, + "loss": 1.4922, + "theoretical_loss": 4.243156515125234, + "tokens_seen": 261423104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440561685055166, + "loss": 1.6475, + "theoretical_loss": 4.243029781249625, + "tokens_seen": 261488640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440401203610833, + "loss": 1.5878, + "theoretical_loss": 4.2429030880239775, + "tokens_seen": 261554176 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074402407221665, + "loss": 1.6059, + "theoretical_loss": 4.242776435425072, + "tokens_seen": 261619712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440080240722167, + "loss": 1.5873, + "theoretical_loss": 4.24264982342971, + "tokens_seen": 261685248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439919759277834, + "loss": 1.5429, + "theoretical_loss": 4.242523252014709, + "tokens_seen": 261750784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439759277833501, + "loss": 1.6651, + "theoretical_loss": 4.242396721156908, + "tokens_seen": 261816320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439598796389168, + "loss": 1.7161, + "theoretical_loss": 4.242270230833164, + "tokens_seen": 261881856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439438314944835, + "loss": 1.5095, + "theoretical_loss": 4.242143781020353, + "tokens_seen": 261947392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439277833500501, + "loss": 1.6434, + "theoretical_loss": 4.24201737169537, + "tokens_seen": 262012928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439117352056169, + "loss": 1.6189, + "theoretical_loss": 4.24189100283513, + "tokens_seen": 262078464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 93423, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6364566087722778, + "objective/train/theoretical_loss": 4.241764674416565, + "objective/train/tokens_used": 282604000, + "theoretical_loss": 4.241764674416565, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438956870611835, + "loss": 1.6476, + "theoretical_loss": 4.241764674416565, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438796389167503, + "loss": 1.5735, + "theoretical_loss": 4.241638386416628, + "tokens_seen": 262209536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438635907723169, + "loss": 1.6634, + "theoretical_loss": 4.241512138812288, + "tokens_seen": 262275072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438475426278837, + "loss": 1.5158, + "theoretical_loss": 4.2413859315805365, + "tokens_seen": 262340608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438314944834503, + "loss": 1.5613, + "theoretical_loss": 4.241259764698381, + "tokens_seen": 262406144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438154463390171, + "loss": 1.5698, + "theoretical_loss": 4.24113363814285, + "tokens_seen": 262471680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437993981945837, + "loss": 1.7, + "theoretical_loss": 4.241007551890988, + "tokens_seen": 262537216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437833500501505, + "loss": 1.5795, + "theoretical_loss": 4.2408815059198615, + "tokens_seen": 262602752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437673019057171, + "loss": 1.6302, + "theoretical_loss": 4.240755500206554, + "tokens_seen": 262668288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437512537612839, + "loss": 1.6311, + "theoretical_loss": 4.240629534728167, + "tokens_seen": 262733824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437352056168505, + "loss": 1.6255, + "theoretical_loss": 4.240503609461822, + "tokens_seen": 262799360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437191574724173, + "loss": 1.8019, + "theoretical_loss": 4.24037772438466, + "tokens_seen": 262864896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437031093279839, + "loss": 1.5956, + "theoretical_loss": 4.240251879473839, + "tokens_seen": 262930432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436870611835507, + "loss": 1.6039, + "theoretical_loss": 4.240126074706536, + "tokens_seen": 262995968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436710130391173, + "loss": 1.6342, + "theoretical_loss": 4.240000310059948, + "tokens_seen": 263061504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436549648946841, + "loss": 1.5717, + "theoretical_loss": 4.239874585511288, + "tokens_seen": 263127040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436389167502507, + "loss": 1.7075, + "theoretical_loss": 4.239748901037791, + "tokens_seen": 263192576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436228686058175, + "loss": 1.5863, + "theoretical_loss": 4.2396232566167065, + "tokens_seen": 263258112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436068204613841, + "loss": 1.7253, + "theoretical_loss": 4.239497652225307, + "tokens_seen": 263323648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435907723169509, + "loss": 1.5759, + "theoretical_loss": 4.23937208784088, + "tokens_seen": 263389184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435747241725175, + "loss": 1.5407, + "theoretical_loss": 4.239246563440734, + "tokens_seen": 263454720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435586760280843, + "loss": 1.512, + "theoretical_loss": 4.239121079002194, + "tokens_seen": 263520256 + }, + { + "epoch": 0.08, + "learning_rate": 0.000743542627883651, + "loss": 1.5218, + "theoretical_loss": 4.238995634502606, + "tokens_seen": 263585792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435265797392177, + "loss": 1.6162, + "theoretical_loss": 4.238870229919331, + "tokens_seen": 263651328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435105315947844, + "loss": 1.5465, + "theoretical_loss": 4.2387448652297515, + "tokens_seen": 263716864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434944834503511, + "loss": 1.6953, + "theoretical_loss": 4.238619540411268, + "tokens_seen": 263782400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434784353059178, + "loss": 1.6432, + "theoretical_loss": 4.238494255441296, + "tokens_seen": 263847936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434623871614845, + "loss": 1.6252, + "theoretical_loss": 4.238369010297275, + "tokens_seen": 263913472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434463390170512, + "loss": 1.6152, + "theoretical_loss": 4.238243804956658, + "tokens_seen": 263979008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434302908726179, + "loss": 1.6624, + "theoretical_loss": 4.23811863939692, + "tokens_seen": 264044544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434142427281846, + "loss": 1.605, + "theoretical_loss": 4.237993513595552, + "tokens_seen": 264110080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433981945837513, + "loss": 1.5313, + "theoretical_loss": 4.2378684275300635, + "tokens_seen": 264175616 + }, + { + "epoch": 0.08, + "learning_rate": 0.000743382146439318, + "loss": 1.5915, + "theoretical_loss": 4.237743381177983, + "tokens_seen": 264241152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433660982948847, + "loss": 1.6108, + "theoretical_loss": 4.237618374516858, + "tokens_seen": 264306688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433500501504514, + "loss": 1.6528, + "theoretical_loss": 4.237493407524253, + "tokens_seen": 264372224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433340020060181, + "loss": 1.5117, + "theoretical_loss": 4.23736848017775, + "tokens_seen": 264437760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433179538615848, + "loss": 1.6246, + "theoretical_loss": 4.237243592454951, + "tokens_seen": 264503296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433019057171515, + "loss": 1.6768, + "theoretical_loss": 4.2371187443334755, + "tokens_seen": 264568832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432858575727182, + "loss": 1.7698, + "theoretical_loss": 4.236993935790962, + "tokens_seen": 264634368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432698094282849, + "loss": 1.5859, + "theoretical_loss": 4.236869166805064, + "tokens_seen": 264699904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432537612838516, + "loss": 1.4231, + "theoretical_loss": 4.236744437353458, + "tokens_seen": 264765440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432377131394183, + "loss": 1.7275, + "theoretical_loss": 4.236619747413836, + "tokens_seen": 264830976 + }, + { + "epoch": 0.08, + "learning_rate": 0.000743221664994985, + "loss": 1.5382, + "theoretical_loss": 4.236495096963906, + "tokens_seen": 264896512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432056168505517, + "loss": 1.6893, + "theoretical_loss": 4.236370485981398, + "tokens_seen": 264962048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431895687061184, + "loss": 1.676, + "theoretical_loss": 4.236245914444058, + "tokens_seen": 265027584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431735205616851, + "loss": 1.5495, + "theoretical_loss": 4.2361213823296495, + "tokens_seen": 265093120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431574724172518, + "loss": 1.6253, + "theoretical_loss": 4.235996889615957, + "tokens_seen": 265158656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431414242728185, + "loss": 1.646, + "theoretical_loss": 4.235872436280779, + "tokens_seen": 265224192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431253761283852, + "loss": 1.7312, + "theoretical_loss": 4.235748022301935, + "tokens_seen": 265289728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431093279839519, + "loss": 1.5934, + "theoretical_loss": 4.235623647657261, + "tokens_seen": 265355264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 94744, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6758512258529663, + "objective/train/theoretical_loss": 4.235499312324611, + "objective/train/tokens_used": 285880800, + "theoretical_loss": 4.235499312324611, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430932798395186, + "loss": 1.6044, + "theoretical_loss": 4.235499312324611, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430772316950853, + "loss": 1.6172, + "theoretical_loss": 4.235375016281858, + "tokens_seen": 265486336 + }, + { + "epoch": 0.08, + "learning_rate": 0.000743061183550652, + "loss": 1.5535, + "theoretical_loss": 4.235250759506892, + "tokens_seen": 265551872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430451354062187, + "loss": 1.4944, + "theoretical_loss": 4.235126541977621, + "tokens_seen": 265617408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430290872617854, + "loss": 1.6137, + "theoretical_loss": 4.2350023636719705, + "tokens_seen": 265682944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430130391173521, + "loss": 1.4967, + "theoretical_loss": 4.234878224567885, + "tokens_seen": 265748480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429969909729188, + "loss": 1.5846, + "theoretical_loss": 4.2347541246433265, + "tokens_seen": 265814016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429809428284855, + "loss": 1.6215, + "theoretical_loss": 4.234630063876273, + "tokens_seen": 265879552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429648946840523, + "loss": 1.5888, + "theoretical_loss": 4.234506042244723, + "tokens_seen": 265945088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429488465396189, + "loss": 1.6198, + "theoretical_loss": 4.234382059726691, + "tokens_seen": 266010624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429327983951857, + "loss": 1.635, + "theoretical_loss": 4.234258116300211, + "tokens_seen": 266076160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429167502507523, + "loss": 1.5904, + "theoretical_loss": 4.234134211943332, + "tokens_seen": 266141696 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742900702106319, + "loss": 1.6317, + "theoretical_loss": 4.234010346634123, + "tokens_seen": 266207232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428846539618857, + "loss": 1.6059, + "theoretical_loss": 4.23388652035067, + "tokens_seen": 266272768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428686058174525, + "loss": 1.5262, + "theoretical_loss": 4.233762733071076, + "tokens_seen": 266338304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428525576730191, + "loss": 1.6322, + "theoretical_loss": 4.233638984773464, + "tokens_seen": 266403840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428365095285859, + "loss": 1.7282, + "theoretical_loss": 4.2335152754359715, + "tokens_seen": 266469376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428204613841525, + "loss": 1.5823, + "theoretical_loss": 4.233391605036756, + "tokens_seen": 266534912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428044132397193, + "loss": 1.555, + "theoretical_loss": 4.233267973553991, + "tokens_seen": 266600448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427883650952859, + "loss": 1.6832, + "theoretical_loss": 4.233144380965869, + "tokens_seen": 266665984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427723169508527, + "loss": 1.5752, + "theoretical_loss": 4.2330208272505985, + "tokens_seen": 266731520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427562688064193, + "loss": 1.6642, + "theoretical_loss": 4.232897312386408, + "tokens_seen": 266797056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427402206619861, + "loss": 1.6123, + "theoretical_loss": 4.232773836351541, + "tokens_seen": 266862592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427241725175527, + "loss": 1.6768, + "theoretical_loss": 4.2326503991242586, + "tokens_seen": 266928128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427081243731195, + "loss": 1.5605, + "theoretical_loss": 4.232527000682843, + "tokens_seen": 266993664 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426920762286861, + "loss": 1.6535, + "theoretical_loss": 4.232403641005589, + "tokens_seen": 267059200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426760280842528, + "loss": 1.6531, + "theoretical_loss": 4.232280320070812, + "tokens_seen": 267124736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426599799398195, + "loss": 1.6343, + "theoretical_loss": 4.232157037856844, + "tokens_seen": 267190272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426439317953862, + "loss": 1.6088, + "theoretical_loss": 4.232033794342033, + "tokens_seen": 267255808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426278836509529, + "loss": 1.5571, + "theoretical_loss": 4.231910589504748, + "tokens_seen": 267321344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426118355065196, + "loss": 1.626, + "theoretical_loss": 4.2317874233233725, + "tokens_seen": 267386880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007425957873620863, + "loss": 1.561, + "theoretical_loss": 4.231664295776307, + "tokens_seen": 267452416 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742579739217653, + "loss": 1.5373, + "theoretical_loss": 4.231541206841971, + "tokens_seen": 267517952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007425636910732197, + "loss": 1.5562, + "theoretical_loss": 4.2314181564988, + "tokens_seen": 267583488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007425476429287864, + "loss": 1.614, + "theoretical_loss": 4.23129514472525, + "tokens_seen": 267649024 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742531594784353, + "loss": 1.5684, + "theoretical_loss": 4.231172171499789, + "tokens_seen": 267714560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007425155466399198, + "loss": 1.6038, + "theoretical_loss": 4.231049236800907, + "tokens_seen": 267780096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424994984954864, + "loss": 1.5656, + "theoretical_loss": 4.230926340607109, + "tokens_seen": 267845632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424834503510532, + "loss": 1.6124, + "theoretical_loss": 4.230803482896918, + "tokens_seen": 267911168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424674022066198, + "loss": 1.6741, + "theoretical_loss": 4.230680663648872, + "tokens_seen": 267976704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424513540621866, + "loss": 1.6197, + "theoretical_loss": 4.230557882841531, + "tokens_seen": 268042240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424353059177532, + "loss": 1.7328, + "theoretical_loss": 4.2304351404534675, + "tokens_seen": 268107776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074241925777332, + "loss": 1.6919, + "theoretical_loss": 4.230312436463272, + "tokens_seen": 268173312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424032096288866, + "loss": 1.6116, + "theoretical_loss": 4.230189770849556, + "tokens_seen": 268238848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423871614844534, + "loss": 1.6098, + "theoretical_loss": 4.230067143590943, + "tokens_seen": 268304384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074237111334002, + "loss": 1.5206, + "theoretical_loss": 4.229944554666076, + "tokens_seen": 268369920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423550651955868, + "loss": 1.6026, + "theoretical_loss": 4.229822004053617, + "tokens_seen": 268435456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423390170511535, + "loss": 1.6134, + "theoretical_loss": 4.229699491732242, + "tokens_seen": 268500992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423229689067202, + "loss": 1.5862, + "theoretical_loss": 4.2295770176806435, + "tokens_seen": 268566528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423069207622869, + "loss": 1.6335, + "theoretical_loss": 4.229454581877535, + "tokens_seen": 268632064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 95678, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.629549503326416, + "objective/train/theoretical_loss": 4.229332184301644, + "objective/train/tokens_used": 289157600, + "theoretical_loss": 4.229332184301644, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422908726178536, + "loss": 1.6398, + "theoretical_loss": 4.229332184301644, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422748244734203, + "loss": 1.4887, + "theoretical_loss": 4.229209824931714, + "tokens_seen": 268763136 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742258776328987, + "loss": 1.6316, + "theoretical_loss": 4.22908750374651, + "tokens_seen": 268828672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422427281845537, + "loss": 1.5414, + "theoretical_loss": 4.22896522072481, + "tokens_seen": 268894208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422266800401204, + "loss": 1.6666, + "theoretical_loss": 4.22884297584541, + "tokens_seen": 268959744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422106318956871, + "loss": 1.6226, + "theoretical_loss": 4.228720769087124, + "tokens_seen": 269025280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421945837512538, + "loss": 1.4824, + "theoretical_loss": 4.22859860042878, + "tokens_seen": 269090816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421785356068205, + "loss": 1.574, + "theoretical_loss": 4.2284764698492285, + "tokens_seen": 269156352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421624874623872, + "loss": 1.6074, + "theoretical_loss": 4.22835437732733, + "tokens_seen": 269221888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421464393179539, + "loss": 1.6837, + "theoretical_loss": 4.228232322841967, + "tokens_seen": 269287424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421303911735206, + "loss": 1.5557, + "theoretical_loss": 4.228110306372036, + "tokens_seen": 269352960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421143430290873, + "loss": 1.5514, + "theoretical_loss": 4.227988327896453, + "tokens_seen": 269418496 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742098294884654, + "loss": 1.6616, + "theoretical_loss": 4.2278663873941476, + "tokens_seen": 269484032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420822467402207, + "loss": 1.5448, + "theoretical_loss": 4.227744484844068, + "tokens_seen": 269549568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420661985957874, + "loss": 1.5115, + "theoretical_loss": 4.227622620225182, + "tokens_seen": 269615104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420501504513541, + "loss": 1.6058, + "theoretical_loss": 4.227500793516466, + "tokens_seen": 269680640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420341023069208, + "loss": 1.6111, + "theoretical_loss": 4.227379004696923, + "tokens_seen": 269746176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420180541624875, + "loss": 1.5459, + "theoretical_loss": 4.227257253745566, + "tokens_seen": 269811712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420020060180542, + "loss": 1.6436, + "theoretical_loss": 4.227135540641426, + "tokens_seen": 269877248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419859578736209, + "loss": 1.6299, + "theoretical_loss": 4.227013865363553, + "tokens_seen": 269942784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419699097291876, + "loss": 1.5194, + "theoretical_loss": 4.226892227891012, + "tokens_seen": 270008320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419538615847543, + "loss": 1.5954, + "theoretical_loss": 4.2267706282028845, + "tokens_seen": 270073856 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741937813440321, + "loss": 1.596, + "theoretical_loss": 4.226649066278269, + "tokens_seen": 270139392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419217652958877, + "loss": 1.7046, + "theoretical_loss": 4.226527542096282, + "tokens_seen": 270204928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419057171514544, + "loss": 1.6551, + "theoretical_loss": 4.226406055636053, + "tokens_seen": 270270464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418896690070211, + "loss": 1.6659, + "theoretical_loss": 4.226284606876731, + "tokens_seen": 270336000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418736208625878, + "loss": 1.5405, + "theoretical_loss": 4.226163195797481, + "tokens_seen": 270401536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418575727181545, + "loss": 1.6297, + "theoretical_loss": 4.226041822377486, + "tokens_seen": 270467072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418415245737212, + "loss": 1.5975, + "theoretical_loss": 4.225920486595941, + "tokens_seen": 270532608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418254764292879, + "loss": 1.6092, + "theoretical_loss": 4.225799188432063, + "tokens_seen": 270598144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418094282848546, + "loss": 1.665, + "theoretical_loss": 4.225677927865083, + "tokens_seen": 270663680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417933801404213, + "loss": 1.5838, + "theoretical_loss": 4.225556704874247, + "tokens_seen": 270729216 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741777331995988, + "loss": 1.6257, + "theoretical_loss": 4.22543551943882, + "tokens_seen": 270794752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417612838515547, + "loss": 1.5903, + "theoretical_loss": 4.225314371538083, + "tokens_seen": 270860288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417452357071214, + "loss": 1.5667, + "theoretical_loss": 4.225193261151332, + "tokens_seen": 270925824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417291875626881, + "loss": 1.5673, + "theoretical_loss": 4.22507218825788, + "tokens_seen": 270991360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417131394182548, + "loss": 1.5828, + "theoretical_loss": 4.224951152837058, + "tokens_seen": 271056896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416970912738215, + "loss": 1.6085, + "theoretical_loss": 4.2248301548682115, + "tokens_seen": 271122432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416810431293882, + "loss": 1.5875, + "theoretical_loss": 4.2247091943307025, + "tokens_seen": 271187968 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741664994984955, + "loss": 1.5357, + "theoretical_loss": 4.224588271203911, + "tokens_seen": 271253504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416489468405216, + "loss": 1.7111, + "theoretical_loss": 4.22446738546723, + "tokens_seen": 271319040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416328986960883, + "loss": 1.6451, + "theoretical_loss": 4.224346537100074, + "tokens_seen": 271384576 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741616850551655, + "loss": 1.4726, + "theoretical_loss": 4.22422572608187, + "tokens_seen": 271450112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416008024072218, + "loss": 1.5835, + "theoretical_loss": 4.224104952392061, + "tokens_seen": 271515648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415847542627884, + "loss": 1.6244, + "theoretical_loss": 4.223984216010107, + "tokens_seen": 271581184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415687061183552, + "loss": 1.6205, + "theoretical_loss": 4.223863516915486, + "tokens_seen": 271646720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415526579739218, + "loss": 1.5239, + "theoretical_loss": 4.223742855087691, + "tokens_seen": 271712256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415366098294886, + "loss": 1.5946, + "theoretical_loss": 4.223622230506231, + "tokens_seen": 271777792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415205616850552, + "loss": 1.7028, + "theoretical_loss": 4.22350164315063, + "tokens_seen": 271843328 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741504513540622, + "loss": 1.5819, + "theoretical_loss": 4.22338109300043, + "tokens_seen": 271908864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 96867, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5667804479599, + "objective/train/theoretical_loss": 4.22326058003519, + "objective/train/tokens_used": 292434400, + "theoretical_loss": 4.22326058003519, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414884653961886, + "loss": 1.5744, + "theoretical_loss": 4.22326058003519, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414724172517554, + "loss": 1.6947, + "theoretical_loss": 4.223140104234482, + "tokens_seen": 272039936 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741456369107322, + "loss": 1.5442, + "theoretical_loss": 4.2230196655778975, + "tokens_seen": 272105472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414403209628888, + "loss": 1.707, + "theoretical_loss": 4.222899264045042, + "tokens_seen": 272171008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414242728184554, + "loss": 1.635, + "theoretical_loss": 4.2227788996155375, + "tokens_seen": 272236544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414082246740222, + "loss": 1.6033, + "theoretical_loss": 4.222658572269022, + "tokens_seen": 272302080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413921765295888, + "loss": 1.6143, + "theoretical_loss": 4.222538281985151, + "tokens_seen": 272367616 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413761283851556, + "loss": 1.6715, + "theoretical_loss": 4.2224180287435935, + "tokens_seen": 272433152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413600802407222, + "loss": 1.5306, + "theoretical_loss": 4.222297812524037, + "tokens_seen": 272498688 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741344032096289, + "loss": 1.6365, + "theoretical_loss": 4.222177633306183, + "tokens_seen": 272564224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413279839518556, + "loss": 1.657, + "theoretical_loss": 4.222057491069751, + "tokens_seen": 272629760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413119358074224, + "loss": 1.5467, + "theoretical_loss": 4.221937385794474, + "tokens_seen": 272695296 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741295887662989, + "loss": 1.5622, + "theoretical_loss": 4.221817317460104, + "tokens_seen": 272760832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412798395185557, + "loss": 1.53, + "theoretical_loss": 4.221697286046407, + "tokens_seen": 272826368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412637913741224, + "loss": 1.4924, + "theoretical_loss": 4.221577291533165, + "tokens_seen": 272891904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412477432296891, + "loss": 1.583, + "theoretical_loss": 4.221457333900176, + "tokens_seen": 272957440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412316950852558, + "loss": 1.5246, + "theoretical_loss": 4.221337413127254, + "tokens_seen": 273022976 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412156469408225, + "loss": 1.42, + "theoretical_loss": 4.221217529194231, + "tokens_seen": 273088512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411995987963892, + "loss": 1.6641, + "theoretical_loss": 4.22109768208095, + "tokens_seen": 273154048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411835506519559, + "loss": 1.5788, + "theoretical_loss": 4.220977871767275, + "tokens_seen": 273219584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411675025075226, + "loss": 1.6302, + "theoretical_loss": 4.220858098233082, + "tokens_seen": 273285120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411514543630893, + "loss": 1.5654, + "theoretical_loss": 4.220738361458266, + "tokens_seen": 273350656 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741135406218656, + "loss": 1.5599, + "theoretical_loss": 4.220618661422735, + "tokens_seen": 273416192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411193580742227, + "loss": 1.6897, + "theoretical_loss": 4.2204989981064145, + "tokens_seen": 273481728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411033099297894, + "loss": 1.5683, + "theoretical_loss": 4.220379371489246, + "tokens_seen": 273547264 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410872617853561, + "loss": 1.5457, + "theoretical_loss": 4.220259781551185, + "tokens_seen": 273612800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410712136409228, + "loss": 1.6194, + "theoretical_loss": 4.2201402282722045, + "tokens_seen": 273678336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410551654964895, + "loss": 1.6366, + "theoretical_loss": 4.2200207116322925, + "tokens_seen": 273743872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410391173520562, + "loss": 1.6136, + "theoretical_loss": 4.219901231611452, + "tokens_seen": 273809408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410230692076229, + "loss": 1.6047, + "theoretical_loss": 4.219781788189703, + "tokens_seen": 273874944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410070210631896, + "loss": 1.4999, + "theoretical_loss": 4.2196623813470815, + "tokens_seen": 273940480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409909729187563, + "loss": 1.6752, + "theoretical_loss": 4.219543011063637, + "tokens_seen": 274006016 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740974924774323, + "loss": 1.7212, + "theoretical_loss": 4.219423677319437, + "tokens_seen": 274071552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409588766298897, + "loss": 1.6442, + "theoretical_loss": 4.219304380094562, + "tokens_seen": 274137088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409428284854564, + "loss": 1.6143, + "theoretical_loss": 4.2191851193691114, + "tokens_seen": 274202624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409267803410231, + "loss": 1.6529, + "theoretical_loss": 4.219065895123197, + "tokens_seen": 274268160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409107321965898, + "loss": 1.6902, + "theoretical_loss": 4.218946707336949, + "tokens_seen": 274333696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408946840521565, + "loss": 1.5987, + "theoretical_loss": 4.2188275559905115, + "tokens_seen": 274399232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408786359077232, + "loss": 1.527, + "theoretical_loss": 4.218708441064044, + "tokens_seen": 274464768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408625877632899, + "loss": 1.5728, + "theoretical_loss": 4.218589362537723, + "tokens_seen": 274530304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408465396188566, + "loss": 1.5688, + "theoretical_loss": 4.218470320391739, + "tokens_seen": 274595840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408304914744233, + "loss": 1.5903, + "theoretical_loss": 4.218351314606298, + "tokens_seen": 274661376 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074081444332999, + "loss": 1.6266, + "theoretical_loss": 4.218232345161622, + "tokens_seen": 274726912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407983951855567, + "loss": 1.5244, + "theoretical_loss": 4.21811341203795, + "tokens_seen": 274792448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407823470411234, + "loss": 1.6264, + "theoretical_loss": 4.217994515215534, + "tokens_seen": 274857984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407662988966901, + "loss": 1.6983, + "theoretical_loss": 4.217875654674642, + "tokens_seen": 274923520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407502507522568, + "loss": 1.5818, + "theoretical_loss": 4.217756830395559, + "tokens_seen": 274989056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407342026078235, + "loss": 1.6596, + "theoretical_loss": 4.217638042358583, + "tokens_seen": 275054592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407181544633902, + "loss": 1.6276, + "theoretical_loss": 4.21751929054403, + "tokens_seen": 275120128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407021063189569, + "loss": 1.6848, + "theoretical_loss": 4.2174005749322285, + "tokens_seen": 275185664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 97725, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.526691198348999, + "objective/train/theoretical_loss": 4.2172818955035245, + "objective/train/tokens_used": 295711200, + "theoretical_loss": 4.2172818955035245, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406860581745236, + "loss": 1.5652, + "theoretical_loss": 4.2172818955035245, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406700100300903, + "loss": 1.5369, + "theoretical_loss": 4.217163252238279, + "tokens_seen": 275316736 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740653961885657, + "loss": 1.4076, + "theoretical_loss": 4.217044645116869, + "tokens_seen": 275382272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406379137412237, + "loss": 1.6474, + "theoretical_loss": 4.216926074119684, + "tokens_seen": 275447808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406218655967904, + "loss": 1.5036, + "theoretical_loss": 4.216807539227132, + "tokens_seen": 275513344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406058174523571, + "loss": 1.6137, + "theoretical_loss": 4.2166890404196335, + "tokens_seen": 275578880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405897693079238, + "loss": 1.6188, + "theoretical_loss": 4.216570577677627, + "tokens_seen": 275644416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405737211634905, + "loss": 1.6735, + "theoretical_loss": 4.216452150981566, + "tokens_seen": 275709952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405576730190572, + "loss": 1.4854, + "theoretical_loss": 4.216333760311916, + "tokens_seen": 275775488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405416248746239, + "loss": 1.6522, + "theoretical_loss": 4.216215405649161, + "tokens_seen": 275841024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405255767301906, + "loss": 1.5212, + "theoretical_loss": 4.2160970869738, + "tokens_seen": 275906560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405095285857573, + "loss": 1.7428, + "theoretical_loss": 4.215978804266345, + "tokens_seen": 275972096 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740493480441324, + "loss": 1.6548, + "theoretical_loss": 4.215860557507324, + "tokens_seen": 276037632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404774322968907, + "loss": 1.4961, + "theoretical_loss": 4.215742346677283, + "tokens_seen": 276103168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404613841524574, + "loss": 1.5168, + "theoretical_loss": 4.215624171756779, + "tokens_seen": 276168704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404453360080241, + "loss": 1.5582, + "theoretical_loss": 4.215506032726386, + "tokens_seen": 276234240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404292878635908, + "loss": 1.6843, + "theoretical_loss": 4.215387929566694, + "tokens_seen": 276299776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404132397191575, + "loss": 1.6185, + "theoretical_loss": 4.215269862258307, + "tokens_seen": 276365312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403971915747242, + "loss": 1.4809, + "theoretical_loss": 4.215151830781843, + "tokens_seen": 276430848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403811434302909, + "loss": 1.4883, + "theoretical_loss": 4.2150338351179375, + "tokens_seen": 276496384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403650952858576, + "loss": 1.6069, + "theoretical_loss": 4.21491587524724, + "tokens_seen": 276561920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403490471414243, + "loss": 1.7038, + "theoretical_loss": 4.214797951150415, + "tokens_seen": 276627456 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740332998996991, + "loss": 1.581, + "theoretical_loss": 4.214680062808142, + "tokens_seen": 276692992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403169508525577, + "loss": 1.542, + "theoretical_loss": 4.214562210201114, + "tokens_seen": 276758528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403009027081244, + "loss": 1.4822, + "theoretical_loss": 4.214444393310042, + "tokens_seen": 276824064 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402848545636911, + "loss": 1.5861, + "theoretical_loss": 4.2143266121156495, + "tokens_seen": 276889600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402688064192578, + "loss": 1.6427, + "theoretical_loss": 4.2142088665986766, + "tokens_seen": 276955136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402527582748245, + "loss": 1.535, + "theoretical_loss": 4.214091156739878, + "tokens_seen": 277020672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402367101303912, + "loss": 1.6281, + "theoretical_loss": 4.213973482520021, + "tokens_seen": 277086208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402206619859579, + "loss": 1.6517, + "theoretical_loss": 4.213855843919891, + "tokens_seen": 277151744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402046138415247, + "loss": 1.6089, + "theoretical_loss": 4.2137382409202875, + "tokens_seen": 277217280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401885656970913, + "loss": 1.6273, + "theoretical_loss": 4.2136206735020245, + "tokens_seen": 277282816 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740172517552658, + "loss": 1.5309, + "theoretical_loss": 4.21350314164593, + "tokens_seen": 277348352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401564694082247, + "loss": 1.612, + "theoretical_loss": 4.213385645332848, + "tokens_seen": 277413888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401404212637915, + "loss": 1.7019, + "theoretical_loss": 4.213268184543638, + "tokens_seen": 277479424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401243731193581, + "loss": 1.5525, + "theoretical_loss": 4.213150759259172, + "tokens_seen": 277544960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401083249749249, + "loss": 1.601, + "theoretical_loss": 4.213033369460339, + "tokens_seen": 277610496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400922768304915, + "loss": 1.55, + "theoretical_loss": 4.212916015128041, + "tokens_seen": 277676032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400762286860583, + "loss": 1.6188, + "theoretical_loss": 4.212798696243197, + "tokens_seen": 277741568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400601805416249, + "loss": 1.6865, + "theoretical_loss": 4.212681412786738, + "tokens_seen": 277807104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400441323971917, + "loss": 1.6119, + "theoretical_loss": 4.212564164739613, + "tokens_seen": 277872640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400280842527583, + "loss": 1.653, + "theoretical_loss": 4.212446952082783, + "tokens_seen": 277938176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400120361083251, + "loss": 1.5911, + "theoretical_loss": 4.212329774797225, + "tokens_seen": 278003712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399959879638917, + "loss": 1.5933, + "theoretical_loss": 4.212212632863931, + "tokens_seen": 278069248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399799398194585, + "loss": 1.5824, + "theoretical_loss": 4.212095526263905, + "tokens_seen": 278134784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399638916750251, + "loss": 1.593, + "theoretical_loss": 4.211978454978171, + "tokens_seen": 278200320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399478435305919, + "loss": 1.4985, + "theoretical_loss": 4.211861418987762, + "tokens_seen": 278265856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399317953861585, + "loss": 1.7031, + "theoretical_loss": 4.211744418273728, + "tokens_seen": 278331392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399157472417253, + "loss": 1.4938, + "theoretical_loss": 4.211627452817136, + "tokens_seen": 278396928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398996990972919, + "loss": 1.5638, + "theoretical_loss": 4.211510522599063, + "tokens_seen": 278462464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 98910, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5239521265029907, + "objective/train/theoretical_loss": 4.2113936276006045, + "objective/train/tokens_used": 298988000, + "theoretical_loss": 4.2113936276006045, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398836509528586, + "loss": 1.5001, + "theoretical_loss": 4.2113936276006045, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398676028084253, + "loss": 1.4508, + "theoretical_loss": 4.211276767802868, + "tokens_seen": 278593536 + }, + { + "epoch": 0.08, + "learning_rate": 0.000739851554663992, + "loss": 1.5051, + "theoretical_loss": 4.211159943186978, + "tokens_seen": 278659072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398355065195587, + "loss": 1.5914, + "theoretical_loss": 4.211043153734071, + "tokens_seen": 278724608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398194583751254, + "loss": 1.5629, + "theoretical_loss": 4.2109263994253, + "tokens_seen": 278790144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398034102306921, + "loss": 1.5097, + "theoretical_loss": 4.21080968024183, + "tokens_seen": 278855680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397873620862588, + "loss": 1.6827, + "theoretical_loss": 4.210692996164845, + "tokens_seen": 278921216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397713139418255, + "loss": 1.539, + "theoretical_loss": 4.210576347175539, + "tokens_seen": 278986752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397552657973922, + "loss": 1.6543, + "theoretical_loss": 4.210459733255123, + "tokens_seen": 279052288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397392176529589, + "loss": 1.5957, + "theoretical_loss": 4.2103431543848195, + "tokens_seen": 279117824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397231695085256, + "loss": 1.5742, + "theoretical_loss": 4.210226610545872, + "tokens_seen": 279183360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397071213640923, + "loss": 1.6426, + "theoretical_loss": 4.21011010171953, + "tokens_seen": 279248896 + }, + { + "epoch": 0.08, + "learning_rate": 0.000739691073219659, + "loss": 1.5525, + "theoretical_loss": 4.209993627887064, + "tokens_seen": 279314432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396750250752257, + "loss": 1.5869, + "theoretical_loss": 4.209877189029756, + "tokens_seen": 279379968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396589769307924, + "loss": 1.6383, + "theoretical_loss": 4.209760785128903, + "tokens_seen": 279445504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396429287863591, + "loss": 1.638, + "theoretical_loss": 4.2096444161658155, + "tokens_seen": 279511040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396268806419258, + "loss": 1.5023, + "theoretical_loss": 4.209528082121819, + "tokens_seen": 279576576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396108324974925, + "loss": 1.5516, + "theoretical_loss": 4.209411782978256, + "tokens_seen": 279642112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395947843530592, + "loss": 1.5677, + "theoretical_loss": 4.209295518716477, + "tokens_seen": 279707648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395787362086259, + "loss": 1.5189, + "theoretical_loss": 4.209179289317854, + "tokens_seen": 279773184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395626880641926, + "loss": 1.5775, + "theoretical_loss": 4.209063094763767, + "tokens_seen": 279838720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395466399197593, + "loss": 1.5707, + "theoretical_loss": 4.208946935035616, + "tokens_seen": 279904256 + }, + { + "epoch": 0.08, + "learning_rate": 0.000739530591775326, + "loss": 1.5592, + "theoretical_loss": 4.2088308101148115, + "tokens_seen": 279969792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395145436308927, + "loss": 1.6149, + "theoretical_loss": 4.208714719982778, + "tokens_seen": 280035328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394984954864594, + "loss": 1.5742, + "theoretical_loss": 4.208598664620958, + "tokens_seen": 280100864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394824473420261, + "loss": 1.6079, + "theoretical_loss": 4.208482644010804, + "tokens_seen": 280166400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394663991975928, + "loss": 1.5717, + "theoretical_loss": 4.2083666581337855, + "tokens_seen": 280231936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394503510531595, + "loss": 1.5169, + "theoretical_loss": 4.208250706971385, + "tokens_seen": 280297472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394343029087262, + "loss": 1.6327, + "theoretical_loss": 4.208134790505099, + "tokens_seen": 280363008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394182547642929, + "loss": 1.6121, + "theoretical_loss": 4.20801890871644, + "tokens_seen": 280428544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394022066198596, + "loss": 1.5979, + "theoretical_loss": 4.207903061586931, + "tokens_seen": 280494080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393861584754263, + "loss": 1.6794, + "theoretical_loss": 4.207787249098113, + "tokens_seen": 280559616 + }, + { + "epoch": 0.09, + "learning_rate": 0.000739370110330993, + "loss": 1.5258, + "theoretical_loss": 4.20767147123154, + "tokens_seen": 280625152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393540621865597, + "loss": 1.6724, + "theoretical_loss": 4.20755572796878, + "tokens_seen": 280690688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393380140421264, + "loss": 1.5395, + "theoretical_loss": 4.207440019291413, + "tokens_seen": 280756224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393219658976931, + "loss": 1.6244, + "theoretical_loss": 4.2073243451810365, + "tokens_seen": 280821760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393059177532598, + "loss": 1.6882, + "theoretical_loss": 4.20720870561926, + "tokens_seen": 280887296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392898696088265, + "loss": 1.4866, + "theoretical_loss": 4.2070931005877075, + "tokens_seen": 280952832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392738214643932, + "loss": 1.5088, + "theoretical_loss": 4.206977530068017, + "tokens_seen": 281018368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392577733199599, + "loss": 1.583, + "theoretical_loss": 4.206861994041842, + "tokens_seen": 281083904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392417251755266, + "loss": 1.5661, + "theoretical_loss": 4.206746492490847, + "tokens_seen": 281149440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392256770310933, + "loss": 1.6379, + "theoretical_loss": 4.206631025396714, + "tokens_seen": 281214976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00073920962888666, + "loss": 1.587, + "theoretical_loss": 4.206515592741135, + "tokens_seen": 281280512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391935807422267, + "loss": 1.6539, + "theoretical_loss": 4.20640019450582, + "tokens_seen": 281346048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391775325977934, + "loss": 1.6555, + "theoretical_loss": 4.206284830672491, + "tokens_seen": 281411584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391614844533601, + "loss": 1.6364, + "theoretical_loss": 4.206169501222885, + "tokens_seen": 281477120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391454363089268, + "loss": 1.5291, + "theoretical_loss": 4.20605420613875, + "tokens_seen": 281542656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391293881644935, + "loss": 1.5918, + "theoretical_loss": 4.205938945401851, + "tokens_seen": 281608192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391133400200602, + "loss": 1.5979, + "theoretical_loss": 4.205823718993966, + "tokens_seen": 281673728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390972918756269, + "loss": 1.6177, + "theoretical_loss": 4.2057085268968875, + "tokens_seen": 281739264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 99932, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.517197608947754, + "objective/train/theoretical_loss": 4.205593369092421, + "objective/train/tokens_used": 302264800, + "theoretical_loss": 4.205593369092421, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390812437311936, + "loss": 1.5729, + "theoretical_loss": 4.205593369092421, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390651955867603, + "loss": 1.6341, + "theoretical_loss": 4.205478245562386, + "tokens_seen": 281870336 + }, + { + "epoch": 0.09, + "learning_rate": 0.000739049147442327, + "loss": 1.6788, + "theoretical_loss": 4.205363156288615, + "tokens_seen": 281935872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390330992978937, + "loss": 1.621, + "theoretical_loss": 4.205248101252957, + "tokens_seen": 282001408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390170511534604, + "loss": 1.544, + "theoretical_loss": 4.205133080437273, + "tokens_seen": 282066944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390010030090271, + "loss": 1.6125, + "theoretical_loss": 4.205018093823437, + "tokens_seen": 282132480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389849548645938, + "loss": 1.5787, + "theoretical_loss": 4.204903141393338, + "tokens_seen": 282198016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389689067201605, + "loss": 1.6345, + "theoretical_loss": 4.204788223128879, + "tokens_seen": 282263552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389528585757272, + "loss": 1.5378, + "theoretical_loss": 4.204673339011976, + "tokens_seen": 282329088 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738936810431294, + "loss": 1.6958, + "theoretical_loss": 4.204558489024561, + "tokens_seen": 282394624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389207622868606, + "loss": 1.5529, + "theoretical_loss": 4.204443673148575, + "tokens_seen": 282460160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389047141424273, + "loss": 1.4651, + "theoretical_loss": 4.204328891365977, + "tokens_seen": 282525696 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738888665997994, + "loss": 1.551, + "theoretical_loss": 4.2042141436587395, + "tokens_seen": 282591232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388726178535607, + "loss": 1.5604, + "theoretical_loss": 4.204099430008846, + "tokens_seen": 282656768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388565697091274, + "loss": 1.5569, + "theoretical_loss": 4.203984750398296, + "tokens_seen": 282722304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388405215646942, + "loss": 1.6475, + "theoretical_loss": 4.203870104809101, + "tokens_seen": 282787840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388244734202608, + "loss": 1.5684, + "theoretical_loss": 4.203755493223289, + "tokens_seen": 282853376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388084252758276, + "loss": 1.7048, + "theoretical_loss": 4.203640915622898, + "tokens_seen": 282918912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387923771313942, + "loss": 1.6332, + "theoretical_loss": 4.203526371989983, + "tokens_seen": 282984448 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738776328986961, + "loss": 1.6873, + "theoretical_loss": 4.203411862306609, + "tokens_seen": 283049984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387602808425276, + "loss": 1.6564, + "theoretical_loss": 4.203297386554859, + "tokens_seen": 283115520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387442326980944, + "loss": 1.6088, + "theoretical_loss": 4.203182944716826, + "tokens_seen": 283181056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387281845536611, + "loss": 1.6262, + "theoretical_loss": 4.203068536774619, + "tokens_seen": 283246592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387121364092278, + "loss": 1.5406, + "theoretical_loss": 4.202954162710357, + "tokens_seen": 283312128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386960882647945, + "loss": 1.731, + "theoretical_loss": 4.202839822506178, + "tokens_seen": 283377664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386800401203612, + "loss": 1.6566, + "theoretical_loss": 4.202725516144228, + "tokens_seen": 283443200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386639919759279, + "loss": 1.6085, + "theoretical_loss": 4.202611243606671, + "tokens_seen": 283508736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386479438314946, + "loss": 1.7107, + "theoretical_loss": 4.202497004875681, + "tokens_seen": 283574272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386318956870613, + "loss": 1.6096, + "theoretical_loss": 4.202382799933449, + "tokens_seen": 283639808 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738615847542628, + "loss": 1.4388, + "theoretical_loss": 4.2022686287621775, + "tokens_seen": 283705344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385997993981946, + "loss": 1.588, + "theoretical_loss": 4.20215449134408, + "tokens_seen": 283770880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385837512537614, + "loss": 1.634, + "theoretical_loss": 4.202040387661389, + "tokens_seen": 283836416 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738567703109328, + "loss": 1.6546, + "theoretical_loss": 4.201926317696347, + "tokens_seen": 283901952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385516549648948, + "loss": 1.656, + "theoretical_loss": 4.201812281431209, + "tokens_seen": 283967488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385356068204614, + "loss": 1.6949, + "theoretical_loss": 4.201698278848246, + "tokens_seen": 284033024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385195586760282, + "loss": 1.5907, + "theoretical_loss": 4.201584309929743, + "tokens_seen": 284098560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385035105315948, + "loss": 1.6153, + "theoretical_loss": 4.201470374657993, + "tokens_seen": 284164096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384874623871615, + "loss": 1.6676, + "theoretical_loss": 4.201356473015308, + "tokens_seen": 284229632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384714142427282, + "loss": 1.6801, + "theoretical_loss": 4.201242604984013, + "tokens_seen": 284295168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384553660982949, + "loss": 1.7536, + "theoretical_loss": 4.201128770546442, + "tokens_seen": 284360704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384393179538616, + "loss": 1.6957, + "theoretical_loss": 4.201014969684948, + "tokens_seen": 284426240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384232698094283, + "loss": 1.6393, + "theoretical_loss": 4.200901202381893, + "tokens_seen": 284491776 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738407221664995, + "loss": 1.6103, + "theoretical_loss": 4.200787468619653, + "tokens_seen": 284557312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383911735205617, + "loss": 1.6311, + "theoretical_loss": 4.200673768380621, + "tokens_seen": 284622848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383751253761284, + "loss": 1.6207, + "theoretical_loss": 4.200560101647198, + "tokens_seen": 284688384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383590772316951, + "loss": 1.5322, + "theoretical_loss": 4.2004464684018, + "tokens_seen": 284753920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383430290872618, + "loss": 1.5648, + "theoretical_loss": 4.200332868626861, + "tokens_seen": 284819456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383269809428285, + "loss": 1.4811, + "theoretical_loss": 4.200219302304821, + "tokens_seen": 284884992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383109327983952, + "loss": 1.5884, + "theoretical_loss": 4.200105769418138, + "tokens_seen": 284950528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382948846539619, + "loss": 1.6323, + "theoretical_loss": 4.199992269949281, + "tokens_seen": 285016064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 101193, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5234239101409912, + "objective/train/theoretical_loss": 4.199878803880733, + "objective/train/tokens_used": 305541600, + "theoretical_loss": 4.199878803880733, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382788365095286, + "loss": 1.4773, + "theoretical_loss": 4.199878803880733, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382627883650953, + "loss": 1.4435, + "theoretical_loss": 4.1997653711949905, + "tokens_seen": 285147136 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738246740220662, + "loss": 1.5471, + "theoretical_loss": 4.199651971874562, + "tokens_seen": 285212672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382306920762287, + "loss": 1.5007, + "theoretical_loss": 4.199538605901973, + "tokens_seen": 285278208 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382146439317954, + "loss": 1.6037, + "theoretical_loss": 4.199425273259756, + "tokens_seen": 285343744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381985957873621, + "loss": 1.5669, + "theoretical_loss": 4.199311973930461, + "tokens_seen": 285409280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381825476429288, + "loss": 1.5726, + "theoretical_loss": 4.199198707896651, + "tokens_seen": 285474816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381664994984955, + "loss": 1.5804, + "theoretical_loss": 4.199085475140899, + "tokens_seen": 285540352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381504513540622, + "loss": 1.6743, + "theoretical_loss": 4.1989722756457954, + "tokens_seen": 285605888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381344032096289, + "loss": 1.5639, + "theoretical_loss": 4.198859109393941, + "tokens_seen": 285671424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381183550651956, + "loss": 1.5105, + "theoretical_loss": 4.198745976367949, + "tokens_seen": 285736960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381023069207623, + "loss": 1.5629, + "theoretical_loss": 4.198632876550449, + "tokens_seen": 285802496 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738086258776329, + "loss": 1.5286, + "theoretical_loss": 4.19851980992408, + "tokens_seen": 285868032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380702106318957, + "loss": 1.6526, + "theoretical_loss": 4.198406776471497, + "tokens_seen": 285933568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380541624874624, + "loss": 1.7248, + "theoretical_loss": 4.198293776175365, + "tokens_seen": 285999104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380381143430291, + "loss": 1.4311, + "theoretical_loss": 4.198180809018366, + "tokens_seen": 286064640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380220661985958, + "loss": 1.5445, + "theoretical_loss": 4.198067874983192, + "tokens_seen": 286130176 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380060180541625, + "loss": 1.58, + "theoretical_loss": 4.197954974052546, + "tokens_seen": 286195712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379899699097292, + "loss": 1.5693, + "theoretical_loss": 4.19784210620915, + "tokens_seen": 286261248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379739217652959, + "loss": 1.6604, + "theoretical_loss": 4.197729271435736, + "tokens_seen": 286326784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379578736208626, + "loss": 1.6563, + "theoretical_loss": 4.197616469715046, + "tokens_seen": 286392320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379418254764293, + "loss": 1.75, + "theoretical_loss": 4.19750370102984, + "tokens_seen": 286457856 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737925777331996, + "loss": 1.7021, + "theoretical_loss": 4.197390965362887, + "tokens_seen": 286523392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379097291875627, + "loss": 1.567, + "theoretical_loss": 4.197278262696971, + "tokens_seen": 286588928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378936810431294, + "loss": 1.5716, + "theoretical_loss": 4.197165593014889, + "tokens_seen": 286654464 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378776328986961, + "loss": 1.6708, + "theoretical_loss": 4.197052956299449, + "tokens_seen": 286720000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378615847542628, + "loss": 1.5052, + "theoretical_loss": 4.196940352533474, + "tokens_seen": 286785536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378455366098295, + "loss": 1.5812, + "theoretical_loss": 4.196827781699799, + "tokens_seen": 286851072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378294884653962, + "loss": 1.6132, + "theoretical_loss": 4.196715243781273, + "tokens_seen": 286916608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378134403209629, + "loss": 1.6742, + "theoretical_loss": 4.196602738760754, + "tokens_seen": 286982144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377973921765296, + "loss": 1.6104, + "theoretical_loss": 4.196490266621119, + "tokens_seen": 287047680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377813440320963, + "loss": 1.6694, + "theoretical_loss": 4.196377827345252, + "tokens_seen": 287113216 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737765295887663, + "loss": 1.5387, + "theoretical_loss": 4.196265420916053, + "tokens_seen": 287178752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377492477432297, + "loss": 1.6054, + "theoretical_loss": 4.196153047316433, + "tokens_seen": 287244288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377331995987964, + "loss": 1.5317, + "theoretical_loss": 4.196040706529319, + "tokens_seen": 287309824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377171514543631, + "loss": 1.6415, + "theoretical_loss": 4.195928398537647, + "tokens_seen": 287375360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377011033099298, + "loss": 1.6341, + "theoretical_loss": 4.195816123324367, + "tokens_seen": 287440896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376850551654965, + "loss": 1.6622, + "theoretical_loss": 4.195703880872443, + "tokens_seen": 287506432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376690070210632, + "loss": 1.5524, + "theoretical_loss": 4.195591671164852, + "tokens_seen": 287571968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376529588766299, + "loss": 1.5814, + "theoretical_loss": 4.195479494184581, + "tokens_seen": 287637504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376369107321966, + "loss": 1.5762, + "theoretical_loss": 4.195367349914631, + "tokens_seen": 287703040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376208625877633, + "loss": 1.5798, + "theoretical_loss": 4.195255238338016, + "tokens_seen": 287768576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00073760481444333, + "loss": 1.5229, + "theoretical_loss": 4.195143159437764, + "tokens_seen": 287834112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375887662988967, + "loss": 1.5433, + "theoretical_loss": 4.195031113196913, + "tokens_seen": 287899648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375727181544634, + "loss": 1.6659, + "theoretical_loss": 4.194919099598516, + "tokens_seen": 287965184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375566700100301, + "loss": 1.5775, + "theoretical_loss": 4.194807118625636, + "tokens_seen": 288030720 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375406218655968, + "loss": 1.6525, + "theoretical_loss": 4.194695170261353, + "tokens_seen": 288096256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375245737211636, + "loss": 1.5722, + "theoretical_loss": 4.194583254488754, + "tokens_seen": 288161792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375085255767302, + "loss": 1.5539, + "theoretical_loss": 4.194471371290944, + "tokens_seen": 288227328 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737492477432297, + "loss": 1.6119, + "theoretical_loss": 4.194359520651036, + "tokens_seen": 288292864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 102045, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5714504718780518, + "objective/train/theoretical_loss": 4.1942477025521585, + "objective/train/tokens_used": 308818400, + "theoretical_loss": 4.1942477025521585, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007374764292878637, + "loss": 1.5811, + "theoretical_loss": 4.1942477025521585, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007374603811434304, + "loss": 1.6291, + "theoretical_loss": 4.194135916977452, + "tokens_seen": 288423936 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737444332998997, + "loss": 1.6362, + "theoretical_loss": 4.19402416391007, + "tokens_seen": 288489472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007374282848545638, + "loss": 1.5195, + "theoretical_loss": 4.193912443333177, + "tokens_seen": 288555008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007374122367101305, + "loss": 1.474, + "theoretical_loss": 4.193800755229951, + "tokens_seen": 288620544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373961885656972, + "loss": 1.6996, + "theoretical_loss": 4.193689099583582, + "tokens_seen": 288686080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373801404212639, + "loss": 1.5586, + "theoretical_loss": 4.193577476377275, + "tokens_seen": 288751616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373640922768306, + "loss": 1.5385, + "theoretical_loss": 4.193465885594242, + "tokens_seen": 288817152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373480441323973, + "loss": 1.5508, + "theoretical_loss": 4.1933543272177145, + "tokens_seen": 288882688 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737331995987964, + "loss": 1.6673, + "theoretical_loss": 4.193242801230931, + "tokens_seen": 288948224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373159478435307, + "loss": 1.6181, + "theoretical_loss": 4.193131307617145, + "tokens_seen": 289013760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372998996990974, + "loss": 1.6803, + "theoretical_loss": 4.193019846359622, + "tokens_seen": 289079296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372838515546641, + "loss": 1.6104, + "theoretical_loss": 4.192908417441639, + "tokens_seen": 289144832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372678034102308, + "loss": 1.6341, + "theoretical_loss": 4.192797020846487, + "tokens_seen": 289210368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372517552657975, + "loss": 1.5669, + "theoretical_loss": 4.192685656557468, + "tokens_seen": 289275904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372357071213642, + "loss": 1.6607, + "theoretical_loss": 4.1925743245578975, + "tokens_seen": 289341440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372196589769309, + "loss": 1.5765, + "theoretical_loss": 4.1924630248311034, + "tokens_seen": 289406976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372036108324975, + "loss": 1.48, + "theoretical_loss": 4.192351757360425, + "tokens_seen": 289472512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371875626880643, + "loss": 1.5329, + "theoretical_loss": 4.1922405221292145, + "tokens_seen": 289538048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371715145436309, + "loss": 1.4819, + "theoretical_loss": 4.192129319120838, + "tokens_seen": 289603584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371554663991977, + "loss": 1.6302, + "theoretical_loss": 4.1920181483186685, + "tokens_seen": 289669120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371394182547643, + "loss": 1.5838, + "theoretical_loss": 4.1919070097060995, + "tokens_seen": 289734656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371233701103311, + "loss": 1.5787, + "theoretical_loss": 4.19179590326653, + "tokens_seen": 289800192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371073219658977, + "loss": 1.5751, + "theoretical_loss": 4.191684828983375, + "tokens_seen": 289865728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370912738214645, + "loss": 1.664, + "theoretical_loss": 4.191573786840061, + "tokens_seen": 289931264 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370752256770311, + "loss": 1.5749, + "theoretical_loss": 4.191462776820025, + "tokens_seen": 289996800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370591775325978, + "loss": 1.604, + "theoretical_loss": 4.191351798906719, + "tokens_seen": 290062336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370431293881645, + "loss": 1.7533, + "theoretical_loss": 4.191240853083604, + "tokens_seen": 290127872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370270812437312, + "loss": 1.6221, + "theoretical_loss": 4.191129939334159, + "tokens_seen": 290193408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370110330992979, + "loss": 1.5681, + "theoretical_loss": 4.191019057641868, + "tokens_seen": 290258944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369949849548646, + "loss": 1.6289, + "theoretical_loss": 4.1909082079902324, + "tokens_seen": 290324480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369789368104313, + "loss": 1.7017, + "theoretical_loss": 4.190797390362764, + "tokens_seen": 290390016 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736962888665998, + "loss": 1.5707, + "theoretical_loss": 4.190686604742986, + "tokens_seen": 290455552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369468405215647, + "loss": 1.6355, + "theoretical_loss": 4.190575851114437, + "tokens_seen": 290521088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369307923771314, + "loss": 1.7072, + "theoretical_loss": 4.190465129460662, + "tokens_seen": 290586624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369147442326981, + "loss": 1.6748, + "theoretical_loss": 4.190354439765224, + "tokens_seen": 290652160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368986960882648, + "loss": 1.5803, + "theoretical_loss": 4.190243782011696, + "tokens_seen": 290717696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368826479438315, + "loss": 1.6931, + "theoretical_loss": 4.190133156183663, + "tokens_seen": 290783232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368665997993982, + "loss": 1.5436, + "theoretical_loss": 4.190022562264721, + "tokens_seen": 290848768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368505516549649, + "loss": 1.6706, + "theoretical_loss": 4.189912000238479, + "tokens_seen": 290914304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368345035105316, + "loss": 1.5295, + "theoretical_loss": 4.189801470088559, + "tokens_seen": 290979840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368184553660983, + "loss": 1.6353, + "theoretical_loss": 4.189690971798596, + "tokens_seen": 291045376 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736802407221665, + "loss": 1.6281, + "theoretical_loss": 4.189580505352232, + "tokens_seen": 291110912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367863590772317, + "loss": 1.6095, + "theoretical_loss": 4.189470070733129, + "tokens_seen": 291176448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367703109327984, + "loss": 1.6304, + "theoretical_loss": 4.189359667924952, + "tokens_seen": 291241984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367542627883651, + "loss": 1.6004, + "theoretical_loss": 4.189249296911386, + "tokens_seen": 291307520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367382146439318, + "loss": 1.6946, + "theoretical_loss": 4.189138957676125, + "tokens_seen": 291373056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367221664994985, + "loss": 1.7297, + "theoretical_loss": 4.1890286502028715, + "tokens_seen": 291438592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367061183550652, + "loss": 1.6308, + "theoretical_loss": 4.188918374475347, + "tokens_seen": 291504128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366900702106319, + "loss": 1.6745, + "theoretical_loss": 4.188808130477279, + "tokens_seen": 291569664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 103221, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6348395347595215, + "objective/train/theoretical_loss": 4.1886979181924096, + "objective/train/tokens_used": 312095200, + "theoretical_loss": 4.1886979181924096, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366740220661986, + "loss": 1.647, + "theoretical_loss": 4.1886979181924096, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366579739217653, + "loss": 1.7113, + "theoretical_loss": 4.188587737604493, + "tokens_seen": 291700736 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736641925777332, + "loss": 1.6427, + "theoretical_loss": 4.188477588697295, + "tokens_seen": 291766272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366258776328987, + "loss": 1.5969, + "theoretical_loss": 4.188367471454593, + "tokens_seen": 291831808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366098294884654, + "loss": 1.4803, + "theoretical_loss": 4.188257385860177, + "tokens_seen": 291897344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365937813440321, + "loss": 1.6042, + "theoretical_loss": 4.188147331897849, + "tokens_seen": 291962880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365777331995988, + "loss": 1.6471, + "theoretical_loss": 4.1880373095514205, + "tokens_seen": 292028416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365616850551655, + "loss": 1.6034, + "theoretical_loss": 4.18792731880472, + "tokens_seen": 292093952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365456369107322, + "loss": 1.5614, + "theoretical_loss": 4.187817359641581, + "tokens_seen": 292159488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365295887662989, + "loss": 1.5428, + "theoretical_loss": 4.1877074320458565, + "tokens_seen": 292225024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365135406218656, + "loss": 1.5486, + "theoretical_loss": 4.187597536001406, + "tokens_seen": 292290560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364974924774323, + "loss": 1.5498, + "theoretical_loss": 4.187487671492102, + "tokens_seen": 292356096 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736481444332999, + "loss": 1.6196, + "theoretical_loss": 4.18737783850183, + "tokens_seen": 292421632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364653961885657, + "loss": 1.6716, + "theoretical_loss": 4.187268037014486, + "tokens_seen": 292487168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364493480441324, + "loss": 1.6067, + "theoretical_loss": 4.187158267013979, + "tokens_seen": 292552704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364332998996991, + "loss": 1.517, + "theoretical_loss": 4.18704852848423, + "tokens_seen": 292618240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364172517552658, + "loss": 1.6312, + "theoretical_loss": 4.18693882140917, + "tokens_seen": 292683776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364012036108325, + "loss": 1.4959, + "theoretical_loss": 4.186829145772743, + "tokens_seen": 292749312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363851554663992, + "loss": 1.5707, + "theoretical_loss": 4.186719501558905, + "tokens_seen": 292814848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363691073219659, + "loss": 1.5797, + "theoretical_loss": 4.186609888751623, + "tokens_seen": 292880384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363530591775326, + "loss": 1.6421, + "theoretical_loss": 4.186500307334878, + "tokens_seen": 292945920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363370110330993, + "loss": 1.6258, + "theoretical_loss": 4.186390757292659, + "tokens_seen": 293011456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363209628886661, + "loss": 1.5723, + "theoretical_loss": 4.18628123860897, + "tokens_seen": 293076992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363049147442327, + "loss": 1.638, + "theoretical_loss": 4.186171751267825, + "tokens_seen": 293142528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362888665997995, + "loss": 1.6085, + "theoretical_loss": 4.186062295253249, + "tokens_seen": 293208064 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362728184553661, + "loss": 1.6619, + "theoretical_loss": 4.185952870549283, + "tokens_seen": 293273600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362567703109329, + "loss": 1.5903, + "theoretical_loss": 4.185843477139974, + "tokens_seen": 293339136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362407221664995, + "loss": 1.555, + "theoretical_loss": 4.185734115009383, + "tokens_seen": 293404672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362246740220663, + "loss": 1.5025, + "theoretical_loss": 4.185624784141585, + "tokens_seen": 293470208 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736208625877633, + "loss": 1.5115, + "theoretical_loss": 4.185515484520664, + "tokens_seen": 293535744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361925777331997, + "loss": 1.6593, + "theoretical_loss": 4.1854062161307155, + "tokens_seen": 293601280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361765295887663, + "loss": 1.7188, + "theoretical_loss": 4.185296978955848, + "tokens_seen": 293666816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361604814443331, + "loss": 1.5893, + "theoretical_loss": 4.185187772980181, + "tokens_seen": 293732352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361444332998997, + "loss": 1.4278, + "theoretical_loss": 4.185078598187846, + "tokens_seen": 293797888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361283851554665, + "loss": 1.4631, + "theoretical_loss": 4.184969454562985, + "tokens_seen": 293863424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361123370110332, + "loss": 1.5732, + "theoretical_loss": 4.1848603420897525, + "tokens_seen": 293928960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360962888665999, + "loss": 1.5025, + "theoretical_loss": 4.184751260752316, + "tokens_seen": 293994496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360802407221666, + "loss": 1.5542, + "theoretical_loss": 4.18464221053485, + "tokens_seen": 294060032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360641925777333, + "loss": 1.6657, + "theoretical_loss": 4.184533191421547, + "tokens_seen": 294125568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360481444333, + "loss": 1.7267, + "theoretical_loss": 4.184424203396606, + "tokens_seen": 294191104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360320962888667, + "loss": 1.4338, + "theoretical_loss": 4.184315246444239, + "tokens_seen": 294256640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360160481444334, + "loss": 1.5626, + "theoretical_loss": 4.18420632054867, + "tokens_seen": 294322176 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360000000000001, + "loss": 1.6043, + "theoretical_loss": 4.1840974256941355, + "tokens_seen": 294387712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359839518555668, + "loss": 1.5946, + "theoretical_loss": 4.183988561864879, + "tokens_seen": 294453248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359679037111335, + "loss": 1.5876, + "theoretical_loss": 4.183879729045163, + "tokens_seen": 294518784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359518555667002, + "loss": 1.6075, + "theoretical_loss": 4.183770927219255, + "tokens_seen": 294584320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359358074222669, + "loss": 1.5569, + "theoretical_loss": 4.183662156371436, + "tokens_seen": 294649856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359197592778336, + "loss": 1.6115, + "theoretical_loss": 4.183553416485999, + "tokens_seen": 294715392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359037111334003, + "loss": 1.5446, + "theoretical_loss": 4.183444707547249, + "tokens_seen": 294780928 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735887662988967, + "loss": 1.6602, + "theoretical_loss": 4.1833360295395, + "tokens_seen": 294846464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 104159, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.72190523147583, + "objective/train/theoretical_loss": 4.183227382447081, + "objective/train/tokens_used": 315372000, + "theoretical_loss": 4.183227382447081, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358716148445337, + "loss": 1.6478, + "theoretical_loss": 4.183227382447081, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358555667001004, + "loss": 1.6181, + "theoretical_loss": 4.183118766254328, + "tokens_seen": 294977536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358395185556671, + "loss": 1.6082, + "theoretical_loss": 4.183010180945593, + "tokens_seen": 295043072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358234704112338, + "loss": 1.5291, + "theoretical_loss": 4.182901626505237, + "tokens_seen": 295108608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358074222668004, + "loss": 1.6135, + "theoretical_loss": 4.182793102917632, + "tokens_seen": 295174144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357913741223672, + "loss": 1.5915, + "theoretical_loss": 4.182684610167162, + "tokens_seen": 295239680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357753259779338, + "loss": 1.6561, + "theoretical_loss": 4.1825761482382235, + "tokens_seen": 295305216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357592778335006, + "loss": 1.7016, + "theoretical_loss": 4.182467717115221, + "tokens_seen": 295370752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357432296890672, + "loss": 1.6305, + "theoretical_loss": 4.182359316782576, + "tokens_seen": 295436288 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735727181544634, + "loss": 1.5485, + "theoretical_loss": 4.1822509472247145, + "tokens_seen": 295501824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357111334002006, + "loss": 1.6077, + "theoretical_loss": 4.18214260842608, + "tokens_seen": 295567360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356950852557674, + "loss": 1.5885, + "theoretical_loss": 4.182034300371122, + "tokens_seen": 295632896 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735679037111334, + "loss": 1.564, + "theoretical_loss": 4.181926023044306, + "tokens_seen": 295698432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356629889669007, + "loss": 1.5082, + "theoretical_loss": 4.181817776430106, + "tokens_seen": 295763968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356469408224674, + "loss": 1.711, + "theoretical_loss": 4.181709560513008, + "tokens_seen": 295829504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356308926780341, + "loss": 1.616, + "theoretical_loss": 4.181601375277509, + "tokens_seen": 295895040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356148445336008, + "loss": 1.6577, + "theoretical_loss": 4.181493220708117, + "tokens_seen": 295960576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355987963891675, + "loss": 1.6458, + "theoretical_loss": 4.181385096789353, + "tokens_seen": 296026112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355827482447342, + "loss": 1.6822, + "theoretical_loss": 4.181277003505747, + "tokens_seen": 296091648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355667001003009, + "loss": 1.5702, + "theoretical_loss": 4.181168940841843, + "tokens_seen": 296157184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355506519558676, + "loss": 1.6557, + "theoretical_loss": 4.181060908782191, + "tokens_seen": 296222720 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355346038114343, + "loss": 1.5546, + "theoretical_loss": 4.18095290731136, + "tokens_seen": 296288256 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735518555667001, + "loss": 1.53, + "theoretical_loss": 4.180844936413922, + "tokens_seen": 296353792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355025075225677, + "loss": 1.562, + "theoretical_loss": 4.180736996074465, + "tokens_seen": 296419328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354864593781344, + "loss": 1.7042, + "theoretical_loss": 4.180629086277589, + "tokens_seen": 296484864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354704112337011, + "loss": 1.5895, + "theoretical_loss": 4.180521207007902, + "tokens_seen": 296550400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354543630892678, + "loss": 1.6513, + "theoretical_loss": 4.180413358250024, + "tokens_seen": 296615936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354383149448345, + "loss": 1.6866, + "theoretical_loss": 4.180305539988588, + "tokens_seen": 296681472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354222668004012, + "loss": 1.5783, + "theoretical_loss": 4.180197752208235, + "tokens_seen": 296747008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354062186559679, + "loss": 1.6043, + "theoretical_loss": 4.1800899948936205, + "tokens_seen": 296812544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353901705115346, + "loss": 1.565, + "theoretical_loss": 4.179982268029409, + "tokens_seen": 296878080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353741223671013, + "loss": 1.5138, + "theoretical_loss": 4.179874571600277, + "tokens_seen": 296943616 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735358074222668, + "loss": 1.6457, + "theoretical_loss": 4.1797669055909115, + "tokens_seen": 297009152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353420260782347, + "loss": 1.6078, + "theoretical_loss": 4.1796592699860105, + "tokens_seen": 297074688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353259779338014, + "loss": 1.691, + "theoretical_loss": 4.179551664770283, + "tokens_seen": 297140224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353099297893681, + "loss": 1.5473, + "theoretical_loss": 4.1794440899284515, + "tokens_seen": 297205760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352938816449348, + "loss": 1.5952, + "theoretical_loss": 4.179336545445245, + "tokens_seen": 297271296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352778335005015, + "loss": 1.5846, + "theoretical_loss": 4.1792290313054075, + "tokens_seen": 297336832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352617853560682, + "loss": 1.5803, + "theoretical_loss": 4.179121547493692, + "tokens_seen": 297402368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352457372116349, + "loss": 1.6163, + "theoretical_loss": 4.179014093994862, + "tokens_seen": 297467904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352296890672016, + "loss": 1.59, + "theoretical_loss": 4.178906670793695, + "tokens_seen": 297533440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352136409227683, + "loss": 1.6357, + "theoretical_loss": 4.178799277874977, + "tokens_seen": 297598976 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735197592778335, + "loss": 1.6383, + "theoretical_loss": 4.178691915223505, + "tokens_seen": 297664512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351815446339017, + "loss": 1.5541, + "theoretical_loss": 4.178584582824088, + "tokens_seen": 297730048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351654964894684, + "loss": 1.7037, + "theoretical_loss": 4.1784772806615464, + "tokens_seen": 297795584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351494483450351, + "loss": 1.6525, + "theoretical_loss": 4.178370008720709, + "tokens_seen": 297861120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351334002006018, + "loss": 1.5659, + "theoretical_loss": 4.178262766986418, + "tokens_seen": 297926656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351173520561686, + "loss": 1.5114, + "theoretical_loss": 4.178155555443525, + "tokens_seen": 297992192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351013039117352, + "loss": 1.7272, + "theoretical_loss": 4.178048374076894, + "tokens_seen": 298057728 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735085255767302, + "loss": 1.6131, + "theoretical_loss": 4.1779412228714, + "tokens_seen": 298123264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 105423, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5215253829956055, + "objective/train/theoretical_loss": 4.177834101811927, + "objective/train/tokens_used": 318648800, + "theoretical_loss": 4.177834101811927, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007350692076228686, + "loss": 1.5549, + "theoretical_loss": 4.177834101811927, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007350531594784354, + "loss": 1.4696, + "theoretical_loss": 4.1777270108833715, + "tokens_seen": 298254336 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735037111334002, + "loss": 1.6553, + "theoretical_loss": 4.177619950070639, + "tokens_seen": 298319872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007350210631895688, + "loss": 1.6148, + "theoretical_loss": 4.177512919358649, + "tokens_seen": 298385408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007350050150451354, + "loss": 1.6731, + "theoretical_loss": 4.17740591873233, + "tokens_seen": 298450944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349889669007022, + "loss": 1.6382, + "theoretical_loss": 4.177298948176619, + "tokens_seen": 298516480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349729187562688, + "loss": 1.4472, + "theoretical_loss": 4.17719200767647, + "tokens_seen": 298582016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349568706118356, + "loss": 1.6397, + "theoretical_loss": 4.177085097216842, + "tokens_seen": 298647552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349408224674022, + "loss": 1.6669, + "theoretical_loss": 4.1769782167827065, + "tokens_seen": 298713088 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734924774322969, + "loss": 1.6121, + "theoretical_loss": 4.176871366359047, + "tokens_seen": 298778624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349087261785356, + "loss": 1.4917, + "theoretical_loss": 4.176764545930858, + "tokens_seen": 298844160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348926780341024, + "loss": 1.5772, + "theoretical_loss": 4.176657755483143, + "tokens_seen": 298909696 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734876629889669, + "loss": 1.6754, + "theoretical_loss": 4.176550995000917, + "tokens_seen": 298975232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348605817452358, + "loss": 1.6027, + "theoretical_loss": 4.176444264469206, + "tokens_seen": 299040768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348445336008024, + "loss": 1.7495, + "theoretical_loss": 4.176337563873046, + "tokens_seen": 299106304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348284854563692, + "loss": 1.6497, + "theoretical_loss": 4.176230893197486, + "tokens_seen": 299171840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348124373119358, + "loss": 1.51, + "theoretical_loss": 4.1761242524275834, + "tokens_seen": 299237376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347963891675026, + "loss": 1.5526, + "theoretical_loss": 4.176017641548407, + "tokens_seen": 299302912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347803410230692, + "loss": 1.7034, + "theoretical_loss": 4.175911060545037, + "tokens_seen": 299368448 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734764292878636, + "loss": 1.6063, + "theoretical_loss": 4.175804509402562, + "tokens_seen": 299433984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347482447342027, + "loss": 1.4916, + "theoretical_loss": 4.175697988106085, + "tokens_seen": 299499520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347321965897694, + "loss": 1.7321, + "theoretical_loss": 4.175591496640718, + "tokens_seen": 299565056 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734716148445336, + "loss": 1.5392, + "theoretical_loss": 4.175485034991581, + "tokens_seen": 299630592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347001003009028, + "loss": 1.5464, + "theoretical_loss": 4.17537860314381, + "tokens_seen": 299696128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346840521564695, + "loss": 1.5023, + "theoretical_loss": 4.175272201082547, + "tokens_seen": 299761664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346680040120362, + "loss": 1.5762, + "theoretical_loss": 4.175165828792946, + "tokens_seen": 299827200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346519558676029, + "loss": 1.6361, + "theoretical_loss": 4.1750594862601735, + "tokens_seen": 299892736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346359077231696, + "loss": 1.6891, + "theoretical_loss": 4.1749531734694045, + "tokens_seen": 299958272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346198595787363, + "loss": 1.5498, + "theoretical_loss": 4.174846890405825, + "tokens_seen": 300023808 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734603811434303, + "loss": 1.5768, + "theoretical_loss": 4.174740637054632, + "tokens_seen": 300089344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345877632898697, + "loss": 1.5761, + "theoretical_loss": 4.174634413401034, + "tokens_seen": 300154880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345717151454364, + "loss": 1.5583, + "theoretical_loss": 4.174528219430247, + "tokens_seen": 300220416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345556670010031, + "loss": 1.5313, + "theoretical_loss": 4.174422055127502, + "tokens_seen": 300285952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345396188565698, + "loss": 1.6252, + "theoretical_loss": 4.174315920478037, + "tokens_seen": 300351488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345235707121365, + "loss": 1.5832, + "theoretical_loss": 4.174209815467102, + "tokens_seen": 300417024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345075225677032, + "loss": 1.5806, + "theoretical_loss": 4.174103740079958, + "tokens_seen": 300482560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344914744232699, + "loss": 1.6372, + "theoretical_loss": 4.1739976943018755, + "tokens_seen": 300548096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344754262788366, + "loss": 1.7141, + "theoretical_loss": 4.173891678118135, + "tokens_seen": 300613632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344593781344033, + "loss": 1.592, + "theoretical_loss": 4.173785691514031, + "tokens_seen": 300679168 + }, + { + "epoch": 0.09, + "learning_rate": 0.00073444332998997, + "loss": 1.6567, + "theoretical_loss": 4.173679734474862, + "tokens_seen": 300744704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344272818455367, + "loss": 1.5908, + "theoretical_loss": 4.173573806985945, + "tokens_seen": 300810240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344112337011033, + "loss": 1.6119, + "theoretical_loss": 4.173467909032602, + "tokens_seen": 300875776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343951855566701, + "loss": 1.6013, + "theoretical_loss": 4.173362040600166, + "tokens_seen": 300941312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343791374122367, + "loss": 1.6848, + "theoretical_loss": 4.173256201673983, + "tokens_seen": 301006848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343630892678035, + "loss": 1.5417, + "theoretical_loss": 4.173150392239406, + "tokens_seen": 301072384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343470411233701, + "loss": 1.5092, + "theoretical_loss": 4.173044612281802, + "tokens_seen": 301137920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343309929789369, + "loss": 1.5109, + "theoretical_loss": 4.172938861786546, + "tokens_seen": 301203456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343149448345035, + "loss": 1.5317, + "theoretical_loss": 4.1728331407390264, + "tokens_seen": 301268992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342988966900703, + "loss": 1.58, + "theoretical_loss": 4.172727449124636, + "tokens_seen": 301334528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342828485456369, + "loss": 1.625, + "theoretical_loss": 4.172621786928786, + "tokens_seen": 301400064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 106715, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5944786071777344, + "objective/train/theoretical_loss": 4.17251615413689, + "objective/train/tokens_used": 321925600, + "theoretical_loss": 4.17251615413689, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342668004012036, + "loss": 1.6601, + "theoretical_loss": 4.17251615413689, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342507522567703, + "loss": 1.5005, + "theoretical_loss": 4.172410550734378, + "tokens_seen": 301531136 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734234704112337, + "loss": 1.5192, + "theoretical_loss": 4.172304976706689, + "tokens_seen": 301596672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342186559679037, + "loss": 1.5185, + "theoretical_loss": 4.17219943203927, + "tokens_seen": 301662208 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342026078234704, + "loss": 1.7129, + "theoretical_loss": 4.172093916717581, + "tokens_seen": 301727744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341865596790371, + "loss": 1.5986, + "theoretical_loss": 4.171988430727091, + "tokens_seen": 301793280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341705115346038, + "loss": 1.5291, + "theoretical_loss": 4.17188297405328, + "tokens_seen": 301858816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341544633901705, + "loss": 1.6157, + "theoretical_loss": 4.171777546681638, + "tokens_seen": 301924352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341384152457372, + "loss": 1.6012, + "theoretical_loss": 4.171672148597665, + "tokens_seen": 301989888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341223671013039, + "loss": 1.7156, + "theoretical_loss": 4.171566779786872, + "tokens_seen": 302055424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341063189568706, + "loss": 1.5974, + "theoretical_loss": 4.17146144023478, + "tokens_seen": 302120960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340902708124373, + "loss": 1.5717, + "theoretical_loss": 4.171356129926921, + "tokens_seen": 302186496 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734074222668004, + "loss": 1.647, + "theoretical_loss": 4.171250848848835, + "tokens_seen": 302252032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340581745235707, + "loss": 1.7013, + "theoretical_loss": 4.171145596986076, + "tokens_seen": 302317568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340421263791374, + "loss": 1.5124, + "theoretical_loss": 4.171040374324204, + "tokens_seen": 302383104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340260782347041, + "loss": 1.5927, + "theoretical_loss": 4.170935180848793, + "tokens_seen": 302448640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340100300902708, + "loss": 1.5633, + "theoretical_loss": 4.170830016545425, + "tokens_seen": 302514176 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339939819458375, + "loss": 1.7269, + "theoretical_loss": 4.1707248813996936, + "tokens_seen": 302579712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339779338014042, + "loss": 1.6634, + "theoretical_loss": 4.170619775397201, + "tokens_seen": 302645248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339618856569709, + "loss": 1.7733, + "theoretical_loss": 4.1705146985235615, + "tokens_seen": 302710784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339458375125376, + "loss": 1.6171, + "theoretical_loss": 4.170409650764397, + "tokens_seen": 302776320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339297893681043, + "loss": 1.49, + "theoretical_loss": 4.170304632105344, + "tokens_seen": 302841856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339137412236711, + "loss": 1.6787, + "theoretical_loss": 4.170199642532045, + "tokens_seen": 302907392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338976930792377, + "loss": 1.5294, + "theoretical_loss": 4.170094682030154, + "tokens_seen": 302972928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338816449348045, + "loss": 1.5229, + "theoretical_loss": 4.169989750585335, + "tokens_seen": 303038464 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338655967903711, + "loss": 1.6258, + "theoretical_loss": 4.169884848183264, + "tokens_seen": 303104000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338495486459379, + "loss": 1.5401, + "theoretical_loss": 4.169779974809624, + "tokens_seen": 303169536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338335005015045, + "loss": 1.6634, + "theoretical_loss": 4.169675130450111, + "tokens_seen": 303235072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338174523570713, + "loss": 1.5992, + "theoretical_loss": 4.16957031509043, + "tokens_seen": 303300608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338014042126379, + "loss": 1.601, + "theoretical_loss": 4.169465528716295, + "tokens_seen": 303366144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337853560682047, + "loss": 1.4711, + "theoretical_loss": 4.169360771313434, + "tokens_seen": 303431680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337693079237713, + "loss": 1.7924, + "theoretical_loss": 4.169256042867579, + "tokens_seen": 303497216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337532597793381, + "loss": 1.6994, + "theoretical_loss": 4.169151343364476, + "tokens_seen": 303562752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337372116349047, + "loss": 1.5778, + "theoretical_loss": 4.169046672789882, + "tokens_seen": 303628288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337211634904715, + "loss": 1.6038, + "theoretical_loss": 4.168942031129562, + "tokens_seen": 303693824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337051153460381, + "loss": 1.6144, + "theoretical_loss": 4.168837418369292, + "tokens_seen": 303759360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336890672016049, + "loss": 1.6138, + "theoretical_loss": 4.168732834494857, + "tokens_seen": 303824896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336730190571715, + "loss": 1.5502, + "theoretical_loss": 4.1686282794920535, + "tokens_seen": 303890432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336569709127383, + "loss": 1.7256, + "theoretical_loss": 4.168523753346687, + "tokens_seen": 303955968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336409227683049, + "loss": 1.5377, + "theoretical_loss": 4.168419256044573, + "tokens_seen": 304021504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336248746238717, + "loss": 1.632, + "theoretical_loss": 4.168314787571538, + "tokens_seen": 304087040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336088264794383, + "loss": 1.626, + "theoretical_loss": 4.168210347913418, + "tokens_seen": 304152576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335927783350051, + "loss": 1.6017, + "theoretical_loss": 4.168105937056059, + "tokens_seen": 304218112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335767301905717, + "loss": 1.7149, + "theoretical_loss": 4.168001554985317, + "tokens_seen": 304283648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335606820461385, + "loss": 1.6081, + "theoretical_loss": 4.167897201687056, + "tokens_seen": 304349184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335446339017051, + "loss": 1.6175, + "theoretical_loss": 4.167792877147155, + "tokens_seen": 304414720 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335285857572719, + "loss": 1.5451, + "theoretical_loss": 4.1676885813514986, + "tokens_seen": 304480256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335125376128385, + "loss": 1.6038, + "theoretical_loss": 4.167584314285982, + "tokens_seen": 304545792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334964894684053, + "loss": 1.5534, + "theoretical_loss": 4.167480075936512, + "tokens_seen": 304611328 + }, + { + "epoch": 0.09, + "learning_rate": 0.000733480441323972, + "loss": 1.5889, + "theoretical_loss": 4.167375866289004, + "tokens_seen": 304676864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 107694, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6512261629104614, + "objective/train/theoretical_loss": 4.167271685329383, + "objective/train/tokens_used": 325202400, + "theoretical_loss": 4.167271685329383, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334643931795387, + "loss": 1.5713, + "theoretical_loss": 4.167271685329383, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334483450351053, + "loss": 1.6476, + "theoretical_loss": 4.167167533043585, + "tokens_seen": 304807936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334322968906721, + "loss": 1.63, + "theoretical_loss": 4.1670634094175565, + "tokens_seen": 304873472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334162487462387, + "loss": 1.541, + "theoretical_loss": 4.166959314437252, + "tokens_seen": 304939008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334002006018055, + "loss": 1.6109, + "theoretical_loss": 4.166855248088637, + "tokens_seen": 305004544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333841524573721, + "loss": 1.6243, + "theoretical_loss": 4.1667512103576865, + "tokens_seen": 305070080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333681043129389, + "loss": 1.6751, + "theoretical_loss": 4.166647201230386, + "tokens_seen": 305135616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333520561685056, + "loss": 1.6589, + "theoretical_loss": 4.166543220692731, + "tokens_seen": 305201152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333360080240723, + "loss": 1.5901, + "theoretical_loss": 4.166439268730724, + "tokens_seen": 305266688 + }, + { + "epoch": 0.09, + "learning_rate": 0.000733319959879639, + "loss": 1.653, + "theoretical_loss": 4.166335345330383, + "tokens_seen": 305332224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333039117352057, + "loss": 1.6598, + "theoretical_loss": 4.16623145047773, + "tokens_seen": 305397760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332878635907724, + "loss": 1.5896, + "theoretical_loss": 4.166127584158801, + "tokens_seen": 305463296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332718154463391, + "loss": 1.6481, + "theoretical_loss": 4.166023746359639, + "tokens_seen": 305528832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332557673019058, + "loss": 1.6581, + "theoretical_loss": 4.165919937066299, + "tokens_seen": 305594368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332397191574725, + "loss": 1.6001, + "theoretical_loss": 4.165816156264844, + "tokens_seen": 305659904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332236710130392, + "loss": 1.7288, + "theoretical_loss": 4.165712403941348, + "tokens_seen": 305725440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332076228686059, + "loss": 1.5835, + "theoretical_loss": 4.1656086800818946, + "tokens_seen": 305790976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331915747241726, + "loss": 1.6093, + "theoretical_loss": 4.165504984672577, + "tokens_seen": 305856512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331755265797393, + "loss": 1.4599, + "theoretical_loss": 4.165401317699498, + "tokens_seen": 305922048 + }, + { + "epoch": 0.09, + "learning_rate": 0.000733159478435306, + "loss": 1.6132, + "theoretical_loss": 4.16529767914877, + "tokens_seen": 305987584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331434302908727, + "loss": 1.5185, + "theoretical_loss": 4.165194069006516, + "tokens_seen": 306053120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331273821464394, + "loss": 1.6518, + "theoretical_loss": 4.165090487258867, + "tokens_seen": 306118656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331113340020061, + "loss": 1.5807, + "theoretical_loss": 4.164986933891968, + "tokens_seen": 306184192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330952858575728, + "loss": 1.6028, + "theoretical_loss": 4.164883408891968, + "tokens_seen": 306249728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330792377131395, + "loss": 1.6098, + "theoretical_loss": 4.164779912245029, + "tokens_seen": 306315264 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330631895687062, + "loss": 1.5847, + "theoretical_loss": 4.1646764439373225, + "tokens_seen": 306380800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330471414242729, + "loss": 1.5105, + "theoretical_loss": 4.164573003955029, + "tokens_seen": 306446336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330310932798396, + "loss": 1.5924, + "theoretical_loss": 4.164469592284338, + "tokens_seen": 306511872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330150451354062, + "loss": 1.7551, + "theoretical_loss": 4.164366208911453, + "tokens_seen": 306577408 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732998996990973, + "loss": 1.7007, + "theoretical_loss": 4.1642628538225805, + "tokens_seen": 306642944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329829488465396, + "loss": 1.6358, + "theoretical_loss": 4.1641595270039415, + "tokens_seen": 306708480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329669007021064, + "loss": 1.6819, + "theoretical_loss": 4.1640562284417655, + "tokens_seen": 306774016 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732950852557673, + "loss": 1.5444, + "theoretical_loss": 4.16395295812229, + "tokens_seen": 306839552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329348044132398, + "loss": 1.6675, + "theoretical_loss": 4.1638497160317645, + "tokens_seen": 306905088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329187562688064, + "loss": 1.6466, + "theoretical_loss": 4.163746502156448, + "tokens_seen": 306970624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329027081243732, + "loss": 1.4845, + "theoretical_loss": 4.163643316482606, + "tokens_seen": 307036160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328866599799398, + "loss": 1.6061, + "theoretical_loss": 4.1635401589965175, + "tokens_seen": 307101696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328706118355065, + "loss": 1.6429, + "theoretical_loss": 4.163437029684469, + "tokens_seen": 307167232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328545636910732, + "loss": 1.6264, + "theoretical_loss": 4.163333928532758, + "tokens_seen": 307232768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328385155466399, + "loss": 1.6924, + "theoretical_loss": 4.163230855527688, + "tokens_seen": 307298304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328224674022066, + "loss": 1.6224, + "theoretical_loss": 4.163127810655578, + "tokens_seen": 307363840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328064192577733, + "loss": 1.6408, + "theoretical_loss": 4.163024793902752, + "tokens_seen": 307429376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00073279037111334, + "loss": 1.7319, + "theoretical_loss": 4.162921805255543, + "tokens_seen": 307494912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327743229689067, + "loss": 1.501, + "theoretical_loss": 4.162818844700298, + "tokens_seen": 307560448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327582748244734, + "loss": 1.6282, + "theoretical_loss": 4.162715912223369, + "tokens_seen": 307625984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327422266800401, + "loss": 1.5315, + "theoretical_loss": 4.162613007811122, + "tokens_seen": 307691520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327261785356068, + "loss": 1.5266, + "theoretical_loss": 4.1625101314499275, + "tokens_seen": 307757056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327101303911736, + "loss": 1.5706, + "theoretical_loss": 4.162407283126169, + "tokens_seen": 307822592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007326940822467402, + "loss": 1.5297, + "theoretical_loss": 4.1623044628262384, + "tokens_seen": 307888128 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732678034102307, + "loss": 1.6028, + "theoretical_loss": 4.162201670536537, + "tokens_seen": 307953664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 108952, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5811710357666016, + "objective/train/theoretical_loss": 4.162098906243477, + "objective/train/tokens_used": 328479200, + "theoretical_loss": 4.162098906243477, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007326619859578736, + "loss": 1.6008, + "theoretical_loss": 4.162098906243477, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007326459378134404, + "loss": 1.5509, + "theoretical_loss": 4.161996169933477, + "tokens_seen": 308084736 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732629889669007, + "loss": 1.6285, + "theoretical_loss": 4.161893461592968, + "tokens_seen": 308150272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007326138415245738, + "loss": 1.6352, + "theoretical_loss": 4.16179078120839, + "tokens_seen": 308215808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325977933801404, + "loss": 1.5552, + "theoretical_loss": 4.161688128766191, + "tokens_seen": 308281344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325817452357072, + "loss": 1.5523, + "theoretical_loss": 4.16158550425283, + "tokens_seen": 308346880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325656970912738, + "loss": 1.6311, + "theoretical_loss": 4.161482907654775, + "tokens_seen": 308412416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325496489468406, + "loss": 1.6177, + "theoretical_loss": 4.161380338958502, + "tokens_seen": 308477952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325336008024072, + "loss": 1.6076, + "theoretical_loss": 4.161277798150498, + "tokens_seen": 308543488 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732517552657974, + "loss": 1.5029, + "theoretical_loss": 4.16117528521726, + "tokens_seen": 308609024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325015045135406, + "loss": 1.6536, + "theoretical_loss": 4.161072800145292, + "tokens_seen": 308674560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324854563691074, + "loss": 1.5278, + "theoretical_loss": 4.16097034292111, + "tokens_seen": 308740096 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732469408224674, + "loss": 1.4431, + "theoretical_loss": 4.160867913531238, + "tokens_seen": 308805632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324533600802408, + "loss": 1.6229, + "theoretical_loss": 4.160765511962209, + "tokens_seen": 308871168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324373119358074, + "loss": 1.5469, + "theoretical_loss": 4.160663138200567, + "tokens_seen": 308936704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324212637913742, + "loss": 1.6081, + "theoretical_loss": 4.1605607922328645, + "tokens_seen": 309002240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324052156469408, + "loss": 1.6369, + "theoretical_loss": 4.160458474045662, + "tokens_seen": 309067776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007323891675025076, + "loss": 1.736, + "theoretical_loss": 4.16035618362553, + "tokens_seen": 309133312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007323731193580742, + "loss": 1.4865, + "theoretical_loss": 4.16025392095905, + "tokens_seen": 309198848 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732357071213641, + "loss": 1.6449, + "theoretical_loss": 4.160151686032814, + "tokens_seen": 309264384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007323410230692076, + "loss": 1.622, + "theoretical_loss": 4.160049478833416, + "tokens_seen": 309329920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007323249749247744, + "loss": 1.681, + "theoretical_loss": 4.159947299347468, + "tokens_seen": 309395456 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732308926780341, + "loss": 1.5762, + "theoretical_loss": 4.159845147561587, + "tokens_seen": 309460992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322928786359078, + "loss": 1.5544, + "theoretical_loss": 4.1597430234624, + "tokens_seen": 309526528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322768304914744, + "loss": 1.4294, + "theoretical_loss": 4.159640927036541, + "tokens_seen": 309592064 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322607823470412, + "loss": 1.5333, + "theoretical_loss": 4.159538858270659, + "tokens_seen": 309657600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322447342026078, + "loss": 1.5659, + "theoretical_loss": 4.159436817151407, + "tokens_seen": 309723136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322286860581746, + "loss": 1.6312, + "theoretical_loss": 4.159334803665448, + "tokens_seen": 309788672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322126379137412, + "loss": 1.5842, + "theoretical_loss": 4.159232817799458, + "tokens_seen": 309854208 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732196589769308, + "loss": 1.5808, + "theoretical_loss": 4.1591308595401175, + "tokens_seen": 309919744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321805416248746, + "loss": 1.5817, + "theoretical_loss": 4.159028928874118, + "tokens_seen": 309985280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321644934804414, + "loss": 1.6191, + "theoretical_loss": 4.158927025788162, + "tokens_seen": 310050816 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732148445336008, + "loss": 1.624, + "theoretical_loss": 4.158825150268958, + "tokens_seen": 310116352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321323971915748, + "loss": 1.5717, + "theoretical_loss": 4.158723302303227, + "tokens_seen": 310181888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321163490471414, + "loss": 1.6036, + "theoretical_loss": 4.158621481877697, + "tokens_seen": 310247424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321003009027082, + "loss": 1.6511, + "theoretical_loss": 4.158519688979106, + "tokens_seen": 310312960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320842527582748, + "loss": 1.5586, + "theoretical_loss": 4.1584179235942, + "tokens_seen": 310378496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320682046138416, + "loss": 1.6026, + "theoretical_loss": 4.158316185709737, + "tokens_seen": 310444032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320521564694082, + "loss": 1.7038, + "theoretical_loss": 4.158214475312481, + "tokens_seen": 310509568 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732036108324975, + "loss": 1.7325, + "theoretical_loss": 4.158112792389206, + "tokens_seen": 310575104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320200601805416, + "loss": 1.6455, + "theoretical_loss": 4.158011136926698, + "tokens_seen": 310640640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320040120361084, + "loss": 1.5333, + "theoretical_loss": 4.157909508911747, + "tokens_seen": 310706176 + }, + { + "epoch": 0.09, + "learning_rate": 0.000731987963891675, + "loss": 1.663, + "theoretical_loss": 4.157807908331157, + "tokens_seen": 310771712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319719157472418, + "loss": 1.6373, + "theoretical_loss": 4.157706335171738, + "tokens_seen": 310837248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319558676028085, + "loss": 1.6038, + "theoretical_loss": 4.15760478942031, + "tokens_seen": 310902784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319398194583752, + "loss": 1.6834, + "theoretical_loss": 4.157503271063703, + "tokens_seen": 310968320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319237713139419, + "loss": 1.529, + "theoretical_loss": 4.157401780088756, + "tokens_seen": 311033856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319077231695086, + "loss": 1.671, + "theoretical_loss": 4.157300316482315, + "tokens_seen": 311099392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318916750250753, + "loss": 1.6511, + "theoretical_loss": 4.157198880231238, + "tokens_seen": 311164928 + }, + { + "epoch": 0.09, + "learning_rate": 0.000731875626880642, + "loss": 1.5657, + "theoretical_loss": 4.157097471322389, + "tokens_seen": 311230464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 110010, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5765893459320068, + "objective/train/theoretical_loss": 4.156996089742645, + "objective/train/tokens_used": 331756000, + "theoretical_loss": 4.156996089742645, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318595787362087, + "loss": 1.56, + "theoretical_loss": 4.156996089742645, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318435305917754, + "loss": 1.7442, + "theoretical_loss": 4.156894735478888, + "tokens_seen": 311361536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318274824473421, + "loss": 1.5361, + "theoretical_loss": 4.156793408518011, + "tokens_seen": 311427072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318114343029088, + "loss": 1.6328, + "theoretical_loss": 4.156692108846916, + "tokens_seen": 311492608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317953861584755, + "loss": 1.6673, + "theoretical_loss": 4.156590836452514, + "tokens_seen": 311558144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317793380140422, + "loss": 1.7033, + "theoretical_loss": 4.156489591321726, + "tokens_seen": 311623680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317632898696089, + "loss": 1.4632, + "theoretical_loss": 4.15638837344148, + "tokens_seen": 311689216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317472417251756, + "loss": 1.6694, + "theoretical_loss": 4.156287182798714, + "tokens_seen": 311754752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317311935807423, + "loss": 1.552, + "theoretical_loss": 4.156186019380375, + "tokens_seen": 311820288 + }, + { + "epoch": 0.09, + "learning_rate": 0.000731715145436309, + "loss": 1.6538, + "theoretical_loss": 4.156084883173419, + "tokens_seen": 311885824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316990972918757, + "loss": 1.5271, + "theoretical_loss": 4.155983774164811, + "tokens_seen": 311951360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316830491474424, + "loss": 1.4608, + "theoretical_loss": 4.1558826923415255, + "tokens_seen": 312016896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316670010030091, + "loss": 1.6252, + "theoretical_loss": 4.155781637690545, + "tokens_seen": 312082432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316509528585758, + "loss": 1.5328, + "theoretical_loss": 4.1556806101988615, + "tokens_seen": 312147968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316349047141425, + "loss": 1.5291, + "theoretical_loss": 4.155579609853476, + "tokens_seen": 312213504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316188565697092, + "loss": 1.5796, + "theoretical_loss": 4.155478636641398, + "tokens_seen": 312279040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316028084252759, + "loss": 1.7283, + "theoretical_loss": 4.155377690549646, + "tokens_seen": 312344576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315867602808425, + "loss": 1.5384, + "theoretical_loss": 4.155276771565248, + "tokens_seen": 312410112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315707121364093, + "loss": 1.6876, + "theoretical_loss": 4.155175879675241, + "tokens_seen": 312475648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315546639919759, + "loss": 1.5711, + "theoretical_loss": 4.155075014866671, + "tokens_seen": 312541184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315386158475427, + "loss": 1.6038, + "theoretical_loss": 4.15497417712659, + "tokens_seen": 312606720 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315225677031093, + "loss": 1.6181, + "theoretical_loss": 4.1548733664420645, + "tokens_seen": 312672256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315065195586761, + "loss": 1.6023, + "theoretical_loss": 4.154772582800165, + "tokens_seen": 312737792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314904714142427, + "loss": 1.6421, + "theoretical_loss": 4.154671826187972, + "tokens_seen": 312803328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314744232698095, + "loss": 1.6457, + "theoretical_loss": 4.154571096592576, + "tokens_seen": 312868864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314583751253761, + "loss": 1.6198, + "theoretical_loss": 4.154470394001077, + "tokens_seen": 312934400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314423269809429, + "loss": 1.5943, + "theoretical_loss": 4.154369718400583, + "tokens_seen": 312999936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314262788365095, + "loss": 1.5763, + "theoretical_loss": 4.154269069778207, + "tokens_seen": 313065472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314102306920763, + "loss": 1.6197, + "theoretical_loss": 4.154168448121078, + "tokens_seen": 313131008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313941825476429, + "loss": 1.6308, + "theoretical_loss": 4.15406785341633, + "tokens_seen": 313196544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313781344032097, + "loss": 1.488, + "theoretical_loss": 4.153967285651105, + "tokens_seen": 313262080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313620862587763, + "loss": 1.7241, + "theoretical_loss": 4.153866744812555, + "tokens_seen": 313327616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313460381143431, + "loss": 1.488, + "theoretical_loss": 4.1537662308878405, + "tokens_seen": 313393152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313299899699097, + "loss": 1.685, + "theoretical_loss": 4.153665743864131, + "tokens_seen": 313458688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007313139418254765, + "loss": 1.5907, + "theoretical_loss": 4.153565283728606, + "tokens_seen": 313524224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312978936810431, + "loss": 1.6424, + "theoretical_loss": 4.153464850468452, + "tokens_seen": 313589760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312818455366099, + "loss": 1.6169, + "theoretical_loss": 4.1533644440708635, + "tokens_seen": 313655296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312657973921765, + "loss": 1.6373, + "theoretical_loss": 4.153264064523047, + "tokens_seen": 313720832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312497492477433, + "loss": 1.4888, + "theoretical_loss": 4.1531637118122156, + "tokens_seen": 313786368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312337011033099, + "loss": 1.6178, + "theoretical_loss": 4.153063385925591, + "tokens_seen": 313851904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312176529588767, + "loss": 1.6062, + "theoretical_loss": 4.152963086850405, + "tokens_seen": 313917440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312016048144433, + "loss": 1.7214, + "theoretical_loss": 4.152862814573895, + "tokens_seen": 313982976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311855566700101, + "loss": 1.5936, + "theoretical_loss": 4.152762569083313, + "tokens_seen": 314048512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311695085255767, + "loss": 1.6886, + "theoretical_loss": 4.1526623503659135, + "tokens_seen": 314114048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311534603811435, + "loss": 1.543, + "theoretical_loss": 4.152562158408962, + "tokens_seen": 314179584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311374122367101, + "loss": 1.5865, + "theoretical_loss": 4.1524619931997355, + "tokens_seen": 314245120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311213640922769, + "loss": 1.5747, + "theoretical_loss": 4.152361854725515, + "tokens_seen": 314310656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311053159478435, + "loss": 1.6538, + "theoretical_loss": 4.152261742973594, + "tokens_seen": 314376192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310892678034103, + "loss": 1.6593, + "theoretical_loss": 4.152161657931273, + "tokens_seen": 314441728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310732196589769, + "loss": 1.6103, + "theoretical_loss": 4.15206159958586, + "tokens_seen": 314507264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 111213, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5043652057647705, + "objective/train/theoretical_loss": 4.1519615679246735, + "objective/train/tokens_used": 335032800, + "theoretical_loss": 4.1519615679246735, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310571715145437, + "loss": 1.5245, + "theoretical_loss": 4.1519615679246735, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310411233701103, + "loss": 1.5779, + "theoretical_loss": 4.151861562935041, + "tokens_seen": 314638336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310250752256771, + "loss": 1.5589, + "theoretical_loss": 4.151761584604298, + "tokens_seen": 314703872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310090270812437, + "loss": 1.5793, + "theoretical_loss": 4.1516616329197875, + "tokens_seen": 314769408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309929789368105, + "loss": 1.6619, + "theoretical_loss": 4.151561707868863, + "tokens_seen": 314834944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309769307923771, + "loss": 1.6171, + "theoretical_loss": 4.151461809438885, + "tokens_seen": 314900480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309608826479439, + "loss": 1.5683, + "theoretical_loss": 4.1513619376172235, + "tokens_seen": 314966016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309448345035105, + "loss": 1.6547, + "theoretical_loss": 4.151262092391257, + "tokens_seen": 315031552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309287863590773, + "loss": 1.6485, + "theoretical_loss": 4.1511622737483735, + "tokens_seen": 315097088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309127382146439, + "loss": 1.5209, + "theoretical_loss": 4.151062481675968, + "tokens_seen": 315162624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308966900702107, + "loss": 1.5688, + "theoretical_loss": 4.150962716161445, + "tokens_seen": 315228160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308806419257773, + "loss": 1.5452, + "theoretical_loss": 4.150862977192217, + "tokens_seen": 315293696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308645937813441, + "loss": 1.555, + "theoretical_loss": 4.150763264755706, + "tokens_seen": 315359232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308485456369107, + "loss": 1.7569, + "theoretical_loss": 4.150663578839342, + "tokens_seen": 315424768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308324974924775, + "loss": 1.598, + "theoretical_loss": 4.150563919430562, + "tokens_seen": 315490304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308164493480441, + "loss": 1.4641, + "theoretical_loss": 4.150464286516816, + "tokens_seen": 315555840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308004012036109, + "loss": 1.5865, + "theoretical_loss": 4.150364680085558, + "tokens_seen": 315621376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307843530591775, + "loss": 1.5584, + "theoretical_loss": 4.150265100124253, + "tokens_seen": 315686912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307683049147443, + "loss": 1.6787, + "theoretical_loss": 4.150165546620373, + "tokens_seen": 315752448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307522567703109, + "loss": 1.7157, + "theoretical_loss": 4.1500660195614, + "tokens_seen": 315817984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307362086258777, + "loss": 1.5774, + "theoretical_loss": 4.149966518934824, + "tokens_seen": 315883520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307201604814443, + "loss": 1.5534, + "theoretical_loss": 4.149867044728143, + "tokens_seen": 315949056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307041123370111, + "loss": 1.597, + "theoretical_loss": 4.149767596928863, + "tokens_seen": 316014592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306880641925777, + "loss": 1.8193, + "theoretical_loss": 4.149668175524502, + "tokens_seen": 316080128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306720160481445, + "loss": 1.5885, + "theoretical_loss": 4.14956878050258, + "tokens_seen": 316145664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306559679037111, + "loss": 1.6594, + "theoretical_loss": 4.1494694118506334, + "tokens_seen": 316211200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306399197592779, + "loss": 1.6892, + "theoretical_loss": 4.1493700695562, + "tokens_seen": 316276736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306238716148446, + "loss": 1.5594, + "theoretical_loss": 4.14927075360683, + "tokens_seen": 316342272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306078234704113, + "loss": 1.6457, + "theoretical_loss": 4.149171463990082, + "tokens_seen": 316407808 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730591775325978, + "loss": 1.5986, + "theoretical_loss": 4.149072200693521, + "tokens_seen": 316473344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305757271815447, + "loss": 1.6219, + "theoretical_loss": 4.148972963704722, + "tokens_seen": 316538880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305596790371114, + "loss": 1.6861, + "theoretical_loss": 4.148873753011269, + "tokens_seen": 316604416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305436308926781, + "loss": 1.6092, + "theoretical_loss": 4.148774568600752, + "tokens_seen": 316669952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305275827482448, + "loss": 1.5524, + "theoretical_loss": 4.148675410460772, + "tokens_seen": 316735488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305115346038115, + "loss": 1.5586, + "theoretical_loss": 4.148576278578937, + "tokens_seen": 316801024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304954864593782, + "loss": 1.5904, + "theoretical_loss": 4.148477172942863, + "tokens_seen": 316866560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304794383149449, + "loss": 1.745, + "theoretical_loss": 4.1483780935401775, + "tokens_seen": 316932096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304633901705116, + "loss": 1.5839, + "theoretical_loss": 4.148279040358511, + "tokens_seen": 316997632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304473420260783, + "loss": 1.5903, + "theoretical_loss": 4.148180013385507, + "tokens_seen": 317063168 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730431293881645, + "loss": 1.7813, + "theoretical_loss": 4.148081012608817, + "tokens_seen": 317128704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304152457372117, + "loss": 1.8107, + "theoretical_loss": 4.147982038016096, + "tokens_seen": 317194240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303991975927784, + "loss": 1.7651, + "theoretical_loss": 4.147883089595015, + "tokens_seen": 317259776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303831494483451, + "loss": 1.8327, + "theoretical_loss": 4.147784167333247, + "tokens_seen": 317325312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303671013039118, + "loss": 1.8399, + "theoretical_loss": 4.147685271218477, + "tokens_seen": 317390848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303510531594785, + "loss": 1.8931, + "theoretical_loss": 4.147586401238396, + "tokens_seen": 317456384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303350050150452, + "loss": 1.8639, + "theoretical_loss": 4.147487557380705, + "tokens_seen": 317521920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303189568706119, + "loss": 1.8167, + "theoretical_loss": 4.147388739633113, + "tokens_seen": 317587456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303029087261786, + "loss": 1.86, + "theoretical_loss": 4.147289947983337, + "tokens_seen": 317652992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302868605817453, + "loss": 1.9685, + "theoretical_loss": 4.1471911824191015, + "tokens_seen": 317718528 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730270812437312, + "loss": 1.7142, + "theoretical_loss": 4.147092442928141, + "tokens_seen": 317784064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 112065, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.024470567703247, + "objective/train/theoretical_loss": 4.146993729498197, + "objective/train/tokens_used": 338309600, + "theoretical_loss": 4.146993729498197, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302547642928787, + "loss": 1.9022, + "theoretical_loss": 4.146993729498197, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302387161484454, + "loss": 1.9061, + "theoretical_loss": 4.146895042117021, + "tokens_seen": 317915136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302226680040121, + "loss": 1.9165, + "theoretical_loss": 4.146796380772369, + "tokens_seen": 317980672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302066198595788, + "loss": 1.9143, + "theoretical_loss": 4.146697745452011, + "tokens_seen": 318046208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301905717151454, + "loss": 1.8095, + "theoretical_loss": 4.146599136143719, + "tokens_seen": 318111744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301745235707122, + "loss": 1.7185, + "theoretical_loss": 4.146500552835278, + "tokens_seen": 318177280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301584754262788, + "loss": 1.8265, + "theoretical_loss": 4.146401995514479, + "tokens_seen": 318242816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301424272818456, + "loss": 1.8853, + "theoretical_loss": 4.146303464169123, + "tokens_seen": 318308352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301263791374122, + "loss": 1.8499, + "theoretical_loss": 4.146204958787017, + "tokens_seen": 318373888 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730110330992979, + "loss": 1.6803, + "theoretical_loss": 4.146106479355978, + "tokens_seen": 318439424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300942828485456, + "loss": 1.8647, + "theoretical_loss": 4.1460080258638285, + "tokens_seen": 318504960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300782347041124, + "loss": 1.8665, + "theoretical_loss": 4.145909598298404, + "tokens_seen": 318570496 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730062186559679, + "loss": 1.8231, + "theoretical_loss": 4.145811196647544, + "tokens_seen": 318636032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300461384152458, + "loss": 1.8109, + "theoretical_loss": 4.145712820899098, + "tokens_seen": 318701568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300300902708124, + "loss": 1.7429, + "theoretical_loss": 4.145614471040923, + "tokens_seen": 318767104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300140421263792, + "loss": 1.8226, + "theoretical_loss": 4.145516147060884, + "tokens_seen": 318832640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299979939819458, + "loss": 1.701, + "theoretical_loss": 4.145417848946857, + "tokens_seen": 318898176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299819458375126, + "loss": 1.8668, + "theoretical_loss": 4.145319576686721, + "tokens_seen": 318963712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299658976930792, + "loss": 1.6904, + "theoretical_loss": 4.1452213302683685, + "tokens_seen": 319029248 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729949849548646, + "loss": 1.9191, + "theoretical_loss": 4.145123109679696, + "tokens_seen": 319094784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299338014042126, + "loss": 1.9909, + "theoretical_loss": 4.1450249149086105, + "tokens_seen": 319160320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299177532597794, + "loss": 1.9859, + "theoretical_loss": 4.144926745943026, + "tokens_seen": 319225856 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729901705115346, + "loss": 1.8796, + "theoretical_loss": 4.1448286027708665, + "tokens_seen": 319291392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298856569709128, + "loss": 1.7589, + "theoretical_loss": 4.144730485380062, + "tokens_seen": 319356928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298696088264794, + "loss": 1.881, + "theoretical_loss": 4.144632393758551, + "tokens_seen": 319422464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298535606820462, + "loss": 1.9404, + "theoretical_loss": 4.144534327894281, + "tokens_seen": 319488000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298375125376128, + "loss": 1.8255, + "theoretical_loss": 4.144436287775206, + "tokens_seen": 319553536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298214643931796, + "loss": 1.8244, + "theoretical_loss": 4.144338273389291, + "tokens_seen": 319619072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298054162487462, + "loss": 1.7481, + "theoretical_loss": 4.144240284724505, + "tokens_seen": 319684608 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729789368104313, + "loss": 1.8876, + "theoretical_loss": 4.144142321768831, + "tokens_seen": 319750144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007297733199598796, + "loss": 1.7792, + "theoretical_loss": 4.144044384510253, + "tokens_seen": 319815680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007297572718154464, + "loss": 1.7007, + "theoretical_loss": 4.1439464729367685, + "tokens_seen": 319881216 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729741223671013, + "loss": 1.5911, + "theoretical_loss": 4.14384858703638, + "tokens_seen": 319946752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007297251755265798, + "loss": 1.7342, + "theoretical_loss": 4.1437507267971, + "tokens_seen": 320012288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007297091273821464, + "loss": 1.7468, + "theoretical_loss": 4.143652892206948, + "tokens_seen": 320077824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296930792377132, + "loss": 1.7422, + "theoretical_loss": 4.1435550832539505, + "tokens_seen": 320143360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296770310932798, + "loss": 1.6767, + "theoretical_loss": 4.143457299926146, + "tokens_seen": 320208896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296609829488466, + "loss": 1.7012, + "theoretical_loss": 4.143359542211576, + "tokens_seen": 320274432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296449348044132, + "loss": 1.6575, + "theoretical_loss": 4.143261810098293, + "tokens_seen": 320339968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00072962888665998, + "loss": 1.8258, + "theoretical_loss": 4.1431641035743585, + "tokens_seen": 320405504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296128385155466, + "loss": 1.7058, + "theoretical_loss": 4.143066422627838, + "tokens_seen": 320471040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295967903711134, + "loss": 1.7022, + "theoretical_loss": 4.142968767246808, + "tokens_seen": 320536576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00072958074222668, + "loss": 1.658, + "theoretical_loss": 4.142871137419354, + "tokens_seen": 320602112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295646940822468, + "loss": 1.7651, + "theoretical_loss": 4.142773533133565, + "tokens_seen": 320667648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295486459378134, + "loss": 1.7998, + "theoretical_loss": 4.142675954377543, + "tokens_seen": 320733184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295325977933802, + "loss": 1.8644, + "theoretical_loss": 4.142578401139396, + "tokens_seen": 320798720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295165496489468, + "loss": 1.778, + "theoretical_loss": 4.142480873407239, + "tokens_seen": 320864256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295005015045136, + "loss": 1.7293, + "theoretical_loss": 4.1423833711691955, + "tokens_seen": 320929792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007294844533600802, + "loss": 1.6623, + "theoretical_loss": 4.142285894413398, + "tokens_seen": 320995328 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729468405215647, + "loss": 1.768, + "theoretical_loss": 4.142188443127985, + "tokens_seen": 321060864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 113306, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5380922555923462, + "objective/train/theoretical_loss": 4.142091017301105, + "objective/train/tokens_used": 341586400, + "theoretical_loss": 4.142091017301105, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007294523570712136, + "loss": 1.6093, + "theoretical_loss": 4.142091017301105, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007294363089267804, + "loss": 1.6369, + "theoretical_loss": 4.141993616920914, + "tokens_seen": 321191936 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729420260782347, + "loss": 1.71, + "theoretical_loss": 4.141896241975575, + "tokens_seen": 321257472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007294042126379138, + "loss": 1.4959, + "theoretical_loss": 4.141798892453259, + "tokens_seen": 321323008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293881644934804, + "loss": 1.7129, + "theoretical_loss": 4.141701568342145, + "tokens_seen": 321388544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293721163490472, + "loss": 1.6549, + "theoretical_loss": 4.141604269630422, + "tokens_seen": 321454080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293560682046138, + "loss": 1.5904, + "theoretical_loss": 4.1415069963062825, + "tokens_seen": 321519616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293400200601806, + "loss": 1.6822, + "theoretical_loss": 4.1414097483579315, + "tokens_seen": 321585152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293239719157472, + "loss": 1.6719, + "theoretical_loss": 4.1413125257735794, + "tokens_seen": 321650688 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729307923771314, + "loss": 1.5757, + "theoretical_loss": 4.141215328541445, + "tokens_seen": 321716224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292918756268806, + "loss": 1.5815, + "theoretical_loss": 4.141118156649753, + "tokens_seen": 321781760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292758274824474, + "loss": 1.6802, + "theoretical_loss": 4.1410210100867415, + "tokens_seen": 321847296 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729259779338014, + "loss": 1.6105, + "theoretical_loss": 4.14092388884065, + "tokens_seen": 321912832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292437311935808, + "loss": 1.5999, + "theoretical_loss": 4.1408267928997295, + "tokens_seen": 321978368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292276830491475, + "loss": 1.6539, + "theoretical_loss": 4.140729722252239, + "tokens_seen": 322043904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292116349047142, + "loss": 1.6241, + "theoretical_loss": 4.140632676886442, + "tokens_seen": 322109440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291955867602809, + "loss": 1.6331, + "theoretical_loss": 4.140535656790615, + "tokens_seen": 322174976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291795386158476, + "loss": 1.6988, + "theoretical_loss": 4.140438661953038, + "tokens_seen": 322240512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291634904714143, + "loss": 1.4855, + "theoretical_loss": 4.140341692362, + "tokens_seen": 322306048 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729147442326981, + "loss": 1.682, + "theoretical_loss": 4.140244748005799, + "tokens_seen": 322371584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291313941825477, + "loss": 1.7468, + "theoretical_loss": 4.14014782887274, + "tokens_seen": 322437120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291153460381144, + "loss": 1.6284, + "theoretical_loss": 4.140050934951135, + "tokens_seen": 322502656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290992978936812, + "loss": 1.6126, + "theoretical_loss": 4.139954066229304, + "tokens_seen": 322568192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290832497492478, + "loss": 1.6776, + "theoretical_loss": 4.139857222695578, + "tokens_seen": 322633728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290672016048146, + "loss": 1.6517, + "theoretical_loss": 4.139760404338291, + "tokens_seen": 322699264 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290511534603812, + "loss": 1.6009, + "theoretical_loss": 4.139663611145786, + "tokens_seen": 322764800 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729035105315948, + "loss": 1.6975, + "theoretical_loss": 4.139566843106417, + "tokens_seen": 322830336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290190571715146, + "loss": 1.6493, + "theoretical_loss": 4.139470100208542, + "tokens_seen": 322895872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290030090270814, + "loss": 1.7209, + "theoretical_loss": 4.139373382440528, + "tokens_seen": 322961408 + }, + { + "epoch": 0.1, + "learning_rate": 0.000728986960882648, + "loss": 1.6022, + "theoretical_loss": 4.1392766897907505, + "tokens_seen": 323026944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289709127382148, + "loss": 1.6735, + "theoretical_loss": 4.139180022247592, + "tokens_seen": 323092480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289548645937814, + "loss": 1.5358, + "theoretical_loss": 4.139083379799443, + "tokens_seen": 323158016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289388164493481, + "loss": 1.5424, + "theoretical_loss": 4.1389867624347, + "tokens_seen": 323223552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289227683049148, + "loss": 1.6485, + "theoretical_loss": 4.138890170141771, + "tokens_seen": 323289088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289067201604815, + "loss": 1.5949, + "theoretical_loss": 4.138793602909068, + "tokens_seen": 323354624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288906720160482, + "loss": 1.7056, + "theoretical_loss": 4.138697060725013, + "tokens_seen": 323420160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288746238716149, + "loss": 1.6545, + "theoretical_loss": 4.138600543578034, + "tokens_seen": 323485696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288585757271816, + "loss": 1.7005, + "theoretical_loss": 4.138504051456568, + "tokens_seen": 323551232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288425275827483, + "loss": 1.6943, + "theoretical_loss": 4.13840758434906, + "tokens_seen": 323616768 + }, + { + "epoch": 0.1, + "learning_rate": 0.000728826479438315, + "loss": 1.5391, + "theoretical_loss": 4.1383111422439605, + "tokens_seen": 323682304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288104312938817, + "loss": 1.6891, + "theoretical_loss": 4.13821472512973, + "tokens_seen": 323747840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287943831494483, + "loss": 1.6866, + "theoretical_loss": 4.138118332994837, + "tokens_seen": 323813376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287783350050151, + "loss": 1.69, + "theoretical_loss": 4.138021965827753, + "tokens_seen": 323878912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287622868605817, + "loss": 1.5937, + "theoretical_loss": 4.1379256236169635, + "tokens_seen": 323944448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287462387161485, + "loss": 1.6, + "theoretical_loss": 4.137829306350957, + "tokens_seen": 324009984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287301905717151, + "loss": 1.6746, + "theoretical_loss": 4.137733014018233, + "tokens_seen": 324075520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287141424272819, + "loss": 1.6548, + "theoretical_loss": 4.137636746607295, + "tokens_seen": 324141056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286980942828485, + "loss": 1.5518, + "theoretical_loss": 4.137540504106657, + "tokens_seen": 324206592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286820461384153, + "loss": 1.7232, + "theoretical_loss": 4.137444286504841, + "tokens_seen": 324272128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286659979939819, + "loss": 1.6377, + "theoretical_loss": 4.137348093790372, + "tokens_seen": 324337664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 114269, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6439099311828613, + "objective/train/theoretical_loss": 4.13725192595179, + "objective/train/tokens_used": 344863200, + "theoretical_loss": 4.13725192595179, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286499498495487, + "loss": 1.5966, + "theoretical_loss": 4.13725192595179, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286339017051153, + "loss": 1.7042, + "theoretical_loss": 4.1371557829776355, + "tokens_seen": 324468736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286178535606821, + "loss": 1.6878, + "theoretical_loss": 4.13705966485646, + "tokens_seen": 324534272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286018054162487, + "loss": 1.641, + "theoretical_loss": 4.136963571576825, + "tokens_seen": 324599808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285857572718155, + "loss": 1.6288, + "theoretical_loss": 4.136867503127292, + "tokens_seen": 324665344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285697091273821, + "loss": 1.5733, + "theoretical_loss": 4.136771459496439, + "tokens_seen": 324730880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285536609829489, + "loss": 1.5923, + "theoretical_loss": 4.136675440672844, + "tokens_seen": 324796416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285376128385155, + "loss": 1.7097, + "theoretical_loss": 4.136579446645098, + "tokens_seen": 324861952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285215646940823, + "loss": 1.6223, + "theoretical_loss": 4.136483477401798, + "tokens_seen": 324927488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285055165496489, + "loss": 1.6171, + "theoretical_loss": 4.136387532931546, + "tokens_seen": 324993024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284894684052157, + "loss": 1.555, + "theoretical_loss": 4.136291613222955, + "tokens_seen": 325058560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284734202607823, + "loss": 1.5637, + "theoretical_loss": 4.136195718264644, + "tokens_seen": 325124096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284573721163491, + "loss": 1.5865, + "theoretical_loss": 4.136099848045239, + "tokens_seen": 325189632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284413239719157, + "loss": 1.695, + "theoretical_loss": 4.136004002553375, + "tokens_seen": 325255168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284252758274825, + "loss": 1.5485, + "theoretical_loss": 4.135908181777693, + "tokens_seen": 325320704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284092276830491, + "loss": 1.6128, + "theoretical_loss": 4.135812385706842, + "tokens_seen": 325386240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283931795386159, + "loss": 1.7098, + "theoretical_loss": 4.135716614329479, + "tokens_seen": 325451776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283771313941825, + "loss": 1.7113, + "theoretical_loss": 4.135620867634269, + "tokens_seen": 325517312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283610832497493, + "loss": 1.6371, + "theoretical_loss": 4.135525145609881, + "tokens_seen": 325582848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283450351053159, + "loss": 1.7067, + "theoretical_loss": 4.135429448244997, + "tokens_seen": 325648384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283289869608827, + "loss": 1.609, + "theoretical_loss": 4.135333775528302, + "tokens_seen": 325713920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283129388164493, + "loss": 1.4926, + "theoretical_loss": 4.13523812744849, + "tokens_seen": 325779456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282968906720161, + "loss": 1.7475, + "theoretical_loss": 4.135142503994263, + "tokens_seen": 325844992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282808425275827, + "loss": 1.539, + "theoretical_loss": 4.13504690515433, + "tokens_seen": 325910528 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282647943831495, + "loss": 1.6961, + "theoretical_loss": 4.134951330917408, + "tokens_seen": 325976064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282487462387161, + "loss": 1.5638, + "theoretical_loss": 4.134855781272218, + "tokens_seen": 326041600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282326980942829, + "loss": 1.6864, + "theoretical_loss": 4.134760256207495, + "tokens_seen": 326107136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282166499498495, + "loss": 1.4862, + "theoretical_loss": 4.1346647557119764, + "tokens_seen": 326172672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282006018054163, + "loss": 1.6338, + "theoretical_loss": 4.134569279774407, + "tokens_seen": 326238208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281845536609829, + "loss": 1.6086, + "theoretical_loss": 4.134473828383541, + "tokens_seen": 326303744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281685055165497, + "loss": 1.4595, + "theoretical_loss": 4.13437840152814, + "tokens_seen": 326369280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281524573721163, + "loss": 1.6468, + "theoretical_loss": 4.134282999196973, + "tokens_seen": 326434816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281364092276831, + "loss": 1.6356, + "theoretical_loss": 4.134187621378813, + "tokens_seen": 326500352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281203610832497, + "loss": 1.546, + "theoretical_loss": 4.134092268062446, + "tokens_seen": 326565888 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281043129388165, + "loss": 1.4938, + "theoretical_loss": 4.133996939236661, + "tokens_seen": 326631424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280882647943831, + "loss": 1.6343, + "theoretical_loss": 4.133901634890256, + "tokens_seen": 326696960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280722166499499, + "loss": 1.5978, + "theoretical_loss": 4.1338063550120365, + "tokens_seen": 326762496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280561685055165, + "loss": 1.7451, + "theoretical_loss": 4.133711099590815, + "tokens_seen": 326828032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280401203610833, + "loss": 1.6231, + "theoretical_loss": 4.133615868615411, + "tokens_seen": 326893568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280240722166499, + "loss": 1.6108, + "theoretical_loss": 4.133520662074654, + "tokens_seen": 326959104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280080240722167, + "loss": 1.6044, + "theoretical_loss": 4.133425479957375, + "tokens_seen": 327024640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279919759277833, + "loss": 1.6349, + "theoretical_loss": 4.133330322252419, + "tokens_seen": 327090176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279759277833501, + "loss": 1.6286, + "theoretical_loss": 4.1332351889486345, + "tokens_seen": 327155712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279598796389167, + "loss": 1.4971, + "theoretical_loss": 4.133140080034878, + "tokens_seen": 327221248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279438314944835, + "loss": 1.5259, + "theoretical_loss": 4.133044995500013, + "tokens_seen": 327286784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279277833500503, + "loss": 1.6473, + "theoretical_loss": 4.1329499353329116, + "tokens_seen": 327352320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279117352056169, + "loss": 1.5083, + "theoretical_loss": 4.132854899522453, + "tokens_seen": 327417856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278956870611837, + "loss": 1.7212, + "theoretical_loss": 4.132759888057521, + "tokens_seen": 327483392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278796389167503, + "loss": 1.6026, + "theoretical_loss": 4.13266490092701, + "tokens_seen": 327548928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278635907723171, + "loss": 1.5403, + "theoretical_loss": 4.132569938119821, + "tokens_seen": 327614464 + }, + { + "debugging/Compilability": 0.0, + "debugging/distinct-1-grams": 0.7371483758339369, + "debugging/entropy-1-grams": 5.361149495192262, + "debugging/length": 581.1666666666666, + "debugging/num_segments": 12, + "debugging/score": 0.06659683474910605, + "debugging/score_std": 0.015715120837281396, + "epoch": 0.1, + "objective/train/docs_used": 115553, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6277862787246704, + "objective/train/theoretical_loss": 4.132474999624861, + "objective/train/tokens_used": 348140000, + "theoretical_loss": 4.132474999624861, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278475426278837, + "loss": 1.6737, + "theoretical_loss": 4.132474999624861, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278314944834505, + "loss": 1.5806, + "theoretical_loss": 4.132380085431046, + "tokens_seen": 327745536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278154463390171, + "loss": 1.641, + "theoretical_loss": 4.132285195527297, + "tokens_seen": 327811072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277993981945839, + "loss": 1.6191, + "theoretical_loss": 4.132190329902544, + "tokens_seen": 327876608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277833500501505, + "loss": 1.5115, + "theoretical_loss": 4.132095488545724, + "tokens_seen": 327942144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277673019057173, + "loss": 1.6416, + "theoretical_loss": 4.132000671445781, + "tokens_seen": 328007680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277512537612839, + "loss": 1.5818, + "theoretical_loss": 4.131905878591667, + "tokens_seen": 328073216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277352056168507, + "loss": 1.5892, + "theoretical_loss": 4.13181110997234, + "tokens_seen": 328138752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277191574724173, + "loss": 1.6217, + "theoretical_loss": 4.131716365576765, + "tokens_seen": 328204288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277031093279841, + "loss": 1.4981, + "theoretical_loss": 4.131621645393917, + "tokens_seen": 328269824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276870611835507, + "loss": 1.5857, + "theoretical_loss": 4.131526949412774, + "tokens_seen": 328335360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276710130391175, + "loss": 1.53, + "theoretical_loss": 4.131432277622325, + "tokens_seen": 328400896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276549648946841, + "loss": 1.5972, + "theoretical_loss": 4.131337630011564, + "tokens_seen": 328466432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276389167502509, + "loss": 1.5262, + "theoretical_loss": 4.131243006569493, + "tokens_seen": 328531968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276228686058175, + "loss": 1.5769, + "theoretical_loss": 4.131148407285121, + "tokens_seen": 328597504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276068204613843, + "loss": 1.6371, + "theoretical_loss": 4.131053832147465, + "tokens_seen": 328663040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275907723169509, + "loss": 1.6341, + "theoretical_loss": 4.130959281145547, + "tokens_seen": 328728576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275747241725177, + "loss": 1.6886, + "theoretical_loss": 4.130864754268399, + "tokens_seen": 328794112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275586760280843, + "loss": 1.5881, + "theoretical_loss": 4.130770251505058, + "tokens_seen": 328859648 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727542627883651, + "loss": 1.5361, + "theoretical_loss": 4.130675772844569, + "tokens_seen": 328925184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275265797392177, + "loss": 1.6476, + "theoretical_loss": 4.130581318275984, + "tokens_seen": 328990720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275105315947844, + "loss": 1.6211, + "theoretical_loss": 4.130486887788362, + "tokens_seen": 329056256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274944834503511, + "loss": 1.5303, + "theoretical_loss": 4.130392481370771, + "tokens_seen": 329121792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274784353059178, + "loss": 1.5864, + "theoretical_loss": 4.130298099012283, + "tokens_seen": 329187328 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274623871614845, + "loss": 1.5658, + "theoretical_loss": 4.130203740701979, + "tokens_seen": 329252864 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274463390170512, + "loss": 1.6121, + "theoretical_loss": 4.1301094064289465, + "tokens_seen": 329318400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274302908726179, + "loss": 1.7239, + "theoretical_loss": 4.130015096182282, + "tokens_seen": 329383936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274142427281846, + "loss": 1.5618, + "theoretical_loss": 4.129920809951085, + "tokens_seen": 329449472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273981945837512, + "loss": 1.594, + "theoretical_loss": 4.129826547724467, + "tokens_seen": 329515008 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727382146439318, + "loss": 1.6085, + "theoretical_loss": 4.129732309491542, + "tokens_seen": 329580544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273660982948846, + "loss": 1.7381, + "theoretical_loss": 4.129638095241436, + "tokens_seen": 329646080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273500501504514, + "loss": 1.608, + "theoretical_loss": 4.129543904963278, + "tokens_seen": 329711616 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727334002006018, + "loss": 1.6218, + "theoretical_loss": 4.129449738646205, + "tokens_seen": 329777152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273179538615848, + "loss": 1.5262, + "theoretical_loss": 4.129355596279362, + "tokens_seen": 329842688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273019057171514, + "loss": 1.585, + "theoretical_loss": 4.129261477851902, + "tokens_seen": 329908224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272858575727182, + "loss": 1.517, + "theoretical_loss": 4.129167383352982, + "tokens_seen": 329973760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272698094282848, + "loss": 1.5909, + "theoretical_loss": 4.129073312771768, + "tokens_seen": 330039296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272537612838516, + "loss": 1.6328, + "theoretical_loss": 4.128979266097433, + "tokens_seen": 330104832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272377131394182, + "loss": 1.5654, + "theoretical_loss": 4.128885243319158, + "tokens_seen": 330170368 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727221664994985, + "loss": 1.6423, + "theoretical_loss": 4.12879124442613, + "tokens_seen": 330235904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272056168505516, + "loss": 1.5612, + "theoretical_loss": 4.128697269407541, + "tokens_seen": 330301440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271895687061184, + "loss": 1.6453, + "theoretical_loss": 4.128603318252593, + "tokens_seen": 330366976 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727173520561685, + "loss": 1.6937, + "theoretical_loss": 4.128509390950494, + "tokens_seen": 330432512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271574724172518, + "loss": 1.6754, + "theoretical_loss": 4.128415487490461, + "tokens_seen": 330498048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271414242728184, + "loss": 1.5799, + "theoretical_loss": 4.128321607861713, + "tokens_seen": 330563584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271253761283852, + "loss": 1.6589, + "theoretical_loss": 4.128227752053482, + "tokens_seen": 330629120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271093279839518, + "loss": 1.5626, + "theoretical_loss": 4.128133920055001, + "tokens_seen": 330694656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007270932798395186, + "loss": 1.5973, + "theoretical_loss": 4.1280401118555154, + "tokens_seen": 330760192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007270772316950852, + "loss": 1.5759, + "theoretical_loss": 4.127946327444274, + "tokens_seen": 330825728 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727061183550652, + "loss": 1.4935, + "theoretical_loss": 4.127852566810535, + "tokens_seen": 330891264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 116426, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7715214490890503, + "objective/train/theoretical_loss": 4.127758829943562, + "objective/train/tokens_used": 351416800, + "theoretical_loss": 4.127758829943562, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007270451354062186, + "loss": 1.6378, + "theoretical_loss": 4.127758829943562, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007270290872617854, + "loss": 1.6115, + "theoretical_loss": 4.127665116832626, + "tokens_seen": 331022336 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727013039117352, + "loss": 1.4952, + "theoretical_loss": 4.127571427467005, + "tokens_seen": 331087872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269969909729188, + "loss": 1.5364, + "theoretical_loss": 4.1274777618359835, + "tokens_seen": 331153408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269809428284854, + "loss": 1.5265, + "theoretical_loss": 4.127384119928854, + "tokens_seen": 331218944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269648946840522, + "loss": 1.5606, + "theoretical_loss": 4.127290501734914, + "tokens_seen": 331284480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269488465396188, + "loss": 1.6189, + "theoretical_loss": 4.127196907243472, + "tokens_seen": 331350016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269327983951856, + "loss": 1.6621, + "theoretical_loss": 4.127103336443839, + "tokens_seen": 331415552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269167502507522, + "loss": 1.6471, + "theoretical_loss": 4.127009789325334, + "tokens_seen": 331481088 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726900702106319, + "loss": 1.5859, + "theoretical_loss": 4.126916265877284, + "tokens_seen": 331546624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268846539618856, + "loss": 1.5848, + "theoretical_loss": 4.126822766089024, + "tokens_seen": 331612160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268686058174524, + "loss": 1.7124, + "theoretical_loss": 4.126729289949893, + "tokens_seen": 331677696 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726852557673019, + "loss": 1.582, + "theoretical_loss": 4.126635837449238, + "tokens_seen": 331743232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268365095285858, + "loss": 1.5718, + "theoretical_loss": 4.126542408576414, + "tokens_seen": 331808768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268204613841524, + "loss": 1.5834, + "theoretical_loss": 4.126449003320783, + "tokens_seen": 331874304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268044132397192, + "loss": 1.5014, + "theoretical_loss": 4.126355621671711, + "tokens_seen": 331939840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267883650952858, + "loss": 1.4939, + "theoretical_loss": 4.126262263618574, + "tokens_seen": 332005376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267723169508526, + "loss": 1.5887, + "theoretical_loss": 4.1261689291507535, + "tokens_seen": 332070912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267562688064192, + "loss": 1.6777, + "theoretical_loss": 4.126075618257639, + "tokens_seen": 332136448 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726740220661986, + "loss": 1.6688, + "theoretical_loss": 4.125982330928624, + "tokens_seen": 332201984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267241725175528, + "loss": 1.6503, + "theoretical_loss": 4.125889067153113, + "tokens_seen": 332267520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267081243731194, + "loss": 1.5699, + "theoretical_loss": 4.125795826920514, + "tokens_seen": 332333056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266920762286862, + "loss": 1.5449, + "theoretical_loss": 4.125702610220244, + "tokens_seen": 332398592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266760280842528, + "loss": 1.6834, + "theoretical_loss": 4.1256094170417255, + "tokens_seen": 332464128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266599799398196, + "loss": 1.6401, + "theoretical_loss": 4.125516247374387, + "tokens_seen": 332529664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266439317953862, + "loss": 1.7152, + "theoretical_loss": 4.125423101207668, + "tokens_seen": 332595200 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726627883650953, + "loss": 1.4565, + "theoretical_loss": 4.125329978531009, + "tokens_seen": 332660736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266118355065196, + "loss": 1.6387, + "theoretical_loss": 4.125236879333861, + "tokens_seen": 332726272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265957873620864, + "loss": 1.5454, + "theoretical_loss": 4.125143803605683, + "tokens_seen": 332791808 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726579739217653, + "loss": 1.6302, + "theoretical_loss": 4.125050751335936, + "tokens_seen": 332857344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265636910732198, + "loss": 1.6075, + "theoretical_loss": 4.124957722514091, + "tokens_seen": 332922880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265476429287864, + "loss": 1.6286, + "theoretical_loss": 4.124864717129629, + "tokens_seen": 332988416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265315947843532, + "loss": 1.4963, + "theoretical_loss": 4.124771735172031, + "tokens_seen": 333053952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265155466399198, + "loss": 1.559, + "theoretical_loss": 4.1246787766307875, + "tokens_seen": 333119488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264994984954866, + "loss": 1.6646, + "theoretical_loss": 4.124585841495398, + "tokens_seen": 333185024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264834503510532, + "loss": 1.4918, + "theoretical_loss": 4.124492929755367, + "tokens_seen": 333250560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00072646740220662, + "loss": 1.5617, + "theoretical_loss": 4.1244000414002056, + "tokens_seen": 333316096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264513540621866, + "loss": 1.6774, + "theoretical_loss": 4.124307176419432, + "tokens_seen": 333381632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264353059177534, + "loss": 1.5307, + "theoretical_loss": 4.12421433480257, + "tokens_seen": 333447168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00072641925777332, + "loss": 1.5039, + "theoretical_loss": 4.124121516539153, + "tokens_seen": 333512704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264032096288868, + "loss": 1.6459, + "theoretical_loss": 4.124028721618719, + "tokens_seen": 333578240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263871614844534, + "loss": 1.5771, + "theoretical_loss": 4.123935950030812, + "tokens_seen": 333643776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263711133400202, + "loss": 1.5749, + "theoretical_loss": 4.123843201764984, + "tokens_seen": 333709312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263550651955868, + "loss": 1.5776, + "theoretical_loss": 4.1237504768107955, + "tokens_seen": 333774848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263390170511536, + "loss": 1.6285, + "theoretical_loss": 4.12365777515781, + "tokens_seen": 333840384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263229689067202, + "loss": 1.739, + "theoretical_loss": 4.1235650967956, + "tokens_seen": 333905920 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726306920762287, + "loss": 1.3803, + "theoretical_loss": 4.123472441713743, + "tokens_seen": 333971456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262908726178536, + "loss": 1.6053, + "theoretical_loss": 4.123379809901827, + "tokens_seen": 334036992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262748244734204, + "loss": 1.5184, + "theoretical_loss": 4.123287201349443, + "tokens_seen": 334102528 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726258776328987, + "loss": 1.4989, + "theoretical_loss": 4.123194616046189, + "tokens_seen": 334168064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 117616, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7517592906951904, + "objective/train/theoretical_loss": 4.123102053981672, + "objective/train/tokens_used": 354693600, + "theoretical_loss": 4.123102053981672, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262427281845538, + "loss": 1.6572, + "theoretical_loss": 4.123102053981672, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262266800401204, + "loss": 1.5332, + "theoretical_loss": 4.123009515145503, + "tokens_seen": 334299136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262106318956872, + "loss": 1.5541, + "theoretical_loss": 4.122916999527301, + "tokens_seen": 334364672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261945837512538, + "loss": 1.6208, + "theoretical_loss": 4.122824507116691, + "tokens_seen": 334430208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261785356068206, + "loss": 1.5833, + "theoretical_loss": 4.122732037903308, + "tokens_seen": 334495744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261624874623872, + "loss": 1.6827, + "theoretical_loss": 4.122639591876789, + "tokens_seen": 334561280 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726146439317954, + "loss": 1.5785, + "theoretical_loss": 4.122547169026778, + "tokens_seen": 334626816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261303911735206, + "loss": 1.6094, + "theoretical_loss": 4.12245476934293, + "tokens_seen": 334692352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261143430290873, + "loss": 1.5737, + "theoretical_loss": 4.122362392814903, + "tokens_seen": 334757888 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726098294884654, + "loss": 1.5096, + "theoretical_loss": 4.122270039432362, + "tokens_seen": 334823424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260822467402207, + "loss": 1.5614, + "theoretical_loss": 4.122177709184978, + "tokens_seen": 334888960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260661985957874, + "loss": 1.6424, + "theoretical_loss": 4.1220854020624325, + "tokens_seen": 334954496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260501504513541, + "loss": 1.6124, + "theoretical_loss": 4.121993118054409, + "tokens_seen": 335020032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260341023069208, + "loss": 1.5596, + "theoretical_loss": 4.1219008571505995, + "tokens_seen": 335085568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260180541624875, + "loss": 1.6681, + "theoretical_loss": 4.121808619340704, + "tokens_seen": 335151104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260020060180541, + "loss": 1.5883, + "theoretical_loss": 4.121716404614426, + "tokens_seen": 335216640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259859578736209, + "loss": 1.4965, + "theoretical_loss": 4.121624212961478, + "tokens_seen": 335282176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259699097291875, + "loss": 1.6037, + "theoretical_loss": 4.121532044371579, + "tokens_seen": 335347712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259538615847543, + "loss": 1.7225, + "theoretical_loss": 4.121439898834454, + "tokens_seen": 335413248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259378134403209, + "loss": 1.69, + "theoretical_loss": 4.1213477763398325, + "tokens_seen": 335478784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259217652958877, + "loss": 1.5917, + "theoretical_loss": 4.121255676877455, + "tokens_seen": 335544320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259057171514543, + "loss": 1.7592, + "theoretical_loss": 4.1211636004370655, + "tokens_seen": 335609856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258896690070211, + "loss": 1.722, + "theoretical_loss": 4.121071547008415, + "tokens_seen": 335675392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258736208625877, + "loss": 1.5877, + "theoretical_loss": 4.120979516581262, + "tokens_seen": 335740928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258575727181545, + "loss": 1.5069, + "theoretical_loss": 4.120887509145371, + "tokens_seen": 335806464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258415245737211, + "loss": 1.5241, + "theoretical_loss": 4.120795524690512, + "tokens_seen": 335872000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258254764292879, + "loss": 1.5952, + "theoretical_loss": 4.1207035632064635, + "tokens_seen": 335937536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258094282848545, + "loss": 1.4847, + "theoretical_loss": 4.120611624683008, + "tokens_seen": 336003072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257933801404213, + "loss": 1.7648, + "theoretical_loss": 4.120519709109938, + "tokens_seen": 336068608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257773319959879, + "loss": 1.5907, + "theoretical_loss": 4.12042781647705, + "tokens_seen": 336134144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257612838515547, + "loss": 1.4965, + "theoretical_loss": 4.120335946774147, + "tokens_seen": 336199680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257452357071213, + "loss": 1.6617, + "theoretical_loss": 4.12024409999104, + "tokens_seen": 336265216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257291875626881, + "loss": 1.5381, + "theoretical_loss": 4.120152276117545, + "tokens_seen": 336330752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257131394182547, + "loss": 1.5592, + "theoretical_loss": 4.120060475143486, + "tokens_seen": 336396288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256970912738215, + "loss": 1.5193, + "theoretical_loss": 4.119968697058692, + "tokens_seen": 336461824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256810431293881, + "loss": 1.5876, + "theoretical_loss": 4.119876941852999, + "tokens_seen": 336527360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256649949849549, + "loss": 1.652, + "theoretical_loss": 4.11978520951625, + "tokens_seen": 336592896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256489468405215, + "loss": 1.5175, + "theoretical_loss": 4.119693500038294, + "tokens_seen": 336658432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256328986960883, + "loss": 1.5785, + "theoretical_loss": 4.119601813408988, + "tokens_seen": 336723968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256168505516549, + "loss": 1.6262, + "theoretical_loss": 4.1195101496181925, + "tokens_seen": 336789504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256008024072217, + "loss": 1.5974, + "theoretical_loss": 4.119418508655777, + "tokens_seen": 336855040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255847542627883, + "loss": 1.5781, + "theoretical_loss": 4.119326890511615, + "tokens_seen": 336920576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255687061183551, + "loss": 1.4977, + "theoretical_loss": 4.1192352951755895, + "tokens_seen": 336986112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255526579739217, + "loss": 1.5431, + "theoretical_loss": 4.119143722637588, + "tokens_seen": 337051648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255366098294885, + "loss": 1.5899, + "theoretical_loss": 4.119052172887506, + "tokens_seen": 337117184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255205616850552, + "loss": 1.6135, + "theoretical_loss": 4.118960645915243, + "tokens_seen": 337182720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255045135406219, + "loss": 1.6049, + "theoretical_loss": 4.118869141710706, + "tokens_seen": 337248256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254884653961886, + "loss": 1.6426, + "theoretical_loss": 4.11877766026381, + "tokens_seen": 337313792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254724172517553, + "loss": 1.5858, + "theoretical_loss": 4.118686201564475, + "tokens_seen": 337379328 + }, + { + "epoch": 0.1, + "learning_rate": 0.000725456369107322, + "loss": 1.7204, + "theoretical_loss": 4.1185947656026265, + "tokens_seen": 337444864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 118525, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7721240520477295, + "objective/train/theoretical_loss": 4.118503352368197, + "objective/train/tokens_used": 357970400, + "theoretical_loss": 4.118503352368197, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254403209628887, + "loss": 1.6626, + "theoretical_loss": 4.118503352368197, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254242728184554, + "loss": 1.506, + "theoretical_loss": 4.118411961851129, + "tokens_seen": 337575936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254082246740221, + "loss": 1.5272, + "theoretical_loss": 4.118320594041366, + "tokens_seen": 337641472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253921765295888, + "loss": 1.5772, + "theoretical_loss": 4.118229248928859, + "tokens_seen": 337707008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253761283851555, + "loss": 1.5166, + "theoretical_loss": 4.11813792650357, + "tokens_seen": 337772544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253600802407223, + "loss": 1.6504, + "theoretical_loss": 4.118046626755461, + "tokens_seen": 337838080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253440320962889, + "loss": 1.543, + "theoretical_loss": 4.117955349674505, + "tokens_seen": 337903616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253279839518557, + "loss": 1.6382, + "theoretical_loss": 4.117864095250679, + "tokens_seen": 337969152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253119358074223, + "loss": 1.5598, + "theoretical_loss": 4.117772863473968, + "tokens_seen": 338034688 + }, + { + "epoch": 0.1, + "learning_rate": 0.000725295887662989, + "loss": 1.626, + "theoretical_loss": 4.117681654334361, + "tokens_seen": 338100224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252798395185557, + "loss": 1.5688, + "theoretical_loss": 4.117590467821857, + "tokens_seen": 338165760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252637913741225, + "loss": 1.5726, + "theoretical_loss": 4.117499303926457, + "tokens_seen": 338231296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252477432296891, + "loss": 1.5728, + "theoretical_loss": 4.1174081626381716, + "tokens_seen": 338296832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252316950852559, + "loss": 1.5736, + "theoretical_loss": 4.117317043947017, + "tokens_seen": 338362368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252156469408225, + "loss": 1.6962, + "theoretical_loss": 4.117225947843014, + "tokens_seen": 338427904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251995987963893, + "loss": 1.5178, + "theoretical_loss": 4.117134874316194, + "tokens_seen": 338493440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251835506519559, + "loss": 1.5646, + "theoretical_loss": 4.117043823356589, + "tokens_seen": 338558976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251675025075227, + "loss": 1.7531, + "theoretical_loss": 4.116952794954242, + "tokens_seen": 338624512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251514543630893, + "loss": 1.4881, + "theoretical_loss": 4.1168617890992, + "tokens_seen": 338690048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251354062186561, + "loss": 1.5268, + "theoretical_loss": 4.116770805781517, + "tokens_seen": 338755584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251193580742227, + "loss": 1.404, + "theoretical_loss": 4.116679844991253, + "tokens_seen": 338821120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251033099297895, + "loss": 1.6254, + "theoretical_loss": 4.116588906718475, + "tokens_seen": 338886656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250872617853561, + "loss": 1.5061, + "theoretical_loss": 4.116497990953255, + "tokens_seen": 338952192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250712136409229, + "loss": 1.4944, + "theoretical_loss": 4.1164070976856735, + "tokens_seen": 339017728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250551654964895, + "loss": 1.6102, + "theoretical_loss": 4.1163162269058144, + "tokens_seen": 339083264 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250391173520563, + "loss": 1.5782, + "theoretical_loss": 4.11622537860377, + "tokens_seen": 339148800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250230692076229, + "loss": 1.7109, + "theoretical_loss": 4.116134552769638, + "tokens_seen": 339214336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250070210631897, + "loss": 1.6032, + "theoretical_loss": 4.116043749393523, + "tokens_seen": 339279872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249909729187563, + "loss": 1.533, + "theoretical_loss": 4.115952968465535, + "tokens_seen": 339345408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249749247743231, + "loss": 1.4966, + "theoretical_loss": 4.115862209975791, + "tokens_seen": 339410944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249588766298897, + "loss": 1.5367, + "theoretical_loss": 4.115771473914415, + "tokens_seen": 339476480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249428284854565, + "loss": 1.6097, + "theoretical_loss": 4.115680760271535, + "tokens_seen": 339542016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249267803410231, + "loss": 1.67, + "theoretical_loss": 4.115590069037287, + "tokens_seen": 339607552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249107321965899, + "loss": 1.7401, + "theoretical_loss": 4.115499400201813, + "tokens_seen": 339673088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248946840521565, + "loss": 1.6455, + "theoretical_loss": 4.1154087537552595, + "tokens_seen": 339738624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248786359077233, + "loss": 1.6652, + "theoretical_loss": 4.1153181296877825, + "tokens_seen": 339804160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248625877632899, + "loss": 1.4987, + "theoretical_loss": 4.115227527989543, + "tokens_seen": 339869696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248465396188567, + "loss": 1.6819, + "theoretical_loss": 4.115136948650705, + "tokens_seen": 339935232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248304914744233, + "loss": 1.512, + "theoretical_loss": 4.115046391661443, + "tokens_seen": 340000768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248144433299901, + "loss": 1.5822, + "theoretical_loss": 4.114955857011937, + "tokens_seen": 340066304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247983951855567, + "loss": 1.6232, + "theoretical_loss": 4.114865344692371, + "tokens_seen": 340131840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247823470411235, + "loss": 1.5821, + "theoretical_loss": 4.1147748546929375, + "tokens_seen": 340197376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247662988966901, + "loss": 1.484, + "theoretical_loss": 4.114684387003833, + "tokens_seen": 340262912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247502507522569, + "loss": 1.6424, + "theoretical_loss": 4.114593941615262, + "tokens_seen": 340328448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247342026078235, + "loss": 1.4232, + "theoretical_loss": 4.114503518517435, + "tokens_seen": 340393984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247181544633902, + "loss": 1.6698, + "theoretical_loss": 4.114413117700568, + "tokens_seen": 340459520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247021063189569, + "loss": 1.6268, + "theoretical_loss": 4.114322739154883, + "tokens_seen": 340525056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246860581745236, + "loss": 1.7974, + "theoretical_loss": 4.114232382870609, + "tokens_seen": 340590592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246700100300903, + "loss": 1.6961, + "theoretical_loss": 4.114142048837982, + "tokens_seen": 340656128 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724653961885657, + "loss": 1.6857, + "theoretical_loss": 4.114051737047241, + "tokens_seen": 340721664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 119921, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6314363479614258, + "objective/train/theoretical_loss": 4.1139614474886335, + "objective/train/tokens_used": 361247200, + "theoretical_loss": 4.1139614474886335, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246379137412237, + "loss": 1.5892, + "theoretical_loss": 4.1139614474886335, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246218655967904, + "loss": 1.6219, + "theoretical_loss": 4.1138711801524135, + "tokens_seen": 340852736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246058174523571, + "loss": 1.5791, + "theoretical_loss": 4.11378093502884, + "tokens_seen": 340918272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245897693079238, + "loss": 1.498, + "theoretical_loss": 4.113690712108179, + "tokens_seen": 340983808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245737211634904, + "loss": 1.5305, + "theoretical_loss": 4.113600511380701, + "tokens_seen": 341049344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245576730190572, + "loss": 1.4932, + "theoretical_loss": 4.113510332836685, + "tokens_seen": 341114880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245416248746238, + "loss": 1.5771, + "theoretical_loss": 4.1134201764664144, + "tokens_seen": 341180416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245255767301906, + "loss": 1.6052, + "theoretical_loss": 4.113330042260181, + "tokens_seen": 341245952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245095285857572, + "loss": 1.4641, + "theoretical_loss": 4.113239930208278, + "tokens_seen": 341311488 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724493480441324, + "loss": 1.6998, + "theoretical_loss": 4.113149840301009, + "tokens_seen": 341377024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007244774322968906, + "loss": 1.6401, + "theoretical_loss": 4.113059772528683, + "tokens_seen": 341442560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007244613841524574, + "loss": 1.5859, + "theoretical_loss": 4.112969726881613, + "tokens_seen": 341508096 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724445336008024, + "loss": 1.6477, + "theoretical_loss": 4.112879703350122, + "tokens_seen": 341573632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007244292878635908, + "loss": 1.701, + "theoretical_loss": 4.112789701924535, + "tokens_seen": 341639168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007244132397191574, + "loss": 1.5066, + "theoretical_loss": 4.112699722595185, + "tokens_seen": 341704704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243971915747242, + "loss": 1.5769, + "theoretical_loss": 4.11260976535241, + "tokens_seen": 341770240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243811434302908, + "loss": 1.613, + "theoretical_loss": 4.112519830186557, + "tokens_seen": 341835776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243650952858576, + "loss": 1.5651, + "theoretical_loss": 4.112429917087974, + "tokens_seen": 341901312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243490471414242, + "loss": 1.6294, + "theoretical_loss": 4.112340026047022, + "tokens_seen": 341966848 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724332998996991, + "loss": 1.5895, + "theoretical_loss": 4.112250157054061, + "tokens_seen": 342032384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243169508525577, + "loss": 1.8001, + "theoretical_loss": 4.112160310099461, + "tokens_seen": 342097920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243009027081244, + "loss": 1.5778, + "theoretical_loss": 4.112070485173598, + "tokens_seen": 342163456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242848545636911, + "loss": 1.6557, + "theoretical_loss": 4.111980682266853, + "tokens_seen": 342228992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242688064192578, + "loss": 1.5965, + "theoretical_loss": 4.111890901369612, + "tokens_seen": 342294528 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242527582748245, + "loss": 1.6546, + "theoretical_loss": 4.11180114247227, + "tokens_seen": 342360064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242367101303912, + "loss": 1.6265, + "theoretical_loss": 4.1117114055652255, + "tokens_seen": 342425600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242206619859579, + "loss": 1.5941, + "theoretical_loss": 4.111621690638884, + "tokens_seen": 342491136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242046138415246, + "loss": 1.4372, + "theoretical_loss": 4.111531997683658, + "tokens_seen": 342556672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241885656970913, + "loss": 1.5377, + "theoretical_loss": 4.1114423266899625, + "tokens_seen": 342622208 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724172517552658, + "loss": 1.6013, + "theoretical_loss": 4.111352677648224, + "tokens_seen": 342687744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241564694082247, + "loss": 1.5588, + "theoretical_loss": 4.111263050548869, + "tokens_seen": 342753280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241404212637914, + "loss": 1.7092, + "theoretical_loss": 4.111173445382335, + "tokens_seen": 342818816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241243731193581, + "loss": 1.5877, + "theoretical_loss": 4.111083862139063, + "tokens_seen": 342884352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241083249749248, + "loss": 1.569, + "theoretical_loss": 4.110994300809501, + "tokens_seen": 342949888 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007240922768304915, + "loss": 1.5495, + "theoretical_loss": 4.1109047613841, + "tokens_seen": 343015424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007240762286860582, + "loss": 1.5503, + "theoretical_loss": 4.110815243853323, + "tokens_seen": 343080960 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724060180541625, + "loss": 1.6105, + "theoretical_loss": 4.110725748207633, + "tokens_seen": 343146496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007240441323971916, + "loss": 1.4616, + "theoretical_loss": 4.110636274437502, + "tokens_seen": 343212032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007240280842527583, + "loss": 1.6878, + "theoretical_loss": 4.110546822533407, + "tokens_seen": 343277568 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724012036108325, + "loss": 1.6503, + "theoretical_loss": 4.110457392485832, + "tokens_seen": 343343104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239959879638917, + "loss": 1.7304, + "theoretical_loss": 4.110367984285265, + "tokens_seen": 343408640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239799398194584, + "loss": 1.6316, + "theoretical_loss": 4.1102785979222025, + "tokens_seen": 343474176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239638916750252, + "loss": 1.6789, + "theoretical_loss": 4.110189233387146, + "tokens_seen": 343539712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239478435305918, + "loss": 1.4762, + "theoretical_loss": 4.1100998906706, + "tokens_seen": 343605248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239317953861586, + "loss": 1.6151, + "theoretical_loss": 4.110010569763081, + "tokens_seen": 343670784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239157472417252, + "loss": 1.5582, + "theoretical_loss": 4.109921270655106, + "tokens_seen": 343736320 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723899699097292, + "loss": 1.5535, + "theoretical_loss": 4.109831993337201, + "tokens_seen": 343801856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238836509528586, + "loss": 1.6652, + "theoretical_loss": 4.109742737799895, + "tokens_seen": 343867392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238676028084254, + "loss": 1.5522, + "theoretical_loss": 4.109653504033726, + "tokens_seen": 343932928 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723851554663992, + "loss": 1.4251, + "theoretical_loss": 4.109564292029235, + "tokens_seen": 343998464 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 121082, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.569899082183838, + "objective/train/theoretical_loss": 4.109475101776974, + "objective/train/tokens_used": 364524000, + "theoretical_loss": 4.109475101776974, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238355065195588, + "loss": 1.6178, + "theoretical_loss": 4.109475101776974, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238194583751254, + "loss": 1.6877, + "theoretical_loss": 4.109385933267495, + "tokens_seen": 344129536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238034102306922, + "loss": 1.4787, + "theoretical_loss": 4.109296786491359, + "tokens_seen": 344195072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237873620862588, + "loss": 1.5375, + "theoretical_loss": 4.1092076614391315, + "tokens_seen": 344260608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237713139418256, + "loss": 1.7169, + "theoretical_loss": 4.109118558101386, + "tokens_seen": 344326144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237552657973922, + "loss": 1.5425, + "theoretical_loss": 4.1090294764687, + "tokens_seen": 344391680 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723739217652959, + "loss": 1.6181, + "theoretical_loss": 4.108940416531656, + "tokens_seen": 344457216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237231695085256, + "loss": 1.5927, + "theoretical_loss": 4.108851378280846, + "tokens_seen": 344522752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237071213640924, + "loss": 1.7046, + "theoretical_loss": 4.108762361706865, + "tokens_seen": 344588288 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723691073219659, + "loss": 1.6399, + "theoretical_loss": 4.108673366800313, + "tokens_seen": 344653824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236750250752258, + "loss": 1.5645, + "theoretical_loss": 4.1085843935518, + "tokens_seen": 344719360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236589769307924, + "loss": 1.4864, + "theoretical_loss": 4.108495441951937, + "tokens_seen": 344784896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236429287863592, + "loss": 1.6584, + "theoretical_loss": 4.108406511991344, + "tokens_seen": 344850432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236268806419258, + "loss": 1.5342, + "theoretical_loss": 4.108317603660646, + "tokens_seen": 344915968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236108324974926, + "loss": 1.6394, + "theoretical_loss": 4.108228716950474, + "tokens_seen": 344981504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007235947843530592, + "loss": 1.5845, + "theoretical_loss": 4.108139851851464, + "tokens_seen": 345047040 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723578736208626, + "loss": 1.4446, + "theoretical_loss": 4.108051008354258, + "tokens_seen": 345112576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007235626880641926, + "loss": 1.5017, + "theoretical_loss": 4.107962186449506, + "tokens_seen": 345178112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007235466399197594, + "loss": 1.5043, + "theoretical_loss": 4.107873386127861, + "tokens_seen": 345243648 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723530591775326, + "loss": 1.4736, + "theoretical_loss": 4.107784607379983, + "tokens_seen": 345309184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007235145436308928, + "loss": 1.4901, + "theoretical_loss": 4.107695850196539, + "tokens_seen": 345374720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234984954864594, + "loss": 1.5762, + "theoretical_loss": 4.107607114568197, + "tokens_seen": 345440256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234824473420262, + "loss": 1.5355, + "theoretical_loss": 4.107518400485638, + "tokens_seen": 345505792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234663991975928, + "loss": 1.6054, + "theoretical_loss": 4.107429707939545, + "tokens_seen": 345571328 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234503510531596, + "loss": 1.6409, + "theoretical_loss": 4.107341036920604, + "tokens_seen": 345636864 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234343029087262, + "loss": 1.5958, + "theoretical_loss": 4.107252387419513, + "tokens_seen": 345702400 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723418254764293, + "loss": 1.6453, + "theoretical_loss": 4.107163759426971, + "tokens_seen": 345767936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234022066198596, + "loss": 1.6439, + "theoretical_loss": 4.1070751529336835, + "tokens_seen": 345833472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233861584754264, + "loss": 1.5924, + "theoretical_loss": 4.106986567930364, + "tokens_seen": 345899008 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723370110330993, + "loss": 1.6914, + "theoretical_loss": 4.1068980044077295, + "tokens_seen": 345964544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233540621865598, + "loss": 1.5754, + "theoretical_loss": 4.1068094623565035, + "tokens_seen": 346030080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233380140421264, + "loss": 1.5374, + "theoretical_loss": 4.106720941767417, + "tokens_seen": 346095616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233219658976931, + "loss": 1.6939, + "theoretical_loss": 4.106632442631203, + "tokens_seen": 346161152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233059177532598, + "loss": 1.6309, + "theoretical_loss": 4.106543964938604, + "tokens_seen": 346226688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007232898696088265, + "loss": 1.5436, + "theoretical_loss": 4.106455508680366, + "tokens_seen": 346292224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007232738214643932, + "loss": 1.5128, + "theoretical_loss": 4.106367073847241, + "tokens_seen": 346357760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007232577733199599, + "loss": 1.4556, + "theoretical_loss": 4.106278660429988, + "tokens_seen": 346423296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007232417251755266, + "loss": 1.5254, + "theoretical_loss": 4.10619026841937, + "tokens_seen": 346488832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007232256770310933, + "loss": 1.7047, + "theoretical_loss": 4.106101897806156, + "tokens_seen": 346554368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00072320962888666, + "loss": 1.5841, + "theoretical_loss": 4.1060135485811236, + "tokens_seen": 346619904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231935807422267, + "loss": 1.6565, + "theoretical_loss": 4.105925220735052, + "tokens_seen": 346685440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231775325977933, + "loss": 1.5367, + "theoretical_loss": 4.105836914258728, + "tokens_seen": 346750976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231614844533601, + "loss": 1.5487, + "theoretical_loss": 4.105748629142945, + "tokens_seen": 346816512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231454363089267, + "loss": 1.5987, + "theoretical_loss": 4.1056603653785, + "tokens_seen": 346882048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231293881644935, + "loss": 1.6058, + "theoretical_loss": 4.1055721229561986, + "tokens_seen": 346947584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231133400200602, + "loss": 1.6939, + "theoretical_loss": 4.105483901866848, + "tokens_seen": 347013120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230972918756269, + "loss": 1.628, + "theoretical_loss": 4.105395702101266, + "tokens_seen": 347078656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230812437311936, + "loss": 1.6857, + "theoretical_loss": 4.1053075236502705, + "tokens_seen": 347144192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230651955867603, + "loss": 1.535, + "theoretical_loss": 4.1052193665046905, + "tokens_seen": 347209728 + }, + { + "epoch": 0.11, + "learning_rate": 0.000723049147442327, + "loss": 1.5932, + "theoretical_loss": 4.105131230655358, + "tokens_seen": 347275264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 122020, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6148931980133057, + "objective/train/theoretical_loss": 4.1050431160931105, + "objective/train/tokens_used": 367800800, + "theoretical_loss": 4.1050431160931105, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230330992978937, + "loss": 1.5267, + "theoretical_loss": 4.1050431160931105, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230170511534604, + "loss": 1.6963, + "theoretical_loss": 4.104955022808792, + "tokens_seen": 347406336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230010030090271, + "loss": 1.597, + "theoretical_loss": 4.104866950793251, + "tokens_seen": 347471872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229849548645938, + "loss": 1.6458, + "theoretical_loss": 4.104778900037343, + "tokens_seen": 347537408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229689067201605, + "loss": 1.5932, + "theoretical_loss": 4.104690870531929, + "tokens_seen": 347602944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229528585757272, + "loss": 1.6048, + "theoretical_loss": 4.104602862267875, + "tokens_seen": 347668480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229368104312939, + "loss": 1.5327, + "theoretical_loss": 4.1045148752360525, + "tokens_seen": 347734016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229207622868606, + "loss": 1.6515, + "theoretical_loss": 4.104426909427339, + "tokens_seen": 347799552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229047141424273, + "loss": 1.7427, + "theoretical_loss": 4.104338964832619, + "tokens_seen": 347865088 + }, + { + "epoch": 0.11, + "learning_rate": 0.000722888665997994, + "loss": 1.4355, + "theoretical_loss": 4.10425104144278, + "tokens_seen": 347930624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228726178535607, + "loss": 1.5693, + "theoretical_loss": 4.104163139248717, + "tokens_seen": 347996160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228565697091274, + "loss": 1.7103, + "theoretical_loss": 4.1040752582413305, + "tokens_seen": 348061696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228405215646941, + "loss": 1.5576, + "theoretical_loss": 4.103987398411524, + "tokens_seen": 348127232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228244734202608, + "loss": 1.5204, + "theoretical_loss": 4.103899559750213, + "tokens_seen": 348192768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228084252758275, + "loss": 1.5165, + "theoretical_loss": 4.10381174224831, + "tokens_seen": 348258304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227923771313942, + "loss": 1.4986, + "theoretical_loss": 4.10372394589674, + "tokens_seen": 348323840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227763289869609, + "loss": 1.6011, + "theoretical_loss": 4.10363617068643, + "tokens_seen": 348389376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227602808425276, + "loss": 1.5675, + "theoretical_loss": 4.103548416608316, + "tokens_seen": 348454912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227442326980943, + "loss": 1.6869, + "theoretical_loss": 4.103460683653335, + "tokens_seen": 348520448 + }, + { + "epoch": 0.11, + "learning_rate": 0.000722728184553661, + "loss": 1.5663, + "theoretical_loss": 4.103372971812432, + "tokens_seen": 348585984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227121364092277, + "loss": 1.605, + "theoretical_loss": 4.103285281076558, + "tokens_seen": 348651520 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226960882647944, + "loss": 1.5792, + "theoretical_loss": 4.10319761143667, + "tokens_seen": 348717056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226800401203611, + "loss": 1.5252, + "theoretical_loss": 4.103109962883728, + "tokens_seen": 348782592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226639919759278, + "loss": 1.57, + "theoretical_loss": 4.1030223354087, + "tokens_seen": 348848128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226479438314945, + "loss": 1.5449, + "theoretical_loss": 4.102934729002559, + "tokens_seen": 348913664 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226318956870612, + "loss": 1.4946, + "theoretical_loss": 4.102847143656284, + "tokens_seen": 348979200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226158475426279, + "loss": 1.548, + "theoretical_loss": 4.102759579360857, + "tokens_seen": 349044736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225997993981947, + "loss": 1.6362, + "theoretical_loss": 4.102672036107269, + "tokens_seen": 349110272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225837512537613, + "loss": 1.5507, + "theoretical_loss": 4.102584513886514, + "tokens_seen": 349175808 + }, + { + "epoch": 0.11, + "learning_rate": 0.000722567703109328, + "loss": 1.5163, + "theoretical_loss": 4.102497012689594, + "tokens_seen": 349241344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225516549648947, + "loss": 1.6004, + "theoretical_loss": 4.102409532507513, + "tokens_seen": 349306880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225356068204615, + "loss": 1.4954, + "theoretical_loss": 4.102322073331285, + "tokens_seen": 349372416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225195586760281, + "loss": 1.4697, + "theoretical_loss": 4.102234635151925, + "tokens_seen": 349437952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225035105315949, + "loss": 1.5609, + "theoretical_loss": 4.102147217960457, + "tokens_seen": 349503488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224874623871615, + "loss": 1.5016, + "theoretical_loss": 4.1020598217479085, + "tokens_seen": 349569024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224714142427283, + "loss": 1.6255, + "theoretical_loss": 4.101972446505314, + "tokens_seen": 349634560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224553660982949, + "loss": 1.5068, + "theoretical_loss": 4.101885092223712, + "tokens_seen": 349700096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224393179538617, + "loss": 1.6375, + "theoretical_loss": 4.101797758894147, + "tokens_seen": 349765632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224232698094283, + "loss": 1.5521, + "theoretical_loss": 4.101710446507669, + "tokens_seen": 349831168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224072216649951, + "loss": 1.5221, + "theoretical_loss": 4.101623155055336, + "tokens_seen": 349896704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223911735205617, + "loss": 1.5779, + "theoretical_loss": 4.101535884528206, + "tokens_seen": 349962240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223751253761285, + "loss": 1.5925, + "theoretical_loss": 4.1014486349173485, + "tokens_seen": 350027776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223590772316951, + "loss": 1.6624, + "theoretical_loss": 4.101361406213834, + "tokens_seen": 350093312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223430290872619, + "loss": 1.5511, + "theoretical_loss": 4.101274198408741, + "tokens_seen": 350158848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223269809428285, + "loss": 1.6351, + "theoretical_loss": 4.101187011493153, + "tokens_seen": 350224384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223109327983953, + "loss": 1.6788, + "theoretical_loss": 4.101099845458156, + "tokens_seen": 350289920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222948846539619, + "loss": 1.6737, + "theoretical_loss": 4.101012700294848, + "tokens_seen": 350355456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222788365095287, + "loss": 1.6357, + "theoretical_loss": 4.100925575994325, + "tokens_seen": 350420992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222627883650953, + "loss": 1.5646, + "theoretical_loss": 4.100838472547695, + "tokens_seen": 350486528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222467402206621, + "loss": 1.5503, + "theoretical_loss": 4.100751389946067, + "tokens_seen": 350552064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 123333, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5236725807189941, + "objective/train/theoretical_loss": 4.100664328180557, + "objective/train/tokens_used": 371077600, + "theoretical_loss": 4.100664328180557, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222306920762287, + "loss": 1.5919, + "theoretical_loss": 4.100664328180557, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222146439317955, + "loss": 1.6033, + "theoretical_loss": 4.100577287242286, + "tokens_seen": 350683136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221985957873621, + "loss": 1.4229, + "theoretical_loss": 4.100490267122381, + "tokens_seen": 350748672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221825476429289, + "loss": 1.4672, + "theoretical_loss": 4.100403267811975, + "tokens_seen": 350814208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221664994984955, + "loss": 1.6758, + "theoretical_loss": 4.100316289302206, + "tokens_seen": 350879744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221504513540623, + "loss": 1.5765, + "theoretical_loss": 4.100229331584215, + "tokens_seen": 350945280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221344032096289, + "loss": 1.6201, + "theoretical_loss": 4.100142394649152, + "tokens_seen": 351010816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221183550651957, + "loss": 1.5798, + "theoretical_loss": 4.100055478488171, + "tokens_seen": 351076352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221023069207623, + "loss": 1.5945, + "theoretical_loss": 4.099968583092431, + "tokens_seen": 351141888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220862587763291, + "loss": 1.7934, + "theoretical_loss": 4.099881708453097, + "tokens_seen": 351207424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220702106318957, + "loss": 1.5929, + "theoretical_loss": 4.099794854561339, + "tokens_seen": 351272960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220541624874625, + "loss": 1.5479, + "theoretical_loss": 4.0997080214083335, + "tokens_seen": 351338496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220381143430291, + "loss": 1.5969, + "theoretical_loss": 4.09962120898526, + "tokens_seen": 351404032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220220661985959, + "loss": 1.8095, + "theoretical_loss": 4.099534417283305, + "tokens_seen": 351469568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220060180541625, + "loss": 1.5545, + "theoretical_loss": 4.099447646293662, + "tokens_seen": 351535104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219899699097293, + "loss": 1.4616, + "theoretical_loss": 4.099360896007527, + "tokens_seen": 351600640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219739217652959, + "loss": 1.5338, + "theoretical_loss": 4.099274166416102, + "tokens_seen": 351666176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219578736208627, + "loss": 1.5553, + "theoretical_loss": 4.099187457510595, + "tokens_seen": 351731712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219418254764293, + "loss": 1.6277, + "theoretical_loss": 4.099100769282221, + "tokens_seen": 351797248 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721925777331996, + "loss": 1.5872, + "theoretical_loss": 4.099014101722196, + "tokens_seen": 351862784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219097291875627, + "loss": 1.549, + "theoretical_loss": 4.098927454821747, + "tokens_seen": 351928320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218936810431294, + "loss": 1.7034, + "theoretical_loss": 4.098840828572101, + "tokens_seen": 351993856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218776328986961, + "loss": 1.5264, + "theoretical_loss": 4.0987542229644935, + "tokens_seen": 352059392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218615847542628, + "loss": 1.5467, + "theoretical_loss": 4.0986676379901645, + "tokens_seen": 352124928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218455366098295, + "loss": 1.5454, + "theoretical_loss": 4.098581073640361, + "tokens_seen": 352190464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218294884653962, + "loss": 1.6108, + "theoretical_loss": 4.098494529906331, + "tokens_seen": 352256000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218134403209629, + "loss": 1.625, + "theoretical_loss": 4.098408006779332, + "tokens_seen": 352321536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217973921765296, + "loss": 1.6397, + "theoretical_loss": 4.098321504250626, + "tokens_seen": 352387072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217813440320963, + "loss": 1.651, + "theoretical_loss": 4.098235022311479, + "tokens_seen": 352452608 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721765295887663, + "loss": 1.5686, + "theoretical_loss": 4.098148560953163, + "tokens_seen": 352518144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217492477432297, + "loss": 1.5725, + "theoretical_loss": 4.098062120166956, + "tokens_seen": 352583680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217331995987964, + "loss": 1.5605, + "theoretical_loss": 4.097975699944141, + "tokens_seen": 352649216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217171514543631, + "loss": 1.4939, + "theoretical_loss": 4.097889300276005, + "tokens_seen": 352714752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217011033099298, + "loss": 1.425, + "theoretical_loss": 4.097802921153842, + "tokens_seen": 352780288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216850551654965, + "loss": 1.5532, + "theoretical_loss": 4.097716562568951, + "tokens_seen": 352845824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216690070210632, + "loss": 1.6839, + "theoretical_loss": 4.097630224512635, + "tokens_seen": 352911360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216529588766299, + "loss": 1.6294, + "theoretical_loss": 4.097543906976204, + "tokens_seen": 352976896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216369107321966, + "loss": 1.5198, + "theoretical_loss": 4.097457609950973, + "tokens_seen": 353042432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216208625877633, + "loss": 1.5761, + "theoretical_loss": 4.09737133342826, + "tokens_seen": 353107968 + }, + { + "epoch": 0.11, + "learning_rate": 0.00072160481444333, + "loss": 1.6117, + "theoretical_loss": 4.097285077399392, + "tokens_seen": 353173504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215887662988967, + "loss": 1.6125, + "theoretical_loss": 4.097198841855699, + "tokens_seen": 353239040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215727181544634, + "loss": 1.6285, + "theoretical_loss": 4.097112626788516, + "tokens_seen": 353304576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215566700100301, + "loss": 1.5738, + "theoretical_loss": 4.097026432189185, + "tokens_seen": 353370112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215406218655968, + "loss": 1.6719, + "theoretical_loss": 4.096940258049052, + "tokens_seen": 353435648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215245737211635, + "loss": 1.5116, + "theoretical_loss": 4.096854104359467, + "tokens_seen": 353501184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215085255767302, + "loss": 1.4971, + "theoretical_loss": 4.096767971111789, + "tokens_seen": 353566720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214924774322969, + "loss": 1.6234, + "theoretical_loss": 4.096681858297378, + "tokens_seen": 353632256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214764292878636, + "loss": 1.5845, + "theoretical_loss": 4.096595765907603, + "tokens_seen": 353697792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214603811434303, + "loss": 1.4026, + "theoretical_loss": 4.096509693933835, + "tokens_seen": 353763328 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721444332998997, + "loss": 1.745, + "theoretical_loss": 4.096423642367453, + "tokens_seen": 353828864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 124136, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5353221893310547, + "objective/train/theoretical_loss": 4.096337611199838, + "objective/train/tokens_used": 374354400, + "theoretical_loss": 4.096337611199838, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214282848545637, + "loss": 1.4744, + "theoretical_loss": 4.096337611199838, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214122367101304, + "loss": 1.5854, + "theoretical_loss": 4.096251600422381, + "tokens_seen": 353959936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213961885656971, + "loss": 1.5289, + "theoretical_loss": 4.096165610026474, + "tokens_seen": 354025472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213801404212638, + "loss": 1.5319, + "theoretical_loss": 4.096079640003515, + "tokens_seen": 354091008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213640922768305, + "loss": 1.469, + "theoretical_loss": 4.095993690344908, + "tokens_seen": 354156544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213480441323972, + "loss": 1.5532, + "theoretical_loss": 4.0959077610420636, + "tokens_seen": 354222080 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721331995987964, + "loss": 1.503, + "theoretical_loss": 4.095821852086394, + "tokens_seen": 354287616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213159478435306, + "loss": 1.4657, + "theoretical_loss": 4.095735963469322, + "tokens_seen": 354353152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212998996990973, + "loss": 1.4184, + "theoretical_loss": 4.095650095182268, + "tokens_seen": 354418688 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721283851554664, + "loss": 1.525, + "theoretical_loss": 4.0955642472166645, + "tokens_seen": 354484224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212678034102307, + "loss": 1.524, + "theoretical_loss": 4.095478419563947, + "tokens_seen": 354549760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212517552657974, + "loss": 1.4833, + "theoretical_loss": 4.095392612215555, + "tokens_seen": 354615296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212357071213642, + "loss": 1.486, + "theoretical_loss": 4.095306825162933, + "tokens_seen": 354680832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212196589769308, + "loss": 1.6026, + "theoretical_loss": 4.095221058397534, + "tokens_seen": 354746368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212036108324976, + "loss": 1.493, + "theoretical_loss": 4.095135311910812, + "tokens_seen": 354811904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007211875626880642, + "loss": 1.4913, + "theoretical_loss": 4.095049585694229, + "tokens_seen": 354877440 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721171514543631, + "loss": 1.5869, + "theoretical_loss": 4.09496387973925, + "tokens_seen": 354942976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007211554663991976, + "loss": 1.4811, + "theoretical_loss": 4.094878194037348, + "tokens_seen": 355008512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007211394182547644, + "loss": 1.5131, + "theoretical_loss": 4.094792528579999, + "tokens_seen": 355074048 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721123370110331, + "loss": 1.5777, + "theoretical_loss": 4.094706883358684, + "tokens_seen": 355139584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007211073219658978, + "loss": 1.5027, + "theoretical_loss": 4.09462125836489, + "tokens_seen": 355205120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210912738214644, + "loss": 1.4404, + "theoretical_loss": 4.0945356535901105, + "tokens_seen": 355270656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210752256770312, + "loss": 1.5339, + "theoretical_loss": 4.094450069025841, + "tokens_seen": 355336192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210591775325978, + "loss": 1.454, + "theoretical_loss": 4.094364504663584, + "tokens_seen": 355401728 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210431293881646, + "loss": 1.6805, + "theoretical_loss": 4.094278960494847, + "tokens_seen": 355467264 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210270812437312, + "loss": 1.4953, + "theoretical_loss": 4.094193436511143, + "tokens_seen": 355532800 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721011033099298, + "loss": 1.4764, + "theoretical_loss": 4.094107932703991, + "tokens_seen": 355598336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209949849548646, + "loss": 1.6513, + "theoretical_loss": 4.09402244906491, + "tokens_seen": 355663872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209789368104314, + "loss": 1.6311, + "theoretical_loss": 4.093936985585432, + "tokens_seen": 355729408 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720962888665998, + "loss": 1.6119, + "theoretical_loss": 4.093851542257088, + "tokens_seen": 355794944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209468405215648, + "loss": 1.6171, + "theoretical_loss": 4.093766119071416, + "tokens_seen": 355860480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209307923771314, + "loss": 1.6012, + "theoretical_loss": 4.093680716019961, + "tokens_seen": 355926016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209147442326982, + "loss": 1.5785, + "theoretical_loss": 4.093595333094269, + "tokens_seen": 355991552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208986960882648, + "loss": 1.5399, + "theoretical_loss": 4.093509970285895, + "tokens_seen": 356057088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208826479438316, + "loss": 1.5283, + "theoretical_loss": 4.093424627586399, + "tokens_seen": 356122624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208665997993982, + "loss": 1.4739, + "theoretical_loss": 4.093339304987341, + "tokens_seen": 356188160 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720850551654965, + "loss": 1.4281, + "theoretical_loss": 4.093254002480293, + "tokens_seen": 356253696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208345035105316, + "loss": 1.7098, + "theoretical_loss": 4.093168720056828, + "tokens_seen": 356319232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208184553660984, + "loss": 1.4742, + "theoretical_loss": 4.093083457708524, + "tokens_seen": 356384768 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720802407221665, + "loss": 1.6874, + "theoretical_loss": 4.092998215426967, + "tokens_seen": 356450304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207863590772318, + "loss": 1.5701, + "theoretical_loss": 4.092912993203744, + "tokens_seen": 356515840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207703109327984, + "loss": 1.5932, + "theoretical_loss": 4.09282779103045, + "tokens_seen": 356581376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207542627883652, + "loss": 1.5365, + "theoretical_loss": 4.092742608898685, + "tokens_seen": 356646912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207382146439318, + "loss": 1.6133, + "theoretical_loss": 4.092657446800052, + "tokens_seen": 356712448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207221664994986, + "loss": 1.5243, + "theoretical_loss": 4.092572304726161, + "tokens_seen": 356777984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207061183550653, + "loss": 1.5389, + "theoretical_loss": 4.092487182668627, + "tokens_seen": 356843520 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720690070210632, + "loss": 1.6216, + "theoretical_loss": 4.092402080619069, + "tokens_seen": 356909056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007206740220661987, + "loss": 1.7329, + "theoretical_loss": 4.09231699856911, + "tokens_seen": 356974592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007206579739217654, + "loss": 1.5615, + "theoretical_loss": 4.092231936510383, + "tokens_seen": 357040128 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720641925777332, + "loss": 1.5868, + "theoretical_loss": 4.09214689443452, + "tokens_seen": 357105664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 125479, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5331692695617676, + "objective/train/theoretical_loss": 4.0920618723331605, + "objective/train/tokens_used": 377631200, + "theoretical_loss": 4.0920618723331605, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007206258776328988, + "loss": 1.4486, + "theoretical_loss": 4.0920618723331605, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007206098294884654, + "loss": 1.621, + "theoretical_loss": 4.091976870197951, + "tokens_seen": 357236736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205937813440322, + "loss": 1.5364, + "theoretical_loss": 4.091891888020539, + "tokens_seen": 357302272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205777331995988, + "loss": 1.4859, + "theoretical_loss": 4.091806925792582, + "tokens_seen": 357367808 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205616850551656, + "loss": 1.6564, + "theoretical_loss": 4.091721983505737, + "tokens_seen": 357433344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205456369107322, + "loss": 1.4599, + "theoretical_loss": 4.091637061151671, + "tokens_seen": 357498880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205295887662989, + "loss": 1.4924, + "theoretical_loss": 4.091552158722052, + "tokens_seen": 357564416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205135406218656, + "loss": 1.4918, + "theoretical_loss": 4.091467276208554, + "tokens_seen": 357629952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204974924774323, + "loss": 1.5315, + "theoretical_loss": 4.091382413602861, + "tokens_seen": 357695488 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720481444332999, + "loss": 1.6625, + "theoretical_loss": 4.0912975708966535, + "tokens_seen": 357761024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204653961885657, + "loss": 1.5066, + "theoretical_loss": 4.091212748081622, + "tokens_seen": 357826560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204493480441324, + "loss": 1.5501, + "theoretical_loss": 4.091127945149463, + "tokens_seen": 357892096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204332998996991, + "loss": 1.5951, + "theoretical_loss": 4.091043162091876, + "tokens_seen": 357957632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204172517552658, + "loss": 1.6305, + "theoretical_loss": 4.090958398900564, + "tokens_seen": 358023168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204012036108325, + "loss": 1.4798, + "theoretical_loss": 4.090873655567238, + "tokens_seen": 358088704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203851554663992, + "loss": 1.5176, + "theoretical_loss": 4.090788932083613, + "tokens_seen": 358154240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203691073219659, + "loss": 1.5244, + "theoretical_loss": 4.090704228441408, + "tokens_seen": 358219776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203530591775326, + "loss": 1.4846, + "theoretical_loss": 4.090619544632347, + "tokens_seen": 358285312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203370110330993, + "loss": 1.5317, + "theoretical_loss": 4.0905348806481605, + "tokens_seen": 358350848 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720320962888666, + "loss": 1.5263, + "theoretical_loss": 4.090450236480583, + "tokens_seen": 358416384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203049147442327, + "loss": 1.4949, + "theoretical_loss": 4.090365612121355, + "tokens_seen": 358481920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202888665997994, + "loss": 1.5811, + "theoretical_loss": 4.0902810075622185, + "tokens_seen": 358547456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202728184553661, + "loss": 1.5137, + "theoretical_loss": 4.090196422794925, + "tokens_seen": 358612992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202567703109328, + "loss": 1.6153, + "theoretical_loss": 4.0901118578112285, + "tokens_seen": 358678528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202407221664995, + "loss": 1.5427, + "theoretical_loss": 4.090027312602888, + "tokens_seen": 358744064 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202246740220662, + "loss": 1.5705, + "theoretical_loss": 4.089942787161667, + "tokens_seen": 358809600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202086258776329, + "loss": 1.6536, + "theoretical_loss": 4.089858281479335, + "tokens_seen": 358875136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201925777331996, + "loss": 1.5034, + "theoretical_loss": 4.089773795547668, + "tokens_seen": 358940672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201765295887663, + "loss": 1.6133, + "theoretical_loss": 4.089689329358443, + "tokens_seen": 359006208 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720160481444333, + "loss": 1.6104, + "theoretical_loss": 4.089604882903444, + "tokens_seen": 359071744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201444332998997, + "loss": 1.5082, + "theoretical_loss": 4.089520456174461, + "tokens_seen": 359137280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201283851554664, + "loss": 1.6146, + "theoretical_loss": 4.089436049163287, + "tokens_seen": 359202816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201123370110331, + "loss": 1.6244, + "theoretical_loss": 4.089351661861722, + "tokens_seen": 359268352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200962888665998, + "loss": 1.5419, + "theoretical_loss": 4.089267294261566, + "tokens_seen": 359333888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200802407221665, + "loss": 1.6289, + "theoretical_loss": 4.089182946354631, + "tokens_seen": 359399424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200641925777332, + "loss": 1.5434, + "theoretical_loss": 4.08909861813273, + "tokens_seen": 359464960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200481444332999, + "loss": 1.4964, + "theoretical_loss": 4.0890143095876805, + "tokens_seen": 359530496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200320962888666, + "loss": 1.6228, + "theoretical_loss": 4.088930020711305, + "tokens_seen": 359596032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200160481444333, + "loss": 1.5421, + "theoretical_loss": 4.088845751495433, + "tokens_seen": 359661568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00072, + "loss": 1.4741, + "theoretical_loss": 4.088761501931897, + "tokens_seen": 359727104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199839518555667, + "loss": 1.6506, + "theoretical_loss": 4.088677272012535, + "tokens_seen": 359792640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199679037111334, + "loss": 1.6194, + "theoretical_loss": 4.088593061729188, + "tokens_seen": 359858176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199518555667001, + "loss": 1.5317, + "theoretical_loss": 4.0885088710737065, + "tokens_seen": 359923712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199358074222668, + "loss": 1.6348, + "theoretical_loss": 4.088424700037941, + "tokens_seen": 359989248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199197592778335, + "loss": 1.5538, + "theoretical_loss": 4.08834054861375, + "tokens_seen": 360054784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199037111334002, + "loss": 1.5956, + "theoretical_loss": 4.088256416792994, + "tokens_seen": 360120320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198876629889669, + "loss": 1.4913, + "theoretical_loss": 4.088172304567541, + "tokens_seen": 360185856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198716148445337, + "loss": 1.6122, + "theoretical_loss": 4.088088211929263, + "tokens_seen": 360251392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198555667001003, + "loss": 1.4953, + "theoretical_loss": 4.088004138870037, + "tokens_seen": 360316928 + }, + { + "epoch": 0.11, + "learning_rate": 0.000719839518555667, + "loss": 1.5233, + "theoretical_loss": 4.087920085381745, + "tokens_seen": 360382464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 126440, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6889604330062866, + "objective/train/theoretical_loss": 4.0878360514562715, + "objective/train/tokens_used": 380908000, + "theoretical_loss": 4.0878360514562715, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198234704112337, + "loss": 1.6539, + "theoretical_loss": 4.0878360514562715, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198074222668005, + "loss": 1.4704, + "theoretical_loss": 4.087752037085509, + "tokens_seen": 360513536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197913741223671, + "loss": 1.5366, + "theoretical_loss": 4.087668042261353, + "tokens_seen": 360579072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197753259779339, + "loss": 1.4443, + "theoretical_loss": 4.087584066975706, + "tokens_seen": 360644608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197592778335005, + "loss": 1.5863, + "theoretical_loss": 4.087500111220472, + "tokens_seen": 360710144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197432296890673, + "loss": 1.5266, + "theoretical_loss": 4.087416174987563, + "tokens_seen": 360775680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197271815446339, + "loss": 1.6049, + "theoretical_loss": 4.087332258268893, + "tokens_seen": 360841216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197111334002007, + "loss": 1.5389, + "theoretical_loss": 4.087248361056384, + "tokens_seen": 360906752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196950852557673, + "loss": 1.7153, + "theoretical_loss": 4.087164483341959, + "tokens_seen": 360972288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196790371113341, + "loss": 1.4725, + "theoretical_loss": 4.087080625117549, + "tokens_seen": 361037824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196629889669007, + "loss": 1.4997, + "theoretical_loss": 4.086996786375089, + "tokens_seen": 361103360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196469408224675, + "loss": 1.5327, + "theoretical_loss": 4.0869129671065165, + "tokens_seen": 361168896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196308926780341, + "loss": 1.5279, + "theoretical_loss": 4.086829167303778, + "tokens_seen": 361234432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196148445336009, + "loss": 1.6238, + "theoretical_loss": 4.086745386958821, + "tokens_seen": 361299968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195987963891675, + "loss": 1.6077, + "theoretical_loss": 4.0866616260636, + "tokens_seen": 361365504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195827482447343, + "loss": 1.5136, + "theoretical_loss": 4.086577884610074, + "tokens_seen": 361431040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195667001003009, + "loss": 1.5304, + "theoretical_loss": 4.086494162590206, + "tokens_seen": 361496576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195506519558677, + "loss": 1.6102, + "theoretical_loss": 4.086410459995964, + "tokens_seen": 361562112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195346038114343, + "loss": 1.6007, + "theoretical_loss": 4.0863267768193205, + "tokens_seen": 361627648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195185556670011, + "loss": 1.5662, + "theoretical_loss": 4.086243113052254, + "tokens_seen": 361693184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195025075225678, + "loss": 1.4989, + "theoretical_loss": 4.0861594686867475, + "tokens_seen": 361758720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194864593781345, + "loss": 1.5979, + "theoretical_loss": 4.086075843714786, + "tokens_seen": 361824256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194704112337012, + "loss": 1.6645, + "theoretical_loss": 4.085992238128364, + "tokens_seen": 361889792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194543630892679, + "loss": 1.7224, + "theoretical_loss": 4.085908651919477, + "tokens_seen": 361955328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194383149448346, + "loss": 1.5489, + "theoretical_loss": 4.085825085080128, + "tokens_seen": 362020864 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194222668004013, + "loss": 1.5459, + "theoretical_loss": 4.08574153760232, + "tokens_seen": 362086400 + }, + { + "epoch": 0.11, + "learning_rate": 0.000719406218655968, + "loss": 1.7234, + "theoretical_loss": 4.085658009478067, + "tokens_seen": 362151936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193901705115347, + "loss": 1.4414, + "theoretical_loss": 4.0855745006993835, + "tokens_seen": 362217472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193741223671014, + "loss": 1.5663, + "theoretical_loss": 4.08549101125829, + "tokens_seen": 362283008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193580742226681, + "loss": 1.6014, + "theoretical_loss": 4.085407541146812, + "tokens_seen": 362348544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193420260782348, + "loss": 1.6279, + "theoretical_loss": 4.085324090356979, + "tokens_seen": 362414080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193259779338015, + "loss": 1.531, + "theoretical_loss": 4.085240658880827, + "tokens_seen": 362479616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193099297893682, + "loss": 1.5496, + "theoretical_loss": 4.085157246710394, + "tokens_seen": 362545152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192938816449349, + "loss": 1.5263, + "theoretical_loss": 4.0850738538377245, + "tokens_seen": 362610688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192778335005016, + "loss": 1.6056, + "theoretical_loss": 4.084990480254867, + "tokens_seen": 362676224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192617853560683, + "loss": 1.6295, + "theoretical_loss": 4.084907125953876, + "tokens_seen": 362741760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192457372116349, + "loss": 1.5451, + "theoretical_loss": 4.0848237909268095, + "tokens_seen": 362807296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192296890672017, + "loss": 1.6121, + "theoretical_loss": 4.084740475165729, + "tokens_seen": 362872832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192136409227683, + "loss": 1.5199, + "theoretical_loss": 4.084657178662704, + "tokens_seen": 362938368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191975927783351, + "loss": 1.5887, + "theoretical_loss": 4.084573901409807, + "tokens_seen": 363003904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191815446339017, + "loss": 1.6182, + "theoretical_loss": 4.084490643399113, + "tokens_seen": 363069440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191654964894685, + "loss": 1.5459, + "theoretical_loss": 4.084407404622706, + "tokens_seen": 363134976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191494483450351, + "loss": 1.6282, + "theoretical_loss": 4.084324185072671, + "tokens_seen": 363200512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191334002006019, + "loss": 1.6513, + "theoretical_loss": 4.084240984741099, + "tokens_seen": 363266048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191173520561685, + "loss": 1.5754, + "theoretical_loss": 4.084157803620086, + "tokens_seen": 363331584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191013039117352, + "loss": 1.6044, + "theoretical_loss": 4.084074641701734, + "tokens_seen": 363397120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190852557673019, + "loss": 1.6218, + "theoretical_loss": 4.083991498978146, + "tokens_seen": 363462656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190692076228686, + "loss": 1.549, + "theoretical_loss": 4.083908375441433, + "tokens_seen": 363528192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190531594784353, + "loss": 1.5066, + "theoretical_loss": 4.083825271083709, + "tokens_seen": 363593728 + }, + { + "epoch": 0.11, + "learning_rate": 0.000719037111334002, + "loss": 1.7009, + "theoretical_loss": 4.083742185897093, + "tokens_seen": 363659264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 127545, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4759583473205566, + "objective/train/theoretical_loss": 4.08365911987371, + "objective/train/tokens_used": 384184800, + "theoretical_loss": 4.08365911987371, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190210631895687, + "loss": 1.4025, + "theoretical_loss": 4.08365911987371, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190050150451354, + "loss": 1.6018, + "theoretical_loss": 4.083576073005687, + "tokens_seen": 363790336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189889669007021, + "loss": 1.381, + "theoretical_loss": 4.083493045285158, + "tokens_seen": 363855872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189729187562688, + "loss": 1.4797, + "theoretical_loss": 4.08341003670426, + "tokens_seen": 363921408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189568706118355, + "loss": 1.477, + "theoretical_loss": 4.0833270472551355, + "tokens_seen": 363986944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189408224674022, + "loss": 1.6203, + "theoretical_loss": 4.083244076929932, + "tokens_seen": 364052480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189247743229689, + "loss": 1.669, + "theoretical_loss": 4.083161125720801, + "tokens_seen": 364118016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189087261785356, + "loss": 1.5458, + "theoretical_loss": 4.083078193619899, + "tokens_seen": 364183552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188926780341023, + "loss": 1.6065, + "theoretical_loss": 4.082995280619386, + "tokens_seen": 364249088 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718876629889669, + "loss": 1.4797, + "theoretical_loss": 4.0829123867114285, + "tokens_seen": 364314624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188605817452357, + "loss": 1.752, + "theoretical_loss": 4.082829511888197, + "tokens_seen": 364380160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188445336008024, + "loss": 1.5957, + "theoretical_loss": 4.082746656141865, + "tokens_seen": 364445696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188284854563691, + "loss": 1.6762, + "theoretical_loss": 4.082663819464613, + "tokens_seen": 364511232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188124373119358, + "loss": 1.6709, + "theoretical_loss": 4.082581001848624, + "tokens_seen": 364576768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187963891675025, + "loss": 1.4828, + "theoretical_loss": 4.082498203286088, + "tokens_seen": 364642304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187803410230692, + "loss": 1.5422, + "theoretical_loss": 4.082415423769197, + "tokens_seen": 364707840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187642928786359, + "loss": 1.4115, + "theoretical_loss": 4.082332663290149, + "tokens_seen": 364773376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187482447342026, + "loss": 1.5521, + "theoretical_loss": 4.082249921841147, + "tokens_seen": 364838912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187321965897693, + "loss": 1.5435, + "theoretical_loss": 4.082167199414398, + "tokens_seen": 364904448 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718716148445336, + "loss": 1.5592, + "theoretical_loss": 4.082084496002113, + "tokens_seen": 364969984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187001003009027, + "loss": 1.521, + "theoretical_loss": 4.082001811596509, + "tokens_seen": 365035520 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186840521564694, + "loss": 1.5971, + "theoretical_loss": 4.081919146189806, + "tokens_seen": 365101056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186680040120361, + "loss": 1.3846, + "theoretical_loss": 4.08183649977423, + "tokens_seen": 365166592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186519558676028, + "loss": 1.6659, + "theoretical_loss": 4.08175387234201, + "tokens_seen": 365232128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186359077231695, + "loss": 1.5476, + "theoretical_loss": 4.081671263885381, + "tokens_seen": 365297664 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186198595787362, + "loss": 1.501, + "theoretical_loss": 4.081588674396583, + "tokens_seen": 365363200 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718603811434303, + "loss": 1.5564, + "theoretical_loss": 4.081506103867858, + "tokens_seen": 365428736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185877632898696, + "loss": 1.6682, + "theoretical_loss": 4.081423552291456, + "tokens_seen": 365494272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185717151454363, + "loss": 1.5907, + "theoretical_loss": 4.081341019659628, + "tokens_seen": 365559808 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718555667001003, + "loss": 1.5072, + "theoretical_loss": 4.081258505964633, + "tokens_seen": 365625344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185396188565697, + "loss": 1.629, + "theoretical_loss": 4.081176011198731, + "tokens_seen": 365690880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185235707121364, + "loss": 1.6701, + "theoretical_loss": 4.08109353535419, + "tokens_seen": 365756416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185075225677032, + "loss": 1.6575, + "theoretical_loss": 4.0810110784232805, + "tokens_seen": 365821952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184914744232698, + "loss": 1.6648, + "theoretical_loss": 4.080928640398278, + "tokens_seen": 365887488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184754262788366, + "loss": 1.5334, + "theoretical_loss": 4.080846221271461, + "tokens_seen": 365953024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184593781344032, + "loss": 1.6324, + "theoretical_loss": 4.0807638210351165, + "tokens_seen": 366018560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00071844332998997, + "loss": 1.7469, + "theoretical_loss": 4.080681439681533, + "tokens_seen": 366084096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184272818455366, + "loss": 1.6183, + "theoretical_loss": 4.080599077203003, + "tokens_seen": 366149632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184112337011034, + "loss": 1.4391, + "theoretical_loss": 4.080516733591827, + "tokens_seen": 366215168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00071839518555667, + "loss": 1.578, + "theoretical_loss": 4.080434408840304, + "tokens_seen": 366280704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183791374122368, + "loss": 1.6214, + "theoretical_loss": 4.080352102940744, + "tokens_seen": 366346240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183630892678034, + "loss": 1.583, + "theoretical_loss": 4.080269815885458, + "tokens_seen": 366411776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183470411233702, + "loss": 1.6004, + "theoretical_loss": 4.080187547666763, + "tokens_seen": 366477312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183309929789368, + "loss": 1.5257, + "theoretical_loss": 4.080105298276978, + "tokens_seen": 366542848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183149448345036, + "loss": 1.4851, + "theoretical_loss": 4.08002306770843, + "tokens_seen": 366608384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182988966900703, + "loss": 1.554, + "theoretical_loss": 4.079940855953447, + "tokens_seen": 366673920 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718282848545637, + "loss": 1.5353, + "theoretical_loss": 4.079858663004364, + "tokens_seen": 366739456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182668004012037, + "loss": 1.5238, + "theoretical_loss": 4.079776488853521, + "tokens_seen": 366804992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182507522567704, + "loss": 1.5154, + "theoretical_loss": 4.079694333493259, + "tokens_seen": 366870528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182347041123371, + "loss": 1.6315, + "theoretical_loss": 4.079612196915927, + "tokens_seen": 366936064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 128594, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5575971603393555, + "objective/train/theoretical_loss": 4.079530079113876, + "objective/train/tokens_used": 387461600, + "theoretical_loss": 4.079530079113876, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182186559679038, + "loss": 1.559, + "theoretical_loss": 4.079530079113876, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182026078234705, + "loss": 1.6243, + "theoretical_loss": 4.079447980079465, + "tokens_seen": 367067136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181865596790372, + "loss": 1.5679, + "theoretical_loss": 4.079365899805053, + "tokens_seen": 367132672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181705115346039, + "loss": 1.5415, + "theoretical_loss": 4.079283838283007, + "tokens_seen": 367198208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181544633901706, + "loss": 1.63, + "theoretical_loss": 4.079201795505695, + "tokens_seen": 367263744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181384152457373, + "loss": 1.5294, + "theoretical_loss": 4.079119771465494, + "tokens_seen": 367329280 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718122367101304, + "loss": 1.5443, + "theoretical_loss": 4.079037766154781, + "tokens_seen": 367394816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181063189568707, + "loss": 1.6435, + "theoretical_loss": 4.078955779565941, + "tokens_seen": 367460352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180902708124374, + "loss": 1.4562, + "theoretical_loss": 4.078873811691361, + "tokens_seen": 367525888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180742226680041, + "loss": 1.6974, + "theoretical_loss": 4.078791862523433, + "tokens_seen": 367591424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180581745235708, + "loss": 1.5949, + "theoretical_loss": 4.078709932054554, + "tokens_seen": 367656960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180421263791375, + "loss": 1.5187, + "theoretical_loss": 4.078628020277128, + "tokens_seen": 367722496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180260782347042, + "loss": 1.547, + "theoretical_loss": 4.078546127183556, + "tokens_seen": 367788032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180100300902709, + "loss": 1.5075, + "theoretical_loss": 4.078464252766252, + "tokens_seen": 367853568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179939819458376, + "loss": 1.4956, + "theoretical_loss": 4.078382397017628, + "tokens_seen": 367919104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179779338014043, + "loss": 1.6077, + "theoretical_loss": 4.0783005599301045, + "tokens_seen": 367984640 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717961885656971, + "loss": 1.6948, + "theoretical_loss": 4.078218741496104, + "tokens_seen": 368050176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179458375125377, + "loss": 1.5008, + "theoretical_loss": 4.078136941708055, + "tokens_seen": 368115712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179297893681044, + "loss": 1.4755, + "theoretical_loss": 4.078055160558391, + "tokens_seen": 368181248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179137412236711, + "loss": 1.6208, + "theoretical_loss": 4.0779733980395445, + "tokens_seen": 368246784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178976930792378, + "loss": 1.5518, + "theoretical_loss": 4.077891654143961, + "tokens_seen": 368312320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178816449348045, + "loss": 1.746, + "theoretical_loss": 4.0778099288640846, + "tokens_seen": 368377856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178655967903712, + "loss": 1.4664, + "theoretical_loss": 4.0777282221923645, + "tokens_seen": 368443392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178495486459378, + "loss": 1.6716, + "theoretical_loss": 4.077646534121256, + "tokens_seen": 368508928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178335005015046, + "loss": 1.6311, + "theoretical_loss": 4.077564864643217, + "tokens_seen": 368574464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178174523570712, + "loss": 1.5638, + "theoretical_loss": 4.077483213750711, + "tokens_seen": 368640000 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717801404212638, + "loss": 1.6408, + "theoretical_loss": 4.077401581436206, + "tokens_seen": 368705536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177853560682046, + "loss": 1.5993, + "theoretical_loss": 4.077319967692173, + "tokens_seen": 368771072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177693079237714, + "loss": 1.5674, + "theoretical_loss": 4.077238372511088, + "tokens_seen": 368836608 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717753259779338, + "loss": 1.5769, + "theoretical_loss": 4.077156795885434, + "tokens_seen": 368902144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177372116349048, + "loss": 1.4508, + "theoretical_loss": 4.077075237807693, + "tokens_seen": 368967680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177211634904714, + "loss": 1.5182, + "theoretical_loss": 4.076993698270357, + "tokens_seen": 369033216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177051153460381, + "loss": 1.5911, + "theoretical_loss": 4.0769121772659185, + "tokens_seen": 369098752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176890672016048, + "loss": 1.5392, + "theoretical_loss": 4.076830674786876, + "tokens_seen": 369164288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176730190571715, + "loss": 1.5293, + "theoretical_loss": 4.076749190825732, + "tokens_seen": 369229824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176569709127382, + "loss": 1.5865, + "theoretical_loss": 4.076667725374993, + "tokens_seen": 369295360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176409227683049, + "loss": 1.5412, + "theoretical_loss": 4.076586278427171, + "tokens_seen": 369360896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176248746238716, + "loss": 1.4512, + "theoretical_loss": 4.076504849974781, + "tokens_seen": 369426432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176088264794383, + "loss": 1.6467, + "theoretical_loss": 4.076423440010343, + "tokens_seen": 369491968 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717592778335005, + "loss": 1.5657, + "theoretical_loss": 4.076342048526382, + "tokens_seen": 369557504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175767301905717, + "loss": 1.5009, + "theoretical_loss": 4.076260675515427, + "tokens_seen": 369623040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175606820461384, + "loss": 1.4412, + "theoretical_loss": 4.07617932097001, + "tokens_seen": 369688576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175446339017051, + "loss": 1.5958, + "theoretical_loss": 4.076097984882669, + "tokens_seen": 369754112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175285857572718, + "loss": 1.5223, + "theoretical_loss": 4.076016667245945, + "tokens_seen": 369819648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175125376128385, + "loss": 1.5297, + "theoretical_loss": 4.075935368052384, + "tokens_seen": 369885184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174964894684052, + "loss": 1.5543, + "theoretical_loss": 4.075854087294538, + "tokens_seen": 369950720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174804413239719, + "loss": 1.4428, + "theoretical_loss": 4.075772824964961, + "tokens_seen": 370016256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174643931795386, + "loss": 1.483, + "theoretical_loss": 4.07569158105621, + "tokens_seen": 370081792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174483450351053, + "loss": 1.5444, + "theoretical_loss": 4.075610355560851, + "tokens_seen": 370147328 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717432296890672, + "loss": 1.5556, + "theoretical_loss": 4.075529148471451, + "tokens_seen": 370212864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 129872, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5275757312774658, + "objective/train/theoretical_loss": 4.075447959780581, + "objective/train/tokens_used": 390738400, + "theoretical_loss": 4.075447959780581, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174162487462387, + "loss": 1.517, + "theoretical_loss": 4.075447959780581, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174002006018054, + "loss": 1.5282, + "theoretical_loss": 4.075366789480818, + "tokens_seen": 370343936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173841524573721, + "loss": 1.5652, + "theoretical_loss": 4.075285637564743, + "tokens_seen": 370409472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173681043129388, + "loss": 1.5969, + "theoretical_loss": 4.07520450402494, + "tokens_seen": 370475008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173520561685055, + "loss": 1.4542, + "theoretical_loss": 4.075123388854, + "tokens_seen": 370540544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173360080240722, + "loss": 1.619, + "theoretical_loss": 4.075042292044515, + "tokens_seen": 370606080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173199598796389, + "loss": 1.5211, + "theoretical_loss": 4.074961213589082, + "tokens_seen": 370671616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173039117352056, + "loss": 1.6464, + "theoretical_loss": 4.074880153480304, + "tokens_seen": 370737152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172878635907723, + "loss": 1.528, + "theoretical_loss": 4.074799111710789, + "tokens_seen": 370802688 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717271815446339, + "loss": 1.5398, + "theoretical_loss": 4.074718088273144, + "tokens_seen": 370868224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172557673019057, + "loss": 1.496, + "theoretical_loss": 4.074637083159988, + "tokens_seen": 370933760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172397191574724, + "loss": 1.4893, + "theoretical_loss": 4.0745560963639385, + "tokens_seen": 370999296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172236710130391, + "loss": 1.5709, + "theoretical_loss": 4.0744751278776175, + "tokens_seen": 371064832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172076228686058, + "loss": 1.6092, + "theoretical_loss": 4.074394177693655, + "tokens_seen": 371130368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171915747241725, + "loss": 1.464, + "theoretical_loss": 4.074313245804683, + "tokens_seen": 371195904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171755265797392, + "loss": 1.5461, + "theoretical_loss": 4.074232332203336, + "tokens_seen": 371261440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171594784353059, + "loss": 1.4682, + "theoretical_loss": 4.074151436882256, + "tokens_seen": 371326976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171434302908726, + "loss": 1.5951, + "theoretical_loss": 4.074070559834088, + "tokens_seen": 371392512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171273821464393, + "loss": 1.5379, + "theoretical_loss": 4.073989701051481, + "tokens_seen": 371458048 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717111334002006, + "loss": 1.5889, + "theoretical_loss": 4.073908860527087, + "tokens_seen": 371523584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170952858575728, + "loss": 1.5812, + "theoretical_loss": 4.0738280382535645, + "tokens_seen": 371589120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170792377131395, + "loss": 1.403, + "theoretical_loss": 4.0737472342235765, + "tokens_seen": 371654656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170631895687062, + "loss": 1.5452, + "theoretical_loss": 4.073666448429787, + "tokens_seen": 371720192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170471414242729, + "loss": 1.5192, + "theoretical_loss": 4.073585680864868, + "tokens_seen": 371785728 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170310932798396, + "loss": 1.4643, + "theoretical_loss": 4.073504931521493, + "tokens_seen": 371851264 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170150451354063, + "loss": 1.5878, + "theoretical_loss": 4.073424200392342, + "tokens_seen": 371916800 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716998996990973, + "loss": 1.6744, + "theoretical_loss": 4.073343487470097, + "tokens_seen": 371982336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169829488465397, + "loss": 1.5849, + "theoretical_loss": 4.073262792747445, + "tokens_seen": 372047872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169669007021064, + "loss": 1.5167, + "theoretical_loss": 4.073182116217079, + "tokens_seen": 372113408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169508525576731, + "loss": 1.5442, + "theoretical_loss": 4.073101457871693, + "tokens_seen": 372178944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169348044132398, + "loss": 1.5109, + "theoretical_loss": 4.073020817703988, + "tokens_seen": 372244480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169187562688065, + "loss": 1.5497, + "theoretical_loss": 4.072940195706668, + "tokens_seen": 372310016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169027081243732, + "loss": 1.5475, + "theoretical_loss": 4.072859591872441, + "tokens_seen": 372375552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168866599799399, + "loss": 1.5128, + "theoretical_loss": 4.07277900619402, + "tokens_seen": 372441088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168706118355066, + "loss": 1.5664, + "theoretical_loss": 4.072698438664121, + "tokens_seen": 372506624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168545636910733, + "loss": 1.5994, + "theoretical_loss": 4.072617889275467, + "tokens_seen": 372572160 + }, + { + "epoch": 0.11, + "learning_rate": 0.00071683851554664, + "loss": 1.5367, + "theoretical_loss": 4.07253735802078, + "tokens_seen": 372637696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168224674022067, + "loss": 1.595, + "theoretical_loss": 4.072456844892792, + "tokens_seen": 372703232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168064192577734, + "loss": 1.5945, + "theoretical_loss": 4.072376349884236, + "tokens_seen": 372768768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167903711133401, + "loss": 1.4935, + "theoretical_loss": 4.072295872987848, + "tokens_seen": 372834304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167743229689068, + "loss": 1.5228, + "theoretical_loss": 4.072215414196373, + "tokens_seen": 372899840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167582748244735, + "loss": 1.5967, + "theoretical_loss": 4.072134973502554, + "tokens_seen": 372965376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167422266800402, + "loss": 1.5616, + "theoretical_loss": 4.072054550899144, + "tokens_seen": 373030912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167261785356069, + "loss": 1.4757, + "theoretical_loss": 4.071974146378896, + "tokens_seen": 373096448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167101303911736, + "loss": 1.5998, + "theoretical_loss": 4.071893759934568, + "tokens_seen": 373161984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166940822467403, + "loss": 1.4561, + "theoretical_loss": 4.0718133915589245, + "tokens_seen": 373227520 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716678034102307, + "loss": 1.5062, + "theoretical_loss": 4.071733041244732, + "tokens_seen": 373293056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166619859578737, + "loss": 1.5552, + "theoretical_loss": 4.071652708984761, + "tokens_seen": 373358592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166459378134404, + "loss": 1.4439, + "theoretical_loss": 4.0715723947717875, + "tokens_seen": 373424128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166298896690071, + "loss": 1.5569, + "theoretical_loss": 4.07149209859859, + "tokens_seen": 373489664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 131043, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4238359928131104, + "objective/train/theoretical_loss": 4.0714118204579535, + "objective/train/tokens_used": 394015200, + "theoretical_loss": 4.0714118204579535, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166138415245738, + "loss": 1.432, + "theoretical_loss": 4.0714118204579535, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165977933801405, + "loss": 1.4711, + "theoretical_loss": 4.0713315603426645, + "tokens_seen": 373620736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165817452357072, + "loss": 1.7222, + "theoretical_loss": 4.071251318245516, + "tokens_seen": 373686272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165656970912739, + "loss": 1.4663, + "theoretical_loss": 4.071171094159304, + "tokens_seen": 373751808 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165496489468406, + "loss": 1.5914, + "theoretical_loss": 4.071090888076828, + "tokens_seen": 373817344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165336008024073, + "loss": 1.4531, + "theoretical_loss": 4.071010699990892, + "tokens_seen": 373882880 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716517552657974, + "loss": 1.5228, + "theoretical_loss": 4.070930529894306, + "tokens_seen": 373948416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165015045135407, + "loss": 1.6131, + "theoretical_loss": 4.070850377779882, + "tokens_seen": 374013952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164854563691074, + "loss": 1.578, + "theoretical_loss": 4.070770243640436, + "tokens_seen": 374079488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164694082246741, + "loss": 1.6162, + "theoretical_loss": 4.07069012746879, + "tokens_seen": 374145024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164533600802407, + "loss": 1.4642, + "theoretical_loss": 4.070610029257768, + "tokens_seen": 374210560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164373119358075, + "loss": 1.596, + "theoretical_loss": 4.0705299490002, + "tokens_seen": 374276096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164212637913741, + "loss": 1.5887, + "theoretical_loss": 4.070449886688919, + "tokens_seen": 374341632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164052156469409, + "loss": 1.466, + "theoretical_loss": 4.0703698423167625, + "tokens_seen": 374407168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163891675025075, + "loss": 1.55, + "theoretical_loss": 4.070289815876571, + "tokens_seen": 374472704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163731193580743, + "loss": 1.6113, + "theoretical_loss": 4.070209807361191, + "tokens_seen": 374538240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163570712136409, + "loss": 1.5242, + "theoretical_loss": 4.0701298167634725, + "tokens_seen": 374603776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163410230692077, + "loss": 1.5492, + "theoretical_loss": 4.0700498440762685, + "tokens_seen": 374669312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163249749247743, + "loss": 1.5498, + "theoretical_loss": 4.069969889292437, + "tokens_seen": 374734848 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716308926780341, + "loss": 1.6181, + "theoretical_loss": 4.069889952404841, + "tokens_seen": 374800384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162928786359077, + "loss": 1.4905, + "theoretical_loss": 4.069810033406345, + "tokens_seen": 374865920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162768304914744, + "loss": 1.6594, + "theoretical_loss": 4.06973013228982, + "tokens_seen": 374931456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162607823470411, + "loss": 1.59, + "theoretical_loss": 4.0696502490481405, + "tokens_seen": 374996992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162447342026078, + "loss": 1.5982, + "theoretical_loss": 4.069570383674185, + "tokens_seen": 375062528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162286860581745, + "loss": 1.5867, + "theoretical_loss": 4.069490536160835, + "tokens_seen": 375128064 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162126379137412, + "loss": 1.5181, + "theoretical_loss": 4.069410706500978, + "tokens_seen": 375193600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161965897693079, + "loss": 1.5475, + "theoretical_loss": 4.0693308946875035, + "tokens_seen": 375259136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161805416248746, + "loss": 1.5137, + "theoretical_loss": 4.069251100713307, + "tokens_seen": 375324672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161644934804413, + "loss": 1.558, + "theoretical_loss": 4.069171324571288, + "tokens_seen": 375390208 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716148445336008, + "loss": 1.5597, + "theoretical_loss": 4.069091566254347, + "tokens_seen": 375455744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161323971915747, + "loss": 1.5059, + "theoretical_loss": 4.069011825755393, + "tokens_seen": 375521280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161163490471414, + "loss": 1.4777, + "theoretical_loss": 4.068932103067335, + "tokens_seen": 375586816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161003009027081, + "loss": 1.5031, + "theoretical_loss": 4.06885239818309, + "tokens_seen": 375652352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160842527582748, + "loss": 1.5192, + "theoretical_loss": 4.068772711095575, + "tokens_seen": 375717888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160682046138415, + "loss": 1.6938, + "theoretical_loss": 4.068693041797715, + "tokens_seen": 375783424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160521564694082, + "loss": 1.6846, + "theoretical_loss": 4.068613390282437, + "tokens_seen": 375848960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160361083249749, + "loss": 1.6436, + "theoretical_loss": 4.06853375654267, + "tokens_seen": 375914496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160200601805416, + "loss": 1.4518, + "theoretical_loss": 4.068454140571351, + "tokens_seen": 375980032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160040120361083, + "loss": 1.6677, + "theoretical_loss": 4.068374542361418, + "tokens_seen": 376045568 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715987963891675, + "loss": 1.6108, + "theoretical_loss": 4.068294961905817, + "tokens_seen": 376111104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159719157472417, + "loss": 1.5951, + "theoretical_loss": 4.068215399197492, + "tokens_seen": 376176640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159558676028084, + "loss": 1.6626, + "theoretical_loss": 4.068135854229396, + "tokens_seen": 376242176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159398194583751, + "loss": 1.5851, + "theoretical_loss": 4.068056326994485, + "tokens_seen": 376307712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159237713139418, + "loss": 1.549, + "theoretical_loss": 4.067976817485717, + "tokens_seen": 376373248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159077231695085, + "loss": 1.534, + "theoretical_loss": 4.067897325696056, + "tokens_seen": 376438784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158916750250753, + "loss": 1.6247, + "theoretical_loss": 4.06781785161847, + "tokens_seen": 376504320 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715875626880642, + "loss": 1.574, + "theoretical_loss": 4.067738395245929, + "tokens_seen": 376569856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158595787362087, + "loss": 1.5068, + "theoretical_loss": 4.067658956571409, + "tokens_seen": 376635392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158435305917753, + "loss": 1.56, + "theoretical_loss": 4.067579535587891, + "tokens_seen": 376700928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158274824473421, + "loss": 1.5091, + "theoretical_loss": 4.067500132288357, + "tokens_seen": 376766464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 131935, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4308035373687744, + "objective/train/theoretical_loss": 4.067420746665794, + "objective/train/tokens_used": 397292000, + "theoretical_loss": 4.067420746665794, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158114343029087, + "loss": 1.5559, + "theoretical_loss": 4.067420746665794, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157953861584755, + "loss": 1.5028, + "theoretical_loss": 4.067341378713195, + "tokens_seen": 376897536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157793380140421, + "loss": 1.5996, + "theoretical_loss": 4.0672620284235546, + "tokens_seen": 376963072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157632898696089, + "loss": 1.5491, + "theoretical_loss": 4.067182695789872, + "tokens_seen": 377028608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157472417251756, + "loss": 1.5053, + "theoretical_loss": 4.06710338080515, + "tokens_seen": 377094144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157311935807423, + "loss": 1.5987, + "theoretical_loss": 4.067024083462399, + "tokens_seen": 377159680 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715715145436309, + "loss": 1.4151, + "theoretical_loss": 4.066944803754628, + "tokens_seen": 377225216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156990972918757, + "loss": 1.4533, + "theoretical_loss": 4.0668655416748525, + "tokens_seen": 377290752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156830491474424, + "loss": 1.5566, + "theoretical_loss": 4.066786297216092, + "tokens_seen": 377356288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156670010030091, + "loss": 1.4728, + "theoretical_loss": 4.066707070371371, + "tokens_seen": 377421824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156509528585758, + "loss": 1.5469, + "theoretical_loss": 4.066627861133716, + "tokens_seen": 377487360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156349047141425, + "loss": 1.4933, + "theoretical_loss": 4.066548669496157, + "tokens_seen": 377552896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156188565697092, + "loss": 1.5501, + "theoretical_loss": 4.066469495451732, + "tokens_seen": 377618432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156028084252759, + "loss": 1.6296, + "theoretical_loss": 4.066390338993479, + "tokens_seen": 377683968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155867602808426, + "loss": 1.6332, + "theoretical_loss": 4.0663112001144395, + "tokens_seen": 377749504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155707121364093, + "loss": 1.6292, + "theoretical_loss": 4.066232078807664, + "tokens_seen": 377815040 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715554663991976, + "loss": 1.5593, + "theoretical_loss": 4.0661529750662, + "tokens_seen": 377880576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155386158475427, + "loss": 1.5156, + "theoretical_loss": 4.066073888883105, + "tokens_seen": 377946112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155225677031094, + "loss": 1.5514, + "theoretical_loss": 4.065994820251438, + "tokens_seen": 378011648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155065195586761, + "loss": 1.4833, + "theoretical_loss": 4.065915769164262, + "tokens_seen": 378077184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154904714142428, + "loss": 1.6372, + "theoretical_loss": 4.065836735614642, + "tokens_seen": 378142720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154744232698095, + "loss": 1.6646, + "theoretical_loss": 4.06575771959565, + "tokens_seen": 378208256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154583751253762, + "loss": 1.6973, + "theoretical_loss": 4.065678721100362, + "tokens_seen": 378273792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154423269809429, + "loss": 1.6011, + "theoretical_loss": 4.065599740121855, + "tokens_seen": 378339328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154262788365096, + "loss": 1.5382, + "theoretical_loss": 4.065520776653212, + "tokens_seen": 378404864 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154102306920763, + "loss": 1.5479, + "theoretical_loss": 4.06544183068752, + "tokens_seen": 378470400 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715394182547643, + "loss": 1.5046, + "theoretical_loss": 4.0653629022178706, + "tokens_seen": 378535936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153781344032097, + "loss": 1.502, + "theoretical_loss": 4.065283991237355, + "tokens_seen": 378601472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153620862587764, + "loss": 1.553, + "theoretical_loss": 4.065205097739075, + "tokens_seen": 378667008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153460381143431, + "loss": 1.5149, + "theoretical_loss": 4.065126221716131, + "tokens_seen": 378732544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153299899699098, + "loss": 1.5736, + "theoretical_loss": 4.0650473631616295, + "tokens_seen": 378798080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153139418254765, + "loss": 1.5831, + "theoretical_loss": 4.064968522068681, + "tokens_seen": 378863616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152978936810432, + "loss": 1.6488, + "theoretical_loss": 4.0648896984304, + "tokens_seen": 378929152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152818455366099, + "loss": 1.5114, + "theoretical_loss": 4.064810892239903, + "tokens_seen": 378994688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152657973921766, + "loss": 1.4931, + "theoretical_loss": 4.064732103490312, + "tokens_seen": 379060224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152497492477433, + "loss": 1.5598, + "theoretical_loss": 4.064653332174753, + "tokens_seen": 379125760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00071523370110331, + "loss": 1.6029, + "theoretical_loss": 4.064574578286356, + "tokens_seen": 379191296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152176529588767, + "loss": 1.6075, + "theoretical_loss": 4.064495841818255, + "tokens_seen": 379256832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152016048144434, + "loss": 1.5738, + "theoretical_loss": 4.064417122763587, + "tokens_seen": 379322368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007151855566700101, + "loss": 1.5304, + "theoretical_loss": 4.064338421115491, + "tokens_seen": 379387904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007151695085255768, + "loss": 1.5089, + "theoretical_loss": 4.064259736867116, + "tokens_seen": 379453440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007151534603811435, + "loss": 1.5295, + "theoretical_loss": 4.0641810700116086, + "tokens_seen": 379518976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007151374122367102, + "loss": 1.553, + "theoretical_loss": 4.064102420542122, + "tokens_seen": 379584512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007151213640922769, + "loss": 1.5785, + "theoretical_loss": 4.064023788451814, + "tokens_seen": 379650048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007151053159478436, + "loss": 1.5725, + "theoretical_loss": 4.063945173733844, + "tokens_seen": 379715584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007150892678034103, + "loss": 1.3953, + "theoretical_loss": 4.063866576381377, + "tokens_seen": 379781120 + }, + { + "epoch": 0.12, + "learning_rate": 0.000715073219658977, + "loss": 1.4806, + "theoretical_loss": 4.063787996387583, + "tokens_seen": 379846656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007150571715145436, + "loss": 1.5389, + "theoretical_loss": 4.063709433745632, + "tokens_seen": 379912192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007150411233701104, + "loss": 1.5711, + "theoretical_loss": 4.063630888448701, + "tokens_seen": 379977728 + }, + { + "epoch": 0.12, + "learning_rate": 0.000715025075225677, + "loss": 1.6166, + "theoretical_loss": 4.06355236048997, + "tokens_seen": 380043264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 133175, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.637108325958252, + "objective/train/theoretical_loss": 4.063473849862623, + "objective/train/tokens_used": 400568800, + "theoretical_loss": 4.063473849862623, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007150090270812438, + "loss": 1.6212, + "theoretical_loss": 4.063473849862623, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149929789368104, + "loss": 1.4958, + "theoretical_loss": 4.063395356559848, + "tokens_seen": 380174336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149769307923772, + "loss": 1.5352, + "theoretical_loss": 4.063316880574836, + "tokens_seen": 380239872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149608826479438, + "loss": 1.5525, + "theoretical_loss": 4.063238421900783, + "tokens_seen": 380305408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149448345035106, + "loss": 1.6395, + "theoretical_loss": 4.063159980530887, + "tokens_seen": 380370944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149287863590772, + "loss": 1.5751, + "theoretical_loss": 4.063081556458353, + "tokens_seen": 380436480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149127382146439, + "loss": 1.5307, + "theoretical_loss": 4.063003149676387, + "tokens_seen": 380502016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148966900702106, + "loss": 1.6268, + "theoretical_loss": 4.062924760178198, + "tokens_seen": 380567552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148806419257773, + "loss": 1.4769, + "theoretical_loss": 4.062846387957005, + "tokens_seen": 380633088 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714864593781344, + "loss": 1.4427, + "theoretical_loss": 4.062768033006023, + "tokens_seen": 380698624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148485456369107, + "loss": 1.5785, + "theoretical_loss": 4.062689695318475, + "tokens_seen": 380764160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148324974924774, + "loss": 1.7042, + "theoretical_loss": 4.062611374887587, + "tokens_seen": 380829696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148164493480441, + "loss": 1.4121, + "theoretical_loss": 4.062533071706589, + "tokens_seen": 380895232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148004012036108, + "loss": 1.529, + "theoretical_loss": 4.062454785768716, + "tokens_seen": 380960768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147843530591775, + "loss": 1.4973, + "theoretical_loss": 4.062376517067204, + "tokens_seen": 381026304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147683049147442, + "loss": 1.5547, + "theoretical_loss": 4.062298265595294, + "tokens_seen": 381091840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147522567703109, + "loss": 1.4552, + "theoretical_loss": 4.062220031346234, + "tokens_seen": 381157376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147362086258776, + "loss": 1.5088, + "theoretical_loss": 4.06214181431327, + "tokens_seen": 381222912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147201604814443, + "loss": 1.5998, + "theoretical_loss": 4.062063614489656, + "tokens_seen": 381288448 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714704112337011, + "loss": 1.3875, + "theoretical_loss": 4.061985431868648, + "tokens_seen": 381353984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146880641925778, + "loss": 1.5849, + "theoretical_loss": 4.061907266443508, + "tokens_seen": 381419520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146720160481444, + "loss": 1.5982, + "theoretical_loss": 4.061829118207498, + "tokens_seen": 381485056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146559679037112, + "loss": 1.5771, + "theoretical_loss": 4.061750987153888, + "tokens_seen": 381550592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146399197592778, + "loss": 1.6583, + "theoretical_loss": 4.061672873275949, + "tokens_seen": 381616128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146238716148446, + "loss": 1.5146, + "theoretical_loss": 4.061594776566958, + "tokens_seen": 381681664 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146078234704112, + "loss": 1.6203, + "theoretical_loss": 4.061516697020191, + "tokens_seen": 381747200 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714591775325978, + "loss": 1.4899, + "theoretical_loss": 4.061438634628933, + "tokens_seen": 381812736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007145757271815446, + "loss": 1.4785, + "theoretical_loss": 4.061360589386473, + "tokens_seen": 381878272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007145596790371114, + "loss": 1.5746, + "theoretical_loss": 4.061282561286098, + "tokens_seen": 381943808 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714543630892678, + "loss": 1.4623, + "theoretical_loss": 4.061204550321104, + "tokens_seen": 382009344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007145275827482448, + "loss": 1.4579, + "theoretical_loss": 4.061126556484791, + "tokens_seen": 382074880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007145115346038114, + "loss": 1.7142, + "theoretical_loss": 4.06104857977046, + "tokens_seen": 382140416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144954864593782, + "loss": 1.4523, + "theoretical_loss": 4.060970620171415, + "tokens_seen": 382205952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144794383149448, + "loss": 1.569, + "theoretical_loss": 4.060892677680967, + "tokens_seen": 382271488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144633901705116, + "loss": 1.56, + "theoretical_loss": 4.06081475229243, + "tokens_seen": 382337024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144473420260782, + "loss": 1.595, + "theoretical_loss": 4.060736843999121, + "tokens_seen": 382402560 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714431293881645, + "loss": 1.4555, + "theoretical_loss": 4.060658952794359, + "tokens_seen": 382468096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144152457372116, + "loss": 1.5569, + "theoretical_loss": 4.06058107867147, + "tokens_seen": 382533632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143991975927784, + "loss": 1.5, + "theoretical_loss": 4.060503221623783, + "tokens_seen": 382599168 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714383149448345, + "loss": 1.5665, + "theoretical_loss": 4.060425381644629, + "tokens_seen": 382664704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143671013039118, + "loss": 1.5303, + "theoretical_loss": 4.060347558727344, + "tokens_seen": 382730240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143510531594785, + "loss": 1.5049, + "theoretical_loss": 4.060269752865268, + "tokens_seen": 382795776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143350050150452, + "loss": 1.5552, + "theoretical_loss": 4.060191964051744, + "tokens_seen": 382861312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143189568706119, + "loss": 1.3659, + "theoretical_loss": 4.060114192280119, + "tokens_seen": 382926848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143029087261786, + "loss": 1.5707, + "theoretical_loss": 4.0600364375437445, + "tokens_seen": 382992384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142868605817453, + "loss": 1.644, + "theoretical_loss": 4.059958699835974, + "tokens_seen": 383057920 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714270812437312, + "loss": 1.5578, + "theoretical_loss": 4.059880979150167, + "tokens_seen": 383123456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142547642928787, + "loss": 1.4528, + "theoretical_loss": 4.059803275479684, + "tokens_seen": 383188992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142387161484454, + "loss": 1.5817, + "theoretical_loss": 4.059725588817891, + "tokens_seen": 383254528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142226680040121, + "loss": 1.4266, + "theoretical_loss": 4.0596479191581585, + "tokens_seen": 383320064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 134130, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6504217386245728, + "objective/train/theoretical_loss": 4.059570266493859, + "objective/train/tokens_used": 403845600, + "theoretical_loss": 4.059570266493859, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142066198595788, + "loss": 1.6386, + "theoretical_loss": 4.059570266493859, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141905717151455, + "loss": 1.5463, + "theoretical_loss": 4.059492630818369, + "tokens_seen": 383451136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141745235707122, + "loss": 1.6065, + "theoretical_loss": 4.05941501212507, + "tokens_seen": 383516672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141584754262789, + "loss": 1.4486, + "theoretical_loss": 4.059337410407346, + "tokens_seen": 383582208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141424272818456, + "loss": 1.5162, + "theoretical_loss": 4.059259825658584, + "tokens_seen": 383647744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141263791374123, + "loss": 1.5191, + "theoretical_loss": 4.059182257872177, + "tokens_seen": 383713280 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714110330992979, + "loss": 1.6177, + "theoretical_loss": 4.059104707041519, + "tokens_seen": 383778816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140942828485457, + "loss": 1.4974, + "theoretical_loss": 4.059027173160011, + "tokens_seen": 383844352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140782347041124, + "loss": 1.5206, + "theoretical_loss": 4.0589496562210545, + "tokens_seen": 383909888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140621865596791, + "loss": 1.5041, + "theoretical_loss": 4.058872156218055, + "tokens_seen": 383975424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140461384152458, + "loss": 1.4481, + "theoretical_loss": 4.058794673144425, + "tokens_seen": 384040960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140300902708125, + "loss": 1.4762, + "theoretical_loss": 4.0587172069935775, + "tokens_seen": 384106496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140140421263792, + "loss": 1.5379, + "theoretical_loss": 4.05863975775893, + "tokens_seen": 384172032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139979939819459, + "loss": 1.6316, + "theoretical_loss": 4.058562325433902, + "tokens_seen": 384237568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139819458375126, + "loss": 1.4715, + "theoretical_loss": 4.058484910011922, + "tokens_seen": 384303104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139658976930793, + "loss": 1.535, + "theoretical_loss": 4.058407511486415, + "tokens_seen": 384368640 + }, + { + "epoch": 0.12, + "learning_rate": 0.000713949849548646, + "loss": 1.591, + "theoretical_loss": 4.058330129850816, + "tokens_seen": 384434176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139338014042127, + "loss": 1.4796, + "theoretical_loss": 4.058252765098558, + "tokens_seen": 384499712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139177532597794, + "loss": 1.3905, + "theoretical_loss": 4.058175417223084, + "tokens_seen": 384565248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139017051153461, + "loss": 1.6414, + "theoretical_loss": 4.058098086217836, + "tokens_seen": 384630784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138856569709128, + "loss": 1.4973, + "theoretical_loss": 4.058020772076259, + "tokens_seen": 384696320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138696088264795, + "loss": 1.6776, + "theoretical_loss": 4.057943474791806, + "tokens_seen": 384761856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138535606820462, + "loss": 1.4589, + "theoretical_loss": 4.0578661943579295, + "tokens_seen": 384827392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138375125376129, + "loss": 1.5205, + "theoretical_loss": 4.05778893076809, + "tokens_seen": 384892928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138214643931796, + "loss": 1.6148, + "theoretical_loss": 4.057711684015747, + "tokens_seen": 384958464 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138054162487463, + "loss": 1.4956, + "theoretical_loss": 4.057634454094365, + "tokens_seen": 385024000 + }, + { + "epoch": 0.12, + "learning_rate": 0.000713789368104313, + "loss": 1.5085, + "theoretical_loss": 4.057557240997415, + "tokens_seen": 385089536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137733199598797, + "loss": 1.4795, + "theoretical_loss": 4.057480044718368, + "tokens_seen": 385155072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137572718154464, + "loss": 1.533, + "theoretical_loss": 4.057402865250701, + "tokens_seen": 385220608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137412236710131, + "loss": 1.5496, + "theoretical_loss": 4.057325702587893, + "tokens_seen": 385286144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137251755265798, + "loss": 1.6184, + "theoretical_loss": 4.057248556723429, + "tokens_seen": 385351680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137091273821465, + "loss": 1.5929, + "theoretical_loss": 4.057171427650793, + "tokens_seen": 385417216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136930792377132, + "loss": 1.5673, + "theoretical_loss": 4.0570943153634795, + "tokens_seen": 385482752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136770310932799, + "loss": 1.4974, + "theoretical_loss": 4.0570172198549805, + "tokens_seen": 385548288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136609829488466, + "loss": 1.5252, + "theoretical_loss": 4.056940141118794, + "tokens_seen": 385613824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136449348044133, + "loss": 1.5163, + "theoretical_loss": 4.056863079148422, + "tokens_seen": 385679360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136288866599799, + "loss": 1.5341, + "theoretical_loss": 4.05678603393737, + "tokens_seen": 385744896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136128385155467, + "loss": 1.5027, + "theoretical_loss": 4.056709005479147, + "tokens_seen": 385810432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135967903711133, + "loss": 1.4236, + "theoretical_loss": 4.056631993767264, + "tokens_seen": 385875968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135807422266801, + "loss": 1.6033, + "theoretical_loss": 4.056554998795239, + "tokens_seen": 385941504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135646940822467, + "loss": 1.5528, + "theoretical_loss": 4.05647802055659, + "tokens_seen": 386007040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135486459378135, + "loss": 1.6238, + "theoretical_loss": 4.056401059044841, + "tokens_seen": 386072576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135325977933801, + "loss": 1.5508, + "theoretical_loss": 4.056324114253519, + "tokens_seen": 386138112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135165496489468, + "loss": 1.4138, + "theoretical_loss": 4.056247186176154, + "tokens_seen": 386203648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135005015045135, + "loss": 1.5629, + "theoretical_loss": 4.056170274806281, + "tokens_seen": 386269184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134844533600803, + "loss": 1.5687, + "theoretical_loss": 4.056093380137436, + "tokens_seen": 386334720 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134684052156469, + "loss": 1.5048, + "theoretical_loss": 4.056016502163162, + "tokens_seen": 386400256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134523570712137, + "loss": 1.6054, + "theoretical_loss": 4.055939640877003, + "tokens_seen": 386465792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134363089267803, + "loss": 1.5434, + "theoretical_loss": 4.055862796272508, + "tokens_seen": 386531328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134202607823471, + "loss": 1.4143, + "theoretical_loss": 4.055785968343228, + "tokens_seen": 386596864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 135378, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6471995115280151, + "objective/train/theoretical_loss": 4.05570915708272, + "objective/train/tokens_used": 407122400, + "theoretical_loss": 4.05570915708272, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134042126379137, + "loss": 1.5954, + "theoretical_loss": 4.05570915708272, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133881644934805, + "loss": 1.367, + "theoretical_loss": 4.055632362484542, + "tokens_seen": 386727936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133721163490471, + "loss": 1.5976, + "theoretical_loss": 4.055555584542257, + "tokens_seen": 386793472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133560682046139, + "loss": 1.4761, + "theoretical_loss": 4.055478823249432, + "tokens_seen": 386859008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133400200601805, + "loss": 1.4676, + "theoretical_loss": 4.055402078599638, + "tokens_seen": 386924544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133239719157473, + "loss": 1.4819, + "theoretical_loss": 4.055325350586445, + "tokens_seen": 386990080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133079237713139, + "loss": 1.3562, + "theoretical_loss": 4.055248639203433, + "tokens_seen": 387055616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132918756268807, + "loss": 1.561, + "theoretical_loss": 4.055171944444182, + "tokens_seen": 387121152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132758274824473, + "loss": 1.5905, + "theoretical_loss": 4.055095266302276, + "tokens_seen": 387186688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132597793380141, + "loss": 1.5024, + "theoretical_loss": 4.055018604771302, + "tokens_seen": 387252224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132437311935807, + "loss": 1.5481, + "theoretical_loss": 4.054941959844853, + "tokens_seen": 387317760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132276830491475, + "loss": 1.5958, + "theoretical_loss": 4.054865331516523, + "tokens_seen": 387383296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132116349047141, + "loss": 1.4962, + "theoretical_loss": 4.054788719779911, + "tokens_seen": 387448832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131955867602809, + "loss": 1.6014, + "theoretical_loss": 4.054712124628617, + "tokens_seen": 387514368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131795386158475, + "loss": 1.569, + "theoretical_loss": 4.054635546056248, + "tokens_seen": 387579904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131634904714143, + "loss": 1.4515, + "theoretical_loss": 4.054558984056414, + "tokens_seen": 387645440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131474423269809, + "loss": 1.509, + "theoretical_loss": 4.054482438622727, + "tokens_seen": 387710976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131313941825477, + "loss": 1.3481, + "theoretical_loss": 4.0544059097488026, + "tokens_seen": 387776512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131153460381143, + "loss": 1.5887, + "theoretical_loss": 4.05432939742826, + "tokens_seen": 387842048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130992978936811, + "loss": 1.5527, + "theoretical_loss": 4.054252901654724, + "tokens_seen": 387907584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130832497492477, + "loss": 1.5288, + "theoretical_loss": 4.054176422421821, + "tokens_seen": 387973120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130672016048145, + "loss": 1.575, + "theoretical_loss": 4.05409995972318, + "tokens_seen": 388038656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130511534603811, + "loss": 1.54, + "theoretical_loss": 4.054023513552437, + "tokens_seen": 388104192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130351053159479, + "loss": 1.476, + "theoretical_loss": 4.0539470839032266, + "tokens_seen": 388169728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130190571715146, + "loss": 1.5659, + "theoretical_loss": 4.053870670769192, + "tokens_seen": 388235264 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130030090270813, + "loss": 1.5637, + "theoretical_loss": 4.053794274143977, + "tokens_seen": 388300800 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712986960882648, + "loss": 1.5868, + "theoretical_loss": 4.053717894021229, + "tokens_seen": 388366336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129709127382147, + "loss": 1.4002, + "theoretical_loss": 4.0536415303946, + "tokens_seen": 388431872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129548645937814, + "loss": 1.591, + "theoretical_loss": 4.053565183257745, + "tokens_seen": 388497408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129388164493481, + "loss": 1.5506, + "theoretical_loss": 4.053488852604322, + "tokens_seen": 388562944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129227683049148, + "loss": 1.483, + "theoretical_loss": 4.053412538427993, + "tokens_seen": 388628480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129067201604815, + "loss": 1.6479, + "theoretical_loss": 4.053336240722423, + "tokens_seen": 388694016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128906720160482, + "loss": 1.4564, + "theoretical_loss": 4.053259959481283, + "tokens_seen": 388759552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128746238716149, + "loss": 1.532, + "theoretical_loss": 4.053183694698244, + "tokens_seen": 388825088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128585757271816, + "loss": 1.4543, + "theoretical_loss": 4.053107446366981, + "tokens_seen": 388890624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128425275827483, + "loss": 1.7257, + "theoretical_loss": 4.053031214481175, + "tokens_seen": 388956160 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712826479438315, + "loss": 1.5432, + "theoretical_loss": 4.052954999034509, + "tokens_seen": 389021696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128104312938817, + "loss": 1.3693, + "theoretical_loss": 4.052878800020668, + "tokens_seen": 389087232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127943831494484, + "loss": 1.5255, + "theoretical_loss": 4.052802617433344, + "tokens_seen": 389152768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127783350050151, + "loss": 1.5014, + "theoretical_loss": 4.0527264512662295, + "tokens_seen": 389218304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127622868605818, + "loss": 1.5613, + "theoretical_loss": 4.052650301513021, + "tokens_seen": 389283840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127462387161485, + "loss": 1.4304, + "theoretical_loss": 4.0525741681674186, + "tokens_seen": 389349376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127301905717152, + "loss": 1.4755, + "theoretical_loss": 4.052498051223127, + "tokens_seen": 389414912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127141424272819, + "loss": 1.7319, + "theoretical_loss": 4.052421950673853, + "tokens_seen": 389480448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126980942828486, + "loss": 1.6418, + "theoretical_loss": 4.052345866513308, + "tokens_seen": 389545984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126820461384153, + "loss": 1.4938, + "theoretical_loss": 4.052269798735207, + "tokens_seen": 389611520 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712665997993982, + "loss": 1.4466, + "theoretical_loss": 4.052193747333265, + "tokens_seen": 389677056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126499498495487, + "loss": 1.6005, + "theoretical_loss": 4.052117712301206, + "tokens_seen": 389742592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126339017051154, + "loss": 1.4522, + "theoretical_loss": 4.052041693632754, + "tokens_seen": 389808128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126178535606821, + "loss": 1.5045, + "theoretical_loss": 4.0519656913216355, + "tokens_seen": 389873664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 136223, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4804840087890625, + "objective/train/theoretical_loss": 4.051889705361584, + "objective/train/tokens_used": 410399200, + "theoretical_loss": 4.051889705361584, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126018054162488, + "loss": 1.5287, + "theoretical_loss": 4.051889705361584, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125857572718155, + "loss": 1.4526, + "theoretical_loss": 4.051813735746334, + "tokens_seen": 390004736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125697091273822, + "loss": 1.6437, + "theoretical_loss": 4.051737782469624, + "tokens_seen": 390070272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125536609829489, + "loss": 1.6544, + "theoretical_loss": 4.051661845525196, + "tokens_seen": 390135808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125376128385156, + "loss": 1.4723, + "theoretical_loss": 4.051585924906796, + "tokens_seen": 390201344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125215646940823, + "loss": 1.7279, + "theoretical_loss": 4.051510020608171, + "tokens_seen": 390266880 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712505516549649, + "loss": 1.5472, + "theoretical_loss": 4.051434132623075, + "tokens_seen": 390332416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124894684052157, + "loss": 1.699, + "theoretical_loss": 4.051358260945262, + "tokens_seen": 390397952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124734202607824, + "loss": 1.6237, + "theoretical_loss": 4.051282405568494, + "tokens_seen": 390463488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124573721163491, + "loss": 1.5859, + "theoretical_loss": 4.051206566486531, + "tokens_seen": 390529024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124413239719158, + "loss": 1.5356, + "theoretical_loss": 4.0511307436931405, + "tokens_seen": 390594560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124252758274825, + "loss": 1.558, + "theoretical_loss": 4.051054937182092, + "tokens_seen": 390660096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124092276830492, + "loss": 1.5667, + "theoretical_loss": 4.0509791469471566, + "tokens_seen": 390725632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123931795386159, + "loss": 1.5082, + "theoretical_loss": 4.050903372982113, + "tokens_seen": 390791168 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123771313941826, + "loss": 1.6031, + "theoretical_loss": 4.050827615280739, + "tokens_seen": 390856704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123610832497493, + "loss": 1.5781, + "theoretical_loss": 4.050751873836819, + "tokens_seen": 390922240 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712345035105316, + "loss": 1.4028, + "theoretical_loss": 4.050676148644139, + "tokens_seen": 390987776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123289869608827, + "loss": 1.6489, + "theoretical_loss": 4.050600439696489, + "tokens_seen": 391053312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123129388164494, + "loss": 1.473, + "theoretical_loss": 4.050524746987664, + "tokens_seen": 391118848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122968906720161, + "loss": 1.5548, + "theoretical_loss": 4.050449070511458, + "tokens_seen": 391184384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122808425275828, + "loss": 1.5028, + "theoretical_loss": 4.050373410261673, + "tokens_seen": 391249920 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122647943831495, + "loss": 1.5715, + "theoretical_loss": 4.050297766232113, + "tokens_seen": 391315456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122487462387162, + "loss": 1.7176, + "theoretical_loss": 4.050222138416583, + "tokens_seen": 391380992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122326980942828, + "loss": 1.5573, + "theoretical_loss": 4.050146526808896, + "tokens_seen": 391446528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122166499498496, + "loss": 1.6053, + "theoretical_loss": 4.050070931402864, + "tokens_seen": 391512064 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122006018054162, + "loss": 1.4945, + "theoretical_loss": 4.049995352192305, + "tokens_seen": 391577600 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712184553660983, + "loss": 1.6051, + "theoretical_loss": 4.04991978917104, + "tokens_seen": 391643136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007121685055165496, + "loss": 1.6117, + "theoretical_loss": 4.049844242332892, + "tokens_seen": 391708672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007121524573721164, + "loss": 1.5436, + "theoretical_loss": 4.049768711671689, + "tokens_seen": 391774208 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712136409227683, + "loss": 1.5156, + "theoretical_loss": 4.049693197181262, + "tokens_seen": 391839744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007121203610832498, + "loss": 1.5343, + "theoretical_loss": 4.049617698855444, + "tokens_seen": 391905280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007121043129388164, + "loss": 1.5006, + "theoretical_loss": 4.049542216688074, + "tokens_seen": 391970816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120882647943832, + "loss": 1.5838, + "theoretical_loss": 4.049466750672994, + "tokens_seen": 392036352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120722166499498, + "loss": 1.6256, + "theoretical_loss": 4.049391300804045, + "tokens_seen": 392101888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120561685055166, + "loss": 1.65, + "theoretical_loss": 4.049315867075077, + "tokens_seen": 392167424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120401203610832, + "loss": 1.4785, + "theoretical_loss": 4.0492404494799406, + "tokens_seen": 392232960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00071202407221665, + "loss": 1.5928, + "theoretical_loss": 4.049165048012489, + "tokens_seen": 392298496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120080240722166, + "loss": 1.4771, + "theoretical_loss": 4.049089662666582, + "tokens_seen": 392364032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119919759277834, + "loss": 1.4962, + "theoretical_loss": 4.04901429343608, + "tokens_seen": 392429568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00071197592778335, + "loss": 1.5464, + "theoretical_loss": 4.048938940314848, + "tokens_seen": 392495104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119598796389168, + "loss": 1.439, + "theoretical_loss": 4.048863603296753, + "tokens_seen": 392560640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119438314944834, + "loss": 1.4998, + "theoretical_loss": 4.048788282375666, + "tokens_seen": 392626176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119277833500502, + "loss": 1.5768, + "theoretical_loss": 4.048712977545463, + "tokens_seen": 392691712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119117352056168, + "loss": 1.4402, + "theoretical_loss": 4.048637688800021, + "tokens_seen": 392757248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007118956870611836, + "loss": 1.5783, + "theoretical_loss": 4.048562416133221, + "tokens_seen": 392822784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007118796389167502, + "loss": 1.5998, + "theoretical_loss": 4.048487159538949, + "tokens_seen": 392888320 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711863590772317, + "loss": 1.5081, + "theoretical_loss": 4.048411919011091, + "tokens_seen": 392953856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007118475426278836, + "loss": 1.5313, + "theoretical_loss": 4.04833669454354, + "tokens_seen": 393019392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007118314944834504, + "loss": 1.5179, + "theoretical_loss": 4.0482614861301895, + "tokens_seen": 393084928 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711815446339017, + "loss": 1.5538, + "theoretical_loss": 4.048186293764939, + "tokens_seen": 393150464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 137415, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5672664642333984, + "objective/train/theoretical_loss": 4.048111117441689, + "objective/train/tokens_used": 413676000, + "theoretical_loss": 4.048111117441689, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117993981945838, + "loss": 1.6009, + "theoretical_loss": 4.048111117441689, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117833500501504, + "loss": 1.6339, + "theoretical_loss": 4.048035957154343, + "tokens_seen": 393281536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117673019057172, + "loss": 1.6105, + "theoretical_loss": 4.047960812896811, + "tokens_seen": 393347072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117512537612838, + "loss": 1.6307, + "theoretical_loss": 4.047885684663004, + "tokens_seen": 393412608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117352056168506, + "loss": 1.5179, + "theoretical_loss": 4.047810572446836, + "tokens_seen": 393478144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117191574724172, + "loss": 1.617, + "theoretical_loss": 4.047735476242225, + "tokens_seen": 393543680 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711703109327984, + "loss": 1.6172, + "theoretical_loss": 4.047660396043092, + "tokens_seen": 393609216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116870611835506, + "loss": 1.4355, + "theoretical_loss": 4.047585331843363, + "tokens_seen": 393674752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116710130391174, + "loss": 1.5347, + "theoretical_loss": 4.047510283636965, + "tokens_seen": 393740288 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711654964894684, + "loss": 1.4901, + "theoretical_loss": 4.04743525141783, + "tokens_seen": 393805824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116389167502508, + "loss": 1.5138, + "theoretical_loss": 4.047360235179891, + "tokens_seen": 393871360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116228686058175, + "loss": 1.5696, + "theoretical_loss": 4.047285234917087, + "tokens_seen": 393936896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116068204613842, + "loss": 1.4289, + "theoretical_loss": 4.04721025062336, + "tokens_seen": 394002432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115907723169509, + "loss": 1.5431, + "theoretical_loss": 4.047135282292653, + "tokens_seen": 394067968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115747241725176, + "loss": 1.5128, + "theoretical_loss": 4.0470603299189145, + "tokens_seen": 394133504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115586760280843, + "loss": 1.5557, + "theoretical_loss": 4.0469853934960955, + "tokens_seen": 394199040 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711542627883651, + "loss": 1.5148, + "theoretical_loss": 4.046910473018151, + "tokens_seen": 394264576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115265797392177, + "loss": 1.6591, + "theoretical_loss": 4.046835568479038, + "tokens_seen": 394330112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115105315947844, + "loss": 1.4581, + "theoretical_loss": 4.046760679872717, + "tokens_seen": 394395648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114944834503511, + "loss": 1.4585, + "theoretical_loss": 4.046685807193153, + "tokens_seen": 394461184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114784353059178, + "loss": 1.5061, + "theoretical_loss": 4.046610950434314, + "tokens_seen": 394526720 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114623871614845, + "loss": 1.5028, + "theoretical_loss": 4.04653610959017, + "tokens_seen": 394592256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114463390170512, + "loss": 1.5819, + "theoretical_loss": 4.046461284654696, + "tokens_seen": 394657792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114302908726179, + "loss": 1.5578, + "theoretical_loss": 4.046386475621869, + "tokens_seen": 394723328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114142427281846, + "loss": 1.5475, + "theoretical_loss": 4.046311682485669, + "tokens_seen": 394788864 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113981945837513, + "loss": 1.5736, + "theoretical_loss": 4.0462369052400815, + "tokens_seen": 394854400 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711382146439318, + "loss": 1.5481, + "theoretical_loss": 4.0461621438790925, + "tokens_seen": 394919936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113660982948847, + "loss": 1.6394, + "theoretical_loss": 4.046087398396693, + "tokens_seen": 394985472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113500501504514, + "loss": 1.5032, + "theoretical_loss": 4.046012668786877, + "tokens_seen": 395051008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113340020060181, + "loss": 1.5094, + "theoretical_loss": 4.045937955043641, + "tokens_seen": 395116544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113179538615848, + "loss": 1.6007, + "theoretical_loss": 4.045863257160986, + "tokens_seen": 395182080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113019057171515, + "loss": 1.4421, + "theoretical_loss": 4.045788575132915, + "tokens_seen": 395247616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112858575727182, + "loss": 1.5609, + "theoretical_loss": 4.045713908953435, + "tokens_seen": 395313152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112698094282849, + "loss": 1.4736, + "theoretical_loss": 4.045639258616557, + "tokens_seen": 395378688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112537612838516, + "loss": 1.5692, + "theoretical_loss": 4.045564624116293, + "tokens_seen": 395444224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112377131394183, + "loss": 1.5839, + "theoretical_loss": 4.04549000544666, + "tokens_seen": 395509760 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711221664994985, + "loss": 1.3912, + "theoretical_loss": 4.045415402601678, + "tokens_seen": 395575296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112056168505517, + "loss": 1.4101, + "theoretical_loss": 4.045340815575372, + "tokens_seen": 395640832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111895687061184, + "loss": 1.582, + "theoretical_loss": 4.045266244361765, + "tokens_seen": 395706368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111735205616851, + "loss": 1.5496, + "theoretical_loss": 4.045191688954889, + "tokens_seen": 395771904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111574724172518, + "loss": 1.5711, + "theoretical_loss": 4.045117149348776, + "tokens_seen": 395837440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111414242728185, + "loss": 1.4717, + "theoretical_loss": 4.045042625537462, + "tokens_seen": 395902976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111253761283852, + "loss": 1.5493, + "theoretical_loss": 4.044968117514987, + "tokens_seen": 395968512 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711109327983952, + "loss": 1.5777, + "theoretical_loss": 4.044893625275392, + "tokens_seen": 396034048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110932798395186, + "loss": 1.4861, + "theoretical_loss": 4.044819148812726, + "tokens_seen": 396099584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110772316950854, + "loss": 1.5666, + "theoretical_loss": 4.044744688121035, + "tokens_seen": 396165120 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711061183550652, + "loss": 1.451, + "theoretical_loss": 4.044670243194372, + "tokens_seen": 396230656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110451354062188, + "loss": 1.5889, + "theoretical_loss": 4.044595814026794, + "tokens_seen": 396296192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110290872617854, + "loss": 1.41, + "theoretical_loss": 4.044521400612357, + "tokens_seen": 396361728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110130391173522, + "loss": 1.5389, + "theoretical_loss": 4.044447002945125, + "tokens_seen": 396427264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 138316, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4225183725357056, + "objective/train/theoretical_loss": 4.044372621019162, + "objective/train/tokens_used": 416952800, + "theoretical_loss": 4.044372621019162, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109969909729188, + "loss": 1.3996, + "theoretical_loss": 4.044372621019162, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109809428284855, + "loss": 1.6224, + "theoretical_loss": 4.044298254828538, + "tokens_seen": 396558336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109648946840522, + "loss": 1.3741, + "theoretical_loss": 4.044223904367324, + "tokens_seen": 396623872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109488465396189, + "loss": 1.5343, + "theoretical_loss": 4.044149569629594, + "tokens_seen": 396689408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109327983951856, + "loss": 1.5944, + "theoretical_loss": 4.044075250609426, + "tokens_seen": 396754944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109167502507523, + "loss": 1.4888, + "theoretical_loss": 4.044000947300901, + "tokens_seen": 396820480 + }, + { + "epoch": 0.12, + "learning_rate": 0.000710900702106319, + "loss": 1.4342, + "theoretical_loss": 4.043926659698106, + "tokens_seen": 396886016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108846539618857, + "loss": 1.6039, + "theoretical_loss": 4.043852387795125, + "tokens_seen": 396951552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108686058174524, + "loss": 1.5694, + "theoretical_loss": 4.043778131586052, + "tokens_seen": 397017088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108525576730191, + "loss": 1.6645, + "theoretical_loss": 4.043703891064979, + "tokens_seen": 397082624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108365095285857, + "loss": 1.5529, + "theoretical_loss": 4.043629666226003, + "tokens_seen": 397148160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108204613841525, + "loss": 1.4129, + "theoretical_loss": 4.043555457063226, + "tokens_seen": 397213696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108044132397191, + "loss": 1.5982, + "theoretical_loss": 4.043481263570751, + "tokens_seen": 397279232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107883650952859, + "loss": 1.5183, + "theoretical_loss": 4.043407085742684, + "tokens_seen": 397344768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107723169508525, + "loss": 1.5224, + "theoretical_loss": 4.0433329235731374, + "tokens_seen": 397410304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107562688064193, + "loss": 1.576, + "theoretical_loss": 4.043258777056221, + "tokens_seen": 397475840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107402206619859, + "loss": 1.5558, + "theoretical_loss": 4.043184646186053, + "tokens_seen": 397541376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107241725175527, + "loss": 1.5146, + "theoretical_loss": 4.043110530956754, + "tokens_seen": 397606912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107081243731193, + "loss": 1.5506, + "theoretical_loss": 4.043036431362444, + "tokens_seen": 397672448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106920762286861, + "loss": 1.5227, + "theoretical_loss": 4.042962347397252, + "tokens_seen": 397737984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106760280842527, + "loss": 1.467, + "theoretical_loss": 4.042888279055304, + "tokens_seen": 397803520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106599799398195, + "loss": 1.4999, + "theoretical_loss": 4.0428142263307345, + "tokens_seen": 397869056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106439317953861, + "loss": 1.5015, + "theoretical_loss": 4.042740189217677, + "tokens_seen": 397934592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106278836509529, + "loss": 1.4486, + "theoretical_loss": 4.042666167710273, + "tokens_seen": 398000128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106118355065195, + "loss": 1.5478, + "theoretical_loss": 4.042592161802662, + "tokens_seen": 398065664 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105957873620863, + "loss": 1.4975, + "theoretical_loss": 4.042518171488989, + "tokens_seen": 398131200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105797392176529, + "loss": 1.5275, + "theoretical_loss": 4.042444196763403, + "tokens_seen": 398196736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105636910732197, + "loss": 1.5028, + "theoretical_loss": 4.042370237620053, + "tokens_seen": 398262272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105476429287863, + "loss": 1.4962, + "theoretical_loss": 4.042296294053097, + "tokens_seen": 398327808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105315947843531, + "loss": 1.6266, + "theoretical_loss": 4.0422223660566905, + "tokens_seen": 398393344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105155466399197, + "loss": 1.5788, + "theoretical_loss": 4.0421484536249945, + "tokens_seen": 398458880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104994984954865, + "loss": 1.7934, + "theoretical_loss": 4.042074556752172, + "tokens_seen": 398524416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104834503510531, + "loss": 1.629, + "theoretical_loss": 4.042000675432391, + "tokens_seen": 398589952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104674022066199, + "loss": 1.4217, + "theoretical_loss": 4.041926809659821, + "tokens_seen": 398655488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104513540621865, + "loss": 1.5271, + "theoretical_loss": 4.0418529594286365, + "tokens_seen": 398721024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104353059177533, + "loss": 1.5935, + "theoretical_loss": 4.041779124733012, + "tokens_seen": 398786560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104192577733199, + "loss": 1.4487, + "theoretical_loss": 4.041705305567129, + "tokens_seen": 398852096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104032096288867, + "loss": 1.7103, + "theoretical_loss": 4.041631501925169, + "tokens_seen": 398917632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103871614844533, + "loss": 1.5326, + "theoretical_loss": 4.041557713801319, + "tokens_seen": 398983168 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103711133400201, + "loss": 1.5504, + "theoretical_loss": 4.041483941189766, + "tokens_seen": 399048704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103550651955867, + "loss": 1.58, + "theoretical_loss": 4.041410184084703, + "tokens_seen": 399114240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103390170511535, + "loss": 1.5606, + "theoretical_loss": 4.041336442480326, + "tokens_seen": 399179776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103229689067201, + "loss": 1.6753, + "theoretical_loss": 4.041262716370833, + "tokens_seen": 399245312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103069207622869, + "loss": 1.6089, + "theoretical_loss": 4.041189005750424, + "tokens_seen": 399310848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102908726178535, + "loss": 1.4364, + "theoretical_loss": 4.041115310613307, + "tokens_seen": 399376384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102748244734203, + "loss": 1.6079, + "theoretical_loss": 4.0410416309536865, + "tokens_seen": 399441920 + }, + { + "epoch": 0.12, + "learning_rate": 0.000710258776328987, + "loss": 1.5491, + "theoretical_loss": 4.040967966765774, + "tokens_seen": 399507456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102427281845537, + "loss": 1.5805, + "theoretical_loss": 4.040894318043785, + "tokens_seen": 399572992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102266800401204, + "loss": 1.6833, + "theoretical_loss": 4.040820684781934, + "tokens_seen": 399638528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102106318956871, + "loss": 1.5103, + "theoretical_loss": 4.040747066974443, + "tokens_seen": 399704064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 139589, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5951530933380127, + "objective/train/theoretical_loss": 4.040673464615535, + "objective/train/tokens_used": 420229600, + "theoretical_loss": 4.040673464615535, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101945837512538, + "loss": 1.5524, + "theoretical_loss": 4.040673464615535, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101785356068205, + "loss": 1.5063, + "theoretical_loss": 4.040599877699437, + "tokens_seen": 399835136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101624874623872, + "loss": 1.5295, + "theoretical_loss": 4.040526306220377, + "tokens_seen": 399900672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101464393179539, + "loss": 1.6562, + "theoretical_loss": 4.040452750172588, + "tokens_seen": 399966208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101303911735206, + "loss": 1.5602, + "theoretical_loss": 4.040379209550307, + "tokens_seen": 400031744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101143430290873, + "loss": 1.5235, + "theoretical_loss": 4.0403056843477705, + "tokens_seen": 400097280 + }, + { + "epoch": 0.12, + "learning_rate": 0.000710098294884654, + "loss": 1.4961, + "theoretical_loss": 4.040232174559223, + "tokens_seen": 400162816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100822467402207, + "loss": 1.5648, + "theoretical_loss": 4.040158680178907, + "tokens_seen": 400228352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100661985957874, + "loss": 1.4735, + "theoretical_loss": 4.040085201201072, + "tokens_seen": 400293888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100501504513541, + "loss": 1.5627, + "theoretical_loss": 4.0400117376199685, + "tokens_seen": 400359424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100341023069208, + "loss": 1.6345, + "theoretical_loss": 4.039938289429852, + "tokens_seen": 400424960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100180541624875, + "loss": 1.6676, + "theoretical_loss": 4.039864856624979, + "tokens_seen": 400490496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100020060180542, + "loss": 1.5924, + "theoretical_loss": 4.039791439199609, + "tokens_seen": 400556032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099859578736209, + "loss": 1.5713, + "theoretical_loss": 4.039718037148006, + "tokens_seen": 400621568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099699097291876, + "loss": 1.5793, + "theoretical_loss": 4.039644650464438, + "tokens_seen": 400687104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099538615847543, + "loss": 1.5998, + "theoretical_loss": 4.039571279143172, + "tokens_seen": 400752640 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709937813440321, + "loss": 1.541, + "theoretical_loss": 4.039497923178484, + "tokens_seen": 400818176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099217652958877, + "loss": 1.4307, + "theoretical_loss": 4.039424582564646, + "tokens_seen": 400883712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099057171514545, + "loss": 1.4884, + "theoretical_loss": 4.03935125729594, + "tokens_seen": 400949248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098896690070211, + "loss": 1.6443, + "theoretical_loss": 4.039277947366647, + "tokens_seen": 401014784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098736208625879, + "loss": 1.5543, + "theoretical_loss": 4.039204652771051, + "tokens_seen": 401080320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098575727181545, + "loss": 1.4671, + "theoretical_loss": 4.039131373503441, + "tokens_seen": 401145856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098415245737213, + "loss": 1.5812, + "theoretical_loss": 4.039058109558107, + "tokens_seen": 401211392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098254764292879, + "loss": 1.6335, + "theoretical_loss": 4.038984860929346, + "tokens_seen": 401276928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098094282848547, + "loss": 1.5945, + "theoretical_loss": 4.038911627611451, + "tokens_seen": 401342464 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097933801404213, + "loss": 1.5385, + "theoretical_loss": 4.038838409598727, + "tokens_seen": 401408000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097773319959881, + "loss": 1.491, + "theoretical_loss": 4.038765206885472, + "tokens_seen": 401473536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097612838515547, + "loss": 1.5328, + "theoretical_loss": 4.038692019465997, + "tokens_seen": 401539072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097452357071215, + "loss": 1.573, + "theoretical_loss": 4.038618847334609, + "tokens_seen": 401604608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097291875626881, + "loss": 1.7025, + "theoretical_loss": 4.038545690485622, + "tokens_seen": 401670144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097131394182549, + "loss": 1.5662, + "theoretical_loss": 4.038472548913349, + "tokens_seen": 401735680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096970912738215, + "loss": 1.59, + "theoretical_loss": 4.038399422612112, + "tokens_seen": 401801216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096810431293883, + "loss": 1.5199, + "theoretical_loss": 4.038326311576229, + "tokens_seen": 401866752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096649949849549, + "loss": 1.542, + "theoretical_loss": 4.038253215800028, + "tokens_seen": 401932288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096489468405217, + "loss": 1.5482, + "theoretical_loss": 4.038180135277834, + "tokens_seen": 401997824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096328986960883, + "loss": 1.6522, + "theoretical_loss": 4.038107070003979, + "tokens_seen": 402063360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096168505516551, + "loss": 1.5936, + "theoretical_loss": 4.038034019972796, + "tokens_seen": 402128896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096008024072217, + "loss": 1.6195, + "theoretical_loss": 4.037960985178623, + "tokens_seen": 402194432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095847542627884, + "loss": 1.5161, + "theoretical_loss": 4.037887965615798, + "tokens_seen": 402259968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095687061183551, + "loss": 1.5262, + "theoretical_loss": 4.0378149612786665, + "tokens_seen": 402325504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095526579739218, + "loss": 1.6219, + "theoretical_loss": 4.037741972161571, + "tokens_seen": 402391040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095366098294885, + "loss": 1.584, + "theoretical_loss": 4.037668998258863, + "tokens_seen": 402456576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095205616850552, + "loss": 1.532, + "theoretical_loss": 4.037596039564892, + "tokens_seen": 402522112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095045135406219, + "loss": 1.6007, + "theoretical_loss": 4.0375230960740165, + "tokens_seen": 402587648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007094884653961886, + "loss": 1.5508, + "theoretical_loss": 4.03745016778059, + "tokens_seen": 402653184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007094724172517553, + "loss": 1.5275, + "theoretical_loss": 4.037377254678977, + "tokens_seen": 402718720 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709456369107322, + "loss": 1.587, + "theoretical_loss": 4.037304356763539, + "tokens_seen": 402784256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007094403209628886, + "loss": 1.5588, + "theoretical_loss": 4.037231474028644, + "tokens_seen": 402849792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007094242728184554, + "loss": 1.4966, + "theoretical_loss": 4.037158606468663, + "tokens_seen": 402915328 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709408224674022, + "loss": 1.5709, + "theoretical_loss": 4.037085754077966, + "tokens_seen": 402980864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 140534, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5789742469787598, + "objective/train/theoretical_loss": 4.037012916850932, + "objective/train/tokens_used": 423506400, + "theoretical_loss": 4.037012916850932, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093921765295888, + "loss": 1.5523, + "theoretical_loss": 4.037012916850932, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093761283851554, + "loss": 1.5749, + "theoretical_loss": 4.036940094781937, + "tokens_seen": 403111936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093600802407222, + "loss": 1.596, + "theoretical_loss": 4.036867287865366, + "tokens_seen": 403177472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093440320962888, + "loss": 1.4619, + "theoretical_loss": 4.036794496095602, + "tokens_seen": 403243008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093279839518556, + "loss": 1.587, + "theoretical_loss": 4.0367217194670335, + "tokens_seen": 403308544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093119358074222, + "loss": 1.5353, + "theoretical_loss": 4.036648957974052, + "tokens_seen": 403374080 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709295887662989, + "loss": 1.5422, + "theoretical_loss": 4.03657621161105, + "tokens_seen": 403439616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007092798395185556, + "loss": 1.6412, + "theoretical_loss": 4.036503480372424, + "tokens_seen": 403505152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007092637913741224, + "loss": 1.5548, + "theoretical_loss": 4.036430764252577, + "tokens_seen": 403570688 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709247743229689, + "loss": 1.5112, + "theoretical_loss": 4.036358063245908, + "tokens_seen": 403636224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007092316950852558, + "loss": 1.515, + "theoretical_loss": 4.036285377346826, + "tokens_seen": 403701760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007092156469408224, + "loss": 1.422, + "theoretical_loss": 4.036212706549738, + "tokens_seen": 403767296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091995987963892, + "loss": 1.6223, + "theoretical_loss": 4.036140050849056, + "tokens_seen": 403832832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091835506519558, + "loss": 1.4715, + "theoretical_loss": 4.036067410239196, + "tokens_seen": 403898368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091675025075226, + "loss": 1.5133, + "theoretical_loss": 4.035994784714574, + "tokens_seen": 403963904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091514543630892, + "loss": 1.4933, + "theoretical_loss": 4.035922174269611, + "tokens_seen": 404029440 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709135406218656, + "loss": 1.3653, + "theoretical_loss": 4.035849578898731, + "tokens_seen": 404094976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091193580742226, + "loss": 1.5697, + "theoretical_loss": 4.035776998596361, + "tokens_seen": 404160512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091033099297894, + "loss": 1.5925, + "theoretical_loss": 4.035704433356932, + "tokens_seen": 404226048 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709087261785356, + "loss": 1.5928, + "theoretical_loss": 4.035631883174873, + "tokens_seen": 404291584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090712136409228, + "loss": 1.51, + "theoretical_loss": 4.035559348044622, + "tokens_seen": 404357120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090551654964894, + "loss": 1.5758, + "theoretical_loss": 4.035486827960617, + "tokens_seen": 404422656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090391173520562, + "loss": 1.4794, + "theoretical_loss": 4.035414322917299, + "tokens_seen": 404488192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090230692076228, + "loss": 1.643, + "theoretical_loss": 4.035341832909114, + "tokens_seen": 404553728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090070210631896, + "loss": 1.5227, + "theoretical_loss": 4.035269357930507, + "tokens_seen": 404619264 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007089909729187562, + "loss": 1.5348, + "theoretical_loss": 4.03519689797593, + "tokens_seen": 404684800 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708974924774323, + "loss": 1.5649, + "theoretical_loss": 4.035124453039836, + "tokens_seen": 404750336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007089588766298896, + "loss": 1.6123, + "theoretical_loss": 4.035052023116679, + "tokens_seen": 404815872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007089428284854564, + "loss": 1.5052, + "theoretical_loss": 4.0349796082009215, + "tokens_seen": 404881408 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708926780341023, + "loss": 1.493, + "theoretical_loss": 4.034907208287024, + "tokens_seen": 404946944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007089107321965898, + "loss": 1.6608, + "theoretical_loss": 4.034834823369451, + "tokens_seen": 405012480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088946840521565, + "loss": 1.5893, + "theoretical_loss": 4.034762453442672, + "tokens_seen": 405078016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088786359077232, + "loss": 1.4565, + "theoretical_loss": 4.034690098501155, + "tokens_seen": 405143552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088625877632899, + "loss": 1.5087, + "theoretical_loss": 4.034617758539377, + "tokens_seen": 405209088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088465396188566, + "loss": 1.5578, + "theoretical_loss": 4.034545433551814, + "tokens_seen": 405274624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088304914744233, + "loss": 1.4697, + "theoretical_loss": 4.034473123532944, + "tokens_seen": 405340160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00070881444332999, + "loss": 1.5152, + "theoretical_loss": 4.034400828477251, + "tokens_seen": 405405696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087983951855567, + "loss": 1.5794, + "theoretical_loss": 4.03432854837922, + "tokens_seen": 405471232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087823470411234, + "loss": 1.4242, + "theoretical_loss": 4.03425628323334, + "tokens_seen": 405536768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087662988966901, + "loss": 1.5615, + "theoretical_loss": 4.034184033034102, + "tokens_seen": 405602304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087502507522568, + "loss": 1.5607, + "theoretical_loss": 4.034111797776001, + "tokens_seen": 405667840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087342026078235, + "loss": 1.5839, + "theoretical_loss": 4.034039577453533, + "tokens_seen": 405733376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087181544633902, + "loss": 1.3925, + "theoretical_loss": 4.033967372061199, + "tokens_seen": 405798912 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708702106318957, + "loss": 1.5731, + "theoretical_loss": 4.033895181593501, + "tokens_seen": 405864448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086860581745236, + "loss": 1.4833, + "theoretical_loss": 4.033823006044947, + "tokens_seen": 405929984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086700100300904, + "loss": 1.5742, + "theoretical_loss": 4.033750845410044, + "tokens_seen": 405995520 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708653961885657, + "loss": 1.5395, + "theoretical_loss": 4.033678699683303, + "tokens_seen": 406061056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086379137412238, + "loss": 1.5878, + "theoretical_loss": 4.033606568859241, + "tokens_seen": 406126592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086218655967904, + "loss": 1.4696, + "theoretical_loss": 4.033534452932376, + "tokens_seen": 406192128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086058174523572, + "loss": 1.5745, + "theoretical_loss": 4.033462351897225, + "tokens_seen": 406257664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 141740, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.756189227104187, + "objective/train/theoretical_loss": 4.033390265748315, + "objective/train/tokens_used": 426783200, + "theoretical_loss": 4.033390265748315, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085897693079238, + "loss": 1.5844, + "theoretical_loss": 4.033390265748315, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085737211634906, + "loss": 1.4169, + "theoretical_loss": 4.033318194480169, + "tokens_seen": 406388736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085576730190572, + "loss": 1.6124, + "theoretical_loss": 4.0332461380873195, + "tokens_seen": 406454272 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708541624874624, + "loss": 1.6184, + "theoretical_loss": 4.033174096564295, + "tokens_seen": 406519808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085255767301906, + "loss": 1.4786, + "theoretical_loss": 4.033102069905635, + "tokens_seen": 406585344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085095285857574, + "loss": 1.4864, + "theoretical_loss": 4.033030058105872, + "tokens_seen": 406650880 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708493480441324, + "loss": 1.5946, + "theoretical_loss": 4.032958061159551, + "tokens_seen": 406716416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084774322968908, + "loss": 1.5697, + "theoretical_loss": 4.032886079061214, + "tokens_seen": 406781952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084613841524574, + "loss": 1.6082, + "theoretical_loss": 4.032814111805407, + "tokens_seen": 406847488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084453360080242, + "loss": 1.6108, + "theoretical_loss": 4.03274215938668, + "tokens_seen": 406913024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084292878635908, + "loss": 1.5825, + "theoretical_loss": 4.0326702217995845, + "tokens_seen": 406978560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084132397191576, + "loss": 1.6002, + "theoretical_loss": 4.0325982990386775, + "tokens_seen": 407044096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083971915747242, + "loss": 1.5136, + "theoretical_loss": 4.032526391098515, + "tokens_seen": 407109632 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708381143430291, + "loss": 1.5522, + "theoretical_loss": 4.032454497973659, + "tokens_seen": 407175168 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083650952858576, + "loss": 1.5036, + "theoretical_loss": 4.032382619658673, + "tokens_seen": 407240704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083490471414244, + "loss": 1.5827, + "theoretical_loss": 4.0323107561481235, + "tokens_seen": 407306240 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708332998996991, + "loss": 1.4544, + "theoretical_loss": 4.0322389074365805, + "tokens_seen": 407371776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083169508525578, + "loss": 1.6027, + "theoretical_loss": 4.032167073518615, + "tokens_seen": 407437312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083009027081244, + "loss": 1.5711, + "theoretical_loss": 4.032095254388803, + "tokens_seen": 407502848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082848545636912, + "loss": 1.5284, + "theoretical_loss": 4.032023450041724, + "tokens_seen": 407568384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082688064192578, + "loss": 1.5628, + "theoretical_loss": 4.031951660471956, + "tokens_seen": 407633920 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082527582748246, + "loss": 1.5589, + "theoretical_loss": 4.031879885674085, + "tokens_seen": 407699456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082367101303912, + "loss": 1.6216, + "theoretical_loss": 4.031808125642697, + "tokens_seen": 407764992 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708220661985958, + "loss": 1.552, + "theoretical_loss": 4.031736380372382, + "tokens_seen": 407830528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082046138415246, + "loss": 1.4943, + "theoretical_loss": 4.031664649857731, + "tokens_seen": 407896064 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081885656970914, + "loss": 1.5147, + "theoretical_loss": 4.031592934093341, + "tokens_seen": 407961600 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708172517552658, + "loss": 1.4208, + "theoretical_loss": 4.031521233073809, + "tokens_seen": 408027136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081564694082247, + "loss": 1.5565, + "theoretical_loss": 4.031449546793736, + "tokens_seen": 408092672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081404212637914, + "loss": 1.5643, + "theoretical_loss": 4.031377875247726, + "tokens_seen": 408158208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081243731193581, + "loss": 1.6523, + "theoretical_loss": 4.031306218430385, + "tokens_seen": 408223744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081083249749248, + "loss": 1.5344, + "theoretical_loss": 4.031234576336322, + "tokens_seen": 408289280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080922768304915, + "loss": 1.5772, + "theoretical_loss": 4.0311629489601515, + "tokens_seen": 408354816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080762286860582, + "loss": 1.6145, + "theoretical_loss": 4.031091336296487, + "tokens_seen": 408420352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080601805416249, + "loss": 1.5051, + "theoretical_loss": 4.031019738339946, + "tokens_seen": 408485888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080441323971915, + "loss": 1.5164, + "theoretical_loss": 4.03094815508515, + "tokens_seen": 408551424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080280842527583, + "loss": 1.5636, + "theoretical_loss": 4.030876586526722, + "tokens_seen": 408616960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080120361083249, + "loss": 1.3939, + "theoretical_loss": 4.0308050326592895, + "tokens_seen": 408682496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079959879638917, + "loss": 1.4653, + "theoretical_loss": 4.030733493477481, + "tokens_seen": 408748032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079799398194583, + "loss": 1.5727, + "theoretical_loss": 4.030661968975929, + "tokens_seen": 408813568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079638916750251, + "loss": 1.5448, + "theoretical_loss": 4.030590459149267, + "tokens_seen": 408879104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079478435305917, + "loss": 1.453, + "theoretical_loss": 4.030518963992135, + "tokens_seen": 408944640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079317953861585, + "loss": 1.5661, + "theoretical_loss": 4.030447483499171, + "tokens_seen": 409010176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079157472417251, + "loss": 1.504, + "theoretical_loss": 4.03037601766502, + "tokens_seen": 409075712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078996990972919, + "loss": 1.4467, + "theoretical_loss": 4.030304566484329, + "tokens_seen": 409141248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078836509528585, + "loss": 1.5275, + "theoretical_loss": 4.030233129951744, + "tokens_seen": 409206784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078676028084253, + "loss": 1.5733, + "theoretical_loss": 4.03016170806192, + "tokens_seen": 409272320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078515546639919, + "loss": 1.4817, + "theoretical_loss": 4.030090300809509, + "tokens_seen": 409337856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078355065195587, + "loss": 1.6054, + "theoretical_loss": 4.03001890818917, + "tokens_seen": 409403392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078194583751253, + "loss": 1.4895, + "theoretical_loss": 4.029947530195563, + "tokens_seen": 409468928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078034102306921, + "loss": 1.547, + "theoretical_loss": 4.029876166823351, + "tokens_seen": 409534464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 142735, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6000092029571533, + "objective/train/theoretical_loss": 4.029804818067199, + "objective/train/tokens_used": 430060000, + "theoretical_loss": 4.029804818067199, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077873620862587, + "loss": 1.5053, + "theoretical_loss": 4.029804818067199, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077713139418255, + "loss": 1.4704, + "theoretical_loss": 4.029733483921777, + "tokens_seen": 409665536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077552657973921, + "loss": 1.5721, + "theoretical_loss": 4.0296621643817545, + "tokens_seen": 409731072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077392176529589, + "loss": 1.6005, + "theoretical_loss": 4.029590859441807, + "tokens_seen": 409796608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077231695085255, + "loss": 1.5829, + "theoretical_loss": 4.029519569096612, + "tokens_seen": 409862144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077071213640923, + "loss": 1.5834, + "theoretical_loss": 4.029448293340847, + "tokens_seen": 409927680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076910732196589, + "loss": 1.4792, + "theoretical_loss": 4.029377032169197, + "tokens_seen": 409993216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076750250752257, + "loss": 1.4813, + "theoretical_loss": 4.029305785576347, + "tokens_seen": 410058752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076589769307923, + "loss": 1.597, + "theoretical_loss": 4.029234553556985, + "tokens_seen": 410124288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076429287863591, + "loss": 1.5541, + "theoretical_loss": 4.029163336105801, + "tokens_seen": 410189824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076268806419257, + "loss": 1.5452, + "theoretical_loss": 4.029092133217489, + "tokens_seen": 410255360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076108324974925, + "loss": 1.4583, + "theoretical_loss": 4.029020944886746, + "tokens_seen": 410320896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075947843530591, + "loss": 1.5313, + "theoretical_loss": 4.0289497711082705, + "tokens_seen": 410386432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075787362086259, + "loss": 1.5252, + "theoretical_loss": 4.028878611876765, + "tokens_seen": 410451968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075626880641925, + "loss": 1.5533, + "theoretical_loss": 4.028807467186935, + "tokens_seen": 410517504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075466399197593, + "loss": 1.5356, + "theoretical_loss": 4.028736337033488, + "tokens_seen": 410583040 + }, + { + "epoch": 0.12, + "learning_rate": 0.000707530591775326, + "loss": 1.4687, + "theoretical_loss": 4.028665221411133, + "tokens_seen": 410648576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075145436308927, + "loss": 1.5098, + "theoretical_loss": 4.028594120314585, + "tokens_seen": 410714112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074984954864595, + "loss": 1.5612, + "theoretical_loss": 4.028523033738558, + "tokens_seen": 410779648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074824473420261, + "loss": 1.4409, + "theoretical_loss": 4.028451961677772, + "tokens_seen": 410845184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074663991975929, + "loss": 1.3957, + "theoretical_loss": 4.028380904126948, + "tokens_seen": 410910720 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074503510531595, + "loss": 1.616, + "theoretical_loss": 4.0283098610808095, + "tokens_seen": 410976256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074343029087263, + "loss": 1.5218, + "theoretical_loss": 4.028238832534084, + "tokens_seen": 411041792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074182547642929, + "loss": 1.5817, + "theoretical_loss": 4.028167818481502, + "tokens_seen": 411107328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074022066198597, + "loss": 1.5122, + "theoretical_loss": 4.0280968189177955, + "tokens_seen": 411172864 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073861584754263, + "loss": 1.5176, + "theoretical_loss": 4.028025833837699, + "tokens_seen": 411238400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073701103309931, + "loss": 1.5894, + "theoretical_loss": 4.027954863235951, + "tokens_seen": 411303936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073540621865597, + "loss": 1.3881, + "theoretical_loss": 4.0278839071072925, + "tokens_seen": 411369472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073380140421265, + "loss": 1.492, + "theoretical_loss": 4.027812965446467, + "tokens_seen": 411435008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073219658976931, + "loss": 1.6257, + "theoretical_loss": 4.02774203824822, + "tokens_seen": 411500544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073059177532599, + "loss": 1.4058, + "theoretical_loss": 4.0276711255073, + "tokens_seen": 411566080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072898696088265, + "loss": 1.4869, + "theoretical_loss": 4.027600227218461, + "tokens_seen": 411631616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072738214643933, + "loss": 1.5341, + "theoretical_loss": 4.027529343376456, + "tokens_seen": 411697152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072577733199599, + "loss": 1.541, + "theoretical_loss": 4.027458473976041, + "tokens_seen": 411762688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072417251755267, + "loss": 1.5091, + "theoretical_loss": 4.027387619011977, + "tokens_seen": 411828224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072256770310933, + "loss": 1.4396, + "theoretical_loss": 4.027316778479027, + "tokens_seen": 411893760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072096288866601, + "loss": 1.5064, + "theoretical_loss": 4.027245952371956, + "tokens_seen": 411959296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071935807422267, + "loss": 1.4344, + "theoretical_loss": 4.027175140685532, + "tokens_seen": 412024832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071775325977935, + "loss": 1.4299, + "theoretical_loss": 4.027104343414527, + "tokens_seen": 412090368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071614844533601, + "loss": 1.509, + "theoretical_loss": 4.027033560553713, + "tokens_seen": 412155904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071454363089269, + "loss": 1.4293, + "theoretical_loss": 4.026962792097867, + "tokens_seen": 412221440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071293881644935, + "loss": 1.423, + "theoretical_loss": 4.026892038041767, + "tokens_seen": 412286976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071133400200603, + "loss": 1.6205, + "theoretical_loss": 4.026821298380197, + "tokens_seen": 412352512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007070972918756269, + "loss": 1.5077, + "theoretical_loss": 4.026750573107939, + "tokens_seen": 412418048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007070812437311937, + "loss": 1.5667, + "theoretical_loss": 4.0266798622197815, + "tokens_seen": 412483584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070651955867603, + "loss": 1.5787, + "theoretical_loss": 4.026609165710514, + "tokens_seen": 412549120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070491474423271, + "loss": 1.5379, + "theoretical_loss": 4.02653848357493, + "tokens_seen": 412614656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070330992978937, + "loss": 1.4793, + "theoretical_loss": 4.026467815807823, + "tokens_seen": 412680192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070170511534605, + "loss": 1.5342, + "theoretical_loss": 4.026397162403994, + "tokens_seen": 412745728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070010030090271, + "loss": 1.5203, + "theoretical_loss": 4.0263265233582395, + "tokens_seen": 412811264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 143929, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.583418369293213, + "objective/train/theoretical_loss": 4.026255898665367, + "objective/train/tokens_used": 433336800, + "theoretical_loss": 4.026255898665367, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069849548645939, + "loss": 1.5316, + "theoretical_loss": 4.026255898665367, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069689067201605, + "loss": 1.5474, + "theoretical_loss": 4.02618528832018, + "tokens_seen": 412942336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069528585757273, + "loss": 1.5566, + "theoretical_loss": 4.026114692317489, + "tokens_seen": 413007872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069368104312939, + "loss": 1.2872, + "theoretical_loss": 4.026044110652105, + "tokens_seen": 413073408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069207622868607, + "loss": 1.5705, + "theoretical_loss": 4.025973543318841, + "tokens_seen": 413138944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069047141424273, + "loss": 1.5252, + "theoretical_loss": 4.025902990312517, + "tokens_seen": 413204480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068886659979941, + "loss": 1.5701, + "theoretical_loss": 4.025832451627949, + "tokens_seen": 413270016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068726178535607, + "loss": 1.5828, + "theoretical_loss": 4.025761927259961, + "tokens_seen": 413335552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068565697091275, + "loss": 1.4946, + "theoretical_loss": 4.0256914172033795, + "tokens_seen": 413401088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068405215646941, + "loss": 1.5515, + "theoretical_loss": 4.025620921453029, + "tokens_seen": 413466624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068244734202609, + "loss": 1.639, + "theoretical_loss": 4.025550440003744, + "tokens_seen": 413532160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068084252758275, + "loss": 1.549, + "theoretical_loss": 4.025479972850354, + "tokens_seen": 413597696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067923771313943, + "loss": 1.4433, + "theoretical_loss": 4.025409519987696, + "tokens_seen": 413663232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067763289869609, + "loss": 1.4926, + "theoretical_loss": 4.025339081410609, + "tokens_seen": 413728768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067602808425276, + "loss": 1.5137, + "theoretical_loss": 4.025268657113934, + "tokens_seen": 413794304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067442326980943, + "loss": 1.6172, + "theoretical_loss": 4.025198247092514, + "tokens_seen": 413859840 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706728184553661, + "loss": 1.5152, + "theoretical_loss": 4.025127851341196, + "tokens_seen": 413925376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067121364092277, + "loss": 1.4704, + "theoretical_loss": 4.025057469854831, + "tokens_seen": 413990912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066960882647944, + "loss": 1.454, + "theoretical_loss": 4.024987102628267, + "tokens_seen": 414056448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066800401203611, + "loss": 1.482, + "theoretical_loss": 4.024916749656362, + "tokens_seen": 414121984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066639919759278, + "loss": 1.4719, + "theoretical_loss": 4.024846410933971, + "tokens_seen": 414187520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066479438314945, + "loss": 1.523, + "theoretical_loss": 4.024776086455955, + "tokens_seen": 414253056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066318956870612, + "loss": 1.4781, + "theoretical_loss": 4.024705776217177, + "tokens_seen": 414318592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066158475426278, + "loss": 1.5754, + "theoretical_loss": 4.024635480212501, + "tokens_seen": 414384128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065997993981946, + "loss": 1.5037, + "theoretical_loss": 4.024565198436796, + "tokens_seen": 414449664 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065837512537612, + "loss": 1.5883, + "theoretical_loss": 4.024494930884931, + "tokens_seen": 414515200 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706567703109328, + "loss": 1.5003, + "theoretical_loss": 4.024424677551781, + "tokens_seen": 414580736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065516549648946, + "loss": 1.4816, + "theoretical_loss": 4.02435443843222, + "tokens_seen": 414646272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065356068204614, + "loss": 1.6653, + "theoretical_loss": 4.024284213521128, + "tokens_seen": 414711808 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706519558676028, + "loss": 1.5012, + "theoretical_loss": 4.024214002813386, + "tokens_seen": 414777344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065035105315948, + "loss": 1.4605, + "theoretical_loss": 4.024143806303877, + "tokens_seen": 414842880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064874623871614, + "loss": 1.4218, + "theoretical_loss": 4.024073623987487, + "tokens_seen": 414908416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064714142427282, + "loss": 1.5412, + "theoretical_loss": 4.024003455859107, + "tokens_seen": 414973952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064553660982948, + "loss": 1.4357, + "theoretical_loss": 4.023933301913628, + "tokens_seen": 415039488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064393179538616, + "loss": 1.5242, + "theoretical_loss": 4.023863162145943, + "tokens_seen": 415105024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064232698094282, + "loss": 1.6922, + "theoretical_loss": 4.023793036550952, + "tokens_seen": 415170560 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706407221664995, + "loss": 1.4891, + "theoretical_loss": 4.023722925123551, + "tokens_seen": 415236096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063911735205616, + "loss": 1.5164, + "theoretical_loss": 4.023652827858646, + "tokens_seen": 415301632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063751253761284, + "loss": 1.537, + "theoretical_loss": 4.023582744751138, + "tokens_seen": 415367168 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706359077231695, + "loss": 1.437, + "theoretical_loss": 4.0235126757959385, + "tokens_seen": 415432704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063430290872618, + "loss": 1.467, + "theoretical_loss": 4.0234426209879555, + "tokens_seen": 415498240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063269809428284, + "loss": 1.5664, + "theoretical_loss": 4.0233725803221025, + "tokens_seen": 415563776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063109327983952, + "loss": 1.5127, + "theoretical_loss": 4.023302553793295, + "tokens_seen": 415629312 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706294884653962, + "loss": 1.4854, + "theoretical_loss": 4.023232541396451, + "tokens_seen": 415694848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007062788365095286, + "loss": 1.5801, + "theoretical_loss": 4.023162543126491, + "tokens_seen": 415760384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007062627883650954, + "loss": 1.5972, + "theoretical_loss": 4.023092558978339, + "tokens_seen": 415825920 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706246740220662, + "loss": 1.5104, + "theoretical_loss": 4.023022588946921, + "tokens_seen": 415891456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007062306920762288, + "loss": 1.3768, + "theoretical_loss": 4.022952633027164, + "tokens_seen": 415956992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007062146439317954, + "loss": 1.5053, + "theoretical_loss": 4.022882691214002, + "tokens_seen": 416022528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061985957873622, + "loss": 1.6381, + "theoretical_loss": 4.022812763502367, + "tokens_seen": 416088064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 144846, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4289838075637817, + "objective/train/theoretical_loss": 4.022742849887195, + "objective/train/tokens_used": 436613600, + "theoretical_loss": 4.022742849887195, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061825476429288, + "loss": 1.5006, + "theoretical_loss": 4.022742849887195, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061664994984956, + "loss": 1.5972, + "theoretical_loss": 4.022672950363427, + "tokens_seen": 416219136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061504513540622, + "loss": 1.5633, + "theoretical_loss": 4.022603064926004, + "tokens_seen": 416284672 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706134403209629, + "loss": 1.578, + "theoretical_loss": 4.02253319356987, + "tokens_seen": 416350208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061183550651956, + "loss": 1.475, + "theoretical_loss": 4.022463336289971, + "tokens_seen": 416415744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061023069207624, + "loss": 1.4692, + "theoretical_loss": 4.022393493081259, + "tokens_seen": 416481280 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706086258776329, + "loss": 1.4357, + "theoretical_loss": 4.022323663938684, + "tokens_seen": 416546816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060702106318958, + "loss": 1.4955, + "theoretical_loss": 4.022253848857201, + "tokens_seen": 416612352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060541624874624, + "loss": 1.544, + "theoretical_loss": 4.022184047831768, + "tokens_seen": 416677888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060381143430292, + "loss": 1.5028, + "theoretical_loss": 4.022114260857345, + "tokens_seen": 416743424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060220661985958, + "loss": 1.523, + "theoretical_loss": 4.022044487928895, + "tokens_seen": 416808960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060060180541626, + "loss": 1.5272, + "theoretical_loss": 4.021974729041381, + "tokens_seen": 416874496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007059899699097292, + "loss": 1.5397, + "theoretical_loss": 4.021904984189773, + "tokens_seen": 416940032 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705973921765296, + "loss": 1.5114, + "theoretical_loss": 4.021835253369041, + "tokens_seen": 417005568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007059578736208626, + "loss": 1.6011, + "theoretical_loss": 4.021765536574157, + "tokens_seen": 417071104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007059418254764294, + "loss": 1.5084, + "theoretical_loss": 4.021695833800098, + "tokens_seen": 417136640 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705925777331996, + "loss": 1.5491, + "theoretical_loss": 4.02162614504184, + "tokens_seen": 417202176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007059097291875628, + "loss": 1.4945, + "theoretical_loss": 4.021556470294366, + "tokens_seen": 417267712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058936810431294, + "loss": 1.5515, + "theoretical_loss": 4.021486809552657, + "tokens_seen": 417333248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058776328986962, + "loss": 1.4881, + "theoretical_loss": 4.021417162811701, + "tokens_seen": 417398784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058615847542628, + "loss": 1.6362, + "theoretical_loss": 4.021347530066485, + "tokens_seen": 417464320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058455366098296, + "loss": 1.5221, + "theoretical_loss": 4.0212779113120005, + "tokens_seen": 417529856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058294884653962, + "loss": 1.5595, + "theoretical_loss": 4.021208306543242, + "tokens_seen": 417595392 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705813440320963, + "loss": 1.5748, + "theoretical_loss": 4.021138715755204, + "tokens_seen": 417660928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057973921765296, + "loss": 1.5799, + "theoretical_loss": 4.021069138942886, + "tokens_seen": 417726464 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057813440320964, + "loss": 1.5296, + "theoretical_loss": 4.020999576101291, + "tokens_seen": 417792000 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705765295887663, + "loss": 1.4946, + "theoretical_loss": 4.02093002722542, + "tokens_seen": 417857536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057492477432298, + "loss": 1.583, + "theoretical_loss": 4.020860492310281, + "tokens_seen": 417923072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057331995987964, + "loss": 1.5426, + "theoretical_loss": 4.020790971350884, + "tokens_seen": 417988608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057171514543632, + "loss": 1.5689, + "theoretical_loss": 4.020721464342238, + "tokens_seen": 418054144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057011033099298, + "loss": 1.4474, + "theoretical_loss": 4.02065197127936, + "tokens_seen": 418119680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007056850551654966, + "loss": 1.5951, + "theoretical_loss": 4.020582492157265, + "tokens_seen": 418185216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007056690070210632, + "loss": 1.5839, + "theoretical_loss": 4.020513026970973, + "tokens_seen": 418250752 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070565295887663, + "loss": 1.5332, + "theoretical_loss": 4.020443575715506, + "tokens_seen": 418316288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007056369107321966, + "loss": 1.5696, + "theoretical_loss": 4.020374138385887, + "tokens_seen": 418381824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007056208625877634, + "loss": 1.6332, + "theoretical_loss": 4.020304714977145, + "tokens_seen": 418447360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070560481444333, + "loss": 1.5723, + "theoretical_loss": 4.020235305484308, + "tokens_seen": 418512896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055887662988968, + "loss": 1.5005, + "theoretical_loss": 4.020165909902409, + "tokens_seen": 418578432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055727181544634, + "loss": 1.4726, + "theoretical_loss": 4.020096528226483, + "tokens_seen": 418643968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055566700100302, + "loss": 1.59, + "theoretical_loss": 4.020027160451566, + "tokens_seen": 418709504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055406218655968, + "loss": 1.4871, + "theoretical_loss": 4.019957806572697, + "tokens_seen": 418775040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055245737211636, + "loss": 1.54, + "theoretical_loss": 4.019888466584921, + "tokens_seen": 418840576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055085255767302, + "loss": 1.4329, + "theoretical_loss": 4.019819140483281, + "tokens_seen": 418906112 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705492477432297, + "loss": 1.613, + "theoretical_loss": 4.019749828262825, + "tokens_seen": 418971648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007054764292878636, + "loss": 1.501, + "theoretical_loss": 4.0196805299186025, + "tokens_seen": 419037184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007054603811434304, + "loss": 1.3925, + "theoretical_loss": 4.019611245445666, + "tokens_seen": 419102720 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705444332998997, + "loss": 1.4554, + "theoretical_loss": 4.01954197483907, + "tokens_seen": 419168256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007054282848545638, + "loss": 1.4464, + "theoretical_loss": 4.019472718093874, + "tokens_seen": 419233792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007054122367101304, + "loss": 1.5303, + "theoretical_loss": 4.019403475205135, + "tokens_seen": 419299328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053961885656972, + "loss": 1.5512, + "theoretical_loss": 4.019334246167919, + "tokens_seen": 419364864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 145965, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.599278211593628, + "objective/train/theoretical_loss": 4.019265030977288, + "objective/train/tokens_used": 439890400, + "theoretical_loss": 4.019265030977288, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053801404212638, + "loss": 1.615, + "theoretical_loss": 4.019265030977288, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053640922768305, + "loss": 1.4472, + "theoretical_loss": 4.019195829628311, + "tokens_seen": 419495936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053480441323972, + "loss": 1.5333, + "theoretical_loss": 4.019126642116059, + "tokens_seen": 419561472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053319959879639, + "loss": 1.4841, + "theoretical_loss": 4.019057468435603, + "tokens_seen": 419627008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053159478435306, + "loss": 1.4736, + "theoretical_loss": 4.01898830858202, + "tokens_seen": 419692544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052998996990973, + "loss": 1.5241, + "theoretical_loss": 4.018919162550386, + "tokens_seen": 419758080 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705283851554664, + "loss": 1.5164, + "theoretical_loss": 4.018850030335783, + "tokens_seen": 419823616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052678034102307, + "loss": 1.4566, + "theoretical_loss": 4.018780911933293, + "tokens_seen": 419889152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052517552657974, + "loss": 1.629, + "theoretical_loss": 4.018711807338001, + "tokens_seen": 419954688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052357071213641, + "loss": 1.5485, + "theoretical_loss": 4.018642716544994, + "tokens_seen": 420020224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052196589769307, + "loss": 1.5464, + "theoretical_loss": 4.018573639549364, + "tokens_seen": 420085760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052036108324975, + "loss": 1.5059, + "theoretical_loss": 4.018504576346203, + "tokens_seen": 420151296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051875626880641, + "loss": 1.5752, + "theoretical_loss": 4.018435526930607, + "tokens_seen": 420216832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051715145436309, + "loss": 1.6088, + "theoretical_loss": 4.018366491297673, + "tokens_seen": 420282368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051554663991975, + "loss": 1.4821, + "theoretical_loss": 4.018297469442502, + "tokens_seen": 420347904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051394182547643, + "loss": 1.4243, + "theoretical_loss": 4.018228461360197, + "tokens_seen": 420413440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051233701103309, + "loss": 1.538, + "theoretical_loss": 4.018159467045862, + "tokens_seen": 420478976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051073219658977, + "loss": 1.4714, + "theoretical_loss": 4.018090486494607, + "tokens_seen": 420544512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050912738214644, + "loss": 1.4927, + "theoretical_loss": 4.0180215197015405, + "tokens_seen": 420610048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050752256770311, + "loss": 1.4405, + "theoretical_loss": 4.017952566661776, + "tokens_seen": 420675584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050591775325978, + "loss": 1.6563, + "theoretical_loss": 4.017883627370431, + "tokens_seen": 420741120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050431293881645, + "loss": 1.5569, + "theoretical_loss": 4.017814701822619, + "tokens_seen": 420806656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050270812437312, + "loss": 1.5939, + "theoretical_loss": 4.017745790013464, + "tokens_seen": 420872192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050110330992979, + "loss": 1.6454, + "theoretical_loss": 4.017676891938088, + "tokens_seen": 420937728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049949849548647, + "loss": 1.5453, + "theoretical_loss": 4.0176080075916145, + "tokens_seen": 421003264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049789368104313, + "loss": 1.5243, + "theoretical_loss": 4.017539136969174, + "tokens_seen": 421068800 + }, + { + "epoch": 0.13, + "learning_rate": 0.000704962888665998, + "loss": 1.6324, + "theoretical_loss": 4.017470280065895, + "tokens_seen": 421134336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049468405215647, + "loss": 1.4929, + "theoretical_loss": 4.017401436876911, + "tokens_seen": 421199872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049307923771315, + "loss": 1.4403, + "theoretical_loss": 4.017332607397358, + "tokens_seen": 421265408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049147442326981, + "loss": 1.4566, + "theoretical_loss": 4.017263791622373, + "tokens_seen": 421330944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048986960882649, + "loss": 1.5641, + "theoretical_loss": 4.017194989547095, + "tokens_seen": 421396480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048826479438315, + "loss": 1.4466, + "theoretical_loss": 4.017126201166668, + "tokens_seen": 421462016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048665997993983, + "loss": 1.6451, + "theoretical_loss": 4.017057426476239, + "tokens_seen": 421527552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048505516549649, + "loss": 1.5085, + "theoretical_loss": 4.016988665470953, + "tokens_seen": 421593088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048345035105317, + "loss": 1.528, + "theoretical_loss": 4.016919918145961, + "tokens_seen": 421658624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048184553660983, + "loss": 1.4832, + "theoretical_loss": 4.016851184496415, + "tokens_seen": 421724160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048024072216651, + "loss": 1.338, + "theoretical_loss": 4.016782464517472, + "tokens_seen": 421789696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047863590772317, + "loss": 1.4554, + "theoretical_loss": 4.016713758204288, + "tokens_seen": 421855232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047703109327985, + "loss": 1.5311, + "theoretical_loss": 4.016645065552023, + "tokens_seen": 421920768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047542627883651, + "loss": 1.4613, + "theoretical_loss": 4.0165763865558395, + "tokens_seen": 421986304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047382146439319, + "loss": 1.3784, + "theoretical_loss": 4.016507721210904, + "tokens_seen": 422051840 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047221664994985, + "loss": 1.5047, + "theoretical_loss": 4.016439069512382, + "tokens_seen": 422117376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047061183550653, + "loss": 1.5729, + "theoretical_loss": 4.016370431455444, + "tokens_seen": 422182912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046900702106319, + "loss": 1.5015, + "theoretical_loss": 4.016301807035264, + "tokens_seen": 422248448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046740220661987, + "loss": 1.4857, + "theoretical_loss": 4.016233196247014, + "tokens_seen": 422313984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046579739217653, + "loss": 1.4861, + "theoretical_loss": 4.016164599085872, + "tokens_seen": 422379520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046419257773321, + "loss": 1.3585, + "theoretical_loss": 4.016096015547021, + "tokens_seen": 422445056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046258776328987, + "loss": 1.7261, + "theoretical_loss": 4.016027445625638, + "tokens_seen": 422510592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046098294884655, + "loss": 1.4888, + "theoretical_loss": 4.015958889316911, + "tokens_seen": 422576128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045937813440321, + "loss": 1.505, + "theoretical_loss": 4.015890346616026, + "tokens_seen": 422641664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 147195, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.464314579963684, + "objective/train/theoretical_loss": 4.015821817518173, + "objective/train/tokens_used": 443167200, + "theoretical_loss": 4.015821817518173, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045777331995989, + "loss": 1.4434, + "theoretical_loss": 4.015821817518173, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045616850551655, + "loss": 1.5134, + "theoretical_loss": 4.015753302018544, + "tokens_seen": 422772736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045456369107323, + "loss": 1.3918, + "theoretical_loss": 4.015684800112332, + "tokens_seen": 422838272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045295887662989, + "loss": 1.4096, + "theoretical_loss": 4.015616311794735, + "tokens_seen": 422903808 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045135406218657, + "loss": 1.5184, + "theoretical_loss": 4.015547837060953, + "tokens_seen": 422969344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044974924774323, + "loss": 1.4947, + "theoretical_loss": 4.015479375906187, + "tokens_seen": 423034880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044814443329991, + "loss": 1.5867, + "theoretical_loss": 4.0154109283256405, + "tokens_seen": 423100416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044653961885657, + "loss": 1.5139, + "theoretical_loss": 4.015342494314521, + "tokens_seen": 423165952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044493480441325, + "loss": 1.5439, + "theoretical_loss": 4.015274073868037, + "tokens_seen": 423231488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044332998996991, + "loss": 1.5311, + "theoretical_loss": 4.015205666981401, + "tokens_seen": 423297024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044172517552659, + "loss": 1.5146, + "theoretical_loss": 4.015137273649826, + "tokens_seen": 423362560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044012036108325, + "loss": 1.6298, + "theoretical_loss": 4.015068893868528, + "tokens_seen": 423428096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043851554663993, + "loss": 1.3893, + "theoretical_loss": 4.015000527632726, + "tokens_seen": 423493632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043691073219659, + "loss": 1.452, + "theoretical_loss": 4.014932174937642, + "tokens_seen": 423559168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043530591775327, + "loss": 1.4128, + "theoretical_loss": 4.014863835778499, + "tokens_seen": 423624704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043370110330993, + "loss": 1.5542, + "theoretical_loss": 4.014795510150522, + "tokens_seen": 423690240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043209628886661, + "loss": 1.6039, + "theoretical_loss": 4.014727198048942, + "tokens_seen": 423755776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043049147442327, + "loss": 1.5696, + "theoretical_loss": 4.014658899468988, + "tokens_seen": 423821312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042888665997995, + "loss": 1.566, + "theoretical_loss": 4.014590614405893, + "tokens_seen": 423886848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042728184553661, + "loss": 1.5941, + "theoretical_loss": 4.014522342854895, + "tokens_seen": 423952384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042567703109329, + "loss": 1.4703, + "theoretical_loss": 4.01445408481123, + "tokens_seen": 424017920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042407221664995, + "loss": 1.5825, + "theoretical_loss": 4.014385840270139, + "tokens_seen": 424083456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042246740220663, + "loss": 1.5623, + "theoretical_loss": 4.014317609226865, + "tokens_seen": 424148992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042086258776329, + "loss": 1.466, + "theoretical_loss": 4.014249391676652, + "tokens_seen": 424214528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041925777331997, + "loss": 1.4422, + "theoretical_loss": 4.014181187614751, + "tokens_seen": 424280064 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041765295887663, + "loss": 1.4518, + "theoretical_loss": 4.01411299703641, + "tokens_seen": 424345600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041604814443331, + "loss": 1.5206, + "theoretical_loss": 4.014044819936882, + "tokens_seen": 424411136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041444332998997, + "loss": 1.4531, + "theoretical_loss": 4.013976656311422, + "tokens_seen": 424476672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041283851554665, + "loss": 1.5651, + "theoretical_loss": 4.013908506155287, + "tokens_seen": 424542208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041123370110331, + "loss": 1.5244, + "theoretical_loss": 4.013840369463737, + "tokens_seen": 424607744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040962888665999, + "loss": 1.5536, + "theoretical_loss": 4.013772246232035, + "tokens_seen": 424673280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040802407221665, + "loss": 1.4733, + "theoretical_loss": 4.013704136455445, + "tokens_seen": 424738816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040641925777333, + "loss": 1.4762, + "theoretical_loss": 4.013636040129233, + "tokens_seen": 424804352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040481444332999, + "loss": 1.4481, + "theoretical_loss": 4.0135679572486715, + "tokens_seen": 424869888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040320962888667, + "loss": 1.5199, + "theoretical_loss": 4.013499887809028, + "tokens_seen": 424935424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040160481444333, + "loss": 1.5524, + "theoretical_loss": 4.01343183180558, + "tokens_seen": 425000960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040000000000001, + "loss": 1.5354, + "theoretical_loss": 4.013363789233602, + "tokens_seen": 425066496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039839518555667, + "loss": 1.4442, + "theoretical_loss": 4.013295760088374, + "tokens_seen": 425132032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039679037111334, + "loss": 1.4176, + "theoretical_loss": 4.013227744365178, + "tokens_seen": 425197568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039518555667001, + "loss": 1.4971, + "theoretical_loss": 4.013159742059296, + "tokens_seen": 425263104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039358074222668, + "loss": 1.4715, + "theoretical_loss": 4.013091753166015, + "tokens_seen": 425328640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039197592778335, + "loss": 1.5929, + "theoretical_loss": 4.013023777680624, + "tokens_seen": 425394176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039037111334002, + "loss": 1.5164, + "theoretical_loss": 4.0129558155984135, + "tokens_seen": 425459712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038876629889669, + "loss": 1.395, + "theoretical_loss": 4.012887866914675, + "tokens_seen": 425525248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038716148445336, + "loss": 1.4983, + "theoretical_loss": 4.0128199316247075, + "tokens_seen": 425590784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038555667001003, + "loss": 1.5055, + "theoretical_loss": 4.0127520097238065, + "tokens_seen": 425656320 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703839518555667, + "loss": 1.5522, + "theoretical_loss": 4.012684101207273, + "tokens_seen": 425721856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038234704112337, + "loss": 1.3876, + "theoretical_loss": 4.01261620607041, + "tokens_seen": 425787392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038074222668004, + "loss": 1.5193, + "theoretical_loss": 4.012548324308522, + "tokens_seen": 425852928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037913741223671, + "loss": 1.5764, + "theoretical_loss": 4.012480455916918, + "tokens_seen": 425918464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 148172, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.435992956161499, + "objective/train/theoretical_loss": 4.012412600890906, + "objective/train/tokens_used": 446444000, + "theoretical_loss": 4.012412600890906, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037753259779338, + "loss": 1.5679, + "theoretical_loss": 4.012412600890906, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037592778335005, + "loss": 1.5018, + "theoretical_loss": 4.012344759225799, + "tokens_seen": 426049536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037432296890672, + "loss": 1.5631, + "theoretical_loss": 4.0122769309169115, + "tokens_seen": 426115072 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703727181544634, + "loss": 1.5465, + "theoretical_loss": 4.0122091159595605, + "tokens_seen": 426180608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037111334002006, + "loss": 1.4451, + "theoretical_loss": 4.0121413143490665, + "tokens_seen": 426246144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036950852557673, + "loss": 1.479, + "theoretical_loss": 4.012073526080749, + "tokens_seen": 426311680 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703679037111334, + "loss": 1.5421, + "theoretical_loss": 4.012005751149934, + "tokens_seen": 426377216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036629889669007, + "loss": 1.416, + "theoretical_loss": 4.011937989551946, + "tokens_seen": 426442752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036469408224674, + "loss": 1.5782, + "theoretical_loss": 4.011870241282114, + "tokens_seen": 426508288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036308926780342, + "loss": 1.4749, + "theoretical_loss": 4.011802506335771, + "tokens_seen": 426573824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036148445336008, + "loss": 1.515, + "theoretical_loss": 4.011734784708249, + "tokens_seen": 426639360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035987963891676, + "loss": 1.5876, + "theoretical_loss": 4.011667076394884, + "tokens_seen": 426704896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035827482447342, + "loss": 1.4823, + "theoretical_loss": 4.011599381391014, + "tokens_seen": 426770432 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703566700100301, + "loss": 1.5256, + "theoretical_loss": 4.0115316996919805, + "tokens_seen": 426835968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035506519558676, + "loss": 1.4832, + "theoretical_loss": 4.011464031293125, + "tokens_seen": 426901504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035346038114344, + "loss": 1.4891, + "theoretical_loss": 4.0113963761897935, + "tokens_seen": 426967040 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703518555667001, + "loss": 1.4984, + "theoretical_loss": 4.011328734377333, + "tokens_seen": 427032576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035025075225678, + "loss": 1.4975, + "theoretical_loss": 4.011261105851093, + "tokens_seen": 427098112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034864593781344, + "loss": 1.51, + "theoretical_loss": 4.011193490606427, + "tokens_seen": 427163648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034704112337012, + "loss": 1.5087, + "theoretical_loss": 4.01112588863869, + "tokens_seen": 427229184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034543630892678, + "loss": 1.4949, + "theoretical_loss": 4.011058299943237, + "tokens_seen": 427294720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034383149448346, + "loss": 1.501, + "theoretical_loss": 4.010990724515427, + "tokens_seen": 427360256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034222668004012, + "loss": 1.4712, + "theoretical_loss": 4.010923162350624, + "tokens_seen": 427425792 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703406218655968, + "loss": 1.5337, + "theoretical_loss": 4.01085561344419, + "tokens_seen": 427491328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033901705115346, + "loss": 1.5073, + "theoretical_loss": 4.010788077791491, + "tokens_seen": 427556864 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033741223671014, + "loss": 1.4828, + "theoretical_loss": 4.0107205553878975, + "tokens_seen": 427622400 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703358074222668, + "loss": 1.5587, + "theoretical_loss": 4.010653046228779, + "tokens_seen": 427687936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033420260782348, + "loss": 1.4948, + "theoretical_loss": 4.010585550309508, + "tokens_seen": 427753472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033259779338014, + "loss": 1.4606, + "theoretical_loss": 4.010518067625461, + "tokens_seen": 427819008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033099297893682, + "loss": 1.5048, + "theoretical_loss": 4.010450598172016, + "tokens_seen": 427884544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032938816449348, + "loss": 1.6048, + "theoretical_loss": 4.010383141944553, + "tokens_seen": 427950080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032778335005016, + "loss": 1.567, + "theoretical_loss": 4.010315698938455, + "tokens_seen": 428015616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032617853560682, + "loss": 1.4507, + "theoretical_loss": 4.0102482691491055, + "tokens_seen": 428081152 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703245737211635, + "loss": 1.5643, + "theoretical_loss": 4.010180852571893, + "tokens_seen": 428146688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032296890672016, + "loss": 1.5148, + "theoretical_loss": 4.010113449202206, + "tokens_seen": 428212224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032136409227684, + "loss": 1.4841, + "theoretical_loss": 4.010046059035437, + "tokens_seen": 428277760 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703197592778335, + "loss": 1.438, + "theoretical_loss": 4.009978682066979, + "tokens_seen": 428343296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031815446339018, + "loss": 1.5299, + "theoretical_loss": 4.00991131829223, + "tokens_seen": 428408832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031654964894684, + "loss": 1.5179, + "theoretical_loss": 4.009843967706586, + "tokens_seen": 428474368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031494483450352, + "loss": 1.4384, + "theoretical_loss": 4.009776630305451, + "tokens_seen": 428539904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031334002006018, + "loss": 1.4572, + "theoretical_loss": 4.009709306084227, + "tokens_seen": 428605440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031173520561686, + "loss": 1.5282, + "theoretical_loss": 4.00964199503832, + "tokens_seen": 428670976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031013039117352, + "loss": 1.476, + "theoretical_loss": 4.009574697163137, + "tokens_seen": 428736512 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703085255767302, + "loss": 1.6019, + "theoretical_loss": 4.00950741245409, + "tokens_seen": 428802048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007030692076228686, + "loss": 1.4104, + "theoretical_loss": 4.009440140906589, + "tokens_seen": 428867584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007030531594784354, + "loss": 1.6, + "theoretical_loss": 4.00937288251605, + "tokens_seen": 428933120 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703037111334002, + "loss": 1.5577, + "theoretical_loss": 4.009305637277892, + "tokens_seen": 428998656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007030210631895688, + "loss": 1.5977, + "theoretical_loss": 4.009238405187531, + "tokens_seen": 429064192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007030050150451354, + "loss": 1.57, + "theoretical_loss": 4.009171186240391, + "tokens_seen": 429129728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029889669007022, + "loss": 1.478, + "theoretical_loss": 4.009103980431896, + "tokens_seen": 429195264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 149293, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.394667625427246, + "objective/train/theoretical_loss": 4.009036787757472, + "objective/train/tokens_used": 449720800, + "theoretical_loss": 4.009036787757472, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029729187562688, + "loss": 1.4451, + "theoretical_loss": 4.009036787757472, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029568706118356, + "loss": 1.6264, + "theoretical_loss": 4.0089696082125466, + "tokens_seen": 429326336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029408224674022, + "loss": 1.6049, + "theoretical_loss": 4.008902441792552, + "tokens_seen": 429391872 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702924774322969, + "loss": 1.6117, + "theoretical_loss": 4.00883528849292, + "tokens_seen": 429457408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029087261785356, + "loss": 1.6004, + "theoretical_loss": 4.008768148309087, + "tokens_seen": 429522944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028926780341024, + "loss": 1.5711, + "theoretical_loss": 4.00870102123649, + "tokens_seen": 429588480 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702876629889669, + "loss": 1.4821, + "theoretical_loss": 4.008633907270571, + "tokens_seen": 429654016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028605817452358, + "loss": 1.5774, + "theoretical_loss": 4.008566806406771, + "tokens_seen": 429719552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028445336008024, + "loss": 1.3801, + "theoretical_loss": 4.0084997186405324, + "tokens_seen": 429785088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028284854563692, + "loss": 1.5135, + "theoretical_loss": 4.008432643967305, + "tokens_seen": 429850624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028124373119358, + "loss": 1.4067, + "theoretical_loss": 4.008365582382538, + "tokens_seen": 429916160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027963891675026, + "loss": 1.6318, + "theoretical_loss": 4.008298533881681, + "tokens_seen": 429981696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027803410230692, + "loss": 1.5392, + "theoretical_loss": 4.008231498460189, + "tokens_seen": 430047232 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702764292878636, + "loss": 1.4467, + "theoretical_loss": 4.008164476113517, + "tokens_seen": 430112768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027482447342026, + "loss": 1.5168, + "theoretical_loss": 4.008097466837124, + "tokens_seen": 430178304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027321965897694, + "loss": 1.5886, + "theoretical_loss": 4.00803047062647, + "tokens_seen": 430243840 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702716148445336, + "loss": 1.5773, + "theoretical_loss": 4.007963487477018, + "tokens_seen": 430309376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027001003009028, + "loss": 1.5376, + "theoretical_loss": 4.007896517384233, + "tokens_seen": 430374912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026840521564694, + "loss": 1.5965, + "theoretical_loss": 4.007829560343582, + "tokens_seen": 430440448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026680040120362, + "loss": 1.5795, + "theoretical_loss": 4.007762616350535, + "tokens_seen": 430505984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026519558676028, + "loss": 1.4647, + "theoretical_loss": 4.007695685400563, + "tokens_seen": 430571520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026359077231696, + "loss": 1.552, + "theoretical_loss": 4.007628767489142, + "tokens_seen": 430637056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026198595787362, + "loss": 1.4869, + "theoretical_loss": 4.007561862611746, + "tokens_seen": 430702592 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702603811434303, + "loss": 1.4319, + "theoretical_loss": 4.007494970763855, + "tokens_seen": 430768128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025877632898696, + "loss": 1.5502, + "theoretical_loss": 4.00742809194095, + "tokens_seen": 430833664 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025717151454363, + "loss": 1.5765, + "theoretical_loss": 4.007361226138513, + "tokens_seen": 430899200 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702555667001003, + "loss": 1.4739, + "theoretical_loss": 4.00729437335203, + "tokens_seen": 430964736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025396188565697, + "loss": 1.3935, + "theoretical_loss": 4.007227533576988, + "tokens_seen": 431030272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025235707121364, + "loss": 1.5863, + "theoretical_loss": 4.007160706808879, + "tokens_seen": 431095808 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025075225677031, + "loss": 1.4851, + "theoretical_loss": 4.007093893043192, + "tokens_seen": 431161344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024914744232698, + "loss": 1.5307, + "theoretical_loss": 4.007027092275424, + "tokens_seen": 431226880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024754262788365, + "loss": 1.5869, + "theoretical_loss": 4.006960304501069, + "tokens_seen": 431292416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024593781344032, + "loss": 1.5409, + "theoretical_loss": 4.0068935297156285, + "tokens_seen": 431357952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024433299899699, + "loss": 1.572, + "theoretical_loss": 4.0068267679146015, + "tokens_seen": 431423488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024272818455366, + "loss": 1.4711, + "theoretical_loss": 4.006760019093493, + "tokens_seen": 431489024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024112337011033, + "loss": 1.4946, + "theoretical_loss": 4.006693283247808, + "tokens_seen": 431554560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070239518555667, + "loss": 1.4717, + "theoretical_loss": 4.0066265603730535, + "tokens_seen": 431620096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023791374122367, + "loss": 1.5803, + "theoretical_loss": 4.006559850464741, + "tokens_seen": 431685632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023630892678034, + "loss": 1.652, + "theoretical_loss": 4.006493153518381, + "tokens_seen": 431751168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023470411233701, + "loss": 1.5243, + "theoretical_loss": 4.006426469529488, + "tokens_seen": 431816704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023309929789368, + "loss": 1.4888, + "theoretical_loss": 4.006359798493582, + "tokens_seen": 431882240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023149448345035, + "loss": 1.5452, + "theoretical_loss": 4.006293140406179, + "tokens_seen": 431947776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022988966900702, + "loss": 1.5458, + "theoretical_loss": 4.0062264952627995, + "tokens_seen": 432013312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022828485456369, + "loss": 1.6067, + "theoretical_loss": 4.00615986305897, + "tokens_seen": 432078848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022668004012037, + "loss": 1.5937, + "theoretical_loss": 4.006093243790214, + "tokens_seen": 432144384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022507522567703, + "loss": 1.6446, + "theoretical_loss": 4.0060266374520594, + "tokens_seen": 432209920 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702234704112337, + "loss": 1.5289, + "theoretical_loss": 4.005960044040037, + "tokens_seen": 432275456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022186559679037, + "loss": 1.5373, + "theoretical_loss": 4.005893463549679, + "tokens_seen": 432340992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022026078234705, + "loss": 1.502, + "theoretical_loss": 4.0058268959765195, + "tokens_seen": 432406528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021865596790371, + "loss": 1.4267, + "theoretical_loss": 4.005760341316096, + "tokens_seen": 432472064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 150155, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2991299629211426, + "objective/train/theoretical_loss": 4.0056937995639466, + "objective/train/tokens_used": 452997600, + "theoretical_loss": 4.0056937995639466, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021705115346039, + "loss": 1.4196, + "theoretical_loss": 4.0056937995639466, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021544633901705, + "loss": 1.5659, + "theoretical_loss": 4.005627270715614, + "tokens_seen": 432603136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021384152457373, + "loss": 1.5304, + "theoretical_loss": 4.00556075476664, + "tokens_seen": 432668672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021223671013039, + "loss": 1.5103, + "theoretical_loss": 4.005494251712571, + "tokens_seen": 432734208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021063189568707, + "loss": 1.4877, + "theoretical_loss": 4.005427761548955, + "tokens_seen": 432799744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020902708124373, + "loss": 1.5152, + "theoretical_loss": 4.005361284271341, + "tokens_seen": 432865280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020742226680041, + "loss": 1.3662, + "theoretical_loss": 4.005294819875283, + "tokens_seen": 432930816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020581745235707, + "loss": 1.5065, + "theoretical_loss": 4.005228368356335, + "tokens_seen": 432996352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020421263791375, + "loss": 1.5918, + "theoretical_loss": 4.005161929710052, + "tokens_seen": 433061888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020260782347041, + "loss": 1.5184, + "theoretical_loss": 4.0050955039319955, + "tokens_seen": 433127424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020100300902709, + "loss": 1.4619, + "theoretical_loss": 4.0050290910177235, + "tokens_seen": 433192960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019939819458375, + "loss": 1.5408, + "theoretical_loss": 4.004962690962802, + "tokens_seen": 433258496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019779338014043, + "loss": 1.4712, + "theoretical_loss": 4.004896303762797, + "tokens_seen": 433324032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019618856569709, + "loss": 1.4964, + "theoretical_loss": 4.0048299294132725, + "tokens_seen": 433389568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019458375125377, + "loss": 1.5561, + "theoretical_loss": 4.0047635679098015, + "tokens_seen": 433455104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019297893681043, + "loss": 1.4142, + "theoretical_loss": 4.004697219247955, + "tokens_seen": 433520640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019137412236711, + "loss": 1.5248, + "theoretical_loss": 4.004630883423308, + "tokens_seen": 433586176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018976930792377, + "loss": 1.5179, + "theoretical_loss": 4.004564560431437, + "tokens_seen": 433651712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018816449348045, + "loss": 1.6083, + "theoretical_loss": 4.004498250267919, + "tokens_seen": 433717248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018655967903711, + "loss": 1.4272, + "theoretical_loss": 4.004431952928337, + "tokens_seen": 433782784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018495486459379, + "loss": 1.5259, + "theoretical_loss": 4.004365668408273, + "tokens_seen": 433848320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018335005015045, + "loss": 1.4605, + "theoretical_loss": 4.004299396703312, + "tokens_seen": 433913856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018174523570713, + "loss": 1.5143, + "theoretical_loss": 4.004233137809042, + "tokens_seen": 433979392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018014042126379, + "loss": 1.5833, + "theoretical_loss": 4.004166891721052, + "tokens_seen": 434044928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017853560682047, + "loss": 1.4719, + "theoretical_loss": 4.004100658434934, + "tokens_seen": 434110464 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017693079237713, + "loss": 1.5535, + "theoretical_loss": 4.004034437946283, + "tokens_seen": 434176000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017532597793381, + "loss": 1.4578, + "theoretical_loss": 4.003968230250694, + "tokens_seen": 434241536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017372116349047, + "loss": 1.5173, + "theoretical_loss": 4.003902035343766, + "tokens_seen": 434307072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017211634904715, + "loss": 1.5396, + "theoretical_loss": 4.003835853221099, + "tokens_seen": 434372608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017051153460381, + "loss": 1.3807, + "theoretical_loss": 4.003769683878296, + "tokens_seen": 434438144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016890672016049, + "loss": 1.4859, + "theoretical_loss": 4.003703527310962, + "tokens_seen": 434503680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016730190571715, + "loss": 1.4641, + "theoretical_loss": 4.003637383514704, + "tokens_seen": 434569216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016569709127383, + "loss": 1.5701, + "theoretical_loss": 4.00357125248513, + "tokens_seen": 434634752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016409227683049, + "loss": 1.5586, + "theoretical_loss": 4.003505134217853, + "tokens_seen": 434700288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016248746238717, + "loss": 1.5719, + "theoretical_loss": 4.0034390287084864, + "tokens_seen": 434765824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016088264794383, + "loss": 1.6704, + "theoretical_loss": 4.003372935952646, + "tokens_seen": 434831360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015927783350051, + "loss": 1.5484, + "theoretical_loss": 4.003306855945948, + "tokens_seen": 434896896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015767301905717, + "loss": 1.4919, + "theoretical_loss": 4.003240788684015, + "tokens_seen": 434962432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015606820461385, + "loss": 1.5402, + "theoretical_loss": 4.003174734162467, + "tokens_seen": 435027968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015446339017051, + "loss": 1.515, + "theoretical_loss": 4.00310869237693, + "tokens_seen": 435093504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015285857572719, + "loss": 1.5989, + "theoretical_loss": 4.003042663323029, + "tokens_seen": 435159040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015125376128385, + "loss": 1.4633, + "theoretical_loss": 4.0029766469963945, + "tokens_seen": 435224576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014964894684053, + "loss": 1.4019, + "theoretical_loss": 4.0029106433926565, + "tokens_seen": 435290112 + }, + { + "epoch": 0.13, + "learning_rate": 0.000701480441323972, + "loss": 1.5274, + "theoretical_loss": 4.002844652507449, + "tokens_seen": 435355648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014643931795387, + "loss": 1.4794, + "theoretical_loss": 4.002778674336405, + "tokens_seen": 435421184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014483450351054, + "loss": 1.5274, + "theoretical_loss": 4.002712708875162, + "tokens_seen": 435486720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014322968906721, + "loss": 1.5422, + "theoretical_loss": 4.002646756119362, + "tokens_seen": 435552256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014162487462388, + "loss": 1.5547, + "theoretical_loss": 4.002580816064645, + "tokens_seen": 435617792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014002006018055, + "loss": 1.5411, + "theoretical_loss": 4.002514888706655, + "tokens_seen": 435683328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013841524573722, + "loss": 1.374, + "theoretical_loss": 4.002448974041037, + "tokens_seen": 435748864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 151381, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.41636323928833, + "objective/train/theoretical_loss": 4.0023830720634415, + "objective/train/tokens_used": 456274400, + "theoretical_loss": 4.0023830720634415, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013681043129389, + "loss": 1.3912, + "theoretical_loss": 4.0023830720634415, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013520561685056, + "loss": 1.4124, + "theoretical_loss": 4.0023171827695165, + "tokens_seen": 435879936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013360080240723, + "loss": 1.4398, + "theoretical_loss": 4.002251306154916, + "tokens_seen": 435945472 + }, + { + "epoch": 0.13, + "learning_rate": 0.000701319959879639, + "loss": 1.5039, + "theoretical_loss": 4.002185442215293, + "tokens_seen": 436011008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013039117352057, + "loss": 1.4421, + "theoretical_loss": 4.002119590946305, + "tokens_seen": 436076544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012878635907723, + "loss": 1.5099, + "theoretical_loss": 4.002053752343611, + "tokens_seen": 436142080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012718154463391, + "loss": 1.523, + "theoretical_loss": 4.001987926402871, + "tokens_seen": 436207616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012557673019057, + "loss": 1.46, + "theoretical_loss": 4.001922113119749, + "tokens_seen": 436273152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012397191574725, + "loss": 1.542, + "theoretical_loss": 4.00185631248991, + "tokens_seen": 436338688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012236710130391, + "loss": 1.4672, + "theoretical_loss": 4.001790524509021, + "tokens_seen": 436404224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012076228686059, + "loss": 1.5367, + "theoretical_loss": 4.001724749172752, + "tokens_seen": 436469760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011915747241725, + "loss": 1.5853, + "theoretical_loss": 4.001658986476775, + "tokens_seen": 436535296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011755265797393, + "loss": 1.4906, + "theoretical_loss": 4.0015932364167615, + "tokens_seen": 436600832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011594784353059, + "loss": 1.5542, + "theoretical_loss": 4.00152749898839, + "tokens_seen": 436666368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011434302908726, + "loss": 1.3937, + "theoretical_loss": 4.001461774187338, + "tokens_seen": 436731904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011273821464393, + "loss": 1.6308, + "theoretical_loss": 4.001396062009285, + "tokens_seen": 436797440 + }, + { + "epoch": 0.13, + "learning_rate": 0.000701111334002006, + "loss": 1.5693, + "theoretical_loss": 4.001330362449914, + "tokens_seen": 436862976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010952858575727, + "loss": 1.4551, + "theoretical_loss": 4.001264675504908, + "tokens_seen": 436928512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010792377131394, + "loss": 1.5887, + "theoretical_loss": 4.001199001169955, + "tokens_seen": 436994048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010631895687061, + "loss": 1.5951, + "theoretical_loss": 4.001133339440743, + "tokens_seen": 437059584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010471414242728, + "loss": 1.448, + "theoretical_loss": 4.001067690312963, + "tokens_seen": 437125120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010310932798395, + "loss": 1.4664, + "theoretical_loss": 4.001002053782307, + "tokens_seen": 437190656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010150451354062, + "loss": 1.4245, + "theoretical_loss": 4.0009364298444705, + "tokens_seen": 437256192 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700998996990973, + "loss": 1.464, + "theoretical_loss": 4.000870818495152, + "tokens_seen": 437321728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009829488465396, + "loss": 1.5072, + "theoretical_loss": 4.000805219730049, + "tokens_seen": 437387264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009669007021063, + "loss": 1.3711, + "theoretical_loss": 4.000739633544863, + "tokens_seen": 437452800 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700950852557673, + "loss": 1.5206, + "theoretical_loss": 4.000674059935298, + "tokens_seen": 437518336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009348044132397, + "loss": 1.5586, + "theoretical_loss": 4.000608498897059, + "tokens_seen": 437583872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009187562688064, + "loss": 1.6071, + "theoretical_loss": 4.000542950425855, + "tokens_seen": 437649408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009027081243731, + "loss": 1.4701, + "theoretical_loss": 4.000477414517394, + "tokens_seen": 437714944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008866599799398, + "loss": 1.4597, + "theoretical_loss": 4.000411891167389, + "tokens_seen": 437780480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008706118355066, + "loss": 1.5, + "theoretical_loss": 4.000346380371554, + "tokens_seen": 437846016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008545636910732, + "loss": 1.4292, + "theoretical_loss": 4.000280882125605, + "tokens_seen": 437911552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070083851554664, + "loss": 1.45, + "theoretical_loss": 4.000215396425259, + "tokens_seen": 437977088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008224674022066, + "loss": 1.5011, + "theoretical_loss": 4.000149923266237, + "tokens_seen": 438042624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008064192577734, + "loss": 1.4882, + "theoretical_loss": 4.0000844626442635, + "tokens_seen": 438108160 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070079037111334, + "loss": 1.5182, + "theoretical_loss": 4.000019014555059, + "tokens_seen": 438173696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007743229689068, + "loss": 1.402, + "theoretical_loss": 3.9999535789943543, + "tokens_seen": 438239232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007582748244734, + "loss": 1.591, + "theoretical_loss": 3.9998881559578754, + "tokens_seen": 438304768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007422266800402, + "loss": 1.4926, + "theoretical_loss": 3.999822745441353, + "tokens_seen": 438370304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007261785356068, + "loss": 1.504, + "theoretical_loss": 3.9997573474405215, + "tokens_seen": 438435840 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007101303911736, + "loss": 1.4095, + "theoretical_loss": 3.9996919619511146, + "tokens_seen": 438501376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007006940822467402, + "loss": 1.5043, + "theoretical_loss": 3.9996265889688702, + "tokens_seen": 438566912 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700678034102307, + "loss": 1.4652, + "theoretical_loss": 3.999561228489527, + "tokens_seen": 438632448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007006619859578736, + "loss": 1.444, + "theoretical_loss": 3.9994958805088268, + "tokens_seen": 438697984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007006459378134404, + "loss": 1.5337, + "theoretical_loss": 3.999430545022512, + "tokens_seen": 438763520 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700629889669007, + "loss": 1.5445, + "theoretical_loss": 3.999365222026329, + "tokens_seen": 438829056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007006138415245738, + "loss": 1.689, + "theoretical_loss": 3.999299911516024, + "tokens_seen": 438894592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005977933801404, + "loss": 1.5859, + "theoretical_loss": 3.9992346134873484, + "tokens_seen": 438960128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005817452357072, + "loss": 1.4005, + "theoretical_loss": 3.999169327936052, + "tokens_seen": 439025664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 152335, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6528295278549194, + "objective/train/theoretical_loss": 3.99910405485789, + "objective/train/tokens_used": 459551200, + "theoretical_loss": 3.99910405485789, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005656970912738, + "loss": 1.5898, + "theoretical_loss": 3.99910405485789, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005496489468406, + "loss": 1.601, + "theoretical_loss": 3.9990387942486167, + "tokens_seen": 439156736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005336008024072, + "loss": 1.5703, + "theoretical_loss": 3.9989735461039917, + "tokens_seen": 439222272 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700517552657974, + "loss": 1.5155, + "theoretical_loss": 3.9989083104197736, + "tokens_seen": 439287808 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005015045135406, + "loss": 1.5189, + "theoretical_loss": 3.9988430871917253, + "tokens_seen": 439353344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004854563691074, + "loss": 1.4124, + "theoretical_loss": 3.9987778764156108, + "tokens_seen": 439418880 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700469408224674, + "loss": 1.4934, + "theoretical_loss": 3.9987126780871955, + "tokens_seen": 439484416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004533600802408, + "loss": 1.4822, + "theoretical_loss": 3.998647492202249, + "tokens_seen": 439549952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004373119358074, + "loss": 1.4562, + "theoretical_loss": 3.99858231875654, + "tokens_seen": 439615488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004212637913742, + "loss": 1.6713, + "theoretical_loss": 3.9985171577458427, + "tokens_seen": 439681024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004052156469408, + "loss": 1.4871, + "theoretical_loss": 3.99845200916593, + "tokens_seen": 439746560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007003891675025076, + "loss": 1.5559, + "theoretical_loss": 3.9983868730125796, + "tokens_seen": 439812096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007003731193580742, + "loss": 1.4817, + "theoretical_loss": 3.998321749281569, + "tokens_seen": 439877632 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700357071213641, + "loss": 1.6303, + "theoretical_loss": 3.99825663796868, + "tokens_seen": 439943168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007003410230692076, + "loss": 1.5342, + "theoretical_loss": 3.998191539069694, + "tokens_seen": 440008704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007003249749247744, + "loss": 1.4689, + "theoretical_loss": 3.9981264525803972, + "tokens_seen": 440074240 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700308926780341, + "loss": 1.636, + "theoretical_loss": 3.9980613784965753, + "tokens_seen": 440139776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002928786359078, + "loss": 1.4691, + "theoretical_loss": 3.9979963168140182, + "tokens_seen": 440205312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002768304914745, + "loss": 1.5015, + "theoretical_loss": 3.997931267528516, + "tokens_seen": 440270848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002607823470412, + "loss": 1.5555, + "theoretical_loss": 3.997866230635862, + "tokens_seen": 440336384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002447342026079, + "loss": 1.4922, + "theoretical_loss": 3.9978012061318515, + "tokens_seen": 440401920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002286860581746, + "loss": 1.5052, + "theoretical_loss": 3.997736194012281, + "tokens_seen": 440467456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002126379137413, + "loss": 1.4865, + "theoretical_loss": 3.99767119427295, + "tokens_seen": 440532992 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700196589769308, + "loss": 1.5147, + "theoretical_loss": 3.9976062069096603, + "tokens_seen": 440598528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001805416248747, + "loss": 1.5291, + "theoretical_loss": 3.9975412319182144, + "tokens_seen": 440664064 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001644934804414, + "loss": 1.4955, + "theoretical_loss": 3.9974762692944172, + "tokens_seen": 440729600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001484453360081, + "loss": 1.5683, + "theoretical_loss": 3.997411319034077, + "tokens_seen": 440795136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001323971915748, + "loss": 1.4643, + "theoretical_loss": 3.9973463811330037, + "tokens_seen": 440860672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001163490471415, + "loss": 1.436, + "theoretical_loss": 3.997281455587007, + "tokens_seen": 440926208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001003009027082, + "loss": 1.6007, + "theoretical_loss": 3.9972165423919015, + "tokens_seen": 440991744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000842527582749, + "loss": 1.4685, + "theoretical_loss": 3.9971516415435024, + "tokens_seen": 441057280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000682046138416, + "loss": 1.5206, + "theoretical_loss": 3.9970867530376273, + "tokens_seen": 441122816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000521564694083, + "loss": 1.4993, + "theoretical_loss": 3.9970218768700967, + "tokens_seen": 441188352 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700036108324975, + "loss": 1.48, + "theoretical_loss": 3.99695701303673, + "tokens_seen": 441253888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000200601805417, + "loss": 1.4757, + "theoretical_loss": 3.9968921615333532, + "tokens_seen": 441319424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000040120361084, + "loss": 1.5006, + "theoretical_loss": 3.996827322355791, + "tokens_seen": 441384960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999879638916751, + "loss": 1.432, + "theoretical_loss": 3.9967624954998713, + "tokens_seen": 441450496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999719157472418, + "loss": 1.7472, + "theoretical_loss": 3.9966976809614234, + "tokens_seen": 441516032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999558676028085, + "loss": 1.4991, + "theoretical_loss": 3.99663287873628, + "tokens_seen": 441581568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999398194583752, + "loss": 1.5962, + "theoretical_loss": 3.9965680888202737, + "tokens_seen": 441647104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999237713139419, + "loss": 1.4266, + "theoretical_loss": 3.996503311209242, + "tokens_seen": 441712640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999077231695086, + "loss": 1.587, + "theoretical_loss": 3.9964385458990206, + "tokens_seen": 441778176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006998916750250752, + "loss": 1.5794, + "theoretical_loss": 3.9963737928854517, + "tokens_seen": 441843712 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699875626880642, + "loss": 1.4391, + "theoretical_loss": 3.996309052164376, + "tokens_seen": 441909248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006998595787362086, + "loss": 1.5746, + "theoretical_loss": 3.996244323731638, + "tokens_seen": 441974784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006998435305917754, + "loss": 1.5798, + "theoretical_loss": 3.996179607583082, + "tokens_seen": 442040320 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699827482447342, + "loss": 1.3749, + "theoretical_loss": 3.9961149037145587, + "tokens_seen": 442105856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006998114343029088, + "loss": 1.554, + "theoretical_loss": 3.996050212121916, + "tokens_seen": 442171392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997953861584754, + "loss": 1.4281, + "theoretical_loss": 3.995985532801007, + "tokens_seen": 442236928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997793380140422, + "loss": 1.5208, + "theoretical_loss": 3.995920865747685, + "tokens_seen": 442302464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 153589, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5176739692687988, + "objective/train/theoretical_loss": 3.9958562109578066, + "objective/train/tokens_used": 462828000, + "theoretical_loss": 3.9958562109578066, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997632898696088, + "loss": 1.4968, + "theoretical_loss": 3.9958562109578066, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997472417251755, + "loss": 1.5038, + "theoretical_loss": 3.9957915684272294, + "tokens_seen": 442433536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997311935807422, + "loss": 1.4439, + "theoretical_loss": 3.995726938151815, + "tokens_seen": 442499072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997151454363089, + "loss": 1.644, + "theoretical_loss": 3.995662320127423, + "tokens_seen": 442564608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996990972918756, + "loss": 1.4451, + "theoretical_loss": 3.9955977143499197, + "tokens_seen": 442630144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996830491474423, + "loss": 1.612, + "theoretical_loss": 3.99553312081517, + "tokens_seen": 442695680 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699667001003009, + "loss": 1.6093, + "theoretical_loss": 3.995468539519042, + "tokens_seen": 442761216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996509528585757, + "loss": 1.4835, + "theoretical_loss": 3.9954039704574065, + "tokens_seen": 442826752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996349047141424, + "loss": 1.6179, + "theoretical_loss": 3.9953394136261355, + "tokens_seen": 442892288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996188565697091, + "loss": 1.5285, + "theoretical_loss": 3.995274869021103, + "tokens_seen": 442957824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996028084252758, + "loss": 1.5937, + "theoretical_loss": 3.995210336638185, + "tokens_seen": 443023360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995867602808425, + "loss": 1.6425, + "theoretical_loss": 3.9951458164732596, + "tokens_seen": 443088896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995707121364092, + "loss": 1.468, + "theoretical_loss": 3.9950813085222068, + "tokens_seen": 443154432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995546639919759, + "loss": 1.4978, + "theoretical_loss": 3.99501681278091, + "tokens_seen": 443219968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995386158475426, + "loss": 1.4505, + "theoretical_loss": 3.994952329245251, + "tokens_seen": 443285504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995225677031093, + "loss": 1.5477, + "theoretical_loss": 3.994887857911118, + "tokens_seen": 443351040 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699506519558676, + "loss": 1.5128, + "theoretical_loss": 3.9948233987743986, + "tokens_seen": 443416576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994904714142427, + "loss": 1.529, + "theoretical_loss": 3.994758951830982, + "tokens_seen": 443482112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994744232698095, + "loss": 1.5579, + "theoretical_loss": 3.9946945170767614, + "tokens_seen": 443547648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994583751253761, + "loss": 1.4624, + "theoretical_loss": 3.994630094507631, + "tokens_seen": 443613184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994423269809429, + "loss": 1.5406, + "theoretical_loss": 3.994565684119486, + "tokens_seen": 443678720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994262788365095, + "loss": 1.5907, + "theoretical_loss": 3.9945012859082247, + "tokens_seen": 443744256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994102306920763, + "loss": 1.4722, + "theoretical_loss": 3.9944368998697475, + "tokens_seen": 443809792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993941825476429, + "loss": 1.4417, + "theoretical_loss": 3.9943725259999563, + "tokens_seen": 443875328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993781344032097, + "loss": 1.5681, + "theoretical_loss": 3.994308164294755, + "tokens_seen": 443940864 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993620862587763, + "loss": 1.3864, + "theoretical_loss": 3.9942438147500496, + "tokens_seen": 444006400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993460381143431, + "loss": 1.5363, + "theoretical_loss": 3.9941794773617487, + "tokens_seen": 444071936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993299899699097, + "loss": 1.572, + "theoretical_loss": 3.9941151521257616, + "tokens_seen": 444137472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993139418254765, + "loss": 1.4912, + "theoretical_loss": 3.9940508390380005, + "tokens_seen": 444203008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992978936810431, + "loss": 1.6135, + "theoretical_loss": 3.9939865380943793, + "tokens_seen": 444268544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992818455366099, + "loss": 1.5042, + "theoretical_loss": 3.993922249290814, + "tokens_seen": 444334080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992657973921765, + "loss": 1.4777, + "theoretical_loss": 3.993857972623222, + "tokens_seen": 444399616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992497492477433, + "loss": 1.5358, + "theoretical_loss": 3.993793708087524, + "tokens_seen": 444465152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992337011033099, + "loss": 1.4613, + "theoretical_loss": 3.9937294556796417, + "tokens_seen": 444530688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992176529588767, + "loss": 1.6225, + "theoretical_loss": 3.993665215395498, + "tokens_seen": 444596224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992016048144433, + "loss": 1.5545, + "theoretical_loss": 3.9936009872310194, + "tokens_seen": 444661760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991855566700101, + "loss": 1.5691, + "theoretical_loss": 3.993536771182134, + "tokens_seen": 444727296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991695085255767, + "loss": 1.4441, + "theoretical_loss": 3.993472567244771, + "tokens_seen": 444792832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991534603811435, + "loss": 1.6013, + "theoretical_loss": 3.9934083754148615, + "tokens_seen": 444858368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991374122367101, + "loss": 1.4713, + "theoretical_loss": 3.9933441956883398, + "tokens_seen": 444923904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991213640922769, + "loss": 1.6447, + "theoretical_loss": 3.9932800280611422, + "tokens_seen": 444989440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991053159478435, + "loss": 1.4422, + "theoretical_loss": 3.9932158725292055, + "tokens_seen": 445054976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990892678034103, + "loss": 1.4835, + "theoretical_loss": 3.993151729088469, + "tokens_seen": 445120512 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699073219658977, + "loss": 1.4059, + "theoretical_loss": 3.993087597734875, + "tokens_seen": 445186048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990571715145437, + "loss": 1.4937, + "theoretical_loss": 3.993023478464366, + "tokens_seen": 445251584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990411233701104, + "loss": 1.4638, + "theoretical_loss": 3.992959371272888, + "tokens_seen": 445317120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990250752256771, + "loss": 1.57, + "theoretical_loss": 3.992895276156388, + "tokens_seen": 445382656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990090270812438, + "loss": 1.4905, + "theoretical_loss": 3.992831193110816, + "tokens_seen": 445448192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989929789368105, + "loss": 1.4754, + "theoretical_loss": 3.992767122132123, + "tokens_seen": 445513728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989769307923772, + "loss": 1.5195, + "theoretical_loss": 3.9927030632162612, + "tokens_seen": 445579264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 154520, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.673126220703125, + "objective/train/theoretical_loss": 3.9926390163591874, + "objective/train/tokens_used": 466104800, + "theoretical_loss": 3.9926390163591874, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989608826479439, + "loss": 1.5465, + "theoretical_loss": 3.9926390163591874, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989448345035106, + "loss": 1.4801, + "theoretical_loss": 3.9925749815568583, + "tokens_seen": 445710336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989287863590773, + "loss": 1.4372, + "theoretical_loss": 3.9925109588052328, + "tokens_seen": 445775872 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698912738214644, + "loss": 1.5426, + "theoretical_loss": 3.9924469481002713, + "tokens_seen": 445841408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988966900702107, + "loss": 1.5802, + "theoretical_loss": 3.9923829494379377, + "tokens_seen": 445906944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988806419257774, + "loss": 1.5393, + "theoretical_loss": 3.9923189628141964, + "tokens_seen": 445972480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988645937813441, + "loss": 1.4943, + "theoretical_loss": 3.9922549882250147, + "tokens_seen": 446038016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988485456369108, + "loss": 1.4797, + "theoretical_loss": 3.9921910256663606, + "tokens_seen": 446103552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988324974924775, + "loss": 1.4582, + "theoretical_loss": 3.9921270751342064, + "tokens_seen": 446169088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988164493480442, + "loss": 1.3838, + "theoretical_loss": 3.992063136624523, + "tokens_seen": 446234624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988004012036109, + "loss": 1.4467, + "theoretical_loss": 3.9919992101332866, + "tokens_seen": 446300160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987843530591776, + "loss": 1.5887, + "theoretical_loss": 3.991935295656473, + "tokens_seen": 446365696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987683049147443, + "loss": 1.4229, + "theoretical_loss": 3.9918713931900602, + "tokens_seen": 446431232 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698752256770311, + "loss": 1.5085, + "theoretical_loss": 3.9918075027300297, + "tokens_seen": 446496768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987362086258777, + "loss": 1.5026, + "theoretical_loss": 3.991743624272363, + "tokens_seen": 446562304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987201604814444, + "loss": 1.5529, + "theoretical_loss": 3.9916797578130456, + "tokens_seen": 446627840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987041123370111, + "loss": 1.4314, + "theoretical_loss": 3.9916159033480625, + "tokens_seen": 446693376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986880641925778, + "loss": 1.5042, + "theoretical_loss": 3.9915520608734028, + "tokens_seen": 446758912 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986720160481445, + "loss": 1.5427, + "theoretical_loss": 3.991488230385056, + "tokens_seen": 446824448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986559679037112, + "loss": 1.5715, + "theoretical_loss": 3.991424411879014, + "tokens_seen": 446889984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986399197592779, + "loss": 1.532, + "theoretical_loss": 3.9913606053512716, + "tokens_seen": 446955520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986238716148446, + "loss": 1.4223, + "theoretical_loss": 3.9912968107978237, + "tokens_seen": 447021056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986078234704113, + "loss": 1.5344, + "theoretical_loss": 3.9912330282146695, + "tokens_seen": 447086592 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698591775325978, + "loss": 1.606, + "theoretical_loss": 3.9911692575978077, + "tokens_seen": 447152128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985757271815447, + "loss": 1.4802, + "theoretical_loss": 3.9911054989432397, + "tokens_seen": 447217664 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985596790371114, + "loss": 1.4954, + "theoretical_loss": 3.9910417522469706, + "tokens_seen": 447283200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985436308926781, + "loss": 1.4643, + "theoretical_loss": 3.990978017505004, + "tokens_seen": 447348736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985275827482448, + "loss": 1.4879, + "theoretical_loss": 3.990914294713348, + "tokens_seen": 447414272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985115346038115, + "loss": 1.3527, + "theoretical_loss": 3.9908505838680126, + "tokens_seen": 447479808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984954864593781, + "loss": 1.5504, + "theoretical_loss": 3.9907868849650088, + "tokens_seen": 447545344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984794383149449, + "loss": 1.4939, + "theoretical_loss": 3.9907231980003495, + "tokens_seen": 447610880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984633901705115, + "loss": 1.4516, + "theoretical_loss": 3.99065952297005, + "tokens_seen": 447676416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984473420260783, + "loss": 1.4114, + "theoretical_loss": 3.990595859870127, + "tokens_seen": 447741952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984312938816449, + "loss": 1.6211, + "theoretical_loss": 3.9905322086966004, + "tokens_seen": 447807488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984152457372117, + "loss": 1.5145, + "theoretical_loss": 3.990468569445489, + "tokens_seen": 447873024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983991975927783, + "loss": 1.5833, + "theoretical_loss": 3.990404942112818, + "tokens_seen": 447938560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983831494483451, + "loss": 1.4963, + "theoretical_loss": 3.9903413266946104, + "tokens_seen": 448004096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983671013039117, + "loss": 1.4331, + "theoretical_loss": 3.9902777231868933, + "tokens_seen": 448069632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983510531594784, + "loss": 1.4788, + "theoretical_loss": 3.990214131585695, + "tokens_seen": 448135168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983350050150451, + "loss": 1.4973, + "theoretical_loss": 3.990150551887047, + "tokens_seen": 448200704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983189568706118, + "loss": 1.4562, + "theoretical_loss": 3.9900869840869793, + "tokens_seen": 448266240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983029087261785, + "loss": 1.6024, + "theoretical_loss": 3.9900234281815283, + "tokens_seen": 448331776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982868605817452, + "loss": 1.4377, + "theoretical_loss": 3.989959884166728, + "tokens_seen": 448397312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982708124373119, + "loss": 1.5387, + "theoretical_loss": 3.9898963520386186, + "tokens_seen": 448462848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982547642928786, + "loss": 1.5395, + "theoretical_loss": 3.9898328317932386, + "tokens_seen": 448528384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982387161484453, + "loss": 1.5434, + "theoretical_loss": 3.98976932342663, + "tokens_seen": 448593920 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698222668004012, + "loss": 1.6005, + "theoretical_loss": 3.9897058269348364, + "tokens_seen": 448659456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982066198595787, + "loss": 1.5173, + "theoretical_loss": 3.9896423423139034, + "tokens_seen": 448724992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981905717151454, + "loss": 1.4321, + "theoretical_loss": 3.989578869559879, + "tokens_seen": 448790528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981745235707121, + "loss": 1.3811, + "theoretical_loss": 3.989515408668812, + "tokens_seen": 448856064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 155690, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5394407510757446, + "objective/train/theoretical_loss": 3.989451959636754, + "objective/train/tokens_used": 469381600, + "theoretical_loss": 3.989451959636754, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981584754262788, + "loss": 1.4618, + "theoretical_loss": 3.989451959636754, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981424272818456, + "loss": 1.5767, + "theoretical_loss": 3.989388522459757, + "tokens_seen": 448987136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981263791374122, + "loss": 1.5553, + "theoretical_loss": 3.989325097133878, + "tokens_seen": 449052672 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698110330992979, + "loss": 1.59, + "theoretical_loss": 3.989261683655172, + "tokens_seen": 449118208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980942828485456, + "loss": 1.5345, + "theoretical_loss": 3.989198282019699, + "tokens_seen": 449183744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980782347041124, + "loss": 1.4384, + "theoretical_loss": 3.989134892223519, + "tokens_seen": 449249280 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698062186559679, + "loss": 1.5084, + "theoretical_loss": 3.9890715142626956, + "tokens_seen": 449314816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980461384152458, + "loss": 1.4859, + "theoretical_loss": 3.9890081481332924, + "tokens_seen": 449380352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980300902708124, + "loss": 1.5314, + "theoretical_loss": 3.988944793831376, + "tokens_seen": 449445888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980140421263792, + "loss": 1.4309, + "theoretical_loss": 3.9888814513530138, + "tokens_seen": 449511424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979979939819458, + "loss": 1.5783, + "theoretical_loss": 3.988818120694277, + "tokens_seen": 449576960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979819458375126, + "loss": 1.423, + "theoretical_loss": 3.988754801851237, + "tokens_seen": 449642496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979658976930792, + "loss": 1.517, + "theoretical_loss": 3.9886914948199683, + "tokens_seen": 449708032 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697949849548646, + "loss": 1.5909, + "theoretical_loss": 3.988628199596546, + "tokens_seen": 449773568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979338014042126, + "loss": 1.4381, + "theoretical_loss": 3.9885649161770473, + "tokens_seen": 449839104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979177532597794, + "loss": 1.4802, + "theoretical_loss": 3.988501644557553, + "tokens_seen": 449904640 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697901705115346, + "loss": 1.4035, + "theoretical_loss": 3.9884383847341436, + "tokens_seen": 449970176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978856569709128, + "loss": 1.5362, + "theoretical_loss": 3.988375136702902, + "tokens_seen": 450035712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978696088264795, + "loss": 1.4519, + "theoretical_loss": 3.988311900459914, + "tokens_seen": 450101248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978535606820462, + "loss": 1.5602, + "theoretical_loss": 3.988248676001267, + "tokens_seen": 450166784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978375125376129, + "loss": 1.4926, + "theoretical_loss": 3.9881854633230485, + "tokens_seen": 450232320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978214643931796, + "loss": 1.5247, + "theoretical_loss": 3.9881222624213497, + "tokens_seen": 450297856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978054162487463, + "loss": 1.5003, + "theoretical_loss": 3.9880590732922636, + "tokens_seen": 450363392 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697789368104313, + "loss": 1.7043, + "theoretical_loss": 3.9879958959318844, + "tokens_seen": 450428928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977733199598797, + "loss": 1.583, + "theoretical_loss": 3.9879327303363086, + "tokens_seen": 450494464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977572718154464, + "loss": 1.5782, + "theoretical_loss": 3.9878695765016334, + "tokens_seen": 450560000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977412236710131, + "loss": 1.4689, + "theoretical_loss": 3.98780643442396, + "tokens_seen": 450625536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977251755265798, + "loss": 1.4843, + "theoretical_loss": 3.9877433040993897, + "tokens_seen": 450691072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977091273821465, + "loss": 1.54, + "theoretical_loss": 3.9876801855240265, + "tokens_seen": 450756608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976930792377132, + "loss": 1.4755, + "theoretical_loss": 3.987617078693975, + "tokens_seen": 450822144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976770310932799, + "loss": 1.4973, + "theoretical_loss": 3.987553983605345, + "tokens_seen": 450887680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976609829488466, + "loss": 1.6195, + "theoretical_loss": 3.9874909002542434, + "tokens_seen": 450953216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976449348044133, + "loss": 1.4774, + "theoretical_loss": 3.987427828636782, + "tokens_seen": 451018752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00069762888665998, + "loss": 1.5001, + "theoretical_loss": 3.9873647687490745, + "tokens_seen": 451084288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976128385155467, + "loss": 1.6324, + "theoretical_loss": 3.9873017205872356, + "tokens_seen": 451149824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975967903711134, + "loss": 1.432, + "theoretical_loss": 3.9872386841473815, + "tokens_seen": 451215360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975807422266801, + "loss": 1.5045, + "theoretical_loss": 3.987175659425631, + "tokens_seen": 451280896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975646940822468, + "loss": 1.5344, + "theoretical_loss": 3.9871126464181046, + "tokens_seen": 451346432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975486459378135, + "loss": 1.5424, + "theoretical_loss": 3.9870496451209245, + "tokens_seen": 451411968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975325977933802, + "loss": 1.605, + "theoretical_loss": 3.986986655530215, + "tokens_seen": 451477504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975165496489469, + "loss": 1.5406, + "theoretical_loss": 3.9869236776421015, + "tokens_seen": 451543040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975005015045136, + "loss": 1.4135, + "theoretical_loss": 3.986860711452713, + "tokens_seen": 451608576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974844533600803, + "loss": 1.5881, + "theoretical_loss": 3.986797756958178, + "tokens_seen": 451674112 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697468405215647, + "loss": 1.5084, + "theoretical_loss": 3.9867348141546284, + "tokens_seen": 451739648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974523570712137, + "loss": 1.5528, + "theoretical_loss": 3.9866718830381975, + "tokens_seen": 451805184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974363089267804, + "loss": 1.4579, + "theoretical_loss": 3.98660896360502, + "tokens_seen": 451870720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974202607823471, + "loss": 1.5445, + "theoretical_loss": 3.9865460558512336, + "tokens_seen": 451936256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974042126379138, + "loss": 1.3889, + "theoretical_loss": 3.986483159772977, + "tokens_seen": 452001792 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973881644934805, + "loss": 1.4828, + "theoretical_loss": 3.986420275366391, + "tokens_seen": 452067328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973721163490472, + "loss": 1.5615, + "theoretical_loss": 3.9863574026276174, + "tokens_seen": 452132864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 156834, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4055612087249756, + "objective/train/theoretical_loss": 3.986294541552801, + "objective/train/tokens_used": 472658400, + "theoretical_loss": 3.986294541552801, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973560682046139, + "loss": 1.4389, + "theoretical_loss": 3.986294541552801, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973400200601806, + "loss": 1.5344, + "theoretical_loss": 3.9862316921380883, + "tokens_seen": 452263936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973239719157473, + "loss": 1.5517, + "theoretical_loss": 3.9861688543796268, + "tokens_seen": 452329472 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697307923771314, + "loss": 1.4465, + "theoretical_loss": 3.986106028273567, + "tokens_seen": 452395008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972918756268807, + "loss": 1.4825, + "theoretical_loss": 3.9860432138160595, + "tokens_seen": 452460544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972758274824474, + "loss": 1.3684, + "theoretical_loss": 3.985980411003259, + "tokens_seen": 452526080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972597793380141, + "loss": 1.4558, + "theoretical_loss": 3.98591761983132, + "tokens_seen": 452591616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972437311935808, + "loss": 1.5356, + "theoretical_loss": 3.9858548402964, + "tokens_seen": 452657152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972276830491475, + "loss": 1.4826, + "theoretical_loss": 3.985792072394658, + "tokens_seen": 452722688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972116349047142, + "loss": 1.4988, + "theoretical_loss": 3.9857293161222542, + "tokens_seen": 452788224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971955867602809, + "loss": 1.5111, + "theoretical_loss": 3.985666571475352, + "tokens_seen": 452853760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971795386158476, + "loss": 1.5831, + "theoretical_loss": 3.985603838450116, + "tokens_seen": 452919296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971634904714143, + "loss": 1.5667, + "theoretical_loss": 3.9855411170427115, + "tokens_seen": 452984832 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697147442326981, + "loss": 1.4883, + "theoretical_loss": 3.985478407249307, + "tokens_seen": 453050368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971313941825477, + "loss": 1.5357, + "theoretical_loss": 3.985415709066073, + "tokens_seen": 453115904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971153460381144, + "loss": 1.4585, + "theoretical_loss": 3.98535302248918, + "tokens_seen": 453181440 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697099297893681, + "loss": 1.4994, + "theoretical_loss": 3.9852903475148027, + "tokens_seen": 453246976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970832497492478, + "loss": 1.5574, + "theoretical_loss": 3.985227684139116, + "tokens_seen": 453312512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970672016048144, + "loss": 1.5326, + "theoretical_loss": 3.9851650323582977, + "tokens_seen": 453378048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970511534603812, + "loss": 1.4839, + "theoretical_loss": 3.9851023921685256, + "tokens_seen": 453443584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970351053159478, + "loss": 1.471, + "theoretical_loss": 3.9850397635659807, + "tokens_seen": 453509120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970190571715146, + "loss": 1.4931, + "theoretical_loss": 3.9849771465468464, + "tokens_seen": 453574656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970030090270812, + "loss": 1.5389, + "theoretical_loss": 3.9849145411073064, + "tokens_seen": 453640192 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696986960882648, + "loss": 1.6396, + "theoretical_loss": 3.984851947243547, + "tokens_seen": 453705728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006969709127382146, + "loss": 1.4131, + "theoretical_loss": 3.984789364951757, + "tokens_seen": 453771264 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006969548645937813, + "loss": 1.534, + "theoretical_loss": 3.9847267942281253, + "tokens_seen": 453836800 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696938816449348, + "loss": 1.4727, + "theoretical_loss": 3.984664235068843, + "tokens_seen": 453902336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006969227683049147, + "loss": 1.4447, + "theoretical_loss": 3.9846016874701053, + "tokens_seen": 453967872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006969067201604814, + "loss": 1.5291, + "theoretical_loss": 3.9845391514281063, + "tokens_seen": 454033408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968906720160481, + "loss": 1.465, + "theoretical_loss": 3.9844766269390437, + "tokens_seen": 454098944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968746238716148, + "loss": 1.5212, + "theoretical_loss": 3.984414113999115, + "tokens_seen": 454164480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968585757271815, + "loss": 1.4242, + "theoretical_loss": 3.984351612604522, + "tokens_seen": 454230016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968425275827482, + "loss": 1.4277, + "theoretical_loss": 3.984289122751467, + "tokens_seen": 454295552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968264794383149, + "loss": 1.5631, + "theoretical_loss": 3.984226644436154, + "tokens_seen": 454361088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968104312938816, + "loss": 1.4569, + "theoretical_loss": 3.9841641776547885, + "tokens_seen": 454426624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967943831494483, + "loss": 1.4869, + "theoretical_loss": 3.98410172240358, + "tokens_seen": 454492160 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696778335005015, + "loss": 1.4457, + "theoretical_loss": 3.984039278678736, + "tokens_seen": 454557696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967622868605817, + "loss": 1.5251, + "theoretical_loss": 3.9839768464764695, + "tokens_seen": 454623232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967462387161485, + "loss": 1.6093, + "theoretical_loss": 3.9839144257929933, + "tokens_seen": 454688768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967301905717151, + "loss": 1.409, + "theoretical_loss": 3.983852016624522, + "tokens_seen": 454754304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967141424272819, + "loss": 1.5454, + "theoretical_loss": 3.9837896189672724, + "tokens_seen": 454819840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966980942828485, + "loss": 1.5249, + "theoretical_loss": 3.9837272328174635, + "tokens_seen": 454885376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966820461384153, + "loss": 1.4779, + "theoretical_loss": 3.9836648581713154, + "tokens_seen": 454950912 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696665997993982, + "loss": 1.5053, + "theoretical_loss": 3.9836024950250497, + "tokens_seen": 455016448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966499498495487, + "loss": 1.5391, + "theoretical_loss": 3.983540143374891, + "tokens_seen": 455081984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966339017051154, + "loss": 1.5406, + "theoretical_loss": 3.983477803217066, + "tokens_seen": 455147520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966178535606821, + "loss": 1.6182, + "theoretical_loss": 3.9834154745478, + "tokens_seen": 455213056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966018054162488, + "loss": 1.3834, + "theoretical_loss": 3.9833531573633234, + "tokens_seen": 455278592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965857572718155, + "loss": 1.5062, + "theoretical_loss": 3.9832908516598673, + "tokens_seen": 455344128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965697091273822, + "loss": 1.509, + "theoretical_loss": 3.983228557433664, + "tokens_seen": 455409664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 157689, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5005377531051636, + "objective/train/theoretical_loss": 3.9831662746809493, + "objective/train/tokens_used": 475935200, + "theoretical_loss": 3.9831662746809493, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965536609829489, + "loss": 1.5773, + "theoretical_loss": 3.9831662746809493, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965376128385156, + "loss": 1.516, + "theoretical_loss": 3.9831040033979583, + "tokens_seen": 455540736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965215646940823, + "loss": 1.3727, + "theoretical_loss": 3.9830417435809298, + "tokens_seen": 455606272 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696505516549649, + "loss": 1.4682, + "theoretical_loss": 3.9829794952261035, + "tokens_seen": 455671808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964894684052157, + "loss": 1.5151, + "theoretical_loss": 3.9829172583297208, + "tokens_seen": 455737344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964734202607824, + "loss": 1.4777, + "theoretical_loss": 3.9828550328880263, + "tokens_seen": 455802880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964573721163491, + "loss": 1.6122, + "theoretical_loss": 3.982792818897264, + "tokens_seen": 455868416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964413239719158, + "loss": 1.5951, + "theoretical_loss": 3.9827306163536815, + "tokens_seen": 455933952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964252758274825, + "loss": 1.5475, + "theoretical_loss": 3.982668425253528, + "tokens_seen": 455999488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964092276830492, + "loss": 1.4892, + "theoretical_loss": 3.982606245593053, + "tokens_seen": 456065024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963931795386159, + "loss": 1.5035, + "theoretical_loss": 3.9825440773685092, + "tokens_seen": 456130560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963771313941826, + "loss": 1.493, + "theoretical_loss": 3.9824819205761512, + "tokens_seen": 456196096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963610832497493, + "loss": 1.5549, + "theoretical_loss": 3.982419775212235, + "tokens_seen": 456261632 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696345035105316, + "loss": 1.6392, + "theoretical_loss": 3.9823576412730173, + "tokens_seen": 456327168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963289869608827, + "loss": 1.5156, + "theoretical_loss": 3.982295518754758, + "tokens_seen": 456392704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963129388164494, + "loss": 1.5393, + "theoretical_loss": 3.982233407653718, + "tokens_seen": 456458240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962968906720161, + "loss": 1.5114, + "theoretical_loss": 3.982171307966161, + "tokens_seen": 456523776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962808425275828, + "loss": 1.5031, + "theoretical_loss": 3.9821092196883505, + "tokens_seen": 456589312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962647943831495, + "loss": 1.5498, + "theoretical_loss": 3.982047142816554, + "tokens_seen": 456654848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962487462387162, + "loss": 1.5781, + "theoretical_loss": 3.981985077347039, + "tokens_seen": 456720384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962326980942829, + "loss": 1.5119, + "theoretical_loss": 3.981923023276075, + "tokens_seen": 456785920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962166499498496, + "loss": 1.5188, + "theoretical_loss": 3.9818609805999348, + "tokens_seen": 456851456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962006018054163, + "loss": 1.4882, + "theoretical_loss": 3.981798949314891, + "tokens_seen": 456916992 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696184553660983, + "loss": 1.6035, + "theoretical_loss": 3.98173692941722, + "tokens_seen": 456982528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961685055165497, + "loss": 1.5139, + "theoretical_loss": 3.981674920903197, + "tokens_seen": 457048064 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961524573721164, + "loss": 1.5006, + "theoretical_loss": 3.9816129237691023, + "tokens_seen": 457113600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961364092276831, + "loss": 1.5368, + "theoretical_loss": 3.981550938011215, + "tokens_seen": 457179136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961203610832498, + "loss": 1.4274, + "theoretical_loss": 3.981488963625818, + "tokens_seen": 457244672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961043129388165, + "loss": 1.3543, + "theoretical_loss": 3.9814270006091954, + "tokens_seen": 457310208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960882647943832, + "loss": 1.5751, + "theoretical_loss": 3.981365048957633, + "tokens_seen": 457375744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960722166499499, + "loss": 1.5665, + "theoretical_loss": 3.9813031086674173, + "tokens_seen": 457441280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960561685055166, + "loss": 1.4666, + "theoretical_loss": 3.9812411797348384, + "tokens_seen": 457506816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960401203610833, + "loss": 1.4307, + "theoretical_loss": 3.9811792621561874, + "tokens_seen": 457572352 + }, + { + "epoch": 0.14, + "learning_rate": 0.00069602407221665, + "loss": 1.4939, + "theoretical_loss": 3.981117355927756, + "tokens_seen": 457637888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960080240722167, + "loss": 1.541, + "theoretical_loss": 3.981055461045839, + "tokens_seen": 457703424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959919759277834, + "loss": 1.5948, + "theoretical_loss": 3.980993577506733, + "tokens_seen": 457768960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959759277833501, + "loss": 1.5795, + "theoretical_loss": 3.980931705306735, + "tokens_seen": 457834496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959598796389168, + "loss": 1.519, + "theoretical_loss": 3.9808698444421458, + "tokens_seen": 457900032 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959438314944835, + "loss": 1.5537, + "theoretical_loss": 3.9808079949092656, + "tokens_seen": 457965568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959277833500502, + "loss": 1.5433, + "theoretical_loss": 3.9807461567043987, + "tokens_seen": 458031104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959117352056169, + "loss": 1.4325, + "theoretical_loss": 3.980684329823849, + "tokens_seen": 458096640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958956870611836, + "loss": 1.5263, + "theoretical_loss": 3.980622514263923, + "tokens_seen": 458162176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958796389167503, + "loss": 1.5642, + "theoretical_loss": 3.9805607100209297, + "tokens_seen": 458227712 + }, + { + "epoch": 0.14, + "learning_rate": 0.000695863590772317, + "loss": 1.4736, + "theoretical_loss": 3.980498917091179, + "tokens_seen": 458293248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958475426278837, + "loss": 1.5423, + "theoretical_loss": 3.980437135470982, + "tokens_seen": 458358784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958314944834504, + "loss": 1.5497, + "theoretical_loss": 3.980375365156653, + "tokens_seen": 458424320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958154463390171, + "loss": 1.4201, + "theoretical_loss": 3.9803136061445064, + "tokens_seen": 458489856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957993981945838, + "loss": 1.5075, + "theoretical_loss": 3.9802518584308597, + "tokens_seen": 458555392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957833500501505, + "loss": 1.407, + "theoretical_loss": 3.980190122012032, + "tokens_seen": 458620928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957673019057172, + "loss": 1.4852, + "theoretical_loss": 3.980128396884343, + "tokens_seen": 458686464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 158992, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4671831130981445, + "objective/train/theoretical_loss": 3.9800666830441145, + "objective/train/tokens_used": 479212000, + "theoretical_loss": 3.9800666830441145, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957512537612839, + "loss": 1.5057, + "theoretical_loss": 3.9800666830441145, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957352056168506, + "loss": 1.3742, + "theoretical_loss": 3.9800049804876716, + "tokens_seen": 458817536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957191574724173, + "loss": 1.6351, + "theoretical_loss": 3.9799432892113384, + "tokens_seen": 458883072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957031093279839, + "loss": 1.4647, + "theoretical_loss": 3.9798816092114437, + "tokens_seen": 458948608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956870611835507, + "loss": 1.4541, + "theoretical_loss": 3.9798199404843153, + "tokens_seen": 459014144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956710130391173, + "loss": 1.5378, + "theoretical_loss": 3.9797582830262845, + "tokens_seen": 459079680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956549648946841, + "loss": 1.4761, + "theoretical_loss": 3.9796966368336832, + "tokens_seen": 459145216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956389167502507, + "loss": 1.5889, + "theoretical_loss": 3.979635001902847, + "tokens_seen": 459210752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956228686058175, + "loss": 1.6021, + "theoretical_loss": 3.9795733782301097, + "tokens_seen": 459276288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956068204613841, + "loss": 1.5399, + "theoretical_loss": 3.979511765811811, + "tokens_seen": 459341824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955907723169509, + "loss": 1.5921, + "theoretical_loss": 3.9794501646442884, + "tokens_seen": 459407360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955747241725175, + "loss": 1.5187, + "theoretical_loss": 3.9793885747238837, + "tokens_seen": 459472896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955586760280842, + "loss": 1.5166, + "theoretical_loss": 3.97932699604694, + "tokens_seen": 459538432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955426278836509, + "loss": 1.4833, + "theoretical_loss": 3.979265428609801, + "tokens_seen": 459603968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955265797392176, + "loss": 1.5999, + "theoretical_loss": 3.9792038724088137, + "tokens_seen": 459669504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955105315947843, + "loss": 1.5948, + "theoretical_loss": 3.9791423274403255, + "tokens_seen": 459735040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954944834503511, + "loss": 1.4881, + "theoretical_loss": 3.9790807937006853, + "tokens_seen": 459800576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954784353059177, + "loss": 1.488, + "theoretical_loss": 3.979019271186246, + "tokens_seen": 459866112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954623871614845, + "loss": 1.506, + "theoretical_loss": 3.9789577598933588, + "tokens_seen": 459931648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954463390170511, + "loss": 1.4603, + "theoretical_loss": 3.9788962598183795, + "tokens_seen": 459997184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954302908726179, + "loss": 1.5106, + "theoretical_loss": 3.978834770957665, + "tokens_seen": 460062720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954142427281846, + "loss": 1.5574, + "theoretical_loss": 3.9787732933075715, + "tokens_seen": 460128256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953981945837513, + "loss": 1.6217, + "theoretical_loss": 3.97871182686446, + "tokens_seen": 460193792 + }, + { + "epoch": 0.14, + "learning_rate": 0.000695382146439318, + "loss": 1.4491, + "theoretical_loss": 3.978650371624692, + "tokens_seen": 460259328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953660982948847, + "loss": 1.5272, + "theoretical_loss": 3.9785889275846307, + "tokens_seen": 460324864 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953500501504514, + "loss": 1.5238, + "theoretical_loss": 3.978527494740641, + "tokens_seen": 460390400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953340020060181, + "loss": 1.5595, + "theoretical_loss": 3.9784660730890895, + "tokens_seen": 460455936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953179538615848, + "loss": 1.5279, + "theoretical_loss": 3.978404662626344, + "tokens_seen": 460521472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953019057171515, + "loss": 1.5194, + "theoretical_loss": 3.9783432633487745, + "tokens_seen": 460587008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952858575727182, + "loss": 1.5553, + "theoretical_loss": 3.9782818752527533, + "tokens_seen": 460652544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952698094282849, + "loss": 1.481, + "theoretical_loss": 3.978220498334654, + "tokens_seen": 460718080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952537612838516, + "loss": 1.5989, + "theoretical_loss": 3.97815913259085, + "tokens_seen": 460783616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952377131394183, + "loss": 1.4864, + "theoretical_loss": 3.9780977780177205, + "tokens_seen": 460849152 + }, + { + "epoch": 0.14, + "learning_rate": 0.000695221664994985, + "loss": 1.535, + "theoretical_loss": 3.978036434611642, + "tokens_seen": 460914688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952056168505517, + "loss": 1.4559, + "theoretical_loss": 3.977975102368995, + "tokens_seen": 460980224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951895687061184, + "loss": 1.5792, + "theoretical_loss": 3.977913781286162, + "tokens_seen": 461045760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951735205616851, + "loss": 1.5258, + "theoretical_loss": 3.9778524713595256, + "tokens_seen": 461111296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951574724172518, + "loss": 1.4682, + "theoretical_loss": 3.977791172585472, + "tokens_seen": 461176832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951414242728185, + "loss": 1.4718, + "theoretical_loss": 3.977729884960387, + "tokens_seen": 461242368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951253761283852, + "loss": 1.5455, + "theoretical_loss": 3.9776686084806596, + "tokens_seen": 461307904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951093279839519, + "loss": 1.4778, + "theoretical_loss": 3.9776073431426804, + "tokens_seen": 461373440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950932798395186, + "loss": 1.5595, + "theoretical_loss": 3.977546088942841, + "tokens_seen": 461438976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950772316950853, + "loss": 1.5265, + "theoretical_loss": 3.9774848458775347, + "tokens_seen": 461504512 + }, + { + "epoch": 0.14, + "learning_rate": 0.000695061183550652, + "loss": 1.5641, + "theoretical_loss": 3.9774236139431576, + "tokens_seen": 461570048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950451354062187, + "loss": 1.4548, + "theoretical_loss": 3.977362393136106, + "tokens_seen": 461635584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950290872617854, + "loss": 1.4131, + "theoretical_loss": 3.977301183452779, + "tokens_seen": 461701120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950130391173521, + "loss": 1.4352, + "theoretical_loss": 3.977239984889576, + "tokens_seen": 461766656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949969909729188, + "loss": 1.4112, + "theoretical_loss": 3.9771787974429, + "tokens_seen": 461832192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949809428284855, + "loss": 1.443, + "theoretical_loss": 3.977117621109154, + "tokens_seen": 461897728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949648946840522, + "loss": 1.4848, + "theoretical_loss": 3.977056455884744, + "tokens_seen": 461963264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 159827, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.441860318183899, + "objective/train/theoretical_loss": 3.9769953017660766, + "objective/train/tokens_used": 482488800, + "theoretical_loss": 3.9769953017660766, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949488465396189, + "loss": 1.4389, + "theoretical_loss": 3.9769953017660766, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949327983951856, + "loss": 1.4872, + "theoretical_loss": 3.97693415874956, + "tokens_seen": 462094336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949167502507523, + "loss": 1.4321, + "theoretical_loss": 3.9768730268316057, + "tokens_seen": 462159872 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694900702106319, + "loss": 1.4834, + "theoretical_loss": 3.976811906008625, + "tokens_seen": 462225408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948846539618857, + "loss": 1.4452, + "theoretical_loss": 3.976750796277031, + "tokens_seen": 462290944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948686058174524, + "loss": 1.6206, + "theoretical_loss": 3.9766896976332404, + "tokens_seen": 462356480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948525576730191, + "loss": 1.493, + "theoretical_loss": 3.9766286100736696, + "tokens_seen": 462422016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948365095285858, + "loss": 1.479, + "theoretical_loss": 3.976567533594737, + "tokens_seen": 462487552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948204613841525, + "loss": 1.5357, + "theoretical_loss": 3.9765064681928637, + "tokens_seen": 462553088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948044132397192, + "loss": 1.464, + "theoretical_loss": 3.9764454138644703, + "tokens_seen": 462618624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947883650952859, + "loss": 1.469, + "theoretical_loss": 3.976384370605982, + "tokens_seen": 462684160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947723169508526, + "loss": 1.4872, + "theoretical_loss": 3.9763233384138235, + "tokens_seen": 462749696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947562688064193, + "loss": 1.5855, + "theoretical_loss": 3.976262317284421, + "tokens_seen": 462815232 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694740220661986, + "loss": 1.5168, + "theoretical_loss": 3.976201307214205, + "tokens_seen": 462880768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947241725175527, + "loss": 1.5409, + "theoretical_loss": 3.9761403081996045, + "tokens_seen": 462946304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947081243731194, + "loss": 1.5067, + "theoretical_loss": 3.9760793202370524, + "tokens_seen": 463011840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946920762286861, + "loss": 1.6068, + "theoretical_loss": 3.976018343322981, + "tokens_seen": 463077376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946760280842528, + "loss": 1.4944, + "theoretical_loss": 3.975957377453826, + "tokens_seen": 463142912 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946599799398195, + "loss": 1.4191, + "theoretical_loss": 3.9758964226260254, + "tokens_seen": 463208448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946439317953862, + "loss": 1.5305, + "theoretical_loss": 3.975835478836016, + "tokens_seen": 463273984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946278836509529, + "loss": 1.581, + "theoretical_loss": 3.97577454608024, + "tokens_seen": 463339520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946118355065196, + "loss": 1.5766, + "theoretical_loss": 3.975713624355139, + "tokens_seen": 463405056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945957873620863, + "loss": 1.4439, + "theoretical_loss": 3.975652713657155, + "tokens_seen": 463470592 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694579739217653, + "loss": 1.5426, + "theoretical_loss": 3.975591813982734, + "tokens_seen": 463536128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945636910732197, + "loss": 1.4779, + "theoretical_loss": 3.9755309253283233, + "tokens_seen": 463601664 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945476429287864, + "loss": 1.5188, + "theoretical_loss": 3.9754700476903713, + "tokens_seen": 463667200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945315947843531, + "loss": 1.4893, + "theoretical_loss": 3.975409181065328, + "tokens_seen": 463732736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945155466399198, + "loss": 1.5916, + "theoretical_loss": 3.9753483254496444, + "tokens_seen": 463798272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944994984954865, + "loss": 1.5098, + "theoretical_loss": 3.9752874808397753, + "tokens_seen": 463863808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944834503510532, + "loss": 1.5938, + "theoretical_loss": 3.975226647232175, + "tokens_seen": 463929344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944674022066199, + "loss": 1.5668, + "theoretical_loss": 3.9751658246233, + "tokens_seen": 463994880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944513540621866, + "loss": 1.5402, + "theoretical_loss": 3.9751050130096095, + "tokens_seen": 464060416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944353059177533, + "loss": 1.521, + "theoretical_loss": 3.9750442123875622, + "tokens_seen": 464125952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00069441925777332, + "loss": 1.5605, + "theoretical_loss": 3.9749834227536214, + "tokens_seen": 464191488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944032096288867, + "loss": 1.441, + "theoretical_loss": 3.9749226441042493, + "tokens_seen": 464257024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943871614844534, + "loss": 1.5675, + "theoretical_loss": 3.9748618764359107, + "tokens_seen": 464322560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943711133400201, + "loss": 1.5599, + "theoretical_loss": 3.9748011197450728, + "tokens_seen": 464388096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943550651955868, + "loss": 1.6316, + "theoretical_loss": 3.974740374028203, + "tokens_seen": 464453632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943390170511535, + "loss": 1.4587, + "theoretical_loss": 3.9746796392817716, + "tokens_seen": 464519168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943229689067202, + "loss": 1.4971, + "theoretical_loss": 3.9746189155022504, + "tokens_seen": 464584704 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694306920762287, + "loss": 1.5041, + "theoretical_loss": 3.9745582026861124, + "tokens_seen": 464650240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006942908726178536, + "loss": 1.6098, + "theoretical_loss": 3.9744975008298313, + "tokens_seen": 464715776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006942748244734202, + "loss": 1.5333, + "theoretical_loss": 3.974436809929885, + "tokens_seen": 464781312 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694258776328987, + "loss": 1.5918, + "theoretical_loss": 3.9743761299827494, + "tokens_seen": 464846848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006942427281845536, + "loss": 1.4116, + "theoretical_loss": 3.974315460984906, + "tokens_seen": 464912384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006942266800401204, + "loss": 1.5304, + "theoretical_loss": 3.974254802932836, + "tokens_seen": 464977920 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694210631895687, + "loss": 1.5563, + "theoretical_loss": 3.974194155823021, + "tokens_seen": 465043456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941945837512538, + "loss": 1.5092, + "theoretical_loss": 3.9741335196519465, + "tokens_seen": 465108992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941785356068204, + "loss": 1.5468, + "theoretical_loss": 3.9740728944160977, + "tokens_seen": 465174528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941624874623872, + "loss": 1.4083, + "theoretical_loss": 3.974012280111963, + "tokens_seen": 465240064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 161129, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5128990411758423, + "objective/train/theoretical_loss": 3.973951676736032, + "objective/train/tokens_used": 485765600, + "theoretical_loss": 3.973951676736032, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941464393179538, + "loss": 1.4757, + "theoretical_loss": 3.973951676736032, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941303911735206, + "loss": 1.6992, + "theoretical_loss": 3.973891084284795, + "tokens_seen": 465371136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941143430290872, + "loss": 1.4481, + "theoretical_loss": 3.973830502754745, + "tokens_seen": 465436672 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694098294884654, + "loss": 1.3695, + "theoretical_loss": 3.973769932142376, + "tokens_seen": 465502208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940822467402206, + "loss": 1.4866, + "theoretical_loss": 3.9737093724441843, + "tokens_seen": 465567744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940661985957874, + "loss": 1.6144, + "theoretical_loss": 3.9736488236566667, + "tokens_seen": 465633280 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694050150451354, + "loss": 1.494, + "theoretical_loss": 3.973588285776323, + "tokens_seen": 465698816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940341023069208, + "loss": 1.3829, + "theoretical_loss": 3.9735277587996527, + "tokens_seen": 465764352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940180541624875, + "loss": 1.511, + "theoretical_loss": 3.973467242723159, + "tokens_seen": 465829888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940020060180542, + "loss": 1.5361, + "theoretical_loss": 3.9734067375433466, + "tokens_seen": 465895424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939859578736209, + "loss": 1.4227, + "theoretical_loss": 3.9733462432567195, + "tokens_seen": 465960960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939699097291876, + "loss": 1.5223, + "theoretical_loss": 3.9732857598597855, + "tokens_seen": 466026496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939538615847543, + "loss": 1.587, + "theoretical_loss": 3.9732252873490532, + "tokens_seen": 466092032 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693937813440321, + "loss": 1.4361, + "theoretical_loss": 3.9731648257210335, + "tokens_seen": 466157568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939217652958877, + "loss": 1.5038, + "theoretical_loss": 3.9731043749722375, + "tokens_seen": 466223104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939057171514544, + "loss": 1.5343, + "theoretical_loss": 3.9730439350991795, + "tokens_seen": 466288640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938896690070211, + "loss": 1.5162, + "theoretical_loss": 3.9729835060983745, + "tokens_seen": 466354176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938736208625878, + "loss": 1.4384, + "theoretical_loss": 3.972923087966339, + "tokens_seen": 466419712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938575727181545, + "loss": 1.5003, + "theoretical_loss": 3.9728626806995915, + "tokens_seen": 466485248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938415245737212, + "loss": 1.5118, + "theoretical_loss": 3.9728022842946524, + "tokens_seen": 466550784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938254764292879, + "loss": 1.518, + "theoretical_loss": 3.972741898748043, + "tokens_seen": 466616320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938094282848546, + "loss": 1.5349, + "theoretical_loss": 3.972681524056286, + "tokens_seen": 466681856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937933801404213, + "loss": 1.6516, + "theoretical_loss": 3.9726211602159074, + "tokens_seen": 466747392 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693777331995988, + "loss": 1.5143, + "theoretical_loss": 3.9725608072234326, + "tokens_seen": 466812928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937612838515547, + "loss": 1.5186, + "theoretical_loss": 3.97250046507539, + "tokens_seen": 466878464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937452357071214, + "loss": 1.4036, + "theoretical_loss": 3.972440133768309, + "tokens_seen": 466944000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937291875626881, + "loss": 1.4624, + "theoretical_loss": 3.972379813298721, + "tokens_seen": 467009536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937131394182548, + "loss": 1.4537, + "theoretical_loss": 3.972319503663159, + "tokens_seen": 467075072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936970912738215, + "loss": 1.4609, + "theoretical_loss": 3.972259204858157, + "tokens_seen": 467140608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936810431293882, + "loss": 1.5882, + "theoretical_loss": 3.9721989168802505, + "tokens_seen": 467206144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936649949849549, + "loss": 1.5135, + "theoretical_loss": 3.9721386397259786, + "tokens_seen": 467271680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936489468405216, + "loss": 1.6007, + "theoretical_loss": 3.972078373391879, + "tokens_seen": 467337216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936328986960883, + "loss": 1.3934, + "theoretical_loss": 3.9720181178744935, + "tokens_seen": 467402752 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693616850551655, + "loss": 1.4687, + "theoretical_loss": 3.971957873170364, + "tokens_seen": 467468288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936008024072217, + "loss": 1.4979, + "theoretical_loss": 3.971897639276035, + "tokens_seen": 467533824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935847542627884, + "loss": 1.4511, + "theoretical_loss": 3.9718374161880505, + "tokens_seen": 467599360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935687061183551, + "loss": 1.396, + "theoretical_loss": 3.971777203902959, + "tokens_seen": 467664896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935526579739218, + "loss": 1.5844, + "theoretical_loss": 3.971717002417309, + "tokens_seen": 467730432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935366098294885, + "loss": 1.6607, + "theoretical_loss": 3.9716568117276503, + "tokens_seen": 467795968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935205616850552, + "loss": 1.5169, + "theoretical_loss": 3.9715966318305354, + "tokens_seen": 467861504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935045135406219, + "loss": 1.4717, + "theoretical_loss": 3.9715364627225176, + "tokens_seen": 467927040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934884653961886, + "loss": 1.3652, + "theoretical_loss": 3.9714763044001513, + "tokens_seen": 467992576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934724172517553, + "loss": 1.5736, + "theoretical_loss": 3.971416156859995, + "tokens_seen": 468058112 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693456369107322, + "loss": 1.4701, + "theoretical_loss": 3.9713560200986042, + "tokens_seen": 468123648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934403209628887, + "loss": 1.4428, + "theoretical_loss": 3.9712958941125405, + "tokens_seen": 468189184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934242728184554, + "loss": 1.6038, + "theoretical_loss": 3.971235778898365, + "tokens_seen": 468254720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934082246740221, + "loss": 1.5205, + "theoretical_loss": 3.971175674452641, + "tokens_seen": 468320256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933921765295888, + "loss": 1.5535, + "theoretical_loss": 3.9711155807719316, + "tokens_seen": 468385792 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933761283851555, + "loss": 1.5537, + "theoretical_loss": 3.971055497852804, + "tokens_seen": 468451328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933600802407222, + "loss": 1.4948, + "theoretical_loss": 3.9709954256918265, + "tokens_seen": 468516864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 161996, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3592880964279175, + "objective/train/theoretical_loss": 3.970935364285567, + "objective/train/tokens_used": 489042400, + "theoretical_loss": 3.970935364285567, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933440320962889, + "loss": 1.4138, + "theoretical_loss": 3.970935364285567, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933279839518556, + "loss": 1.438, + "theoretical_loss": 3.9708753136305974, + "tokens_seen": 468647936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933119358074223, + "loss": 1.4692, + "theoretical_loss": 3.9708152737234887, + "tokens_seen": 468713472 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693295887662989, + "loss": 1.4952, + "theoretical_loss": 3.9707552445608165, + "tokens_seen": 468779008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932798395185557, + "loss": 1.482, + "theoretical_loss": 3.9706952261391555, + "tokens_seen": 468844544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932637913741224, + "loss": 1.4899, + "theoretical_loss": 3.970635218455083, + "tokens_seen": 468910080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932477432296891, + "loss": 1.4871, + "theoretical_loss": 3.970575221505178, + "tokens_seen": 468975616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932316950852558, + "loss": 1.5286, + "theoretical_loss": 3.97051523528602, + "tokens_seen": 469041152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932156469408225, + "loss": 1.5471, + "theoretical_loss": 3.970455259794191, + "tokens_seen": 469106688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931995987963892, + "loss": 1.5182, + "theoretical_loss": 3.970395295026275, + "tokens_seen": 469172224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931835506519559, + "loss": 1.6327, + "theoretical_loss": 3.970335340978856, + "tokens_seen": 469237760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931675025075226, + "loss": 1.4887, + "theoretical_loss": 3.970275397648522, + "tokens_seen": 469303296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931514543630893, + "loss": 1.5198, + "theoretical_loss": 3.9702154650318597, + "tokens_seen": 469368832 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693135406218656, + "loss": 1.5937, + "theoretical_loss": 3.9701555431254594, + "tokens_seen": 469434368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931193580742227, + "loss": 1.4071, + "theoretical_loss": 3.9700956319259113, + "tokens_seen": 469499904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931033099297894, + "loss": 1.4972, + "theoretical_loss": 3.97003573142981, + "tokens_seen": 469565440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930872617853562, + "loss": 1.5, + "theoretical_loss": 3.9699758416337483, + "tokens_seen": 469630976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930712136409228, + "loss": 1.561, + "theoretical_loss": 3.9699159625343223, + "tokens_seen": 469696512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930551654964896, + "loss": 1.3793, + "theoretical_loss": 3.96985609412813, + "tokens_seen": 469762048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930391173520562, + "loss": 1.534, + "theoretical_loss": 3.96979623641177, + "tokens_seen": 469827584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930230692076229, + "loss": 1.5069, + "theoretical_loss": 3.9697363893818434, + "tokens_seen": 469893120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930070210631896, + "loss": 1.503, + "theoretical_loss": 3.969676553034951, + "tokens_seen": 469958656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929909729187563, + "loss": 1.5206, + "theoretical_loss": 3.969616727367698, + "tokens_seen": 470024192 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692974924774323, + "loss": 1.5285, + "theoretical_loss": 3.9695569123766887, + "tokens_seen": 470089728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929588766298897, + "loss": 1.4659, + "theoretical_loss": 3.9694971080585297, + "tokens_seen": 470155264 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929428284854564, + "loss": 1.4967, + "theoretical_loss": 3.9694373144098307, + "tokens_seen": 470220800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929267803410231, + "loss": 1.514, + "theoretical_loss": 3.9693775314271997, + "tokens_seen": 470286336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929107321965898, + "loss": 1.4917, + "theoretical_loss": 3.9693177591072493, + "tokens_seen": 470351872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928946840521565, + "loss": 1.4453, + "theoretical_loss": 3.9692579974465922, + "tokens_seen": 470417408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928786359077231, + "loss": 1.4894, + "theoretical_loss": 3.9691982464418434, + "tokens_seen": 470482944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928625877632899, + "loss": 1.5014, + "theoretical_loss": 3.9691385060896174, + "tokens_seen": 470548480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928465396188565, + "loss": 1.5743, + "theoretical_loss": 3.9690787763865334, + "tokens_seen": 470614016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928304914744233, + "loss": 1.5716, + "theoretical_loss": 3.96901905732921, + "tokens_seen": 470679552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928144433299899, + "loss": 1.5425, + "theoretical_loss": 3.968959348914268, + "tokens_seen": 470745088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927983951855567, + "loss": 1.482, + "theoretical_loss": 3.9688996511383294, + "tokens_seen": 470810624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927823470411233, + "loss": 1.4986, + "theoretical_loss": 3.968839963998018, + "tokens_seen": 470876160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927662988966901, + "loss": 1.4271, + "theoretical_loss": 3.9687802874899596, + "tokens_seen": 470941696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927502507522567, + "loss": 1.4772, + "theoretical_loss": 3.9687206216107804, + "tokens_seen": 471007232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927342026078235, + "loss": 1.5334, + "theoretical_loss": 3.968660966357109, + "tokens_seen": 471072768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927181544633901, + "loss": 1.5458, + "theoretical_loss": 3.9686013217255756, + "tokens_seen": 471138304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927021063189569, + "loss": 1.4409, + "theoretical_loss": 3.9685416877128112, + "tokens_seen": 471203840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926860581745235, + "loss": 1.5034, + "theoretical_loss": 3.9684820643154497, + "tokens_seen": 471269376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926700100300903, + "loss": 1.4848, + "theoretical_loss": 3.9684224515301247, + "tokens_seen": 471334912 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692653961885657, + "loss": 1.4413, + "theoretical_loss": 3.968362849353473, + "tokens_seen": 471400448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926379137412237, + "loss": 1.543, + "theoretical_loss": 3.9683032577821313, + "tokens_seen": 471465984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926218655967904, + "loss": 1.4091, + "theoretical_loss": 3.9682436768127394, + "tokens_seen": 471531520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926058174523571, + "loss": 1.5432, + "theoretical_loss": 3.968184106441938, + "tokens_seen": 471597056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925897693079238, + "loss": 1.5376, + "theoretical_loss": 3.9681245466663695, + "tokens_seen": 471662592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925737211634905, + "loss": 1.4423, + "theoretical_loss": 3.9680649974826774, + "tokens_seen": 471728128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925576730190572, + "loss": 1.4998, + "theoretical_loss": 3.9680054588875064, + "tokens_seen": 471793664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 163162, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4037538766860962, + "objective/train/theoretical_loss": 3.9679459308775042, + "objective/train/tokens_used": 492319200, + "theoretical_loss": 3.9679459308775042, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925416248746239, + "loss": 1.4952, + "theoretical_loss": 3.9679459308775042, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925255767301906, + "loss": 1.5488, + "theoretical_loss": 3.9678864134493184, + "tokens_seen": 471924736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925095285857573, + "loss": 1.5842, + "theoretical_loss": 3.9678269065996, + "tokens_seen": 471990272 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692493480441324, + "loss": 1.5198, + "theoretical_loss": 3.967767410324999, + "tokens_seen": 472055808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924774322968907, + "loss": 1.4667, + "theoretical_loss": 3.967707924622169, + "tokens_seen": 472121344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924613841524574, + "loss": 1.5051, + "theoretical_loss": 3.967648449487764, + "tokens_seen": 472186880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924453360080241, + "loss": 1.5073, + "theoretical_loss": 3.9675889849184407, + "tokens_seen": 472252416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924292878635908, + "loss": 1.5508, + "theoretical_loss": 3.9675295309108556, + "tokens_seen": 472317952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924132397191575, + "loss": 1.4599, + "theoretical_loss": 3.9674700874616686, + "tokens_seen": 472383488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923971915747242, + "loss": 1.6051, + "theoretical_loss": 3.9674106545675403, + "tokens_seen": 472449024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923811434302909, + "loss": 1.4511, + "theoretical_loss": 3.967351232225132, + "tokens_seen": 472514560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923650952858576, + "loss": 1.4974, + "theoretical_loss": 3.9672918204311074, + "tokens_seen": 472580096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923490471414243, + "loss": 1.5511, + "theoretical_loss": 3.9672324191821318, + "tokens_seen": 472645632 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692332998996991, + "loss": 1.6074, + "theoretical_loss": 3.967173028474871, + "tokens_seen": 472711168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923169508525577, + "loss": 1.6054, + "theoretical_loss": 3.967113648305995, + "tokens_seen": 472776704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923009027081244, + "loss": 1.5523, + "theoretical_loss": 3.9670542786721716, + "tokens_seen": 472842240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922848545636911, + "loss": 1.4466, + "theoretical_loss": 3.9669949195700727, + "tokens_seen": 472907776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922688064192578, + "loss": 1.5414, + "theoretical_loss": 3.9669355709963705, + "tokens_seen": 472973312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922527582748245, + "loss": 1.4645, + "theoretical_loss": 3.9668762329477403, + "tokens_seen": 473038848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922367101303912, + "loss": 1.4129, + "theoretical_loss": 3.966816905420856, + "tokens_seen": 473104384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922206619859579, + "loss": 1.5528, + "theoretical_loss": 3.966757588412396, + "tokens_seen": 473169920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922046138415246, + "loss": 1.6118, + "theoretical_loss": 3.9666982819190384, + "tokens_seen": 473235456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921885656970913, + "loss": 1.4257, + "theoretical_loss": 3.9666389859374642, + "tokens_seen": 473300992 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692172517552658, + "loss": 1.5467, + "theoretical_loss": 3.9665797004643544, + "tokens_seen": 473366528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921564694082247, + "loss": 1.5235, + "theoretical_loss": 3.966520425496392, + "tokens_seen": 473432064 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921404212637914, + "loss": 1.4722, + "theoretical_loss": 3.9664611610302627, + "tokens_seen": 473497600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921243731193581, + "loss": 1.3878, + "theoretical_loss": 3.966401907062652, + "tokens_seen": 473563136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921083249749248, + "loss": 1.5362, + "theoretical_loss": 3.9663426635902477, + "tokens_seen": 473628672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920922768304915, + "loss": 1.6144, + "theoretical_loss": 3.9662834306097388, + "tokens_seen": 473694208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920762286860582, + "loss": 1.4815, + "theoretical_loss": 3.9662242081178167, + "tokens_seen": 473759744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920601805416249, + "loss": 1.508, + "theoretical_loss": 3.9661649961111727, + "tokens_seen": 473825280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920441323971916, + "loss": 1.4545, + "theoretical_loss": 3.9661057945865017, + "tokens_seen": 473890816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920280842527583, + "loss": 1.565, + "theoretical_loss": 3.966046603540498, + "tokens_seen": 473956352 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692012036108325, + "loss": 1.4851, + "theoretical_loss": 3.9659874229698584, + "tokens_seen": 474021888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919959879638917, + "loss": 1.4032, + "theoretical_loss": 3.9659282528712816, + "tokens_seen": 474087424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919799398194584, + "loss": 1.4323, + "theoretical_loss": 3.965869093241467, + "tokens_seen": 474152960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919638916750251, + "loss": 1.519, + "theoretical_loss": 3.965809944077116, + "tokens_seen": 474218496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919478435305918, + "loss": 1.4708, + "theoretical_loss": 3.965750805374931, + "tokens_seen": 474284032 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919317953861585, + "loss": 1.4264, + "theoretical_loss": 3.965691677131617, + "tokens_seen": 474349568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919157472417252, + "loss": 1.5769, + "theoretical_loss": 3.9656325593438786, + "tokens_seen": 474415104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918996990972919, + "loss": 1.4914, + "theoretical_loss": 3.9655734520084236, + "tokens_seen": 474480640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918836509528587, + "loss": 1.5013, + "theoretical_loss": 3.965514355121961, + "tokens_seen": 474546176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918676028084253, + "loss": 1.5206, + "theoretical_loss": 3.9654552686812004, + "tokens_seen": 474611712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918515546639921, + "loss": 1.452, + "theoretical_loss": 3.9653961926828534, + "tokens_seen": 474677248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918355065195587, + "loss": 1.5568, + "theoretical_loss": 3.9653371271236337, + "tokens_seen": 474742784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918194583751255, + "loss": 1.576, + "theoretical_loss": 3.9652780720002556, + "tokens_seen": 474808320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918034102306921, + "loss": 1.544, + "theoretical_loss": 3.965219027309435, + "tokens_seen": 474873856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917873620862589, + "loss": 1.5643, + "theoretical_loss": 3.9651599930478905, + "tokens_seen": 474939392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917713139418255, + "loss": 1.4463, + "theoretical_loss": 3.9651009692123402, + "tokens_seen": 475004928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917552657973923, + "loss": 1.5332, + "theoretical_loss": 3.9650419557995047, + "tokens_seen": 475070464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 164144, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6853489875793457, + "objective/train/theoretical_loss": 3.964982952806107, + "objective/train/tokens_used": 495596000, + "theoretical_loss": 3.964982952806107, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917392176529589, + "loss": 1.5972, + "theoretical_loss": 3.964982952806107, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917231695085257, + "loss": 1.4829, + "theoretical_loss": 3.9649239602288695, + "tokens_seen": 475201536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917071213640923, + "loss": 1.3827, + "theoretical_loss": 3.9648649780645178, + "tokens_seen": 475267072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916910732196591, + "loss": 1.4239, + "theoretical_loss": 3.9648060063097788, + "tokens_seen": 475332608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916750250752257, + "loss": 1.4822, + "theoretical_loss": 3.9647470449613795, + "tokens_seen": 475398144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916589769307925, + "loss": 1.4893, + "theoretical_loss": 3.9646880940160507, + "tokens_seen": 475463680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916429287863591, + "loss": 1.4965, + "theoretical_loss": 3.9646291534705225, + "tokens_seen": 475529216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916268806419258, + "loss": 1.5566, + "theoretical_loss": 3.9645702233215268, + "tokens_seen": 475594752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916108324974925, + "loss": 1.4053, + "theoretical_loss": 3.964511303565799, + "tokens_seen": 475660288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006915947843530592, + "loss": 1.5627, + "theoretical_loss": 3.964452394200073, + "tokens_seen": 475725824 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691578736208626, + "loss": 1.4794, + "theoretical_loss": 3.9643934952210866, + "tokens_seen": 475791360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006915626880641926, + "loss": 1.484, + "theoretical_loss": 3.9643346066255782, + "tokens_seen": 475856896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006915466399197593, + "loss": 1.5084, + "theoretical_loss": 3.964275728410287, + "tokens_seen": 475922432 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691530591775326, + "loss": 1.4504, + "theoretical_loss": 3.9642168605719545, + "tokens_seen": 475987968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006915145436308928, + "loss": 1.5379, + "theoretical_loss": 3.9641580031073236, + "tokens_seen": 476053504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914984954864594, + "loss": 1.5919, + "theoretical_loss": 3.9640991560131384, + "tokens_seen": 476119040 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691482447342026, + "loss": 1.4496, + "theoretical_loss": 3.964040319286145, + "tokens_seen": 476184576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914663991975928, + "loss": 1.4655, + "theoretical_loss": 3.9639814929230894, + "tokens_seen": 476250112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914503510531594, + "loss": 1.5148, + "theoretical_loss": 3.9639226769207214, + "tokens_seen": 476315648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914343029087262, + "loss": 1.4979, + "theoretical_loss": 3.963863871275791, + "tokens_seen": 476381184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914182547642928, + "loss": 1.5029, + "theoretical_loss": 3.963805075985049, + "tokens_seen": 476446720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914022066198596, + "loss": 1.5035, + "theoretical_loss": 3.9637462910452497, + "tokens_seen": 476512256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006913861584754262, + "loss": 1.6175, + "theoretical_loss": 3.963687516453146, + "tokens_seen": 476577792 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691370110330993, + "loss": 1.4935, + "theoretical_loss": 3.9636287522054947, + "tokens_seen": 476643328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006913540621865596, + "loss": 1.5319, + "theoretical_loss": 3.9635699982990538, + "tokens_seen": 476708864 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006913380140421264, + "loss": 1.3645, + "theoretical_loss": 3.963511254730581, + "tokens_seen": 476774400 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691321965897693, + "loss": 1.4714, + "theoretical_loss": 3.9634525214968375, + "tokens_seen": 476839936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006913059177532598, + "loss": 1.4609, + "theoretical_loss": 3.963393798594585, + "tokens_seen": 476905472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912898696088265, + "loss": 1.5017, + "theoretical_loss": 3.963335086020586, + "tokens_seen": 476971008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912738214643932, + "loss": 1.5675, + "theoretical_loss": 3.9632763837716065, + "tokens_seen": 477036544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912577733199599, + "loss": 1.5274, + "theoretical_loss": 3.9632176918444113, + "tokens_seen": 477102080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912417251755266, + "loss": 1.4453, + "theoretical_loss": 3.9631590102357697, + "tokens_seen": 477167616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912256770310933, + "loss": 1.491, + "theoretical_loss": 3.963100338942449, + "tokens_seen": 477233152 + }, + { + "epoch": 0.14, + "learning_rate": 0.00069120962888666, + "loss": 1.4311, + "theoretical_loss": 3.963041677961221, + "tokens_seen": 477298688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911935807422267, + "loss": 1.5531, + "theoretical_loss": 3.962983027288857, + "tokens_seen": 477364224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911775325977934, + "loss": 1.4859, + "theoretical_loss": 3.9629243869221304, + "tokens_seen": 477429760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911614844533601, + "loss": 1.5019, + "theoretical_loss": 3.962865756857817, + "tokens_seen": 477495296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911454363089268, + "loss": 1.5433, + "theoretical_loss": 3.9628071370926925, + "tokens_seen": 477560832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911293881644935, + "loss": 1.5089, + "theoretical_loss": 3.9627485276235346, + "tokens_seen": 477626368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911133400200602, + "loss": 1.5039, + "theoretical_loss": 3.962689928447123, + "tokens_seen": 477691904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910972918756269, + "loss": 1.6092, + "theoretical_loss": 3.962631339560238, + "tokens_seen": 477757440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910812437311936, + "loss": 1.5068, + "theoretical_loss": 3.962572760959662, + "tokens_seen": 477822976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910651955867603, + "loss": 1.4194, + "theoretical_loss": 3.9625141926421783, + "tokens_seen": 477888512 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691049147442327, + "loss": 1.5435, + "theoretical_loss": 3.9624556346045727, + "tokens_seen": 477954048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910330992978937, + "loss": 1.4924, + "theoretical_loss": 3.962397086843631, + "tokens_seen": 478019584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910170511534604, + "loss": 1.5266, + "theoretical_loss": 3.962338549356141, + "tokens_seen": 478085120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910010030090271, + "loss": 1.5045, + "theoretical_loss": 3.962280022138893, + "tokens_seen": 478150656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909849548645938, + "loss": 1.5396, + "theoretical_loss": 3.962221505188677, + "tokens_seen": 478216192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909689067201605, + "loss": 1.4155, + "theoretical_loss": 3.9621629985022855, + "tokens_seen": 478281728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909528585757272, + "loss": 1.482, + "theoretical_loss": 3.9621045020765124, + "tokens_seen": 478347264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 165388, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.461626648902893, + "objective/train/theoretical_loss": 3.9620460159081525, + "objective/train/tokens_used": 498872800, + "theoretical_loss": 3.9620460159081525, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909368104312939, + "loss": 1.4965, + "theoretical_loss": 3.9620460159081525, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909207622868606, + "loss": 1.5332, + "theoretical_loss": 3.9619875399940025, + "tokens_seen": 478478336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006909047141424273, + "loss": 1.4555, + "theoretical_loss": 3.9619290743308606, + "tokens_seen": 478543872 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690888665997994, + "loss": 1.5502, + "theoretical_loss": 3.961870618915526, + "tokens_seen": 478609408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908726178535607, + "loss": 1.4432, + "theoretical_loss": 3.961812173744801, + "tokens_seen": 478674944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908565697091274, + "loss": 1.4627, + "theoretical_loss": 3.961753738815486, + "tokens_seen": 478740480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908405215646941, + "loss": 1.4945, + "theoretical_loss": 3.961695314124385, + "tokens_seen": 478806016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908244734202608, + "loss": 1.5489, + "theoretical_loss": 3.961636899668304, + "tokens_seen": 478871552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908084252758275, + "loss": 1.5541, + "theoretical_loss": 3.96157849544405, + "tokens_seen": 478937088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907923771313942, + "loss": 1.5218, + "theoretical_loss": 3.96152010144843, + "tokens_seen": 479002624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907763289869609, + "loss": 1.476, + "theoretical_loss": 3.961461717678254, + "tokens_seen": 479068160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907602808425276, + "loss": 1.3915, + "theoretical_loss": 3.961403344130333, + "tokens_seen": 479133696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907442326980943, + "loss": 1.5185, + "theoretical_loss": 3.9613449808014796, + "tokens_seen": 479199232 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690728184553661, + "loss": 1.6288, + "theoretical_loss": 3.9612866276885073, + "tokens_seen": 479264768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907121364092277, + "loss": 1.5618, + "theoretical_loss": 3.961228284788231, + "tokens_seen": 479330304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906960882647944, + "loss": 1.403, + "theoretical_loss": 3.9611699520974684, + "tokens_seen": 479395840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906800401203612, + "loss": 1.5271, + "theoretical_loss": 3.961111629613036, + "tokens_seen": 479461376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906639919759278, + "loss": 1.627, + "theoretical_loss": 3.9610533173317553, + "tokens_seen": 479526912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906479438314946, + "loss": 1.4974, + "theoretical_loss": 3.960995015250446, + "tokens_seen": 479592448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906318956870612, + "loss": 1.4971, + "theoretical_loss": 3.96093672336593, + "tokens_seen": 479657984 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690615847542628, + "loss": 1.6243, + "theoretical_loss": 3.9608784416750322, + "tokens_seen": 479723520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905997993981946, + "loss": 1.4766, + "theoretical_loss": 3.9608201701745775, + "tokens_seen": 479789056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905837512537614, + "loss": 1.5663, + "theoretical_loss": 3.960761908861392, + "tokens_seen": 479854592 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690567703109328, + "loss": 1.4331, + "theoretical_loss": 3.9607036577323047, + "tokens_seen": 479920128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905516549648948, + "loss": 1.3873, + "theoretical_loss": 3.9606454167841445, + "tokens_seen": 479985664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905356068204614, + "loss": 1.5245, + "theoretical_loss": 3.960587186013742, + "tokens_seen": 480051200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905195586760282, + "loss": 1.5018, + "theoretical_loss": 3.96052896541793, + "tokens_seen": 480116736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905035105315948, + "loss": 1.4737, + "theoretical_loss": 3.9604707549935423, + "tokens_seen": 480182272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006904874623871616, + "loss": 1.5611, + "theoretical_loss": 3.9604125547374136, + "tokens_seen": 480247808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006904714142427282, + "loss": 1.4181, + "theoretical_loss": 3.9603543646463812, + "tokens_seen": 480313344 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690455366098295, + "loss": 1.5261, + "theoretical_loss": 3.9602961847172824, + "tokens_seen": 480378880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006904393179538616, + "loss": 1.5351, + "theoretical_loss": 3.960238014946957, + "tokens_seen": 480444416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006904232698094284, + "loss": 1.4876, + "theoretical_loss": 3.960179855332245, + "tokens_seen": 480509952 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690407221664995, + "loss": 1.4839, + "theoretical_loss": 3.96012170586999, + "tokens_seen": 480575488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903911735205618, + "loss": 1.3866, + "theoretical_loss": 3.9600635665570345, + "tokens_seen": 480641024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903751253761284, + "loss": 1.5976, + "theoretical_loss": 3.9600054373902243, + "tokens_seen": 480706560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903590772316952, + "loss": 1.4092, + "theoretical_loss": 3.959947318366405, + "tokens_seen": 480772096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903430290872618, + "loss": 1.4633, + "theoretical_loss": 3.9598892094824256, + "tokens_seen": 480837632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903269809428286, + "loss": 1.5516, + "theoretical_loss": 3.9598311107351347, + "tokens_seen": 480903168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903109327983952, + "loss": 1.5714, + "theoretical_loss": 3.959773022121383, + "tokens_seen": 480968704 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690294884653962, + "loss": 1.4482, + "theoretical_loss": 3.959714943638023, + "tokens_seen": 481034240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006902788365095286, + "loss": 1.4323, + "theoretical_loss": 3.9596568752819072, + "tokens_seen": 481099776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006902627883650954, + "loss": 1.5344, + "theoretical_loss": 3.9595988170498915, + "tokens_seen": 481165312 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690246740220662, + "loss": 1.5042, + "theoretical_loss": 3.959540768938832, + "tokens_seen": 481230848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006902306920762287, + "loss": 1.6065, + "theoretical_loss": 3.9594827309455862, + "tokens_seen": 481296384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006902146439317954, + "loss": 1.5888, + "theoretical_loss": 3.9594247030670133, + "tokens_seen": 481361920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901985957873621, + "loss": 1.4552, + "theoretical_loss": 3.959366685299974, + "tokens_seen": 481427456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901825476429288, + "loss": 1.564, + "theoretical_loss": 3.9593086776413298, + "tokens_seen": 481492992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901664994984955, + "loss": 1.4304, + "theoretical_loss": 3.959250680087945, + "tokens_seen": 481558528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901504513540622, + "loss": 1.4006, + "theoretical_loss": 3.959192692636683, + "tokens_seen": 481624064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 166207, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4316575527191162, + "objective/train/theoretical_loss": 3.959134715284411, + "objective/train/tokens_used": 502149600, + "theoretical_loss": 3.959134715284411, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901344032096289, + "loss": 1.5149, + "theoretical_loss": 3.959134715284411, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901183550651957, + "loss": 1.5353, + "theoretical_loss": 3.959076748027996, + "tokens_seen": 481755136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901023069207623, + "loss": 1.5337, + "theoretical_loss": 3.959018790864307, + "tokens_seen": 481820672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900862587763289, + "loss": 1.535, + "theoretical_loss": 3.9589608437902144, + "tokens_seen": 481886208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900702106318957, + "loss": 1.5224, + "theoretical_loss": 3.9589029068025896, + "tokens_seen": 481951744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900541624874623, + "loss": 1.4818, + "theoretical_loss": 3.9588449798983065, + "tokens_seen": 482017280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900381143430291, + "loss": 1.4084, + "theoretical_loss": 3.9587870630742383, + "tokens_seen": 482082816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900220661985957, + "loss": 1.6073, + "theoretical_loss": 3.958729156327262, + "tokens_seen": 482148352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900060180541625, + "loss": 1.5083, + "theoretical_loss": 3.9586712596542553, + "tokens_seen": 482213888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899899699097291, + "loss": 1.4613, + "theoretical_loss": 3.958613373052095, + "tokens_seen": 482279424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899739217652959, + "loss": 1.4953, + "theoretical_loss": 3.958555496517663, + "tokens_seen": 482344960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899578736208625, + "loss": 1.4788, + "theoretical_loss": 3.95849763004784, + "tokens_seen": 482410496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899418254764293, + "loss": 1.4487, + "theoretical_loss": 3.958439773639509, + "tokens_seen": 482476032 + }, + { + "epoch": 0.15, + "learning_rate": 0.000689925777331996, + "loss": 1.4259, + "theoretical_loss": 3.958381927289554, + "tokens_seen": 482541568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899097291875627, + "loss": 1.4236, + "theoretical_loss": 3.958324090994861, + "tokens_seen": 482607104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898936810431294, + "loss": 1.4207, + "theoretical_loss": 3.958266264752317, + "tokens_seen": 482672640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898776328986961, + "loss": 1.5267, + "theoretical_loss": 3.9582084485588096, + "tokens_seen": 482738176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898615847542628, + "loss": 1.5434, + "theoretical_loss": 3.95815064241123, + "tokens_seen": 482803712 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898455366098295, + "loss": 1.4206, + "theoretical_loss": 3.958092846306468, + "tokens_seen": 482869248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898294884653962, + "loss": 1.5472, + "theoretical_loss": 3.958035060241417, + "tokens_seen": 482934784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898134403209629, + "loss": 1.5298, + "theoretical_loss": 3.957977284212971, + "tokens_seen": 483000320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897973921765296, + "loss": 1.3356, + "theoretical_loss": 3.9579195182180245, + "tokens_seen": 483065856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897813440320963, + "loss": 1.4347, + "theoretical_loss": 3.957861762253475, + "tokens_seen": 483131392 + }, + { + "epoch": 0.15, + "learning_rate": 0.000689765295887663, + "loss": 1.4678, + "theoretical_loss": 3.9578040163162203, + "tokens_seen": 483196928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897492477432297, + "loss": 1.5097, + "theoretical_loss": 3.95774628040316, + "tokens_seen": 483262464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897331995987964, + "loss": 1.4883, + "theoretical_loss": 3.9576885545111944, + "tokens_seen": 483328000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897171514543631, + "loss": 1.4399, + "theoretical_loss": 3.9576308386372263, + "tokens_seen": 483393536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897011033099298, + "loss": 1.6046, + "theoretical_loss": 3.957573132778159, + "tokens_seen": 483459072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896850551654965, + "loss": 1.499, + "theoretical_loss": 3.9575154369308985, + "tokens_seen": 483524608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896690070210632, + "loss": 1.482, + "theoretical_loss": 3.957457751092349, + "tokens_seen": 483590144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896529588766299, + "loss": 1.3449, + "theoretical_loss": 3.9574000752594203, + "tokens_seen": 483655680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896369107321966, + "loss": 1.466, + "theoretical_loss": 3.9573424094290206, + "tokens_seen": 483721216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896208625877633, + "loss": 1.3929, + "theoretical_loss": 3.9572847535980604, + "tokens_seen": 483786752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068960481444333, + "loss": 1.446, + "theoretical_loss": 3.9572271077634515, + "tokens_seen": 483852288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895887662988967, + "loss": 1.5478, + "theoretical_loss": 3.957169471922107, + "tokens_seen": 483917824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895727181544634, + "loss": 1.4874, + "theoretical_loss": 3.9571118460709425, + "tokens_seen": 483983360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895566700100301, + "loss": 1.4437, + "theoretical_loss": 3.9570542302068725, + "tokens_seen": 484048896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895406218655968, + "loss": 1.5434, + "theoretical_loss": 3.9569966243268153, + "tokens_seen": 484114432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895245737211635, + "loss": 1.4222, + "theoretical_loss": 3.9569390284276893, + "tokens_seen": 484179968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895085255767302, + "loss": 1.4055, + "theoretical_loss": 3.9568814425064147, + "tokens_seen": 484245504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894924774322969, + "loss": 1.5564, + "theoretical_loss": 3.9568238665599127, + "tokens_seen": 484311040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894764292878637, + "loss": 1.5019, + "theoretical_loss": 3.9567663005851057, + "tokens_seen": 484376576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894603811434303, + "loss": 1.4021, + "theoretical_loss": 3.956708744578919, + "tokens_seen": 484442112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894443329989971, + "loss": 1.4476, + "theoretical_loss": 3.9566511985382773, + "tokens_seen": 484507648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894282848545637, + "loss": 1.4908, + "theoretical_loss": 3.956593662460108, + "tokens_seen": 484573184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894122367101305, + "loss": 1.5107, + "theoretical_loss": 3.9565361363413385, + "tokens_seen": 484638720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893961885656971, + "loss": 1.4567, + "theoretical_loss": 3.9564786201788995, + "tokens_seen": 484704256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893801404212639, + "loss": 1.5791, + "theoretical_loss": 3.956421113969721, + "tokens_seen": 484769792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893640922768305, + "loss": 1.4383, + "theoretical_loss": 3.9563636177107364, + "tokens_seen": 484835328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893480441323973, + "loss": 1.4124, + "theoretical_loss": 3.956306131398878, + "tokens_seen": 484900864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 167502, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4446344375610352, + "objective/train/theoretical_loss": 3.9562486550310823, + "objective/train/tokens_used": 505426400, + "theoretical_loss": 3.9562486550310823, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893319959879639, + "loss": 1.3573, + "theoretical_loss": 3.9562486550310823, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893159478435307, + "loss": 1.4464, + "theoretical_loss": 3.9561911886042846, + "tokens_seen": 485031936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892998996990973, + "loss": 1.4097, + "theoretical_loss": 3.9561337321154237, + "tokens_seen": 485097472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892838515546641, + "loss": 1.4955, + "theoretical_loss": 3.956076285561438, + "tokens_seen": 485163008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892678034102307, + "loss": 1.4266, + "theoretical_loss": 3.9560188489392676, + "tokens_seen": 485228544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892517552657975, + "loss": 1.4735, + "theoretical_loss": 3.9559614222458555, + "tokens_seen": 485294080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892357071213641, + "loss": 1.4313, + "theoretical_loss": 3.955904005478144, + "tokens_seen": 485359616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892196589769309, + "loss": 1.4503, + "theoretical_loss": 3.9558465986330784, + "tokens_seen": 485425152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892036108324975, + "loss": 1.5692, + "theoretical_loss": 3.9557892017076037, + "tokens_seen": 485490688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891875626880643, + "loss": 1.4592, + "theoretical_loss": 3.9557318146986677, + "tokens_seen": 485556224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891715145436309, + "loss": 1.4865, + "theoretical_loss": 3.9556744376032196, + "tokens_seen": 485621760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891554663991977, + "loss": 1.554, + "theoretical_loss": 3.9556170704182083, + "tokens_seen": 485687296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891394182547643, + "loss": 1.4508, + "theoretical_loss": 3.955559713140585, + "tokens_seen": 485752832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891233701103311, + "loss": 1.4282, + "theoretical_loss": 3.9555023657673036, + "tokens_seen": 485818368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891073219658977, + "loss": 1.4978, + "theoretical_loss": 3.955445028295317, + "tokens_seen": 485883904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890912738214645, + "loss": 1.418, + "theoretical_loss": 3.955387700721581, + "tokens_seen": 485949440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890752256770311, + "loss": 1.5058, + "theoretical_loss": 3.9553303830430524, + "tokens_seen": 486014976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890591775325979, + "loss": 1.5994, + "theoretical_loss": 3.955273075256689, + "tokens_seen": 486080512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890431293881645, + "loss": 1.4623, + "theoretical_loss": 3.9552157773594496, + "tokens_seen": 486146048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890270812437313, + "loss": 1.4523, + "theoretical_loss": 3.9551584893482965, + "tokens_seen": 486211584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890110330992979, + "loss": 1.5572, + "theoretical_loss": 3.9551012112201906, + "tokens_seen": 486277120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889949849548647, + "loss": 1.4486, + "theoretical_loss": 3.9550439429720954, + "tokens_seen": 486342656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889789368104313, + "loss": 1.4769, + "theoretical_loss": 3.9549866846009762, + "tokens_seen": 486408192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889628886659981, + "loss": 1.6685, + "theoretical_loss": 3.9549294361037988, + "tokens_seen": 486473728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889468405215647, + "loss": 1.5421, + "theoretical_loss": 3.9548721974775303, + "tokens_seen": 486539264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889307923771315, + "loss": 1.477, + "theoretical_loss": 3.95481496871914, + "tokens_seen": 486604800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889147442326981, + "loss": 1.5481, + "theoretical_loss": 3.954757749825598, + "tokens_seen": 486670336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888986960882649, + "loss": 1.4039, + "theoretical_loss": 3.9547005407938753, + "tokens_seen": 486735872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888826479438315, + "loss": 1.3672, + "theoretical_loss": 3.9546433416209448, + "tokens_seen": 486801408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888665997993983, + "loss": 1.5134, + "theoretical_loss": 3.9545861523037815, + "tokens_seen": 486866944 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688850551654965, + "loss": 1.5121, + "theoretical_loss": 3.95452897283936, + "tokens_seen": 486932480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888345035105317, + "loss": 1.5094, + "theoretical_loss": 3.9544718032246573, + "tokens_seen": 486998016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888184553660983, + "loss": 1.5549, + "theoretical_loss": 3.9544146434566514, + "tokens_seen": 487063552 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688802407221665, + "loss": 1.4202, + "theoretical_loss": 3.9543574935323216, + "tokens_seen": 487129088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887863590772317, + "loss": 1.4835, + "theoretical_loss": 3.95430035344865, + "tokens_seen": 487194624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887703109327984, + "loss": 1.4217, + "theoretical_loss": 3.954243223202617, + "tokens_seen": 487260160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887542627883652, + "loss": 1.4977, + "theoretical_loss": 3.9541861027912066, + "tokens_seen": 487325696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887382146439318, + "loss": 1.5066, + "theoretical_loss": 3.9541289922114045, + "tokens_seen": 487391232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887221664994986, + "loss": 1.5499, + "theoretical_loss": 3.954071891460196, + "tokens_seen": 487456768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887061183550652, + "loss": 1.4346, + "theoretical_loss": 3.954014800534569, + "tokens_seen": 487522304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886900702106318, + "loss": 1.4414, + "theoretical_loss": 3.9539577194315116, + "tokens_seen": 487587840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886740220661986, + "loss": 1.5233, + "theoretical_loss": 3.953900648148015, + "tokens_seen": 487653376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886579739217652, + "loss": 1.5257, + "theoretical_loss": 3.953843586681069, + "tokens_seen": 487718912 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688641925777332, + "loss": 1.4625, + "theoretical_loss": 3.9537865350276684, + "tokens_seen": 487784448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886258776328986, + "loss": 1.6209, + "theoretical_loss": 3.953729493184806, + "tokens_seen": 487849984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886098294884654, + "loss": 1.4365, + "theoretical_loss": 3.953672461149477, + "tokens_seen": 487915520 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688593781344032, + "loss": 1.5948, + "theoretical_loss": 3.953615438918679, + "tokens_seen": 487981056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885777331995988, + "loss": 1.4913, + "theoretical_loss": 3.9535584264894092, + "tokens_seen": 488046592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885616850551655, + "loss": 1.4834, + "theoretical_loss": 3.9535014238586683, + "tokens_seen": 488112128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885456369107322, + "loss": 1.5498, + "theoretical_loss": 3.953444431023456, + "tokens_seen": 488177664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 168681, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5218801498413086, + "objective/train/theoretical_loss": 3.9533874479807745, + "objective/train/tokens_used": 508703200, + "theoretical_loss": 3.9533874479807745, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885295887662989, + "loss": 1.4728, + "theoretical_loss": 3.9533874479807745, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885135406218656, + "loss": 1.4928, + "theoretical_loss": 3.9533304747276263, + "tokens_seen": 488308736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884974924774323, + "loss": 1.4964, + "theoretical_loss": 3.953273511261018, + "tokens_seen": 488374272 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688481444332999, + "loss": 1.4585, + "theoretical_loss": 3.953216557577954, + "tokens_seen": 488439808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884653961885657, + "loss": 1.5129, + "theoretical_loss": 3.9531596136754423, + "tokens_seen": 488505344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884493480441324, + "loss": 1.3073, + "theoretical_loss": 3.953102679550491, + "tokens_seen": 488570880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884332998996991, + "loss": 1.6385, + "theoretical_loss": 3.953045755200111, + "tokens_seen": 488636416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884172517552658, + "loss": 1.4704, + "theoretical_loss": 3.9529888406213125, + "tokens_seen": 488701952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884012036108325, + "loss": 1.4706, + "theoretical_loss": 3.952931935811108, + "tokens_seen": 488767488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883851554663992, + "loss": 1.4269, + "theoretical_loss": 3.9528750407665125, + "tokens_seen": 488833024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883691073219659, + "loss": 1.6005, + "theoretical_loss": 3.9528181554845405, + "tokens_seen": 488898560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883530591775326, + "loss": 1.3151, + "theoretical_loss": 3.9527612799622087, + "tokens_seen": 488964096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883370110330993, + "loss": 1.3791, + "theoretical_loss": 3.9527044141965346, + "tokens_seen": 489029632 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688320962888666, + "loss": 1.5089, + "theoretical_loss": 3.9526475581845375, + "tokens_seen": 489095168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883049147442327, + "loss": 1.4356, + "theoretical_loss": 3.9525907119232384, + "tokens_seen": 489160704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882888665997994, + "loss": 1.4122, + "theoretical_loss": 3.952533875409658, + "tokens_seen": 489226240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882728184553662, + "loss": 1.6054, + "theoretical_loss": 3.95247704864082, + "tokens_seen": 489291776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882567703109328, + "loss": 1.5323, + "theoretical_loss": 3.952420231613748, + "tokens_seen": 489357312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882407221664996, + "loss": 1.4921, + "theoretical_loss": 3.952363424325469, + "tokens_seen": 489422848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882246740220662, + "loss": 1.5348, + "theoretical_loss": 3.952306626773009, + "tokens_seen": 489488384 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688208625877633, + "loss": 1.2855, + "theoretical_loss": 3.9522498389533967, + "tokens_seen": 489553920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881925777331996, + "loss": 1.4844, + "theoretical_loss": 3.952193060863661, + "tokens_seen": 489619456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881765295887664, + "loss": 1.4188, + "theoretical_loss": 3.9521362925008336, + "tokens_seen": 489684992 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688160481444333, + "loss": 1.3951, + "theoretical_loss": 3.952079533861947, + "tokens_seen": 489750528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881444332998998, + "loss": 1.5724, + "theoretical_loss": 3.9520227849440337, + "tokens_seen": 489816064 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881283851554664, + "loss": 1.3901, + "theoretical_loss": 3.951966045744129, + "tokens_seen": 489881600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881123370110332, + "loss": 1.5279, + "theoretical_loss": 3.9519093162592687, + "tokens_seen": 489947136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880962888665998, + "loss": 1.4731, + "theoretical_loss": 3.95185259648649, + "tokens_seen": 490012672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880802407221666, + "loss": 1.4999, + "theoretical_loss": 3.951795886422833, + "tokens_seen": 490078208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880641925777332, + "loss": 1.5234, + "theoretical_loss": 3.9517391860653364, + "tokens_seen": 490143744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880481444333, + "loss": 1.5716, + "theoretical_loss": 3.951682495411041, + "tokens_seen": 490209280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880320962888666, + "loss": 1.4127, + "theoretical_loss": 3.9516258144569907, + "tokens_seen": 490274816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880160481444334, + "loss": 1.4481, + "theoretical_loss": 3.9515691432002296, + "tokens_seen": 490340352 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688, + "loss": 1.5564, + "theoretical_loss": 3.951512481637801, + "tokens_seen": 490405888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879839518555668, + "loss": 1.5187, + "theoretical_loss": 3.9514558297667532, + "tokens_seen": 490471424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879679037111334, + "loss": 1.5075, + "theoretical_loss": 3.9513991875841334, + "tokens_seen": 490536960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879518555667002, + "loss": 1.4442, + "theoretical_loss": 3.95134255508699, + "tokens_seen": 490602496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879358074222668, + "loss": 1.5834, + "theoretical_loss": 3.9512859322723743, + "tokens_seen": 490668032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879197592778336, + "loss": 1.5289, + "theoretical_loss": 3.9512293191373375, + "tokens_seen": 490733568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879037111334002, + "loss": 1.6062, + "theoretical_loss": 3.9511727156789327, + "tokens_seen": 490799104 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687887662988967, + "loss": 1.5124, + "theoretical_loss": 3.951116121894214, + "tokens_seen": 490864640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006878716148445336, + "loss": 1.4595, + "theoretical_loss": 3.951059537780237, + "tokens_seen": 490930176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006878555667001004, + "loss": 1.5176, + "theoretical_loss": 3.951002963334059, + "tokens_seen": 490995712 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687839518555667, + "loss": 1.4955, + "theoretical_loss": 3.9509463985527375, + "tokens_seen": 491061248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006878234704112338, + "loss": 1.4248, + "theoretical_loss": 3.9508898434333313, + "tokens_seen": 491126784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006878074222668004, + "loss": 1.5181, + "theoretical_loss": 3.9508332979729026, + "tokens_seen": 491192320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877913741223672, + "loss": 1.4302, + "theoretical_loss": 3.950776762168512, + "tokens_seen": 491257856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877753259779338, + "loss": 1.4632, + "theoretical_loss": 3.9507202360172236, + "tokens_seen": 491323392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877592778335006, + "loss": 1.3747, + "theoretical_loss": 3.9506637195161014, + "tokens_seen": 491388928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877432296890672, + "loss": 1.3571, + "theoretical_loss": 3.950607212662211, + "tokens_seen": 491454464 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 169679, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.443345069885254, + "objective/train/theoretical_loss": 3.950550715452621, + "objective/train/tokens_used": 511980000, + "theoretical_loss": 3.950550715452621, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687727181544634, + "loss": 1.4414, + "theoretical_loss": 3.950550715452621, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877111334002006, + "loss": 1.5504, + "theoretical_loss": 3.9504942278843984, + "tokens_seen": 491585536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876950852557674, + "loss": 1.4341, + "theoretical_loss": 3.950437749954613, + "tokens_seen": 491651072 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687679037111334, + "loss": 1.5328, + "theoretical_loss": 3.950381281660336, + "tokens_seen": 491716608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876629889669008, + "loss": 1.5646, + "theoretical_loss": 3.950324822998639, + "tokens_seen": 491782144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876469408224674, + "loss": 1.5351, + "theoretical_loss": 3.950268373966597, + "tokens_seen": 491847680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876308926780342, + "loss": 1.5962, + "theoretical_loss": 3.950211934561284, + "tokens_seen": 491913216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876148445336008, + "loss": 1.5011, + "theoretical_loss": 3.950155504779775, + "tokens_seen": 491978752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875987963891676, + "loss": 1.5318, + "theoretical_loss": 3.9500990846191484, + "tokens_seen": 492044288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875827482447342, + "loss": 1.4562, + "theoretical_loss": 3.950042674076483, + "tokens_seen": 492109824 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687566700100301, + "loss": 1.5587, + "theoretical_loss": 3.949986273148858, + "tokens_seen": 492175360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875506519558676, + "loss": 1.6206, + "theoretical_loss": 3.9499298818333557, + "tokens_seen": 492240896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875346038114344, + "loss": 1.5126, + "theoretical_loss": 3.9498735001270573, + "tokens_seen": 492306432 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687518555667001, + "loss": 1.6095, + "theoretical_loss": 3.949817128027047, + "tokens_seen": 492371968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875025075225678, + "loss": 1.4721, + "theoretical_loss": 3.9497607655304097, + "tokens_seen": 492437504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874864593781344, + "loss": 1.4875, + "theoretical_loss": 3.949704412634232, + "tokens_seen": 492503040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874704112337012, + "loss": 1.5354, + "theoretical_loss": 3.949648069335601, + "tokens_seen": 492568576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874543630892678, + "loss": 1.5265, + "theoretical_loss": 3.9495917356316057, + "tokens_seen": 492634112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874383149448346, + "loss": 1.4206, + "theoretical_loss": 3.9495354115193364, + "tokens_seen": 492699648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874222668004012, + "loss": 1.5544, + "theoretical_loss": 3.9494790969958835, + "tokens_seen": 492765184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874062186559679, + "loss": 1.5174, + "theoretical_loss": 3.949422792058341, + "tokens_seen": 492830720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873901705115347, + "loss": 1.4034, + "theoretical_loss": 3.949366496703802, + "tokens_seen": 492896256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873741223671013, + "loss": 1.5041, + "theoretical_loss": 3.9493102109293616, + "tokens_seen": 492961792 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687358074222668, + "loss": 1.5309, + "theoretical_loss": 3.9492539347321167, + "tokens_seen": 493027328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873420260782347, + "loss": 1.3729, + "theoretical_loss": 3.9491976681091643, + "tokens_seen": 493092864 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873259779338015, + "loss": 1.428, + "theoretical_loss": 3.949141411057604, + "tokens_seen": 493158400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873099297893681, + "loss": 1.5152, + "theoretical_loss": 3.9490851635745354, + "tokens_seen": 493223936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872938816449349, + "loss": 1.4597, + "theoretical_loss": 3.9490289256570605, + "tokens_seen": 493289472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872778335005015, + "loss": 1.5602, + "theoretical_loss": 3.9489726973022816, + "tokens_seen": 493355008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872617853560681, + "loss": 1.4065, + "theoretical_loss": 3.948916478507303, + "tokens_seen": 493420544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872457372116349, + "loss": 1.4266, + "theoretical_loss": 3.94886026926923, + "tokens_seen": 493486080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872296890672015, + "loss": 1.4491, + "theoretical_loss": 3.9488040695851687, + "tokens_seen": 493551616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872136409227683, + "loss": 1.4786, + "theoretical_loss": 3.9487478794522275, + "tokens_seen": 493617152 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687197592778335, + "loss": 1.5208, + "theoretical_loss": 3.948691698867515, + "tokens_seen": 493682688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871815446339017, + "loss": 1.4405, + "theoretical_loss": 3.9486355278281415, + "tokens_seen": 493748224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871654964894684, + "loss": 1.4269, + "theoretical_loss": 3.948579366331219, + "tokens_seen": 493813760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871494483450351, + "loss": 1.524, + "theoretical_loss": 3.94852321437386, + "tokens_seen": 493879296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871334002006018, + "loss": 1.4391, + "theoretical_loss": 3.9484670719531785, + "tokens_seen": 493944832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871173520561685, + "loss": 1.5082, + "theoretical_loss": 3.9484109390662896, + "tokens_seen": 494010368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871013039117352, + "loss": 1.5184, + "theoretical_loss": 3.94835481571031, + "tokens_seen": 494075904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870852557673019, + "loss": 1.4863, + "theoretical_loss": 3.948298701882358, + "tokens_seen": 494141440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870692076228687, + "loss": 1.5287, + "theoretical_loss": 3.9482425975795525, + "tokens_seen": 494206976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870531594784353, + "loss": 1.5385, + "theoretical_loss": 3.9481865027990137, + "tokens_seen": 494272512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870371113340021, + "loss": 1.5284, + "theoretical_loss": 3.9481304175378633, + "tokens_seen": 494338048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870210631895687, + "loss": 1.4945, + "theoretical_loss": 3.948074341793224, + "tokens_seen": 494403584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870050150451355, + "loss": 1.5887, + "theoretical_loss": 3.94801827556222, + "tokens_seen": 494469120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869889669007021, + "loss": 1.4482, + "theoretical_loss": 3.9479622188419765, + "tokens_seen": 494534656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869729187562689, + "loss": 1.4412, + "theoretical_loss": 3.9479061716296204, + "tokens_seen": 494600192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869568706118355, + "loss": 1.5186, + "theoretical_loss": 3.9478501339222793, + "tokens_seen": 494665728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869408224674023, + "loss": 1.4655, + "theoretical_loss": 3.9477941057170822, + "tokens_seen": 494731264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 170930, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5508875846862793, + "objective/train/theoretical_loss": 3.9477380870111602, + "objective/train/tokens_used": 515256800, + "theoretical_loss": 3.9477380870111602, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869247743229689, + "loss": 1.4898, + "theoretical_loss": 3.9477380870111602, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869087261785357, + "loss": 1.5979, + "theoretical_loss": 3.9476820778016437, + "tokens_seen": 494862336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868926780341023, + "loss": 1.5028, + "theoretical_loss": 3.9476260780856665, + "tokens_seen": 494927872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868766298896691, + "loss": 1.5157, + "theoretical_loss": 3.947570087860362, + "tokens_seen": 494993408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868605817452357, + "loss": 1.4887, + "theoretical_loss": 3.947514107122866, + "tokens_seen": 495058944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868445336008025, + "loss": 1.5196, + "theoretical_loss": 3.9474581358703147, + "tokens_seen": 495124480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868284854563691, + "loss": 1.4206, + "theoretical_loss": 3.947402174099847, + "tokens_seen": 495190016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868124373119359, + "loss": 1.4628, + "theoretical_loss": 3.9473462218085995, + "tokens_seen": 495255552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867963891675025, + "loss": 1.468, + "theoretical_loss": 3.947290278993715, + "tokens_seen": 495321088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867803410230693, + "loss": 1.4707, + "theoretical_loss": 3.947234345652334, + "tokens_seen": 495386624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867642928786359, + "loss": 1.4693, + "theoretical_loss": 3.9471784217815995, + "tokens_seen": 495452160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867482447342027, + "loss": 1.524, + "theoretical_loss": 3.9471225073786553, + "tokens_seen": 495517696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867321965897693, + "loss": 1.5436, + "theoretical_loss": 3.9470666024406467, + "tokens_seen": 495583232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867161484453361, + "loss": 1.5024, + "theoretical_loss": 3.9470107069647202, + "tokens_seen": 495648768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867001003009027, + "loss": 1.3839, + "theoretical_loss": 3.946954820948024, + "tokens_seen": 495714304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866840521564695, + "loss": 1.5284, + "theoretical_loss": 3.946898944387706, + "tokens_seen": 495779840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866680040120361, + "loss": 1.5712, + "theoretical_loss": 3.946843077280917, + "tokens_seen": 495845376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866519558676029, + "loss": 1.6285, + "theoretical_loss": 3.946787219624809, + "tokens_seen": 495910912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866359077231695, + "loss": 1.6073, + "theoretical_loss": 3.9467313714165337, + "tokens_seen": 495976448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866198595787363, + "loss": 1.4352, + "theoretical_loss": 3.946675532653246, + "tokens_seen": 496041984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866038114343029, + "loss": 1.514, + "theoretical_loss": 3.9466197033321, + "tokens_seen": 496107520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865877632898697, + "loss": 1.5126, + "theoretical_loss": 3.9465638834502528, + "tokens_seen": 496173056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865717151454363, + "loss": 1.4771, + "theoretical_loss": 3.9465080730048614, + "tokens_seen": 496238592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865556670010031, + "loss": 1.4483, + "theoretical_loss": 3.9464522719930857, + "tokens_seen": 496304128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865396188565697, + "loss": 1.5381, + "theoretical_loss": 3.9463964804120844, + "tokens_seen": 496369664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865235707121365, + "loss": 1.5251, + "theoretical_loss": 3.9463406982590197, + "tokens_seen": 496435200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865075225677031, + "loss": 1.4486, + "theoretical_loss": 3.946284925531054, + "tokens_seen": 496500736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864914744232699, + "loss": 1.5538, + "theoretical_loss": 3.9462291622253503, + "tokens_seen": 496566272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864754262788365, + "loss": 1.3553, + "theoretical_loss": 3.946173408339075, + "tokens_seen": 496631808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864593781344033, + "loss": 1.4294, + "theoretical_loss": 3.9461176638693933, + "tokens_seen": 496697344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864433299899699, + "loss": 1.5686, + "theoretical_loss": 3.9460619288134726, + "tokens_seen": 496762880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864272818455367, + "loss": 1.4742, + "theoretical_loss": 3.9460062031684817, + "tokens_seen": 496828416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864112337011033, + "loss": 1.3954, + "theoretical_loss": 3.945950486931591, + "tokens_seen": 496893952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863951855566701, + "loss": 1.5157, + "theoretical_loss": 3.9458947800999713, + "tokens_seen": 496959488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863791374122367, + "loss": 1.4627, + "theoretical_loss": 3.9458390826707945, + "tokens_seen": 497025024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863630892678035, + "loss": 1.4251, + "theoretical_loss": 3.945783394641235, + "tokens_seen": 497090560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863470411233701, + "loss": 1.6089, + "theoretical_loss": 3.9457277160084665, + "tokens_seen": 497156096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863309929789369, + "loss": 1.5565, + "theoretical_loss": 3.9456720467696655, + "tokens_seen": 497221632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863149448345035, + "loss": 1.5502, + "theoretical_loss": 3.9456163869220093, + "tokens_seen": 497287168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862988966900703, + "loss": 1.4875, + "theoretical_loss": 3.945560736462677, + "tokens_seen": 497352704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862828485456369, + "loss": 1.4912, + "theoretical_loss": 3.945505095388847, + "tokens_seen": 497418240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862668004012037, + "loss": 1.4862, + "theoretical_loss": 3.9454494636977007, + "tokens_seen": 497483776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862507522567703, + "loss": 1.5397, + "theoretical_loss": 3.9453938413864207, + "tokens_seen": 497549312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862347041123371, + "loss": 1.5082, + "theoretical_loss": 3.94533822845219, + "tokens_seen": 497614848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862186559679037, + "loss": 1.5358, + "theoretical_loss": 3.9452826248921924, + "tokens_seen": 497680384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862026078234705, + "loss": 1.4845, + "theoretical_loss": 3.9452270307036144, + "tokens_seen": 497745920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861865596790371, + "loss": 1.4408, + "theoretical_loss": 3.945171445883643, + "tokens_seen": 497811456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861705115346039, + "loss": 1.475, + "theoretical_loss": 3.9451158704294658, + "tokens_seen": 497876992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861544633901705, + "loss": 1.4578, + "theoretical_loss": 3.945060304338273, + "tokens_seen": 497942528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861384152457373, + "loss": 1.4505, + "theoretical_loss": 3.9450047476072547, + "tokens_seen": 498008064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 171822, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6084593534469604, + "objective/train/theoretical_loss": 3.9449492002336024, + "objective/train/tokens_used": 518533600, + "theoretical_loss": 3.9449492002336024, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.000686122367101304, + "loss": 1.5196, + "theoretical_loss": 3.9449492002336024, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861063189568707, + "loss": 1.5035, + "theoretical_loss": 3.9448936622145103, + "tokens_seen": 498139136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860902708124373, + "loss": 1.49, + "theoretical_loss": 3.9448381335471714, + "tokens_seen": 498204672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860742226680041, + "loss": 1.5774, + "theoretical_loss": 3.9447826142287816, + "tokens_seen": 498270208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860581745235707, + "loss": 1.4464, + "theoretical_loss": 3.944727104256537, + "tokens_seen": 498335744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860421263791375, + "loss": 1.5822, + "theoretical_loss": 3.9446716036276364, + "tokens_seen": 498401280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860260782347042, + "loss": 1.4544, + "theoretical_loss": 3.944616112339279, + "tokens_seen": 498466816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860100300902708, + "loss": 1.4174, + "theoretical_loss": 3.944560630388664, + "tokens_seen": 498532352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006859939819458376, + "loss": 1.4449, + "theoretical_loss": 3.9445051577729937, + "tokens_seen": 498597888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006859779338014042, + "loss": 1.4734, + "theoretical_loss": 3.944449694489471, + "tokens_seen": 498663424 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685961885656971, + "loss": 1.5683, + "theoretical_loss": 3.9443942405352987, + "tokens_seen": 498728960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006859458375125376, + "loss": 1.4521, + "theoretical_loss": 3.9443387959076825, + "tokens_seen": 498794496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006859297893681044, + "loss": 1.4705, + "theoretical_loss": 3.944283360603829, + "tokens_seen": 498860032 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685913741223671, + "loss": 1.5388, + "theoretical_loss": 3.9442279346209457, + "tokens_seen": 498925568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858976930792378, + "loss": 1.5262, + "theoretical_loss": 3.944172517956241, + "tokens_seen": 498991104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858816449348044, + "loss": 1.5162, + "theoretical_loss": 3.944117110606925, + "tokens_seen": 499056640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858655967903712, + "loss": 1.4281, + "theoretical_loss": 3.9440617125702087, + "tokens_seen": 499122176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858495486459378, + "loss": 1.4013, + "theoretical_loss": 3.9440063238433045, + "tokens_seen": 499187712 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858335005015046, + "loss": 1.4952, + "theoretical_loss": 3.9439509444234258, + "tokens_seen": 499253248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858174523570712, + "loss": 1.4168, + "theoretical_loss": 3.943895574307788, + "tokens_seen": 499318784 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685801404212638, + "loss": 1.4904, + "theoretical_loss": 3.9438402134936057, + "tokens_seen": 499384320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857853560682046, + "loss": 1.4545, + "theoretical_loss": 3.9437848619780973, + "tokens_seen": 499449856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857693079237714, + "loss": 1.5395, + "theoretical_loss": 3.94372951975848, + "tokens_seen": 499515392 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685753259779338, + "loss": 1.5102, + "theoretical_loss": 3.9436741868319745, + "tokens_seen": 499580928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857372116349048, + "loss": 1.5157, + "theoretical_loss": 3.943618863195801, + "tokens_seen": 499646464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857211634904714, + "loss": 1.4785, + "theoretical_loss": 3.943563548847181, + "tokens_seen": 499712000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857051153460382, + "loss": 1.6383, + "theoretical_loss": 3.943508243783338, + "tokens_seen": 499777536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856890672016048, + "loss": 1.5405, + "theoretical_loss": 3.9434529480014966, + "tokens_seen": 499843072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856730190571716, + "loss": 1.4985, + "theoretical_loss": 3.9433976614988815, + "tokens_seen": 499908608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856569709127382, + "loss": 1.4688, + "theoretical_loss": 3.9433423842727198, + "tokens_seen": 499974144 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685640922768305, + "loss": 1.5811, + "theoretical_loss": 3.9432871163202394, + "tokens_seen": 500039680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856248746238716, + "loss": 1.4328, + "theoretical_loss": 3.9432318576386693, + "tokens_seen": 500105216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856088264794384, + "loss": 1.3487, + "theoretical_loss": 3.94317660822524, + "tokens_seen": 500170752 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685592778335005, + "loss": 1.5404, + "theoretical_loss": 3.9431213680771826, + "tokens_seen": 500236288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855767301905718, + "loss": 1.6125, + "theoretical_loss": 3.94306613719173, + "tokens_seen": 500301824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855606820461384, + "loss": 1.5333, + "theoretical_loss": 3.9430109155661164, + "tokens_seen": 500367360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855446339017052, + "loss": 1.5013, + "theoretical_loss": 3.9429557031975753, + "tokens_seen": 500432896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855285857572718, + "loss": 1.3822, + "theoretical_loss": 3.9429005000833444, + "tokens_seen": 500498432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855125376128386, + "loss": 1.3441, + "theoretical_loss": 3.9428453062206605, + "tokens_seen": 500563968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854964894684052, + "loss": 1.5172, + "theoretical_loss": 3.942790121606763, + "tokens_seen": 500629504 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685480441323972, + "loss": 1.522, + "theoretical_loss": 3.9427349462388905, + "tokens_seen": 500695040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854643931795386, + "loss": 1.5221, + "theoretical_loss": 3.942679780114284, + "tokens_seen": 500760576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854483450351054, + "loss": 1.5553, + "theoretical_loss": 3.9426246232301865, + "tokens_seen": 500826112 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685432296890672, + "loss": 1.4746, + "theoretical_loss": 3.9425694755838405, + "tokens_seen": 500891648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854162487462388, + "loss": 1.5228, + "theoretical_loss": 3.9425143371724913, + "tokens_seen": 500957184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854002006018054, + "loss": 1.5312, + "theoretical_loss": 3.942459207993384, + "tokens_seen": 501022720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853841524573722, + "loss": 1.4847, + "theoretical_loss": 3.9424040880437654, + "tokens_seen": 501088256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853681043129388, + "loss": 1.5158, + "theoretical_loss": 3.9423489773208837, + "tokens_seen": 501153792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853520561685056, + "loss": 1.5185, + "theoretical_loss": 3.9422938758219885, + "tokens_seen": 501219328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853360080240722, + "loss": 1.5378, + "theoretical_loss": 3.9422387835443296, + "tokens_seen": 501284864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 173143, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4856337308883667, + "objective/train/theoretical_loss": 3.9421837004851588, + "objective/train/tokens_used": 521810400, + "theoretical_loss": 3.9421837004851588, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685319959879639, + "loss": 1.4838, + "theoretical_loss": 3.9421837004851588, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853039117352056, + "loss": 1.4919, + "theoretical_loss": 3.942128626641729, + "tokens_seen": 501415936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852878635907724, + "loss": 1.5944, + "theoretical_loss": 3.942073562011294, + "tokens_seen": 501481472 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685271815446339, + "loss": 1.4147, + "theoretical_loss": 3.9420185065911086, + "tokens_seen": 501547008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852557673019058, + "loss": 1.5705, + "theoretical_loss": 3.94196346037843, + "tokens_seen": 501612544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852397191574724, + "loss": 1.4792, + "theoretical_loss": 3.941908423370515, + "tokens_seen": 501678080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852236710130392, + "loss": 1.4602, + "theoretical_loss": 3.941853395564622, + "tokens_seen": 501743616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852076228686058, + "loss": 1.4439, + "theoretical_loss": 3.9417983769580114, + "tokens_seen": 501809152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006851915747241726, + "loss": 1.5512, + "theoretical_loss": 3.941743367547944, + "tokens_seen": 501874688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006851755265797392, + "loss": 1.416, + "theoretical_loss": 3.9416883673316816, + "tokens_seen": 501940224 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685159478435306, + "loss": 1.5061, + "theoretical_loss": 3.9416333763064886, + "tokens_seen": 502005760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006851434302908726, + "loss": 1.4738, + "theoretical_loss": 3.941578394469628, + "tokens_seen": 502071296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006851273821464394, + "loss": 1.4258, + "theoretical_loss": 3.941523421818366, + "tokens_seen": 502136832 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685111334002006, + "loss": 1.5415, + "theoretical_loss": 3.94146845834997, + "tokens_seen": 502202368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850952858575728, + "loss": 1.4216, + "theoretical_loss": 3.9414135040617078, + "tokens_seen": 502267904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850792377131394, + "loss": 1.3867, + "theoretical_loss": 3.9413585589508484, + "tokens_seen": 502333440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850631895687062, + "loss": 1.4369, + "theoretical_loss": 3.9413036230146625, + "tokens_seen": 502398976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850471414242728, + "loss": 1.5165, + "theoretical_loss": 3.941248696250421, + "tokens_seen": 502464512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850310932798396, + "loss": 1.5472, + "theoretical_loss": 3.941193778655397, + "tokens_seen": 502530048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850150451354062, + "loss": 1.5286, + "theoretical_loss": 3.9411388702268644, + "tokens_seen": 502595584 + }, + { + "epoch": 0.15, + "learning_rate": 0.000684998996990973, + "loss": 1.6273, + "theoretical_loss": 3.941083970962098, + "tokens_seen": 502661120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849829488465396, + "loss": 1.4459, + "theoretical_loss": 3.9410290808583746, + "tokens_seen": 502726656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849669007021064, + "loss": 1.4898, + "theoretical_loss": 3.9409741999129704, + "tokens_seen": 502792192 + }, + { + "epoch": 0.15, + "learning_rate": 0.000684950852557673, + "loss": 1.4672, + "theoretical_loss": 3.9409193281231647, + "tokens_seen": 502857728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849348044132398, + "loss": 1.4665, + "theoretical_loss": 3.9408644654862375, + "tokens_seen": 502923264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849187562688064, + "loss": 1.6371, + "theoretical_loss": 3.9408096119994687, + "tokens_seen": 502988800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849027081243732, + "loss": 1.5104, + "theoretical_loss": 3.940754767660141, + "tokens_seen": 503054336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848866599799398, + "loss": 1.4865, + "theoretical_loss": 3.940699932465537, + "tokens_seen": 503119872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848706118355066, + "loss": 1.4007, + "theoretical_loss": 3.9406451064129415, + "tokens_seen": 503185408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848545636910732, + "loss": 1.5524, + "theoretical_loss": 3.94059028949964, + "tokens_seen": 503250944 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068483851554664, + "loss": 1.541, + "theoretical_loss": 3.940535481722919, + "tokens_seen": 503316480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848224674022066, + "loss": 1.4049, + "theoretical_loss": 3.9404806830800667, + "tokens_seen": 503382016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848064192577734, + "loss": 1.4502, + "theoretical_loss": 3.940425893568371, + "tokens_seen": 503447552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068479037111334, + "loss": 1.4578, + "theoretical_loss": 3.9403711131851233, + "tokens_seen": 503513088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847743229689068, + "loss": 1.4604, + "theoretical_loss": 3.940316341927614, + "tokens_seen": 503578624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847582748244734, + "loss": 1.507, + "theoretical_loss": 3.9402615797931357, + "tokens_seen": 503644160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847422266800402, + "loss": 1.5525, + "theoretical_loss": 3.940206826778982, + "tokens_seen": 503709696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847261785356068, + "loss": 1.4689, + "theoretical_loss": 3.9401520828824483, + "tokens_seen": 503775232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847101303911736, + "loss": 1.6258, + "theoretical_loss": 3.9400973481008297, + "tokens_seen": 503840768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006846940822467402, + "loss": 1.5935, + "theoretical_loss": 3.9400426224314233, + "tokens_seen": 503906304 + }, + { + "epoch": 0.15, + "learning_rate": 0.000684678034102307, + "loss": 1.4517, + "theoretical_loss": 3.9399879058715275, + "tokens_seen": 503971840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006846619859578737, + "loss": 1.4911, + "theoretical_loss": 3.9399331984184416, + "tokens_seen": 504037376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006846459378134404, + "loss": 1.5333, + "theoretical_loss": 3.939878500069466, + "tokens_seen": 504102912 + }, + { + "epoch": 0.15, + "learning_rate": 0.000684629889669007, + "loss": 1.5188, + "theoretical_loss": 3.939823810821903, + "tokens_seen": 504168448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006846138415245737, + "loss": 1.5142, + "theoretical_loss": 3.9397691306730547, + "tokens_seen": 504233984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845977933801405, + "loss": 1.5587, + "theoretical_loss": 3.9397144596202254, + "tokens_seen": 504299520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845817452357071, + "loss": 1.4182, + "theoretical_loss": 3.93965979766072, + "tokens_seen": 504365056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845656970912739, + "loss": 1.393, + "theoretical_loss": 3.9396051447918445, + "tokens_seen": 504430592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845496489468405, + "loss": 1.4089, + "theoretical_loss": 3.939550501010907, + "tokens_seen": 504496128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845336008024073, + "loss": 1.4498, + "theoretical_loss": 3.939495866315216, + "tokens_seen": 504561664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 174029, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3834893703460693, + "objective/train/theoretical_loss": 3.93944124070208, + "objective/train/tokens_used": 525087200, + "theoretical_loss": 3.93944124070208, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845175526579739, + "loss": 1.4513, + "theoretical_loss": 3.93944124070208, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845015045135407, + "loss": 1.5259, + "theoretical_loss": 3.9393866241688116, + "tokens_seen": 504692736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844854563691073, + "loss": 1.4621, + "theoretical_loss": 3.9393320167127213, + "tokens_seen": 504758272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844694082246741, + "loss": 1.4741, + "theoretical_loss": 3.939277418331123, + "tokens_seen": 504823808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844533600802407, + "loss": 1.5203, + "theoretical_loss": 3.9392228290213316, + "tokens_seen": 504889344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844373119358075, + "loss": 1.5894, + "theoretical_loss": 3.939168248780661, + "tokens_seen": 504954880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844212637913741, + "loss": 1.4675, + "theoretical_loss": 3.9391136776064286, + "tokens_seen": 505020416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844052156469409, + "loss": 1.5257, + "theoretical_loss": 3.939059115495952, + "tokens_seen": 505085952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843891675025075, + "loss": 1.4885, + "theoretical_loss": 3.93900456244655, + "tokens_seen": 505151488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843731193580743, + "loss": 1.4047, + "theoretical_loss": 3.938950018455543, + "tokens_seen": 505217024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843570712136409, + "loss": 1.4484, + "theoretical_loss": 3.938895483520252, + "tokens_seen": 505282560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843410230692077, + "loss": 1.4234, + "theoretical_loss": 3.938840957637998, + "tokens_seen": 505348096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843249749247743, + "loss": 1.5042, + "theoretical_loss": 3.9387864408061066, + "tokens_seen": 505413632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843089267803411, + "loss": 1.3835, + "theoretical_loss": 3.9387319330219004, + "tokens_seen": 505479168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842928786359077, + "loss": 1.4732, + "theoretical_loss": 3.938677434282706, + "tokens_seen": 505544704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842768304914745, + "loss": 1.5227, + "theoretical_loss": 3.93862294458585, + "tokens_seen": 505610240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842607823470411, + "loss": 1.5028, + "theoretical_loss": 3.938568463928661, + "tokens_seen": 505675776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842447342026079, + "loss": 1.5603, + "theoretical_loss": 3.9385139923084678, + "tokens_seen": 505741312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842286860581745, + "loss": 1.5817, + "theoretical_loss": 3.9384595297225995, + "tokens_seen": 505806848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842126379137413, + "loss": 1.4385, + "theoretical_loss": 3.9384050761683884, + "tokens_seen": 505872384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841965897693079, + "loss": 1.4871, + "theoretical_loss": 3.938350631643168, + "tokens_seen": 505937920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841805416248747, + "loss": 1.5354, + "theoretical_loss": 3.9382961961442695, + "tokens_seen": 506003456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841644934804413, + "loss": 1.593, + "theoretical_loss": 3.93824176966903, + "tokens_seen": 506068992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841484453360081, + "loss": 1.6305, + "theoretical_loss": 3.9381873522147837, + "tokens_seen": 506134528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841323971915747, + "loss": 1.5211, + "theoretical_loss": 3.938132943778869, + "tokens_seen": 506200064 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841163490471415, + "loss": 1.5019, + "theoretical_loss": 3.938078544358623, + "tokens_seen": 506265600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841003009027081, + "loss": 1.4484, + "theoretical_loss": 3.9380241539513854, + "tokens_seen": 506331136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840842527582749, + "loss": 1.5122, + "theoretical_loss": 3.9379697725544967, + "tokens_seen": 506396672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840682046138415, + "loss": 1.5404, + "theoretical_loss": 3.9379154001652985, + "tokens_seen": 506462208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840521564694083, + "loss": 1.4067, + "theoretical_loss": 3.9378610367811326, + "tokens_seen": 506527744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840361083249749, + "loss": 1.5415, + "theoretical_loss": 3.9378066823993443, + "tokens_seen": 506593280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840200601805417, + "loss": 1.5637, + "theoretical_loss": 3.937752337017278, + "tokens_seen": 506658816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840040120361083, + "loss": 1.4408, + "theoretical_loss": 3.9376980006322784, + "tokens_seen": 506724352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839879638916751, + "loss": 1.4735, + "theoretical_loss": 3.937643673241695, + "tokens_seen": 506789888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839719157472417, + "loss": 1.3714, + "theoretical_loss": 3.9375893548428738, + "tokens_seen": 506855424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839558676028085, + "loss": 1.4486, + "theoretical_loss": 3.9375350454331657, + "tokens_seen": 506920960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839398194583751, + "loss": 1.4675, + "theoretical_loss": 3.937480745009921, + "tokens_seen": 506986496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839237713139419, + "loss": 1.5516, + "theoretical_loss": 3.937426453570491, + "tokens_seen": 507052032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839077231695085, + "loss": 1.3864, + "theoretical_loss": 3.9373721711122287, + "tokens_seen": 507117568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838916750250753, + "loss": 1.4253, + "theoretical_loss": 3.937317897632488, + "tokens_seen": 507183104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838756268806419, + "loss": 1.4409, + "theoretical_loss": 3.9372636331286244, + "tokens_seen": 507248640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838595787362087, + "loss": 1.5317, + "theoretical_loss": 3.937209377597993, + "tokens_seen": 507314176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838435305917753, + "loss": 1.4448, + "theoretical_loss": 3.9371551310379527, + "tokens_seen": 507379712 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838274824473421, + "loss": 1.5911, + "theoretical_loss": 3.93710089344586, + "tokens_seen": 507445248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838114343029087, + "loss": 1.4736, + "theoretical_loss": 3.937046664819076, + "tokens_seen": 507510784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837953861584755, + "loss": 1.4326, + "theoretical_loss": 3.93699244515496, + "tokens_seen": 507576320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837793380140421, + "loss": 1.3809, + "theoretical_loss": 3.936938234450875, + "tokens_seen": 507641856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837632898696089, + "loss": 1.4961, + "theoretical_loss": 3.936884032704183, + "tokens_seen": 507707392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837472417251755, + "loss": 1.4297, + "theoretical_loss": 3.9368298399122486, + "tokens_seen": 507772928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837311935807423, + "loss": 1.5158, + "theoretical_loss": 3.9367756560724363, + "tokens_seen": 507838464 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 175381, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2951419353485107, + "objective/train/theoretical_loss": 3.936721481182113, + "objective/train/tokens_used": 528364000, + "theoretical_loss": 3.936721481182113, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837151454363089, + "loss": 1.4057, + "theoretical_loss": 3.936721481182113, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836990972918757, + "loss": 1.4186, + "theoretical_loss": 3.9366673152386458, + "tokens_seen": 507969536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836830491474423, + "loss": 1.4403, + "theoretical_loss": 3.9366131582394024, + "tokens_seen": 508035072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836670010030091, + "loss": 1.5401, + "theoretical_loss": 3.936559010181753, + "tokens_seen": 508100608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836509528585757, + "loss": 1.4831, + "theoretical_loss": 3.936504871063069, + "tokens_seen": 508166144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836349047141425, + "loss": 1.4929, + "theoretical_loss": 3.9364507408807206, + "tokens_seen": 508231680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836188565697091, + "loss": 1.4945, + "theoretical_loss": 3.936396619632082, + "tokens_seen": 508297216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836028084252759, + "loss": 1.4826, + "theoretical_loss": 3.936342507314527, + "tokens_seen": 508362752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835867602808425, + "loss": 1.459, + "theoretical_loss": 3.9362884039254302, + "tokens_seen": 508428288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835707121364093, + "loss": 1.3537, + "theoretical_loss": 3.9362343094621677, + "tokens_seen": 508493824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835546639919759, + "loss": 1.4776, + "theoretical_loss": 3.936180223922118, + "tokens_seen": 508559360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835386158475427, + "loss": 1.5716, + "theoretical_loss": 3.936126147302658, + "tokens_seen": 508624896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835225677031093, + "loss": 1.5438, + "theoretical_loss": 3.9360720796011686, + "tokens_seen": 508690432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835065195586761, + "loss": 1.5372, + "theoretical_loss": 3.93601802081503, + "tokens_seen": 508755968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834904714142427, + "loss": 1.4193, + "theoretical_loss": 3.9359639709416236, + "tokens_seen": 508821504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834744232698095, + "loss": 1.5186, + "theoretical_loss": 3.9359099299783327, + "tokens_seen": 508887040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834583751253763, + "loss": 1.4852, + "theoretical_loss": 3.935855897922541, + "tokens_seen": 508952576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834423269809429, + "loss": 1.4447, + "theoretical_loss": 3.935801874771634, + "tokens_seen": 509018112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834262788365097, + "loss": 1.4547, + "theoretical_loss": 3.935747860522997, + "tokens_seen": 509083648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834102306920763, + "loss": 1.4285, + "theoretical_loss": 3.9356938551740184, + "tokens_seen": 509149184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833941825476431, + "loss": 1.5165, + "theoretical_loss": 3.9356398587220864, + "tokens_seen": 509214720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833781344032097, + "loss": 1.5487, + "theoretical_loss": 3.9355858711645895, + "tokens_seen": 509280256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833620862587765, + "loss": 1.5374, + "theoretical_loss": 3.9355318924989193, + "tokens_seen": 509345792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833460381143431, + "loss": 1.3921, + "theoretical_loss": 3.9354779227224674, + "tokens_seen": 509411328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833299899699097, + "loss": 1.4561, + "theoretical_loss": 3.935423961832626, + "tokens_seen": 509476864 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833139418254765, + "loss": 1.5147, + "theoretical_loss": 3.93537000982679, + "tokens_seen": 509542400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832978936810431, + "loss": 1.4813, + "theoretical_loss": 3.9353160667023532, + "tokens_seen": 509607936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832818455366099, + "loss": 1.4487, + "theoretical_loss": 3.935262132456713, + "tokens_seen": 509673472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832657973921766, + "loss": 1.4581, + "theoretical_loss": 3.9352082070872654, + "tokens_seen": 509739008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832497492477433, + "loss": 1.5631, + "theoretical_loss": 3.9351542905914094, + "tokens_seen": 509804544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068323370110331, + "loss": 1.4613, + "theoretical_loss": 3.935100382966544, + "tokens_seen": 509870080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832176529588766, + "loss": 1.4773, + "theoretical_loss": 3.93504648421007, + "tokens_seen": 509935616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832016048144434, + "loss": 1.49, + "theoretical_loss": 3.9349925943193886, + "tokens_seen": 510001152 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068318555667001, + "loss": 1.457, + "theoretical_loss": 3.934938713291903, + "tokens_seen": 510066688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831695085255768, + "loss": 1.4519, + "theoretical_loss": 3.934884841125017, + "tokens_seen": 510132224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831534603811434, + "loss": 1.3485, + "theoretical_loss": 3.9348309778161346, + "tokens_seen": 510197760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831374122367102, + "loss": 1.4214, + "theoretical_loss": 3.9347771233626627, + "tokens_seen": 510263296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831213640922768, + "loss": 1.365, + "theoretical_loss": 3.934723277762008, + "tokens_seen": 510328832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831053159478436, + "loss": 1.4693, + "theoretical_loss": 3.934669441011579, + "tokens_seen": 510394368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006830892678034102, + "loss": 1.4993, + "theoretical_loss": 3.934615613108784, + "tokens_seen": 510459904 + }, + { + "epoch": 0.15, + "learning_rate": 0.000683073219658977, + "loss": 1.4738, + "theoretical_loss": 3.9345617940510342, + "tokens_seen": 510525440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006830571715145436, + "loss": 1.3746, + "theoretical_loss": 3.934507983835741, + "tokens_seen": 510590976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006830411233701104, + "loss": 1.4135, + "theoretical_loss": 3.9344541824603163, + "tokens_seen": 510656512 + }, + { + "epoch": 0.15, + "learning_rate": 0.000683025075225677, + "loss": 1.5538, + "theoretical_loss": 3.934400389922174, + "tokens_seen": 510722048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006830090270812438, + "loss": 1.5079, + "theoretical_loss": 3.934346606218729, + "tokens_seen": 510787584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829929789368104, + "loss": 1.5737, + "theoretical_loss": 3.9342928313473964, + "tokens_seen": 510853120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829769307923772, + "loss": 1.4594, + "theoretical_loss": 3.9342390653055945, + "tokens_seen": 510918656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829608826479438, + "loss": 1.4723, + "theoretical_loss": 3.93418530809074, + "tokens_seen": 510984192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829448345035106, + "loss": 1.4413, + "theoretical_loss": 3.9341315597002517, + "tokens_seen": 511049728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829287863590772, + "loss": 1.4883, + "theoretical_loss": 3.934077820131551, + "tokens_seen": 511115264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 176389, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4067481756210327, + "objective/train/theoretical_loss": 3.934024089382058, + "objective/train/tokens_used": 531640800, + "theoretical_loss": 3.934024089382058, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.000682912738214644, + "loss": 1.4437, + "theoretical_loss": 3.934024089382058, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006828966900702106, + "loss": 1.4399, + "theoretical_loss": 3.933970367449195, + "tokens_seen": 511246336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006828806419257774, + "loss": 1.5355, + "theoretical_loss": 3.9339166543303863, + "tokens_seen": 511311872 + }, + { + "epoch": 0.15, + "learning_rate": 0.000682864593781344, + "loss": 1.4865, + "theoretical_loss": 3.9338629500230553, + "tokens_seen": 511377408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006828485456369108, + "loss": 1.4984, + "theoretical_loss": 3.9338092545246286, + "tokens_seen": 511442944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006828324974924774, + "loss": 1.4916, + "theoretical_loss": 3.933755567832532, + "tokens_seen": 511508480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006828164493480442, + "loss": 1.5456, + "theoretical_loss": 3.933701889944194, + "tokens_seen": 511574016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006828004012036108, + "loss": 1.5706, + "theoretical_loss": 3.933648220857042, + "tokens_seen": 511639552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006827843530591776, + "loss": 1.4834, + "theoretical_loss": 3.9335945605685074, + "tokens_seen": 511705088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006827683049147442, + "loss": 1.4373, + "theoretical_loss": 3.9335409090760196, + "tokens_seen": 511770624 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682752256770311, + "loss": 1.443, + "theoretical_loss": 3.933487266377012, + "tokens_seen": 511836160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006827362086258776, + "loss": 1.4371, + "theoretical_loss": 3.933433632468917, + "tokens_seen": 511901696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006827201604814444, + "loss": 1.5022, + "theoretical_loss": 3.9333800073491694, + "tokens_seen": 511967232 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682704112337011, + "loss": 1.4562, + "theoretical_loss": 3.9333263910152034, + "tokens_seen": 512032768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826880641925778, + "loss": 1.441, + "theoretical_loss": 3.9332727834644565, + "tokens_seen": 512098304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826720160481444, + "loss": 1.4132, + "theoretical_loss": 3.933219184694365, + "tokens_seen": 512163840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826559679037112, + "loss": 1.5127, + "theoretical_loss": 3.933165594702368, + "tokens_seen": 512229376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826399197592778, + "loss": 1.4125, + "theoretical_loss": 3.9331120134859052, + "tokens_seen": 512294912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826238716148446, + "loss": 1.4094, + "theoretical_loss": 3.9330584410424168, + "tokens_seen": 512360448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826078234704112, + "loss": 1.5539, + "theoretical_loss": 3.9330048773693447, + "tokens_seen": 512425984 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682591775325978, + "loss": 1.5014, + "theoretical_loss": 3.932951322464132, + "tokens_seen": 512491520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006825757271815446, + "loss": 1.4989, + "theoretical_loss": 3.9328977763242223, + "tokens_seen": 512557056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006825596790371114, + "loss": 1.5426, + "theoretical_loss": 3.93284423894706, + "tokens_seen": 512622592 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682543630892678, + "loss": 1.4491, + "theoretical_loss": 3.932790710330092, + "tokens_seen": 512688128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006825275827482448, + "loss": 1.4457, + "theoretical_loss": 3.932737190470765, + "tokens_seen": 512753664 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006825115346038114, + "loss": 1.4772, + "theoretical_loss": 3.9326836793665265, + "tokens_seen": 512819200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824954864593782, + "loss": 1.6298, + "theoretical_loss": 3.9326301770148264, + "tokens_seen": 512884736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824794383149448, + "loss": 1.5865, + "theoretical_loss": 3.9325766834131155, + "tokens_seen": 512950272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824633901705116, + "loss": 1.4053, + "theoretical_loss": 3.932523198558844, + "tokens_seen": 513015808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824473420260782, + "loss": 1.3849, + "theoretical_loss": 3.932469722449464, + "tokens_seen": 513081344 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682431293881645, + "loss": 1.3922, + "theoretical_loss": 3.932416255082431, + "tokens_seen": 513146880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824152457372116, + "loss": 1.4653, + "theoretical_loss": 3.932362796455198, + "tokens_seen": 513212416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823991975927784, + "loss": 1.3947, + "theoretical_loss": 3.932309346565221, + "tokens_seen": 513277952 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682383149448345, + "loss": 1.4863, + "theoretical_loss": 3.9322559054099564, + "tokens_seen": 513343488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823671013039118, + "loss": 1.5967, + "theoretical_loss": 3.932202472986862, + "tokens_seen": 513409024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823510531594784, + "loss": 1.4689, + "theoretical_loss": 3.932149049293397, + "tokens_seen": 513474560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823350050150452, + "loss": 1.3834, + "theoretical_loss": 3.9320956343270206, + "tokens_seen": 513540096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823189568706118, + "loss": 1.4431, + "theoretical_loss": 3.9320422280851943, + "tokens_seen": 513605632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823029087261786, + "loss": 1.4312, + "theoretical_loss": 3.9319888305653805, + "tokens_seen": 513671168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822868605817452, + "loss": 1.4436, + "theoretical_loss": 3.9319354417650416, + "tokens_seen": 513736704 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682270812437312, + "loss": 1.4754, + "theoretical_loss": 3.931882061681641, + "tokens_seen": 513802240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822547642928787, + "loss": 1.4687, + "theoretical_loss": 3.9318286903126456, + "tokens_seen": 513867776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822387161484454, + "loss": 1.536, + "theoretical_loss": 3.93177532765552, + "tokens_seen": 513933312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822226680040121, + "loss": 1.4208, + "theoretical_loss": 3.9317219737077327, + "tokens_seen": 513998848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822066198595788, + "loss": 1.5384, + "theoretical_loss": 3.9316686284667517, + "tokens_seen": 514064384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006821905717151455, + "loss": 1.5302, + "theoretical_loss": 3.9316152919300458, + "tokens_seen": 514129920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006821745235707122, + "loss": 1.5152, + "theoretical_loss": 3.9315619640950863, + "tokens_seen": 514195456 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682158475426279, + "loss": 1.4262, + "theoretical_loss": 3.9315086449593446, + "tokens_seen": 514260992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006821424272818456, + "loss": 1.5165, + "theoretical_loss": 3.931455334520293, + "tokens_seen": 514326528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006821263791374124, + "loss": 1.5204, + "theoretical_loss": 3.931402032775405, + "tokens_seen": 514392064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 177587, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4545185565948486, + "objective/train/theoretical_loss": 3.9313487397221554, + "objective/train/tokens_used": 534917600, + "theoretical_loss": 3.9313487397221554, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682110330992979, + "loss": 1.442, + "theoretical_loss": 3.9313487397221554, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820942828485458, + "loss": 1.4798, + "theoretical_loss": 3.9312954553580206, + "tokens_seen": 514523136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820782347041124, + "loss": 1.5157, + "theoretical_loss": 3.931242179680477, + "tokens_seen": 514588672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820621865596792, + "loss": 1.4057, + "theoretical_loss": 3.931188912687002, + "tokens_seen": 514654208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820461384152458, + "loss": 1.4713, + "theoretical_loss": 3.931135654375075, + "tokens_seen": 514719744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820300902708126, + "loss": 1.4986, + "theoretical_loss": 3.9310824047421757, + "tokens_seen": 514785280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820140421263792, + "loss": 1.5335, + "theoretical_loss": 3.9310291637857855, + "tokens_seen": 514850816 + }, + { + "epoch": 0.16, + "learning_rate": 0.000681997993981946, + "loss": 1.4282, + "theoretical_loss": 3.930975931503386, + "tokens_seen": 514916352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006819819458375126, + "loss": 1.5608, + "theoretical_loss": 3.930922707892461, + "tokens_seen": 514981888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006819658976930794, + "loss": 1.3664, + "theoretical_loss": 3.930869492950494, + "tokens_seen": 515047424 + }, + { + "epoch": 0.16, + "learning_rate": 0.000681949849548646, + "loss": 1.4341, + "theoretical_loss": 3.9308162866749705, + "tokens_seen": 515112960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006819338014042126, + "loss": 1.3829, + "theoretical_loss": 3.930763089063377, + "tokens_seen": 515178496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006819177532597794, + "loss": 1.517, + "theoretical_loss": 3.9307099001132, + "tokens_seen": 515244032 + }, + { + "epoch": 0.16, + "learning_rate": 0.000681901705115346, + "loss": 1.5803, + "theoretical_loss": 3.930656719821929, + "tokens_seen": 515309568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818856569709128, + "loss": 1.4556, + "theoretical_loss": 3.9306035481870527, + "tokens_seen": 515375104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818696088264795, + "loss": 1.5231, + "theoretical_loss": 3.9305503852060615, + "tokens_seen": 515440640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818535606820462, + "loss": 1.5268, + "theoretical_loss": 3.9304972308764476, + "tokens_seen": 515506176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818375125376129, + "loss": 1.4497, + "theoretical_loss": 3.9304440851957025, + "tokens_seen": 515571712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818214643931796, + "loss": 1.5811, + "theoretical_loss": 3.930390948161321, + "tokens_seen": 515637248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818054162487463, + "loss": 1.5048, + "theoretical_loss": 3.9303378197707968, + "tokens_seen": 515702784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817893681043129, + "loss": 1.4945, + "theoretical_loss": 3.9302847000216263, + "tokens_seen": 515768320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817733199598797, + "loss": 1.5569, + "theoretical_loss": 3.9302315889113055, + "tokens_seen": 515833856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817572718154463, + "loss": 1.4553, + "theoretical_loss": 3.930178486437333, + "tokens_seen": 515899392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817412236710131, + "loss": 1.3959, + "theoretical_loss": 3.930125392597207, + "tokens_seen": 515964928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817251755265797, + "loss": 1.3709, + "theoretical_loss": 3.9300723073884276, + "tokens_seen": 516030464 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817091273821465, + "loss": 1.4881, + "theoretical_loss": 3.9300192308084956, + "tokens_seen": 516096000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816930792377131, + "loss": 1.447, + "theoretical_loss": 3.929966162854913, + "tokens_seen": 516161536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816770310932799, + "loss": 1.5391, + "theoretical_loss": 3.929913103525183, + "tokens_seen": 516227072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816609829488465, + "loss": 1.4131, + "theoretical_loss": 3.929860052816809, + "tokens_seen": 516292608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816449348044133, + "loss": 1.5676, + "theoretical_loss": 3.9298070107272967, + "tokens_seen": 516358144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816288866599799, + "loss": 1.5717, + "theoretical_loss": 3.929753977254152, + "tokens_seen": 516423680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816128385155467, + "loss": 1.5477, + "theoretical_loss": 3.929700952394882, + "tokens_seen": 516489216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815967903711133, + "loss": 1.616, + "theoretical_loss": 3.9296479361469947, + "tokens_seen": 516554752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815807422266801, + "loss": 1.5208, + "theoretical_loss": 3.929594928508, + "tokens_seen": 516620288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815646940822467, + "loss": 1.5779, + "theoretical_loss": 3.9295419294754077, + "tokens_seen": 516685824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815486459378135, + "loss": 1.5485, + "theoretical_loss": 3.9294889390467285, + "tokens_seen": 516751360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815325977933801, + "loss": 1.4398, + "theoretical_loss": 3.929435957219476, + "tokens_seen": 516816896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815165496489469, + "loss": 1.4488, + "theoretical_loss": 3.9293829839911623, + "tokens_seen": 516882432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815005015045135, + "loss": 1.4929, + "theoretical_loss": 3.9293300193593024, + "tokens_seen": 516947968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814844533600803, + "loss": 1.5023, + "theoretical_loss": 3.929277063321412, + "tokens_seen": 517013504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814684052156469, + "loss": 1.5235, + "theoretical_loss": 3.929224115875007, + "tokens_seen": 517079040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814523570712137, + "loss": 1.4841, + "theoretical_loss": 3.929171177017605, + "tokens_seen": 517144576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814363089267803, + "loss": 1.5463, + "theoretical_loss": 3.9291182467467247, + "tokens_seen": 517210112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814202607823471, + "loss": 1.4908, + "theoretical_loss": 3.9290653250598857, + "tokens_seen": 517275648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814042126379137, + "loss": 1.4972, + "theoretical_loss": 3.9290124119546084, + "tokens_seen": 517341184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813881644934805, + "loss": 1.4464, + "theoretical_loss": 3.9289595074284143, + "tokens_seen": 517406720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813721163490471, + "loss": 1.5389, + "theoretical_loss": 3.9289066114788267, + "tokens_seen": 517472256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813560682046139, + "loss": 1.4443, + "theoretical_loss": 3.9288537241033685, + "tokens_seen": 517537792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813400200601805, + "loss": 1.404, + "theoretical_loss": 3.9288008452995644, + "tokens_seen": 517603328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813239719157473, + "loss": 1.5549, + "theoretical_loss": 3.9287479750649412, + "tokens_seen": 517668864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 178536, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4981567859649658, + "objective/train/theoretical_loss": 3.9286951133970245, + "objective/train/tokens_used": 538194400, + "theoretical_loss": 3.9286951133970245, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813079237713139, + "loss": 1.4896, + "theoretical_loss": 3.9286951133970245, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812918756268807, + "loss": 1.3853, + "theoretical_loss": 3.9286422602933424, + "tokens_seen": 517799936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812758274824473, + "loss": 1.4381, + "theoretical_loss": 3.928589415751424, + "tokens_seen": 517865472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812597793380141, + "loss": 1.4726, + "theoretical_loss": 3.9285365797687986, + "tokens_seen": 517931008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812437311935807, + "loss": 1.5329, + "theoretical_loss": 3.9284837523429976, + "tokens_seen": 517996544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812276830491475, + "loss": 1.3644, + "theoretical_loss": 3.9284309334715526, + "tokens_seen": 518062080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812116349047141, + "loss": 1.516, + "theoretical_loss": 3.9283781231519965, + "tokens_seen": 518127616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811955867602809, + "loss": 1.5407, + "theoretical_loss": 3.9283253213818634, + "tokens_seen": 518193152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811795386158475, + "loss": 1.5017, + "theoretical_loss": 3.9282725281586885, + "tokens_seen": 518258688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811634904714143, + "loss": 1.5143, + "theoretical_loss": 3.928219743480007, + "tokens_seen": 518324224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811474423269809, + "loss": 1.5884, + "theoretical_loss": 3.9281669673433566, + "tokens_seen": 518389760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811313941825477, + "loss": 1.552, + "theoretical_loss": 3.9281141997462754, + "tokens_seen": 518455296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811153460381143, + "loss": 1.5766, + "theoretical_loss": 3.9280614406863013, + "tokens_seen": 518520832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810992978936811, + "loss": 1.474, + "theoretical_loss": 3.9280086901609756, + "tokens_seen": 518586368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810832497492477, + "loss": 1.4442, + "theoretical_loss": 3.927955948167839, + "tokens_seen": 518651904 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810672016048145, + "loss": 1.4731, + "theoretical_loss": 3.9279032147044335, + "tokens_seen": 518717440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810511534603812, + "loss": 1.4481, + "theoretical_loss": 3.9278504897683018, + "tokens_seen": 518782976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810351053159479, + "loss": 1.5885, + "theoretical_loss": 3.927797773356989, + "tokens_seen": 518848512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810190571715146, + "loss": 1.3918, + "theoretical_loss": 3.9277450654680397, + "tokens_seen": 518914048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810030090270813, + "loss": 1.4471, + "theoretical_loss": 3.9276923660990004, + "tokens_seen": 518979584 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680986960882648, + "loss": 1.5669, + "theoretical_loss": 3.9276396752474176, + "tokens_seen": 519045120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809709127382147, + "loss": 1.5003, + "theoretical_loss": 3.9275869929108405, + "tokens_seen": 519110656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809548645937814, + "loss": 1.4697, + "theoretical_loss": 3.927534319086817, + "tokens_seen": 519176192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809388164493481, + "loss": 1.438, + "theoretical_loss": 3.9274816537728983, + "tokens_seen": 519241728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809227683049148, + "loss": 1.428, + "theoretical_loss": 3.9274289969666354, + "tokens_seen": 519307264 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809067201604815, + "loss": 1.5598, + "theoretical_loss": 3.9273763486655806, + "tokens_seen": 519372800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808906720160482, + "loss": 1.4526, + "theoretical_loss": 3.9273237088672874, + "tokens_seen": 519438336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808746238716149, + "loss": 1.4842, + "theoretical_loss": 3.92727107756931, + "tokens_seen": 519503872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808585757271816, + "loss": 1.4134, + "theoretical_loss": 3.927218454769203, + "tokens_seen": 519569408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808425275827483, + "loss": 1.4679, + "theoretical_loss": 3.9271658404645233, + "tokens_seen": 519634944 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680826479438315, + "loss": 1.5984, + "theoretical_loss": 3.9271132346528286, + "tokens_seen": 519700480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808104312938817, + "loss": 1.5113, + "theoretical_loss": 3.9270606373316763, + "tokens_seen": 519766016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807943831494484, + "loss": 1.5097, + "theoretical_loss": 3.9270080484986267, + "tokens_seen": 519831552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807783350050151, + "loss": 1.5268, + "theoretical_loss": 3.9269554681512395, + "tokens_seen": 519897088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807622868605819, + "loss": 1.4653, + "theoretical_loss": 3.926902896287076, + "tokens_seen": 519962624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807462387161485, + "loss": 1.4649, + "theoretical_loss": 3.9268503329036992, + "tokens_seen": 520028160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807301905717153, + "loss": 1.5718, + "theoretical_loss": 3.926797777998672, + "tokens_seen": 520093696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807141424272819, + "loss": 1.357, + "theoretical_loss": 3.9267452315695586, + "tokens_seen": 520159232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806980942828487, + "loss": 1.468, + "theoretical_loss": 3.9266926936139246, + "tokens_seen": 520224768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806820461384153, + "loss": 1.4507, + "theoretical_loss": 3.9266401641293367, + "tokens_seen": 520290304 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680665997993982, + "loss": 1.3782, + "theoretical_loss": 3.926587643113362, + "tokens_seen": 520355840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806499498495487, + "loss": 1.4731, + "theoretical_loss": 3.9265351305635687, + "tokens_seen": 520421376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806339017051155, + "loss": 1.4242, + "theoretical_loss": 3.9264826264775268, + "tokens_seen": 520486912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806178535606821, + "loss": 1.5955, + "theoretical_loss": 3.9264301308528067, + "tokens_seen": 520552448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806018054162489, + "loss": 1.4677, + "theoretical_loss": 3.926377643686979, + "tokens_seen": 520617984 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805857572718155, + "loss": 1.539, + "theoretical_loss": 3.926325164977616, + "tokens_seen": 520683520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805697091273823, + "loss": 1.4778, + "theoretical_loss": 3.9262726947222926, + "tokens_seen": 520749056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805536609829489, + "loss": 1.4355, + "theoretical_loss": 3.926220232918582, + "tokens_seen": 520814592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805376128385156, + "loss": 1.5164, + "theoretical_loss": 3.92616777956406, + "tokens_seen": 520880128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805215646940823, + "loss": 1.3718, + "theoretical_loss": 3.926115334656303, + "tokens_seen": 520945664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 179706, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5349757671356201, + "objective/train/theoretical_loss": 3.926062898192888, + "objective/train/tokens_used": 541471200, + "theoretical_loss": 3.926062898192888, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680505516549649, + "loss": 1.5216, + "theoretical_loss": 3.926062898192888, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804894684052157, + "loss": 1.4301, + "theoretical_loss": 3.926010470171394, + "tokens_seen": 521076736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804734202607824, + "loss": 1.5091, + "theoretical_loss": 3.9259580505894007, + "tokens_seen": 521142272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804573721163491, + "loss": 1.4445, + "theoretical_loss": 3.925905639444488, + "tokens_seen": 521207808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804413239719158, + "loss": 1.5386, + "theoretical_loss": 3.9258532367342367, + "tokens_seen": 521273344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804252758274825, + "loss": 1.4605, + "theoretical_loss": 3.9258008424562307, + "tokens_seen": 521338880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804092276830492, + "loss": 1.5114, + "theoretical_loss": 3.9257484566080523, + "tokens_seen": 521404416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803931795386158, + "loss": 1.5215, + "theoretical_loss": 3.9256960791872855, + "tokens_seen": 521469952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803771313941826, + "loss": 1.4821, + "theoretical_loss": 3.925643710191517, + "tokens_seen": 521535488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803610832497492, + "loss": 1.4289, + "theoretical_loss": 3.9255913496183332, + "tokens_seen": 521601024 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680345035105316, + "loss": 1.4774, + "theoretical_loss": 3.9255389974653196, + "tokens_seen": 521666560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803289869608826, + "loss": 1.496, + "theoretical_loss": 3.9254866537300668, + "tokens_seen": 521732096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803129388164494, + "loss": 1.5232, + "theoretical_loss": 3.925434318410163, + "tokens_seen": 521797632 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680296890672016, + "loss": 1.5735, + "theoretical_loss": 3.9253819915031993, + "tokens_seen": 521863168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802808425275828, + "loss": 1.4249, + "theoretical_loss": 3.9253296730067664, + "tokens_seen": 521928704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802647943831494, + "loss": 1.5218, + "theoretical_loss": 3.9252773629184565, + "tokens_seen": 521994240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802487462387162, + "loss": 1.394, + "theoretical_loss": 3.925225061235863, + "tokens_seen": 522059776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802326980942828, + "loss": 1.3936, + "theoretical_loss": 3.925172767956581, + "tokens_seen": 522125312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802166499498496, + "loss": 1.6192, + "theoretical_loss": 3.9251204830782056, + "tokens_seen": 522190848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802006018054162, + "loss": 1.3342, + "theoretical_loss": 3.925068206598332, + "tokens_seen": 522256384 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680184553660983, + "loss": 1.5168, + "theoretical_loss": 3.9250159385145595, + "tokens_seen": 522321920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006801685055165496, + "loss": 1.4312, + "theoretical_loss": 3.9249636788244846, + "tokens_seen": 522387456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006801524573721164, + "loss": 1.4457, + "theoretical_loss": 3.924911427525708, + "tokens_seen": 522452992 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680136409227683, + "loss": 1.4418, + "theoretical_loss": 3.9248591846158285, + "tokens_seen": 522518528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006801203610832498, + "loss": 1.4766, + "theoretical_loss": 3.9248069500924485, + "tokens_seen": 522584064 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006801043129388164, + "loss": 1.4706, + "theoretical_loss": 3.9247547239531695, + "tokens_seen": 522649600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800882647943832, + "loss": 1.3625, + "theoretical_loss": 3.924702506195595, + "tokens_seen": 522715136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800722166499498, + "loss": 1.6163, + "theoretical_loss": 3.9246502968173296, + "tokens_seen": 522780672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800561685055166, + "loss": 1.4221, + "theoretical_loss": 3.9245980958159787, + "tokens_seen": 522846208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800401203610832, + "loss": 1.5205, + "theoretical_loss": 3.924545903189147, + "tokens_seen": 522911744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00068002407221665, + "loss": 1.5592, + "theoretical_loss": 3.9244937189344435, + "tokens_seen": 522977280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800080240722166, + "loss": 1.4894, + "theoretical_loss": 3.9244415430494755, + "tokens_seen": 523042816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799919759277834, + "loss": 1.4424, + "theoretical_loss": 3.9243893755318515, + "tokens_seen": 523108352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00067997592778335, + "loss": 1.3891, + "theoretical_loss": 3.924337216379183, + "tokens_seen": 523173888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799598796389168, + "loss": 1.4588, + "theoretical_loss": 3.92428506558908, + "tokens_seen": 523239424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799438314944834, + "loss": 1.5871, + "theoretical_loss": 3.924232923159155, + "tokens_seen": 523304960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799277833500502, + "loss": 1.4269, + "theoretical_loss": 3.9241807890870213, + "tokens_seen": 523370496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799117352056168, + "loss": 1.5134, + "theoretical_loss": 3.9241286633702925, + "tokens_seen": 523436032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798956870611836, + "loss": 1.5179, + "theoretical_loss": 3.9240765460065834, + "tokens_seen": 523501568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798796389167503, + "loss": 1.515, + "theoretical_loss": 3.924024436993511, + "tokens_seen": 523567104 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679863590772317, + "loss": 1.481, + "theoretical_loss": 3.9239723363286916, + "tokens_seen": 523632640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798475426278837, + "loss": 1.4266, + "theoretical_loss": 3.923920244009743, + "tokens_seen": 523698176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798314944834504, + "loss": 1.4077, + "theoretical_loss": 3.9238681600342846, + "tokens_seen": 523763712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798154463390171, + "loss": 1.4432, + "theoretical_loss": 3.9238160843999363, + "tokens_seen": 523829248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797993981945838, + "loss": 1.4181, + "theoretical_loss": 3.923764017104318, + "tokens_seen": 523894784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797833500501505, + "loss": 1.4425, + "theoretical_loss": 3.9237119581450535, + "tokens_seen": 523960320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797673019057172, + "loss": 1.5611, + "theoretical_loss": 3.923659907519764, + "tokens_seen": 524025856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797512537612839, + "loss": 1.3971, + "theoretical_loss": 3.9236078652260735, + "tokens_seen": 524091392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797352056168506, + "loss": 1.5388, + "theoretical_loss": 3.9235558312616075, + "tokens_seen": 524156928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797191574724173, + "loss": 1.4835, + "theoretical_loss": 3.9235038056239917, + "tokens_seen": 524222464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 180943, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5657446384429932, + "objective/train/theoretical_loss": 3.9234517883108517, + "objective/train/tokens_used": 544748000, + "theoretical_loss": 3.9234517883108517, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679703109327984, + "loss": 1.5641, + "theoretical_loss": 3.9234517883108517, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796870611835507, + "loss": 1.4348, + "theoretical_loss": 3.9233997793198165, + "tokens_seen": 524353536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796710130391174, + "loss": 1.5001, + "theoretical_loss": 3.9233477786485143, + "tokens_seen": 524419072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796549648946841, + "loss": 1.5435, + "theoretical_loss": 3.923295786294575, + "tokens_seen": 524484608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796389167502508, + "loss": 1.4204, + "theoretical_loss": 3.9232438022556293, + "tokens_seen": 524550144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796228686058175, + "loss": 1.4743, + "theoretical_loss": 3.923191826529308, + "tokens_seen": 524615680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796068204613842, + "loss": 1.471, + "theoretical_loss": 3.9231398591132445, + "tokens_seen": 524681216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795907723169509, + "loss": 1.4699, + "theoretical_loss": 3.9230879000050716, + "tokens_seen": 524746752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795747241725176, + "loss": 1.526, + "theoretical_loss": 3.9230359492024247, + "tokens_seen": 524812288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795586760280843, + "loss": 1.5139, + "theoretical_loss": 3.9229840067029382, + "tokens_seen": 524877824 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679542627883651, + "loss": 1.4791, + "theoretical_loss": 3.92293207250425, + "tokens_seen": 524943360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795265797392177, + "loss": 1.4533, + "theoretical_loss": 3.9228801466039958, + "tokens_seen": 525008896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795105315947844, + "loss": 1.4722, + "theoretical_loss": 3.9228282289998155, + "tokens_seen": 525074432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794944834503511, + "loss": 1.358, + "theoretical_loss": 3.9227763196893477, + "tokens_seen": 525139968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794784353059178, + "loss": 1.5151, + "theoretical_loss": 3.9227244186702332, + "tokens_seen": 525205504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794623871614845, + "loss": 1.4097, + "theoretical_loss": 3.9226725259401123, + "tokens_seen": 525271040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794463390170512, + "loss": 1.5695, + "theoretical_loss": 3.9226206414966285, + "tokens_seen": 525336576 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679430290872618, + "loss": 1.3796, + "theoretical_loss": 3.9225687653374237, + "tokens_seen": 525402112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794142427281846, + "loss": 1.4475, + "theoretical_loss": 3.922516897460143, + "tokens_seen": 525467648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793981945837513, + "loss": 1.4249, + "theoretical_loss": 3.9224650378624313, + "tokens_seen": 525533184 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679382146439318, + "loss": 1.5164, + "theoretical_loss": 3.9224131865419345, + "tokens_seen": 525598720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793660982948848, + "loss": 1.4282, + "theoretical_loss": 3.9223613434963003, + "tokens_seen": 525664256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793500501504514, + "loss": 1.4594, + "theoretical_loss": 3.922309508723176, + "tokens_seen": 525729792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793340020060182, + "loss": 1.4641, + "theoretical_loss": 3.9222576822202107, + "tokens_seen": 525795328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793179538615848, + "loss": 1.5236, + "theoretical_loss": 3.922205863985055, + "tokens_seen": 525860864 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793019057171516, + "loss": 1.4442, + "theoretical_loss": 3.9221540540153588, + "tokens_seen": 525926400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006792858575727182, + "loss": 1.3884, + "theoretical_loss": 3.922102252308775, + "tokens_seen": 525991936 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679269809428285, + "loss": 1.5972, + "theoretical_loss": 3.9220504588629552, + "tokens_seen": 526057472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006792537612838516, + "loss": 1.4631, + "theoretical_loss": 3.9219986736755548, + "tokens_seen": 526123008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006792377131394184, + "loss": 1.5174, + "theoretical_loss": 3.921946896744227, + "tokens_seen": 526188544 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679221664994985, + "loss": 1.3494, + "theoretical_loss": 3.9218951280666285, + "tokens_seen": 526254080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006792056168505518, + "loss": 1.3861, + "theoretical_loss": 3.9218433676404154, + "tokens_seen": 526319616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791895687061184, + "loss": 1.3822, + "theoretical_loss": 3.9217916154632455, + "tokens_seen": 526385152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791735205616852, + "loss": 1.5359, + "theoretical_loss": 3.9217398715327776, + "tokens_seen": 526450688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791574724172518, + "loss": 1.5608, + "theoretical_loss": 3.9216881358466713, + "tokens_seen": 526516224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791414242728185, + "loss": 1.3272, + "theoretical_loss": 3.9216364084025868, + "tokens_seen": 526581760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791253761283852, + "loss": 1.5005, + "theoretical_loss": 3.921584689198185, + "tokens_seen": 526647296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791093279839519, + "loss": 1.3935, + "theoretical_loss": 3.921532978231129, + "tokens_seen": 526712832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790932798395186, + "loss": 1.4883, + "theoretical_loss": 3.921481275499082, + "tokens_seen": 526778368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790772316950853, + "loss": 1.5002, + "theoretical_loss": 3.921429580999709, + "tokens_seen": 526843904 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679061183550652, + "loss": 1.538, + "theoretical_loss": 3.921377894730674, + "tokens_seen": 526909440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790451354062187, + "loss": 1.5555, + "theoretical_loss": 3.9213262166896445, + "tokens_seen": 526974976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790290872617854, + "loss": 1.4427, + "theoretical_loss": 3.9212745468742867, + "tokens_seen": 527040512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790130391173521, + "loss": 1.3753, + "theoretical_loss": 3.921222885282268, + "tokens_seen": 527106048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789969909729187, + "loss": 1.3694, + "theoretical_loss": 3.9211712319112597, + "tokens_seen": 527171584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789809428284855, + "loss": 1.4033, + "theoretical_loss": 3.92111958675893, + "tokens_seen": 527237120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789648946840521, + "loss": 1.5529, + "theoretical_loss": 3.9210679498229504, + "tokens_seen": 527302656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789488465396189, + "loss": 1.5309, + "theoretical_loss": 3.9210163211009927, + "tokens_seen": 527368192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789327983951855, + "loss": 1.4576, + "theoretical_loss": 3.92096470059073, + "tokens_seen": 527433728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789167502507523, + "loss": 1.4784, + "theoretical_loss": 3.9209130882898364, + "tokens_seen": 527499264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 181844, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4468753337860107, + "objective/train/theoretical_loss": 3.920861484195986, + "objective/train/tokens_used": 548024800, + "theoretical_loss": 3.920861484195986, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789007021063189, + "loss": 1.4834, + "theoretical_loss": 3.920861484195986, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788846539618857, + "loss": 1.3464, + "theoretical_loss": 3.920809888306854, + "tokens_seen": 527630336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788686058174523, + "loss": 1.4429, + "theoretical_loss": 3.9207583006201183, + "tokens_seen": 527695872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788525576730191, + "loss": 1.5239, + "theoretical_loss": 3.920706721133456, + "tokens_seen": 527761408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788365095285857, + "loss": 1.4154, + "theoretical_loss": 3.9206551498445465, + "tokens_seen": 527826944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788204613841525, + "loss": 1.3948, + "theoretical_loss": 3.920603586751068, + "tokens_seen": 527892480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788044132397191, + "loss": 1.5827, + "theoretical_loss": 3.9205520318507006, + "tokens_seen": 527958016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787883650952859, + "loss": 1.4211, + "theoretical_loss": 3.9205004851411274, + "tokens_seen": 528023552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787723169508525, + "loss": 1.378, + "theoretical_loss": 3.920448946620029, + "tokens_seen": 528089088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787562688064193, + "loss": 1.4033, + "theoretical_loss": 3.9203974162850903, + "tokens_seen": 528154624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787402206619859, + "loss": 1.5083, + "theoretical_loss": 3.920345894133994, + "tokens_seen": 528220160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787241725175527, + "loss": 1.4078, + "theoretical_loss": 3.9202943801644263, + "tokens_seen": 528285696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787081243731193, + "loss": 1.5383, + "theoretical_loss": 3.9202428743740727, + "tokens_seen": 528351232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786920762286861, + "loss": 1.3915, + "theoretical_loss": 3.92019137676062, + "tokens_seen": 528416768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786760280842528, + "loss": 1.4579, + "theoretical_loss": 3.920139887321757, + "tokens_seen": 528482304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786599799398195, + "loss": 1.4671, + "theoretical_loss": 3.9200884060551724, + "tokens_seen": 528547840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786439317953862, + "loss": 1.3536, + "theoretical_loss": 3.9200369329585563, + "tokens_seen": 528613376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786278836509529, + "loss": 1.4441, + "theoretical_loss": 3.9199854680295982, + "tokens_seen": 528678912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786118355065196, + "loss": 1.3964, + "theoretical_loss": 3.9199340112659913, + "tokens_seen": 528744448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785957873620863, + "loss": 1.4516, + "theoretical_loss": 3.9198825626654275, + "tokens_seen": 528809984 + }, + { + "epoch": 0.16, + "learning_rate": 0.000678579739217653, + "loss": 1.4887, + "theoretical_loss": 3.9198311222256, + "tokens_seen": 528875520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785636910732197, + "loss": 1.5368, + "theoretical_loss": 3.919779689944205, + "tokens_seen": 528941056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785476429287864, + "loss": 1.4402, + "theoretical_loss": 3.9197282658189363, + "tokens_seen": 529006592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785315947843531, + "loss": 1.5339, + "theoretical_loss": 3.919676849847491, + "tokens_seen": 529072128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785155466399198, + "loss": 1.4247, + "theoretical_loss": 3.9196254420275665, + "tokens_seen": 529137664 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784994984954865, + "loss": 1.4668, + "theoretical_loss": 3.919574042356861, + "tokens_seen": 529203200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784834503510532, + "loss": 1.4803, + "theoretical_loss": 3.9195226508330734, + "tokens_seen": 529268736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784674022066199, + "loss": 1.3684, + "theoretical_loss": 3.919471267453905, + "tokens_seen": 529334272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784513540621866, + "loss": 1.524, + "theoretical_loss": 3.919419892217055, + "tokens_seen": 529399808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784353059177533, + "loss": 1.4664, + "theoretical_loss": 3.919368525120227, + "tokens_seen": 529465344 + }, + { + "epoch": 0.16, + "learning_rate": 0.00067841925777332, + "loss": 1.4618, + "theoretical_loss": 3.9193171661611235, + "tokens_seen": 529530880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784032096288867, + "loss": 1.4584, + "theoretical_loss": 3.9192658153374484, + "tokens_seen": 529596416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783871614844534, + "loss": 1.4449, + "theoretical_loss": 3.9192144726469063, + "tokens_seen": 529661952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783711133400201, + "loss": 1.5014, + "theoretical_loss": 3.919163138087203, + "tokens_seen": 529727488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783550651955868, + "loss": 1.5635, + "theoretical_loss": 3.919111811656046, + "tokens_seen": 529793024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783390170511535, + "loss": 1.4419, + "theoretical_loss": 3.9190604933511417, + "tokens_seen": 529858560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783229689067202, + "loss": 1.448, + "theoretical_loss": 3.9190091831701994, + "tokens_seen": 529924096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783069207622869, + "loss": 1.3953, + "theoretical_loss": 3.918957881110928, + "tokens_seen": 529989632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782908726178536, + "loss": 1.5078, + "theoretical_loss": 3.9189065871710387, + "tokens_seen": 530055168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782748244734203, + "loss": 1.3386, + "theoretical_loss": 3.918855301348242, + "tokens_seen": 530120704 + }, + { + "epoch": 0.16, + "learning_rate": 0.000678258776328987, + "loss": 1.5389, + "theoretical_loss": 3.9188040236402513, + "tokens_seen": 530186240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782427281845537, + "loss": 1.4875, + "theoretical_loss": 3.918752754044778, + "tokens_seen": 530251776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782266800401204, + "loss": 1.3773, + "theoretical_loss": 3.9187014925595385, + "tokens_seen": 530317312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782106318956871, + "loss": 1.564, + "theoretical_loss": 3.918650239182246, + "tokens_seen": 530382848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781945837512538, + "loss": 1.5289, + "theoretical_loss": 3.918598993910617, + "tokens_seen": 530448384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781785356068205, + "loss": 1.4389, + "theoretical_loss": 3.9185477567423685, + "tokens_seen": 530513920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781624874623872, + "loss": 1.5537, + "theoretical_loss": 3.9184965276752184, + "tokens_seen": 530579456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781464393179539, + "loss": 1.385, + "theoretical_loss": 3.9184453067068854, + "tokens_seen": 530644992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781303911735206, + "loss": 1.3913, + "theoretical_loss": 3.918394093835089, + "tokens_seen": 530710528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781143430290873, + "loss": 1.4841, + "theoretical_loss": 3.91834288905755, + "tokens_seen": 530776064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 183006, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5242764949798584, + "objective/train/theoretical_loss": 3.91829169237199, + "objective/train/tokens_used": 551301600, + "theoretical_loss": 3.91829169237199, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.000678098294884654, + "loss": 1.5453, + "theoretical_loss": 3.91829169237199, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780822467402207, + "loss": 1.5267, + "theoretical_loss": 3.918240503776131, + "tokens_seen": 530907136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780661985957874, + "loss": 1.5107, + "theoretical_loss": 3.9181893232676965, + "tokens_seen": 530972672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780501504513541, + "loss": 1.4461, + "theoretical_loss": 3.9181381508444106, + "tokens_seen": 531038208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780341023069208, + "loss": 1.4501, + "theoretical_loss": 3.918086986504, + "tokens_seen": 531103744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780180541624875, + "loss": 1.46, + "theoretical_loss": 3.9180358302441887, + "tokens_seen": 531169280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780020060180543, + "loss": 1.388, + "theoretical_loss": 3.9179846820627047, + "tokens_seen": 531234816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779859578736209, + "loss": 1.3978, + "theoretical_loss": 3.917933541957276, + "tokens_seen": 531300352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779699097291877, + "loss": 1.5003, + "theoretical_loss": 3.9178824099256317, + "tokens_seen": 531365888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779538615847543, + "loss": 1.5086, + "theoretical_loss": 3.9178312859655016, + "tokens_seen": 531431424 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677937813440321, + "loss": 1.5874, + "theoretical_loss": 3.9177801700746153, + "tokens_seen": 531496960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779217652958877, + "loss": 1.4059, + "theoretical_loss": 3.9177290622507055, + "tokens_seen": 531562496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779057171514545, + "loss": 1.5381, + "theoretical_loss": 3.9176779624915055, + "tokens_seen": 531628032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778896690070211, + "loss": 1.4416, + "theoretical_loss": 3.917626870794747, + "tokens_seen": 531693568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778736208625879, + "loss": 1.4973, + "theoretical_loss": 3.9175757871581647, + "tokens_seen": 531759104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778575727181545, + "loss": 1.4864, + "theoretical_loss": 3.917524711579495, + "tokens_seen": 531824640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778415245737213, + "loss": 1.5042, + "theoretical_loss": 3.9174736440564732, + "tokens_seen": 531890176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778254764292879, + "loss": 1.6653, + "theoretical_loss": 3.917422584586837, + "tokens_seen": 531955712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778094282848547, + "loss": 1.408, + "theoretical_loss": 3.9173715331683243, + "tokens_seen": 532021248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777933801404213, + "loss": 1.3939, + "theoretical_loss": 3.9173204897986738, + "tokens_seen": 532086784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777773319959881, + "loss": 1.4191, + "theoretical_loss": 3.9172694544756252, + "tokens_seen": 532152320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777612838515547, + "loss": 1.4525, + "theoretical_loss": 3.91721842719692, + "tokens_seen": 532217856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777452357071214, + "loss": 1.4859, + "theoretical_loss": 3.9171674079603, + "tokens_seen": 532283392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777291875626881, + "loss": 1.4312, + "theoretical_loss": 3.9171163967635065, + "tokens_seen": 532348928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777131394182548, + "loss": 1.5149, + "theoretical_loss": 3.9170653936042843, + "tokens_seen": 532414464 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776970912738215, + "loss": 1.4395, + "theoretical_loss": 3.917014398480377, + "tokens_seen": 532480000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776810431293882, + "loss": 1.5004, + "theoretical_loss": 3.9169634113895304, + "tokens_seen": 532545536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776649949849549, + "loss": 1.4336, + "theoretical_loss": 3.9169124323294913, + "tokens_seen": 532611072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776489468405216, + "loss": 1.5267, + "theoretical_loss": 3.916861461298005, + "tokens_seen": 532676608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776328986960883, + "loss": 1.5136, + "theoretical_loss": 3.916810498292822, + "tokens_seen": 532742144 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677616850551655, + "loss": 1.4758, + "theoretical_loss": 3.9167595433116897, + "tokens_seen": 532807680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776008024072216, + "loss": 1.4975, + "theoretical_loss": 3.9167085963523585, + "tokens_seen": 532873216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775847542627884, + "loss": 1.6303, + "theoretical_loss": 3.916657657412579, + "tokens_seen": 532938752 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677568706118355, + "loss": 1.5686, + "theoretical_loss": 3.9166067264901026, + "tokens_seen": 533004288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775526579739218, + "loss": 1.5364, + "theoretical_loss": 3.916555803582683, + "tokens_seen": 533069824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775366098294884, + "loss": 1.4351, + "theoretical_loss": 3.9165048886880722, + "tokens_seen": 533135360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775205616850552, + "loss": 1.5138, + "theoretical_loss": 3.916453981804026, + "tokens_seen": 533200896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775045135406218, + "loss": 1.3943, + "theoretical_loss": 3.916403082928299, + "tokens_seen": 533266432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774884653961886, + "loss": 1.4264, + "theoretical_loss": 3.916352192058648, + "tokens_seen": 533331968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774724172517553, + "loss": 1.4411, + "theoretical_loss": 3.9163013091928294, + "tokens_seen": 533397504 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677456369107322, + "loss": 1.4073, + "theoretical_loss": 3.9162504343286013, + "tokens_seen": 533463040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774403209628887, + "loss": 1.4156, + "theoretical_loss": 3.9161995674637233, + "tokens_seen": 533528576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774242728184554, + "loss": 1.4649, + "theoretical_loss": 3.916148708595955, + "tokens_seen": 533594112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774082246740221, + "loss": 1.4853, + "theoretical_loss": 3.9160978577230567, + "tokens_seen": 533659648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773921765295888, + "loss": 1.5875, + "theoretical_loss": 3.91604701484279, + "tokens_seen": 533725184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773761283851555, + "loss": 1.4837, + "theoretical_loss": 3.915996179952918, + "tokens_seen": 533790720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773600802407222, + "loss": 1.44, + "theoretical_loss": 3.915945353051205, + "tokens_seen": 533856256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773440320962889, + "loss": 1.5419, + "theoretical_loss": 3.9158945341354134, + "tokens_seen": 533921792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773279839518556, + "loss": 1.5078, + "theoretical_loss": 3.9158437232033094, + "tokens_seen": 533987328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773119358074223, + "loss": 1.4417, + "theoretical_loss": 3.9157929202526596, + "tokens_seen": 534052864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 184018, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4757671356201172, + "objective/train/theoretical_loss": 3.9157421252812306, + "objective/train/tokens_used": 554578400, + "theoretical_loss": 3.9157421252812306, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677295887662989, + "loss": 1.4674, + "theoretical_loss": 3.9157421252812306, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772798395185557, + "loss": 1.5966, + "theoretical_loss": 3.9156913382867895, + "tokens_seen": 534183936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772637913741224, + "loss": 1.4379, + "theoretical_loss": 3.915640559267107, + "tokens_seen": 534249472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772477432296891, + "loss": 1.5897, + "theoretical_loss": 3.9155897882199513, + "tokens_seen": 534315008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772316950852558, + "loss": 1.393, + "theoretical_loss": 3.9155390251430937, + "tokens_seen": 534380544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772156469408225, + "loss": 1.5403, + "theoretical_loss": 3.915488270034306, + "tokens_seen": 534446080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771995987963892, + "loss": 1.4583, + "theoretical_loss": 3.9154375228913594, + "tokens_seen": 534511616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771835506519559, + "loss": 1.3791, + "theoretical_loss": 3.9153867837120284, + "tokens_seen": 534577152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771675025075226, + "loss": 1.4518, + "theoretical_loss": 3.9153360524940872, + "tokens_seen": 534642688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771514543630893, + "loss": 1.4759, + "theoretical_loss": 3.915285329235311, + "tokens_seen": 534708224 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677135406218656, + "loss": 1.5145, + "theoretical_loss": 3.915234613933475, + "tokens_seen": 534773760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771193580742227, + "loss": 1.5178, + "theoretical_loss": 3.915183906586357, + "tokens_seen": 534839296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771033099297894, + "loss": 1.3855, + "theoretical_loss": 3.915133207191734, + "tokens_seen": 534904832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770872617853561, + "loss": 1.4519, + "theoretical_loss": 3.915082515747385, + "tokens_seen": 534970368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770712136409228, + "loss": 1.5169, + "theoretical_loss": 3.9150318322510893, + "tokens_seen": 535035904 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770551654964895, + "loss": 1.4437, + "theoretical_loss": 3.9149811567006285, + "tokens_seen": 535101440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770391173520562, + "loss": 1.4334, + "theoretical_loss": 3.9149304890937833, + "tokens_seen": 535166976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770230692076229, + "loss": 1.4117, + "theoretical_loss": 3.9148798294283353, + "tokens_seen": 535232512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770070210631896, + "loss": 1.5312, + "theoretical_loss": 3.9148291777020683, + "tokens_seen": 535298048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769909729187563, + "loss": 1.4469, + "theoretical_loss": 3.9147785339127665, + "tokens_seen": 535363584 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676974924774323, + "loss": 1.5636, + "theoretical_loss": 3.914727898058215, + "tokens_seen": 535429120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769588766298897, + "loss": 1.4036, + "theoretical_loss": 3.914677270136199, + "tokens_seen": 535494656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769428284854564, + "loss": 1.5086, + "theoretical_loss": 3.914626650144505, + "tokens_seen": 535560192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769267803410231, + "loss": 1.3943, + "theoretical_loss": 3.914576038080922, + "tokens_seen": 535625728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769107321965898, + "loss": 1.3243, + "theoretical_loss": 3.9145254339432367, + "tokens_seen": 535691264 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768946840521565, + "loss": 1.4929, + "theoretical_loss": 3.9144748377292395, + "tokens_seen": 535756800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768786359077232, + "loss": 1.4033, + "theoretical_loss": 3.9144242494367205, + "tokens_seen": 535822336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768625877632899, + "loss": 1.4823, + "theoretical_loss": 3.914373669063471, + "tokens_seen": 535887872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768465396188566, + "loss": 1.4225, + "theoretical_loss": 3.914323096607283, + "tokens_seen": 535953408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768304914744233, + "loss": 1.4116, + "theoretical_loss": 3.914272532065949, + "tokens_seen": 536018944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00067681444332999, + "loss": 1.4862, + "theoretical_loss": 3.914221975437264, + "tokens_seen": 536084480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767983951855567, + "loss": 1.4409, + "theoretical_loss": 3.914171426719021, + "tokens_seen": 536150016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767823470411234, + "loss": 1.5067, + "theoretical_loss": 3.9141208859090164, + "tokens_seen": 536215552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767662988966901, + "loss": 1.4402, + "theoretical_loss": 3.9140703530050467, + "tokens_seen": 536281088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767502507522568, + "loss": 1.6256, + "theoretical_loss": 3.914019828004909, + "tokens_seen": 536346624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767342026078235, + "loss": 1.4947, + "theoretical_loss": 3.9139693109064018, + "tokens_seen": 536412160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767181544633902, + "loss": 1.4468, + "theoretical_loss": 3.9139188017073243, + "tokens_seen": 536477696 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676702106318957, + "loss": 1.4703, + "theoretical_loss": 3.9138683004054764, + "tokens_seen": 536543232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766860581745236, + "loss": 1.3957, + "theoretical_loss": 3.913817806998659, + "tokens_seen": 536608768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766700100300903, + "loss": 1.4385, + "theoretical_loss": 3.9137673214846735, + "tokens_seen": 536674304 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676653961885657, + "loss": 1.3962, + "theoretical_loss": 3.913716843861323, + "tokens_seen": 536739840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766379137412238, + "loss": 1.5503, + "theoretical_loss": 3.91366637412641, + "tokens_seen": 536805376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766218655967904, + "loss": 1.571, + "theoretical_loss": 3.9136159122777405, + "tokens_seen": 536870912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766058174523572, + "loss": 1.546, + "theoretical_loss": 3.9135654583131183, + "tokens_seen": 536936448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765897693079238, + "loss": 1.5006, + "theoretical_loss": 3.9135150122303504, + "tokens_seen": 537001984 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765737211634906, + "loss": 1.5824, + "theoretical_loss": 3.913464574027244, + "tokens_seen": 537067520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765576730190572, + "loss": 1.5226, + "theoretical_loss": 3.913414143701606, + "tokens_seen": 537133056 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676541624874624, + "loss": 1.5175, + "theoretical_loss": 3.913363721251246, + "tokens_seen": 537198592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765255767301906, + "loss": 1.3869, + "theoretical_loss": 3.913313306673974, + "tokens_seen": 537264128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765095285857574, + "loss": 1.5287, + "theoretical_loss": 3.913262899967599, + "tokens_seen": 537329664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 185299, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3653303384780884, + "objective/train/theoretical_loss": 3.9132125011299337, + "objective/train/tokens_used": 557855200, + "theoretical_loss": 3.9132125011299337, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676493480441324, + "loss": 1.3709, + "theoretical_loss": 3.9132125011299337, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764774322968908, + "loss": 1.4701, + "theoretical_loss": 3.9131621101587903, + "tokens_seen": 537460736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764613841524574, + "loss": 1.4026, + "theoretical_loss": 3.913111727051982, + "tokens_seen": 537526272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764453360080242, + "loss": 1.6027, + "theoretical_loss": 3.9130613518073227, + "tokens_seen": 537591808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764292878635908, + "loss": 1.4285, + "theoretical_loss": 3.913010984422627, + "tokens_seen": 537657344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764132397191576, + "loss": 1.4628, + "theoretical_loss": 3.9129606248957103, + "tokens_seen": 537722880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763971915747242, + "loss": 1.5315, + "theoretical_loss": 3.91291027322439, + "tokens_seen": 537788416 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676381143430291, + "loss": 1.3342, + "theoretical_loss": 3.9128599294064834, + "tokens_seen": 537853952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763650952858576, + "loss": 1.6502, + "theoretical_loss": 3.9128095934398095, + "tokens_seen": 537919488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763490471414244, + "loss": 1.4484, + "theoretical_loss": 3.9127592653221868, + "tokens_seen": 537985024 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676332998996991, + "loss": 1.4775, + "theoretical_loss": 3.912708945051435, + "tokens_seen": 538050560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763169508525577, + "loss": 1.6435, + "theoretical_loss": 3.9126586326253765, + "tokens_seen": 538116096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763009027081244, + "loss": 1.5078, + "theoretical_loss": 3.9126083280418325, + "tokens_seen": 538181632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762848545636911, + "loss": 1.4297, + "theoretical_loss": 3.9125580312986257, + "tokens_seen": 538247168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762688064192578, + "loss": 1.4096, + "theoretical_loss": 3.9125077423935792, + "tokens_seen": 538312704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762527582748245, + "loss": 1.4451, + "theoretical_loss": 3.912457461324519, + "tokens_seen": 538378240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762367101303912, + "loss": 1.5723, + "theoretical_loss": 3.9124071880892686, + "tokens_seen": 538443776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762206619859579, + "loss": 1.4255, + "theoretical_loss": 3.912356922685655, + "tokens_seen": 538509312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762046138415246, + "loss": 1.6908, + "theoretical_loss": 3.912306665111506, + "tokens_seen": 538574848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761885656970913, + "loss": 1.4579, + "theoretical_loss": 3.9122564153646486, + "tokens_seen": 538640384 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676172517552658, + "loss": 1.4806, + "theoretical_loss": 3.9122061734429128, + "tokens_seen": 538705920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761564694082247, + "loss": 1.3943, + "theoretical_loss": 3.9121559393441268, + "tokens_seen": 538771456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761404212637914, + "loss": 1.4735, + "theoretical_loss": 3.912105713066122, + "tokens_seen": 538836992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761243731193581, + "loss": 1.5643, + "theoretical_loss": 3.91205549460673, + "tokens_seen": 538902528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761083249749248, + "loss": 1.4788, + "theoretical_loss": 3.9120052839637824, + "tokens_seen": 538968064 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760922768304915, + "loss": 1.5398, + "theoretical_loss": 3.9119550811351127, + "tokens_seen": 539033600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760762286860582, + "loss": 1.5037, + "theoretical_loss": 3.9119048861185552, + "tokens_seen": 539099136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760601805416249, + "loss": 1.4558, + "theoretical_loss": 3.9118546989119447, + "tokens_seen": 539164672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760441323971916, + "loss": 1.5235, + "theoretical_loss": 3.9118045195131166, + "tokens_seen": 539230208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760280842527583, + "loss": 1.4225, + "theoretical_loss": 3.911754347919908, + "tokens_seen": 539295744 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676012036108325, + "loss": 1.4986, + "theoretical_loss": 3.911704184130156, + "tokens_seen": 539361280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759959879638917, + "loss": 1.4661, + "theoretical_loss": 3.911654028141699, + "tokens_seen": 539426816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759799398194584, + "loss": 1.4318, + "theoretical_loss": 3.9116038799523762, + "tokens_seen": 539492352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759638916750251, + "loss": 1.478, + "theoretical_loss": 3.911553739560028, + "tokens_seen": 539557888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759478435305918, + "loss": 1.4314, + "theoretical_loss": 3.9115036069624947, + "tokens_seen": 539623424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759317953861585, + "loss": 1.471, + "theoretical_loss": 3.9114534821576186, + "tokens_seen": 539688960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759157472417252, + "loss": 1.3858, + "theoretical_loss": 3.9114033651432414, + "tokens_seen": 539754496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758996990972919, + "loss": 1.4939, + "theoretical_loss": 3.9113532559172084, + "tokens_seen": 539820032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758836509528586, + "loss": 1.5082, + "theoretical_loss": 3.9113031544773618, + "tokens_seen": 539885568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758676028084253, + "loss": 1.545, + "theoretical_loss": 3.9112530608215486, + "tokens_seen": 539951104 + }, + { + "epoch": 0.16, + "learning_rate": 0.000675851554663992, + "loss": 1.5214, + "theoretical_loss": 3.9112029749476145, + "tokens_seen": 540016640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758355065195587, + "loss": 1.5001, + "theoretical_loss": 3.9111528968534053, + "tokens_seen": 540082176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758194583751254, + "loss": 1.5725, + "theoretical_loss": 3.91110282653677, + "tokens_seen": 540147712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758034102306921, + "loss": 1.4774, + "theoretical_loss": 3.9110527639955563, + "tokens_seen": 540213248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757873620862588, + "loss": 1.5428, + "theoretical_loss": 3.911002709227615, + "tokens_seen": 540278784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757713139418255, + "loss": 1.5055, + "theoretical_loss": 3.9109526622307955, + "tokens_seen": 540344320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757552657973922, + "loss": 1.409, + "theoretical_loss": 3.910902623002949, + "tokens_seen": 540409856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757392176529589, + "loss": 1.4916, + "theoretical_loss": 3.9108525915419277, + "tokens_seen": 540475392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757231695085256, + "loss": 1.4927, + "theoretical_loss": 3.9108025678455847, + "tokens_seen": 540540928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757071213640923, + "loss": 1.4452, + "theoretical_loss": 3.9107525519117736, + "tokens_seen": 540606464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 186273, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.601438283920288, + "objective/train/theoretical_loss": 3.910702543738349, + "objective/train/tokens_used": 561132000, + "theoretical_loss": 3.910702543738349, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.000675691073219659, + "loss": 1.559, + "theoretical_loss": 3.910702543738349, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756750250752257, + "loss": 1.4619, + "theoretical_loss": 3.9106525433231663, + "tokens_seen": 540737536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756589769307924, + "loss": 1.4786, + "theoretical_loss": 3.9106025506640822, + "tokens_seen": 540803072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756429287863591, + "loss": 1.4277, + "theoretical_loss": 3.9105525657589535, + "tokens_seen": 540868608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756268806419258, + "loss": 1.5143, + "theoretical_loss": 3.910502588605638, + "tokens_seen": 540934144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756108324974925, + "loss": 1.5267, + "theoretical_loss": 3.9104526192019957, + "tokens_seen": 540999680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755947843530592, + "loss": 1.4738, + "theoretical_loss": 3.9104026575458857, + "tokens_seen": 541065216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755787362086259, + "loss": 1.4764, + "theoretical_loss": 3.9103527036351684, + "tokens_seen": 541130752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755626880641926, + "loss": 1.4191, + "theoretical_loss": 3.910302757467705, + "tokens_seen": 541196288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755466399197593, + "loss": 1.3898, + "theoretical_loss": 3.910252819041358, + "tokens_seen": 541261824 + }, + { + "epoch": 0.16, + "learning_rate": 0.000675530591775326, + "loss": 1.4358, + "theoretical_loss": 3.910202888353991, + "tokens_seen": 541327360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755145436308927, + "loss": 1.5174, + "theoretical_loss": 3.9101529654034675, + "tokens_seen": 541392896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754984954864594, + "loss": 1.5144, + "theoretical_loss": 3.9101030501876526, + "tokens_seen": 541458432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754824473420261, + "loss": 1.4981, + "theoretical_loss": 3.9100531427044114, + "tokens_seen": 541523968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754663991975928, + "loss": 1.4326, + "theoretical_loss": 3.9100032429516114, + "tokens_seen": 541589504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754503510531595, + "loss": 1.4579, + "theoretical_loss": 3.909953350927119, + "tokens_seen": 541655040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754343029087262, + "loss": 1.4463, + "theoretical_loss": 3.909903466628803, + "tokens_seen": 541720576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754182547642929, + "loss": 1.5862, + "theoretical_loss": 3.9098535900545324, + "tokens_seen": 541786112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754022066198596, + "loss": 1.5375, + "theoretical_loss": 3.909803721202177, + "tokens_seen": 541851648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753861584754263, + "loss": 1.5433, + "theoretical_loss": 3.909753860069608, + "tokens_seen": 541917184 + }, + { + "epoch": 0.16, + "learning_rate": 0.000675370110330993, + "loss": 1.5082, + "theoretical_loss": 3.909704006654695, + "tokens_seen": 541982720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753540621865597, + "loss": 1.4676, + "theoretical_loss": 3.9096541609553137, + "tokens_seen": 542048256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753380140421264, + "loss": 1.4795, + "theoretical_loss": 3.909604322969335, + "tokens_seen": 542113792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753219658976931, + "loss": 1.5508, + "theoretical_loss": 3.9095544926946344, + "tokens_seen": 542179328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753059177532598, + "loss": 1.4308, + "theoretical_loss": 3.909504670129085, + "tokens_seen": 542244864 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752898696088265, + "loss": 1.4758, + "theoretical_loss": 3.909454855270565, + "tokens_seen": 542310400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752738214643933, + "loss": 1.4732, + "theoretical_loss": 3.909405048116949, + "tokens_seen": 542375936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752577733199599, + "loss": 1.4179, + "theoretical_loss": 3.909355248666116, + "tokens_seen": 542441472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752417251755267, + "loss": 1.4859, + "theoretical_loss": 3.9093054569159436, + "tokens_seen": 542507008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752256770310933, + "loss": 1.4269, + "theoretical_loss": 3.909255672864311, + "tokens_seen": 542572544 + }, + { + "epoch": 0.16, + "learning_rate": 0.00067520962888666, + "loss": 1.4713, + "theoretical_loss": 3.909205896509098, + "tokens_seen": 542638080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751935807422267, + "loss": 1.4976, + "theoretical_loss": 3.9091561278481866, + "tokens_seen": 542703616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751775325977935, + "loss": 1.5367, + "theoretical_loss": 3.909106366879457, + "tokens_seen": 542769152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751614844533601, + "loss": 1.4783, + "theoretical_loss": 3.909056613600793, + "tokens_seen": 542834688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751454363089269, + "loss": 1.5162, + "theoretical_loss": 3.9090068680100765, + "tokens_seen": 542900224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751293881644935, + "loss": 1.4326, + "theoretical_loss": 3.9089571301051933, + "tokens_seen": 542965760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751133400200603, + "loss": 1.45, + "theoretical_loss": 3.9089073998840274, + "tokens_seen": 543031296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750972918756269, + "loss": 1.5227, + "theoretical_loss": 3.908857677344465, + "tokens_seen": 543096832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750812437311937, + "loss": 1.3709, + "theoretical_loss": 3.908807962484393, + "tokens_seen": 543162368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750651955867603, + "loss": 1.5308, + "theoretical_loss": 3.908758255301699, + "tokens_seen": 543227904 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750491474423271, + "loss": 1.471, + "theoretical_loss": 3.9087085557942713, + "tokens_seen": 543293440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750330992978937, + "loss": 1.5801, + "theoretical_loss": 3.908658863959999, + "tokens_seen": 543358976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750170511534605, + "loss": 1.4352, + "theoretical_loss": 3.9086091797967724, + "tokens_seen": 543424512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750010030090271, + "loss": 1.3739, + "theoretical_loss": 3.908559503302482, + "tokens_seen": 543490048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749849548645939, + "loss": 1.474, + "theoretical_loss": 3.9085098344750193, + "tokens_seen": 543555584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749689067201605, + "loss": 1.3797, + "theoretical_loss": 3.908460173312278, + "tokens_seen": 543621120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749528585757273, + "loss": 1.5149, + "theoretical_loss": 3.9084105198121506, + "tokens_seen": 543686656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749368104312939, + "loss": 1.5111, + "theoretical_loss": 3.9083608739725317, + "tokens_seen": 543752192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749207622868606, + "loss": 1.4539, + "theoretical_loss": 3.9083112357913166, + "tokens_seen": 543817728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749047141424273, + "loss": 1.4421, + "theoretical_loss": 3.9082616052664, + "tokens_seen": 543883264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 187447, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5180284976959229, + "objective/train/theoretical_loss": 3.90821198239568, + "objective/train/tokens_used": 564408800, + "theoretical_loss": 3.90821198239568, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.000674888665997994, + "loss": 1.4843, + "theoretical_loss": 3.90821198239568, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748726178535607, + "loss": 1.3792, + "theoretical_loss": 3.9081623671770536, + "tokens_seen": 544014336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748565697091274, + "loss": 1.6008, + "theoretical_loss": 3.9081127596084193, + "tokens_seen": 544079872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748405215646941, + "loss": 1.4053, + "theoretical_loss": 3.908063159687676, + "tokens_seen": 544145408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748244734202608, + "loss": 1.474, + "theoretical_loss": 3.9080135674127243, + "tokens_seen": 544210944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748084252758275, + "loss": 1.532, + "theoretical_loss": 3.9079639827814647, + "tokens_seen": 544276480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006747923771313942, + "loss": 1.4896, + "theoretical_loss": 3.907914405791799, + "tokens_seen": 544342016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006747763289869609, + "loss": 1.4168, + "theoretical_loss": 3.9078648364416297, + "tokens_seen": 544407552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006747602808425276, + "loss": 1.4467, + "theoretical_loss": 3.9078152747288604, + "tokens_seen": 544473088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006747442326980943, + "loss": 1.5824, + "theoretical_loss": 3.907765720651395, + "tokens_seen": 544538624 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674728184553661, + "loss": 1.5888, + "theoretical_loss": 3.907716174207139, + "tokens_seen": 544604160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006747121364092277, + "loss": 1.5042, + "theoretical_loss": 3.9076666353939977, + "tokens_seen": 544669696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746960882647944, + "loss": 1.5183, + "theoretical_loss": 3.9076171042098777, + "tokens_seen": 544735232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746800401203611, + "loss": 1.4646, + "theoretical_loss": 3.907567580652687, + "tokens_seen": 544800768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746639919759278, + "loss": 1.4199, + "theoretical_loss": 3.9075180647203336, + "tokens_seen": 544866304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746479438314945, + "loss": 1.4331, + "theoretical_loss": 3.907468556410727, + "tokens_seen": 544931840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746318956870612, + "loss": 1.4481, + "theoretical_loss": 3.907419055721777, + "tokens_seen": 544997376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746158475426279, + "loss": 1.4933, + "theoretical_loss": 3.907369562651394, + "tokens_seen": 545062912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745997993981946, + "loss": 1.4835, + "theoretical_loss": 3.9073200771974905, + "tokens_seen": 545128448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745837512537613, + "loss": 1.4513, + "theoretical_loss": 3.907270599357978, + "tokens_seen": 545193984 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674567703109328, + "loss": 1.5083, + "theoretical_loss": 3.9072211291307704, + "tokens_seen": 545259520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745516549648947, + "loss": 1.4462, + "theoretical_loss": 3.907171666513781, + "tokens_seen": 545325056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745356068204614, + "loss": 1.4709, + "theoretical_loss": 3.907122211504926, + "tokens_seen": 545390592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745195586760281, + "loss": 1.5978, + "theoretical_loss": 3.9070727641021206, + "tokens_seen": 545456128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745035105315948, + "loss": 1.4448, + "theoretical_loss": 3.907023324303281, + "tokens_seen": 545521664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744874623871615, + "loss": 1.4641, + "theoretical_loss": 3.9069738921063246, + "tokens_seen": 545587200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744714142427282, + "loss": 1.5723, + "theoretical_loss": 3.9069244675091697, + "tokens_seen": 545652736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744553660982949, + "loss": 1.4894, + "theoretical_loss": 3.9068750505097354, + "tokens_seen": 545718272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744393179538616, + "loss": 1.4264, + "theoretical_loss": 3.906825641105942, + "tokens_seen": 545783808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744232698094283, + "loss": 1.4719, + "theoretical_loss": 3.9067762392957093, + "tokens_seen": 545849344 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674407221664995, + "loss": 1.468, + "theoretical_loss": 3.90672684507696, + "tokens_seen": 545914880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743911735205617, + "loss": 1.3939, + "theoretical_loss": 3.9066774584476147, + "tokens_seen": 545980416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743751253761284, + "loss": 1.4936, + "theoretical_loss": 3.9066280794055976, + "tokens_seen": 546045952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743590772316951, + "loss": 1.4917, + "theoretical_loss": 3.9065787079488326, + "tokens_seen": 546111488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743430290872618, + "loss": 1.5062, + "theoretical_loss": 3.9065293440752447, + "tokens_seen": 546177024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743269809428285, + "loss": 1.4176, + "theoretical_loss": 3.9064799877827587, + "tokens_seen": 546242560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743109327983952, + "loss": 1.5045, + "theoretical_loss": 3.9064306390693018, + "tokens_seen": 546308096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742948846539619, + "loss": 1.5332, + "theoretical_loss": 3.9063812979328, + "tokens_seen": 546373632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742788365095286, + "loss": 1.4321, + "theoretical_loss": 3.9063319643711827, + "tokens_seen": 546439168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742627883650953, + "loss": 1.4583, + "theoretical_loss": 3.906282638382377, + "tokens_seen": 546504704 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674246740220662, + "loss": 1.4618, + "theoretical_loss": 3.906233319964315, + "tokens_seen": 546570240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742306920762287, + "loss": 1.4734, + "theoretical_loss": 3.9061840091149254, + "tokens_seen": 546635776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742146439317954, + "loss": 1.4433, + "theoretical_loss": 3.90613470583214, + "tokens_seen": 546701312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741985957873621, + "loss": 1.4171, + "theoretical_loss": 3.9060854101138904, + "tokens_seen": 546766848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741825476429288, + "loss": 1.552, + "theoretical_loss": 3.90603612195811, + "tokens_seen": 546832384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741664994984955, + "loss": 1.4249, + "theoretical_loss": 3.9059868413627328, + "tokens_seen": 546897920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741504513540622, + "loss": 1.4977, + "theoretical_loss": 3.905937568325692, + "tokens_seen": 546963456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741344032096289, + "loss": 1.4874, + "theoretical_loss": 3.9058883028449247, + "tokens_seen": 547028992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741183550651956, + "loss": 1.573, + "theoretical_loss": 3.905839044918366, + "tokens_seen": 547094528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741023069207623, + "loss": 1.4674, + "theoretical_loss": 3.905789794543953, + "tokens_seen": 547160064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 188319, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3917874097824097, + "objective/train/theoretical_loss": 3.9057405517196235, + "objective/train/tokens_used": 567685600, + "theoretical_loss": 3.9057405517196235, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674086258776329, + "loss": 1.3673, + "theoretical_loss": 3.9057405517196235, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740702106318957, + "loss": 1.4805, + "theoretical_loss": 3.905691316443316, + "tokens_seen": 547291136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740541624874624, + "loss": 1.4456, + "theoretical_loss": 3.90564208871297, + "tokens_seen": 547356672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740381143430291, + "loss": 1.4466, + "theoretical_loss": 3.9055928685265258, + "tokens_seen": 547422208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740220661985958, + "loss": 1.3941, + "theoretical_loss": 3.9055436558819245, + "tokens_seen": 547487744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740060180541625, + "loss": 1.3788, + "theoretical_loss": 3.905494450777107, + "tokens_seen": 547553280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006739899699097292, + "loss": 1.6164, + "theoretical_loss": 3.9054452532100177, + "tokens_seen": 547618816 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673973921765296, + "loss": 1.4857, + "theoretical_loss": 3.905396063178598, + "tokens_seen": 547684352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006739578736208626, + "loss": 1.4982, + "theoretical_loss": 3.905346880680794, + "tokens_seen": 547749888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006739418254764293, + "loss": 1.355, + "theoretical_loss": 3.905297705714549, + "tokens_seen": 547815424 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673925777331996, + "loss": 1.496, + "theoretical_loss": 3.9052485382778097, + "tokens_seen": 547880960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006739097291875627, + "loss": 1.4519, + "theoretical_loss": 3.905199378368523, + "tokens_seen": 547946496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738936810431294, + "loss": 1.5708, + "theoretical_loss": 3.9051502259846362, + "tokens_seen": 548012032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738776328986962, + "loss": 1.4104, + "theoretical_loss": 3.9051010811240974, + "tokens_seen": 548077568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738615847542629, + "loss": 1.6264, + "theoretical_loss": 3.905051943784856, + "tokens_seen": 548143104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738455366098296, + "loss": 1.4225, + "theoretical_loss": 3.905002813964861, + "tokens_seen": 548208640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738294884653963, + "loss": 1.4701, + "theoretical_loss": 3.904953691662064, + "tokens_seen": 548274176 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673813440320963, + "loss": 1.4369, + "theoretical_loss": 3.904904576874416, + "tokens_seen": 548339712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737973921765297, + "loss": 1.4516, + "theoretical_loss": 3.90485546959987, + "tokens_seen": 548405248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737813440320964, + "loss": 1.4014, + "theoretical_loss": 3.904806369836378, + "tokens_seen": 548470784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737652958876631, + "loss": 1.4709, + "theoretical_loss": 3.904757277581895, + "tokens_seen": 548536320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737492477432298, + "loss": 1.55, + "theoretical_loss": 3.9047081928343745, + "tokens_seen": 548601856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737331995987965, + "loss": 1.497, + "theoretical_loss": 3.904659115591773, + "tokens_seen": 548667392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737171514543632, + "loss": 1.4167, + "theoretical_loss": 3.904610045852046, + "tokens_seen": 548732928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737011033099299, + "loss": 1.5021, + "theoretical_loss": 3.904560983613152, + "tokens_seen": 548798464 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006736850551654966, + "loss": 1.4396, + "theoretical_loss": 3.9045119288730468, + "tokens_seen": 548864000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006736690070210632, + "loss": 1.5089, + "theoretical_loss": 3.9044628816296907, + "tokens_seen": 548929536 + }, + { + "epoch": 0.17, + "learning_rate": 0.00067365295887663, + "loss": 1.3947, + "theoretical_loss": 3.9044138418810426, + "tokens_seen": 548995072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006736369107321966, + "loss": 1.4812, + "theoretical_loss": 3.904364809625063, + "tokens_seen": 549060608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006736208625877634, + "loss": 1.3479, + "theoretical_loss": 3.9043157848597128, + "tokens_seen": 549126144 + }, + { + "epoch": 0.17, + "learning_rate": 0.00067360481444333, + "loss": 1.3436, + "theoretical_loss": 3.904266767582954, + "tokens_seen": 549191680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735887662988968, + "loss": 1.4552, + "theoretical_loss": 3.9042177577927495, + "tokens_seen": 549257216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735727181544634, + "loss": 1.4793, + "theoretical_loss": 3.904168755487062, + "tokens_seen": 549322752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735566700100302, + "loss": 1.4728, + "theoretical_loss": 3.9041197606638565, + "tokens_seen": 549388288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735406218655968, + "loss": 1.5343, + "theoretical_loss": 3.9040707733210978, + "tokens_seen": 549453824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735245737211635, + "loss": 1.5119, + "theoretical_loss": 3.9040217934567525, + "tokens_seen": 549519360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735085255767302, + "loss": 1.4605, + "theoretical_loss": 3.903972821068786, + "tokens_seen": 549584896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734924774322969, + "loss": 1.3218, + "theoretical_loss": 3.903923856155167, + "tokens_seen": 549650432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734764292878636, + "loss": 1.4982, + "theoretical_loss": 3.9038748987138625, + "tokens_seen": 549715968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734603811434303, + "loss": 1.4314, + "theoretical_loss": 3.9038259487428424, + "tokens_seen": 549781504 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673444332998997, + "loss": 1.4938, + "theoretical_loss": 3.9037770062400767, + "tokens_seen": 549847040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734282848545637, + "loss": 1.4628, + "theoretical_loss": 3.903728071203535, + "tokens_seen": 549912576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734122367101304, + "loss": 1.3642, + "theoretical_loss": 3.90367914363119, + "tokens_seen": 549978112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733961885656971, + "loss": 1.4482, + "theoretical_loss": 3.903630223521013, + "tokens_seen": 550043648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733801404212638, + "loss": 1.5198, + "theoretical_loss": 3.903581310870978, + "tokens_seen": 550109184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733640922768305, + "loss": 1.4351, + "theoretical_loss": 3.9035324056790577, + "tokens_seen": 550174720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733480441323972, + "loss": 1.4349, + "theoretical_loss": 3.9034835079432275, + "tokens_seen": 550240256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733319959879639, + "loss": 1.5413, + "theoretical_loss": 3.9034346176614623, + "tokens_seen": 550305792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733159478435306, + "loss": 1.5968, + "theoretical_loss": 3.9033857348317387, + "tokens_seen": 550371328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732998996990973, + "loss": 1.5534, + "theoretical_loss": 3.9033368594520335, + "tokens_seen": 550436864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 189606, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5754977464675903, + "objective/train/theoretical_loss": 3.903287991520324, + "objective/train/tokens_used": 570962400, + "theoretical_loss": 3.903287991520324, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673283851554664, + "loss": 1.502, + "theoretical_loss": 3.903287991520324, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732678034102307, + "loss": 1.4596, + "theoretical_loss": 3.903239131034589, + "tokens_seen": 550567936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732517552657974, + "loss": 1.4343, + "theoretical_loss": 3.9031902779928087, + "tokens_seen": 550633472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732357071213641, + "loss": 1.4176, + "theoretical_loss": 3.903141432392962, + "tokens_seen": 550699008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732196589769308, + "loss": 1.4912, + "theoretical_loss": 3.903092594233031, + "tokens_seen": 550764544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732036108324975, + "loss": 1.4471, + "theoretical_loss": 3.903043763510996, + "tokens_seen": 550830080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731875626880642, + "loss": 1.4121, + "theoretical_loss": 3.9029949402248407, + "tokens_seen": 550895616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731715145436309, + "loss": 1.4698, + "theoretical_loss": 3.902946124372548, + "tokens_seen": 550961152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731554663991976, + "loss": 1.4843, + "theoretical_loss": 3.902897315952102, + "tokens_seen": 551026688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731394182547643, + "loss": 1.4583, + "theoretical_loss": 3.9028485149614873, + "tokens_seen": 551092224 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673123370110331, + "loss": 1.4265, + "theoretical_loss": 3.90279972139869, + "tokens_seen": 551157760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731073219658977, + "loss": 1.4766, + "theoretical_loss": 3.902750935261696, + "tokens_seen": 551223296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730912738214644, + "loss": 1.5156, + "theoretical_loss": 3.902702156548493, + "tokens_seen": 551288832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730752256770311, + "loss": 1.5538, + "theoretical_loss": 3.9026533852570684, + "tokens_seen": 551354368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730591775325978, + "loss": 1.4477, + "theoretical_loss": 3.9026046213854118, + "tokens_seen": 551419904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730431293881645, + "loss": 1.4707, + "theoretical_loss": 3.9025558649315117, + "tokens_seen": 551485440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730270812437312, + "loss": 1.5043, + "theoretical_loss": 3.9025071158933597, + "tokens_seen": 551550976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730110330992979, + "loss": 1.4054, + "theoretical_loss": 3.9024583742689463, + "tokens_seen": 551616512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729949849548646, + "loss": 1.3994, + "theoretical_loss": 3.902409640056263, + "tokens_seen": 551682048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729789368104313, + "loss": 1.427, + "theoretical_loss": 3.902360913253303, + "tokens_seen": 551747584 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672962888665998, + "loss": 1.4008, + "theoretical_loss": 3.9023121938580605, + "tokens_seen": 551813120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729468405215647, + "loss": 1.4376, + "theoretical_loss": 3.9022634818685282, + "tokens_seen": 551878656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729307923771314, + "loss": 1.554, + "theoretical_loss": 3.902214777282702, + "tokens_seen": 551944192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729147442326981, + "loss": 1.3726, + "theoretical_loss": 3.902166080098578, + "tokens_seen": 552009728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728986960882648, + "loss": 1.394, + "theoretical_loss": 3.902117390314152, + "tokens_seen": 552075264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728826479438315, + "loss": 1.5451, + "theoretical_loss": 3.902068707927422, + "tokens_seen": 552140800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728665997993982, + "loss": 1.5443, + "theoretical_loss": 3.9020200329363863, + "tokens_seen": 552206336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728505516549649, + "loss": 1.4525, + "theoretical_loss": 3.901971365339043, + "tokens_seen": 552271872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728345035105316, + "loss": 1.345, + "theoretical_loss": 3.9019227051333925, + "tokens_seen": 552337408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728184553660983, + "loss": 1.4898, + "theoretical_loss": 3.901874052317435, + "tokens_seen": 552402944 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672802407221665, + "loss": 1.5711, + "theoretical_loss": 3.9018254068891727, + "tokens_seen": 552468480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727863590772317, + "loss": 1.6135, + "theoretical_loss": 3.901776768846606, + "tokens_seen": 552534016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727703109327984, + "loss": 1.6138, + "theoretical_loss": 3.901728138187739, + "tokens_seen": 552599552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727542627883651, + "loss": 1.3739, + "theoretical_loss": 3.9016795149105747, + "tokens_seen": 552665088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727382146439318, + "loss": 1.3964, + "theoretical_loss": 3.901630899013118, + "tokens_seen": 552730624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727221664994985, + "loss": 1.4253, + "theoretical_loss": 3.901582290493374, + "tokens_seen": 552796160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727061183550652, + "loss": 1.4719, + "theoretical_loss": 3.901533689349348, + "tokens_seen": 552861696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726900702106319, + "loss": 1.4899, + "theoretical_loss": 3.9014850955790474, + "tokens_seen": 552927232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726740220661986, + "loss": 1.3868, + "theoretical_loss": 3.9014365091804795, + "tokens_seen": 552992768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726579739217654, + "loss": 1.3792, + "theoretical_loss": 3.9013879301516523, + "tokens_seen": 553058304 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672641925777332, + "loss": 1.5142, + "theoretical_loss": 3.901339358490575, + "tokens_seen": 553123840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726258776328988, + "loss": 1.3746, + "theoretical_loss": 3.9012907941952575, + "tokens_seen": 553189376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726098294884654, + "loss": 1.4594, + "theoretical_loss": 3.9012422372637103, + "tokens_seen": 553254912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725937813440322, + "loss": 1.5213, + "theoretical_loss": 3.901193687693945, + "tokens_seen": 553320448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725777331995988, + "loss": 1.4227, + "theoretical_loss": 3.9011451454839734, + "tokens_seen": 553385984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725616850551656, + "loss": 1.5528, + "theoretical_loss": 3.901096610631808, + "tokens_seen": 553451520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725456369107322, + "loss": 1.4144, + "theoretical_loss": 3.901048083135464, + "tokens_seen": 553517056 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672529588766299, + "loss": 1.4536, + "theoretical_loss": 3.900999562992954, + "tokens_seen": 553582592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725135406218657, + "loss": 1.4655, + "theoretical_loss": 3.9009510502022944, + "tokens_seen": 553648128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724974924774324, + "loss": 1.5144, + "theoretical_loss": 3.9009025447615002, + "tokens_seen": 553713664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 190485, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3175182342529297, + "objective/train/theoretical_loss": 3.9008540466685897, + "objective/train/tokens_used": 574239200, + "theoretical_loss": 3.9008540466685897, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672481444332999, + "loss": 1.3439, + "theoretical_loss": 3.9008540466685897, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724653961885658, + "loss": 1.4714, + "theoretical_loss": 3.900805555921579, + "tokens_seen": 553844736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724493480441325, + "loss": 1.4252, + "theoretical_loss": 3.9007570725184872, + "tokens_seen": 553910272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724332998996992, + "loss": 1.4653, + "theoretical_loss": 3.900708596457333, + "tokens_seen": 553975808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724172517552659, + "loss": 1.4083, + "theoretical_loss": 3.900660127736136, + "tokens_seen": 554041344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724012036108326, + "loss": 1.4722, + "theoretical_loss": 3.9006116663529173, + "tokens_seen": 554106880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723851554663993, + "loss": 1.476, + "theoretical_loss": 3.9005632123056984, + "tokens_seen": 554172416 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672369107321966, + "loss": 1.5502, + "theoretical_loss": 3.9005147655925008, + "tokens_seen": 554237952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723530591775327, + "loss": 1.4592, + "theoretical_loss": 3.9004663262113475, + "tokens_seen": 554303488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723370110330994, + "loss": 1.5817, + "theoretical_loss": 3.900417894160263, + "tokens_seen": 554369024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723209628886661, + "loss": 1.4163, + "theoretical_loss": 3.900369469437271, + "tokens_seen": 554434560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723049147442328, + "loss": 1.4168, + "theoretical_loss": 3.900321052040397, + "tokens_seen": 554500096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722888665997995, + "loss": 1.3777, + "theoretical_loss": 3.900272641967666, + "tokens_seen": 554565632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722728184553661, + "loss": 1.4316, + "theoretical_loss": 3.900224239217106, + "tokens_seen": 554631168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722567703109329, + "loss": 1.4393, + "theoretical_loss": 3.9001758437867435, + "tokens_seen": 554696704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722407221664995, + "loss": 1.5377, + "theoretical_loss": 3.900127455674608, + "tokens_seen": 554762240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722246740220663, + "loss": 1.371, + "theoretical_loss": 3.9000790748787275, + "tokens_seen": 554827776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722086258776329, + "loss": 1.4876, + "theoretical_loss": 3.9000307013971325, + "tokens_seen": 554893312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721925777331997, + "loss": 1.5017, + "theoretical_loss": 3.899982335227852, + "tokens_seen": 554958848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721765295887663, + "loss": 1.4599, + "theoretical_loss": 3.89993397636892, + "tokens_seen": 555024384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721604814443331, + "loss": 1.5532, + "theoretical_loss": 3.899885624818366, + "tokens_seen": 555089920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721444332998997, + "loss": 1.4082, + "theoretical_loss": 3.8998372805742245, + "tokens_seen": 555155456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721283851554664, + "loss": 1.4126, + "theoretical_loss": 3.8997889436345283, + "tokens_seen": 555220992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721123370110331, + "loss": 1.5716, + "theoretical_loss": 3.899740613997312, + "tokens_seen": 555286528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720962888665998, + "loss": 1.5568, + "theoretical_loss": 3.8996922916606103, + "tokens_seen": 555352064 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720802407221665, + "loss": 1.4742, + "theoretical_loss": 3.89964397662246, + "tokens_seen": 555417600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720641925777332, + "loss": 1.4739, + "theoretical_loss": 3.899595668880897, + "tokens_seen": 555483136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720481444332999, + "loss": 1.418, + "theoretical_loss": 3.899547368433959, + "tokens_seen": 555548672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720320962888666, + "loss": 1.4476, + "theoretical_loss": 3.899499075279684, + "tokens_seen": 555614208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720160481444333, + "loss": 1.4526, + "theoretical_loss": 3.899450789416111, + "tokens_seen": 555679744 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672, + "loss": 1.522, + "theoretical_loss": 3.8994025108412793, + "tokens_seen": 555745280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719839518555667, + "loss": 1.4267, + "theoretical_loss": 3.8993542395532304, + "tokens_seen": 555810816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719679037111334, + "loss": 1.5176, + "theoretical_loss": 3.8993059755500044, + "tokens_seen": 555876352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719518555667001, + "loss": 1.4775, + "theoretical_loss": 3.8992577188296433, + "tokens_seen": 555941888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719358074222668, + "loss": 1.5034, + "theoretical_loss": 3.8992094693901906, + "tokens_seen": 556007424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719197592778335, + "loss": 1.428, + "theoretical_loss": 3.8991612272296896, + "tokens_seen": 556072960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719037111334002, + "loss": 1.4193, + "theoretical_loss": 3.899112992346184, + "tokens_seen": 556138496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718876629889669, + "loss": 1.4228, + "theoretical_loss": 3.899064764737719, + "tokens_seen": 556204032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718716148445336, + "loss": 1.4621, + "theoretical_loss": 3.8990165444023406, + "tokens_seen": 556269568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718555667001003, + "loss": 1.5343, + "theoretical_loss": 3.8989683313380947, + "tokens_seen": 556335104 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671839518555667, + "loss": 1.4186, + "theoretical_loss": 3.8989201255430292, + "tokens_seen": 556400640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718234704112337, + "loss": 1.5301, + "theoretical_loss": 3.8988719270151924, + "tokens_seen": 556466176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718074222668004, + "loss": 1.4505, + "theoretical_loss": 3.8988237357526314, + "tokens_seen": 556531712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717913741223671, + "loss": 1.4219, + "theoretical_loss": 3.8987755517533973, + "tokens_seen": 556597248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717753259779338, + "loss": 1.5398, + "theoretical_loss": 3.8987273750155396, + "tokens_seen": 556662784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717592778335005, + "loss": 1.539, + "theoretical_loss": 3.89867920553711, + "tokens_seen": 556728320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717432296890672, + "loss": 1.4986, + "theoretical_loss": 3.8986310433161595, + "tokens_seen": 556793856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717271815446339, + "loss": 1.4106, + "theoretical_loss": 3.898582888350741, + "tokens_seen": 556859392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717111334002006, + "loss": 1.4857, + "theoretical_loss": 3.8985347406389077, + "tokens_seen": 556924928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716950852557673, + "loss": 1.3531, + "theoretical_loss": 3.898486600178714, + "tokens_seen": 556990464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 191608, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5120327472686768, + "objective/train/theoretical_loss": 3.898438466968214, + "objective/train/tokens_used": 577516000, + "theoretical_loss": 3.898438466968214, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671679037111334, + "loss": 1.4921, + "theoretical_loss": 3.898438466968214, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716629889669007, + "loss": 1.4352, + "theoretical_loss": 3.8983903410054634, + "tokens_seen": 557121536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716469408224674, + "loss": 1.4466, + "theoretical_loss": 3.8983422222885187, + "tokens_seen": 557187072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716308926780341, + "loss": 1.4628, + "theoretical_loss": 3.8982941108154368, + "tokens_seen": 557252608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716148445336008, + "loss": 1.3886, + "theoretical_loss": 3.8982460065842757, + "tokens_seen": 557318144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715987963891675, + "loss": 1.4199, + "theoretical_loss": 3.8981979095930934, + "tokens_seen": 557383680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715827482447342, + "loss": 1.5233, + "theoretical_loss": 3.89814981983995, + "tokens_seen": 557449216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715667001003009, + "loss": 1.39, + "theoretical_loss": 3.8981017373229045, + "tokens_seen": 557514752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715506519558676, + "loss": 1.4756, + "theoretical_loss": 3.8980536620400184, + "tokens_seen": 557580288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715346038114343, + "loss": 1.3975, + "theoretical_loss": 3.8980055939893523, + "tokens_seen": 557645824 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671518555667001, + "loss": 1.4887, + "theoretical_loss": 3.89795753316897, + "tokens_seen": 557711360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715025075225677, + "loss": 1.4431, + "theoretical_loss": 3.8979094795769336, + "tokens_seen": 557776896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714864593781344, + "loss": 1.5638, + "theoretical_loss": 3.8978614332113066, + "tokens_seen": 557842432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714704112337011, + "loss": 1.5003, + "theoretical_loss": 3.8978133940701536, + "tokens_seen": 557907968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714543630892679, + "loss": 1.3973, + "theoretical_loss": 3.8977653621515405, + "tokens_seen": 557973504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714383149448345, + "loss": 1.4663, + "theoretical_loss": 3.8977173374535328, + "tokens_seen": 558039040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714222668004013, + "loss": 1.5457, + "theoretical_loss": 3.8976693199741974, + "tokens_seen": 558104576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714062186559679, + "loss": 1.4807, + "theoretical_loss": 3.8976213097116013, + "tokens_seen": 558170112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713901705115347, + "loss": 1.4396, + "theoretical_loss": 3.8975733066638134, + "tokens_seen": 558235648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713741223671013, + "loss": 1.5034, + "theoretical_loss": 3.897525310828902, + "tokens_seen": 558301184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713580742226681, + "loss": 1.3895, + "theoretical_loss": 3.8974773222049377, + "tokens_seen": 558366720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713420260782347, + "loss": 1.4724, + "theoretical_loss": 3.89742934078999, + "tokens_seen": 558432256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713259779338015, + "loss": 1.3561, + "theoretical_loss": 3.8973813665821306, + "tokens_seen": 558497792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713099297893681, + "loss": 1.5061, + "theoretical_loss": 3.8973333995794315, + "tokens_seen": 558563328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712938816449349, + "loss": 1.4293, + "theoretical_loss": 3.897285439779965, + "tokens_seen": 558628864 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712778335005015, + "loss": 1.3869, + "theoretical_loss": 3.8972374871818047, + "tokens_seen": 558694400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712617853560683, + "loss": 1.5328, + "theoretical_loss": 3.8971895417830247, + "tokens_seen": 558759936 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671245737211635, + "loss": 1.4658, + "theoretical_loss": 3.8971416035817006, + "tokens_seen": 558825472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712296890672017, + "loss": 1.4214, + "theoretical_loss": 3.8970936725759073, + "tokens_seen": 558891008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712136409227683, + "loss": 1.4339, + "theoretical_loss": 3.8970457487637216, + "tokens_seen": 558956544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711975927783351, + "loss": 1.4222, + "theoretical_loss": 3.89699783214322, + "tokens_seen": 559022080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711815446339017, + "loss": 1.5009, + "theoretical_loss": 3.8969499227124804, + "tokens_seen": 559087616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711654964894685, + "loss": 1.3898, + "theoretical_loss": 3.8969020204695823, + "tokens_seen": 559153152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711494483450352, + "loss": 1.5702, + "theoretical_loss": 3.8968541254126046, + "tokens_seen": 559218688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711334002006019, + "loss": 1.4097, + "theoretical_loss": 3.896806237539627, + "tokens_seen": 559284224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711173520561686, + "loss": 1.5051, + "theoretical_loss": 3.8967583568487307, + "tokens_seen": 559349760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711013039117353, + "loss": 1.4303, + "theoretical_loss": 3.8967104833379973, + "tokens_seen": 559415296 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671085255767302, + "loss": 1.5536, + "theoretical_loss": 3.8966626170055085, + "tokens_seen": 559480832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710692076228687, + "loss": 1.4769, + "theoretical_loss": 3.896614757849348, + "tokens_seen": 559546368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710531594784354, + "loss": 1.4104, + "theoretical_loss": 3.8965669058675996, + "tokens_seen": 559611904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710371113340021, + "loss": 1.5694, + "theoretical_loss": 3.8965190610583473, + "tokens_seen": 559677440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710210631895688, + "loss": 1.4871, + "theoretical_loss": 3.896471223419677, + "tokens_seen": 559742976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710050150451355, + "loss": 1.3676, + "theoretical_loss": 3.896423392949674, + "tokens_seen": 559808512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709889669007022, + "loss": 1.4637, + "theoretical_loss": 3.896375569646425, + "tokens_seen": 559874048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709729187562689, + "loss": 1.3481, + "theoretical_loss": 3.896327753508018, + "tokens_seen": 559939584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709568706118356, + "loss": 1.5005, + "theoretical_loss": 3.896279944532541, + "tokens_seen": 560005120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709408224674023, + "loss": 1.5627, + "theoretical_loss": 3.8962321427180826, + "tokens_seen": 560070656 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670924774322969, + "loss": 1.4232, + "theoretical_loss": 3.8961843480627323, + "tokens_seen": 560136192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709087261785357, + "loss": 1.4844, + "theoretical_loss": 3.8961365605645817, + "tokens_seen": 560201728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708926780341024, + "loss": 1.4625, + "theoretical_loss": 3.89608878022172, + "tokens_seen": 560267264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 192944, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4706947803497314, + "objective/train/theoretical_loss": 3.896041007032241, + "objective/train/tokens_used": 580792800, + "theoretical_loss": 3.896041007032241, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708766298896691, + "loss": 1.4728, + "theoretical_loss": 3.896041007032241, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708605817452358, + "loss": 1.5247, + "theoretical_loss": 3.895993240994236, + "tokens_seen": 560398336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708445336008024, + "loss": 1.4447, + "theoretical_loss": 3.895945482105798, + "tokens_seen": 560463872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708284854563692, + "loss": 1.4128, + "theoretical_loss": 3.8958977303650224, + "tokens_seen": 560529408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708124373119358, + "loss": 1.5002, + "theoretical_loss": 3.8958499857700035, + "tokens_seen": 560594944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707963891675026, + "loss": 1.424, + "theoretical_loss": 3.8958022483188364, + "tokens_seen": 560660480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707803410230692, + "loss": 1.5017, + "theoretical_loss": 3.895754518009617, + "tokens_seen": 560726016 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670764292878636, + "loss": 1.4005, + "theoretical_loss": 3.8957067948404434, + "tokens_seen": 560791552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707482447342026, + "loss": 1.5501, + "theoretical_loss": 3.895659078809412, + "tokens_seen": 560857088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707321965897693, + "loss": 1.5037, + "theoretical_loss": 3.895611369914622, + "tokens_seen": 560922624 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670716148445336, + "loss": 1.4869, + "theoretical_loss": 3.895563668154173, + "tokens_seen": 560988160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707001003009027, + "loss": 1.5724, + "theoretical_loss": 3.8955159735261637, + "tokens_seen": 561053696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706840521564694, + "loss": 1.4572, + "theoretical_loss": 3.895468286028695, + "tokens_seen": 561119232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706680040120361, + "loss": 1.3411, + "theoretical_loss": 3.8954206056598695, + "tokens_seen": 561184768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706519558676028, + "loss": 1.4937, + "theoretical_loss": 3.8953729324177875, + "tokens_seen": 561250304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706359077231695, + "loss": 1.436, + "theoretical_loss": 3.895325266300553, + "tokens_seen": 561315840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706198595787362, + "loss": 1.545, + "theoretical_loss": 3.8952776073062685, + "tokens_seen": 561381376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706038114343029, + "loss": 1.404, + "theoretical_loss": 3.895229955433039, + "tokens_seen": 561446912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705877632898696, + "loss": 1.3153, + "theoretical_loss": 3.8951823106789694, + "tokens_seen": 561512448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705717151454363, + "loss": 1.4576, + "theoretical_loss": 3.895134673042165, + "tokens_seen": 561577984 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670555667001003, + "loss": 1.4926, + "theoretical_loss": 3.895087042520733, + "tokens_seen": 561643520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705396188565697, + "loss": 1.4651, + "theoretical_loss": 3.895039419112779, + "tokens_seen": 561709056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705235707121364, + "loss": 1.4288, + "theoretical_loss": 3.8949918028164126, + "tokens_seen": 561774592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705075225677031, + "loss": 1.4602, + "theoretical_loss": 3.8949441936297413, + "tokens_seen": 561840128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704914744232698, + "loss": 1.4691, + "theoretical_loss": 3.894896591550875, + "tokens_seen": 561905664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704754262788365, + "loss": 1.4379, + "theoretical_loss": 3.894848996577924, + "tokens_seen": 561971200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704593781344032, + "loss": 1.5564, + "theoretical_loss": 3.8948014087089975, + "tokens_seen": 562036736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704433299899699, + "loss": 1.414, + "theoretical_loss": 3.8947538279422083, + "tokens_seen": 562102272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704272818455366, + "loss": 1.2931, + "theoretical_loss": 3.8947062542756683, + "tokens_seen": 562167808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704112337011033, + "loss": 1.5324, + "theoretical_loss": 3.8946586877074907, + "tokens_seen": 562233344 + }, + { + "epoch": 0.17, + "learning_rate": 0.00067039518555667, + "loss": 1.5057, + "theoretical_loss": 3.894611128235789, + "tokens_seen": 562298880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703791374122367, + "loss": 1.4521, + "theoretical_loss": 3.894563575858677, + "tokens_seen": 562364416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703630892678034, + "loss": 1.5724, + "theoretical_loss": 3.8945160305742705, + "tokens_seen": 562429952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703470411233701, + "loss": 1.5203, + "theoretical_loss": 3.8944684923806845, + "tokens_seen": 562495488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703309929789368, + "loss": 1.4232, + "theoretical_loss": 3.8944209612760368, + "tokens_seen": 562561024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703149448345035, + "loss": 1.4636, + "theoretical_loss": 3.894373437258443, + "tokens_seen": 562626560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702988966900702, + "loss": 1.454, + "theoretical_loss": 3.8943259203260228, + "tokens_seen": 562692096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702828485456369, + "loss": 1.4675, + "theoretical_loss": 3.8942784104768937, + "tokens_seen": 562757632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702668004012036, + "loss": 1.3461, + "theoretical_loss": 3.8942309077091757, + "tokens_seen": 562823168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702507522567704, + "loss": 1.4379, + "theoretical_loss": 3.894183412020988, + "tokens_seen": 562888704 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670234704112337, + "loss": 1.5108, + "theoretical_loss": 3.894135923410453, + "tokens_seen": 562954240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702186559679038, + "loss": 1.418, + "theoretical_loss": 3.8940884418756907, + "tokens_seen": 563019776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702026078234704, + "loss": 1.491, + "theoretical_loss": 3.894040967414824, + "tokens_seen": 563085312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701865596790372, + "loss": 1.3897, + "theoretical_loss": 3.893993500025976, + "tokens_seen": 563150848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701705115346038, + "loss": 1.53, + "theoretical_loss": 3.893946039707271, + "tokens_seen": 563216384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701544633901706, + "loss": 1.5299, + "theoretical_loss": 3.893898586456832, + "tokens_seen": 563281920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701384152457372, + "loss": 1.3898, + "theoretical_loss": 3.893851140272785, + "tokens_seen": 563347456 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670122367101304, + "loss": 1.4234, + "theoretical_loss": 3.8938037011532556, + "tokens_seen": 563412992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701063189568706, + "loss": 1.506, + "theoretical_loss": 3.893756269096371, + "tokens_seen": 563478528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700902708124374, + "loss": 1.5202, + "theoretical_loss": 3.8937088441002574, + "tokens_seen": 563544064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 193835, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5278165340423584, + "objective/train/theoretical_loss": 3.893661426163044, + "objective/train/tokens_used": 584069600, + "theoretical_loss": 3.893661426163044, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670074222668004, + "loss": 1.5765, + "theoretical_loss": 3.893661426163044, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700581745235708, + "loss": 1.502, + "theoretical_loss": 3.8936140152828584, + "tokens_seen": 563675136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700421263791374, + "loss": 1.4192, + "theoretical_loss": 3.8935666114578305, + "tokens_seen": 563740672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700260782347042, + "loss": 1.4857, + "theoretical_loss": 3.8935192146860906, + "tokens_seen": 563806208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700100300902708, + "loss": 1.5107, + "theoretical_loss": 3.8934718249657694, + "tokens_seen": 563871744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006699939819458376, + "loss": 1.4932, + "theoretical_loss": 3.8934244422949984, + "tokens_seen": 563937280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006699779338014042, + "loss": 1.4328, + "theoretical_loss": 3.89337706667191, + "tokens_seen": 564002816 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669961885656971, + "loss": 1.4154, + "theoretical_loss": 3.8933296980946372, + "tokens_seen": 564068352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006699458375125376, + "loss": 1.4524, + "theoretical_loss": 3.8932823365613136, + "tokens_seen": 564133888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006699297893681044, + "loss": 1.5674, + "theoretical_loss": 3.893234982070074, + "tokens_seen": 564199424 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669913741223671, + "loss": 1.478, + "theoretical_loss": 3.8931876346190517, + "tokens_seen": 564264960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698976930792378, + "loss": 1.48, + "theoretical_loss": 3.8931402942063853, + "tokens_seen": 564330496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698816449348044, + "loss": 1.5694, + "theoretical_loss": 3.89309296083021, + "tokens_seen": 564396032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698655967903712, + "loss": 1.4293, + "theoretical_loss": 3.8930456344886624, + "tokens_seen": 564461568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698495486459378, + "loss": 1.3529, + "theoretical_loss": 3.8929983151798817, + "tokens_seen": 564527104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698335005015046, + "loss": 1.4832, + "theoretical_loss": 3.892951002902006, + "tokens_seen": 564592640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698174523570712, + "loss": 1.5263, + "theoretical_loss": 3.892903697653174, + "tokens_seen": 564658176 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669801404212638, + "loss": 1.4369, + "theoretical_loss": 3.8928563994315266, + "tokens_seen": 564723712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697853560682047, + "loss": 1.3109, + "theoretical_loss": 3.8928091082352045, + "tokens_seen": 564789248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697693079237714, + "loss": 1.3746, + "theoretical_loss": 3.8927618240623496, + "tokens_seen": 564854784 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669753259779338, + "loss": 1.524, + "theoretical_loss": 3.8927145469111033, + "tokens_seen": 564920320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697372116349048, + "loss": 1.4817, + "theoretical_loss": 3.892667276779609, + "tokens_seen": 564985856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697211634904715, + "loss": 1.4604, + "theoretical_loss": 3.89262001366601, + "tokens_seen": 565051392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697051153460382, + "loss": 1.4828, + "theoretical_loss": 3.892572757568451, + "tokens_seen": 565116928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696890672016049, + "loss": 1.419, + "theoretical_loss": 3.892525508485077, + "tokens_seen": 565182464 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696730190571716, + "loss": 1.4302, + "theoretical_loss": 3.892478266414033, + "tokens_seen": 565248000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696569709127383, + "loss": 1.4737, + "theoretical_loss": 3.8924310313534667, + "tokens_seen": 565313536 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669640922768305, + "loss": 1.436, + "theoretical_loss": 3.892383803301524, + "tokens_seen": 565379072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696248746238717, + "loss": 1.385, + "theoretical_loss": 3.8923365822563536, + "tokens_seen": 565444608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696088264794384, + "loss": 1.3827, + "theoretical_loss": 3.8922893682161037, + "tokens_seen": 565510144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695927783350051, + "loss": 1.4165, + "theoretical_loss": 3.892242161178924, + "tokens_seen": 565575680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695767301905718, + "loss": 1.3998, + "theoretical_loss": 3.892194961142964, + "tokens_seen": 565641216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695606820461385, + "loss": 1.4822, + "theoretical_loss": 3.8921477681063736, + "tokens_seen": 565706752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695446339017052, + "loss": 1.5143, + "theoretical_loss": 3.892100582067306, + "tokens_seen": 565772288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695285857572719, + "loss": 1.5652, + "theoretical_loss": 3.8920534030239122, + "tokens_seen": 565837824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695125376128386, + "loss": 1.4798, + "theoretical_loss": 3.892006230974345, + "tokens_seen": 565903360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694964894684053, + "loss": 1.423, + "theoretical_loss": 3.8919590659167578, + "tokens_seen": 565968896 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669480441323972, + "loss": 1.4732, + "theoretical_loss": 3.8919119078493045, + "tokens_seen": 566034432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694643931795387, + "loss": 1.4357, + "theoretical_loss": 3.8918647567701417, + "tokens_seen": 566099968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694483450351053, + "loss": 1.481, + "theoretical_loss": 3.8918176126774227, + "tokens_seen": 566165504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694322968906721, + "loss": 1.513, + "theoretical_loss": 3.891770475569305, + "tokens_seen": 566231040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694162487462387, + "loss": 1.4065, + "theoretical_loss": 3.8917233454439453, + "tokens_seen": 566296576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694002006018055, + "loss": 1.4289, + "theoretical_loss": 3.8916762222995014, + "tokens_seen": 566362112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693841524573721, + "loss": 1.4075, + "theoretical_loss": 3.8916291061341317, + "tokens_seen": 566427648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693681043129389, + "loss": 1.3731, + "theoretical_loss": 3.8915819969459946, + "tokens_seen": 566493184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693520561685055, + "loss": 1.4602, + "theoretical_loss": 3.891534894733251, + "tokens_seen": 566558720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693360080240723, + "loss": 1.4231, + "theoretical_loss": 3.891487799494061, + "tokens_seen": 566624256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693199598796389, + "loss": 1.3516, + "theoretical_loss": 3.891440711226585, + "tokens_seen": 566689792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693039117352056, + "loss": 1.4852, + "theoretical_loss": 3.891393629928986, + "tokens_seen": 566755328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692878635907723, + "loss": 1.5724, + "theoretical_loss": 3.8913465555994255, + "tokens_seen": 566820864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 195104, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5167372226715088, + "objective/train/theoretical_loss": 3.8912994882360676, + "objective/train/tokens_used": 587346400, + "theoretical_loss": 3.8912994882360676, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669271815446339, + "loss": 1.5297, + "theoretical_loss": 3.8912994882360676, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692557673019057, + "loss": 1.4265, + "theoretical_loss": 3.891252427837076, + "tokens_seen": 566951936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692397191574724, + "loss": 1.4251, + "theoretical_loss": 3.8912053744006156, + "tokens_seen": 567017472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692236710130391, + "loss": 1.5477, + "theoretical_loss": 3.8911583279248507, + "tokens_seen": 567083008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692076228686058, + "loss": 1.4827, + "theoretical_loss": 3.8911112884079486, + "tokens_seen": 567148544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691915747241725, + "loss": 1.3542, + "theoretical_loss": 3.8910642558480752, + "tokens_seen": 567214080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691755265797392, + "loss": 1.5154, + "theoretical_loss": 3.8910172302433987, + "tokens_seen": 567279616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691594784353059, + "loss": 1.3885, + "theoretical_loss": 3.8909702115920863, + "tokens_seen": 567345152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691434302908726, + "loss": 1.4874, + "theoretical_loss": 3.890923199892308, + "tokens_seen": 567410688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691273821464393, + "loss": 1.4088, + "theoretical_loss": 3.890876195142232, + "tokens_seen": 567476224 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669111334002006, + "loss": 1.4489, + "theoretical_loss": 3.8908291973400297, + "tokens_seen": 567541760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690952858575727, + "loss": 1.3928, + "theoretical_loss": 3.890782206483871, + "tokens_seen": 567607296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690792377131394, + "loss": 1.4766, + "theoretical_loss": 3.8907352225719283, + "tokens_seen": 567672832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690631895687061, + "loss": 1.3147, + "theoretical_loss": 3.890688245602373, + "tokens_seen": 567738368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690471414242729, + "loss": 1.4016, + "theoretical_loss": 3.890641275573379, + "tokens_seen": 567803904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690310932798395, + "loss": 1.3566, + "theoretical_loss": 3.8905943124831195, + "tokens_seen": 567869440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690150451354063, + "loss": 1.3658, + "theoretical_loss": 3.890547356329769, + "tokens_seen": 567934976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689989969909729, + "loss": 1.4371, + "theoretical_loss": 3.890500407111502, + "tokens_seen": 568000512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689829488465397, + "loss": 1.4648, + "theoretical_loss": 3.8904534648264955, + "tokens_seen": 568066048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689669007021063, + "loss": 1.3354, + "theoretical_loss": 3.8904065294729246, + "tokens_seen": 568131584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689508525576731, + "loss": 1.5189, + "theoretical_loss": 3.890359601048967, + "tokens_seen": 568197120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689348044132397, + "loss": 1.5132, + "theoretical_loss": 3.8903126795528014, + "tokens_seen": 568262656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689187562688065, + "loss": 1.4662, + "theoretical_loss": 3.8902657649826047, + "tokens_seen": 568328192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689027081243731, + "loss": 1.4783, + "theoretical_loss": 3.890218857336557, + "tokens_seen": 568393728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688866599799399, + "loss": 1.3859, + "theoretical_loss": 3.890171956612838, + "tokens_seen": 568459264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688706118355065, + "loss": 1.4601, + "theoretical_loss": 3.890125062809628, + "tokens_seen": 568524800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688545636910733, + "loss": 1.4535, + "theoretical_loss": 3.8900781759251086, + "tokens_seen": 568590336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688385155466399, + "loss": 1.3373, + "theoretical_loss": 3.8900312959574617, + "tokens_seen": 568655872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688224674022067, + "loss": 1.4347, + "theoretical_loss": 3.88998442290487, + "tokens_seen": 568721408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688064192577733, + "loss": 1.4231, + "theoretical_loss": 3.889937556765517, + "tokens_seen": 568786944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687903711133401, + "loss": 1.3824, + "theoretical_loss": 3.8898906975375858, + "tokens_seen": 568852480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687743229689067, + "loss": 1.4538, + "theoretical_loss": 3.8898438452192616, + "tokens_seen": 568918016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687582748244735, + "loss": 1.4741, + "theoretical_loss": 3.8897969998087305, + "tokens_seen": 568983552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687422266800401, + "loss": 1.4618, + "theoretical_loss": 3.8897501613041783, + "tokens_seen": 569049088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687261785356069, + "loss": 1.4505, + "theoretical_loss": 3.8897033297037904, + "tokens_seen": 569114624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687101303911735, + "loss": 1.429, + "theoretical_loss": 3.889656505005756, + "tokens_seen": 569180160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686940822467403, + "loss": 1.5221, + "theoretical_loss": 3.889609687208262, + "tokens_seen": 569245696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686780341023069, + "loss": 1.4228, + "theoretical_loss": 3.8895628763094976, + "tokens_seen": 569311232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686619859578737, + "loss": 1.47, + "theoretical_loss": 3.8895160723076527, + "tokens_seen": 569376768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686459378134403, + "loss": 1.4073, + "theoretical_loss": 3.889469275200917, + "tokens_seen": 569442304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686298896690071, + "loss": 1.4508, + "theoretical_loss": 3.889422484987481, + "tokens_seen": 569507840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686138415245737, + "loss": 1.3578, + "theoretical_loss": 3.889375701665537, + "tokens_seen": 569573376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685977933801405, + "loss": 1.4829, + "theoretical_loss": 3.8893289252332766, + "tokens_seen": 569638912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685817452357071, + "loss": 1.4044, + "theoretical_loss": 3.8892821556888935, + "tokens_seen": 569704448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685656970912739, + "loss": 1.4279, + "theoretical_loss": 3.8892353930305807, + "tokens_seen": 569769984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685496489468405, + "loss": 1.6804, + "theoretical_loss": 3.8891886372565327, + "tokens_seen": 569835520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685336008024073, + "loss": 1.4291, + "theoretical_loss": 3.889141888364944, + "tokens_seen": 569901056 + }, + { + "epoch": 0.17, + "learning_rate": 0.000668517552657974, + "loss": 1.4354, + "theoretical_loss": 3.8890951463540104, + "tokens_seen": 569966592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685015045135407, + "loss": 1.4495, + "theoretical_loss": 3.8890484112219283, + "tokens_seen": 570032128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684854563691073, + "loss": 1.4318, + "theoretical_loss": 3.889001682966895, + "tokens_seen": 570097664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 195990, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5162267684936523, + "objective/train/theoretical_loss": 3.8889549615871077, + "objective/train/tokens_used": 590623200, + "theoretical_loss": 3.8889549615871077, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684694082246741, + "loss": 1.4552, + "theoretical_loss": 3.8889549615871077, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684533600802407, + "loss": 1.4672, + "theoretical_loss": 3.8889082470807645, + "tokens_seen": 570228736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684373119358075, + "loss": 1.4537, + "theoretical_loss": 3.888861539446066, + "tokens_seen": 570294272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684212637913742, + "loss": 1.4429, + "theoretical_loss": 3.88881483868121, + "tokens_seen": 570359808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684052156469409, + "loss": 1.4702, + "theoretical_loss": 3.8887681447843976, + "tokens_seen": 570425344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683891675025076, + "loss": 1.4375, + "theoretical_loss": 3.88872145775383, + "tokens_seen": 570490880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683731193580743, + "loss": 1.4581, + "theoretical_loss": 3.8886747775877084, + "tokens_seen": 570556416 + }, + { + "epoch": 0.17, + "learning_rate": 0.000668357071213641, + "loss": 1.3596, + "theoretical_loss": 3.888628104284236, + "tokens_seen": 570621952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683410230692077, + "loss": 1.5164, + "theoretical_loss": 3.888581437841615, + "tokens_seen": 570687488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683249749247744, + "loss": 1.4334, + "theoretical_loss": 3.8885347782580504, + "tokens_seen": 570753024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683089267803411, + "loss": 1.4766, + "theoretical_loss": 3.8884881255317456, + "tokens_seen": 570818560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682928786359078, + "loss": 1.4858, + "theoretical_loss": 3.888441479660906, + "tokens_seen": 570884096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682768304914745, + "loss": 1.532, + "theoretical_loss": 3.8883948406437376, + "tokens_seen": 570949632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682607823470412, + "loss": 1.4114, + "theoretical_loss": 3.8883482084784466, + "tokens_seen": 571015168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682447342026079, + "loss": 1.4769, + "theoretical_loss": 3.8883015831632406, + "tokens_seen": 571080704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682286860581746, + "loss": 1.5367, + "theoretical_loss": 3.8882549646963263, + "tokens_seen": 571146240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682126379137413, + "loss": 1.3905, + "theoretical_loss": 3.888208353075914, + "tokens_seen": 571211776 + }, + { + "epoch": 0.17, + "learning_rate": 0.000668196589769308, + "loss": 1.4432, + "theoretical_loss": 3.8881617483002113, + "tokens_seen": 571277312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681805416248747, + "loss": 1.4893, + "theoretical_loss": 3.8881151503674283, + "tokens_seen": 571342848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681644934804414, + "loss": 1.375, + "theoretical_loss": 3.888068559275776, + "tokens_seen": 571408384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681484453360081, + "loss": 1.4334, + "theoretical_loss": 3.8880219750234657, + "tokens_seen": 571473920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681323971915748, + "loss": 1.5079, + "theoretical_loss": 3.8879753976087086, + "tokens_seen": 571539456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681163490471415, + "loss": 1.4816, + "theoretical_loss": 3.8879288270297176, + "tokens_seen": 571604992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681003009027082, + "loss": 1.4003, + "theoretical_loss": 3.8878822632847063, + "tokens_seen": 571670528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680842527582749, + "loss": 1.5134, + "theoretical_loss": 3.887835706371888, + "tokens_seen": 571736064 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680682046138416, + "loss": 1.4662, + "theoretical_loss": 3.887789156289477, + "tokens_seen": 571801600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680521564694082, + "loss": 1.4893, + "theoretical_loss": 3.887742613035689, + "tokens_seen": 571867136 + }, + { + "epoch": 0.17, + "learning_rate": 0.000668036108324975, + "loss": 1.5036, + "theoretical_loss": 3.8876960766087407, + "tokens_seen": 571932672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680200601805416, + "loss": 1.4776, + "theoretical_loss": 3.8876495470068475, + "tokens_seen": 571998208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680040120361084, + "loss": 1.4827, + "theoretical_loss": 3.8876030242282265, + "tokens_seen": 572063744 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667987963891675, + "loss": 1.3667, + "theoretical_loss": 3.8875565082710963, + "tokens_seen": 572129280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679719157472418, + "loss": 1.436, + "theoretical_loss": 3.887509999133675, + "tokens_seen": 572194816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679558676028084, + "loss": 1.4718, + "theoretical_loss": 3.887463496814182, + "tokens_seen": 572260352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679398194583752, + "loss": 1.5012, + "theoretical_loss": 3.887417001310838, + "tokens_seen": 572325888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679237713139418, + "loss": 1.5284, + "theoretical_loss": 3.887370512621862, + "tokens_seen": 572391424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679077231695085, + "loss": 1.3893, + "theoretical_loss": 3.887324030745477, + "tokens_seen": 572456960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678916750250752, + "loss": 1.4374, + "theoretical_loss": 3.887277555679903, + "tokens_seen": 572522496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678756268806419, + "loss": 1.3893, + "theoretical_loss": 3.8872310874233644, + "tokens_seen": 572588032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678595787362086, + "loss": 1.4841, + "theoretical_loss": 3.887184625974083, + "tokens_seen": 572653568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678435305917754, + "loss": 1.4397, + "theoretical_loss": 3.8871381713302835, + "tokens_seen": 572719104 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667827482447342, + "loss": 1.4047, + "theoretical_loss": 3.8870917234901907, + "tokens_seen": 572784640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678114343029088, + "loss": 1.4294, + "theoretical_loss": 3.8870452824520294, + "tokens_seen": 572850176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677953861584754, + "loss": 1.6088, + "theoretical_loss": 3.8869988482140254, + "tokens_seen": 572915712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677793380140422, + "loss": 1.5765, + "theoretical_loss": 3.886952420774406, + "tokens_seen": 572981248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677632898696088, + "loss": 1.4694, + "theoretical_loss": 3.8869060001313973, + "tokens_seen": 573046784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677472417251756, + "loss": 1.4458, + "theoretical_loss": 3.8868595862832285, + "tokens_seen": 573112320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677311935807422, + "loss": 1.4668, + "theoretical_loss": 3.886813179228127, + "tokens_seen": 573177856 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667715145436309, + "loss": 1.3994, + "theoretical_loss": 3.886766778964323, + "tokens_seen": 573243392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676990972918756, + "loss": 1.4859, + "theoretical_loss": 3.8867203854900456, + "tokens_seen": 573308928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676830491474424, + "loss": 1.3269, + "theoretical_loss": 3.886673998803526, + "tokens_seen": 573374464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 197322, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.453871488571167, + "objective/train/theoretical_loss": 3.886627618902995, + "objective/train/tokens_used": 593900000, + "theoretical_loss": 3.886627618902995, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667667001003009, + "loss": 1.5145, + "theoretical_loss": 3.886627618902995, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676509528585758, + "loss": 1.4646, + "theoretical_loss": 3.886581245786685, + "tokens_seen": 573505536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676349047141424, + "loss": 1.252, + "theoretical_loss": 3.8865348794528276, + "tokens_seen": 573571072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676188565697092, + "loss": 1.4531, + "theoretical_loss": 3.8864885198996575, + "tokens_seen": 573636608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676028084252758, + "loss": 1.4677, + "theoretical_loss": 3.8864421671254066, + "tokens_seen": 573702144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006675867602808426, + "loss": 1.4749, + "theoretical_loss": 3.8863958211283114, + "tokens_seen": 573767680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006675707121364092, + "loss": 1.471, + "theoretical_loss": 3.8863494819066062, + "tokens_seen": 573833216 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667554663991976, + "loss": 1.5011, + "theoretical_loss": 3.8863031494585267, + "tokens_seen": 573898752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006675386158475426, + "loss": 1.5202, + "theoretical_loss": 3.8862568237823103, + "tokens_seen": 573964288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006675225677031094, + "loss": 1.4638, + "theoretical_loss": 3.8862105048761926, + "tokens_seen": 574029824 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667506519558676, + "loss": 1.5591, + "theoretical_loss": 3.8861641927384127, + "tokens_seen": 574095360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674904714142428, + "loss": 1.3434, + "theoretical_loss": 3.8861178873672095, + "tokens_seen": 574160896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674744232698094, + "loss": 1.4476, + "theoretical_loss": 3.886071588760821, + "tokens_seen": 574226432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674583751253762, + "loss": 1.4522, + "theoretical_loss": 3.886025296917487, + "tokens_seen": 574291968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674423269809428, + "loss": 1.4525, + "theoretical_loss": 3.8859790118354494, + "tokens_seen": 574357504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674262788365096, + "loss": 1.3981, + "theoretical_loss": 3.8859327335129477, + "tokens_seen": 574423040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674102306920762, + "loss": 1.4895, + "theoretical_loss": 3.8858864619482247, + "tokens_seen": 574488576 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667394182547643, + "loss": 1.4367, + "theoretical_loss": 3.8858401971395224, + "tokens_seen": 574554112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006673781344032096, + "loss": 1.4269, + "theoretical_loss": 3.885793939085085, + "tokens_seen": 574619648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006673620862587764, + "loss": 1.4373, + "theoretical_loss": 3.8857476877831543, + "tokens_seen": 574685184 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667346038114343, + "loss": 1.4741, + "theoretical_loss": 3.8857014432319765, + "tokens_seen": 574750720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006673299899699098, + "loss": 1.5079, + "theoretical_loss": 3.8856552054297957, + "tokens_seen": 574816256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006673139418254764, + "loss": 1.5025, + "theoretical_loss": 3.8856089743748576, + "tokens_seen": 574881792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672978936810432, + "loss": 1.3655, + "theoretical_loss": 3.885562750065409, + "tokens_seen": 574947328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672818455366098, + "loss": 1.6031, + "theoretical_loss": 3.8855165324996976, + "tokens_seen": 575012864 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672657973921766, + "loss": 1.432, + "theoretical_loss": 3.88547032167597, + "tokens_seen": 575078400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672497492477432, + "loss": 1.5351, + "theoretical_loss": 3.885424117592475, + "tokens_seen": 575143936 + }, + { + "epoch": 0.17, + "learning_rate": 0.00066723370110331, + "loss": 1.4087, + "theoretical_loss": 3.885377920247461, + "tokens_seen": 575209472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672176529588766, + "loss": 1.5188, + "theoretical_loss": 3.885331729639179, + "tokens_seen": 575275008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672016048144434, + "loss": 1.5046, + "theoretical_loss": 3.8852855457658784, + "tokens_seen": 575340544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00066718555667001, + "loss": 1.4645, + "theoretical_loss": 3.88523936862581, + "tokens_seen": 575406080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671695085255768, + "loss": 1.4773, + "theoretical_loss": 3.885193198217226, + "tokens_seen": 575471616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671534603811434, + "loss": 1.5131, + "theoretical_loss": 3.8851470345383783, + "tokens_seen": 575537152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671374122367102, + "loss": 1.4086, + "theoretical_loss": 3.8851008775875204, + "tokens_seen": 575602688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671213640922768, + "loss": 1.4833, + "theoretical_loss": 3.8850547273629052, + "tokens_seen": 575668224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671053159478436, + "loss": 1.3597, + "theoretical_loss": 3.8850085838627875, + "tokens_seen": 575733760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006670892678034102, + "loss": 1.4577, + "theoretical_loss": 3.8849624470854223, + "tokens_seen": 575799296 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667073219658977, + "loss": 1.3629, + "theoretical_loss": 3.8849163170290644, + "tokens_seen": 575864832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006670571715145436, + "loss": 1.3028, + "theoretical_loss": 3.8848701936919703, + "tokens_seen": 575930368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006670411233701104, + "loss": 1.54, + "theoretical_loss": 3.8848240770723974, + "tokens_seen": 575995904 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667025075225677, + "loss": 1.4641, + "theoretical_loss": 3.8847779671686022, + "tokens_seen": 576061440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006670090270812438, + "loss": 1.403, + "theoretical_loss": 3.8847318639788435, + "tokens_seen": 576126976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669929789368105, + "loss": 1.4338, + "theoretical_loss": 3.8846857675013804, + "tokens_seen": 576192512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669769307923772, + "loss": 1.5129, + "theoretical_loss": 3.884639677734471, + "tokens_seen": 576258048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669608826479439, + "loss": 1.3515, + "theoretical_loss": 3.884593594676377, + "tokens_seen": 576323584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669448345035106, + "loss": 1.3883, + "theoretical_loss": 3.884547518325359, + "tokens_seen": 576389120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669287863590773, + "loss": 1.4969, + "theoretical_loss": 3.8845014486796776, + "tokens_seen": 576454656 + }, + { + "epoch": 0.17, + "learning_rate": 0.000666912738214644, + "loss": 1.3977, + "theoretical_loss": 3.8844553857375947, + "tokens_seen": 576520192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668966900702107, + "loss": 1.3918, + "theoretical_loss": 3.884409329497374, + "tokens_seen": 576585728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668806419257774, + "loss": 1.3791, + "theoretical_loss": 3.884363279957278, + "tokens_seen": 576651264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 198260, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4905140399932861, + "objective/train/theoretical_loss": 3.884317237115571, + "objective/train/tokens_used": 597176800, + "theoretical_loss": 3.884317237115571, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668645937813441, + "loss": 1.4027, + "theoretical_loss": 3.884317237115571, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668485456369108, + "loss": 1.4067, + "theoretical_loss": 3.884271200970517, + "tokens_seen": 576782336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668324974924775, + "loss": 1.5032, + "theoretical_loss": 3.8842251715203826, + "tokens_seen": 576847872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668164493480442, + "loss": 1.4525, + "theoretical_loss": 3.8841791487634327, + "tokens_seen": 576913408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668004012036109, + "loss": 1.5495, + "theoretical_loss": 3.8841331326979347, + "tokens_seen": 576978944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667843530591776, + "loss": 1.4502, + "theoretical_loss": 3.8840871233221548, + "tokens_seen": 577044480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667683049147443, + "loss": 1.49, + "theoretical_loss": 3.8840411206343615, + "tokens_seen": 577110016 + }, + { + "epoch": 0.17, + "learning_rate": 0.000666752256770311, + "loss": 1.4969, + "theoretical_loss": 3.883995124632823, + "tokens_seen": 577175552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667362086258777, + "loss": 1.495, + "theoretical_loss": 3.8839491353158087, + "tokens_seen": 577241088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667201604814444, + "loss": 1.5002, + "theoretical_loss": 3.8839031526815884, + "tokens_seen": 577306624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667041123370111, + "loss": 1.4853, + "theoretical_loss": 3.8838571767284327, + "tokens_seen": 577372160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006666880641925778, + "loss": 1.4603, + "theoretical_loss": 3.883811207454612, + "tokens_seen": 577437696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666720160481445, + "loss": 1.3529, + "theoretical_loss": 3.883765244858399, + "tokens_seen": 577503232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666559679037111, + "loss": 1.4831, + "theoretical_loss": 3.8837192889380647, + "tokens_seen": 577568768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666399197592779, + "loss": 1.4656, + "theoretical_loss": 3.8836733396918834, + "tokens_seen": 577634304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666238716148445, + "loss": 1.4363, + "theoretical_loss": 3.8836273971181283, + "tokens_seen": 577699840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666078234704113, + "loss": 1.44, + "theoretical_loss": 3.883581461215074, + "tokens_seen": 577765376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665917753259779, + "loss": 1.4905, + "theoretical_loss": 3.8835355319809954, + "tokens_seen": 577830912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665757271815447, + "loss": 1.5494, + "theoretical_loss": 3.883489609414167, + "tokens_seen": 577896448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665596790371113, + "loss": 1.4153, + "theoretical_loss": 3.8834436935128664, + "tokens_seen": 577961984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665436308926781, + "loss": 1.3772, + "theoretical_loss": 3.8833977842753704, + "tokens_seen": 578027520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665275827482447, + "loss": 1.4353, + "theoretical_loss": 3.883351881699956, + "tokens_seen": 578093056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665115346038115, + "loss": 1.5437, + "theoretical_loss": 3.883305985784901, + "tokens_seen": 578158592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664954864593781, + "loss": 1.4388, + "theoretical_loss": 3.8832600965284847, + "tokens_seen": 578224128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664794383149449, + "loss": 1.4035, + "theoretical_loss": 3.8832142139289862, + "tokens_seen": 578289664 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664633901705115, + "loss": 1.614, + "theoretical_loss": 3.883168337984686, + "tokens_seen": 578355200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664473420260783, + "loss": 1.4377, + "theoretical_loss": 3.883122468693865, + "tokens_seen": 578420736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664312938816449, + "loss": 1.4435, + "theoretical_loss": 3.8830766060548036, + "tokens_seen": 578486272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664152457372117, + "loss": 1.4986, + "theoretical_loss": 3.8830307500657844, + "tokens_seen": 578551808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663991975927783, + "loss": 1.37, + "theoretical_loss": 3.8829849007250905, + "tokens_seen": 578617344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663831494483451, + "loss": 1.489, + "theoretical_loss": 3.8829390580310044, + "tokens_seen": 578682880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663671013039117, + "loss": 1.4244, + "theoretical_loss": 3.88289322198181, + "tokens_seen": 578748416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663510531594785, + "loss": 1.4809, + "theoretical_loss": 3.8828473925757923, + "tokens_seen": 578813952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663350050150451, + "loss": 1.5996, + "theoretical_loss": 3.882801569811236, + "tokens_seen": 578879488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663189568706119, + "loss": 1.4475, + "theoretical_loss": 3.8827557536864274, + "tokens_seen": 578945024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663029087261785, + "loss": 1.5018, + "theoretical_loss": 3.882709944199653, + "tokens_seen": 579010560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662868605817453, + "loss": 1.5429, + "theoretical_loss": 3.8826641413491987, + "tokens_seen": 579076096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662708124373119, + "loss": 1.4644, + "theoretical_loss": 3.882618345133354, + "tokens_seen": 579141632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662547642928787, + "loss": 1.4266, + "theoretical_loss": 3.882572555550406, + "tokens_seen": 579207168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662387161484453, + "loss": 1.5529, + "theoretical_loss": 3.882526772598644, + "tokens_seen": 579272704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662226680040121, + "loss": 1.4612, + "theoretical_loss": 3.8824809962763576, + "tokens_seen": 579338240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662066198595787, + "loss": 1.4295, + "theoretical_loss": 3.882435226581837, + "tokens_seen": 579403776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661905717151455, + "loss": 1.3675, + "theoretical_loss": 3.8823894635133738, + "tokens_seen": 579469312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661745235707121, + "loss": 1.4033, + "theoretical_loss": 3.8823437070692584, + "tokens_seen": 579534848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661584754262789, + "loss": 1.3765, + "theoretical_loss": 3.882297957247784, + "tokens_seen": 579600384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661424272818455, + "loss": 1.4589, + "theoretical_loss": 3.8822522140472424, + "tokens_seen": 579665920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661263791374123, + "loss": 1.4647, + "theoretical_loss": 3.882206477465928, + "tokens_seen": 579731456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661103309929789, + "loss": 1.3975, + "theoretical_loss": 3.882160747502134, + "tokens_seen": 579796992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660942828485457, + "loss": 1.5154, + "theoretical_loss": 3.8821150241541558, + "tokens_seen": 579862528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660782347041123, + "loss": 1.5099, + "theoretical_loss": 3.8820693074202888, + "tokens_seen": 579928064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 199439, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4916222095489502, + "objective/train/theoretical_loss": 3.8820235972988275, + "objective/train/tokens_used": 600453600, + "theoretical_loss": 3.8820235972988275, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660621865596791, + "loss": 1.4725, + "theoretical_loss": 3.8820235972988275, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660461384152457, + "loss": 1.5312, + "theoretical_loss": 3.8819778937880702, + "tokens_seen": 580059136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660300902708125, + "loss": 1.481, + "theoretical_loss": 3.8819321968863134, + "tokens_seen": 580124672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660140421263791, + "loss": 1.4392, + "theoretical_loss": 3.8818865065918553, + "tokens_seen": 580190208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659979939819459, + "loss": 1.282, + "theoretical_loss": 3.8818408229029937, + "tokens_seen": 580255744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659819458375125, + "loss": 1.43, + "theoretical_loss": 3.881795145818028, + "tokens_seen": 580321280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659658976930793, + "loss": 1.5045, + "theoretical_loss": 3.8817494753352584, + "tokens_seen": 580386816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659498495486459, + "loss": 1.3841, + "theoretical_loss": 3.881703811452985, + "tokens_seen": 580452352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659338014042127, + "loss": 1.4778, + "theoretical_loss": 3.8816581541695085, + "tokens_seen": 580517888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659177532597793, + "loss": 1.4307, + "theoretical_loss": 3.8816125034831312, + "tokens_seen": 580583424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659017051153461, + "loss": 1.3677, + "theoretical_loss": 3.881566859392154, + "tokens_seen": 580648960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658856569709127, + "loss": 1.4426, + "theoretical_loss": 3.8815212218948814, + "tokens_seen": 580714496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658696088264795, + "loss": 1.3904, + "theoretical_loss": 3.8814755909896164, + "tokens_seen": 580780032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658535606820461, + "loss": 1.4756, + "theoretical_loss": 3.881429966674663, + "tokens_seen": 580845568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658375125376129, + "loss": 1.4131, + "theoretical_loss": 3.881384348948325, + "tokens_seen": 580911104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658214643931795, + "loss": 1.3903, + "theoretical_loss": 3.8813387378089095, + "tokens_seen": 580976640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658054162487463, + "loss": 1.3678, + "theoretical_loss": 3.8812931332547214, + "tokens_seen": 581042176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657893681043129, + "loss": 1.4463, + "theoretical_loss": 3.881247535284068, + "tokens_seen": 581107712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657733199598797, + "loss": 1.3848, + "theoretical_loss": 3.881201943895256, + "tokens_seen": 581173248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657572718154463, + "loss": 1.4144, + "theoretical_loss": 3.8811563590865936, + "tokens_seen": 581238784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657412236710131, + "loss": 1.304, + "theoretical_loss": 3.8811107808563894, + "tokens_seen": 581304320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657251755265797, + "loss": 1.3599, + "theoretical_loss": 3.8810652092029523, + "tokens_seen": 581369856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657091273821465, + "loss": 1.4867, + "theoretical_loss": 3.881019644124593, + "tokens_seen": 581435392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656930792377131, + "loss": 1.3985, + "theoretical_loss": 3.8809740856196204, + "tokens_seen": 581500928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656770310932799, + "loss": 1.4878, + "theoretical_loss": 3.880928533686346, + "tokens_seen": 581566464 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656609829488466, + "loss": 1.5318, + "theoretical_loss": 3.8808829883230826, + "tokens_seen": 581632000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656449348044133, + "loss": 1.4466, + "theoretical_loss": 3.880837449528141, + "tokens_seen": 581697536 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066562888665998, + "loss": 1.4171, + "theoretical_loss": 3.8807919172998346, + "tokens_seen": 581763072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656128385155467, + "loss": 1.3872, + "theoretical_loss": 3.8807463916364773, + "tokens_seen": 581828608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655967903711134, + "loss": 1.508, + "theoretical_loss": 3.8807008725363836, + "tokens_seen": 581894144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655807422266801, + "loss": 1.506, + "theoretical_loss": 3.880655359997867, + "tokens_seen": 581959680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655646940822468, + "loss": 1.2942, + "theoretical_loss": 3.8806098540192435, + "tokens_seen": 582025216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655486459378135, + "loss": 1.5053, + "theoretical_loss": 3.880564354598829, + "tokens_seen": 582090752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655325977933802, + "loss": 1.4198, + "theoretical_loss": 3.8805188617349406, + "tokens_seen": 582156288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655165496489469, + "loss": 1.5112, + "theoretical_loss": 3.880473375425895, + "tokens_seen": 582221824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655005015045136, + "loss": 1.468, + "theoretical_loss": 3.880427895670011, + "tokens_seen": 582287360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654844533600803, + "loss": 1.4688, + "theoretical_loss": 3.8803824224656056, + "tokens_seen": 582352896 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665468405215647, + "loss": 1.3934, + "theoretical_loss": 3.880336955810999, + "tokens_seen": 582418432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654523570712137, + "loss": 1.5792, + "theoretical_loss": 3.880291495704511, + "tokens_seen": 582483968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654363089267805, + "loss": 1.3663, + "theoretical_loss": 3.880246042144461, + "tokens_seen": 582549504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654202607823471, + "loss": 1.4836, + "theoretical_loss": 3.880200595129171, + "tokens_seen": 582615040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654042126379139, + "loss": 1.4792, + "theoretical_loss": 3.880155154656962, + "tokens_seen": 582680576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653881644934805, + "loss": 1.3691, + "theoretical_loss": 3.880109720726157, + "tokens_seen": 582746112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653721163490472, + "loss": 1.4147, + "theoretical_loss": 3.880064293335078, + "tokens_seen": 582811648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653560682046139, + "loss": 1.422, + "theoretical_loss": 3.880018872482049, + "tokens_seen": 582877184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653400200601806, + "loss": 1.4849, + "theoretical_loss": 3.8799734581653933, + "tokens_seen": 582942720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653239719157473, + "loss": 1.5962, + "theoretical_loss": 3.8799280503834366, + "tokens_seen": 583008256 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665307923771314, + "loss": 1.4192, + "theoretical_loss": 3.879882649134503, + "tokens_seen": 583073792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652918756268807, + "loss": 1.4484, + "theoretical_loss": 3.8798372544169197, + "tokens_seen": 583139328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652758274824474, + "loss": 1.4309, + "theoretical_loss": 3.879791866229013, + "tokens_seen": 583204864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 200344, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4507158994674683, + "objective/train/theoretical_loss": 3.879746484569109, + "objective/train/tokens_used": 603730400, + "theoretical_loss": 3.879746484569109, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665259779338014, + "loss": 1.5196, + "theoretical_loss": 3.879746484569109, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652437311935808, + "loss": 1.4805, + "theoretical_loss": 3.879701109435537, + "tokens_seen": 583335936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652276830491474, + "loss": 1.4719, + "theoretical_loss": 3.8796557408266237, + "tokens_seen": 583401472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652116349047142, + "loss": 1.5439, + "theoretical_loss": 3.8796103787406997, + "tokens_seen": 583467008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651955867602808, + "loss": 1.4825, + "theoretical_loss": 3.8795650231760934, + "tokens_seen": 583532544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651795386158476, + "loss": 1.4274, + "theoretical_loss": 3.879519674131136, + "tokens_seen": 583598080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651634904714142, + "loss": 1.3955, + "theoretical_loss": 3.8794743316041584, + "tokens_seen": 583663616 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665147442326981, + "loss": 1.4537, + "theoretical_loss": 3.879428995593491, + "tokens_seen": 583729152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651313941825476, + "loss": 1.468, + "theoretical_loss": 3.8793836660974668, + "tokens_seen": 583794688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651153460381144, + "loss": 1.552, + "theoretical_loss": 3.8793383431144175, + "tokens_seen": 583860224 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665099297893681, + "loss": 1.4288, + "theoretical_loss": 3.879293026642678, + "tokens_seen": 583925760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650832497492478, + "loss": 1.3514, + "theoretical_loss": 3.879247716680581, + "tokens_seen": 583991296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650672016048144, + "loss": 1.4898, + "theoretical_loss": 3.8792024132264613, + "tokens_seen": 584056832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650511534603812, + "loss": 1.5033, + "theoretical_loss": 3.8791571162786544, + "tokens_seen": 584122368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650351053159478, + "loss": 1.3391, + "theoretical_loss": 3.8791118258354955, + "tokens_seen": 584187904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650190571715146, + "loss": 1.3986, + "theoretical_loss": 3.8790665418953214, + "tokens_seen": 584253440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650030090270812, + "loss": 1.4383, + "theoretical_loss": 3.879021264456469, + "tokens_seen": 584318976 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664986960882648, + "loss": 1.5654, + "theoretical_loss": 3.878975993517276, + "tokens_seen": 584384512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006649709127382146, + "loss": 1.5445, + "theoretical_loss": 3.8789307290760804, + "tokens_seen": 584450048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006649548645937814, + "loss": 1.4391, + "theoretical_loss": 3.8788854711312206, + "tokens_seen": 584515584 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664938816449348, + "loss": 1.4836, + "theoretical_loss": 3.878840219681037, + "tokens_seen": 584581120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006649227683049148, + "loss": 1.5231, + "theoretical_loss": 3.8787949747238697, + "tokens_seen": 584646656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006649067201604814, + "loss": 1.5907, + "theoretical_loss": 3.878749736258058, + "tokens_seen": 584712192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648906720160482, + "loss": 1.4663, + "theoretical_loss": 3.8787045042819446, + "tokens_seen": 584777728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648746238716148, + "loss": 1.4545, + "theoretical_loss": 3.8786592787938705, + "tokens_seen": 584843264 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648585757271816, + "loss": 1.4241, + "theoretical_loss": 3.878614059792178, + "tokens_seen": 584908800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648425275827482, + "loss": 1.5496, + "theoretical_loss": 3.878568847275212, + "tokens_seen": 584974336 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664826479438315, + "loss": 1.379, + "theoretical_loss": 3.8785236412413138, + "tokens_seen": 585039872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648104312938816, + "loss": 1.5821, + "theoretical_loss": 3.878478441688829, + "tokens_seen": 585105408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647943831494484, + "loss": 1.5253, + "theoretical_loss": 3.878433248616102, + "tokens_seen": 585170944 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664778335005015, + "loss": 1.4069, + "theoretical_loss": 3.878388062021479, + "tokens_seen": 585236480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647622868605818, + "loss": 1.3906, + "theoretical_loss": 3.8783428819033055, + "tokens_seen": 585302016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647462387161484, + "loss": 1.5373, + "theoretical_loss": 3.8782977082599284, + "tokens_seen": 585367552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647301905717152, + "loss": 1.4043, + "theoretical_loss": 3.8782525410896955, + "tokens_seen": 585433088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647141424272818, + "loss": 1.3978, + "theoretical_loss": 3.8782073803909545, + "tokens_seen": 585498624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646980942828486, + "loss": 1.443, + "theoretical_loss": 3.878162226162053, + "tokens_seen": 585564160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646820461384152, + "loss": 1.4232, + "theoretical_loss": 3.878117078401342, + "tokens_seen": 585629696 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664665997993982, + "loss": 1.4794, + "theoretical_loss": 3.87807193710717, + "tokens_seen": 585695232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646499498495486, + "loss": 1.4249, + "theoretical_loss": 3.8780268022778874, + "tokens_seen": 585760768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646339017051154, + "loss": 1.4592, + "theoretical_loss": 3.877981673911846, + "tokens_seen": 585826304 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664617853560682, + "loss": 1.4521, + "theoretical_loss": 3.8779365520073963, + "tokens_seen": 585891840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646018054162488, + "loss": 1.5234, + "theoretical_loss": 3.8778914365628916, + "tokens_seen": 585957376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645857572718154, + "loss": 1.5314, + "theoretical_loss": 3.877846327576684, + "tokens_seen": 586022912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645697091273822, + "loss": 1.4303, + "theoretical_loss": 3.8778012250471265, + "tokens_seen": 586088448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645536609829488, + "loss": 1.4924, + "theoretical_loss": 3.8777561289725746, + "tokens_seen": 586153984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645376128385156, + "loss": 1.3857, + "theoretical_loss": 3.8777110393513814, + "tokens_seen": 586219520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645215646940822, + "loss": 1.4557, + "theoretical_loss": 3.877665956181903, + "tokens_seen": 586285056 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664505516549649, + "loss": 1.4093, + "theoretical_loss": 3.8776208794624942, + "tokens_seen": 586350592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644894684052156, + "loss": 1.4833, + "theoretical_loss": 3.8775758091915127, + "tokens_seen": 586416128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644734202607824, + "loss": 1.6214, + "theoretical_loss": 3.8775307453673147, + "tokens_seen": 586481664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 201447, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.557558298110962, + "objective/train/theoretical_loss": 3.8774856879882584, + "objective/train/tokens_used": 607007200, + "theoretical_loss": 3.8774856879882584, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664457372116349, + "loss": 1.5159, + "theoretical_loss": 3.8774856879882584, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644413239719158, + "loss": 1.4372, + "theoretical_loss": 3.8774406370527013, + "tokens_seen": 586612736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644252758274824, + "loss": 1.5486, + "theoretical_loss": 3.8773955925590036, + "tokens_seen": 586678272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644092276830492, + "loss": 1.4192, + "theoretical_loss": 3.8773505545055227, + "tokens_seen": 586743808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643931795386158, + "loss": 1.4344, + "theoretical_loss": 3.87730552289062, + "tokens_seen": 586809344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643771313941826, + "loss": 1.5196, + "theoretical_loss": 3.8772604977126557, + "tokens_seen": 586874880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643610832497492, + "loss": 1.3887, + "theoretical_loss": 3.8772154789699913, + "tokens_seen": 586940416 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664345035105316, + "loss": 1.3559, + "theoretical_loss": 3.8771704666609885, + "tokens_seen": 587005952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643289869608826, + "loss": 1.3598, + "theoretical_loss": 3.8771254607840095, + "tokens_seen": 587071488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643129388164494, + "loss": 1.3921, + "theoretical_loss": 3.877080461337418, + "tokens_seen": 587137024 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664296890672016, + "loss": 1.4247, + "theoretical_loss": 3.877035468319577, + "tokens_seen": 587202560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642808425275828, + "loss": 1.4363, + "theoretical_loss": 3.8769904817288507, + "tokens_seen": 587268096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642647943831495, + "loss": 1.5142, + "theoretical_loss": 3.8769455015636045, + "tokens_seen": 587333632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642487462387162, + "loss": 1.4592, + "theoretical_loss": 3.8769005278222033, + "tokens_seen": 587399168 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664232698094283, + "loss": 1.5649, + "theoretical_loss": 3.876855560503013, + "tokens_seen": 587464704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642166499498496, + "loss": 1.435, + "theoretical_loss": 3.8768105996044007, + "tokens_seen": 587530240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642006018054164, + "loss": 1.3991, + "theoretical_loss": 3.876765645124734, + "tokens_seen": 587595776 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664184553660983, + "loss": 1.4028, + "theoretical_loss": 3.876720697062379, + "tokens_seen": 587661312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641685055165498, + "loss": 1.4663, + "theoretical_loss": 3.876675755415706, + "tokens_seen": 587726848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641524573721164, + "loss": 1.4308, + "theoretical_loss": 3.8766308201830837, + "tokens_seen": 587792384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641364092276832, + "loss": 1.5021, + "theoretical_loss": 3.8765858913628803, + "tokens_seen": 587857920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641203610832498, + "loss": 1.5041, + "theoretical_loss": 3.8765409689534676, + "tokens_seen": 587923456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641043129388166, + "loss": 1.4928, + "theoretical_loss": 3.8764960529532155, + "tokens_seen": 587988992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006640882647943832, + "loss": 1.4278, + "theoretical_loss": 3.876451143360496, + "tokens_seen": 588054528 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066407221664995, + "loss": 1.534, + "theoretical_loss": 3.8764062401736803, + "tokens_seen": 588120064 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006640561685055166, + "loss": 1.4744, + "theoretical_loss": 3.8763613433911415, + "tokens_seen": 588185600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006640401203610834, + "loss": 1.4814, + "theoretical_loss": 3.8763164530112535, + "tokens_seen": 588251136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066402407221665, + "loss": 1.4645, + "theoretical_loss": 3.8762715690323883, + "tokens_seen": 588316672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006640080240722168, + "loss": 1.4931, + "theoretical_loss": 3.876226691452922, + "tokens_seen": 588382208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639919759277834, + "loss": 1.3968, + "theoretical_loss": 3.8761818202712286, + "tokens_seen": 588447744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639759277833501, + "loss": 1.5262, + "theoretical_loss": 3.8761369554856837, + "tokens_seen": 588513280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639598796389168, + "loss": 1.3893, + "theoretical_loss": 3.8760920970946637, + "tokens_seen": 588578816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639438314944835, + "loss": 1.3683, + "theoretical_loss": 3.8760472450965455, + "tokens_seen": 588644352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639277833500502, + "loss": 1.4467, + "theoretical_loss": 3.876002399489706, + "tokens_seen": 588709888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639117352056169, + "loss": 1.4814, + "theoretical_loss": 3.8759575602725236, + "tokens_seen": 588775424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638956870611836, + "loss": 1.4286, + "theoretical_loss": 3.8759127274433762, + "tokens_seen": 588840960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638796389167503, + "loss": 1.4914, + "theoretical_loss": 3.8758679010006434, + "tokens_seen": 588906496 + }, + { + "epoch": 0.18, + "learning_rate": 0.000663863590772317, + "loss": 1.38, + "theoretical_loss": 3.875823080942705, + "tokens_seen": 588972032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638475426278837, + "loss": 1.5693, + "theoretical_loss": 3.875778267267941, + "tokens_seen": 589037568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638314944834503, + "loss": 1.4688, + "theoretical_loss": 3.8757334599747324, + "tokens_seen": 589103104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638154463390171, + "loss": 1.3994, + "theoretical_loss": 3.8756886590614608, + "tokens_seen": 589168640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637993981945837, + "loss": 1.4926, + "theoretical_loss": 3.8756438645265074, + "tokens_seen": 589234176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637833500501505, + "loss": 1.4596, + "theoretical_loss": 3.8755990763682564, + "tokens_seen": 589299712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637673019057171, + "loss": 1.4927, + "theoretical_loss": 3.87555429458509, + "tokens_seen": 589365248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637512537612839, + "loss": 1.4579, + "theoretical_loss": 3.8755095191753917, + "tokens_seen": 589430784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637352056168505, + "loss": 1.4657, + "theoretical_loss": 3.8754647501375468, + "tokens_seen": 589496320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637191574724173, + "loss": 1.4161, + "theoretical_loss": 3.8754199874699404, + "tokens_seen": 589561856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637031093279839, + "loss": 1.5285, + "theoretical_loss": 3.875375231170957, + "tokens_seen": 589627392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636870611835507, + "loss": 1.4242, + "theoretical_loss": 3.8753304812389837, + "tokens_seen": 589692928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636710130391173, + "loss": 1.5375, + "theoretical_loss": 3.8752857376724075, + "tokens_seen": 589758464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 202342, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.514643907546997, + "objective/train/theoretical_loss": 3.875241000469615, + "objective/train/tokens_used": 610284000, + "theoretical_loss": 3.875241000469615, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636549648946841, + "loss": 1.5381, + "theoretical_loss": 3.875241000469615, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636389167502507, + "loss": 1.4772, + "theoretical_loss": 3.8751962696289945, + "tokens_seen": 589889536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636228686058175, + "loss": 1.4867, + "theoretical_loss": 3.8751515451489347, + "tokens_seen": 589955072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636068204613841, + "loss": 1.4871, + "theoretical_loss": 3.875106827027824, + "tokens_seen": 590020608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635907723169509, + "loss": 1.4657, + "theoretical_loss": 3.875062115264053, + "tokens_seen": 590086144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635747241725175, + "loss": 1.4551, + "theoretical_loss": 3.875017409856012, + "tokens_seen": 590151680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635586760280843, + "loss": 1.4035, + "theoretical_loss": 3.8749727108020915, + "tokens_seen": 590217216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635426278836509, + "loss": 1.4181, + "theoretical_loss": 3.8749280181006824, + "tokens_seen": 590282752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635265797392177, + "loss": 1.5073, + "theoretical_loss": 3.874883331750178, + "tokens_seen": 590348288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635105315947843, + "loss": 1.537, + "theoretical_loss": 3.874838651748971, + "tokens_seen": 590413824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634944834503511, + "loss": 1.5114, + "theoretical_loss": 3.874793978095453, + "tokens_seen": 590479360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634784353059177, + "loss": 1.3964, + "theoretical_loss": 3.874749310788019, + "tokens_seen": 590544896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634623871614845, + "loss": 1.4324, + "theoretical_loss": 3.8747046498250626, + "tokens_seen": 590610432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634463390170511, + "loss": 1.474, + "theoretical_loss": 3.87465999520498, + "tokens_seen": 590675968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634302908726179, + "loss": 1.5118, + "theoretical_loss": 3.8746153469261664, + "tokens_seen": 590741504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634142427281845, + "loss": 1.4831, + "theoretical_loss": 3.8745707049870175, + "tokens_seen": 590807040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633981945837513, + "loss": 1.4809, + "theoretical_loss": 3.8745260693859302, + "tokens_seen": 590872576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633821464393179, + "loss": 1.319, + "theoretical_loss": 3.8744814401213024, + "tokens_seen": 590938112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633660982948847, + "loss": 1.4283, + "theoretical_loss": 3.8744368171915307, + "tokens_seen": 591003648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633500501504513, + "loss": 1.4634, + "theoretical_loss": 3.8743922005950147, + "tokens_seen": 591069184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633340020060181, + "loss": 1.4687, + "theoretical_loss": 3.8743475903301534, + "tokens_seen": 591134720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633179538615847, + "loss": 1.451, + "theoretical_loss": 3.874302986395346, + "tokens_seen": 591200256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633019057171515, + "loss": 1.5493, + "theoretical_loss": 3.8742583887889928, + "tokens_seen": 591265792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632858575727181, + "loss": 1.5102, + "theoretical_loss": 3.874213797509495, + "tokens_seen": 591331328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632698094282849, + "loss": 1.41, + "theoretical_loss": 3.874169212555253, + "tokens_seen": 591396864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632537612838515, + "loss": 1.3676, + "theoretical_loss": 3.87412463392467, + "tokens_seen": 591462400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632377131394183, + "loss": 1.4752, + "theoretical_loss": 3.8740800616161484, + "tokens_seen": 591527936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632216649949849, + "loss": 1.4781, + "theoretical_loss": 3.8740354956280907, + "tokens_seen": 591593472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632056168505517, + "loss": 1.4841, + "theoretical_loss": 3.873990935958901, + "tokens_seen": 591659008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631895687061183, + "loss": 1.5448, + "theoretical_loss": 3.8739463826069835, + "tokens_seen": 591724544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631735205616851, + "loss": 1.3107, + "theoretical_loss": 3.8739018355707433, + "tokens_seen": 591790080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631574724172517, + "loss": 1.4157, + "theoretical_loss": 3.8738572948485848, + "tokens_seen": 591855616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631414242728185, + "loss": 1.4669, + "theoretical_loss": 3.873812760438916, + "tokens_seen": 591921152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631253761283851, + "loss": 1.4281, + "theoretical_loss": 3.873768232340142, + "tokens_seen": 591986688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631093279839519, + "loss": 1.4568, + "theoretical_loss": 3.87372371055067, + "tokens_seen": 592052224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630932798395185, + "loss": 1.5104, + "theoretical_loss": 3.8736791950689087, + "tokens_seen": 592117760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630772316950853, + "loss": 1.4712, + "theoretical_loss": 3.8736346858932658, + "tokens_seen": 592183296 + }, + { + "epoch": 0.18, + "learning_rate": 0.000663061183550652, + "loss": 1.4616, + "theoretical_loss": 3.8735901830221495, + "tokens_seen": 592248832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630451354062187, + "loss": 1.4809, + "theoretical_loss": 3.873545686453971, + "tokens_seen": 592314368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630290872617855, + "loss": 1.4827, + "theoretical_loss": 3.8735011961871395, + "tokens_seen": 592379904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630130391173521, + "loss": 1.5461, + "theoretical_loss": 3.873456712220066, + "tokens_seen": 592445440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629969909729189, + "loss": 1.4825, + "theoretical_loss": 3.873412234551161, + "tokens_seen": 592510976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629809428284855, + "loss": 1.4295, + "theoretical_loss": 3.8733677631788366, + "tokens_seen": 592576512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629648946840523, + "loss": 1.491, + "theoretical_loss": 3.8733232981015058, + "tokens_seen": 592642048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629488465396189, + "loss": 1.5089, + "theoretical_loss": 3.8732788393175808, + "tokens_seen": 592707584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629327983951857, + "loss": 1.5842, + "theoretical_loss": 3.8732343868254753, + "tokens_seen": 592773120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629167502507523, + "loss": 1.384, + "theoretical_loss": 3.873189940623604, + "tokens_seen": 592838656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629007021063191, + "loss": 1.4859, + "theoretical_loss": 3.8731455007103808, + "tokens_seen": 592904192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628846539618857, + "loss": 1.4207, + "theoretical_loss": 3.873101067084221, + "tokens_seen": 592969728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628686058174525, + "loss": 1.2965, + "theoretical_loss": 3.873056639743541, + "tokens_seen": 593035264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 203552, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4285786151885986, + "objective/train/theoretical_loss": 3.8730122186867573, + "objective/train/tokens_used": 613560800, + "theoretical_loss": 3.8730122186867573, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628525576730191, + "loss": 1.394, + "theoretical_loss": 3.8730122186867573, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628365095285859, + "loss": 1.6074, + "theoretical_loss": 3.872967803912286, + "tokens_seen": 593166336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628204613841525, + "loss": 1.3419, + "theoretical_loss": 3.872923395418546, + "tokens_seen": 593231872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628044132397193, + "loss": 1.4308, + "theoretical_loss": 3.872878993203954, + "tokens_seen": 593297408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627883650952859, + "loss": 1.4721, + "theoretical_loss": 3.8728345972669294, + "tokens_seen": 593362944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627723169508527, + "loss": 1.4272, + "theoretical_loss": 3.872790207605891, + "tokens_seen": 593428480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627562688064193, + "loss": 1.3887, + "theoretical_loss": 3.87274582421926, + "tokens_seen": 593494016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627402206619861, + "loss": 1.4862, + "theoretical_loss": 3.872701447105455, + "tokens_seen": 593559552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627241725175527, + "loss": 1.5783, + "theoretical_loss": 3.8726570762628985, + "tokens_seen": 593625088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627081243731195, + "loss": 1.4866, + "theoretical_loss": 3.872612711690011, + "tokens_seen": 593690624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626920762286861, + "loss": 1.46, + "theoretical_loss": 3.8725683533852155, + "tokens_seen": 593756160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626760280842529, + "loss": 1.4521, + "theoretical_loss": 3.8725240013469344, + "tokens_seen": 593821696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626599799398195, + "loss": 1.5239, + "theoretical_loss": 3.8724796555735903, + "tokens_seen": 593887232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626439317953863, + "loss": 1.5429, + "theoretical_loss": 3.8724353160636085, + "tokens_seen": 593952768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626278836509529, + "loss": 1.4103, + "theoretical_loss": 3.872390982815412, + "tokens_seen": 594018304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626118355065197, + "loss": 1.3815, + "theoretical_loss": 3.8723466558274264, + "tokens_seen": 594083840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625957873620863, + "loss": 1.5167, + "theoretical_loss": 3.872302335098077, + "tokens_seen": 594149376 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662579739217653, + "loss": 1.4964, + "theoretical_loss": 3.872258020625791, + "tokens_seen": 594214912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625636910732197, + "loss": 1.5179, + "theoretical_loss": 3.8722137124089935, + "tokens_seen": 594280448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625476429287864, + "loss": 1.4576, + "theoretical_loss": 3.8721694104461126, + "tokens_seen": 594345984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625315947843531, + "loss": 1.4038, + "theoretical_loss": 3.872125114735576, + "tokens_seen": 594411520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625155466399198, + "loss": 1.4124, + "theoretical_loss": 3.8720808252758125, + "tokens_seen": 594477056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624994984954865, + "loss": 1.5148, + "theoretical_loss": 3.872036542065251, + "tokens_seen": 594542592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624834503510532, + "loss": 1.4547, + "theoretical_loss": 3.87199226510232, + "tokens_seen": 594608128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624674022066199, + "loss": 1.5369, + "theoretical_loss": 3.871947994385451, + "tokens_seen": 594673664 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624513540621866, + "loss": 1.5483, + "theoretical_loss": 3.871903729913074, + "tokens_seen": 594739200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624353059177532, + "loss": 1.4984, + "theoretical_loss": 3.8718594716836203, + "tokens_seen": 594804736 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066241925777332, + "loss": 1.5162, + "theoretical_loss": 3.871815219695522, + "tokens_seen": 594870272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624032096288866, + "loss": 1.3593, + "theoretical_loss": 3.8717709739472106, + "tokens_seen": 594935808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623871614844534, + "loss": 1.4981, + "theoretical_loss": 3.87172673443712, + "tokens_seen": 595001344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066237111334002, + "loss": 1.4377, + "theoretical_loss": 3.8716825011636837, + "tokens_seen": 595066880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623550651955868, + "loss": 1.5133, + "theoretical_loss": 3.871638274125335, + "tokens_seen": 595132416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623390170511534, + "loss": 1.3823, + "theoretical_loss": 3.871594053320509, + "tokens_seen": 595197952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623229689067202, + "loss": 1.4658, + "theoretical_loss": 3.871549838747641, + "tokens_seen": 595263488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623069207622868, + "loss": 1.4668, + "theoretical_loss": 3.871505630405167, + "tokens_seen": 595329024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006622908726178536, + "loss": 1.5038, + "theoretical_loss": 3.8714614282915227, + "tokens_seen": 595394560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006622748244734202, + "loss": 1.5095, + "theoretical_loss": 3.8714172324051455, + "tokens_seen": 595460096 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662258776328987, + "loss": 1.4693, + "theoretical_loss": 3.8713730427444726, + "tokens_seen": 595525632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006622427281845536, + "loss": 1.4081, + "theoretical_loss": 3.8713288593079422, + "tokens_seen": 595591168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006622266800401204, + "loss": 1.3935, + "theoretical_loss": 3.871284682093993, + "tokens_seen": 595656704 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662210631895687, + "loss": 1.4838, + "theoretical_loss": 3.871240511101064, + "tokens_seen": 595722240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621945837512538, + "loss": 1.491, + "theoretical_loss": 3.8711963463275945, + "tokens_seen": 595787776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621785356068204, + "loss": 1.4342, + "theoretical_loss": 3.871152187772026, + "tokens_seen": 595853312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621624874623872, + "loss": 1.4372, + "theoretical_loss": 3.8711080354327976, + "tokens_seen": 595918848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621464393179538, + "loss": 1.4873, + "theoretical_loss": 3.8710638893083518, + "tokens_seen": 595984384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621303911735206, + "loss": 1.3888, + "theoretical_loss": 3.8710197493971306, + "tokens_seen": 596049920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621143430290872, + "loss": 1.5513, + "theoretical_loss": 3.870975615697576, + "tokens_seen": 596115456 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662098294884654, + "loss": 1.3557, + "theoretical_loss": 3.8709314882081323, + "tokens_seen": 596180992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620822467402206, + "loss": 1.4748, + "theoretical_loss": 3.870887366927242, + "tokens_seen": 596246528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620661985957874, + "loss": 1.4486, + "theoretical_loss": 3.8708432518533495, + "tokens_seen": 596312064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 204530, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5145456790924072, + "objective/train/theoretical_loss": 3.8707991429848994, + "objective/train/tokens_used": 616837600, + "theoretical_loss": 3.8707991429848994, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662050150451354, + "loss": 1.4601, + "theoretical_loss": 3.8707991429848994, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620341023069208, + "loss": 1.4813, + "theoretical_loss": 3.8707550403203377, + "tokens_seen": 596443136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620180541624874, + "loss": 1.4676, + "theoretical_loss": 3.8707109438581098, + "tokens_seen": 596508672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620020060180542, + "loss": 1.429, + "theoretical_loss": 3.8706668535966626, + "tokens_seen": 596574208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619859578736208, + "loss": 1.4921, + "theoretical_loss": 3.8706227695344424, + "tokens_seen": 596639744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619699097291876, + "loss": 1.4169, + "theoretical_loss": 3.8705786916698974, + "tokens_seen": 596705280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619538615847542, + "loss": 1.411, + "theoretical_loss": 3.870534620001476, + "tokens_seen": 596770816 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661937813440321, + "loss": 1.4635, + "theoretical_loss": 3.870490554527626, + "tokens_seen": 596836352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619217652958876, + "loss": 1.4659, + "theoretical_loss": 3.8704464952467967, + "tokens_seen": 596901888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619057171514544, + "loss": 1.3666, + "theoretical_loss": 3.870402442157439, + "tokens_seen": 596967424 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661889669007021, + "loss": 1.6077, + "theoretical_loss": 3.8703583952580023, + "tokens_seen": 597032960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006618736208625878, + "loss": 1.4423, + "theoretical_loss": 3.8703143545469376, + "tokens_seen": 597098496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006618575727181545, + "loss": 1.5461, + "theoretical_loss": 3.870270320022697, + "tokens_seen": 597164032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006618415245737212, + "loss": 1.3763, + "theoretical_loss": 3.8702262916837316, + "tokens_seen": 597229568 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661825476429288, + "loss": 1.5186, + "theoretical_loss": 3.8701822695284953, + "tokens_seen": 597295104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006618094282848546, + "loss": 1.4728, + "theoretical_loss": 3.8701382535554396, + "tokens_seen": 597360640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617933801404213, + "loss": 1.4291, + "theoretical_loss": 3.8700942437630195, + "tokens_seen": 597426176 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661777331995988, + "loss": 1.51, + "theoretical_loss": 3.870050240149689, + "tokens_seen": 597491712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617612838515548, + "loss": 1.4125, + "theoretical_loss": 3.870006242713903, + "tokens_seen": 597557248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617452357071214, + "loss": 1.5887, + "theoretical_loss": 3.8699622514541163, + "tokens_seen": 597622784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617291875626882, + "loss": 1.5154, + "theoretical_loss": 3.8699182663687854, + "tokens_seen": 597688320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617131394182548, + "loss": 1.4929, + "theoretical_loss": 3.8698742874563665, + "tokens_seen": 597753856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616970912738216, + "loss": 1.4577, + "theoretical_loss": 3.8698303147153172, + "tokens_seen": 597819392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616810431293882, + "loss": 1.3801, + "theoretical_loss": 3.8697863481440944, + "tokens_seen": 597884928 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661664994984955, + "loss": 1.5343, + "theoretical_loss": 3.8697423877411565, + "tokens_seen": 597950464 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616489468405216, + "loss": 1.4261, + "theoretical_loss": 3.8696984335049622, + "tokens_seen": 598016000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616328986960884, + "loss": 1.3522, + "theoretical_loss": 3.869654485433971, + "tokens_seen": 598081536 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661616850551655, + "loss": 1.4609, + "theoretical_loss": 3.869610543526643, + "tokens_seen": 598147072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616008024072218, + "loss": 1.4074, + "theoretical_loss": 3.8695666077814375, + "tokens_seen": 598212608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615847542627884, + "loss": 1.4669, + "theoretical_loss": 3.869522678196816, + "tokens_seen": 598278144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615687061183552, + "loss": 1.5038, + "theoretical_loss": 3.86947875477124, + "tokens_seen": 598343680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615526579739218, + "loss": 1.4047, + "theoretical_loss": 3.8694348375031717, + "tokens_seen": 598409216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615366098294886, + "loss": 1.3909, + "theoretical_loss": 3.8693909263910737, + "tokens_seen": 598474752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615205616850552, + "loss": 1.4707, + "theoretical_loss": 3.869347021433409, + "tokens_seen": 598540288 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661504513540622, + "loss": 1.4068, + "theoretical_loss": 3.869303122628641, + "tokens_seen": 598605824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614884653961886, + "loss": 1.378, + "theoretical_loss": 3.869259229975235, + "tokens_seen": 598671360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614724172517554, + "loss": 1.4765, + "theoretical_loss": 3.869215343471654, + "tokens_seen": 598736896 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661456369107322, + "loss": 1.5062, + "theoretical_loss": 3.8691714631163645, + "tokens_seen": 598802432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614403209628888, + "loss": 1.4873, + "theoretical_loss": 3.8691275889078325, + "tokens_seen": 598867968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614242728184554, + "loss": 1.5145, + "theoretical_loss": 3.8690837208445243, + "tokens_seen": 598933504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614082246740222, + "loss": 1.5306, + "theoretical_loss": 3.869039858924906, + "tokens_seen": 598999040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613921765295888, + "loss": 1.5118, + "theoretical_loss": 3.868996003147447, + "tokens_seen": 599064576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613761283851556, + "loss": 1.3758, + "theoretical_loss": 3.868952153510614, + "tokens_seen": 599130112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613600802407222, + "loss": 1.4249, + "theoretical_loss": 3.868908310012876, + "tokens_seen": 599195648 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661344032096289, + "loss": 1.481, + "theoretical_loss": 3.868864472652702, + "tokens_seen": 599261184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613279839518556, + "loss": 1.3103, + "theoretical_loss": 3.868820641428562, + "tokens_seen": 599326720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613119358074224, + "loss": 1.5188, + "theoretical_loss": 3.868776816338926, + "tokens_seen": 599392256 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661295887662989, + "loss": 1.5063, + "theoretical_loss": 3.8687329973822653, + "tokens_seen": 599457792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612798395185558, + "loss": 1.4514, + "theoretical_loss": 3.868689184557051, + "tokens_seen": 599523328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612637913741224, + "loss": 1.5114, + "theoretical_loss": 3.8686453778617547, + "tokens_seen": 599588864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 205743, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4836275577545166, + "objective/train/theoretical_loss": 3.868601577294849, + "objective/train/tokens_used": 620114400, + "theoretical_loss": 3.868601577294849, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612477432296892, + "loss": 1.5095, + "theoretical_loss": 3.868601577294849, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612316950852558, + "loss": 1.2991, + "theoretical_loss": 3.8685577828548086, + "tokens_seen": 599719936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612156469408226, + "loss": 1.3201, + "theoretical_loss": 3.8685139945401046, + "tokens_seen": 599785472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611995987963892, + "loss": 1.4232, + "theoretical_loss": 3.8684702123492123, + "tokens_seen": 599851008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611835506519559, + "loss": 1.5699, + "theoretical_loss": 3.8684264362806062, + "tokens_seen": 599916544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611675025075226, + "loss": 1.5524, + "theoretical_loss": 3.868382666332762, + "tokens_seen": 599982080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611514543630893, + "loss": 1.4625, + "theoretical_loss": 3.8683389025041546, + "tokens_seen": 600047616 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661135406218656, + "loss": 1.4177, + "theoretical_loss": 3.8682951447932608, + "tokens_seen": 600113152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611193580742227, + "loss": 1.3461, + "theoretical_loss": 3.8682513931985576, + "tokens_seen": 600178688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611033099297894, + "loss": 1.3989, + "theoretical_loss": 3.8682076477185223, + "tokens_seen": 600244224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610872617853561, + "loss": 1.4005, + "theoretical_loss": 3.868163908351632, + "tokens_seen": 600309760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610712136409228, + "loss": 1.4583, + "theoretical_loss": 3.868120175096367, + "tokens_seen": 600375296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610551654964895, + "loss": 1.5047, + "theoretical_loss": 3.868076447951205, + "tokens_seen": 600440832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610391173520561, + "loss": 1.4884, + "theoretical_loss": 3.8680327269146257, + "tokens_seen": 600506368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610230692076229, + "loss": 1.347, + "theoretical_loss": 3.8679890119851095, + "tokens_seen": 600571904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610070210631895, + "loss": 1.466, + "theoretical_loss": 3.867945303161137, + "tokens_seen": 600637440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609909729187563, + "loss": 1.4438, + "theoretical_loss": 3.8679016004411895, + "tokens_seen": 600702976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609749247743229, + "loss": 1.4553, + "theoretical_loss": 3.867857903823748, + "tokens_seen": 600768512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609588766298897, + "loss": 1.4764, + "theoretical_loss": 3.8678142133072964, + "tokens_seen": 600834048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609428284854563, + "loss": 1.4829, + "theoretical_loss": 3.8677705288903166, + "tokens_seen": 600899584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609267803410231, + "loss": 1.5542, + "theoretical_loss": 3.867726850571291, + "tokens_seen": 600965120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609107321965897, + "loss": 1.3085, + "theoretical_loss": 3.867683178348705, + "tokens_seen": 601030656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608946840521565, + "loss": 1.4878, + "theoretical_loss": 3.867639512221043, + "tokens_seen": 601096192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608786359077231, + "loss": 1.4221, + "theoretical_loss": 3.8675958521867893, + "tokens_seen": 601161728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608625877632899, + "loss": 1.4264, + "theoretical_loss": 3.8675521982444296, + "tokens_seen": 601227264 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608465396188565, + "loss": 1.5051, + "theoretical_loss": 3.8675085503924507, + "tokens_seen": 601292800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608304914744233, + "loss": 1.4914, + "theoretical_loss": 3.867464908629338, + "tokens_seen": 601358336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608144433299899, + "loss": 1.4987, + "theoretical_loss": 3.86742127295358, + "tokens_seen": 601423872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607983951855567, + "loss": 1.3915, + "theoretical_loss": 3.8673776433636635, + "tokens_seen": 601489408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607823470411233, + "loss": 1.5461, + "theoretical_loss": 3.8673340198580766, + "tokens_seen": 601554944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607662988966901, + "loss": 1.3752, + "theoretical_loss": 3.867290402435309, + "tokens_seen": 601620480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607502507522567, + "loss": 1.5197, + "theoretical_loss": 3.8672467910938497, + "tokens_seen": 601686016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607342026078235, + "loss": 1.4824, + "theoretical_loss": 3.8672031858321887, + "tokens_seen": 601751552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607181544633901, + "loss": 1.3691, + "theoretical_loss": 3.8671595866488158, + "tokens_seen": 601817088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607021063189569, + "loss": 1.4666, + "theoretical_loss": 3.867115993542222, + "tokens_seen": 601882624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606860581745235, + "loss": 1.5184, + "theoretical_loss": 3.8670724065109, + "tokens_seen": 601948160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606700100300903, + "loss": 1.4481, + "theoretical_loss": 3.8670288255533403, + "tokens_seen": 602013696 + }, + { + "epoch": 0.18, + "learning_rate": 0.000660653961885657, + "loss": 1.4513, + "theoretical_loss": 3.8669852506680362, + "tokens_seen": 602079232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606379137412237, + "loss": 1.4266, + "theoretical_loss": 3.866941681853481, + "tokens_seen": 602144768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606218655967904, + "loss": 1.4738, + "theoretical_loss": 3.866898119108168, + "tokens_seen": 602210304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606058174523571, + "loss": 1.4137, + "theoretical_loss": 3.866854562430592, + "tokens_seen": 602275840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605897693079238, + "loss": 1.4907, + "theoretical_loss": 3.8668110118192462, + "tokens_seen": 602341376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605737211634905, + "loss": 1.4957, + "theoretical_loss": 3.8667674672726275, + "tokens_seen": 602406912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605576730190572, + "loss": 1.465, + "theoretical_loss": 3.866723928789231, + "tokens_seen": 602472448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605416248746239, + "loss": 1.4293, + "theoretical_loss": 3.8666803963675536, + "tokens_seen": 602537984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605255767301906, + "loss": 1.4752, + "theoretical_loss": 3.866636870006091, + "tokens_seen": 602603520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605095285857573, + "loss": 1.4357, + "theoretical_loss": 3.866593349703342, + "tokens_seen": 602669056 + }, + { + "epoch": 0.18, + "learning_rate": 0.000660493480441324, + "loss": 1.4409, + "theoretical_loss": 3.8665498354578034, + "tokens_seen": 602734592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604774322968907, + "loss": 1.5654, + "theoretical_loss": 3.866506327267974, + "tokens_seen": 602800128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604613841524574, + "loss": 1.4332, + "theoretical_loss": 3.866462825132353, + "tokens_seen": 602865664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 206667, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4438321590423584, + "objective/train/theoretical_loss": 3.86641932904944, + "objective/train/tokens_used": 623391200, + "theoretical_loss": 3.86641932904944, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604453360080241, + "loss": 1.458, + "theoretical_loss": 3.86641932904944, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604292878635908, + "loss": 1.3835, + "theoretical_loss": 3.866375839017735, + "tokens_seen": 602996736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604132397191575, + "loss": 1.3458, + "theoretical_loss": 3.8663323550357385, + "tokens_seen": 603062272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603971915747243, + "loss": 1.3798, + "theoretical_loss": 3.866288877101952, + "tokens_seen": 603127808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603811434302909, + "loss": 1.4673, + "theoretical_loss": 3.866245405214877, + "tokens_seen": 603193344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603650952858577, + "loss": 1.4339, + "theoretical_loss": 3.8662019393730156, + "tokens_seen": 603258880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603490471414243, + "loss": 1.4291, + "theoretical_loss": 3.86615847957487, + "tokens_seen": 603324416 + }, + { + "epoch": 0.18, + "learning_rate": 0.000660332998996991, + "loss": 1.4796, + "theoretical_loss": 3.866115025818945, + "tokens_seen": 603389952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603169508525577, + "loss": 1.2908, + "theoretical_loss": 3.8660715781037434, + "tokens_seen": 603455488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603009027081245, + "loss": 1.6529, + "theoretical_loss": 3.8660281364277695, + "tokens_seen": 603521024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602848545636911, + "loss": 1.4889, + "theoretical_loss": 3.865984700789528, + "tokens_seen": 603586560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602688064192579, + "loss": 1.397, + "theoretical_loss": 3.865941271187525, + "tokens_seen": 603652096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602527582748245, + "loss": 1.415, + "theoretical_loss": 3.865897847620266, + "tokens_seen": 603717632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602367101303913, + "loss": 1.479, + "theoretical_loss": 3.865854430086258, + "tokens_seen": 603783168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602206619859579, + "loss": 1.3538, + "theoretical_loss": 3.865811018584007, + "tokens_seen": 603848704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602046138415247, + "loss": 1.4362, + "theoretical_loss": 3.8657676131120224, + "tokens_seen": 603914240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601885656970913, + "loss": 1.3837, + "theoretical_loss": 3.86572421366881, + "tokens_seen": 603979776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601725175526581, + "loss": 1.4328, + "theoretical_loss": 3.86568082025288, + "tokens_seen": 604045312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601564694082247, + "loss": 1.3513, + "theoretical_loss": 3.8656374328627408, + "tokens_seen": 604110848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601404212637915, + "loss": 1.4515, + "theoretical_loss": 3.8655940514969025, + "tokens_seen": 604176384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601243731193581, + "loss": 1.4263, + "theoretical_loss": 3.865550676153875, + "tokens_seen": 604241920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601083249749249, + "loss": 1.3591, + "theoretical_loss": 3.8655073068321695, + "tokens_seen": 604307456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600922768304915, + "loss": 1.5243, + "theoretical_loss": 3.8654639435302967, + "tokens_seen": 604372992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600762286860583, + "loss": 1.4241, + "theoretical_loss": 3.865420586246768, + "tokens_seen": 604438528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600601805416249, + "loss": 1.4579, + "theoretical_loss": 3.865377234980097, + "tokens_seen": 604504064 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600441323971917, + "loss": 1.3609, + "theoretical_loss": 3.865333889728795, + "tokens_seen": 604569600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600280842527583, + "loss": 1.392, + "theoretical_loss": 3.8652905504913777, + "tokens_seen": 604635136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600120361083251, + "loss": 1.3966, + "theoretical_loss": 3.865247217266356, + "tokens_seen": 604700672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599959879638917, + "loss": 1.4724, + "theoretical_loss": 3.865203890052247, + "tokens_seen": 604766208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599799398194585, + "loss": 1.3838, + "theoretical_loss": 3.865160568847564, + "tokens_seen": 604831744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599638916750251, + "loss": 1.427, + "theoretical_loss": 3.865117253650823, + "tokens_seen": 604897280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599478435305919, + "loss": 1.382, + "theoretical_loss": 3.8650739444605406, + "tokens_seen": 604962816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599317953861585, + "loss": 1.4716, + "theoretical_loss": 3.8650306412752324, + "tokens_seen": 605028352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599157472417253, + "loss": 1.4328, + "theoretical_loss": 3.8649873440934157, + "tokens_seen": 605093888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598996990972919, + "loss": 1.4793, + "theoretical_loss": 3.8649440529136085, + "tokens_seen": 605159424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598836509528587, + "loss": 1.3659, + "theoretical_loss": 3.864900767734329, + "tokens_seen": 605224960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598676028084253, + "loss": 1.4324, + "theoretical_loss": 3.8648574885540947, + "tokens_seen": 605290496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598515546639921, + "loss": 1.3815, + "theoretical_loss": 3.8648142153714264, + "tokens_seen": 605356032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598355065195587, + "loss": 1.4118, + "theoretical_loss": 3.8647709481848427, + "tokens_seen": 605421568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598194583751255, + "loss": 1.4245, + "theoretical_loss": 3.8647276869928637, + "tokens_seen": 605487104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598034102306921, + "loss": 1.4302, + "theoretical_loss": 3.864684431794011, + "tokens_seen": 605552640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597873620862588, + "loss": 1.4232, + "theoretical_loss": 3.864641182586806, + "tokens_seen": 605618176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597713139418255, + "loss": 1.4132, + "theoretical_loss": 3.8645979393697694, + "tokens_seen": 605683712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597552657973922, + "loss": 1.4686, + "theoretical_loss": 3.864554702141424, + "tokens_seen": 605749248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597392176529589, + "loss": 1.3692, + "theoretical_loss": 3.8645114709002932, + "tokens_seen": 605814784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597231695085256, + "loss": 1.4305, + "theoretical_loss": 3.8644682456448995, + "tokens_seen": 605880320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597071213640923, + "loss": 1.3826, + "theoretical_loss": 3.864425026373768, + "tokens_seen": 605945856 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659691073219659, + "loss": 1.464, + "theoretical_loss": 3.864381813085422, + "tokens_seen": 606011392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006596750250752257, + "loss": 1.3892, + "theoretical_loss": 3.8643386057783866, + "tokens_seen": 606076928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006596589769307924, + "loss": 1.4343, + "theoretical_loss": 3.8642954044511875, + "tokens_seen": 606142464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 207838, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6236754655838013, + "objective/train/theoretical_loss": 3.8642522091023506, + "objective/train/tokens_used": 626668000, + "theoretical_loss": 3.8642522091023506, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659642928786359, + "loss": 1.5245, + "theoretical_loss": 3.8642522091023506, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006596268806419258, + "loss": 1.5009, + "theoretical_loss": 3.864209019730403, + "tokens_seen": 606273536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006596108324974924, + "loss": 1.3572, + "theoretical_loss": 3.8641658363338705, + "tokens_seen": 606339072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595947843530592, + "loss": 1.4147, + "theoretical_loss": 3.8641226589112816, + "tokens_seen": 606404608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595787362086258, + "loss": 1.3962, + "theoretical_loss": 3.8640794874611646, + "tokens_seen": 606470144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595626880641926, + "loss": 1.4121, + "theoretical_loss": 3.8640363219820473, + "tokens_seen": 606535680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595466399197592, + "loss": 1.4212, + "theoretical_loss": 3.8639931624724593, + "tokens_seen": 606601216 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659530591775326, + "loss": 1.5439, + "theoretical_loss": 3.8639500089309307, + "tokens_seen": 606666752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595145436308926, + "loss": 1.4606, + "theoretical_loss": 3.86390686135599, + "tokens_seen": 606732288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594984954864594, + "loss": 1.4124, + "theoretical_loss": 3.86386371974617, + "tokens_seen": 606797824 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659482447342026, + "loss": 1.4786, + "theoretical_loss": 3.863820584100001, + "tokens_seen": 606863360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594663991975928, + "loss": 1.4325, + "theoretical_loss": 3.863777454416014, + "tokens_seen": 606928896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594503510531595, + "loss": 1.4714, + "theoretical_loss": 3.863734330692742, + "tokens_seen": 606994432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594343029087262, + "loss": 1.4178, + "theoretical_loss": 3.8636912129287175, + "tokens_seen": 607059968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594182547642929, + "loss": 1.4254, + "theoretical_loss": 3.8636481011224744, + "tokens_seen": 607125504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594022066198596, + "loss": 1.5808, + "theoretical_loss": 3.8636049952725466, + "tokens_seen": 607191040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593861584754263, + "loss": 1.3405, + "theoretical_loss": 3.863561895377467, + "tokens_seen": 607256576 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659370110330993, + "loss": 1.4353, + "theoretical_loss": 3.8635188014357715, + "tokens_seen": 607322112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593540621865597, + "loss": 1.4427, + "theoretical_loss": 3.8634757134459954, + "tokens_seen": 607387648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593380140421264, + "loss": 1.5434, + "theoretical_loss": 3.8634326314066745, + "tokens_seen": 607453184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593219658976931, + "loss": 1.4577, + "theoretical_loss": 3.8633895553163455, + "tokens_seen": 607518720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593059177532598, + "loss": 1.4115, + "theoretical_loss": 3.8633464851735444, + "tokens_seen": 607584256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592898696088265, + "loss": 1.512, + "theoretical_loss": 3.8633034209768096, + "tokens_seen": 607649792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592738214643932, + "loss": 1.4788, + "theoretical_loss": 3.8632603627246787, + "tokens_seen": 607715328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592577733199599, + "loss": 1.3012, + "theoretical_loss": 3.86321731041569, + "tokens_seen": 607780864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592417251755266, + "loss": 1.341, + "theoretical_loss": 3.863174264048383, + "tokens_seen": 607846400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592256770310933, + "loss": 1.3701, + "theoretical_loss": 3.8631312236212967, + "tokens_seen": 607911936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00065920962888666, + "loss": 1.3869, + "theoretical_loss": 3.863088189132971, + "tokens_seen": 607977472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591935807422267, + "loss": 1.4, + "theoretical_loss": 3.8630451605819474, + "tokens_seen": 608043008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591775325977934, + "loss": 1.4417, + "theoretical_loss": 3.8630021379667654, + "tokens_seen": 608108544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591614844533601, + "loss": 1.3807, + "theoretical_loss": 3.862959121285968, + "tokens_seen": 608174080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591454363089268, + "loss": 1.4167, + "theoretical_loss": 3.8629161105380962, + "tokens_seen": 608239616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591293881644935, + "loss": 1.367, + "theoretical_loss": 3.862873105721694, + "tokens_seen": 608305152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591133400200602, + "loss": 1.3104, + "theoretical_loss": 3.862830106835303, + "tokens_seen": 608370688 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659097291875627, + "loss": 1.4652, + "theoretical_loss": 3.8627871138774674, + "tokens_seen": 608436224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590812437311936, + "loss": 1.5179, + "theoretical_loss": 3.862744126846732, + "tokens_seen": 608501760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590651955867603, + "loss": 1.3854, + "theoretical_loss": 3.8627011457416405, + "tokens_seen": 608567296 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659049147442327, + "loss": 1.5049, + "theoretical_loss": 3.862658170560738, + "tokens_seen": 608632832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590330992978938, + "loss": 1.4641, + "theoretical_loss": 3.862615201302571, + "tokens_seen": 608698368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590170511534604, + "loss": 1.4137, + "theoretical_loss": 3.8625722379656855, + "tokens_seen": 608763904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590010030090272, + "loss": 1.4818, + "theoretical_loss": 3.862529280548628, + "tokens_seen": 608829440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589849548645938, + "loss": 1.403, + "theoretical_loss": 3.8624863290499456, + "tokens_seen": 608894976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589689067201606, + "loss": 1.4422, + "theoretical_loss": 3.862443383468186, + "tokens_seen": 608960512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589528585757272, + "loss": 1.4226, + "theoretical_loss": 3.8624004438018984, + "tokens_seen": 609026048 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658936810431294, + "loss": 1.3911, + "theoretical_loss": 3.8623575100496303, + "tokens_seen": 609091584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589207622868606, + "loss": 1.4543, + "theoretical_loss": 3.8623145822099314, + "tokens_seen": 609157120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589047141424274, + "loss": 1.4233, + "theoretical_loss": 3.8622716602813516, + "tokens_seen": 609222656 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658888665997994, + "loss": 1.5068, + "theoretical_loss": 3.862228744262441, + "tokens_seen": 609288192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588726178535608, + "loss": 1.3251, + "theoretical_loss": 3.8621858341517514, + "tokens_seen": 609353728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588565697091274, + "loss": 1.4483, + "theoretical_loss": 3.8621429299478325, + "tokens_seen": 609419264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 208904, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2397621870040894, + "objective/train/theoretical_loss": 3.862100031649237, + "objective/train/tokens_used": 629944800, + "theoretical_loss": 3.862100031649237, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588405215646942, + "loss": 1.297, + "theoretical_loss": 3.862100031649237, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588244734202608, + "loss": 1.3689, + "theoretical_loss": 3.8620571392545173, + "tokens_seen": 609550336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588084252758276, + "loss": 1.5308, + "theoretical_loss": 3.8620142527622265, + "tokens_seen": 609615872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006587923771313942, + "loss": 1.3965, + "theoretical_loss": 3.8619713721709172, + "tokens_seen": 609681408 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658776328986961, + "loss": 1.3237, + "theoretical_loss": 3.861928497479144, + "tokens_seen": 609746944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006587602808425276, + "loss": 1.4447, + "theoretical_loss": 3.86188562868546, + "tokens_seen": 609812480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006587442326980944, + "loss": 1.4676, + "theoretical_loss": 3.8618427657884222, + "tokens_seen": 609878016 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658728184553661, + "loss": 1.4838, + "theoretical_loss": 3.861799908786584, + "tokens_seen": 609943552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006587121364092278, + "loss": 1.5459, + "theoretical_loss": 3.8617570576785027, + "tokens_seen": 610009088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586960882647944, + "loss": 1.315, + "theoretical_loss": 3.8617142124627346, + "tokens_seen": 610074624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586800401203612, + "loss": 1.5473, + "theoretical_loss": 3.8616713731378356, + "tokens_seen": 610140160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586639919759278, + "loss": 1.4366, + "theoretical_loss": 3.8616285397023637, + "tokens_seen": 610205696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586479438314946, + "loss": 1.4457, + "theoretical_loss": 3.861585712154877, + "tokens_seen": 610271232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586318956870612, + "loss": 1.4372, + "theoretical_loss": 3.8615428904939337, + "tokens_seen": 610336768 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658615847542628, + "loss": 1.4573, + "theoretical_loss": 3.861500074718093, + "tokens_seen": 610402304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006585997993981946, + "loss": 1.4509, + "theoretical_loss": 3.8614572648259142, + "tokens_seen": 610467840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585837512537614, + "loss": 1.4814, + "theoretical_loss": 3.8614144608159577, + "tokens_seen": 610533376 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658567703109328, + "loss": 1.4871, + "theoretical_loss": 3.8613716626867833, + "tokens_seen": 610598912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585516549648948, + "loss": 1.5432, + "theoretical_loss": 3.861328870436952, + "tokens_seen": 610664448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585356068204614, + "loss": 1.502, + "theoretical_loss": 3.8612860840650267, + "tokens_seen": 610729984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585195586760282, + "loss": 1.391, + "theoretical_loss": 3.8612433035695672, + "tokens_seen": 610795520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585035105315948, + "loss": 1.4072, + "theoretical_loss": 3.861200528949138, + "tokens_seen": 610861056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006584874623871616, + "loss": 1.378, + "theoretical_loss": 3.8611577602023006, + "tokens_seen": 610926592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006584714142427282, + "loss": 1.4051, + "theoretical_loss": 3.8611149973276193, + "tokens_seen": 610992128 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658455366098295, + "loss": 1.3886, + "theoretical_loss": 3.861072240323658, + "tokens_seen": 611057664 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006584393179538616, + "loss": 1.4285, + "theoretical_loss": 3.861029489188981, + "tokens_seen": 611123200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006584232698094284, + "loss": 1.3538, + "theoretical_loss": 3.8609867439221537, + "tokens_seen": 611188736 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658407221664995, + "loss": 1.4264, + "theoretical_loss": 3.860944004521742, + "tokens_seen": 611254272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583911735205618, + "loss": 1.4412, + "theoretical_loss": 3.860901270986311, + "tokens_seen": 611319808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583751253761284, + "loss": 1.4681, + "theoretical_loss": 3.8608585433144276, + "tokens_seen": 611385344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583590772316951, + "loss": 1.4123, + "theoretical_loss": 3.860815821504659, + "tokens_seen": 611450880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583430290872618, + "loss": 1.3016, + "theoretical_loss": 3.860773105555573, + "tokens_seen": 611516416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583269809428285, + "loss": 1.4889, + "theoretical_loss": 3.860730395465737, + "tokens_seen": 611581952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583109327983952, + "loss": 1.3215, + "theoretical_loss": 3.86068769123372, + "tokens_seen": 611647488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582948846539619, + "loss": 1.4208, + "theoretical_loss": 3.860644992858091, + "tokens_seen": 611713024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582788365095286, + "loss": 1.459, + "theoretical_loss": 3.8606023003374195, + "tokens_seen": 611778560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582627883650953, + "loss": 1.3311, + "theoretical_loss": 3.860559613670276, + "tokens_seen": 611844096 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658246740220662, + "loss": 1.4441, + "theoretical_loss": 3.86051693285523, + "tokens_seen": 611909632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582306920762287, + "loss": 1.3947, + "theoretical_loss": 3.8604742578908535, + "tokens_seen": 611975168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582146439317954, + "loss": 1.4688, + "theoretical_loss": 3.860431588775718, + "tokens_seen": 612040704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581985957873621, + "loss": 1.4368, + "theoretical_loss": 3.860388925508395, + "tokens_seen": 612106240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581825476429288, + "loss": 1.397, + "theoretical_loss": 3.8603462680874583, + "tokens_seen": 612171776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581664994984955, + "loss": 1.4094, + "theoretical_loss": 3.8603036165114797, + "tokens_seen": 612237312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581504513540622, + "loss": 1.5308, + "theoretical_loss": 3.8602609707790334, + "tokens_seen": 612302848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581344032096289, + "loss": 1.4203, + "theoretical_loss": 3.860218330888693, + "tokens_seen": 612368384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581183550651956, + "loss": 1.4991, + "theoretical_loss": 3.8601756968390335, + "tokens_seen": 612433920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581023069207623, + "loss": 1.4507, + "theoretical_loss": 3.86013306862863, + "tokens_seen": 612499456 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658086258776329, + "loss": 1.3386, + "theoretical_loss": 3.8600904462560583, + "tokens_seen": 612564992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580702106318957, + "loss": 1.5372, + "theoretical_loss": 3.8600478297198944, + "tokens_seen": 612630528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580541624874624, + "loss": 1.551, + "theoretical_loss": 3.860005219018714, + "tokens_seen": 612696064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 210170, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.420013427734375, + "objective/train/theoretical_loss": 3.8599626141510948, + "objective/train/tokens_used": 633221600, + "theoretical_loss": 3.8599626141510948, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580381143430291, + "loss": 1.4537, + "theoretical_loss": 3.8599626141510948, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580220661985958, + "loss": 1.4975, + "theoretical_loss": 3.859920015115615, + "tokens_seen": 612827136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580060180541625, + "loss": 1.5048, + "theoretical_loss": 3.8598774219108516, + "tokens_seen": 612892672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579899699097292, + "loss": 1.4598, + "theoretical_loss": 3.859834834535384, + "tokens_seen": 612958208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579739217652959, + "loss": 1.5072, + "theoretical_loss": 3.859792252987791, + "tokens_seen": 613023744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579578736208626, + "loss": 1.5103, + "theoretical_loss": 3.859749677266652, + "tokens_seen": 613089280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579418254764293, + "loss": 1.3005, + "theoretical_loss": 3.8597071073705473, + "tokens_seen": 613154816 + }, + { + "epoch": 0.19, + "learning_rate": 0.000657925777331996, + "loss": 1.4161, + "theoretical_loss": 3.8596645432980567, + "tokens_seen": 613220352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579097291875627, + "loss": 1.415, + "theoretical_loss": 3.859621985047763, + "tokens_seen": 613285888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578936810431294, + "loss": 1.4236, + "theoretical_loss": 3.859579432618246, + "tokens_seen": 613351424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578776328986961, + "loss": 1.4362, + "theoretical_loss": 3.8595368860080894, + "tokens_seen": 613416960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578615847542628, + "loss": 1.3682, + "theoretical_loss": 3.8594943452158743, + "tokens_seen": 613482496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578455366098295, + "loss": 1.3953, + "theoretical_loss": 3.8594518102401842, + "tokens_seen": 613548032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578294884653962, + "loss": 1.3524, + "theoretical_loss": 3.8594092810796026, + "tokens_seen": 613613568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578134403209629, + "loss": 1.4545, + "theoretical_loss": 3.859366757732714, + "tokens_seen": 613679104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577973921765296, + "loss": 1.3618, + "theoretical_loss": 3.8593242401981027, + "tokens_seen": 613744640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577813440320963, + "loss": 1.4137, + "theoretical_loss": 3.859281728474354, + "tokens_seen": 613810176 + }, + { + "epoch": 0.19, + "learning_rate": 0.000657765295887663, + "loss": 1.4826, + "theoretical_loss": 3.8592392225600527, + "tokens_seen": 613875712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577492477432297, + "loss": 1.5189, + "theoretical_loss": 3.859196722453785, + "tokens_seen": 613941248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577331995987964, + "loss": 1.4141, + "theoretical_loss": 3.859154228154138, + "tokens_seen": 614006784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577171514543631, + "loss": 1.3756, + "theoretical_loss": 3.8591117396596983, + "tokens_seen": 614072320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577011033099298, + "loss": 1.3612, + "theoretical_loss": 3.859069256969054, + "tokens_seen": 614137856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576850551654965, + "loss": 1.3814, + "theoretical_loss": 3.859026780080792, + "tokens_seen": 614203392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576690070210633, + "loss": 1.4247, + "theoretical_loss": 3.8589843089935014, + "tokens_seen": 614268928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576529588766299, + "loss": 1.4333, + "theoretical_loss": 3.8589418437057716, + "tokens_seen": 614334464 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576369107321967, + "loss": 1.4784, + "theoretical_loss": 3.8588993842161923, + "tokens_seen": 614400000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576208625877633, + "loss": 1.4326, + "theoretical_loss": 3.858856930523352, + "tokens_seen": 614465536 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065760481444333, + "loss": 1.5097, + "theoretical_loss": 3.8588144826258426, + "tokens_seen": 614531072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575887662988967, + "loss": 1.3983, + "theoretical_loss": 3.8587720405222545, + "tokens_seen": 614596608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575727181544635, + "loss": 1.3622, + "theoretical_loss": 3.8587296042111796, + "tokens_seen": 614662144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575566700100301, + "loss": 1.5156, + "theoretical_loss": 3.8586871736912087, + "tokens_seen": 614727680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575406218655969, + "loss": 1.3586, + "theoretical_loss": 3.858644748960936, + "tokens_seen": 614793216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575245737211635, + "loss": 1.5279, + "theoretical_loss": 3.858602330018953, + "tokens_seen": 614858752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575085255767303, + "loss": 1.5332, + "theoretical_loss": 3.858559916863853, + "tokens_seen": 614924288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574924774322969, + "loss": 1.4379, + "theoretical_loss": 3.8585175094942317, + "tokens_seen": 614989824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574764292878637, + "loss": 1.5811, + "theoretical_loss": 3.8584751079086814, + "tokens_seen": 615055360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574603811434303, + "loss": 1.5058, + "theoretical_loss": 3.8584327121057984, + "tokens_seen": 615120896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574443329989971, + "loss": 1.4936, + "theoretical_loss": 3.858390322084178, + "tokens_seen": 615186432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574282848545637, + "loss": 1.4568, + "theoretical_loss": 3.8583479378424155, + "tokens_seen": 615251968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574122367101305, + "loss": 1.4072, + "theoretical_loss": 3.858305559379107, + "tokens_seen": 615317504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573961885656971, + "loss": 1.5034, + "theoretical_loss": 3.8582631866928505, + "tokens_seen": 615383040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573801404212639, + "loss": 1.4569, + "theoretical_loss": 3.858220819782243, + "tokens_seen": 615448576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573640922768305, + "loss": 1.4396, + "theoretical_loss": 3.858178458645882, + "tokens_seen": 615514112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573480441323973, + "loss": 1.4184, + "theoretical_loss": 3.858136103282366, + "tokens_seen": 615579648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573319959879639, + "loss": 1.4304, + "theoretical_loss": 3.8580937536902935, + "tokens_seen": 615645184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573159478435307, + "loss": 1.4899, + "theoretical_loss": 3.8580514098682643, + "tokens_seen": 615710720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572998996990973, + "loss": 1.5058, + "theoretical_loss": 3.8580090718148776, + "tokens_seen": 615776256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572838515546641, + "loss": 1.3675, + "theoretical_loss": 3.8579667395287345, + "tokens_seen": 615841792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572678034102307, + "loss": 1.5073, + "theoretical_loss": 3.8579244130084356, + "tokens_seen": 615907328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572517552657975, + "loss": 1.3504, + "theoretical_loss": 3.8578820922525816, + "tokens_seen": 615972864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 211365, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4504954814910889, + "objective/train/theoretical_loss": 3.857839777259775, + "objective/train/tokens_used": 636498400, + "theoretical_loss": 3.857839777259775, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572357071213641, + "loss": 1.4745, + "theoretical_loss": 3.857839777259775, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572196589769309, + "loss": 1.4646, + "theoretical_loss": 3.857797468028618, + "tokens_seen": 616103936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572036108324975, + "loss": 1.4291, + "theoretical_loss": 3.8577551645577124, + "tokens_seen": 616169472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571875626880643, + "loss": 1.4413, + "theoretical_loss": 3.857712866845662, + "tokens_seen": 616235008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571715145436309, + "loss": 1.417, + "theoretical_loss": 3.857670574891071, + "tokens_seen": 616300544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571554663991977, + "loss": 1.469, + "theoretical_loss": 3.857628288692543, + "tokens_seen": 616366080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571394182547643, + "loss": 1.4186, + "theoretical_loss": 3.857586008248683, + "tokens_seen": 616431616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571233701103311, + "loss": 1.5126, + "theoretical_loss": 3.8575437335580958, + "tokens_seen": 616497152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571073219658977, + "loss": 1.418, + "theoretical_loss": 3.8575014646193875, + "tokens_seen": 616562688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570912738214645, + "loss": 1.4638, + "theoretical_loss": 3.8574592014311637, + "tokens_seen": 616628224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570752256770311, + "loss": 1.4218, + "theoretical_loss": 3.857416943992032, + "tokens_seen": 616693760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570591775325979, + "loss": 1.5737, + "theoretical_loss": 3.8573746923005983, + "tokens_seen": 616759296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570431293881645, + "loss": 1.5094, + "theoretical_loss": 3.857332446355471, + "tokens_seen": 616824832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570270812437313, + "loss": 1.3835, + "theoretical_loss": 3.857290206155258, + "tokens_seen": 616890368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570110330992979, + "loss": 1.3138, + "theoretical_loss": 3.8572479716985675, + "tokens_seen": 616955904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569949849548647, + "loss": 1.416, + "theoretical_loss": 3.857205742984009, + "tokens_seen": 617021440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569789368104313, + "loss": 1.3857, + "theoretical_loss": 3.857163520010192, + "tokens_seen": 617086976 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656962888665998, + "loss": 1.4585, + "theoretical_loss": 3.8571213027757274, + "tokens_seen": 617152512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569468405215647, + "loss": 1.3681, + "theoretical_loss": 3.857079091279224, + "tokens_seen": 617218048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569307923771314, + "loss": 1.4211, + "theoretical_loss": 3.857036885519293, + "tokens_seen": 617283584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569147442326981, + "loss": 1.4748, + "theoretical_loss": 3.8569946854945467, + "tokens_seen": 617349120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568986960882648, + "loss": 1.4468, + "theoretical_loss": 3.8569524912035975, + "tokens_seen": 617414656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568826479438315, + "loss": 1.4331, + "theoretical_loss": 3.8569103026450566, + "tokens_seen": 617480192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568665997993982, + "loss": 1.4306, + "theoretical_loss": 3.8568681198175376, + "tokens_seen": 617545728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568505516549649, + "loss": 1.5051, + "theoretical_loss": 3.856825942719654, + "tokens_seen": 617611264 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568345035105316, + "loss": 1.4273, + "theoretical_loss": 3.856783771350019, + "tokens_seen": 617676800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568184553660983, + "loss": 1.4189, + "theoretical_loss": 3.8567416057072474, + "tokens_seen": 617742336 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656802407221665, + "loss": 1.39, + "theoretical_loss": 3.8566994457899546, + "tokens_seen": 617807872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567863590772317, + "loss": 1.4286, + "theoretical_loss": 3.8566572915967554, + "tokens_seen": 617873408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567703109327984, + "loss": 1.449, + "theoretical_loss": 3.8566151431262656, + "tokens_seen": 617938944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567542627883651, + "loss": 1.4484, + "theoretical_loss": 3.856573000377101, + "tokens_seen": 618004480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567382146439318, + "loss": 1.4595, + "theoretical_loss": 3.8565308633478796, + "tokens_seen": 618070016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567221664994985, + "loss": 1.543, + "theoretical_loss": 3.856488732037218, + "tokens_seen": 618135552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567061183550652, + "loss": 1.4479, + "theoretical_loss": 3.856446606443734, + "tokens_seen": 618201088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566900702106319, + "loss": 1.404, + "theoretical_loss": 3.8564044865660456, + "tokens_seen": 618266624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566740220661986, + "loss": 1.3884, + "theoretical_loss": 3.8563623724027716, + "tokens_seen": 618332160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566579739217653, + "loss": 1.5086, + "theoretical_loss": 3.856320263952531, + "tokens_seen": 618397696 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656641925777332, + "loss": 1.4269, + "theoretical_loss": 3.856278161213944, + "tokens_seen": 618463232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566258776328987, + "loss": 1.4405, + "theoretical_loss": 3.856236064185631, + "tokens_seen": 618528768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566098294884654, + "loss": 1.4679, + "theoretical_loss": 3.856193972866212, + "tokens_seen": 618594304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565937813440321, + "loss": 1.4031, + "theoretical_loss": 3.8561518872543075, + "tokens_seen": 618659840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565777331995988, + "loss": 1.3028, + "theoretical_loss": 3.85610980734854, + "tokens_seen": 618725376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565616850551655, + "loss": 1.4489, + "theoretical_loss": 3.8560677331475315, + "tokens_seen": 618790912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565456369107322, + "loss": 1.5921, + "theoretical_loss": 3.8560256646499047, + "tokens_seen": 618856448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565295887662989, + "loss": 1.418, + "theoretical_loss": 3.855983601854282, + "tokens_seen": 618921984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565135406218656, + "loss": 1.5715, + "theoretical_loss": 3.8559415447592875, + "tokens_seen": 618987520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564974924774323, + "loss": 1.4975, + "theoretical_loss": 3.8558994933635447, + "tokens_seen": 619053056 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656481444332999, + "loss": 1.4568, + "theoretical_loss": 3.855857447665678, + "tokens_seen": 619118592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564653961885657, + "loss": 1.4231, + "theoretical_loss": 3.855815407664313, + "tokens_seen": 619184128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564493480441324, + "loss": 1.4654, + "theoretical_loss": 3.8557733733580744, + "tokens_seen": 619249664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 212287, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3307487964630127, + "objective/train/theoretical_loss": 3.855731344745589, + "objective/train/tokens_used": 639775200, + "theoretical_loss": 3.855731344745589, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564332998996991, + "loss": 1.3697, + "theoretical_loss": 3.855731344745589, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564172517552658, + "loss": 1.3096, + "theoretical_loss": 3.8556893218254817, + "tokens_seen": 619380736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564012036108325, + "loss": 1.4491, + "theoretical_loss": 3.855647304596381, + "tokens_seen": 619446272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006563851554663992, + "loss": 1.5931, + "theoretical_loss": 3.8556052930569127, + "tokens_seen": 619511808 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656369107321966, + "loss": 1.4521, + "theoretical_loss": 3.8555632872057055, + "tokens_seen": 619577344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006563530591775326, + "loss": 1.4756, + "theoretical_loss": 3.8555212870413875, + "tokens_seen": 619642880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006563370110330993, + "loss": 1.3477, + "theoretical_loss": 3.8554792925625874, + "tokens_seen": 619708416 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656320962888666, + "loss": 1.4574, + "theoretical_loss": 3.855437303767934, + "tokens_seen": 619773952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006563049147442327, + "loss": 1.4968, + "theoretical_loss": 3.855395320656058, + "tokens_seen": 619839488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562888665997994, + "loss": 1.4919, + "theoretical_loss": 3.8553533432255884, + "tokens_seen": 619905024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562728184553662, + "loss": 1.4488, + "theoretical_loss": 3.855311371475157, + "tokens_seen": 619970560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562567703109328, + "loss": 1.4551, + "theoretical_loss": 3.8552694054033942, + "tokens_seen": 620036096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562407221664996, + "loss": 1.5415, + "theoretical_loss": 3.855227445008931, + "tokens_seen": 620101632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562246740220662, + "loss": 1.4233, + "theoretical_loss": 3.8551854902904017, + "tokens_seen": 620167168 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656208625877633, + "loss": 1.44, + "theoretical_loss": 3.855143541246436, + "tokens_seen": 620232704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561925777331996, + "loss": 1.3843, + "theoretical_loss": 3.855101597875669, + "tokens_seen": 620298240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561765295887664, + "loss": 1.511, + "theoretical_loss": 3.8550596601767326, + "tokens_seen": 620363776 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656160481444333, + "loss": 1.4461, + "theoretical_loss": 3.8550177281482627, + "tokens_seen": 620429312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561444332998998, + "loss": 1.3921, + "theoretical_loss": 3.8549758017888918, + "tokens_seen": 620494848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561283851554664, + "loss": 1.3798, + "theoretical_loss": 3.8549338810972564, + "tokens_seen": 620560384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561123370110332, + "loss": 1.4377, + "theoretical_loss": 3.8548919660719907, + "tokens_seen": 620625920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560962888665998, + "loss": 1.333, + "theoretical_loss": 3.854850056711731, + "tokens_seen": 620691456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560802407221666, + "loss": 1.5228, + "theoretical_loss": 3.8548081530151137, + "tokens_seen": 620756992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560641925777332, + "loss": 1.5282, + "theoretical_loss": 3.8547662549807757, + "tokens_seen": 620822528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560481444333, + "loss": 1.3841, + "theoretical_loss": 3.854724362607354, + "tokens_seen": 620888064 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560320962888666, + "loss": 1.4249, + "theoretical_loss": 3.854682475893486, + "tokens_seen": 620953600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560160481444334, + "loss": 1.4984, + "theoretical_loss": 3.854640594837811, + "tokens_seen": 621019136 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656, + "loss": 1.5547, + "theoretical_loss": 3.8545987194389664, + "tokens_seen": 621084672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559839518555668, + "loss": 1.4241, + "theoretical_loss": 3.8545568496955926, + "tokens_seen": 621150208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559679037111334, + "loss": 1.6195, + "theoretical_loss": 3.854514985606328, + "tokens_seen": 621215744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559518555667002, + "loss": 1.3812, + "theoretical_loss": 3.854473127169814, + "tokens_seen": 621281280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559358074222668, + "loss": 1.3515, + "theoretical_loss": 3.85443127438469, + "tokens_seen": 621346816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559197592778336, + "loss": 1.4354, + "theoretical_loss": 3.8543894272495978, + "tokens_seen": 621412352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559037111334002, + "loss": 1.4532, + "theoretical_loss": 3.8543475857631786, + "tokens_seen": 621477888 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655887662988967, + "loss": 1.4251, + "theoretical_loss": 3.854305749924075, + "tokens_seen": 621543424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558716148445336, + "loss": 1.5207, + "theoretical_loss": 3.854263919730928, + "tokens_seen": 621608960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558555667001004, + "loss": 1.4856, + "theoretical_loss": 3.8542220951823816, + "tokens_seen": 621674496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558395185556671, + "loss": 1.5301, + "theoretical_loss": 3.854180276277079, + "tokens_seen": 621740032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558234704112338, + "loss": 1.4044, + "theoretical_loss": 3.8541384630136646, + "tokens_seen": 621805568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558074222668005, + "loss": 1.3659, + "theoretical_loss": 3.854096655390781, + "tokens_seen": 621871104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557913741223672, + "loss": 1.4789, + "theoretical_loss": 3.8540548534070753, + "tokens_seen": 621936640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557753259779339, + "loss": 1.3902, + "theoretical_loss": 3.854013057061191, + "tokens_seen": 622002176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557592778335006, + "loss": 1.5189, + "theoretical_loss": 3.853971266351775, + "tokens_seen": 622067712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557432296890673, + "loss": 1.4863, + "theoretical_loss": 3.853929481277472, + "tokens_seen": 622133248 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655727181544634, + "loss": 1.5079, + "theoretical_loss": 3.8538877018369306, + "tokens_seen": 622198784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557111334002006, + "loss": 1.387, + "theoretical_loss": 3.8538459280287967, + "tokens_seen": 622264320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556950852557674, + "loss": 1.4602, + "theoretical_loss": 3.853804159851718, + "tokens_seen": 622329856 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655679037111334, + "loss": 1.5986, + "theoretical_loss": 3.853762397304343, + "tokens_seen": 622395392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556629889669008, + "loss": 1.4307, + "theoretical_loss": 3.853720640385319, + "tokens_seen": 622460928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556469408224674, + "loss": 1.4352, + "theoretical_loss": 3.853678889093297, + "tokens_seen": 622526464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 213453, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5051805973052979, + "objective/train/theoretical_loss": 3.8536371434269254, + "objective/train/tokens_used": 643052000, + "theoretical_loss": 3.8536371434269254, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556308926780342, + "loss": 1.5377, + "theoretical_loss": 3.8536371434269254, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556148445336008, + "loss": 1.4414, + "theoretical_loss": 3.853595403384854, + "tokens_seen": 622657536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555987963891676, + "loss": 1.5108, + "theoretical_loss": 3.853553668965733, + "tokens_seen": 622723072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555827482447342, + "loss": 1.486, + "theoretical_loss": 3.8535119401682136, + "tokens_seen": 622788608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555667001003009, + "loss": 1.4433, + "theoretical_loss": 3.8534702169909476, + "tokens_seen": 622854144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555506519558676, + "loss": 1.4521, + "theoretical_loss": 3.853428499432586, + "tokens_seen": 622919680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555346038114343, + "loss": 1.4889, + "theoretical_loss": 3.8533867874917815, + "tokens_seen": 622985216 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655518555667001, + "loss": 1.395, + "theoretical_loss": 3.8533450811671877, + "tokens_seen": 623050752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555025075225677, + "loss": 1.3459, + "theoretical_loss": 3.8533033804574557, + "tokens_seen": 623116288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554864593781344, + "loss": 1.3917, + "theoretical_loss": 3.8532616853612405, + "tokens_seen": 623181824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554704112337011, + "loss": 1.5538, + "theoretical_loss": 3.853219995877196, + "tokens_seen": 623247360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554543630892678, + "loss": 1.418, + "theoretical_loss": 3.853178312003977, + "tokens_seen": 623312896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554383149448345, + "loss": 1.491, + "theoretical_loss": 3.853136633740239, + "tokens_seen": 623378432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554222668004012, + "loss": 1.5011, + "theoretical_loss": 3.853094961084636, + "tokens_seen": 623443968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554062186559679, + "loss": 1.503, + "theoretical_loss": 3.853053294035825, + "tokens_seen": 623509504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553901705115346, + "loss": 1.4062, + "theoretical_loss": 3.8530116325924624, + "tokens_seen": 623575040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553741223671013, + "loss": 1.479, + "theoretical_loss": 3.8529699767532053, + "tokens_seen": 623640576 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655358074222668, + "loss": 1.406, + "theoretical_loss": 3.85292832651671, + "tokens_seen": 623706112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553420260782347, + "loss": 1.4656, + "theoretical_loss": 3.8528866818816354, + "tokens_seen": 623771648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553259779338014, + "loss": 1.4556, + "theoretical_loss": 3.8528450428466403, + "tokens_seen": 623837184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553099297893681, + "loss": 1.3951, + "theoretical_loss": 3.852803409410382, + "tokens_seen": 623902720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552938816449348, + "loss": 1.6086, + "theoretical_loss": 3.85276178157152, + "tokens_seen": 623968256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552778335005015, + "loss": 1.5793, + "theoretical_loss": 3.852720159328715, + "tokens_seen": 624033792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552617853560682, + "loss": 1.5246, + "theoretical_loss": 3.852678542680626, + "tokens_seen": 624099328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552457372116349, + "loss": 1.4312, + "theoretical_loss": 3.8526369316259137, + "tokens_seen": 624164864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552296890672016, + "loss": 1.3875, + "theoretical_loss": 3.8525953261632404, + "tokens_seen": 624230400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552136409227683, + "loss": 1.5285, + "theoretical_loss": 3.852553726291266, + "tokens_seen": 624295936 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655197592778335, + "loss": 1.4572, + "theoretical_loss": 3.8525121320086537, + "tokens_seen": 624361472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551815446339017, + "loss": 1.4146, + "theoretical_loss": 3.8524705433140656, + "tokens_seen": 624427008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551654964894684, + "loss": 1.4862, + "theoretical_loss": 3.8524289602061645, + "tokens_seen": 624492544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551494483450351, + "loss": 1.4295, + "theoretical_loss": 3.8523873826836135, + "tokens_seen": 624558080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551334002006018, + "loss": 1.3939, + "theoretical_loss": 3.8523458107450765, + "tokens_seen": 624623616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551173520561685, + "loss": 1.5243, + "theoretical_loss": 3.8523042443892184, + "tokens_seen": 624689152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551013039117352, + "loss": 1.4286, + "theoretical_loss": 3.852262683614703, + "tokens_seen": 624754688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550852557673019, + "loss": 1.5194, + "theoretical_loss": 3.8522211284201964, + "tokens_seen": 624820224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550692076228686, + "loss": 1.4788, + "theoretical_loss": 3.852179578804364, + "tokens_seen": 624885760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550531594784353, + "loss": 1.404, + "theoretical_loss": 3.852138034765872, + "tokens_seen": 624951296 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655037111334002, + "loss": 1.3126, + "theoretical_loss": 3.8520964963033864, + "tokens_seen": 625016832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550210631895687, + "loss": 1.3926, + "theoretical_loss": 3.8520549634155747, + "tokens_seen": 625082368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550050150451354, + "loss": 1.4594, + "theoretical_loss": 3.852013436101104, + "tokens_seen": 625147904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549889669007021, + "loss": 1.5262, + "theoretical_loss": 3.851971914358643, + "tokens_seen": 625213440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549729187562688, + "loss": 1.4859, + "theoretical_loss": 3.85193039818686, + "tokens_seen": 625278976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549568706118355, + "loss": 1.4388, + "theoretical_loss": 3.8518888875844226, + "tokens_seen": 625344512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549408224674022, + "loss": 1.4641, + "theoretical_loss": 3.8518473825500017, + "tokens_seen": 625410048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549247743229689, + "loss": 1.4174, + "theoretical_loss": 3.8518058830822666, + "tokens_seen": 625475584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549087261785357, + "loss": 1.4281, + "theoretical_loss": 3.8517643891798867, + "tokens_seen": 625541120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548926780341023, + "loss": 1.4682, + "theoretical_loss": 3.8517229008415343, + "tokens_seen": 625606656 + }, + { + "epoch": 0.19, + "learning_rate": 0.000654876629889669, + "loss": 1.4707, + "theoretical_loss": 3.8516814180658794, + "tokens_seen": 625672192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548605817452357, + "loss": 1.5598, + "theoretical_loss": 3.8516399408515936, + "tokens_seen": 625737728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548445336008025, + "loss": 1.549, + "theoretical_loss": 3.8515984691973495, + "tokens_seen": 625803264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 214373, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3765876293182373, + "objective/train/theoretical_loss": 3.85155700310182, + "objective/train/tokens_used": 646328800, + "theoretical_loss": 3.85155700310182, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548284854563691, + "loss": 1.4324, + "theoretical_loss": 3.85155700310182, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548124373119359, + "loss": 1.4596, + "theoretical_loss": 3.8515155425636767, + "tokens_seen": 625934336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547963891675025, + "loss": 1.373, + "theoretical_loss": 3.851474087581594, + "tokens_seen": 625999872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547803410230693, + "loss": 1.5331, + "theoretical_loss": 3.8514326381542454, + "tokens_seen": 626065408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547642928786359, + "loss": 1.4168, + "theoretical_loss": 3.851391194280305, + "tokens_seen": 626130944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547482447342027, + "loss": 1.4006, + "theoretical_loss": 3.8513497559584495, + "tokens_seen": 626196480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547321965897693, + "loss": 1.4852, + "theoretical_loss": 3.851308323187352, + "tokens_seen": 626262016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547161484453361, + "loss": 1.4906, + "theoretical_loss": 3.851266895965689, + "tokens_seen": 626327552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547001003009027, + "loss": 1.4673, + "theoretical_loss": 3.8512254742921357, + "tokens_seen": 626393088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546840521564695, + "loss": 1.4863, + "theoretical_loss": 3.8511840581653707, + "tokens_seen": 626458624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546680040120361, + "loss": 1.3973, + "theoretical_loss": 3.85114264758407, + "tokens_seen": 626524160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546519558676029, + "loss": 1.3883, + "theoretical_loss": 3.8511012425469104, + "tokens_seen": 626589696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546359077231696, + "loss": 1.4545, + "theoretical_loss": 3.851059843052571, + "tokens_seen": 626655232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546198595787363, + "loss": 1.3686, + "theoretical_loss": 3.85101844909973, + "tokens_seen": 626720768 + }, + { + "epoch": 0.19, + "learning_rate": 0.000654603811434303, + "loss": 1.4696, + "theoretical_loss": 3.8509770606870664, + "tokens_seen": 626786304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545877632898697, + "loss": 1.4343, + "theoretical_loss": 3.8509356778132586, + "tokens_seen": 626851840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545717151454364, + "loss": 1.442, + "theoretical_loss": 3.8508943004769876, + "tokens_seen": 626917376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545556670010031, + "loss": 1.3759, + "theoretical_loss": 3.8508529286769324, + "tokens_seen": 626982912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545396188565698, + "loss": 1.4723, + "theoretical_loss": 3.850811562411775, + "tokens_seen": 627048448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545235707121365, + "loss": 1.3319, + "theoretical_loss": 3.8507702016801963, + "tokens_seen": 627113984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545075225677032, + "loss": 1.4292, + "theoretical_loss": 3.850728846480877, + "tokens_seen": 627179520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544914744232699, + "loss": 1.4316, + "theoretical_loss": 3.8506874968125, + "tokens_seen": 627245056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544754262788366, + "loss": 1.3812, + "theoretical_loss": 3.8506461526737477, + "tokens_seen": 627310592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544593781344033, + "loss": 1.4015, + "theoretical_loss": 3.8506048140633027, + "tokens_seen": 627376128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065444332998997, + "loss": 1.4431, + "theoretical_loss": 3.850563480979849, + "tokens_seen": 627441664 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544272818455367, + "loss": 1.4909, + "theoretical_loss": 3.8505221534220695, + "tokens_seen": 627507200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544112337011034, + "loss": 1.395, + "theoretical_loss": 3.8504808313886496, + "tokens_seen": 627572736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543951855566701, + "loss": 1.4019, + "theoretical_loss": 3.850439514878273, + "tokens_seen": 627638272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543791374122368, + "loss": 1.5165, + "theoretical_loss": 3.8503982038896263, + "tokens_seen": 627703808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543630892678035, + "loss": 1.3704, + "theoretical_loss": 3.850356898421394, + "tokens_seen": 627769344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543470411233702, + "loss": 1.3597, + "theoretical_loss": 3.850315598472262, + "tokens_seen": 627834880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543309929789369, + "loss": 1.4165, + "theoretical_loss": 3.8502743040409184, + "tokens_seen": 627900416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543149448345035, + "loss": 1.4233, + "theoretical_loss": 3.850233015126048, + "tokens_seen": 627965952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542988966900703, + "loss": 1.3924, + "theoretical_loss": 3.8501917317263405, + "tokens_seen": 628031488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542828485456369, + "loss": 1.4651, + "theoretical_loss": 3.8501504538404827, + "tokens_seen": 628097024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542668004012037, + "loss": 1.4985, + "theoretical_loss": 3.850109181467163, + "tokens_seen": 628162560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542507522567703, + "loss": 1.4034, + "theoretical_loss": 3.85006791460507, + "tokens_seen": 628228096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542347041123371, + "loss": 1.4475, + "theoretical_loss": 3.850026653252894, + "tokens_seen": 628293632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542186559679037, + "loss": 1.4138, + "theoretical_loss": 3.8499853974093234, + "tokens_seen": 628359168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542026078234705, + "loss": 1.3889, + "theoretical_loss": 3.8499441470730487, + "tokens_seen": 628424704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541865596790371, + "loss": 1.3595, + "theoretical_loss": 3.849902902242761, + "tokens_seen": 628490240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541705115346038, + "loss": 1.4252, + "theoretical_loss": 3.849861662917151, + "tokens_seen": 628555776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541544633901705, + "loss": 1.5239, + "theoretical_loss": 3.8498204290949105, + "tokens_seen": 628621312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541384152457372, + "loss": 1.3816, + "theoretical_loss": 3.849779200774731, + "tokens_seen": 628686848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541223671013039, + "loss": 1.5023, + "theoretical_loss": 3.8497379779553054, + "tokens_seen": 628752384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541063189568706, + "loss": 1.518, + "theoretical_loss": 3.849696760635326, + "tokens_seen": 628817920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540902708124373, + "loss": 1.4259, + "theoretical_loss": 3.8496555488134865, + "tokens_seen": 628883456 + }, + { + "epoch": 0.19, + "learning_rate": 0.000654074222668004, + "loss": 1.534, + "theoretical_loss": 3.84961434248848, + "tokens_seen": 628948992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540581745235707, + "loss": 1.4677, + "theoretical_loss": 3.8495731416590018, + "tokens_seen": 629014528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540421263791374, + "loss": 1.4557, + "theoretical_loss": 3.8495319463237454, + "tokens_seen": 629080064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 215608, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4194252490997314, + "objective/train/theoretical_loss": 3.8494907564814067, + "objective/train/tokens_used": 649605600, + "theoretical_loss": 3.8494907564814067, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540260782347041, + "loss": 1.4274, + "theoretical_loss": 3.8494907564814067, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540100300902708, + "loss": 1.487, + "theoretical_loss": 3.8494495721306805, + "tokens_seen": 629211136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539939819458375, + "loss": 1.5867, + "theoretical_loss": 3.849408393270264, + "tokens_seen": 629276672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539779338014042, + "loss": 1.3724, + "theoretical_loss": 3.849367219898852, + "tokens_seen": 629342208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539618856569709, + "loss": 1.4578, + "theoretical_loss": 3.849326052015142, + "tokens_seen": 629407744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539458375125376, + "loss": 1.5947, + "theoretical_loss": 3.8492848896178318, + "tokens_seen": 629473280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539297893681043, + "loss": 1.4659, + "theoretical_loss": 3.8492437327056184, + "tokens_seen": 629538816 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653913741223671, + "loss": 1.4527, + "theoretical_loss": 3.8492025812772006, + "tokens_seen": 629604352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538976930792377, + "loss": 1.4258, + "theoretical_loss": 3.849161435331277, + "tokens_seen": 629669888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538816449348044, + "loss": 1.429, + "theoretical_loss": 3.849120294866546, + "tokens_seen": 629735424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538655967903711, + "loss": 1.4194, + "theoretical_loss": 3.8490791598817076, + "tokens_seen": 629800960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538495486459378, + "loss": 1.5264, + "theoretical_loss": 3.8490380303754623, + "tokens_seen": 629866496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538335005015045, + "loss": 1.3783, + "theoretical_loss": 3.8489969063465095, + "tokens_seen": 629932032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538174523570712, + "loss": 1.4903, + "theoretical_loss": 3.8489557877935505, + "tokens_seen": 629997568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538014042126379, + "loss": 1.5286, + "theoretical_loss": 3.848914674715287, + "tokens_seen": 630063104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537853560682046, + "loss": 1.4062, + "theoretical_loss": 3.84887356711042, + "tokens_seen": 630128640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537693079237713, + "loss": 1.4923, + "theoretical_loss": 3.8488324649776517, + "tokens_seen": 630194176 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653753259779338, + "loss": 1.3488, + "theoretical_loss": 3.848791368315686, + "tokens_seen": 630259712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537372116349047, + "loss": 1.5031, + "theoretical_loss": 3.848750277123224, + "tokens_seen": 630325248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537211634904714, + "loss": 1.4806, + "theoretical_loss": 3.848709191398971, + "tokens_seen": 630390784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537051153460381, + "loss": 1.3267, + "theoretical_loss": 3.8486681111416297, + "tokens_seen": 630456320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536890672016048, + "loss": 1.4047, + "theoretical_loss": 3.848627036349905, + "tokens_seen": 630521856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536730190571715, + "loss": 1.4194, + "theoretical_loss": 3.848585967022502, + "tokens_seen": 630587392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536569709127382, + "loss": 1.408, + "theoretical_loss": 3.8485449031581256, + "tokens_seen": 630652928 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653640922768305, + "loss": 1.3323, + "theoretical_loss": 3.8485038447554816, + "tokens_seen": 630718464 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536248746238716, + "loss": 1.5932, + "theoretical_loss": 3.848462791813276, + "tokens_seen": 630784000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536088264794383, + "loss": 1.4501, + "theoretical_loss": 3.8484217443302153, + "tokens_seen": 630849536 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653592778335005, + "loss": 1.5799, + "theoretical_loss": 3.8483807023050076, + "tokens_seen": 630915072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535767301905717, + "loss": 1.4477, + "theoretical_loss": 3.848339665736359, + "tokens_seen": 630980608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535606820461384, + "loss": 1.3818, + "theoretical_loss": 3.8482986346229784, + "tokens_seen": 631046144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535446339017052, + "loss": 1.4074, + "theoretical_loss": 3.8482576089635736, + "tokens_seen": 631111680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535285857572718, + "loss": 1.5077, + "theoretical_loss": 3.848216588756854, + "tokens_seen": 631177216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535125376128386, + "loss": 1.4928, + "theoretical_loss": 3.8481755740015275, + "tokens_seen": 631242752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534964894684052, + "loss": 1.4117, + "theoretical_loss": 3.848134564696305, + "tokens_seen": 631308288 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653480441323972, + "loss": 1.4229, + "theoretical_loss": 3.8480935608398967, + "tokens_seen": 631373824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534643931795386, + "loss": 1.3704, + "theoretical_loss": 3.8480525624310125, + "tokens_seen": 631439360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534483450351054, + "loss": 1.4856, + "theoretical_loss": 3.8480115694683636, + "tokens_seen": 631504896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534322968906721, + "loss": 1.4555, + "theoretical_loss": 3.847970581950662, + "tokens_seen": 631570432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534162487462388, + "loss": 1.5628, + "theoretical_loss": 3.8479295998766188, + "tokens_seen": 631635968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534002006018055, + "loss": 1.393, + "theoretical_loss": 3.8478886232449465, + "tokens_seen": 631701504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533841524573722, + "loss": 1.4871, + "theoretical_loss": 3.847847652054358, + "tokens_seen": 631767040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533681043129389, + "loss": 1.4766, + "theoretical_loss": 3.8478066863035663, + "tokens_seen": 631832576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533520561685056, + "loss": 1.4443, + "theoretical_loss": 3.8477657259912856, + "tokens_seen": 631898112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533360080240723, + "loss": 1.4973, + "theoretical_loss": 3.847724771116229, + "tokens_seen": 631963648 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653319959879639, + "loss": 1.4456, + "theoretical_loss": 3.847683821677112, + "tokens_seen": 632029184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533039117352057, + "loss": 1.4088, + "theoretical_loss": 3.847642877672649, + "tokens_seen": 632094720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532878635907724, + "loss": 1.4612, + "theoretical_loss": 3.847601939101555, + "tokens_seen": 632160256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532718154463391, + "loss": 1.5592, + "theoretical_loss": 3.8475610059625467, + "tokens_seen": 632225792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532557673019058, + "loss": 1.484, + "theoretical_loss": 3.84752007825434, + "tokens_seen": 632291328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532397191574725, + "loss": 1.384, + "theoretical_loss": 3.8474791559756514, + "tokens_seen": 632356864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 216545, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.624903678894043, + "objective/train/theoretical_loss": 3.8474382391251982, + "objective/train/tokens_used": 652882400, + "theoretical_loss": 3.8474382391251982, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532236710130392, + "loss": 1.5181, + "theoretical_loss": 3.8474382391251982, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532076228686059, + "loss": 1.3753, + "theoretical_loss": 3.8473973277016977, + "tokens_seen": 632487936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531915747241726, + "loss": 1.4691, + "theoretical_loss": 3.847356421703868, + "tokens_seen": 632553472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531755265797393, + "loss": 1.4561, + "theoretical_loss": 3.8473155211304277, + "tokens_seen": 632619008 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653159478435306, + "loss": 1.507, + "theoretical_loss": 3.847274625980096, + "tokens_seen": 632684544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531434302908727, + "loss": 1.4212, + "theoretical_loss": 3.8472337362515914, + "tokens_seen": 632750080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531273821464394, + "loss": 1.4166, + "theoretical_loss": 3.8471928519436345, + "tokens_seen": 632815616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531113340020061, + "loss": 1.5466, + "theoretical_loss": 3.8471519730549444, + "tokens_seen": 632881152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530952858575728, + "loss": 1.5033, + "theoretical_loss": 3.847111099584242, + "tokens_seen": 632946688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530792377131395, + "loss": 1.443, + "theoretical_loss": 3.8470702315302496, + "tokens_seen": 633012224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530631895687062, + "loss": 1.3511, + "theoretical_loss": 3.847029368891687, + "tokens_seen": 633077760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530471414242729, + "loss": 1.4817, + "theoretical_loss": 3.846988511667277, + "tokens_seen": 633143296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530310932798396, + "loss": 1.3648, + "theoretical_loss": 3.8469476598557417, + "tokens_seen": 633208832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530150451354063, + "loss": 1.353, + "theoretical_loss": 3.8469068134558038, + "tokens_seen": 633274368 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652998996990973, + "loss": 1.3605, + "theoretical_loss": 3.846865972466187, + "tokens_seen": 633339904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529829488465397, + "loss": 1.4737, + "theoretical_loss": 3.846825136885615, + "tokens_seen": 633405440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529669007021064, + "loss": 1.4733, + "theoretical_loss": 3.8467843067128102, + "tokens_seen": 633470976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529508525576731, + "loss": 1.4118, + "theoretical_loss": 3.8467434819464996, + "tokens_seen": 633536512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529348044132398, + "loss": 1.3678, + "theoretical_loss": 3.8467026625854066, + "tokens_seen": 633602048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529187562688064, + "loss": 1.4033, + "theoretical_loss": 3.8466618486282567, + "tokens_seen": 633667584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529187562688064, + "loss": 1.4708, + "theoretical_loss": 3.8466210400737757, + "tokens_seen": 633733120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529027081243732, + "loss": 1.4761, + "theoretical_loss": 3.8465802369206905, + "tokens_seen": 633798656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528866599799398, + "loss": 1.4281, + "theoretical_loss": 3.846539439167727, + "tokens_seen": 633864192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528706118355066, + "loss": 1.5256, + "theoretical_loss": 3.846498646813613, + "tokens_seen": 633929728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528545636910732, + "loss": 1.5862, + "theoretical_loss": 3.8464578598570753, + "tokens_seen": 633995264 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065283851554664, + "loss": 1.442, + "theoretical_loss": 3.8464170782968417, + "tokens_seen": 634060800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528224674022066, + "loss": 1.5478, + "theoretical_loss": 3.8463763021316417, + "tokens_seen": 634126336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528064192577734, + "loss": 1.432, + "theoretical_loss": 3.8463355313602037, + "tokens_seen": 634191872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065279037111334, + "loss": 1.4755, + "theoretical_loss": 3.8462947659812565, + "tokens_seen": 634257408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527743229689067, + "loss": 1.6248, + "theoretical_loss": 3.84625400599353, + "tokens_seen": 634322944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527582748244734, + "loss": 1.3956, + "theoretical_loss": 3.846213251395755, + "tokens_seen": 634388480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527422266800401, + "loss": 1.5417, + "theoretical_loss": 3.846172502186661, + "tokens_seen": 634454016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527261785356068, + "loss": 1.459, + "theoretical_loss": 3.8461317583649794, + "tokens_seen": 634519552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527101303911735, + "loss": 1.4051, + "theoretical_loss": 3.8460910199294416, + "tokens_seen": 634585088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526940822467402, + "loss": 1.447, + "theoretical_loss": 3.8460502868787794, + "tokens_seen": 634650624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526780341023069, + "loss": 1.4647, + "theoretical_loss": 3.8460095592117254, + "tokens_seen": 634716160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526619859578736, + "loss": 1.4076, + "theoretical_loss": 3.845968836927012, + "tokens_seen": 634781696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526459378134403, + "loss": 1.4685, + "theoretical_loss": 3.8459281200233724, + "tokens_seen": 634847232 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652629889669007, + "loss": 1.5148, + "theoretical_loss": 3.8458874084995402, + "tokens_seen": 634912768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526138415245737, + "loss": 1.3509, + "theoretical_loss": 3.8458467023542493, + "tokens_seen": 634978304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525977933801404, + "loss": 1.4074, + "theoretical_loss": 3.845806001586234, + "tokens_seen": 635043840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525817452357071, + "loss": 1.3696, + "theoretical_loss": 3.8457653061942296, + "tokens_seen": 635109376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525656970912738, + "loss": 1.5072, + "theoretical_loss": 3.8457246161769705, + "tokens_seen": 635174912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525496489468405, + "loss": 1.5341, + "theoretical_loss": 3.8456839315331934, + "tokens_seen": 635240448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525336008024072, + "loss": 1.4519, + "theoretical_loss": 3.8456432522616337, + "tokens_seen": 635305984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525175526579739, + "loss": 1.4432, + "theoretical_loss": 3.845602578361029, + "tokens_seen": 635371520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525015045135406, + "loss": 1.5209, + "theoretical_loss": 3.8455619098301144, + "tokens_seen": 635437056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524854563691073, + "loss": 1.3726, + "theoretical_loss": 3.8455212466676287, + "tokens_seen": 635502592 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652469408224674, + "loss": 1.4938, + "theoretical_loss": 3.8454805888723094, + "tokens_seen": 635568128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524533600802407, + "loss": 1.4569, + "theoretical_loss": 3.845439936442895, + "tokens_seen": 635633664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 217757, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.47322678565979, + "objective/train/theoretical_loss": 3.8453992893781246, + "objective/train/tokens_used": 656159200, + "theoretical_loss": 3.8453992893781246, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524373119358074, + "loss": 1.4815, + "theoretical_loss": 3.8453992893781246, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524212637913741, + "loss": 1.4739, + "theoretical_loss": 3.845358647676736, + "tokens_seen": 635764736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524052156469408, + "loss": 1.4726, + "theoretical_loss": 3.8453180113374694, + "tokens_seen": 635830272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523891675025075, + "loss": 1.434, + "theoretical_loss": 3.845277380359065, + "tokens_seen": 635895808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523731193580742, + "loss": 1.5007, + "theoretical_loss": 3.845236754740263, + "tokens_seen": 635961344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523570712136409, + "loss": 1.4894, + "theoretical_loss": 3.8451961344798042, + "tokens_seen": 636026880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523410230692076, + "loss": 1.3507, + "theoretical_loss": 3.84515551957643, + "tokens_seen": 636092416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523249749247743, + "loss": 1.4423, + "theoretical_loss": 3.845114910028882, + "tokens_seen": 636157952 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652308926780341, + "loss": 1.5087, + "theoretical_loss": 3.8450743058359023, + "tokens_seen": 636223488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522928786359077, + "loss": 1.4043, + "theoretical_loss": 3.845033706996233, + "tokens_seen": 636289024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522768304914744, + "loss": 1.3376, + "theoretical_loss": 3.8449931135086177, + "tokens_seen": 636354560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522607823470411, + "loss": 1.4942, + "theoretical_loss": 3.8449525253717995, + "tokens_seen": 636420096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522447342026078, + "loss": 1.354, + "theoretical_loss": 3.844911942584522, + "tokens_seen": 636485632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522286860581746, + "loss": 1.4262, + "theoretical_loss": 3.84487136514553, + "tokens_seen": 636551168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522126379137412, + "loss": 1.5158, + "theoretical_loss": 3.8448307930535672, + "tokens_seen": 636616704 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652196589769308, + "loss": 1.5942, + "theoretical_loss": 3.8447902263073797, + "tokens_seen": 636682240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521805416248747, + "loss": 1.4652, + "theoretical_loss": 3.8447496649057125, + "tokens_seen": 636747776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521644934804414, + "loss": 1.5467, + "theoretical_loss": 3.844709108847311, + "tokens_seen": 636813312 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652148445336008, + "loss": 1.5092, + "theoretical_loss": 3.8446685581309223, + "tokens_seen": 636878848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521323971915748, + "loss": 1.3187, + "theoretical_loss": 3.844628012755293, + "tokens_seen": 636944384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521163490471415, + "loss": 1.4752, + "theoretical_loss": 3.8445874727191702, + "tokens_seen": 637009920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521003009027082, + "loss": 1.3897, + "theoretical_loss": 3.8445469380213013, + "tokens_seen": 637075456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520842527582749, + "loss": 1.5081, + "theoretical_loss": 3.844506408660435, + "tokens_seen": 637140992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520682046138416, + "loss": 1.519, + "theoretical_loss": 3.844465884635319, + "tokens_seen": 637206528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520521564694083, + "loss": 1.4463, + "theoretical_loss": 3.8444253659447023, + "tokens_seen": 637272064 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652036108324975, + "loss": 1.4525, + "theoretical_loss": 3.8443848525873343, + "tokens_seen": 637337600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520200601805417, + "loss": 1.4409, + "theoretical_loss": 3.844344344561965, + "tokens_seen": 637403136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520040120361084, + "loss": 1.4772, + "theoretical_loss": 3.844303841867344, + "tokens_seen": 637468672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519879638916751, + "loss": 1.4162, + "theoretical_loss": 3.8442633445022225, + "tokens_seen": 637534208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519719157472418, + "loss": 1.399, + "theoretical_loss": 3.844222852465351, + "tokens_seen": 637599744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519558676028085, + "loss": 1.4368, + "theoretical_loss": 3.844182365755481, + "tokens_seen": 637665280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519398194583752, + "loss": 1.4406, + "theoretical_loss": 3.844141884371364, + "tokens_seen": 637730816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519237713139419, + "loss": 1.5643, + "theoretical_loss": 3.8441014083117535, + "tokens_seen": 637796352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519077231695086, + "loss": 1.5508, + "theoretical_loss": 3.8440609375754007, + "tokens_seen": 637861888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518916750250753, + "loss": 1.3945, + "theoretical_loss": 3.844020472161059, + "tokens_seen": 637927424 + }, + { + "epoch": 0.19, + "learning_rate": 0.000651875626880642, + "loss": 1.3856, + "theoretical_loss": 3.8439800120674827, + "tokens_seen": 637992960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518595787362087, + "loss": 1.4326, + "theoretical_loss": 3.843939557293425, + "tokens_seen": 638058496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518435305917754, + "loss": 1.5456, + "theoretical_loss": 3.8438991078376405, + "tokens_seen": 638124032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518274824473421, + "loss": 1.4321, + "theoretical_loss": 3.843858663698884, + "tokens_seen": 638189568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518114343029088, + "loss": 1.3336, + "theoretical_loss": 3.84381822487591, + "tokens_seen": 638255104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517953861584755, + "loss": 1.3373, + "theoretical_loss": 3.843777791367475, + "tokens_seen": 638320640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517793380140422, + "loss": 1.4302, + "theoretical_loss": 3.843737363172335, + "tokens_seen": 638386176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517632898696089, + "loss": 1.4137, + "theoretical_loss": 3.843696940289246, + "tokens_seen": 638451712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517472417251756, + "loss": 1.396, + "theoretical_loss": 3.843656522716965, + "tokens_seen": 638517248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517311935807423, + "loss": 1.4537, + "theoretical_loss": 3.843616110454249, + "tokens_seen": 638582784 + }, + { + "epoch": 0.19, + "learning_rate": 0.000651715145436309, + "loss": 1.4281, + "theoretical_loss": 3.843575703499856, + "tokens_seen": 638648320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516990972918757, + "loss": 1.3999, + "theoretical_loss": 3.843535301852544, + "tokens_seen": 638713856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516830491474424, + "loss": 1.3964, + "theoretical_loss": 3.843494905511072, + "tokens_seen": 638779392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516670010030091, + "loss": 1.3862, + "theoretical_loss": 3.8434545144741983, + "tokens_seen": 638844928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516509528585758, + "loss": 1.5054, + "theoretical_loss": 3.8434141287406827, + "tokens_seen": 638910464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 218641, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4308626651763916, + "objective/train/theoretical_loss": 3.8433737483092845, + "objective/train/tokens_used": 659436000, + "theoretical_loss": 3.8433737483092845, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516349047141425, + "loss": 1.4943, + "theoretical_loss": 3.8433737483092845, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516188565697092, + "loss": 1.4438, + "theoretical_loss": 3.8433333731787647, + "tokens_seen": 639041536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516028084252759, + "loss": 1.5739, + "theoretical_loss": 3.843293003347883, + "tokens_seen": 639107072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515867602808426, + "loss": 1.3414, + "theoretical_loss": 3.843252638815401, + "tokens_seen": 639172608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515707121364093, + "loss": 1.4464, + "theoretical_loss": 3.8432122795800803, + "tokens_seen": 639238144 + }, + { + "epoch": 0.19, + "learning_rate": 0.000651554663991976, + "loss": 1.51, + "theoretical_loss": 3.843171925640682, + "tokens_seen": 639303680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515386158475427, + "loss": 1.4377, + "theoretical_loss": 3.8431315769959697, + "tokens_seen": 639369216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515225677031094, + "loss": 1.4204, + "theoretical_loss": 3.8430912336447047, + "tokens_seen": 639434752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515065195586761, + "loss": 1.5045, + "theoretical_loss": 3.8430508955856504, + "tokens_seen": 639500288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514904714142427, + "loss": 1.4237, + "theoretical_loss": 3.843010562817571, + "tokens_seen": 639565824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514744232698095, + "loss": 1.4702, + "theoretical_loss": 3.84297023533923, + "tokens_seen": 639631360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514583751253761, + "loss": 1.4988, + "theoretical_loss": 3.842929913149392, + "tokens_seen": 639696896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514423269809429, + "loss": 1.4406, + "theoretical_loss": 3.8428895962468212, + "tokens_seen": 639762432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514262788365095, + "loss": 1.4651, + "theoretical_loss": 3.842849284630284, + "tokens_seen": 639827968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514102306920763, + "loss": 1.4929, + "theoretical_loss": 3.8428089782985446, + "tokens_seen": 639893504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513941825476429, + "loss": 1.4309, + "theoretical_loss": 3.84276867725037, + "tokens_seen": 639959040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513781344032097, + "loss": 1.4411, + "theoretical_loss": 3.842728381484526, + "tokens_seen": 640024576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513620862587763, + "loss": 1.3425, + "theoretical_loss": 3.8426880909997796, + "tokens_seen": 640090112 + }, + { + "epoch": 0.19, + "learning_rate": 0.000651346038114343, + "loss": 1.5662, + "theoretical_loss": 3.8426478057948983, + "tokens_seen": 640155648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513299899699097, + "loss": 1.5147, + "theoretical_loss": 3.8426075258686496, + "tokens_seen": 640221184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513139418254764, + "loss": 1.4709, + "theoretical_loss": 3.842567251219802, + "tokens_seen": 640286720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512978936810431, + "loss": 1.4913, + "theoretical_loss": 3.8425269818471235, + "tokens_seen": 640352256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512818455366098, + "loss": 1.4533, + "theoretical_loss": 3.8424867177493836, + "tokens_seen": 640417792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512657973921765, + "loss": 1.3171, + "theoretical_loss": 3.8424464589253504, + "tokens_seen": 640483328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512497492477432, + "loss": 1.42, + "theoretical_loss": 3.842406205373795, + "tokens_seen": 640548864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512337011033099, + "loss": 1.474, + "theoretical_loss": 3.8423659570934867, + "tokens_seen": 640614400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512176529588766, + "loss": 1.3579, + "theoretical_loss": 3.842325714083197, + "tokens_seen": 640679936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512016048144433, + "loss": 1.4301, + "theoretical_loss": 3.842285476341696, + "tokens_seen": 640745472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065118555667001, + "loss": 1.3874, + "theoretical_loss": 3.842245243867755, + "tokens_seen": 640811008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511695085255767, + "loss": 1.4085, + "theoretical_loss": 3.842205016660147, + "tokens_seen": 640876544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511534603811434, + "loss": 1.4344, + "theoretical_loss": 3.8421647947176423, + "tokens_seen": 640942080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511374122367101, + "loss": 1.438, + "theoretical_loss": 3.842124578039016, + "tokens_seen": 641007616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511213640922768, + "loss": 1.5147, + "theoretical_loss": 3.842084366623039, + "tokens_seen": 641073152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511053159478435, + "loss": 1.4703, + "theoretical_loss": 3.842044160468485, + "tokens_seen": 641138688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510892678034102, + "loss": 1.4338, + "theoretical_loss": 3.8420039595741295, + "tokens_seen": 641204224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510732196589769, + "loss": 1.3835, + "theoretical_loss": 3.841963763938746, + "tokens_seen": 641269760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510571715145436, + "loss": 1.4765, + "theoretical_loss": 3.8419235735611075, + "tokens_seen": 641335296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510411233701103, + "loss": 1.5916, + "theoretical_loss": 3.8418833884399914, + "tokens_seen": 641400832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510250752256771, + "loss": 1.4546, + "theoretical_loss": 3.841843208574172, + "tokens_seen": 641466368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510090270812437, + "loss": 1.5065, + "theoretical_loss": 3.8418030339624254, + "tokens_seen": 641531904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509929789368105, + "loss": 1.4235, + "theoretical_loss": 3.8417628646035284, + "tokens_seen": 641597440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509769307923771, + "loss": 1.413, + "theoretical_loss": 3.8417227004962573, + "tokens_seen": 641662976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509608826479439, + "loss": 1.4879, + "theoretical_loss": 3.8416825416393894, + "tokens_seen": 641728512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509448345035105, + "loss": 1.531, + "theoretical_loss": 3.841642388031702, + "tokens_seen": 641794048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509287863590773, + "loss": 1.4144, + "theoretical_loss": 3.8416022396719733, + "tokens_seen": 641859584 + }, + { + "epoch": 0.19, + "learning_rate": 0.000650912738214644, + "loss": 1.4167, + "theoretical_loss": 3.841562096558982, + "tokens_seen": 641925120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508966900702107, + "loss": 1.363, + "theoretical_loss": 3.841521958691506, + "tokens_seen": 641990656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508806419257773, + "loss": 1.3944, + "theoretical_loss": 3.841481826068325, + "tokens_seen": 642056192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508645937813441, + "loss": 1.3854, + "theoretical_loss": 3.841441698688219, + "tokens_seen": 642121728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508485456369107, + "loss": 1.439, + "theoretical_loss": 3.8414015765499676, + "tokens_seen": 642187264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 219942, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3430824279785156, + "objective/train/theoretical_loss": 3.841361459652351, + "objective/train/tokens_used": 662712800, + "theoretical_loss": 3.841361459652351, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508324974924775, + "loss": 1.3155, + "theoretical_loss": 3.841361459652351, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508164493480441, + "loss": 1.4824, + "theoretical_loss": 3.8413213479941506, + "tokens_seen": 642318336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508004012036109, + "loss": 1.3852, + "theoretical_loss": 3.841281241574147, + "tokens_seen": 642383872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507843530591776, + "loss": 1.5093, + "theoretical_loss": 3.841241140391123, + "tokens_seen": 642449408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507683049147443, + "loss": 1.4804, + "theoretical_loss": 3.8412010444438587, + "tokens_seen": 642514944 + }, + { + "epoch": 0.19, + "learning_rate": 0.000650752256770311, + "loss": 1.4636, + "theoretical_loss": 3.8411609537311384, + "tokens_seen": 642580480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507362086258777, + "loss": 1.3214, + "theoretical_loss": 3.841120868251744, + "tokens_seen": 642646016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507201604814444, + "loss": 1.4039, + "theoretical_loss": 3.8410807880044593, + "tokens_seen": 642711552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507041123370111, + "loss": 1.5107, + "theoretical_loss": 3.8410407129880673, + "tokens_seen": 642777088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506880641925778, + "loss": 1.4625, + "theoretical_loss": 3.841000643201353, + "tokens_seen": 642842624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506720160481445, + "loss": 1.4997, + "theoretical_loss": 3.8409605786431, + "tokens_seen": 642908160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506559679037112, + "loss": 1.2775, + "theoretical_loss": 3.840920519312094, + "tokens_seen": 642973696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506399197592779, + "loss": 1.4313, + "theoretical_loss": 3.8408804652071193, + "tokens_seen": 643039232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506238716148446, + "loss": 1.4374, + "theoretical_loss": 3.8408404163269627, + "tokens_seen": 643104768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506078234704113, + "loss": 1.419, + "theoretical_loss": 3.84080037267041, + "tokens_seen": 643170304 + }, + { + "epoch": 0.19, + "learning_rate": 0.000650591775325978, + "loss": 1.4437, + "theoretical_loss": 3.840760334236247, + "tokens_seen": 643235840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006505757271815447, + "loss": 1.4948, + "theoretical_loss": 3.8407203010232616, + "tokens_seen": 643301376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006505596790371114, + "loss": 1.4523, + "theoretical_loss": 3.840680273030241, + "tokens_seen": 643366912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006505436308926781, + "loss": 1.5176, + "theoretical_loss": 3.840640250255972, + "tokens_seen": 643432448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006505275827482448, + "loss": 1.4363, + "theoretical_loss": 3.840600232699244, + "tokens_seen": 643497984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006505115346038115, + "loss": 1.4636, + "theoretical_loss": 3.8405602203588445, + "tokens_seen": 643563520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504954864593782, + "loss": 1.4343, + "theoretical_loss": 3.8405202132335634, + "tokens_seen": 643629056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504794383149449, + "loss": 1.3896, + "theoretical_loss": 3.8404802113221894, + "tokens_seen": 643694592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504633901705116, + "loss": 1.3487, + "theoretical_loss": 3.8404402146235124, + "tokens_seen": 643760128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504473420260783, + "loss": 1.3639, + "theoretical_loss": 3.8404002231363226, + "tokens_seen": 643825664 + }, + { + "epoch": 0.2, + "learning_rate": 0.000650431293881645, + "loss": 1.398, + "theoretical_loss": 3.8403602368594107, + "tokens_seen": 643891200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504152457372117, + "loss": 1.4373, + "theoretical_loss": 3.8403202557915677, + "tokens_seen": 643956736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503991975927784, + "loss": 1.3552, + "theoretical_loss": 3.8402802799315845, + "tokens_seen": 644022272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503831494483451, + "loss": 1.3018, + "theoretical_loss": 3.8402403092782533, + "tokens_seen": 644087808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503671013039118, + "loss": 1.4913, + "theoretical_loss": 3.8402003438303662, + "tokens_seen": 644153344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503510531594785, + "loss": 1.5308, + "theoretical_loss": 3.840160383586716, + "tokens_seen": 644218880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503350050150452, + "loss": 1.5119, + "theoretical_loss": 3.8401204285460957, + "tokens_seen": 644284416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503189568706119, + "loss": 1.3809, + "theoretical_loss": 3.840080478707298, + "tokens_seen": 644349952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503029087261786, + "loss": 1.4947, + "theoretical_loss": 3.840040534069118, + "tokens_seen": 644415488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502868605817453, + "loss": 1.4467, + "theoretical_loss": 3.8400005946303484, + "tokens_seen": 644481024 + }, + { + "epoch": 0.2, + "learning_rate": 0.000650270812437312, + "loss": 1.4591, + "theoretical_loss": 3.8399606603897847, + "tokens_seen": 644546560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502547642928787, + "loss": 1.4746, + "theoretical_loss": 3.8399207313462216, + "tokens_seen": 644612096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502387161484454, + "loss": 1.4329, + "theoretical_loss": 3.8398808074984547, + "tokens_seen": 644677632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502226680040121, + "loss": 1.3211, + "theoretical_loss": 3.8398408888452797, + "tokens_seen": 644743168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502066198595788, + "loss": 1.4165, + "theoretical_loss": 3.8398009753854927, + "tokens_seen": 644808704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501905717151455, + "loss": 1.3759, + "theoretical_loss": 3.839761067117891, + "tokens_seen": 644874240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501745235707122, + "loss": 1.4048, + "theoretical_loss": 3.8397211640412703, + "tokens_seen": 644939776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501584754262789, + "loss": 1.4058, + "theoretical_loss": 3.839681266154429, + "tokens_seen": 645005312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501424272818456, + "loss": 1.3807, + "theoretical_loss": 3.839641373456165, + "tokens_seen": 645070848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501263791374123, + "loss": 1.535, + "theoretical_loss": 3.8396014859452756, + "tokens_seen": 645136384 + }, + { + "epoch": 0.2, + "learning_rate": 0.000650110330992979, + "loss": 1.3981, + "theoretical_loss": 3.8395616036205604, + "tokens_seen": 645201920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500942828485456, + "loss": 1.52, + "theoretical_loss": 3.839521726480818, + "tokens_seen": 645267456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500782347041124, + "loss": 1.5095, + "theoretical_loss": 3.8394818545248475, + "tokens_seen": 645332992 + }, + { + "epoch": 0.2, + "learning_rate": 0.000650062186559679, + "loss": 1.4019, + "theoretical_loss": 3.8394419877514494, + "tokens_seen": 645398528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500461384152458, + "loss": 1.4695, + "theoretical_loss": 3.8394021261594236, + "tokens_seen": 645464064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 220851, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.384150505065918, + "objective/train/theoretical_loss": 3.8393622697475704, + "objective/train/tokens_used": 665989600, + "theoretical_loss": 3.8393622697475704, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500300902708124, + "loss": 1.34, + "theoretical_loss": 3.8393622697475704, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500140421263792, + "loss": 1.4082, + "theoretical_loss": 3.839322418514691, + "tokens_seen": 645595136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499979939819458, + "loss": 1.4156, + "theoretical_loss": 3.839282572459587, + "tokens_seen": 645660672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499819458375126, + "loss": 1.4836, + "theoretical_loss": 3.83924273158106, + "tokens_seen": 645726208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499658976930792, + "loss": 1.5027, + "theoretical_loss": 3.8392028958779125, + "tokens_seen": 645791744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499498495486459, + "loss": 1.4867, + "theoretical_loss": 3.8391630653489472, + "tokens_seen": 645857280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499338014042126, + "loss": 1.4544, + "theoretical_loss": 3.8391232399929667, + "tokens_seen": 645922816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499177532597793, + "loss": 1.4677, + "theoretical_loss": 3.839083419808774, + "tokens_seen": 645988352 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649901705115346, + "loss": 1.4005, + "theoretical_loss": 3.839043604795174, + "tokens_seen": 646053888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498856569709127, + "loss": 1.3997, + "theoretical_loss": 3.83900379495097, + "tokens_seen": 646119424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498696088264794, + "loss": 1.3995, + "theoretical_loss": 3.8389639902749675, + "tokens_seen": 646184960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498535606820461, + "loss": 1.4303, + "theoretical_loss": 3.83892419076597, + "tokens_seen": 646250496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498375125376128, + "loss": 1.4978, + "theoretical_loss": 3.8388843964227846, + "tokens_seen": 646316032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498214643931796, + "loss": 1.473, + "theoretical_loss": 3.8388446072442157, + "tokens_seen": 646381568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498054162487462, + "loss": 1.5414, + "theoretical_loss": 3.8388048232290704, + "tokens_seen": 646447104 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649789368104313, + "loss": 1.4878, + "theoretical_loss": 3.8387650443761556, + "tokens_seen": 646512640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006497733199598796, + "loss": 1.434, + "theoretical_loss": 3.838725270684277, + "tokens_seen": 646578176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006497572718154464, + "loss": 1.5353, + "theoretical_loss": 3.8386855021522424, + "tokens_seen": 646643712 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649741223671013, + "loss": 1.3599, + "theoretical_loss": 3.83864573877886, + "tokens_seen": 646709248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006497251755265798, + "loss": 1.4096, + "theoretical_loss": 3.8386059805629382, + "tokens_seen": 646774784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006497091273821464, + "loss": 1.4407, + "theoretical_loss": 3.838566227503285, + "tokens_seen": 646840320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496930792377132, + "loss": 1.4622, + "theoretical_loss": 3.838526479598709, + "tokens_seen": 646905856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496770310932798, + "loss": 1.3577, + "theoretical_loss": 3.838486736848021, + "tokens_seen": 646971392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496609829488466, + "loss": 1.4514, + "theoretical_loss": 3.8384469992500296, + "tokens_seen": 647036928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496449348044132, + "loss": 1.4493, + "theoretical_loss": 3.8384072668035447, + "tokens_seen": 647102464 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064962888665998, + "loss": 1.5102, + "theoretical_loss": 3.8383675395073777, + "tokens_seen": 647168000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496128385155466, + "loss": 1.3916, + "theoretical_loss": 3.838327817360339, + "tokens_seen": 647233536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495967903711134, + "loss": 1.3993, + "theoretical_loss": 3.8382881003612406, + "tokens_seen": 647299072 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064958074222668, + "loss": 1.4586, + "theoretical_loss": 3.8382483885088927, + "tokens_seen": 647364608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495646940822468, + "loss": 1.5188, + "theoretical_loss": 3.8382086818021097, + "tokens_seen": 647430144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495486459378134, + "loss": 1.485, + "theoretical_loss": 3.838168980239702, + "tokens_seen": 647495680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495325977933802, + "loss": 1.5177, + "theoretical_loss": 3.838129283820484, + "tokens_seen": 647561216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495165496489468, + "loss": 1.4066, + "theoretical_loss": 3.838089592543268, + "tokens_seen": 647626752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495005015045136, + "loss": 1.3561, + "theoretical_loss": 3.8380499064068685, + "tokens_seen": 647692288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006494844533600802, + "loss": 1.4377, + "theoretical_loss": 3.8380102254100983, + "tokens_seen": 647757824 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649468405215647, + "loss": 1.3284, + "theoretical_loss": 3.8379705495517733, + "tokens_seen": 647823360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006494523570712136, + "loss": 1.4504, + "theoretical_loss": 3.8379308788307083, + "tokens_seen": 647888896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006494363089267804, + "loss": 1.3618, + "theoretical_loss": 3.837891213245718, + "tokens_seen": 647954432 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649420260782347, + "loss": 1.4628, + "theoretical_loss": 3.837851552795618, + "tokens_seen": 648019968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006494042126379138, + "loss": 1.5271, + "theoretical_loss": 3.8378118974792246, + "tokens_seen": 648085504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493881644934805, + "loss": 1.4865, + "theoretical_loss": 3.837772247295354, + "tokens_seen": 648151040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493721163490472, + "loss": 1.5861, + "theoretical_loss": 3.8377326022428235, + "tokens_seen": 648216576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493560682046139, + "loss": 1.4137, + "theoretical_loss": 3.8376929623204505, + "tokens_seen": 648282112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493400200601806, + "loss": 1.4333, + "theoretical_loss": 3.8376533275270517, + "tokens_seen": 648347648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493239719157473, + "loss": 1.453, + "theoretical_loss": 3.837613697861446, + "tokens_seen": 648413184 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649307923771314, + "loss": 1.5266, + "theoretical_loss": 3.8375740733224513, + "tokens_seen": 648478720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492918756268807, + "loss": 1.4287, + "theoretical_loss": 3.8375344539088867, + "tokens_seen": 648544256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492758274824474, + "loss": 1.495, + "theoretical_loss": 3.837494839619571, + "tokens_seen": 648609792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492597793380141, + "loss": 1.3091, + "theoretical_loss": 3.8374552304533243, + "tokens_seen": 648675328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492437311935808, + "loss": 1.4309, + "theoretical_loss": 3.8374156264089665, + "tokens_seen": 648740864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 222090, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4386017322540283, + "objective/train/theoretical_loss": 3.8373760274853175, + "objective/train/tokens_used": 669266400, + "theoretical_loss": 3.8373760274853175, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492276830491475, + "loss": 1.4186, + "theoretical_loss": 3.8373760274853175, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492116349047142, + "loss": 1.5391, + "theoretical_loss": 3.8373364336811986, + "tokens_seen": 648871936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491955867602809, + "loss": 1.4377, + "theoretical_loss": 3.8372968449954308, + "tokens_seen": 648937472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491795386158476, + "loss": 1.3458, + "theoretical_loss": 3.8372572614268354, + "tokens_seen": 649003008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491634904714143, + "loss": 1.4029, + "theoretical_loss": 3.8372176829742344, + "tokens_seen": 649068544 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649147442326981, + "loss": 1.4743, + "theoretical_loss": 3.837178109636451, + "tokens_seen": 649134080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491313941825477, + "loss": 1.5013, + "theoretical_loss": 3.837138541412306, + "tokens_seen": 649199616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491153460381144, + "loss": 1.3991, + "theoretical_loss": 3.837098978300624, + "tokens_seen": 649265152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490992978936811, + "loss": 1.533, + "theoretical_loss": 3.8370594203002284, + "tokens_seen": 649330688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490832497492478, + "loss": 1.4891, + "theoretical_loss": 3.8370198674099427, + "tokens_seen": 649396224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490672016048145, + "loss": 1.4534, + "theoretical_loss": 3.8369803196285917, + "tokens_seen": 649461760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490511534603812, + "loss": 1.469, + "theoretical_loss": 3.8369407769549992, + "tokens_seen": 649527296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490351053159479, + "loss": 1.3051, + "theoretical_loss": 3.8369012393879904, + "tokens_seen": 649592832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490190571715146, + "loss": 1.443, + "theoretical_loss": 3.836861706926392, + "tokens_seen": 649658368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490030090270813, + "loss": 1.463, + "theoretical_loss": 3.836822179569028, + "tokens_seen": 649723904 + }, + { + "epoch": 0.2, + "learning_rate": 0.000648986960882648, + "loss": 1.3889, + "theoretical_loss": 3.8367826573147257, + "tokens_seen": 649789440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489709127382147, + "loss": 1.5045, + "theoretical_loss": 3.836743140162312, + "tokens_seen": 649854976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489548645937814, + "loss": 1.3668, + "theoretical_loss": 3.8367036281106124, + "tokens_seen": 649920512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489388164493481, + "loss": 1.4691, + "theoretical_loss": 3.836664121158456, + "tokens_seen": 649986048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489227683049148, + "loss": 1.4015, + "theoretical_loss": 3.83662461930467, + "tokens_seen": 650051584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489067201604815, + "loss": 1.473, + "theoretical_loss": 3.836585122548082, + "tokens_seen": 650117120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488906720160482, + "loss": 1.3815, + "theoretical_loss": 3.836545630887521, + "tokens_seen": 650182656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488746238716149, + "loss": 1.4541, + "theoretical_loss": 3.836506144321816, + "tokens_seen": 650248192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488585757271816, + "loss": 1.3895, + "theoretical_loss": 3.836466662849795, + "tokens_seen": 650313728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488425275827483, + "loss": 1.4039, + "theoretical_loss": 3.8364271864702904, + "tokens_seen": 650379264 + }, + { + "epoch": 0.2, + "learning_rate": 0.000648826479438315, + "loss": 1.4246, + "theoretical_loss": 3.8363877151821297, + "tokens_seen": 650444800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488104312938817, + "loss": 1.4037, + "theoretical_loss": 3.8363482489841454, + "tokens_seen": 650510336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487943831494484, + "loss": 1.4236, + "theoretical_loss": 3.8363087878751667, + "tokens_seen": 650575872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487783350050151, + "loss": 1.4738, + "theoretical_loss": 3.8362693318540257, + "tokens_seen": 650641408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487622868605818, + "loss": 1.4377, + "theoretical_loss": 3.8362298809195545, + "tokens_seen": 650706944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487462387161485, + "loss": 1.4915, + "theoretical_loss": 3.8361904350705833, + "tokens_seen": 650772480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487301905717152, + "loss": 1.3687, + "theoretical_loss": 3.836150994305947, + "tokens_seen": 650838016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487141424272819, + "loss": 1.4345, + "theoretical_loss": 3.8361115586244763, + "tokens_seen": 650903552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486980942828485, + "loss": 1.4897, + "theoretical_loss": 3.8360721280250054, + "tokens_seen": 650969088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486820461384153, + "loss": 1.3686, + "theoretical_loss": 3.836032702506367, + "tokens_seen": 651034624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486659979939819, + "loss": 1.4817, + "theoretical_loss": 3.8359932820673968, + "tokens_seen": 651100160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486499498495487, + "loss": 1.5604, + "theoretical_loss": 3.8359538667069275, + "tokens_seen": 651165696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486339017051153, + "loss": 1.4949, + "theoretical_loss": 3.8359144564237946, + "tokens_seen": 651231232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486178535606821, + "loss": 1.4686, + "theoretical_loss": 3.8358750512168323, + "tokens_seen": 651296768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486018054162487, + "loss": 1.4227, + "theoretical_loss": 3.8358356510848775, + "tokens_seen": 651362304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485857572718155, + "loss": 1.4537, + "theoretical_loss": 3.8357962560267644, + "tokens_seen": 651427840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485697091273821, + "loss": 1.4057, + "theoretical_loss": 3.835756866041331, + "tokens_seen": 651493376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485536609829489, + "loss": 1.3665, + "theoretical_loss": 3.8357174811274133, + "tokens_seen": 651558912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485376128385155, + "loss": 1.4087, + "theoretical_loss": 3.8356781012838472, + "tokens_seen": 651624448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485215646940823, + "loss": 1.4461, + "theoretical_loss": 3.8356387265094716, + "tokens_seen": 651689984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485055165496489, + "loss": 1.4452, + "theoretical_loss": 3.835599356803124, + "tokens_seen": 651755520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484894684052157, + "loss": 1.3589, + "theoretical_loss": 3.835559992163642, + "tokens_seen": 651821056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484734202607823, + "loss": 1.4137, + "theoretical_loss": 3.835520632589864, + "tokens_seen": 651886592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484573721163491, + "loss": 1.4278, + "theoretical_loss": 3.83548127808063, + "tokens_seen": 651952128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484413239719157, + "loss": 1.5026, + "theoretical_loss": 3.8354419286347783, + "tokens_seen": 652017664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 222990, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3786110877990723, + "objective/train/theoretical_loss": 3.8354025842511494, + "objective/train/tokens_used": 672543200, + "theoretical_loss": 3.8354025842511494, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484252758274825, + "loss": 1.5055, + "theoretical_loss": 3.8354025842511494, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484092276830491, + "loss": 1.4173, + "theoretical_loss": 3.8353632449285833, + "tokens_seen": 652148736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483931795386159, + "loss": 1.3441, + "theoretical_loss": 3.8353239106659194, + "tokens_seen": 652214272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483771313941825, + "loss": 1.5339, + "theoretical_loss": 3.835284581462, + "tokens_seen": 652279808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483610832497493, + "loss": 1.5046, + "theoretical_loss": 3.8352452573156652, + "tokens_seen": 652345344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483450351053159, + "loss": 1.5097, + "theoretical_loss": 3.835205938225757, + "tokens_seen": 652410880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483289869608827, + "loss": 1.4368, + "theoretical_loss": 3.8351666241911175, + "tokens_seen": 652476416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483129388164493, + "loss": 1.4001, + "theoretical_loss": 3.8351273152105896, + "tokens_seen": 652541952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482968906720161, + "loss": 1.4079, + "theoretical_loss": 3.8350880112830152, + "tokens_seen": 652607488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482808425275827, + "loss": 1.4434, + "theoretical_loss": 3.8350487124072377, + "tokens_seen": 652673024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482647943831495, + "loss": 1.5073, + "theoretical_loss": 3.835009418582101, + "tokens_seen": 652738560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482487462387161, + "loss": 1.3897, + "theoretical_loss": 3.8349701298064485, + "tokens_seen": 652804096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482326980942829, + "loss": 1.4405, + "theoretical_loss": 3.834930846079125, + "tokens_seen": 652869632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482166499498495, + "loss": 1.4859, + "theoretical_loss": 3.8348915673989743, + "tokens_seen": 652935168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482006018054163, + "loss": 1.4749, + "theoretical_loss": 3.8348522937648424, + "tokens_seen": 653000704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481845536609829, + "loss": 1.4313, + "theoretical_loss": 3.834813025175574, + "tokens_seen": 653066240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481685055165497, + "loss": 1.4317, + "theoretical_loss": 3.8347737616300153, + "tokens_seen": 653131776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481524573721163, + "loss": 1.4701, + "theoretical_loss": 3.8347345031270126, + "tokens_seen": 653197312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481364092276831, + "loss": 1.4103, + "theoretical_loss": 3.834695249665412, + "tokens_seen": 653262848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481203610832497, + "loss": 1.5055, + "theoretical_loss": 3.8346560012440607, + "tokens_seen": 653328384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481043129388165, + "loss": 1.3859, + "theoretical_loss": 3.8346167578618062, + "tokens_seen": 653393920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480882647943831, + "loss": 1.4491, + "theoretical_loss": 3.834577519517496, + "tokens_seen": 653459456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480722166499499, + "loss": 1.4329, + "theoretical_loss": 3.8345382862099786, + "tokens_seen": 653524992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480561685055166, + "loss": 1.4701, + "theoretical_loss": 3.8344990579381015, + "tokens_seen": 653590528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480401203610833, + "loss": 1.5133, + "theoretical_loss": 3.8344598347007146, + "tokens_seen": 653656064 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064802407221665, + "loss": 1.4859, + "theoretical_loss": 3.834420616496666, + "tokens_seen": 653721600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480080240722167, + "loss": 1.4683, + "theoretical_loss": 3.8343814033248065, + "tokens_seen": 653787136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479919759277834, + "loss": 1.4837, + "theoretical_loss": 3.8343421951839853, + "tokens_seen": 653852672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479759277833501, + "loss": 1.46, + "theoretical_loss": 3.834302992073053, + "tokens_seen": 653918208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479598796389168, + "loss": 1.3495, + "theoretical_loss": 3.8342637939908606, + "tokens_seen": 653983744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479438314944835, + "loss": 1.4656, + "theoretical_loss": 3.8342246009362584, + "tokens_seen": 654049280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479277833500502, + "loss": 1.4019, + "theoretical_loss": 3.8341854129080986, + "tokens_seen": 654114816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479117352056169, + "loss": 1.3941, + "theoretical_loss": 3.8341462299052327, + "tokens_seen": 654180352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478956870611836, + "loss": 1.4368, + "theoretical_loss": 3.8341070519265132, + "tokens_seen": 654245888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478796389167503, + "loss": 1.434, + "theoretical_loss": 3.8340678789707927, + "tokens_seen": 654311424 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647863590772317, + "loss": 1.2979, + "theoretical_loss": 3.834028711036924, + "tokens_seen": 654376960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478475426278837, + "loss": 1.4776, + "theoretical_loss": 3.8339895481237605, + "tokens_seen": 654442496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478314944834504, + "loss": 1.4851, + "theoretical_loss": 3.8339503902301564, + "tokens_seen": 654508032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478154463390171, + "loss": 1.4354, + "theoretical_loss": 3.833911237354965, + "tokens_seen": 654573568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477993981945838, + "loss": 1.5246, + "theoretical_loss": 3.8338720894970413, + "tokens_seen": 654639104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477833500501505, + "loss": 1.4338, + "theoretical_loss": 3.8338329466552405, + "tokens_seen": 654704640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477673019057172, + "loss": 1.458, + "theoretical_loss": 3.833793808828417, + "tokens_seen": 654770176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477512537612839, + "loss": 1.3989, + "theoretical_loss": 3.8337546760154275, + "tokens_seen": 654835712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477352056168506, + "loss": 1.396, + "theoretical_loss": 3.8337155482151264, + "tokens_seen": 654901248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477191574724173, + "loss": 1.4821, + "theoretical_loss": 3.833676425426372, + "tokens_seen": 654966784 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647703109327984, + "loss": 1.3428, + "theoretical_loss": 3.8336373076480204, + "tokens_seen": 655032320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476870611835507, + "loss": 1.2759, + "theoretical_loss": 3.8335981948789275, + "tokens_seen": 655097856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476710130391174, + "loss": 1.3351, + "theoretical_loss": 3.833559087117952, + "tokens_seen": 655163392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476549648946841, + "loss": 1.4337, + "theoretical_loss": 3.833519984363952, + "tokens_seen": 655228928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476389167502508, + "loss": 1.521, + "theoretical_loss": 3.833480886615785, + "tokens_seen": 655294464 + }, + { + "debugging/Compilability": 0.0, + "debugging/distinct-1-grams": 0.6755372198767522, + "debugging/entropy-1-grams": 4.8915199782364125, + "debugging/length": 606.5, + "debugging/num_segments": 6, + "debugging/score": 0.054560030918185586, + "debugging/score_std": 0.025575581829586438, + "epoch": 0.2, + "objective/train/docs_used": 224121, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3385496139526367, + "objective/train/theoretical_loss": 3.83344179387231, + "objective/train/tokens_used": 675820000, + "theoretical_loss": 3.83344179387231, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476228686058175, + "loss": 1.35, + "theoretical_loss": 3.83344179387231, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476068204613842, + "loss": 1.5245, + "theoretical_loss": 3.8334027061323863, + "tokens_seen": 655425536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006475907723169509, + "loss": 1.4587, + "theoretical_loss": 3.8333636233948725, + "tokens_seen": 655491072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006475747241725176, + "loss": 1.4165, + "theoretical_loss": 3.8333245456586287, + "tokens_seen": 655556608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006475586760280843, + "loss": 1.5065, + "theoretical_loss": 3.8332854729225154, + "tokens_seen": 655622144 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647542627883651, + "loss": 1.5041, + "theoretical_loss": 3.8332464051853927, + "tokens_seen": 655687680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006475265797392177, + "loss": 1.4827, + "theoretical_loss": 3.8332073424461215, + "tokens_seen": 655753216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006475105315947844, + "loss": 1.4264, + "theoretical_loss": 3.8331682847035635, + "tokens_seen": 655818752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474944834503511, + "loss": 1.3716, + "theoretical_loss": 3.8331292319565793, + "tokens_seen": 655884288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474784353059178, + "loss": 1.4567, + "theoretical_loss": 3.833090184204032, + "tokens_seen": 655949824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474623871614845, + "loss": 1.399, + "theoretical_loss": 3.833051141444783, + "tokens_seen": 656015360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474463390170512, + "loss": 1.3809, + "theoretical_loss": 3.833012103677696, + "tokens_seen": 656080896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474302908726179, + "loss": 1.3911, + "theoretical_loss": 3.8329730709016334, + "tokens_seen": 656146432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474142427281846, + "loss": 1.3959, + "theoretical_loss": 3.832934043115459, + "tokens_seen": 656211968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473981945837513, + "loss": 1.4487, + "theoretical_loss": 3.8328950203180367, + "tokens_seen": 656277504 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647382146439318, + "loss": 1.439, + "theoretical_loss": 3.8328560025082306, + "tokens_seen": 656343040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473660982948847, + "loss": 1.5272, + "theoretical_loss": 3.832816989684905, + "tokens_seen": 656408576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473500501504514, + "loss": 1.4425, + "theoretical_loss": 3.8327779818469256, + "tokens_seen": 656474112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473340020060181, + "loss": 1.5045, + "theoretical_loss": 3.8327389789931567, + "tokens_seen": 656539648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473179538615848, + "loss": 1.4973, + "theoretical_loss": 3.8326999811224653, + "tokens_seen": 656605184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473019057171514, + "loss": 1.4604, + "theoretical_loss": 3.8326609882337164, + "tokens_seen": 656670720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006472858575727182, + "loss": 1.4072, + "theoretical_loss": 3.8326220003257774, + "tokens_seen": 656736256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006472698094282848, + "loss": 1.4249, + "theoretical_loss": 3.832583017397514, + "tokens_seen": 656801792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006472537612838516, + "loss": 1.3675, + "theoretical_loss": 3.8325440394477943, + "tokens_seen": 656867328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006472377131394182, + "loss": 1.4295, + "theoretical_loss": 3.8325050664754854, + "tokens_seen": 656932864 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647221664994985, + "loss": 1.461, + "theoretical_loss": 3.8324660984794554, + "tokens_seen": 656998400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006472056168505516, + "loss": 1.3973, + "theoretical_loss": 3.832427135458573, + "tokens_seen": 657063936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006471895687061184, + "loss": 1.4038, + "theoretical_loss": 3.832388177411706, + "tokens_seen": 657129472 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647173520561685, + "loss": 1.4553, + "theoretical_loss": 3.832349224337724, + "tokens_seen": 657195008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006471574724172518, + "loss": 1.5236, + "theoretical_loss": 3.832310276235497, + "tokens_seen": 657260544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006471414242728184, + "loss": 1.3477, + "theoretical_loss": 3.8322713331038933, + "tokens_seen": 657326080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006471253761283852, + "loss": 1.417, + "theoretical_loss": 3.8322323949417845, + "tokens_seen": 657391616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006471093279839518, + "loss": 1.4558, + "theoretical_loss": 3.8321934617480404, + "tokens_seen": 657457152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006470932798395186, + "loss": 1.4935, + "theoretical_loss": 3.832154533521532, + "tokens_seen": 657522688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006470772316950852, + "loss": 1.4223, + "theoretical_loss": 3.8321156102611305, + "tokens_seen": 657588224 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647061183550652, + "loss": 1.4744, + "theoretical_loss": 3.832076691965708, + "tokens_seen": 657653760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006470451354062186, + "loss": 1.4337, + "theoretical_loss": 3.832037778634136, + "tokens_seen": 657719296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006470290872617854, + "loss": 1.3169, + "theoretical_loss": 3.831998870265287, + "tokens_seen": 657784832 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647013039117352, + "loss": 1.3151, + "theoretical_loss": 3.831959966858034, + "tokens_seen": 657850368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469969909729188, + "loss": 1.4623, + "theoretical_loss": 3.8319210684112495, + "tokens_seen": 657915904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469809428284854, + "loss": 1.6412, + "theoretical_loss": 3.8318821749238077, + "tokens_seen": 657981440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469648946840522, + "loss": 1.4437, + "theoretical_loss": 3.831843286394582, + "tokens_seen": 658046976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469488465396188, + "loss": 1.388, + "theoretical_loss": 3.8318044028224465, + "tokens_seen": 658112512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469327983951856, + "loss": 1.4706, + "theoretical_loss": 3.831765524206276, + "tokens_seen": 658178048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469167502507522, + "loss": 1.351, + "theoretical_loss": 3.8317266505449457, + "tokens_seen": 658243584 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646900702106319, + "loss": 1.4297, + "theoretical_loss": 3.8316877818373305, + "tokens_seen": 658309120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006468846539618856, + "loss": 1.4541, + "theoretical_loss": 3.8316489180823066, + "tokens_seen": 658374656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006468686058174524, + "loss": 1.4291, + "theoretical_loss": 3.8316100592787494, + "tokens_seen": 658440192 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646852557673019, + "loss": 1.3943, + "theoretical_loss": 3.8315712054255355, + "tokens_seen": 658505728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006468365095285858, + "loss": 1.5116, + "theoretical_loss": 3.8315323565215422, + "tokens_seen": 658571264 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 224939, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3036456108093262, + "objective/train/theoretical_loss": 3.831493512565646, + "objective/train/tokens_used": 679096800, + "theoretical_loss": 3.831493512565646, + "tokens_seen": 658636800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006468204613841524, + "loss": 1.441, + "theoretical_loss": 3.831493512565646, + "tokens_seen": 658636800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006468044132397192, + "loss": 1.3758, + "theoretical_loss": 3.831454673556725, + "tokens_seen": 658702336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006467883650952858, + "loss": 1.5545, + "theoretical_loss": 3.8314158394936557, + "tokens_seen": 658767872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006467723169508526, + "loss": 1.4414, + "theoretical_loss": 3.831377010375318, + "tokens_seen": 658833408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006467562688064192, + "loss": 1.2908, + "theoretical_loss": 3.8313381862005906, + "tokens_seen": 658898944 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646740220661986, + "loss": 1.5868, + "theoretical_loss": 3.8312993669683513, + "tokens_seen": 658964480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006467241725175526, + "loss": 1.486, + "theoretical_loss": 3.83126055267748, + "tokens_seen": 659030016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006467081243731194, + "loss": 1.4725, + "theoretical_loss": 3.8312217433268563, + "tokens_seen": 659095552 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646692076228686, + "loss": 1.3783, + "theoretical_loss": 3.8311829389153607, + "tokens_seen": 659161088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006466760280842528, + "loss": 1.5176, + "theoretical_loss": 3.8311441394418733, + "tokens_seen": 659226624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006466599799398195, + "loss": 1.4197, + "theoretical_loss": 3.831105344905274, + "tokens_seen": 659292160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006466439317953862, + "loss": 1.5111, + "theoretical_loss": 3.831066555304446, + "tokens_seen": 659357696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006466278836509529, + "loss": 1.3525, + "theoretical_loss": 3.8310277706382694, + "tokens_seen": 659423232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006466118355065196, + "loss": 1.4952, + "theoretical_loss": 3.830988990905626, + "tokens_seen": 659488768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006465957873620863, + "loss": 1.4209, + "theoretical_loss": 3.830950216105399, + "tokens_seen": 659554304 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646579739217653, + "loss": 1.3104, + "theoretical_loss": 3.83091144623647, + "tokens_seen": 659619840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006465636910732197, + "loss": 1.3745, + "theoretical_loss": 3.830872681297723, + "tokens_seen": 659685376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006465476429287864, + "loss": 1.5045, + "theoretical_loss": 3.8308339212880407, + "tokens_seen": 659750912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006465315947843531, + "loss": 1.6476, + "theoretical_loss": 3.830795166206307, + "tokens_seen": 659816448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006465155466399198, + "loss": 1.4087, + "theoretical_loss": 3.830756416051406, + "tokens_seen": 659881984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464994984954865, + "loss": 1.4844, + "theoretical_loss": 3.8307176708222217, + "tokens_seen": 659947520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464834503510532, + "loss": 1.426, + "theoretical_loss": 3.8306789305176396, + "tokens_seen": 660013056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464674022066199, + "loss": 1.395, + "theoretical_loss": 3.830640195136545, + "tokens_seen": 660078592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464513540621866, + "loss": 1.3413, + "theoretical_loss": 3.830601464677823, + "tokens_seen": 660144128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464353059177533, + "loss": 1.4125, + "theoretical_loss": 3.8305627391403596, + "tokens_seen": 660209664 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064641925777332, + "loss": 1.6172, + "theoretical_loss": 3.830524018523041, + "tokens_seen": 660275200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464032096288867, + "loss": 1.4359, + "theoretical_loss": 3.8304853028247536, + "tokens_seen": 660340736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463871614844534, + "loss": 1.4082, + "theoretical_loss": 3.830446592044385, + "tokens_seen": 660406272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463711133400201, + "loss": 1.3422, + "theoretical_loss": 3.830407886180822, + "tokens_seen": 660471808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463550651955868, + "loss": 1.4599, + "theoretical_loss": 3.8303691852329522, + "tokens_seen": 660537344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463390170511535, + "loss": 1.451, + "theoretical_loss": 3.830330489199665, + "tokens_seen": 660602880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463229689067202, + "loss": 1.4021, + "theoretical_loss": 3.8302917980798465, + "tokens_seen": 660668416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463069207622869, + "loss": 1.308, + "theoretical_loss": 3.8302531118723877, + "tokens_seen": 660733952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006462908726178536, + "loss": 1.3081, + "theoretical_loss": 3.830214430576177, + "tokens_seen": 660799488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006462748244734203, + "loss": 1.4741, + "theoretical_loss": 3.8301757541901034, + "tokens_seen": 660865024 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646258776328987, + "loss": 1.5243, + "theoretical_loss": 3.8301370827130574, + "tokens_seen": 660930560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006462427281845538, + "loss": 1.4377, + "theoretical_loss": 3.8300984161439295, + "tokens_seen": 660996096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006462266800401204, + "loss": 1.3741, + "theoretical_loss": 3.830059754481609, + "tokens_seen": 661061632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006462106318956872, + "loss": 1.5008, + "theoretical_loss": 3.8300210977249884, + "tokens_seen": 661127168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006461945837512538, + "loss": 1.4381, + "theoretical_loss": 3.8299824458729583, + "tokens_seen": 661192704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006461785356068206, + "loss": 1.3359, + "theoretical_loss": 3.8299437989244103, + "tokens_seen": 661258240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006461624874623872, + "loss": 1.3917, + "theoretical_loss": 3.8299051568782363, + "tokens_seen": 661323776 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646146439317954, + "loss": 1.3676, + "theoretical_loss": 3.8298665197333293, + "tokens_seen": 661389312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006461303911735206, + "loss": 1.4281, + "theoretical_loss": 3.829827887488582, + "tokens_seen": 661454848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006461143430290874, + "loss": 1.4768, + "theoretical_loss": 3.8297892601428867, + "tokens_seen": 661520384 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646098294884654, + "loss": 1.4123, + "theoretical_loss": 3.8297506376951382, + "tokens_seen": 661585920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460822467402208, + "loss": 1.5561, + "theoretical_loss": 3.829712020144229, + "tokens_seen": 661651456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460661985957874, + "loss": 1.4405, + "theoretical_loss": 3.8296734074890546, + "tokens_seen": 661716992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460501504513542, + "loss": 1.5015, + "theoretical_loss": 3.8296347997285083, + "tokens_seen": 661782528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460341023069208, + "loss": 1.3775, + "theoretical_loss": 3.829596196861486, + "tokens_seen": 661848064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 226118, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4302797317504883, + "objective/train/theoretical_loss": 3.8295575988868826, + "objective/train/tokens_used": 682373600, + "theoretical_loss": 3.8295575988868826, + "tokens_seen": 661913600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460180541624875, + "loss": 1.4939, + "theoretical_loss": 3.8295575988868826, + "tokens_seen": 661913600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460020060180542, + "loss": 1.3983, + "theoretical_loss": 3.8295190058035935, + "tokens_seen": 661979136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006459859578736209, + "loss": 1.4613, + "theoretical_loss": 3.8294804176105153, + "tokens_seen": 662044672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006459699097291876, + "loss": 1.4311, + "theoretical_loss": 3.829441834306544, + "tokens_seen": 662110208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006459538615847543, + "loss": 1.5059, + "theoretical_loss": 3.8294032558905764, + "tokens_seen": 662175744 + }, + { + "epoch": 0.2, + "learning_rate": 0.000645937813440321, + "loss": 1.5588, + "theoretical_loss": 3.829364682361509, + "tokens_seen": 662241280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006459217652958877, + "loss": 1.427, + "theoretical_loss": 3.82932611371824, + "tokens_seen": 662306816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006459057171514543, + "loss": 1.3958, + "theoretical_loss": 3.829287549959667, + "tokens_seen": 662372352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458896690070211, + "loss": 1.5076, + "theoretical_loss": 3.8292489910846887, + "tokens_seen": 662437888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458736208625877, + "loss": 1.5154, + "theoretical_loss": 3.829210437092202, + "tokens_seen": 662503424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458575727181545, + "loss": 1.4082, + "theoretical_loss": 3.8291718879811074, + "tokens_seen": 662568960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458415245737211, + "loss": 1.3067, + "theoretical_loss": 3.8291333437503035, + "tokens_seen": 662634496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458254764292879, + "loss": 1.4807, + "theoretical_loss": 3.82909480439869, + "tokens_seen": 662700032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458094282848545, + "loss": 1.3389, + "theoretical_loss": 3.8290562699251662, + "tokens_seen": 662765568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457933801404213, + "loss": 1.4133, + "theoretical_loss": 3.8290177403286334, + "tokens_seen": 662831104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457773319959879, + "loss": 1.4489, + "theoretical_loss": 3.828979215607991, + "tokens_seen": 662896640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457612838515547, + "loss": 1.3352, + "theoretical_loss": 3.8289406957621415, + "tokens_seen": 662962176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457452357071213, + "loss": 1.3634, + "theoretical_loss": 3.828902180789985, + "tokens_seen": 663027712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457291875626881, + "loss": 1.4564, + "theoretical_loss": 3.828863670690424, + "tokens_seen": 663093248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457131394182547, + "loss": 1.46, + "theoretical_loss": 3.8288251654623604, + "tokens_seen": 663158784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456970912738215, + "loss": 1.4153, + "theoretical_loss": 3.8287866651046962, + "tokens_seen": 663224320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456810431293881, + "loss": 1.4752, + "theoretical_loss": 3.8287481696163344, + "tokens_seen": 663289856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456649949849549, + "loss": 1.4587, + "theoretical_loss": 3.828709678996179, + "tokens_seen": 663355392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456489468405215, + "loss": 1.4478, + "theoretical_loss": 3.8286711932431325, + "tokens_seen": 663420928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456328986960883, + "loss": 1.404, + "theoretical_loss": 3.8286327123560984, + "tokens_seen": 663486464 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456168505516549, + "loss": 1.4454, + "theoretical_loss": 3.8285942363339824, + "tokens_seen": 663552000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456008024072217, + "loss": 1.4571, + "theoretical_loss": 3.828555765175688, + "tokens_seen": 663617536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455847542627883, + "loss": 1.4549, + "theoretical_loss": 3.8285172988801195, + "tokens_seen": 663683072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455687061183551, + "loss": 1.3885, + "theoretical_loss": 3.8284788374461836, + "tokens_seen": 663748608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455526579739217, + "loss": 1.4964, + "theoretical_loss": 3.8284403808727854, + "tokens_seen": 663814144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455366098294885, + "loss": 1.435, + "theoretical_loss": 3.8284019291588303, + "tokens_seen": 663879680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455205616850551, + "loss": 1.3356, + "theoretical_loss": 3.8283634823032253, + "tokens_seen": 663945216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455045135406219, + "loss": 1.442, + "theoretical_loss": 3.828325040304877, + "tokens_seen": 664010752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454884653961885, + "loss": 1.4236, + "theoretical_loss": 3.8282866031626925, + "tokens_seen": 664076288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454724172517553, + "loss": 1.4537, + "theoretical_loss": 3.828248170875579, + "tokens_seen": 664141824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454563691073219, + "loss": 1.3599, + "theoretical_loss": 3.828209743442444, + "tokens_seen": 664207360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454403209628887, + "loss": 1.4581, + "theoretical_loss": 3.828171320862196, + "tokens_seen": 664272896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454242728184553, + "loss": 1.4065, + "theoretical_loss": 3.8281329031337434, + "tokens_seen": 664338432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454082246740221, + "loss": 1.478, + "theoretical_loss": 3.828094490255995, + "tokens_seen": 664403968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453921765295887, + "loss": 1.4048, + "theoretical_loss": 3.8280560822278598, + "tokens_seen": 664469504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453761283851555, + "loss": 1.3398, + "theoretical_loss": 3.8280176790482474, + "tokens_seen": 664535040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453600802407221, + "loss": 1.543, + "theoretical_loss": 3.827979280716068, + "tokens_seen": 664600576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453440320962889, + "loss": 1.488, + "theoretical_loss": 3.8279408872302314, + "tokens_seen": 664666112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453279839518556, + "loss": 1.3808, + "theoretical_loss": 3.827902498589648, + "tokens_seen": 664731648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453119358074223, + "loss": 1.3958, + "theoretical_loss": 3.827864114793229, + "tokens_seen": 664797184 + }, + { + "epoch": 0.2, + "learning_rate": 0.000645295887662989, + "loss": 1.4373, + "theoretical_loss": 3.827825735839886, + "tokens_seen": 664862720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006452798395185557, + "loss": 1.4423, + "theoretical_loss": 3.8277873617285305, + "tokens_seen": 664928256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006452637913741224, + "loss": 1.3114, + "theoretical_loss": 3.8277489924580745, + "tokens_seen": 664993792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006452477432296891, + "loss": 1.3278, + "theoretical_loss": 3.82771062802743, + "tokens_seen": 665059328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006452316950852558, + "loss": 1.3394, + "theoretical_loss": 3.827672268435509, + "tokens_seen": 665124864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 227074, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4643185138702393, + "objective/train/theoretical_loss": 3.8276339136812263, + "objective/train/tokens_used": 685650400, + "theoretical_loss": 3.8276339136812263, + "tokens_seen": 665190400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006452156469408225, + "loss": 1.4293, + "theoretical_loss": 3.8276339136812263, + "tokens_seen": 665190400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451995987963892, + "loss": 1.4765, + "theoretical_loss": 3.8275955637634933, + "tokens_seen": 665255936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451835506519559, + "loss": 1.3266, + "theoretical_loss": 3.8275572186812257, + "tokens_seen": 665321472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451675025075226, + "loss": 1.3526, + "theoretical_loss": 3.8275188784333363, + "tokens_seen": 665387008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451514543630893, + "loss": 1.4508, + "theoretical_loss": 3.8274805430187406, + "tokens_seen": 665452544 + }, + { + "epoch": 0.2, + "learning_rate": 0.000645135406218656, + "loss": 1.4785, + "theoretical_loss": 3.8274422124363516, + "tokens_seen": 665518080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451193580742227, + "loss": 1.4408, + "theoretical_loss": 3.8274038866850866, + "tokens_seen": 665583616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451033099297894, + "loss": 1.3977, + "theoretical_loss": 3.8273655657638597, + "tokens_seen": 665649152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450872617853561, + "loss": 1.4323, + "theoretical_loss": 3.827327249671587, + "tokens_seen": 665714688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450712136409228, + "loss": 1.4965, + "theoretical_loss": 3.8272889384071846, + "tokens_seen": 665780224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450551654964895, + "loss": 1.4091, + "theoretical_loss": 3.8272506319695694, + "tokens_seen": 665845760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450391173520563, + "loss": 1.4464, + "theoretical_loss": 3.827212330357658, + "tokens_seen": 665911296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450230692076229, + "loss": 1.3789, + "theoretical_loss": 3.827174033570368, + "tokens_seen": 665976832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450070210631897, + "loss": 1.4906, + "theoretical_loss": 3.8271357416066167, + "tokens_seen": 666042368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449909729187563, + "loss": 1.4139, + "theoretical_loss": 3.827097454465322, + "tokens_seen": 666107904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449749247743231, + "loss": 1.4756, + "theoretical_loss": 3.827059172145402, + "tokens_seen": 666173440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449588766298897, + "loss": 1.457, + "theoretical_loss": 3.827020894645776, + "tokens_seen": 666238976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449428284854565, + "loss": 1.4755, + "theoretical_loss": 3.826982621965363, + "tokens_seen": 666304512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449267803410231, + "loss": 1.4857, + "theoretical_loss": 3.826944354103081, + "tokens_seen": 666370048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449107321965899, + "loss": 1.3627, + "theoretical_loss": 3.826906091057851, + "tokens_seen": 666435584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448946840521565, + "loss": 1.5011, + "theoretical_loss": 3.8268678328285928, + "tokens_seen": 666501120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448786359077233, + "loss": 1.3738, + "theoretical_loss": 3.8268295794142264, + "tokens_seen": 666566656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448625877632899, + "loss": 1.3865, + "theoretical_loss": 3.826791330813673, + "tokens_seen": 666632192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448465396188567, + "loss": 1.4636, + "theoretical_loss": 3.8267530870258533, + "tokens_seen": 666697728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448304914744233, + "loss": 1.341, + "theoretical_loss": 3.826714848049689, + "tokens_seen": 666763264 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448144433299901, + "loss": 1.3991, + "theoretical_loss": 3.8266766138841013, + "tokens_seen": 666828800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447983951855567, + "loss": 1.4756, + "theoretical_loss": 3.826638384528013, + "tokens_seen": 666894336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447823470411235, + "loss": 1.3959, + "theoretical_loss": 3.8266001599803467, + "tokens_seen": 666959872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447662988966901, + "loss": 1.4375, + "theoretical_loss": 3.8265619402400244, + "tokens_seen": 667025408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447502507522569, + "loss": 1.3847, + "theoretical_loss": 3.82652372530597, + "tokens_seen": 667090944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447342026078235, + "loss": 1.3348, + "theoretical_loss": 3.8264855151771067, + "tokens_seen": 667156480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447181544633903, + "loss": 1.3579, + "theoretical_loss": 3.826447309852358, + "tokens_seen": 667222016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447021063189569, + "loss": 1.4032, + "theoretical_loss": 3.826409109330649, + "tokens_seen": 667287552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446860581745237, + "loss": 1.5174, + "theoretical_loss": 3.826370913610903, + "tokens_seen": 667353088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446700100300903, + "loss": 1.4905, + "theoretical_loss": 3.8263327226920465, + "tokens_seen": 667418624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446539618856571, + "loss": 1.4021, + "theoretical_loss": 3.8262945365730032, + "tokens_seen": 667484160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446379137412237, + "loss": 1.421, + "theoretical_loss": 3.8262563552526996, + "tokens_seen": 667549696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446218655967904, + "loss": 1.3896, + "theoretical_loss": 3.8262181787300618, + "tokens_seen": 667615232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446058174523571, + "loss": 1.4655, + "theoretical_loss": 3.8261800070040146, + "tokens_seen": 667680768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445897693079238, + "loss": 1.4998, + "theoretical_loss": 3.8261418400734866, + "tokens_seen": 667746304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445737211634905, + "loss": 1.358, + "theoretical_loss": 3.826103677937403, + "tokens_seen": 667811840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445576730190572, + "loss": 1.3546, + "theoretical_loss": 3.826065520594693, + "tokens_seen": 667877376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445416248746239, + "loss": 1.4208, + "theoretical_loss": 3.8260273680442825, + "tokens_seen": 667942912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445255767301906, + "loss": 1.418, + "theoretical_loss": 3.8259892202851002, + "tokens_seen": 668008448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445095285857573, + "loss": 1.5496, + "theoretical_loss": 3.825951077316075, + "tokens_seen": 668073984 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644493480441324, + "loss": 1.59, + "theoretical_loss": 3.825912939136135, + "tokens_seen": 668139520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006444774322968906, + "loss": 1.3779, + "theoretical_loss": 3.825874805744209, + "tokens_seen": 668205056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006444613841524574, + "loss": 1.378, + "theoretical_loss": 3.825836677139227, + "tokens_seen": 668270592 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644445336008024, + "loss": 1.4298, + "theoretical_loss": 3.8257985533201184, + "tokens_seen": 668336128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006444292878635908, + "loss": 1.4356, + "theoretical_loss": 3.8257604342858134, + "tokens_seen": 668401664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 228265, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3256380558013916, + "objective/train/theoretical_loss": 3.8257223200352417, + "objective/train/tokens_used": 688927200, + "theoretical_loss": 3.8257223200352417, + "tokens_seen": 668467200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006444132397191574, + "loss": 1.4047, + "theoretical_loss": 3.8257223200352417, + "tokens_seen": 668467200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443971915747242, + "loss": 1.4653, + "theoretical_loss": 3.825684210567335, + "tokens_seen": 668532736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443811434302908, + "loss": 1.4851, + "theoretical_loss": 3.8256461058810247, + "tokens_seen": 668598272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443650952858576, + "loss": 1.4715, + "theoretical_loss": 3.825608005975241, + "tokens_seen": 668663808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443490471414242, + "loss": 1.3709, + "theoretical_loss": 3.8255699108489165, + "tokens_seen": 668729344 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644332998996991, + "loss": 1.4355, + "theoretical_loss": 3.8255318205009834, + "tokens_seen": 668794880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443169508525576, + "loss": 1.5044, + "theoretical_loss": 3.825493734930374, + "tokens_seen": 668860416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443009027081244, + "loss": 1.3877, + "theoretical_loss": 3.8254556541360207, + "tokens_seen": 668925952 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644284854563691, + "loss": 1.4145, + "theoretical_loss": 3.8254175781168573, + "tokens_seen": 668991488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006442688064192578, + "loss": 1.3835, + "theoretical_loss": 3.825379506871817, + "tokens_seen": 669057024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006442527582748244, + "loss": 1.3198, + "theoretical_loss": 3.8253414403998334, + "tokens_seen": 669122560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006442367101303912, + "loss": 1.3518, + "theoretical_loss": 3.825303378699841, + "tokens_seen": 669188096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006442206619859578, + "loss": 1.4474, + "theoretical_loss": 3.8252653217707753, + "tokens_seen": 669253632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006442046138415246, + "loss": 1.4523, + "theoretical_loss": 3.8252272696115694, + "tokens_seen": 669319168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006441885656970912, + "loss": 1.5241, + "theoretical_loss": 3.8251892222211596, + "tokens_seen": 669384704 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644172517552658, + "loss": 1.3396, + "theoretical_loss": 3.825151179598481, + "tokens_seen": 669450240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006441564694082246, + "loss": 1.453, + "theoretical_loss": 3.8251131417424693, + "tokens_seen": 669515776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006441404212637914, + "loss": 1.4106, + "theoretical_loss": 3.8250751086520616, + "tokens_seen": 669581312 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644124373119358, + "loss": 1.4388, + "theoretical_loss": 3.825037080326194, + "tokens_seen": 669646848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006441083249749248, + "loss": 1.4614, + "theoretical_loss": 3.8249990567638035, + "tokens_seen": 669712384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006440922768304914, + "loss": 1.256, + "theoretical_loss": 3.8249610379638277, + "tokens_seen": 669777920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006440762286860582, + "loss": 1.4333, + "theoretical_loss": 3.8249230239252032, + "tokens_seen": 669843456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006440601805416248, + "loss": 1.3193, + "theoretical_loss": 3.8248850146468687, + "tokens_seen": 669908992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006440441323971916, + "loss": 1.349, + "theoretical_loss": 3.8248470101277627, + "tokens_seen": 669974528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006440280842527582, + "loss": 1.3975, + "theoretical_loss": 3.824809010366823, + "tokens_seen": 670040064 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644012036108325, + "loss": 1.3833, + "theoretical_loss": 3.8247710153629892, + "tokens_seen": 670105600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006439959879638916, + "loss": 1.3982, + "theoretical_loss": 3.8247330251152007, + "tokens_seen": 670171136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006439799398194584, + "loss": 1.4416, + "theoretical_loss": 3.824695039622397, + "tokens_seen": 670236672 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643963891675025, + "loss": 1.3284, + "theoretical_loss": 3.824657058883518, + "tokens_seen": 670302208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006439478435305918, + "loss": 1.4482, + "theoretical_loss": 3.824619082897504, + "tokens_seen": 670367744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006439317953861585, + "loss": 1.5329, + "theoretical_loss": 3.824581111663296, + "tokens_seen": 670433280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006439157472417252, + "loss": 1.4133, + "theoretical_loss": 3.8245431451798346, + "tokens_seen": 670498816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438996990972919, + "loss": 1.3977, + "theoretical_loss": 3.8245051834460613, + "tokens_seen": 670564352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438836509528586, + "loss": 1.3864, + "theoretical_loss": 3.824467226460918, + "tokens_seen": 670629888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438676028084253, + "loss": 1.4684, + "theoretical_loss": 3.8244292742233457, + "tokens_seen": 670695424 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643851554663992, + "loss": 1.3351, + "theoretical_loss": 3.8243913267322887, + "tokens_seen": 670760960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438355065195588, + "loss": 1.4502, + "theoretical_loss": 3.824353383986688, + "tokens_seen": 670826496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438194583751254, + "loss": 1.4654, + "theoretical_loss": 3.824315445985487, + "tokens_seen": 670892032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438034102306922, + "loss": 1.4255, + "theoretical_loss": 3.8242775127276296, + "tokens_seen": 670957568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006437873620862588, + "loss": 1.3765, + "theoretical_loss": 3.8242395842120596, + "tokens_seen": 671023104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006437713139418256, + "loss": 1.3318, + "theoretical_loss": 3.824201660437721, + "tokens_seen": 671088640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006437552657973922, + "loss": 1.3525, + "theoretical_loss": 3.824163741403557, + "tokens_seen": 671154176 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643739217652959, + "loss": 1.4577, + "theoretical_loss": 3.8241258271085137, + "tokens_seen": 671219712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006437231695085256, + "loss": 1.388, + "theoretical_loss": 3.824087917551536, + "tokens_seen": 671285248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006437071213640924, + "loss": 1.3655, + "theoretical_loss": 3.8240500127315693, + "tokens_seen": 671350784 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643691073219659, + "loss": 1.4919, + "theoretical_loss": 3.8240121126475586, + "tokens_seen": 671416320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006436750250752258, + "loss": 1.3101, + "theoretical_loss": 3.82397421729845, + "tokens_seen": 671481856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006436589769307924, + "loss": 1.4365, + "theoretical_loss": 3.8239363266831914, + "tokens_seen": 671547392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006436429287863592, + "loss": 1.459, + "theoretical_loss": 3.823898440800728, + "tokens_seen": 671612928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006436268806419258, + "loss": 1.3721, + "theoretical_loss": 3.823860559650008, + "tokens_seen": 671678464 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 229250, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3694689273834229, + "objective/train/theoretical_loss": 3.823822683229978, + "objective/train/tokens_used": 692204000, + "theoretical_loss": 3.823822683229978, + "tokens_seen": 671744000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006436108324974926, + "loss": 1.3617, + "theoretical_loss": 3.823822683229978, + "tokens_seen": 671744000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006435947843530592, + "loss": 1.335, + "theoretical_loss": 3.8237848115395865, + "tokens_seen": 671809536 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643578736208626, + "loss": 1.5724, + "theoretical_loss": 3.823746944577781, + "tokens_seen": 671875072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006435626880641926, + "loss": 1.5382, + "theoretical_loss": 3.82370908234351, + "tokens_seen": 671940608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006435466399197594, + "loss": 1.3706, + "theoretical_loss": 3.8236712248357225, + "tokens_seen": 672006144 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643530591775326, + "loss": 1.3624, + "theoretical_loss": 3.8236333720533677, + "tokens_seen": 672071680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006435145436308928, + "loss": 1.3729, + "theoretical_loss": 3.823595523995395, + "tokens_seen": 672137216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434984954864594, + "loss": 1.3791, + "theoretical_loss": 3.823557680660754, + "tokens_seen": 672202752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434824473420262, + "loss": 1.3773, + "theoretical_loss": 3.823519842048395, + "tokens_seen": 672268288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434663991975928, + "loss": 1.4648, + "theoretical_loss": 3.823482008157269, + "tokens_seen": 672333824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434503510531596, + "loss": 1.2812, + "theoretical_loss": 3.8234441789863256, + "tokens_seen": 672399360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434343029087262, + "loss": 1.3632, + "theoretical_loss": 3.8234063545345167, + "tokens_seen": 672464896 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643418254764293, + "loss": 1.4468, + "theoretical_loss": 3.8233685348007933, + "tokens_seen": 672530432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434022066198596, + "loss": 1.4096, + "theoretical_loss": 3.823330719784108, + "tokens_seen": 672595968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006433861584754264, + "loss": 1.4883, + "theoretical_loss": 3.8232929094834125, + "tokens_seen": 672661504 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643370110330993, + "loss": 1.5356, + "theoretical_loss": 3.823255103897659, + "tokens_seen": 672727040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006433540621865598, + "loss": 1.4702, + "theoretical_loss": 3.8232173030258005, + "tokens_seen": 672792576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006433380140421264, + "loss": 1.3244, + "theoretical_loss": 3.823179506866791, + "tokens_seen": 672858112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006433219658976932, + "loss": 1.3574, + "theoretical_loss": 3.823141715419582, + "tokens_seen": 672923648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006433059177532598, + "loss": 1.4943, + "theoretical_loss": 3.823103928683129, + "tokens_seen": 672989184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006432898696088266, + "loss": 1.4458, + "theoretical_loss": 3.8230661466563864, + "tokens_seen": 673054720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006432738214643932, + "loss": 1.3608, + "theoretical_loss": 3.8230283693383074, + "tokens_seen": 673120256 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064325777331996, + "loss": 1.3938, + "theoretical_loss": 3.822990596727847, + "tokens_seen": 673185792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006432417251755266, + "loss": 1.5127, + "theoretical_loss": 3.822952828823961, + "tokens_seen": 673251328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006432256770310933, + "loss": 1.3823, + "theoretical_loss": 3.8229150656256046, + "tokens_seen": 673316864 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064320962888666, + "loss": 1.4841, + "theoretical_loss": 3.8228773071317335, + "tokens_seen": 673382400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431935807422267, + "loss": 1.3449, + "theoretical_loss": 3.822839553341304, + "tokens_seen": 673447936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431775325977934, + "loss": 1.4301, + "theoretical_loss": 3.822801804253273, + "tokens_seen": 673513472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431614844533601, + "loss": 1.3745, + "theoretical_loss": 3.8227640598665964, + "tokens_seen": 673579008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431454363089268, + "loss": 1.47, + "theoretical_loss": 3.822726320180232, + "tokens_seen": 673644544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431293881644935, + "loss": 1.4192, + "theoretical_loss": 3.822688585193137, + "tokens_seen": 673710080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431133400200602, + "loss": 1.3804, + "theoretical_loss": 3.822650854904269, + "tokens_seen": 673775616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430972918756269, + "loss": 1.3837, + "theoretical_loss": 3.822613129312587, + "tokens_seen": 673841152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430812437311935, + "loss": 1.414, + "theoretical_loss": 3.8225754084170482, + "tokens_seen": 673906688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430651955867603, + "loss": 1.4774, + "theoretical_loss": 3.8225376922166125, + "tokens_seen": 673972224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430491474423269, + "loss": 1.4462, + "theoretical_loss": 3.822499980710239, + "tokens_seen": 674037760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430330992978937, + "loss": 1.4094, + "theoretical_loss": 3.8224622738968863, + "tokens_seen": 674103296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430170511534603, + "loss": 1.3966, + "theoretical_loss": 3.822424571775515, + "tokens_seen": 674168832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430010030090271, + "loss": 1.352, + "theoretical_loss": 3.822386874345085, + "tokens_seen": 674234368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429849548645937, + "loss": 1.4683, + "theoretical_loss": 3.822349181604557, + "tokens_seen": 674299904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429689067201605, + "loss": 1.399, + "theoretical_loss": 3.8223114935528915, + "tokens_seen": 674365440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429528585757271, + "loss": 1.3665, + "theoretical_loss": 3.82227381018905, + "tokens_seen": 674430976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429368104312939, + "loss": 1.425, + "theoretical_loss": 3.822236131511993, + "tokens_seen": 674496512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429207622868605, + "loss": 1.3625, + "theoretical_loss": 3.8221984575206833, + "tokens_seen": 674562048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429047141424273, + "loss": 1.365, + "theoretical_loss": 3.8221607882140827, + "tokens_seen": 674627584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428886659979939, + "loss": 1.4393, + "theoretical_loss": 3.8221231235911537, + "tokens_seen": 674693120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428726178535607, + "loss": 1.4967, + "theoretical_loss": 3.822085463650859, + "tokens_seen": 674758656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428565697091273, + "loss": 1.3798, + "theoretical_loss": 3.8220478083921616, + "tokens_seen": 674824192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428405215646941, + "loss": 1.5147, + "theoretical_loss": 3.8220101578140255, + "tokens_seen": 674889728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428244734202607, + "loss": 1.5698, + "theoretical_loss": 3.8219725119154138, + "tokens_seen": 674955264 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 230444, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4336386919021606, + "objective/train/theoretical_loss": 3.821934870695291, + "objective/train/tokens_used": 695480800, + "theoretical_loss": 3.821934870695291, + "tokens_seen": 675020800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428084252758275, + "loss": 1.3804, + "theoretical_loss": 3.821934870695291, + "tokens_seen": 675020800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427923771313941, + "loss": 1.5014, + "theoretical_loss": 3.8218972341526216, + "tokens_seen": 675086336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427763289869609, + "loss": 1.3548, + "theoretical_loss": 3.8218596022863696, + "tokens_seen": 675151872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427602808425275, + "loss": 1.4235, + "theoretical_loss": 3.821821975095501, + "tokens_seen": 675217408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427442326980943, + "loss": 1.4179, + "theoretical_loss": 3.821784352578981, + "tokens_seen": 675282944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427281845536609, + "loss": 1.4752, + "theoretical_loss": 3.8217467347357754, + "tokens_seen": 675348480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427121364092277, + "loss": 1.5204, + "theoretical_loss": 3.82170912156485, + "tokens_seen": 675414016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426960882647943, + "loss": 1.4027, + "theoretical_loss": 3.8216715130651715, + "tokens_seen": 675479552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426800401203611, + "loss": 1.5151, + "theoretical_loss": 3.8216339092357066, + "tokens_seen": 675545088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426639919759277, + "loss": 1.3896, + "theoretical_loss": 3.8215963100754218, + "tokens_seen": 675610624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426479438314945, + "loss": 1.4323, + "theoretical_loss": 3.8215587155832855, + "tokens_seen": 675676160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426318956870613, + "loss": 1.4006, + "theoretical_loss": 3.8215211257582644, + "tokens_seen": 675741696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426158475426279, + "loss": 1.3455, + "theoretical_loss": 3.8214835405993273, + "tokens_seen": 675807232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425997993981947, + "loss": 1.5163, + "theoretical_loss": 3.8214459601054425, + "tokens_seen": 675872768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425837512537613, + "loss": 1.429, + "theoretical_loss": 3.8214083842755784, + "tokens_seen": 675938304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425677031093281, + "loss": 1.4165, + "theoretical_loss": 3.8213708131087043, + "tokens_seen": 676003840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425516549648947, + "loss": 1.5654, + "theoretical_loss": 3.8213332466037895, + "tokens_seen": 676069376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425356068204615, + "loss": 1.4415, + "theoretical_loss": 3.8212956847598036, + "tokens_seen": 676134912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425195586760281, + "loss": 1.3559, + "theoretical_loss": 3.8212581275757165, + "tokens_seen": 676200448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425035105315949, + "loss": 1.5231, + "theoretical_loss": 3.821220575050499, + "tokens_seen": 676265984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006424874623871615, + "loss": 1.3894, + "theoretical_loss": 3.821183027183122, + "tokens_seen": 676331520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006424714142427283, + "loss": 1.4321, + "theoretical_loss": 3.821145483972556, + "tokens_seen": 676397056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006424553660982949, + "loss": 1.4047, + "theoretical_loss": 3.821107945417771, + "tokens_seen": 676462592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006424393179538617, + "loss": 1.4645, + "theoretical_loss": 3.8210704115177414, + "tokens_seen": 676528128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006424232698094283, + "loss": 1.4276, + "theoretical_loss": 3.8210328822714374, + "tokens_seen": 676593664 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006424072216649951, + "loss": 1.4779, + "theoretical_loss": 3.820995357677832, + "tokens_seen": 676659200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423911735205617, + "loss": 1.5168, + "theoretical_loss": 3.8209578377358975, + "tokens_seen": 676724736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423751253761285, + "loss": 1.5245, + "theoretical_loss": 3.8209203224446067, + "tokens_seen": 676790272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423590772316951, + "loss": 1.3363, + "theoretical_loss": 3.820882811802933, + "tokens_seen": 676855808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423430290872619, + "loss": 1.3885, + "theoretical_loss": 3.82084530580985, + "tokens_seen": 676921344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423269809428285, + "loss": 1.4748, + "theoretical_loss": 3.8208078044643328, + "tokens_seen": 676986880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423109327983953, + "loss": 1.5085, + "theoretical_loss": 3.8207703077653536, + "tokens_seen": 677052416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422948846539619, + "loss": 1.4075, + "theoretical_loss": 3.8207328157118887, + "tokens_seen": 677117952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422788365095287, + "loss": 1.4019, + "theoretical_loss": 3.820695328302912, + "tokens_seen": 677183488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422627883650953, + "loss": 1.413, + "theoretical_loss": 3.8206578455373994, + "tokens_seen": 677249024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422467402206621, + "loss": 1.4656, + "theoretical_loss": 3.820620367414326, + "tokens_seen": 677314560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422306920762287, + "loss": 1.3562, + "theoretical_loss": 3.8205828939326683, + "tokens_seen": 677380096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422146439317955, + "loss": 1.4737, + "theoretical_loss": 3.8205454250914013, + "tokens_seen": 677445632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421985957873621, + "loss": 1.4588, + "theoretical_loss": 3.820507960889503, + "tokens_seen": 677511168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421825476429289, + "loss": 1.3709, + "theoretical_loss": 3.82047050132595, + "tokens_seen": 677576704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421664994984955, + "loss": 1.3659, + "theoretical_loss": 3.8204330463997183, + "tokens_seen": 677642240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421504513540623, + "loss": 1.4767, + "theoretical_loss": 3.820395596109787, + "tokens_seen": 677707776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421344032096289, + "loss": 1.3873, + "theoretical_loss": 3.8203581504551325, + "tokens_seen": 677773312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421183550651957, + "loss": 1.4155, + "theoretical_loss": 3.820320709434734, + "tokens_seen": 677838848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421023069207623, + "loss": 1.4827, + "theoretical_loss": 3.8202832730475693, + "tokens_seen": 677904384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420862587763291, + "loss": 1.3759, + "theoretical_loss": 3.820245841292618, + "tokens_seen": 677969920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420702106318957, + "loss": 1.4155, + "theoretical_loss": 3.820208414168859, + "tokens_seen": 678035456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420541624874625, + "loss": 1.3425, + "theoretical_loss": 3.820170991675271, + "tokens_seen": 678100992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420381143430291, + "loss": 1.3834, + "theoretical_loss": 3.8201335738108346, + "tokens_seen": 678166528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420220661985959, + "loss": 1.4239, + "theoretical_loss": 3.82009616057453, + "tokens_seen": 678232064 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 231396, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4644248485565186, + "objective/train/theoretical_loss": 3.8200587519653366, + "objective/train/tokens_used": 698757600, + "theoretical_loss": 3.8200587519653366, + "tokens_seen": 678297600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420060180541625, + "loss": 1.4658, + "theoretical_loss": 3.8200587519653366, + "tokens_seen": 678297600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419899699097293, + "loss": 1.3733, + "theoretical_loss": 3.8200213479822365, + "tokens_seen": 678363136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419739217652959, + "loss": 1.2866, + "theoretical_loss": 3.81998394862421, + "tokens_seen": 678428672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419578736208627, + "loss": 1.4561, + "theoretical_loss": 3.8199465538902384, + "tokens_seen": 678494208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419418254764293, + "loss": 1.5282, + "theoretical_loss": 3.819909163779304, + "tokens_seen": 678559744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419257773319961, + "loss": 1.5327, + "theoretical_loss": 3.8198717782903886, + "tokens_seen": 678625280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419097291875627, + "loss": 1.4462, + "theoretical_loss": 3.8198343974224747, + "tokens_seen": 678690816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418936810431295, + "loss": 1.3364, + "theoretical_loss": 3.8197970211745442, + "tokens_seen": 678756352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418776328986961, + "loss": 1.3884, + "theoretical_loss": 3.819759649545581, + "tokens_seen": 678821888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418615847542629, + "loss": 1.5174, + "theoretical_loss": 3.819722282534568, + "tokens_seen": 678887424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418455366098295, + "loss": 1.3739, + "theoretical_loss": 3.8196849201404888, + "tokens_seen": 678952960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418294884653962, + "loss": 1.3537, + "theoretical_loss": 3.819647562362328, + "tokens_seen": 679018496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418134403209629, + "loss": 1.3341, + "theoretical_loss": 3.819610209199069, + "tokens_seen": 679084032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417973921765296, + "loss": 1.3727, + "theoretical_loss": 3.8195728606496973, + "tokens_seen": 679149568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417813440320963, + "loss": 1.3802, + "theoretical_loss": 3.8195355167131977, + "tokens_seen": 679215104 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641765295887663, + "loss": 1.4435, + "theoretical_loss": 3.819498177388555, + "tokens_seen": 679280640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417492477432297, + "loss": 1.421, + "theoretical_loss": 3.8194608426747543, + "tokens_seen": 679346176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417331995987964, + "loss": 1.5297, + "theoretical_loss": 3.819423512570783, + "tokens_seen": 679411712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417171514543631, + "loss": 1.3532, + "theoretical_loss": 3.8193861870756267, + "tokens_seen": 679477248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417011033099298, + "loss": 1.3038, + "theoretical_loss": 3.819348866188271, + "tokens_seen": 679542784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006416850551654964, + "loss": 1.4278, + "theoretical_loss": 3.819311549907704, + "tokens_seen": 679608320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006416690070210632, + "loss": 1.4793, + "theoretical_loss": 3.8192742382329126, + "tokens_seen": 679673856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006416529588766298, + "loss": 1.4789, + "theoretical_loss": 3.819236931162884, + "tokens_seen": 679739392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006416369107321966, + "loss": 1.3844, + "theoretical_loss": 3.8191996286966066, + "tokens_seen": 679804928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006416208625877632, + "loss": 1.3349, + "theoretical_loss": 3.819162330833068, + "tokens_seen": 679870464 + }, + { + "epoch": 0.21, + "learning_rate": 0.00064160481444333, + "loss": 1.4222, + "theoretical_loss": 3.8191250375712564, + "tokens_seen": 679936000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006415887662988966, + "loss": 1.5197, + "theoretical_loss": 3.819087748910161, + "tokens_seen": 680001536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006415727181544634, + "loss": 1.4457, + "theoretical_loss": 3.8190504648487718, + "tokens_seen": 680067072 + }, + { + "epoch": 0.21, + "learning_rate": 0.00064155667001003, + "loss": 1.4379, + "theoretical_loss": 3.8190131853860763, + "tokens_seen": 680132608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006415406218655968, + "loss": 1.5029, + "theoretical_loss": 3.8189759105210657, + "tokens_seen": 680198144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006415245737211634, + "loss": 1.4406, + "theoretical_loss": 3.8189386402527297, + "tokens_seen": 680263680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006415085255767302, + "loss": 1.3714, + "theoretical_loss": 3.818901374580059, + "tokens_seen": 680329216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006414924774322968, + "loss": 1.5279, + "theoretical_loss": 3.818864113502044, + "tokens_seen": 680394752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006414764292878636, + "loss": 1.4553, + "theoretical_loss": 3.818826857017675, + "tokens_seen": 680460288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006414603811434302, + "loss": 1.4746, + "theoretical_loss": 3.8187896051259442, + "tokens_seen": 680525824 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641444332998997, + "loss": 1.4045, + "theoretical_loss": 3.818752357825843, + "tokens_seen": 680591360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006414282848545638, + "loss": 1.37, + "theoretical_loss": 3.8187151151163636, + "tokens_seen": 680656896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006414122367101304, + "loss": 1.4326, + "theoretical_loss": 3.8186778769964986, + "tokens_seen": 680722432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006413961885656972, + "loss": 1.4775, + "theoretical_loss": 3.81864064346524, + "tokens_seen": 680787968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006413801404212638, + "loss": 1.4301, + "theoretical_loss": 3.818603414521581, + "tokens_seen": 680853504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006413640922768306, + "loss": 1.3632, + "theoretical_loss": 3.818566190164515, + "tokens_seen": 680919040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006413480441323972, + "loss": 1.4633, + "theoretical_loss": 3.8185289703930345, + "tokens_seen": 680984576 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641331995987964, + "loss": 1.4356, + "theoretical_loss": 3.818491755206135, + "tokens_seen": 681050112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006413159478435306, + "loss": 1.4231, + "theoretical_loss": 3.8184545446028095, + "tokens_seen": 681115648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412998996990974, + "loss": 1.4653, + "theoretical_loss": 3.8184173385820532, + "tokens_seen": 681181184 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641283851554664, + "loss": 1.3554, + "theoretical_loss": 3.818380137142861, + "tokens_seen": 681246720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412678034102308, + "loss": 1.5759, + "theoretical_loss": 3.8183429402842277, + "tokens_seen": 681312256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412517552657974, + "loss": 1.2995, + "theoretical_loss": 3.8183057480051485, + "tokens_seen": 681377792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412357071213642, + "loss": 1.4206, + "theoretical_loss": 3.81826856030462, + "tokens_seen": 681443328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412196589769308, + "loss": 1.458, + "theoretical_loss": 3.818231377181638, + "tokens_seen": 681508864 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 232668, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2436425685882568, + "objective/train/theoretical_loss": 3.818194198635198, + "objective/train/tokens_used": 702034400, + "theoretical_loss": 3.818194198635198, + "tokens_seen": 681574400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412036108324976, + "loss": 1.3211, + "theoretical_loss": 3.818194198635198, + "tokens_seen": 681574400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006411875626880642, + "loss": 1.3687, + "theoretical_loss": 3.818157024664298, + "tokens_seen": 681639936 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641171514543631, + "loss": 1.4933, + "theoretical_loss": 3.8181198552679345, + "tokens_seen": 681705472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006411554663991976, + "loss": 1.432, + "theoretical_loss": 3.818082690445105, + "tokens_seen": 681771008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006411394182547644, + "loss": 1.4306, + "theoretical_loss": 3.818045530194807, + "tokens_seen": 681836544 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641123370110331, + "loss": 1.4254, + "theoretical_loss": 3.8180083745160394, + "tokens_seen": 681902080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006411073219658978, + "loss": 1.455, + "theoretical_loss": 3.817971223407799, + "tokens_seen": 681967616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006410912738214644, + "loss": 1.367, + "theoretical_loss": 3.8179340768690855, + "tokens_seen": 682033152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006410752256770312, + "loss": 1.3193, + "theoretical_loss": 3.817896934898897, + "tokens_seen": 682098688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006410591775325978, + "loss": 1.3972, + "theoretical_loss": 3.8178597974962343, + "tokens_seen": 682164224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006410431293881646, + "loss": 1.4552, + "theoretical_loss": 3.817822664660096, + "tokens_seen": 682229760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006410270812437312, + "loss": 1.3903, + "theoretical_loss": 3.817785536389481, + "tokens_seen": 682295296 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641011033099298, + "loss": 1.41, + "theoretical_loss": 3.8177484126833914, + "tokens_seen": 682360832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006409949849548646, + "loss": 1.4212, + "theoretical_loss": 3.8177112935408273, + "tokens_seen": 682426368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006409789368104314, + "loss": 1.4377, + "theoretical_loss": 3.8176741789607886, + "tokens_seen": 682491904 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640962888665998, + "loss": 1.3976, + "theoretical_loss": 3.8176370689422767, + "tokens_seen": 682557440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006409468405215648, + "loss": 1.4831, + "theoretical_loss": 3.8175999634842936, + "tokens_seen": 682622976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006409307923771314, + "loss": 1.4321, + "theoretical_loss": 3.8175628625858407, + "tokens_seen": 682688512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006409147442326982, + "loss": 1.4783, + "theoretical_loss": 3.8175257662459208, + "tokens_seen": 682754048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006408986960882648, + "loss": 1.4594, + "theoretical_loss": 3.8174886744635352, + "tokens_seen": 682819584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006408826479438316, + "loss": 1.4002, + "theoretical_loss": 3.8174515872376875, + "tokens_seen": 682885120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006408665997993982, + "loss": 1.3813, + "theoretical_loss": 3.81741450456738, + "tokens_seen": 682950656 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640850551654965, + "loss": 1.3785, + "theoretical_loss": 3.817377426451617, + "tokens_seen": 683016192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006408345035105316, + "loss": 1.5324, + "theoretical_loss": 3.817340352889402, + "tokens_seen": 683081728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006408184553660984, + "loss": 1.4589, + "theoretical_loss": 3.8173032838797383, + "tokens_seen": 683147264 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640802407221665, + "loss": 1.4205, + "theoretical_loss": 3.8172662194216302, + "tokens_seen": 683212800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407863590772318, + "loss": 1.4372, + "theoretical_loss": 3.817229159514083, + "tokens_seen": 683278336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407703109327984, + "loss": 1.4897, + "theoretical_loss": 3.817192104156101, + "tokens_seen": 683343872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407542627883652, + "loss": 1.4622, + "theoretical_loss": 3.8171550533466903, + "tokens_seen": 683409408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407382146439318, + "loss": 1.3386, + "theoretical_loss": 3.817118007084856, + "tokens_seen": 683474944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407221664994986, + "loss": 1.486, + "theoretical_loss": 3.8170809653696036, + "tokens_seen": 683540480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407061183550652, + "loss": 1.3826, + "theoretical_loss": 3.817043928199939, + "tokens_seen": 683606016 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640690070210632, + "loss": 1.4282, + "theoretical_loss": 3.8170068955748695, + "tokens_seen": 683671552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006406740220661986, + "loss": 1.4732, + "theoretical_loss": 3.8169698674934023, + "tokens_seen": 683737088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006406579739217654, + "loss": 1.4747, + "theoretical_loss": 3.816932843954543, + "tokens_seen": 683802624 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640641925777332, + "loss": 1.4705, + "theoretical_loss": 3.8168958249573004, + "tokens_seen": 683868160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006406258776328988, + "loss": 1.451, + "theoretical_loss": 3.8168588105006815, + "tokens_seen": 683933696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006406098294884654, + "loss": 1.4812, + "theoretical_loss": 3.816821800583695, + "tokens_seen": 683999232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006405937813440322, + "loss": 1.5242, + "theoretical_loss": 3.816784795205348, + "tokens_seen": 684064768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006405777331995988, + "loss": 1.3584, + "theoretical_loss": 3.816747794364651, + "tokens_seen": 684130304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006405616850551656, + "loss": 1.4066, + "theoretical_loss": 3.816710798060611, + "tokens_seen": 684195840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006405456369107322, + "loss": 1.4265, + "theoretical_loss": 3.816673806292239, + "tokens_seen": 684261376 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640529588766299, + "loss": 1.4504, + "theoretical_loss": 3.8166368190585436, + "tokens_seen": 684326912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006405135406218656, + "loss": 1.5065, + "theoretical_loss": 3.816599836358535, + "tokens_seen": 684392448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404974924774324, + "loss": 1.2701, + "theoretical_loss": 3.816562858191223, + "tokens_seen": 684457984 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640481444332999, + "loss": 1.433, + "theoretical_loss": 3.8165258845556194, + "tokens_seen": 684523520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404653961885658, + "loss": 1.4168, + "theoretical_loss": 3.8164889154507335, + "tokens_seen": 684589056 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404493480441324, + "loss": 1.3753, + "theoretical_loss": 3.816451950875578, + "tokens_seen": 684654592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404332998996991, + "loss": 1.4882, + "theoretical_loss": 3.8164149908291627, + "tokens_seen": 684720128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404172517552658, + "loss": 1.4322, + "theoretical_loss": 3.8163780353105006, + "tokens_seen": 684785664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 233601, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3923734426498413, + "objective/train/theoretical_loss": 3.816341084318604, + "objective/train/tokens_used": 705311200, + "theoretical_loss": 3.816341084318604, + "tokens_seen": 684851200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404012036108325, + "loss": 1.4253, + "theoretical_loss": 3.816341084318604, + "tokens_seen": 684851200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006403851554663992, + "loss": 1.5158, + "theoretical_loss": 3.816304137852484, + "tokens_seen": 684916736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006403691073219659, + "loss": 1.2934, + "theoretical_loss": 3.8162671959111543, + "tokens_seen": 684982272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006403530591775326, + "loss": 1.4347, + "theoretical_loss": 3.8162302584936274, + "tokens_seen": 685047808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006403370110330993, + "loss": 1.4166, + "theoretical_loss": 3.8161933255989173, + "tokens_seen": 685113344 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640320962888666, + "loss": 1.4776, + "theoretical_loss": 3.8161563972260373, + "tokens_seen": 685178880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006403049147442327, + "loss": 1.437, + "theoretical_loss": 3.816119473374001, + "tokens_seen": 685244416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402888665997993, + "loss": 1.407, + "theoretical_loss": 3.816082554041823, + "tokens_seen": 685309952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402728184553661, + "loss": 1.546, + "theoretical_loss": 3.816045639228518, + "tokens_seen": 685375488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402567703109327, + "loss": 1.4558, + "theoretical_loss": 3.8160087289331006, + "tokens_seen": 685441024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402407221664995, + "loss": 1.574, + "theoretical_loss": 3.815971823154586, + "tokens_seen": 685506560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402246740220662, + "loss": 1.415, + "theoretical_loss": 3.81593492189199, + "tokens_seen": 685572096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402086258776329, + "loss": 1.4116, + "theoretical_loss": 3.8158980251443277, + "tokens_seen": 685637632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006401925777331996, + "loss": 1.4347, + "theoretical_loss": 3.8158611329106162, + "tokens_seen": 685703168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006401765295887663, + "loss": 1.3947, + "theoretical_loss": 3.815824245189871, + "tokens_seen": 685768704 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640160481444333, + "loss": 1.5681, + "theoretical_loss": 3.81578736198111, + "tokens_seen": 685834240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006401444332998997, + "loss": 1.4623, + "theoretical_loss": 3.815750483283349, + "tokens_seen": 685899776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006401283851554664, + "loss": 1.4556, + "theoretical_loss": 3.8157136090956056, + "tokens_seen": 685965312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006401123370110331, + "loss": 1.5815, + "theoretical_loss": 3.8156767394168982, + "tokens_seen": 686030848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400962888665998, + "loss": 1.4877, + "theoretical_loss": 3.8156398742462443, + "tokens_seen": 686096384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400802407221665, + "loss": 1.3404, + "theoretical_loss": 3.8156030135826615, + "tokens_seen": 686161920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400641925777332, + "loss": 1.3622, + "theoretical_loss": 3.81556615742517, + "tokens_seen": 686227456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400481444332999, + "loss": 1.55, + "theoretical_loss": 3.815529305772787, + "tokens_seen": 686292992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400320962888667, + "loss": 1.4737, + "theoretical_loss": 3.815492458624532, + "tokens_seen": 686358528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400160481444333, + "loss": 1.4545, + "theoretical_loss": 3.8154556159794257, + "tokens_seen": 686424064 + }, + { + "epoch": 0.21, + "learning_rate": 0.00064, + "loss": 1.4074, + "theoretical_loss": 3.815418777836487, + "tokens_seen": 686489600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399839518555667, + "loss": 1.5461, + "theoretical_loss": 3.8153819441947356, + "tokens_seen": 686555136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399679037111335, + "loss": 1.3217, + "theoretical_loss": 3.815345115053193, + "tokens_seen": 686620672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399518555667001, + "loss": 1.4626, + "theoretical_loss": 3.815308290410879, + "tokens_seen": 686686208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399358074222669, + "loss": 1.3287, + "theoretical_loss": 3.815271470266815, + "tokens_seen": 686751744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399197592778335, + "loss": 1.522, + "theoretical_loss": 3.8152346546200224, + "tokens_seen": 686817280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399037111334003, + "loss": 1.5696, + "theoretical_loss": 3.815197843469523, + "tokens_seen": 686882816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398876629889669, + "loss": 1.4541, + "theoretical_loss": 3.815161036814338, + "tokens_seen": 686948352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398716148445337, + "loss": 1.4031, + "theoretical_loss": 3.8151242346534904, + "tokens_seen": 687013888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398555667001003, + "loss": 1.4871, + "theoretical_loss": 3.8150874369860026, + "tokens_seen": 687079424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398395185556671, + "loss": 1.5176, + "theoretical_loss": 3.8150506438108973, + "tokens_seen": 687144960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398234704112337, + "loss": 1.4159, + "theoretical_loss": 3.815013855127198, + "tokens_seen": 687210496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398074222668005, + "loss": 1.3235, + "theoretical_loss": 3.814977070933928, + "tokens_seen": 687276032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397913741223671, + "loss": 1.4435, + "theoretical_loss": 3.814940291230111, + "tokens_seen": 687341568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397753259779339, + "loss": 1.4527, + "theoretical_loss": 3.8149035160147706, + "tokens_seen": 687407104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397592778335005, + "loss": 1.4967, + "theoretical_loss": 3.8148667452869325, + "tokens_seen": 687472640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397432296890673, + "loss": 1.4179, + "theoretical_loss": 3.81482997904562, + "tokens_seen": 687538176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397271815446339, + "loss": 1.3892, + "theoretical_loss": 3.8147932172898598, + "tokens_seen": 687603712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397111334002007, + "loss": 1.5138, + "theoretical_loss": 3.814756460018675, + "tokens_seen": 687669248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396950852557673, + "loss": 1.4843, + "theoretical_loss": 3.814719707231093, + "tokens_seen": 687734784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396790371113341, + "loss": 1.3816, + "theoretical_loss": 3.814682958926139, + "tokens_seen": 687800320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396629889669007, + "loss": 1.456, + "theoretical_loss": 3.8146462151028393, + "tokens_seen": 687865856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396469408224675, + "loss": 1.4038, + "theoretical_loss": 3.8146094757602205, + "tokens_seen": 687931392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396308926780341, + "loss": 1.3949, + "theoretical_loss": 3.814572740897309, + "tokens_seen": 687996928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396148445336009, + "loss": 1.2949, + "theoretical_loss": 3.814536010513133, + "tokens_seen": 688062464 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 234843, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4647266864776611, + "objective/train/theoretical_loss": 3.8144992846067187, + "objective/train/tokens_used": 708588000, + "theoretical_loss": 3.8144992846067187, + "tokens_seen": 688128000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395987963891675, + "loss": 1.4728, + "theoretical_loss": 3.8144992846067187, + "tokens_seen": 688128000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395827482447343, + "loss": 1.4255, + "theoretical_loss": 3.8144625631770945, + "tokens_seen": 688193536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395667001003009, + "loss": 1.496, + "theoretical_loss": 3.8144258462232887, + "tokens_seen": 688259072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395506519558677, + "loss": 1.5404, + "theoretical_loss": 3.8143891337443288, + "tokens_seen": 688324608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395346038114343, + "loss": 1.5205, + "theoretical_loss": 3.8143524257392443, + "tokens_seen": 688390144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395185556670011, + "loss": 1.4784, + "theoretical_loss": 3.814315722207064, + "tokens_seen": 688455680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395025075225677, + "loss": 1.4064, + "theoretical_loss": 3.814279023146817, + "tokens_seen": 688521216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394864593781345, + "loss": 1.3965, + "theoretical_loss": 3.8142423285575324, + "tokens_seen": 688586752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394704112337011, + "loss": 1.4664, + "theoretical_loss": 3.814205638438241, + "tokens_seen": 688652288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394543630892679, + "loss": 1.3477, + "theoretical_loss": 3.814168952787972, + "tokens_seen": 688717824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394383149448345, + "loss": 1.4602, + "theoretical_loss": 3.8141322716057564, + "tokens_seen": 688783360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394222668004013, + "loss": 1.3104, + "theoretical_loss": 3.814095594890625, + "tokens_seen": 688848896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394062186559679, + "loss": 1.4389, + "theoretical_loss": 3.8140589226416095, + "tokens_seen": 688914432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393901705115347, + "loss": 1.4372, + "theoretical_loss": 3.81402225485774, + "tokens_seen": 688979968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393741223671013, + "loss": 1.4144, + "theoretical_loss": 3.8139855915380485, + "tokens_seen": 689045504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393580742226681, + "loss": 1.3738, + "theoretical_loss": 3.813948932681568, + "tokens_seen": 689111040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393420260782347, + "loss": 1.4324, + "theoretical_loss": 3.8139122782873294, + "tokens_seen": 689176576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393259779338015, + "loss": 1.2945, + "theoretical_loss": 3.813875628354366, + "tokens_seen": 689242112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393099297893681, + "loss": 1.5289, + "theoretical_loss": 3.8138389828817116, + "tokens_seen": 689307648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392938816449349, + "loss": 1.4435, + "theoretical_loss": 3.8138023418683975, + "tokens_seen": 689373184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392778335005015, + "loss": 1.5137, + "theoretical_loss": 3.8137657053134584, + "tokens_seen": 689438720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392617853560683, + "loss": 1.3912, + "theoretical_loss": 3.813729073215928, + "tokens_seen": 689504256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392457372116349, + "loss": 1.447, + "theoretical_loss": 3.8136924455748398, + "tokens_seen": 689569792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392296890672017, + "loss": 1.3175, + "theoretical_loss": 3.813655822389229, + "tokens_seen": 689635328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392136409227683, + "loss": 1.4715, + "theoretical_loss": 3.8136192036581296, + "tokens_seen": 689700864 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391975927783351, + "loss": 1.4261, + "theoretical_loss": 3.813582589380577, + "tokens_seen": 689766400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391815446339017, + "loss": 1.4151, + "theoretical_loss": 3.8135459795556064, + "tokens_seen": 689831936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391654964894685, + "loss": 1.5456, + "theoretical_loss": 3.8135093741822534, + "tokens_seen": 689897472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391494483450351, + "loss": 1.5161, + "theoretical_loss": 3.813472773259554, + "tokens_seen": 689963008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391334002006019, + "loss": 1.4062, + "theoretical_loss": 3.8134361767865443, + "tokens_seen": 690028544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391173520561685, + "loss": 1.38, + "theoretical_loss": 3.8133995847622613, + "tokens_seen": 690094080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391013039117353, + "loss": 1.5294, + "theoretical_loss": 3.8133629971857403, + "tokens_seen": 690159616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390852557673019, + "loss": 1.4732, + "theoretical_loss": 3.8133264140560197, + "tokens_seen": 690225152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390692076228687, + "loss": 1.4324, + "theoretical_loss": 3.813289835372137, + "tokens_seen": 690290688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390531594784353, + "loss": 1.4654, + "theoretical_loss": 3.81325326113313, + "tokens_seen": 690356224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390371113340021, + "loss": 1.4102, + "theoretical_loss": 3.8132166913380354, + "tokens_seen": 690421760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390210631895687, + "loss": 1.4409, + "theoretical_loss": 3.8131801259858924, + "tokens_seen": 690487296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390050150451354, + "loss": 1.4745, + "theoretical_loss": 3.8131435650757397, + "tokens_seen": 690552832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389889669007021, + "loss": 1.373, + "theoretical_loss": 3.8131070086066154, + "tokens_seen": 690618368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389729187562688, + "loss": 1.3771, + "theoretical_loss": 3.8130704565775604, + "tokens_seen": 690683904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389568706118355, + "loss": 1.4569, + "theoretical_loss": 3.8130339089876126, + "tokens_seen": 690749440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389408224674022, + "loss": 1.3497, + "theoretical_loss": 3.8129973658358125, + "tokens_seen": 690814976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389247743229689, + "loss": 1.4776, + "theoretical_loss": 3.8129608271211994, + "tokens_seen": 690880512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389087261785356, + "loss": 1.4584, + "theoretical_loss": 3.8129242928428146, + "tokens_seen": 690946048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006388926780341023, + "loss": 1.5172, + "theoretical_loss": 3.812887762999699, + "tokens_seen": 691011584 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638876629889669, + "loss": 1.4696, + "theoretical_loss": 3.8128512375908925, + "tokens_seen": 691077120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006388605817452357, + "loss": 1.4223, + "theoretical_loss": 3.8128147166154376, + "tokens_seen": 691142656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006388445336008024, + "loss": 1.4317, + "theoretical_loss": 3.8127782000723744, + "tokens_seen": 691208192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006388284854563691, + "loss": 1.3205, + "theoretical_loss": 3.812741687960746, + "tokens_seen": 691273728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006388124373119358, + "loss": 1.4841, + "theoretical_loss": 3.8127051802795946, + "tokens_seen": 691339264 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 235714, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3893272876739502, + "objective/train/theoretical_loss": 3.812668677027962, + "objective/train/tokens_used": 711864800, + "theoretical_loss": 3.812668677027962, + "tokens_seen": 691404800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006387963891675025, + "loss": 1.4677, + "theoretical_loss": 3.812668677027962, + "tokens_seen": 691404800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006387803410230692, + "loss": 1.4838, + "theoretical_loss": 3.8126321782048915, + "tokens_seen": 691470336 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638764292878636, + "loss": 1.4568, + "theoretical_loss": 3.8125956838094255, + "tokens_seen": 691535872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006387482447342026, + "loss": 1.5229, + "theoretical_loss": 3.812559193840608, + "tokens_seen": 691601408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006387321965897693, + "loss": 1.4393, + "theoretical_loss": 3.8125227082974824, + "tokens_seen": 691666944 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638716148445336, + "loss": 1.4725, + "theoretical_loss": 3.812486227179093, + "tokens_seen": 691732480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006387001003009027, + "loss": 1.5032, + "theoretical_loss": 3.812449750484484, + "tokens_seen": 691798016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006386840521564694, + "loss": 1.3823, + "theoretical_loss": 3.8124132782126994, + "tokens_seen": 691863552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006386680040120362, + "loss": 1.3504, + "theoretical_loss": 3.812376810362785, + "tokens_seen": 691929088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006386519558676028, + "loss": 1.4185, + "theoretical_loss": 3.8123403469337847, + "tokens_seen": 691994624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006386359077231696, + "loss": 1.5098, + "theoretical_loss": 3.812303887924745, + "tokens_seen": 692060160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006386198595787362, + "loss": 1.5385, + "theoretical_loss": 3.8122674333347115, + "tokens_seen": 692125696 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638603811434303, + "loss": 1.3335, + "theoretical_loss": 3.8122309831627295, + "tokens_seen": 692191232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006385877632898696, + "loss": 1.4043, + "theoretical_loss": 3.8121945374078465, + "tokens_seen": 692256768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006385717151454364, + "loss": 1.481, + "theoretical_loss": 3.8121580960691084, + "tokens_seen": 692322304 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638555667001003, + "loss": 1.5184, + "theoretical_loss": 3.812121659145562, + "tokens_seen": 692387840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006385396188565698, + "loss": 1.5102, + "theoretical_loss": 3.8120852266362553, + "tokens_seen": 692453376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006385235707121364, + "loss": 1.4312, + "theoretical_loss": 3.8120487985402347, + "tokens_seen": 692518912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006385075225677032, + "loss": 1.4521, + "theoretical_loss": 3.8120123748565486, + "tokens_seen": 692584448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006384914744232698, + "loss": 1.3721, + "theoretical_loss": 3.811975955584246, + "tokens_seen": 692649984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006384754262788366, + "loss": 1.4386, + "theoretical_loss": 3.8119395407223737, + "tokens_seen": 692715520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006384593781344032, + "loss": 1.3849, + "theoretical_loss": 3.8119031302699815, + "tokens_seen": 692781056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00063844332998997, + "loss": 1.3678, + "theoretical_loss": 3.8118667242261175, + "tokens_seen": 692846592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006384272818455366, + "loss": 1.3124, + "theoretical_loss": 3.8118303225898327, + "tokens_seen": 692912128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006384112337011034, + "loss": 1.4687, + "theoretical_loss": 3.8117939253601745, + "tokens_seen": 692977664 + }, + { + "epoch": 0.21, + "learning_rate": 0.00063839518555667, + "loss": 1.4331, + "theoretical_loss": 3.811757532536194, + "tokens_seen": 693043200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006383791374122368, + "loss": 1.4734, + "theoretical_loss": 3.811721144116942, + "tokens_seen": 693108736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006383630892678034, + "loss": 1.4495, + "theoretical_loss": 3.8116847601014676, + "tokens_seen": 693174272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006383470411233702, + "loss": 1.3543, + "theoretical_loss": 3.811648380488822, + "tokens_seen": 693239808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006383309929789368, + "loss": 1.483, + "theoretical_loss": 3.811612005278057, + "tokens_seen": 693305344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006383149448345036, + "loss": 1.3364, + "theoretical_loss": 3.8115756344682232, + "tokens_seen": 693370880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006382988966900702, + "loss": 1.3732, + "theoretical_loss": 3.811539268058372, + "tokens_seen": 693436416 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638282848545637, + "loss": 1.3328, + "theoretical_loss": 3.811502906047556, + "tokens_seen": 693501952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006382668004012036, + "loss": 1.2924, + "theoretical_loss": 3.8114665484348276, + "tokens_seen": 693567488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006382507522567704, + "loss": 1.4352, + "theoretical_loss": 3.8114301952192386, + "tokens_seen": 693633024 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638234704112337, + "loss": 1.3933, + "theoretical_loss": 3.8113938463998425, + "tokens_seen": 693698560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006382186559679038, + "loss": 1.4227, + "theoretical_loss": 3.811357501975692, + "tokens_seen": 693764096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006382026078234704, + "loss": 1.4534, + "theoretical_loss": 3.8113211619458403, + "tokens_seen": 693829632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006381865596790372, + "loss": 1.4038, + "theoretical_loss": 3.811284826309342, + "tokens_seen": 693895168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006381705115346038, + "loss": 1.4275, + "theoretical_loss": 3.81124849506525, + "tokens_seen": 693960704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006381544633901706, + "loss": 1.4279, + "theoretical_loss": 3.811212168212619, + "tokens_seen": 694026240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006381384152457372, + "loss": 1.5186, + "theoretical_loss": 3.8111758457505047, + "tokens_seen": 694091776 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638122367101304, + "loss": 1.3418, + "theoretical_loss": 3.8111395276779607, + "tokens_seen": 694157312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006381063189568706, + "loss": 1.4026, + "theoretical_loss": 3.8111032139940417, + "tokens_seen": 694222848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006380902708124374, + "loss": 1.4274, + "theoretical_loss": 3.811066904697805, + "tokens_seen": 694288384 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638074222668004, + "loss": 1.4758, + "theoretical_loss": 3.811030599788304, + "tokens_seen": 694353920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006380581745235708, + "loss": 1.3915, + "theoretical_loss": 3.810994299264597, + "tokens_seen": 694419456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006380421263791374, + "loss": 1.4937, + "theoretical_loss": 3.810958003125739, + "tokens_seen": 694484992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006380260782347042, + "loss": 1.3575, + "theoretical_loss": 3.8109217113707876, + "tokens_seen": 694550528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006380100300902708, + "loss": 1.4403, + "theoretical_loss": 3.8108854239987986, + "tokens_seen": 694616064 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 236782, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2472243309020996, + "objective/train/theoretical_loss": 3.81084914100883, + "objective/train/tokens_used": 715141600, + "theoretical_loss": 3.81084914100883, + "tokens_seen": 694681600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006379939819458376, + "loss": 1.3429, + "theoretical_loss": 3.81084914100883, + "tokens_seen": 694681600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006379779338014042, + "loss": 1.4143, + "theoretical_loss": 3.810812862399939, + "tokens_seen": 694747136 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637961885656971, + "loss": 1.4951, + "theoretical_loss": 3.8107765881711835, + "tokens_seen": 694812672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006379458375125376, + "loss": 1.3626, + "theoretical_loss": 3.8107403183216215, + "tokens_seen": 694878208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006379297893681044, + "loss": 1.453, + "theoretical_loss": 3.810704052850311, + "tokens_seen": 694943744 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637913741223671, + "loss": 1.41, + "theoretical_loss": 3.8106677917563117, + "tokens_seen": 695009280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378976930792378, + "loss": 1.4152, + "theoretical_loss": 3.810631535038682, + "tokens_seen": 695074816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378816449348044, + "loss": 1.4707, + "theoretical_loss": 3.810595282696481, + "tokens_seen": 695140352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378655967903712, + "loss": 1.3896, + "theoretical_loss": 3.810559034728769, + "tokens_seen": 695205888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378495486459378, + "loss": 1.3994, + "theoretical_loss": 3.8105227911346042, + "tokens_seen": 695271424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378335005015046, + "loss": 1.4259, + "theoretical_loss": 3.810486551913048, + "tokens_seen": 695336960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378174523570713, + "loss": 1.3895, + "theoretical_loss": 3.810450317063161, + "tokens_seen": 695402496 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637801404212638, + "loss": 1.4288, + "theoretical_loss": 3.8104140865840033, + "tokens_seen": 695468032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006377853560682047, + "loss": 1.4986, + "theoretical_loss": 3.810377860474636, + "tokens_seen": 695533568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006377693079237714, + "loss": 1.5363, + "theoretical_loss": 3.8103416387341205, + "tokens_seen": 695599104 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637753259779338, + "loss": 1.5343, + "theoretical_loss": 3.8103054213615186, + "tokens_seen": 695664640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006377372116349048, + "loss": 1.3992, + "theoretical_loss": 3.8102692083558916, + "tokens_seen": 695730176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006377211634904714, + "loss": 1.4556, + "theoretical_loss": 3.810232999716302, + "tokens_seen": 695795712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006377051153460382, + "loss": 1.5742, + "theoretical_loss": 3.8101967954418123, + "tokens_seen": 695861248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006376890672016048, + "loss": 1.4954, + "theoretical_loss": 3.8101605955314857, + "tokens_seen": 695926784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006376730190571716, + "loss": 1.4589, + "theoretical_loss": 3.8101243999843843, + "tokens_seen": 695992320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006376569709127382, + "loss": 1.4251, + "theoretical_loss": 3.8100882087995718, + "tokens_seen": 696057856 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637640922768305, + "loss": 1.369, + "theoretical_loss": 3.810052021976112, + "tokens_seen": 696123392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006376248746238716, + "loss": 1.5036, + "theoretical_loss": 3.8100158395130683, + "tokens_seen": 696188928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006376088264794383, + "loss": 1.5799, + "theoretical_loss": 3.809979661409505, + "tokens_seen": 696254464 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637592778335005, + "loss": 1.4678, + "theoretical_loss": 3.809943487664488, + "tokens_seen": 696320000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006375767301905717, + "loss": 1.3974, + "theoretical_loss": 3.8099073182770797, + "tokens_seen": 696385536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006375606820461384, + "loss": 1.3969, + "theoretical_loss": 3.809871153246347, + "tokens_seen": 696451072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006375446339017051, + "loss": 1.3703, + "theoretical_loss": 3.8098349925713544, + "tokens_seen": 696516608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006375285857572718, + "loss": 1.389, + "theoretical_loss": 3.809798836251167, + "tokens_seen": 696582144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006375125376128385, + "loss": 1.4636, + "theoretical_loss": 3.809762684284852, + "tokens_seen": 696647680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374964894684052, + "loss": 1.5068, + "theoretical_loss": 3.809726536671475, + "tokens_seen": 696713216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374804413239719, + "loss": 1.4978, + "theoretical_loss": 3.8096903934101025, + "tokens_seen": 696778752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374643931795386, + "loss": 1.4403, + "theoretical_loss": 3.8096542544998018, + "tokens_seen": 696844288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374483450351053, + "loss": 1.408, + "theoretical_loss": 3.8096181199396386, + "tokens_seen": 696909824 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637432296890672, + "loss": 1.5257, + "theoretical_loss": 3.8095819897286813, + "tokens_seen": 696975360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374162487462387, + "loss": 1.2994, + "theoretical_loss": 3.8095458638659974, + "tokens_seen": 697040896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374002006018054, + "loss": 1.431, + "theoretical_loss": 3.809509742350655, + "tokens_seen": 697106432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373841524573721, + "loss": 1.4339, + "theoretical_loss": 3.809473625181722, + "tokens_seen": 697171968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373681043129388, + "loss": 1.3853, + "theoretical_loss": 3.809437512358267, + "tokens_seen": 697237504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373520561685055, + "loss": 1.4096, + "theoretical_loss": 3.809401403879359, + "tokens_seen": 697303040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373360080240722, + "loss": 1.4918, + "theoretical_loss": 3.8093652997440666, + "tokens_seen": 697368576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373199598796389, + "loss": 1.4135, + "theoretical_loss": 3.80932919995146, + "tokens_seen": 697434112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373039117352057, + "loss": 1.4797, + "theoretical_loss": 3.8092931045006075, + "tokens_seen": 697499648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006372878635907723, + "loss": 1.3962, + "theoretical_loss": 3.8092570133905808, + "tokens_seen": 697565184 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637271815446339, + "loss": 1.5149, + "theoretical_loss": 3.809220926620448, + "tokens_seen": 697630720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006372557673019057, + "loss": 1.5153, + "theoretical_loss": 3.809184844189282, + "tokens_seen": 697696256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006372397191574725, + "loss": 1.4797, + "theoretical_loss": 3.8091487660961514, + "tokens_seen": 697761792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006372236710130391, + "loss": 1.4721, + "theoretical_loss": 3.809112692340129, + "tokens_seen": 697827328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006372076228686059, + "loss": 1.3884, + "theoretical_loss": 3.8090766229202853, + "tokens_seen": 697892864 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 238184, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4217039346694946, + "objective/train/theoretical_loss": 3.8090405578356927, + "objective/train/tokens_used": 718418400, + "theoretical_loss": 3.8090405578356927, + "tokens_seen": 697958400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371915747241725, + "loss": 1.3576, + "theoretical_loss": 3.8090405578356927, + "tokens_seen": 697958400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371755265797393, + "loss": 1.4004, + "theoretical_loss": 3.8090044970854215, + "tokens_seen": 698023936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371594784353059, + "loss": 1.4644, + "theoretical_loss": 3.8089684406685453, + "tokens_seen": 698089472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371434302908727, + "loss": 1.3212, + "theoretical_loss": 3.808932388584137, + "tokens_seen": 698155008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371273821464393, + "loss": 1.3765, + "theoretical_loss": 3.808896340831268, + "tokens_seen": 698220544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371113340020061, + "loss": 1.4736, + "theoretical_loss": 3.808860297409012, + "tokens_seen": 698286080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370952858575727, + "loss": 1.3824, + "theoretical_loss": 3.8088242583164433, + "tokens_seen": 698351616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370792377131395, + "loss": 1.4751, + "theoretical_loss": 3.808788223552634, + "tokens_seen": 698417152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370631895687061, + "loss": 1.5259, + "theoretical_loss": 3.8087521931166592, + "tokens_seen": 698482688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370471414242729, + "loss": 1.4854, + "theoretical_loss": 3.8087161670075926, + "tokens_seen": 698548224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370310932798395, + "loss": 1.5663, + "theoretical_loss": 3.8086801452245087, + "tokens_seen": 698613760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370150451354063, + "loss": 1.4904, + "theoretical_loss": 3.808644127766483, + "tokens_seen": 698679296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369989969909729, + "loss": 1.4431, + "theoretical_loss": 3.808608114632589, + "tokens_seen": 698744832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369829488465397, + "loss": 1.392, + "theoretical_loss": 3.808572105821904, + "tokens_seen": 698810368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369669007021063, + "loss": 1.4019, + "theoretical_loss": 3.8085361013335026, + "tokens_seen": 698875904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369508525576731, + "loss": 1.3516, + "theoretical_loss": 3.8085001011664605, + "tokens_seen": 698941440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369348044132397, + "loss": 1.4292, + "theoretical_loss": 3.8084641053198545, + "tokens_seen": 699006976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369187562688065, + "loss": 1.4497, + "theoretical_loss": 3.808428113792761, + "tokens_seen": 699072512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369027081243731, + "loss": 1.4372, + "theoretical_loss": 3.8083921265842564, + "tokens_seen": 699138048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368866599799399, + "loss": 1.4794, + "theoretical_loss": 3.8083561436934183, + "tokens_seen": 699203584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368706118355065, + "loss": 1.3279, + "theoretical_loss": 3.8083201651193237, + "tokens_seen": 699269120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368545636910733, + "loss": 1.3738, + "theoretical_loss": 3.80828419086105, + "tokens_seen": 699334656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368385155466399, + "loss": 1.4592, + "theoretical_loss": 3.808248220917676, + "tokens_seen": 699400192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368224674022067, + "loss": 1.5415, + "theoretical_loss": 3.8082122552882796, + "tokens_seen": 699465728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368064192577733, + "loss": 1.4874, + "theoretical_loss": 3.808176293971939, + "tokens_seen": 699531264 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367903711133401, + "loss": 1.4419, + "theoretical_loss": 3.8081403369677322, + "tokens_seen": 699596800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367743229689067, + "loss": 1.4602, + "theoretical_loss": 3.8081043842747397, + "tokens_seen": 699662336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367582748244735, + "loss": 1.2952, + "theoretical_loss": 3.8080684358920402, + "tokens_seen": 699727872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367422266800401, + "loss": 1.4549, + "theoretical_loss": 3.8080324918187136, + "tokens_seen": 699793408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367261785356069, + "loss": 1.4145, + "theoretical_loss": 3.8079965520538392, + "tokens_seen": 699858944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367101303911735, + "loss": 1.4283, + "theoretical_loss": 3.8079606165964974, + "tokens_seen": 699924480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366940822467403, + "loss": 1.4507, + "theoretical_loss": 3.8079246854457693, + "tokens_seen": 699990016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366780341023069, + "loss": 1.4945, + "theoretical_loss": 3.807888758600735, + "tokens_seen": 700055552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366619859578737, + "loss": 1.4224, + "theoretical_loss": 3.8078528360604755, + "tokens_seen": 700121088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366459378134403, + "loss": 1.3546, + "theoretical_loss": 3.8078169178240726, + "tokens_seen": 700186624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366298896690071, + "loss": 1.4382, + "theoretical_loss": 3.8077810038906073, + "tokens_seen": 700252160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366138415245738, + "loss": 1.3947, + "theoretical_loss": 3.807745094259162, + "tokens_seen": 700317696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365977933801405, + "loss": 1.3775, + "theoretical_loss": 3.807709188928819, + "tokens_seen": 700383232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365817452357072, + "loss": 1.4585, + "theoretical_loss": 3.80767328789866, + "tokens_seen": 700448768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365656970912739, + "loss": 1.4953, + "theoretical_loss": 3.807637391167768, + "tokens_seen": 700514304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365496489468406, + "loss": 1.5442, + "theoretical_loss": 3.8076014987352265, + "tokens_seen": 700579840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365336008024073, + "loss": 1.4671, + "theoretical_loss": 3.8075656106001183, + "tokens_seen": 700645376 + }, + { + "epoch": 0.21, + "learning_rate": 0.000636517552657974, + "loss": 1.4537, + "theoretical_loss": 3.807529726761527, + "tokens_seen": 700710912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365015045135407, + "loss": 1.4597, + "theoretical_loss": 3.807493847218537, + "tokens_seen": 700776448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364854563691074, + "loss": 1.4342, + "theoretical_loss": 3.8074579719702317, + "tokens_seen": 700841984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364694082246741, + "loss": 1.4122, + "theoretical_loss": 3.8074221010156957, + "tokens_seen": 700907520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364533600802408, + "loss": 1.5221, + "theoretical_loss": 3.8073862343540137, + "tokens_seen": 700973056 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364373119358075, + "loss": 1.4289, + "theoretical_loss": 3.807350371984271, + "tokens_seen": 701038592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364212637913742, + "loss": 1.491, + "theoretical_loss": 3.8073145139055526, + "tokens_seen": 701104128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364052156469409, + "loss": 1.4687, + "theoretical_loss": 3.8072786601169435, + "tokens_seen": 701169664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 239083, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2331576347351074, + "objective/train/theoretical_loss": 3.807242810617531, + "objective/train/tokens_used": 721695200, + "theoretical_loss": 3.807242810617531, + "tokens_seen": 701235200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363891675025076, + "loss": 1.3397, + "theoretical_loss": 3.807242810617531, + "tokens_seen": 701235200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363731193580743, + "loss": 1.5871, + "theoretical_loss": 3.8072069654063996, + "tokens_seen": 701300736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363570712136409, + "loss": 1.4155, + "theoretical_loss": 3.8071711244826365, + "tokens_seen": 701366272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363410230692077, + "loss": 1.4349, + "theoretical_loss": 3.807135287845328, + "tokens_seen": 701431808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363249749247743, + "loss": 1.4464, + "theoretical_loss": 3.807099455493561, + "tokens_seen": 701497344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363089267803411, + "loss": 1.5763, + "theoretical_loss": 3.807063627426424, + "tokens_seen": 701562880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362928786359077, + "loss": 1.5114, + "theoretical_loss": 3.8070278036430025, + "tokens_seen": 701628416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362768304914745, + "loss": 1.3981, + "theoretical_loss": 3.8069919841423854, + "tokens_seen": 701693952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362607823470411, + "loss": 1.504, + "theoretical_loss": 3.8069561689236604, + "tokens_seen": 701759488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362447342026079, + "loss": 1.4841, + "theoretical_loss": 3.8069203579859163, + "tokens_seen": 701825024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362286860581745, + "loss": 1.4901, + "theoretical_loss": 3.806884551328242, + "tokens_seen": 701890560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362126379137412, + "loss": 1.3995, + "theoretical_loss": 3.806848748949725, + "tokens_seen": 701956096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361965897693079, + "loss": 1.4441, + "theoretical_loss": 3.806812950849455, + "tokens_seen": 702021632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361805416248746, + "loss": 1.5151, + "theoretical_loss": 3.806777157026522, + "tokens_seen": 702087168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361644934804413, + "loss": 1.5374, + "theoretical_loss": 3.806741367480016, + "tokens_seen": 702152704 + }, + { + "epoch": 0.21, + "learning_rate": 0.000636148445336008, + "loss": 1.364, + "theoretical_loss": 3.806705582209026, + "tokens_seen": 702218240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361323971915747, + "loss": 1.3221, + "theoretical_loss": 3.8066698012126423, + "tokens_seen": 702283776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361163490471414, + "loss": 1.4006, + "theoretical_loss": 3.8066340244899566, + "tokens_seen": 702349312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361003009027081, + "loss": 1.426, + "theoretical_loss": 3.8065982520400583, + "tokens_seen": 702414848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006360842527582748, + "loss": 1.3471, + "theoretical_loss": 3.80656248386204, + "tokens_seen": 702480384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006360682046138415, + "loss": 1.4601, + "theoretical_loss": 3.8065267199549924, + "tokens_seen": 702545920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006360521564694082, + "loss": 1.4746, + "theoretical_loss": 3.8064909603180066, + "tokens_seen": 702611456 + }, + { + "epoch": 0.21, + "learning_rate": 0.000636036108324975, + "loss": 1.4666, + "theoretical_loss": 3.8064552049501756, + "tokens_seen": 702676992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006360200601805416, + "loss": 1.5028, + "theoretical_loss": 3.806419453850591, + "tokens_seen": 702742528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006360040120361083, + "loss": 1.322, + "theoretical_loss": 3.806383707018345, + "tokens_seen": 702808064 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635987963891675, + "loss": 1.4048, + "theoretical_loss": 3.806347964452531, + "tokens_seen": 702873600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006359719157472417, + "loss": 1.3752, + "theoretical_loss": 3.806312226152242, + "tokens_seen": 702939136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006359558676028084, + "loss": 1.4836, + "theoretical_loss": 3.806276492116571, + "tokens_seen": 703004672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006359398194583752, + "loss": 1.4695, + "theoretical_loss": 3.806240762344612, + "tokens_seen": 703070208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006359237713139418, + "loss": 1.4428, + "theoretical_loss": 3.806205036835459, + "tokens_seen": 703135744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006359077231695086, + "loss": 1.384, + "theoretical_loss": 3.806169315588206, + "tokens_seen": 703201280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006358916750250752, + "loss": 1.408, + "theoretical_loss": 3.8061335986019467, + "tokens_seen": 703266816 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635875626880642, + "loss": 1.4797, + "theoretical_loss": 3.806097885875777, + "tokens_seen": 703332352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006358595787362086, + "loss": 1.439, + "theoretical_loss": 3.8060621774087915, + "tokens_seen": 703397888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006358435305917754, + "loss": 1.4779, + "theoretical_loss": 3.8060264732000846, + "tokens_seen": 703463424 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635827482447342, + "loss": 1.5238, + "theoretical_loss": 3.8059907732487535, + "tokens_seen": 703528960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006358114343029088, + "loss": 1.3687, + "theoretical_loss": 3.8059550775538926, + "tokens_seen": 703594496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006357953861584754, + "loss": 1.5413, + "theoretical_loss": 3.805919386114599, + "tokens_seen": 703660032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006357793380140422, + "loss": 1.3239, + "theoretical_loss": 3.8058836989299683, + "tokens_seen": 703725568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006357632898696088, + "loss": 1.5187, + "theoretical_loss": 3.8058480159990977, + "tokens_seen": 703791104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006357472417251756, + "loss": 1.438, + "theoretical_loss": 3.805812337321084, + "tokens_seen": 703856640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006357311935807422, + "loss": 1.4494, + "theoretical_loss": 3.805776662895024, + "tokens_seen": 703922176 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635715145436309, + "loss": 1.4415, + "theoretical_loss": 3.8057409927200156, + "tokens_seen": 703987712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356990972918756, + "loss": 1.2744, + "theoretical_loss": 3.805705326795157, + "tokens_seen": 704053248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356830491474424, + "loss": 1.4325, + "theoretical_loss": 3.805669665119545, + "tokens_seen": 704118784 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635667001003009, + "loss": 1.3515, + "theoretical_loss": 3.8056340076922797, + "tokens_seen": 704184320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356509528585758, + "loss": 1.5127, + "theoretical_loss": 3.8055983545124583, + "tokens_seen": 704249856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356349047141424, + "loss": 1.4126, + "theoretical_loss": 3.805562705579179, + "tokens_seen": 704315392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356188565697092, + "loss": 1.4153, + "theoretical_loss": 3.805527060891543, + "tokens_seen": 704380928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356028084252758, + "loss": 1.3536, + "theoretical_loss": 3.8054914204486487, + "tokens_seen": 704446464 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 240373, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5571424961090088, + "objective/train/theoretical_loss": 3.8054557842495957, + "objective/train/tokens_used": 724972000, + "theoretical_loss": 3.8054557842495957, + "tokens_seen": 704512000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006355867602808426, + "loss": 1.4782, + "theoretical_loss": 3.8054557842495957, + "tokens_seen": 704512000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006355707121364092, + "loss": 1.4989, + "theoretical_loss": 3.8054201522934843, + "tokens_seen": 704577536 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635554663991976, + "loss": 1.5441, + "theoretical_loss": 3.805384524579414, + "tokens_seen": 704643072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006355386158475426, + "loss": 1.5061, + "theoretical_loss": 3.805348901106486, + "tokens_seen": 704708608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006355225677031094, + "loss": 1.4131, + "theoretical_loss": 3.8053132818738002, + "tokens_seen": 704774144 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635506519558676, + "loss": 1.4338, + "theoretical_loss": 3.805277666880459, + "tokens_seen": 704839680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354904714142428, + "loss": 1.4124, + "theoretical_loss": 3.8052420561255627, + "tokens_seen": 704905216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354744232698094, + "loss": 1.5405, + "theoretical_loss": 3.805206449608214, + "tokens_seen": 704970752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354583751253762, + "loss": 1.5212, + "theoretical_loss": 3.8051708473275134, + "tokens_seen": 705036288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354423269809428, + "loss": 1.4513, + "theoretical_loss": 3.805135249282564, + "tokens_seen": 705101824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354262788365096, + "loss": 1.4789, + "theoretical_loss": 3.8050996554724676, + "tokens_seen": 705167360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354102306920763, + "loss": 1.513, + "theoretical_loss": 3.8050640658963277, + "tokens_seen": 705232896 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635394182547643, + "loss": 1.4029, + "theoretical_loss": 3.8050284805532466, + "tokens_seen": 705298432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006353781344032097, + "loss": 1.5177, + "theoretical_loss": 3.804992899442328, + "tokens_seen": 705363968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006353620862587764, + "loss": 1.4642, + "theoretical_loss": 3.804957322562675, + "tokens_seen": 705429504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006353460381143431, + "loss": 1.3885, + "theoretical_loss": 3.804921749913392, + "tokens_seen": 705495040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006353299899699098, + "loss": 1.4526, + "theoretical_loss": 3.804886181493582, + "tokens_seen": 705560576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006353139418254765, + "loss": 1.3952, + "theoretical_loss": 3.8048506173023506, + "tokens_seen": 705626112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352978936810432, + "loss": 1.3474, + "theoretical_loss": 3.8048150573388018, + "tokens_seen": 705691648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352818455366099, + "loss": 1.2652, + "theoretical_loss": 3.8047795016020407, + "tokens_seen": 705757184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352657973921766, + "loss": 1.4717, + "theoretical_loss": 3.804743950091172, + "tokens_seen": 705822720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352497492477433, + "loss": 1.4024, + "theoretical_loss": 3.8047084028053013, + "tokens_seen": 705888256 + }, + { + "epoch": 0.21, + "learning_rate": 0.00063523370110331, + "loss": 1.3059, + "theoretical_loss": 3.8046728597435346, + "tokens_seen": 705953792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352176529588767, + "loss": 1.4265, + "theoretical_loss": 3.8046373209049777, + "tokens_seen": 706019328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352016048144434, + "loss": 1.4273, + "theoretical_loss": 3.804601786288737, + "tokens_seen": 706084864 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351855566700101, + "loss": 1.4164, + "theoretical_loss": 3.804566255893919, + "tokens_seen": 706150400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351695085255768, + "loss": 1.409, + "theoretical_loss": 3.8045307297196302, + "tokens_seen": 706215936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351534603811435, + "loss": 1.5206, + "theoretical_loss": 3.804495207764978, + "tokens_seen": 706281472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351374122367102, + "loss": 1.4473, + "theoretical_loss": 3.804459690029069, + "tokens_seen": 706347008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351213640922769, + "loss": 1.5032, + "theoretical_loss": 3.804424176511012, + "tokens_seen": 706412544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351053159478436, + "loss": 1.3958, + "theoretical_loss": 3.8043886672099148, + "tokens_seen": 706478080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006350892678034103, + "loss": 1.516, + "theoretical_loss": 3.804353162124884, + "tokens_seen": 706543616 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635073219658977, + "loss": 1.2106, + "theoretical_loss": 3.8043176612550296, + "tokens_seen": 706609152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006350571715145437, + "loss": 1.3856, + "theoretical_loss": 3.80428216459946, + "tokens_seen": 706674688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006350411233701104, + "loss": 1.3575, + "theoretical_loss": 3.8042466721572836, + "tokens_seen": 706740224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006350250752256771, + "loss": 1.3736, + "theoretical_loss": 3.80421118392761, + "tokens_seen": 706805760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006350090270812438, + "loss": 1.3582, + "theoretical_loss": 3.804175699909549, + "tokens_seen": 706871296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006349929789368105, + "loss": 1.4533, + "theoretical_loss": 3.80414022010221, + "tokens_seen": 706936832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006349769307923772, + "loss": 1.4468, + "theoretical_loss": 3.804104744504703, + "tokens_seen": 707002368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006349608826479438, + "loss": 1.4524, + "theoretical_loss": 3.8040692731161383, + "tokens_seen": 707067904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006349448345035106, + "loss": 1.4523, + "theoretical_loss": 3.8040338059356267, + "tokens_seen": 707133440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006349287863590772, + "loss": 1.4276, + "theoretical_loss": 3.803998342962279, + "tokens_seen": 707198976 + }, + { + "epoch": 0.21, + "learning_rate": 0.000634912738214644, + "loss": 1.3684, + "theoretical_loss": 3.8039628841952067, + "tokens_seen": 707264512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348966900702106, + "loss": 1.4081, + "theoretical_loss": 3.8039274296335206, + "tokens_seen": 707330048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348806419257774, + "loss": 1.4177, + "theoretical_loss": 3.803891979276332, + "tokens_seen": 707395584 + }, + { + "epoch": 0.21, + "learning_rate": 0.000634864593781344, + "loss": 1.4522, + "theoretical_loss": 3.8038565331227545, + "tokens_seen": 707461120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348485456369108, + "loss": 1.3709, + "theoretical_loss": 3.8038210911718986, + "tokens_seen": 707526656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348324974924774, + "loss": 1.4075, + "theoretical_loss": 3.803785653422878, + "tokens_seen": 707592192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348164493480441, + "loss": 1.4133, + "theoretical_loss": 3.8037502198748046, + "tokens_seen": 707657728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348004012036108, + "loss": 1.3512, + "theoretical_loss": 3.8037147905267914, + "tokens_seen": 707723264 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 241367, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5710557699203491, + "objective/train/theoretical_loss": 3.8036793653779526, + "objective/train/tokens_used": 728248800, + "theoretical_loss": 3.8036793653779526, + "tokens_seen": 707788800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006347843530591775, + "loss": 1.4849, + "theoretical_loss": 3.8036793653779526, + "tokens_seen": 707788800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006347683049147442, + "loss": 1.4448, + "theoretical_loss": 3.8036439444274004, + "tokens_seen": 707854336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006347522567703109, + "loss": 1.5149, + "theoretical_loss": 3.80360852767425, + "tokens_seen": 707919872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006347362086258776, + "loss": 1.4236, + "theoretical_loss": 3.803573115117615, + "tokens_seen": 707985408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006347201604814443, + "loss": 1.4195, + "theoretical_loss": 3.803537706756609, + "tokens_seen": 708050944 + }, + { + "epoch": 0.21, + "learning_rate": 0.000634704112337011, + "loss": 1.5822, + "theoretical_loss": 3.8035023025903483, + "tokens_seen": 708116480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346880641925777, + "loss": 1.4559, + "theoretical_loss": 3.803466902617946, + "tokens_seen": 708182016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346720160481444, + "loss": 1.4631, + "theoretical_loss": 3.8034315068385185, + "tokens_seen": 708247552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346559679037111, + "loss": 1.388, + "theoretical_loss": 3.8033961152511804, + "tokens_seen": 708313088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346399197592778, + "loss": 1.4286, + "theoretical_loss": 3.8033607278550483, + "tokens_seen": 708378624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346238716148445, + "loss": 1.4811, + "theoretical_loss": 3.803325344649237, + "tokens_seen": 708444160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346078234704112, + "loss": 1.4126, + "theoretical_loss": 3.8032899656328647, + "tokens_seen": 708509696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006345917753259779, + "loss": 1.3167, + "theoretical_loss": 3.8032545908050457, + "tokens_seen": 708575232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006345757271815446, + "loss": 1.5087, + "theoretical_loss": 3.803219220164898, + "tokens_seen": 708640768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006345596790371113, + "loss": 1.4969, + "theoretical_loss": 3.8031838537115386, + "tokens_seen": 708706304 + }, + { + "epoch": 0.21, + "learning_rate": 0.000634543630892678, + "loss": 1.4194, + "theoretical_loss": 3.8031484914440847, + "tokens_seen": 708771840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006345275827482447, + "loss": 1.4666, + "theoretical_loss": 3.803113133361654, + "tokens_seen": 708837376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006345115346038115, + "loss": 1.3104, + "theoretical_loss": 3.8030777794633632, + "tokens_seen": 708902912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344954864593781, + "loss": 1.3697, + "theoretical_loss": 3.8030424297483325, + "tokens_seen": 708968448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344794383149449, + "loss": 1.4544, + "theoretical_loss": 3.803007084215678, + "tokens_seen": 709033984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344633901705115, + "loss": 1.4216, + "theoretical_loss": 3.8029717428645204, + "tokens_seen": 709099520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344473420260783, + "loss": 1.3437, + "theoretical_loss": 3.802936405693978, + "tokens_seen": 709165056 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344312938816449, + "loss": 1.5726, + "theoretical_loss": 3.802901072703169, + "tokens_seen": 709230592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344152457372117, + "loss": 1.5667, + "theoretical_loss": 3.802865743891214, + "tokens_seen": 709296128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006343991975927783, + "loss": 1.5112, + "theoretical_loss": 3.8028304192572326, + "tokens_seen": 709361664 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006343831494483451, + "loss": 1.4424, + "theoretical_loss": 3.802795098800344, + "tokens_seen": 709427200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006343671013039117, + "loss": 1.5096, + "theoretical_loss": 3.8027597825196695, + "tokens_seen": 709492736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006343510531594785, + "loss": 1.4287, + "theoretical_loss": 3.802724470414329, + "tokens_seen": 709558272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006343350050150451, + "loss": 1.3461, + "theoretical_loss": 3.8026891624834427, + "tokens_seen": 709623808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006343189568706119, + "loss": 1.4013, + "theoretical_loss": 3.802653858726133, + "tokens_seen": 709689344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006343029087261785, + "loss": 1.3793, + "theoretical_loss": 3.8026185591415205, + "tokens_seen": 709754880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342868605817453, + "loss": 1.4748, + "theoretical_loss": 3.802583263728727, + "tokens_seen": 709820416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342708124373119, + "loss": 1.4307, + "theoretical_loss": 3.802547972486874, + "tokens_seen": 709885952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342547642928787, + "loss": 1.4739, + "theoretical_loss": 3.802512685415084, + "tokens_seen": 709951488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342387161484453, + "loss": 1.4624, + "theoretical_loss": 3.802477402512479, + "tokens_seen": 710017024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342226680040121, + "loss": 1.4674, + "theoretical_loss": 3.8024421237781825, + "tokens_seen": 710082560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342066198595788, + "loss": 1.5095, + "theoretical_loss": 3.8024068492113163, + "tokens_seen": 710148096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006341905717151455, + "loss": 1.5026, + "theoretical_loss": 3.802371578811004, + "tokens_seen": 710213632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006341745235707122, + "loss": 1.3745, + "theoretical_loss": 3.8023363125763696, + "tokens_seen": 710279168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006341584754262789, + "loss": 1.3888, + "theoretical_loss": 3.8023010505065358, + "tokens_seen": 710344704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006341424272818456, + "loss": 1.4819, + "theoretical_loss": 3.802265792600627, + "tokens_seen": 710410240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006341263791374123, + "loss": 1.4422, + "theoretical_loss": 3.8022305388577684, + "tokens_seen": 710475776 + }, + { + "epoch": 0.22, + "learning_rate": 0.000634110330992979, + "loss": 1.4109, + "theoretical_loss": 3.8021952892770834, + "tokens_seen": 710541312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340942828485457, + "loss": 1.4645, + "theoretical_loss": 3.802160043857697, + "tokens_seen": 710606848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340782347041124, + "loss": 1.4037, + "theoretical_loss": 3.802124802598734, + "tokens_seen": 710672384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340621865596791, + "loss": 1.3728, + "theoretical_loss": 3.8020895654993203, + "tokens_seen": 710737920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340461384152458, + "loss": 1.4698, + "theoretical_loss": 3.8020543325585807, + "tokens_seen": 710803456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340300902708125, + "loss": 1.4225, + "theoretical_loss": 3.802019103775642, + "tokens_seen": 710868992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340140421263792, + "loss": 1.3455, + "theoretical_loss": 3.8019838791496303, + "tokens_seen": 710934528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339979939819459, + "loss": 1.3481, + "theoretical_loss": 3.8019486586796707, + "tokens_seen": 711000064 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 242639, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.499024510383606, + "objective/train/theoretical_loss": 3.8019134423648904, + "objective/train/tokens_used": 731525600, + "theoretical_loss": 3.8019134423648904, + "tokens_seen": 711065600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339819458375126, + "loss": 1.4308, + "theoretical_loss": 3.8019134423648904, + "tokens_seen": 711065600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339658976930793, + "loss": 1.3665, + "theoretical_loss": 3.8018782302044167, + "tokens_seen": 711131136 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633949849548646, + "loss": 1.3976, + "theoretical_loss": 3.801843022197377, + "tokens_seen": 711196672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339338014042127, + "loss": 1.4437, + "theoretical_loss": 3.801807818342898, + "tokens_seen": 711262208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339177532597794, + "loss": 1.4583, + "theoretical_loss": 3.8017726186401077, + "tokens_seen": 711327744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339017051153461, + "loss": 1.3954, + "theoretical_loss": 3.8017374230881336, + "tokens_seen": 711393280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338856569709128, + "loss": 1.4625, + "theoretical_loss": 3.8017022316861047, + "tokens_seen": 711458816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338696088264795, + "loss": 1.4932, + "theoretical_loss": 3.801667044433149, + "tokens_seen": 711524352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338535606820462, + "loss": 1.3748, + "theoretical_loss": 3.801631861328395, + "tokens_seen": 711589888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338375125376129, + "loss": 1.5972, + "theoretical_loss": 3.801596682370972, + "tokens_seen": 711655424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338214643931796, + "loss": 1.3926, + "theoretical_loss": 3.8015615075600095, + "tokens_seen": 711720960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338054162487463, + "loss": 1.4615, + "theoretical_loss": 3.8015263368946366, + "tokens_seen": 711786496 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633789368104313, + "loss": 1.6014, + "theoretical_loss": 3.8014911703739833, + "tokens_seen": 711852032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006337733199598797, + "loss": 1.4169, + "theoretical_loss": 3.8014560079971798, + "tokens_seen": 711917568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006337572718154464, + "loss": 1.3755, + "theoretical_loss": 3.801420849763356, + "tokens_seen": 711983104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006337412236710131, + "loss": 1.4058, + "theoretical_loss": 3.8013856956716428, + "tokens_seen": 712048640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006337251755265798, + "loss": 1.4429, + "theoretical_loss": 3.8013505457211707, + "tokens_seen": 712114176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006337091273821465, + "loss": 1.4631, + "theoretical_loss": 3.801315399911071, + "tokens_seen": 712179712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006336930792377132, + "loss": 1.3433, + "theoretical_loss": 3.8012802582404754, + "tokens_seen": 712245248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006336770310932799, + "loss": 1.5574, + "theoretical_loss": 3.801245120708515, + "tokens_seen": 712310784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006336609829488466, + "loss": 1.4838, + "theoretical_loss": 3.801209987314322, + "tokens_seen": 712376320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006336449348044133, + "loss": 1.4883, + "theoretical_loss": 3.8011748580570286, + "tokens_seen": 712441856 + }, + { + "epoch": 0.22, + "learning_rate": 0.00063362888665998, + "loss": 1.4876, + "theoretical_loss": 3.8011397329357672, + "tokens_seen": 712507392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006336128385155467, + "loss": 1.3607, + "theoretical_loss": 3.80110461194967, + "tokens_seen": 712572928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335967903711134, + "loss": 1.4238, + "theoretical_loss": 3.8010694950978703, + "tokens_seen": 712638464 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335807422266801, + "loss": 1.3884, + "theoretical_loss": 3.801034382379501, + "tokens_seen": 712704000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335646940822468, + "loss": 1.4922, + "theoretical_loss": 3.800999273793696, + "tokens_seen": 712769536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335486459378135, + "loss": 1.4384, + "theoretical_loss": 3.800964169339589, + "tokens_seen": 712835072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335325977933801, + "loss": 1.3729, + "theoretical_loss": 3.8009290690163136, + "tokens_seen": 712900608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335165496489469, + "loss": 1.4204, + "theoretical_loss": 3.8008939728230047, + "tokens_seen": 712966144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335005015045135, + "loss": 1.4367, + "theoretical_loss": 3.8008588807587955, + "tokens_seen": 713031680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006334844533600803, + "loss": 1.4261, + "theoretical_loss": 3.8008237928228223, + "tokens_seen": 713097216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006334684052156469, + "loss": 1.4637, + "theoretical_loss": 3.8007887090142196, + "tokens_seen": 713162752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006334523570712137, + "loss": 1.3785, + "theoretical_loss": 3.800753629332122, + "tokens_seen": 713228288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006334363089267803, + "loss": 1.3275, + "theoretical_loss": 3.800718553775666, + "tokens_seen": 713293824 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633420260782347, + "loss": 1.4474, + "theoretical_loss": 3.8006834823439863, + "tokens_seen": 713359360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006334042126379137, + "loss": 1.456, + "theoretical_loss": 3.80064841503622, + "tokens_seen": 713424896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006333881644934804, + "loss": 1.4341, + "theoretical_loss": 3.800613351851503, + "tokens_seen": 713490432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006333721163490471, + "loss": 1.397, + "theoretical_loss": 3.800578292788972, + "tokens_seen": 713555968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006333560682046138, + "loss": 1.4599, + "theoretical_loss": 3.800543237847764, + "tokens_seen": 713621504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006333400200601805, + "loss": 1.4813, + "theoretical_loss": 3.8005081870270154, + "tokens_seen": 713687040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006333239719157472, + "loss": 1.4561, + "theoretical_loss": 3.8004731403258645, + "tokens_seen": 713752576 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633307923771314, + "loss": 1.3359, + "theoretical_loss": 3.800438097743448, + "tokens_seen": 713818112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006332918756268806, + "loss": 1.4228, + "theoretical_loss": 3.8004030592789046, + "tokens_seen": 713883648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006332758274824473, + "loss": 1.5044, + "theoretical_loss": 3.8003680249313723, + "tokens_seen": 713949184 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633259779338014, + "loss": 1.3843, + "theoretical_loss": 3.800332994699989, + "tokens_seen": 714014720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006332437311935807, + "loss": 1.3726, + "theoretical_loss": 3.8002979685838936, + "tokens_seen": 714080256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006332276830491474, + "loss": 1.4088, + "theoretical_loss": 3.800262946582225, + "tokens_seen": 714145792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006332116349047141, + "loss": 1.4281, + "theoretical_loss": 3.800227928694123, + "tokens_seen": 714211328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006331955867602808, + "loss": 1.5304, + "theoretical_loss": 3.800192914918726, + "tokens_seen": 714276864 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 243571, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.517408847808838, + "objective/train/theoretical_loss": 3.800157905255175, + "objective/train/tokens_used": 734802400, + "theoretical_loss": 3.800157905255175, + "tokens_seen": 714342400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006331795386158476, + "loss": 1.5384, + "theoretical_loss": 3.800157905255175, + "tokens_seen": 714342400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006331634904714142, + "loss": 1.4538, + "theoretical_loss": 3.8001228997026084, + "tokens_seen": 714407936 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633147442326981, + "loss": 1.4125, + "theoretical_loss": 3.800087898260167, + "tokens_seen": 714473472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006331313941825476, + "loss": 1.5228, + "theoretical_loss": 3.800052900926992, + "tokens_seen": 714539008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006331153460381144, + "loss": 1.4806, + "theoretical_loss": 3.8000179077022236, + "tokens_seen": 714604544 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633099297893681, + "loss": 1.5372, + "theoretical_loss": 3.7999829185850027, + "tokens_seen": 714670080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330832497492478, + "loss": 1.3878, + "theoretical_loss": 3.7999479335744706, + "tokens_seen": 714735616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330672016048144, + "loss": 1.4084, + "theoretical_loss": 3.799912952669769, + "tokens_seen": 714801152 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330511534603812, + "loss": 1.3532, + "theoretical_loss": 3.7998779758700394, + "tokens_seen": 714866688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330351053159478, + "loss": 1.4402, + "theoretical_loss": 3.7998430031744235, + "tokens_seen": 714932224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330190571715146, + "loss": 1.4258, + "theoretical_loss": 3.7998080345820644, + "tokens_seen": 714997760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330030090270813, + "loss": 1.3957, + "theoretical_loss": 3.7997730700921046, + "tokens_seen": 715063296 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632986960882648, + "loss": 1.3841, + "theoretical_loss": 3.7997381097036858, + "tokens_seen": 715128832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006329709127382147, + "loss": 1.4965, + "theoretical_loss": 3.7997031534159524, + "tokens_seen": 715194368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006329548645937814, + "loss": 1.412, + "theoretical_loss": 3.799668201228047, + "tokens_seen": 715259904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006329388164493481, + "loss": 1.403, + "theoretical_loss": 3.7996332531391133, + "tokens_seen": 715325440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006329227683049148, + "loss": 1.4338, + "theoretical_loss": 3.7995983091482954, + "tokens_seen": 715390976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006329067201604815, + "loss": 1.384, + "theoretical_loss": 3.799563369254737, + "tokens_seen": 715456512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006328906720160482, + "loss": 1.454, + "theoretical_loss": 3.7995284334575823, + "tokens_seen": 715522048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006328746238716149, + "loss": 1.3987, + "theoretical_loss": 3.7994935017559763, + "tokens_seen": 715587584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006328585757271816, + "loss": 1.4896, + "theoretical_loss": 3.799458574149064, + "tokens_seen": 715653120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006328425275827483, + "loss": 1.4257, + "theoretical_loss": 3.7994236506359895, + "tokens_seen": 715718656 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632826479438315, + "loss": 1.4415, + "theoretical_loss": 3.7993887312158994, + "tokens_seen": 715784192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006328104312938817, + "loss": 1.3706, + "theoretical_loss": 3.799353815887939, + "tokens_seen": 715849728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327943831494484, + "loss": 1.4093, + "theoretical_loss": 3.799318904651254, + "tokens_seen": 715915264 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327783350050151, + "loss": 1.4663, + "theoretical_loss": 3.7992839975049906, + "tokens_seen": 715980800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327622868605818, + "loss": 1.581, + "theoretical_loss": 3.799249094448295, + "tokens_seen": 716046336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327462387161485, + "loss": 1.4882, + "theoretical_loss": 3.799214195480314, + "tokens_seen": 716111872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327301905717152, + "loss": 1.4267, + "theoretical_loss": 3.799179300600194, + "tokens_seen": 716177408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327141424272819, + "loss": 1.37, + "theoretical_loss": 3.7991444098070835, + "tokens_seen": 716242944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326980942828486, + "loss": 1.5027, + "theoretical_loss": 3.7991095231001286, + "tokens_seen": 716308480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326820461384153, + "loss": 1.5437, + "theoretical_loss": 3.7990746404784774, + "tokens_seen": 716374016 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632665997993982, + "loss": 1.39, + "theoretical_loss": 3.7990397619412786, + "tokens_seen": 716439552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326499498495487, + "loss": 1.4869, + "theoretical_loss": 3.799004887487679, + "tokens_seen": 716505088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326339017051154, + "loss": 1.4229, + "theoretical_loss": 3.798970017116828, + "tokens_seen": 716570624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326178535606821, + "loss": 1.323, + "theoretical_loss": 3.7989351508278744, + "tokens_seen": 716636160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326018054162488, + "loss": 1.2717, + "theoretical_loss": 3.7989002886199663, + "tokens_seen": 716701696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006325857572718155, + "loss": 1.3605, + "theoretical_loss": 3.7988654304922536, + "tokens_seen": 716767232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006325697091273822, + "loss": 1.4774, + "theoretical_loss": 3.798830576443886, + "tokens_seen": 716832768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006325536609829489, + "loss": 1.4702, + "theoretical_loss": 3.7987957264740118, + "tokens_seen": 716898304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006325376128385156, + "loss": 1.4558, + "theoretical_loss": 3.7987608805817827, + "tokens_seen": 716963840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006325215646940823, + "loss": 1.4032, + "theoretical_loss": 3.7987260387663486, + "tokens_seen": 717029376 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632505516549649, + "loss": 1.3756, + "theoretical_loss": 3.7986912010268585, + "tokens_seen": 717094912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324894684052157, + "loss": 1.4841, + "theoretical_loss": 3.798656367362465, + "tokens_seen": 717160448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324734202607824, + "loss": 1.3519, + "theoretical_loss": 3.798621537772318, + "tokens_seen": 717225984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324573721163491, + "loss": 1.5479, + "theoretical_loss": 3.7985867122555694, + "tokens_seen": 717291520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324413239719158, + "loss": 1.4504, + "theoretical_loss": 3.79855189081137, + "tokens_seen": 717357056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324252758274825, + "loss": 1.5485, + "theoretical_loss": 3.7985170734388722, + "tokens_seen": 717422592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324092276830492, + "loss": 1.3741, + "theoretical_loss": 3.7984822601372272, + "tokens_seen": 717488128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006323931795386159, + "loss": 1.3699, + "theoretical_loss": 3.7984474509055888, + "tokens_seen": 717553664 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 244811, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4988033771514893, + "objective/train/theoretical_loss": 3.7984126457431078, + "objective/train/tokens_used": 738079200, + "theoretical_loss": 3.7984126457431078, + "tokens_seen": 717619200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006323771313941826, + "loss": 1.4228, + "theoretical_loss": 3.7984126457431078, + "tokens_seen": 717619200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006323610832497493, + "loss": 1.5434, + "theoretical_loss": 3.798377844648938, + "tokens_seen": 717684736 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632345035105316, + "loss": 1.3938, + "theoretical_loss": 3.7983430476222315, + "tokens_seen": 717750272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006323289869608827, + "loss": 1.3932, + "theoretical_loss": 3.7983082546621425, + "tokens_seen": 717815808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006323129388164494, + "loss": 1.4452, + "theoretical_loss": 3.7982734657678243, + "tokens_seen": 717881344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322968906720161, + "loss": 1.4019, + "theoretical_loss": 3.798238680938431, + "tokens_seen": 717946880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322808425275828, + "loss": 1.4289, + "theoretical_loss": 3.798203900173116, + "tokens_seen": 718012416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322647943831495, + "loss": 1.3972, + "theoretical_loss": 3.798169123471034, + "tokens_seen": 718077952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322487462387162, + "loss": 1.3648, + "theoretical_loss": 3.798134350831339, + "tokens_seen": 718143488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322326980942829, + "loss": 1.4046, + "theoretical_loss": 3.7980995822531862, + "tokens_seen": 718209024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322166499498496, + "loss": 1.3455, + "theoretical_loss": 3.7980648177357312, + "tokens_seen": 718274560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322006018054163, + "loss": 1.3513, + "theoretical_loss": 3.7980300572781283, + "tokens_seen": 718340096 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632184553660983, + "loss": 1.4353, + "theoretical_loss": 3.797995300879534, + "tokens_seen": 718405632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006321685055165497, + "loss": 1.3864, + "theoretical_loss": 3.797960548539103, + "tokens_seen": 718471168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006321524573721164, + "loss": 1.4364, + "theoretical_loss": 3.7979258002559924, + "tokens_seen": 718536704 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632136409227683, + "loss": 1.3891, + "theoretical_loss": 3.797891056029358, + "tokens_seen": 718602240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006321203610832498, + "loss": 1.3073, + "theoretical_loss": 3.7978563158583567, + "tokens_seen": 718667776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006321043129388164, + "loss": 1.5076, + "theoretical_loss": 3.797821579742145, + "tokens_seen": 718733312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006320882647943832, + "loss": 1.3615, + "theoretical_loss": 3.7977868476798795, + "tokens_seen": 718798848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006320722166499498, + "loss": 1.5054, + "theoretical_loss": 3.7977521196707187, + "tokens_seen": 718864384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006320561685055166, + "loss": 1.435, + "theoretical_loss": 3.7977173957138195, + "tokens_seen": 718929920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006320401203610832, + "loss": 1.3536, + "theoretical_loss": 3.79768267580834, + "tokens_seen": 718995456 + }, + { + "epoch": 0.22, + "learning_rate": 0.00063202407221665, + "loss": 1.4225, + "theoretical_loss": 3.797647959953438, + "tokens_seen": 719060992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006320080240722166, + "loss": 1.3485, + "theoretical_loss": 3.7976132481482714, + "tokens_seen": 719126528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006319919759277833, + "loss": 1.4536, + "theoretical_loss": 3.7975785403919997, + "tokens_seen": 719192064 + }, + { + "epoch": 0.22, + "learning_rate": 0.00063197592778335, + "loss": 1.3513, + "theoretical_loss": 3.797543836683781, + "tokens_seen": 719257600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006319598796389167, + "loss": 1.399, + "theoretical_loss": 3.797509137022775, + "tokens_seen": 719323136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006319438314944834, + "loss": 1.3285, + "theoretical_loss": 3.797474441408141, + "tokens_seen": 719388672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006319277833500501, + "loss": 1.4618, + "theoretical_loss": 3.797439749839038, + "tokens_seen": 719454208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006319117352056168, + "loss": 1.3587, + "theoretical_loss": 3.7974050623146267, + "tokens_seen": 719519744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006318956870611835, + "loss": 1.409, + "theoretical_loss": 3.7973703788340663, + "tokens_seen": 719585280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006318796389167502, + "loss": 1.3907, + "theoretical_loss": 3.797335699396517, + "tokens_seen": 719650816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006318635907723169, + "loss": 1.4135, + "theoretical_loss": 3.7973010240011407, + "tokens_seen": 719716352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006318475426278836, + "loss": 1.3498, + "theoretical_loss": 3.7972663526470973, + "tokens_seen": 719781888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006318314944834504, + "loss": 1.3264, + "theoretical_loss": 3.797231685333548, + "tokens_seen": 719847424 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631815446339017, + "loss": 1.3904, + "theoretical_loss": 3.7971970220596543, + "tokens_seen": 719912960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317993981945838, + "loss": 1.5169, + "theoretical_loss": 3.7971623628245776, + "tokens_seen": 719978496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317833500501505, + "loss": 1.3087, + "theoretical_loss": 3.79712770762748, + "tokens_seen": 720044032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317673019057172, + "loss": 1.348, + "theoretical_loss": 3.797093056467523, + "tokens_seen": 720109568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317512537612839, + "loss": 1.5558, + "theoretical_loss": 3.7970584093438697, + "tokens_seen": 720175104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317352056168506, + "loss": 1.5001, + "theoretical_loss": 3.7970237662556823, + "tokens_seen": 720240640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317191574724173, + "loss": 1.5201, + "theoretical_loss": 3.7969891272021234, + "tokens_seen": 720306176 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631703109327984, + "loss": 1.4684, + "theoretical_loss": 3.796954492182357, + "tokens_seen": 720371712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316870611835507, + "loss": 1.2957, + "theoretical_loss": 3.796919861195546, + "tokens_seen": 720437248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316710130391174, + "loss": 1.4522, + "theoretical_loss": 3.796885234240853, + "tokens_seen": 720502784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316549648946841, + "loss": 1.3751, + "theoretical_loss": 3.796850611317443, + "tokens_seen": 720568320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316389167502508, + "loss": 1.3219, + "theoretical_loss": 3.79681599242448, + "tokens_seen": 720633856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316228686058175, + "loss": 1.3869, + "theoretical_loss": 3.7967813775611283, + "tokens_seen": 720699392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316068204613842, + "loss": 1.5114, + "theoretical_loss": 3.796746766726552, + "tokens_seen": 720764928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006315907723169509, + "loss": 1.4575, + "theoretical_loss": 3.7967121599199167, + "tokens_seen": 720830464 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 246110, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3479599952697754, + "objective/train/theoretical_loss": 3.7966775571403866, + "objective/train/tokens_used": 741356000, + "theoretical_loss": 3.7966775571403866, + "tokens_seen": 720896000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006315747241725176, + "loss": 1.4673, + "theoretical_loss": 3.7966775571403866, + "tokens_seen": 720896000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006315586760280843, + "loss": 1.4039, + "theoretical_loss": 3.7966429583871277, + "tokens_seen": 720961536 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631542627883651, + "loss": 1.4162, + "theoretical_loss": 3.7966083636593053, + "tokens_seen": 721027072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006315265797392177, + "loss": 1.4734, + "theoretical_loss": 3.7965737729560853, + "tokens_seen": 721092608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006315105315947844, + "loss": 1.4231, + "theoretical_loss": 3.7965391862766342, + "tokens_seen": 721158144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314944834503511, + "loss": 1.4973, + "theoretical_loss": 3.7965046036201175, + "tokens_seen": 721223680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314784353059178, + "loss": 1.4844, + "theoretical_loss": 3.7964700249857026, + "tokens_seen": 721289216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314623871614845, + "loss": 1.5109, + "theoretical_loss": 3.796435450372556, + "tokens_seen": 721354752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314463390170512, + "loss": 1.5156, + "theoretical_loss": 3.7964008797798443, + "tokens_seen": 721420288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314302908726179, + "loss": 1.3812, + "theoretical_loss": 3.796366313206735, + "tokens_seen": 721485824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314142427281846, + "loss": 1.4729, + "theoretical_loss": 3.7963317506523966, + "tokens_seen": 721551360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313981945837513, + "loss": 1.508, + "theoretical_loss": 3.796297192115996, + "tokens_seen": 721616896 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631382146439318, + "loss": 1.4971, + "theoretical_loss": 3.796262637596702, + "tokens_seen": 721682432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313660982948847, + "loss": 1.404, + "theoretical_loss": 3.7962280870936818, + "tokens_seen": 721747968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313500501504514, + "loss": 1.4409, + "theoretical_loss": 3.7961935406061054, + "tokens_seen": 721813504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313340020060181, + "loss": 1.5395, + "theoretical_loss": 3.79615899813314, + "tokens_seen": 721879040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313179538615848, + "loss": 1.3501, + "theoretical_loss": 3.7961244596739556, + "tokens_seen": 721944576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313019057171515, + "loss": 1.3767, + "theoretical_loss": 3.7960899252277214, + "tokens_seen": 722010112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006312858575727182, + "loss": 1.4148, + "theoretical_loss": 3.7960553947936075, + "tokens_seen": 722075648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006312698094282849, + "loss": 1.5147, + "theoretical_loss": 3.796020868370783, + "tokens_seen": 722141184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006312537612838516, + "loss": 1.6155, + "theoretical_loss": 3.795986345958418, + "tokens_seen": 722206720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006312377131394183, + "loss": 1.4997, + "theoretical_loss": 3.7959518275556827, + "tokens_seen": 722272256 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631221664994985, + "loss": 1.4589, + "theoretical_loss": 3.7959173131617483, + "tokens_seen": 722337792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006312056168505517, + "loss": 1.546, + "theoretical_loss": 3.795882802775785, + "tokens_seen": 722403328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311895687061184, + "loss": 1.3899, + "theoretical_loss": 3.7958482963969638, + "tokens_seen": 722468864 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311735205616851, + "loss": 1.4844, + "theoretical_loss": 3.7958137940244567, + "tokens_seen": 722534400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311574724172518, + "loss": 1.4567, + "theoretical_loss": 3.795779295657434, + "tokens_seen": 722599936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311414242728185, + "loss": 1.4734, + "theoretical_loss": 3.7957448012950685, + "tokens_seen": 722665472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311253761283852, + "loss": 1.4436, + "theoretical_loss": 3.795710310936532, + "tokens_seen": 722731008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311093279839519, + "loss": 1.4782, + "theoretical_loss": 3.7956758245809965, + "tokens_seen": 722796544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006310932798395186, + "loss": 1.4097, + "theoretical_loss": 3.795641342227635, + "tokens_seen": 722862080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006310772316950853, + "loss": 1.3761, + "theoretical_loss": 3.7956068638756197, + "tokens_seen": 722927616 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631061183550652, + "loss": 1.3399, + "theoretical_loss": 3.795572389524124, + "tokens_seen": 722993152 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006310451354062187, + "loss": 1.3759, + "theoretical_loss": 3.795537919172321, + "tokens_seen": 723058688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006310290872617854, + "loss": 1.4874, + "theoretical_loss": 3.795503452819384, + "tokens_seen": 723124224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006310130391173521, + "loss": 1.41, + "theoretical_loss": 3.795468990464487, + "tokens_seen": 723189760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309969909729188, + "loss": 1.3574, + "theoretical_loss": 3.795434532106804, + "tokens_seen": 723255296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309809428284855, + "loss": 1.3829, + "theoretical_loss": 3.79540007774551, + "tokens_seen": 723320832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309648946840522, + "loss": 1.5397, + "theoretical_loss": 3.795365627379778, + "tokens_seen": 723386368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309488465396189, + "loss": 1.4052, + "theoretical_loss": 3.795331181008783, + "tokens_seen": 723451904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309327983951856, + "loss": 1.4492, + "theoretical_loss": 3.795296738631701, + "tokens_seen": 723517440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309167502507523, + "loss": 1.565, + "theoretical_loss": 3.795262300247707, + "tokens_seen": 723582976 + }, + { + "epoch": 0.22, + "learning_rate": 0.000630900702106319, + "loss": 1.408, + "theoretical_loss": 3.795227865855975, + "tokens_seen": 723648512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308846539618857, + "loss": 1.341, + "theoretical_loss": 3.7951934354556824, + "tokens_seen": 723714048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308686058174524, + "loss": 1.5522, + "theoretical_loss": 3.7951590090460043, + "tokens_seen": 723779584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308525576730191, + "loss": 1.508, + "theoretical_loss": 3.795124586626117, + "tokens_seen": 723845120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308365095285858, + "loss": 1.4448, + "theoretical_loss": 3.7950901681951974, + "tokens_seen": 723910656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308204613841525, + "loss": 1.4212, + "theoretical_loss": 3.795055753752422, + "tokens_seen": 723976192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308044132397192, + "loss": 1.4526, + "theoretical_loss": 3.795021343296967, + "tokens_seen": 724041728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307883650952859, + "loss": 1.3965, + "theoretical_loss": 3.7949869368280105, + "tokens_seen": 724107264 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 247050, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4974820613861084, + "objective/train/theoretical_loss": 3.7949525343447297, + "objective/train/tokens_used": 744632800, + "theoretical_loss": 3.7949525343447297, + "tokens_seen": 724172800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307723169508526, + "loss": 1.5663, + "theoretical_loss": 3.7949525343447297, + "tokens_seen": 724172800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307562688064193, + "loss": 1.4248, + "theoretical_loss": 3.794918135846302, + "tokens_seen": 724238336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307402206619859, + "loss": 1.4183, + "theoretical_loss": 3.7948837413319056, + "tokens_seen": 724303872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307241725175527, + "loss": 1.4792, + "theoretical_loss": 3.7948493508007184, + "tokens_seen": 724369408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307081243731193, + "loss": 1.4819, + "theoretical_loss": 3.794814964251919, + "tokens_seen": 724434944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306920762286861, + "loss": 1.4506, + "theoretical_loss": 3.7947805816846856, + "tokens_seen": 724500480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306760280842527, + "loss": 1.3404, + "theoretical_loss": 3.7947462030981978, + "tokens_seen": 724566016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306599799398195, + "loss": 1.3182, + "theoretical_loss": 3.794711828491634, + "tokens_seen": 724631552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306439317953861, + "loss": 1.4068, + "theoretical_loss": 3.794677457864174, + "tokens_seen": 724697088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306278836509529, + "loss": 1.297, + "theoretical_loss": 3.7946430912149975, + "tokens_seen": 724762624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306118355065195, + "loss": 1.4566, + "theoretical_loss": 3.794608728543284, + "tokens_seen": 724828160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305957873620863, + "loss": 1.5626, + "theoretical_loss": 3.794574369848214, + "tokens_seen": 724893696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305797392176529, + "loss": 1.3466, + "theoretical_loss": 3.794540015128968, + "tokens_seen": 724959232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305636910732197, + "loss": 1.3054, + "theoretical_loss": 3.7945056643847255, + "tokens_seen": 725024768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305476429287863, + "loss": 1.5714, + "theoretical_loss": 3.7944713176146685, + "tokens_seen": 725090304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305315947843531, + "loss": 1.3652, + "theoretical_loss": 3.794436974817977, + "tokens_seen": 725155840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305155466399197, + "loss": 1.4185, + "theoretical_loss": 3.7944026359938334, + "tokens_seen": 725221376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304994984954865, + "loss": 1.4229, + "theoretical_loss": 3.794368301141419, + "tokens_seen": 725286912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304834503510531, + "loss": 1.356, + "theoretical_loss": 3.794333970259915, + "tokens_seen": 725352448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304674022066199, + "loss": 1.3642, + "theoretical_loss": 3.794299643348504, + "tokens_seen": 725417984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304513540621866, + "loss": 1.4366, + "theoretical_loss": 3.7942653204063683, + "tokens_seen": 725483520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304353059177533, + "loss": 1.5196, + "theoretical_loss": 3.79423100143269, + "tokens_seen": 725549056 + }, + { + "epoch": 0.22, + "learning_rate": 0.00063041925777332, + "loss": 1.4281, + "theoretical_loss": 3.794196686426652, + "tokens_seen": 725614592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304032096288867, + "loss": 1.4114, + "theoretical_loss": 3.7941623753874376, + "tokens_seen": 725680128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303871614844534, + "loss": 1.3399, + "theoretical_loss": 3.79412806831423, + "tokens_seen": 725745664 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303711133400201, + "loss": 1.4427, + "theoretical_loss": 3.7940937652062123, + "tokens_seen": 725811200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303550651955868, + "loss": 1.5487, + "theoretical_loss": 3.794059466062569, + "tokens_seen": 725876736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303390170511535, + "loss": 1.4869, + "theoretical_loss": 3.7940251708824833, + "tokens_seen": 725942272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303229689067202, + "loss": 1.3933, + "theoretical_loss": 3.79399087966514, + "tokens_seen": 726007808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303069207622869, + "loss": 1.474, + "theoretical_loss": 3.7939565924097227, + "tokens_seen": 726073344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006302908726178536, + "loss": 1.4447, + "theoretical_loss": 3.793922309115417, + "tokens_seen": 726138880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006302748244734203, + "loss": 1.3871, + "theoretical_loss": 3.7938880297814075, + "tokens_seen": 726204416 + }, + { + "epoch": 0.22, + "learning_rate": 0.000630258776328987, + "loss": 1.3355, + "theoretical_loss": 3.793853754406879, + "tokens_seen": 726269952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006302427281845537, + "loss": 1.378, + "theoretical_loss": 3.793819482991018, + "tokens_seen": 726335488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006302266800401204, + "loss": 1.4091, + "theoretical_loss": 3.793785215533009, + "tokens_seen": 726401024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006302106318956871, + "loss": 1.3496, + "theoretical_loss": 3.7937509520320387, + "tokens_seen": 726466560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301945837512538, + "loss": 1.3943, + "theoretical_loss": 3.793716692487293, + "tokens_seen": 726532096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301785356068205, + "loss": 1.3716, + "theoretical_loss": 3.7936824368979583, + "tokens_seen": 726597632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301624874623872, + "loss": 1.3933, + "theoretical_loss": 3.793648185263221, + "tokens_seen": 726663168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301464393179539, + "loss": 1.4309, + "theoretical_loss": 3.7936139375822684, + "tokens_seen": 726728704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301303911735206, + "loss": 1.437, + "theoretical_loss": 3.793579693854287, + "tokens_seen": 726794240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301143430290873, + "loss": 1.3149, + "theoretical_loss": 3.7935454540784646, + "tokens_seen": 726859776 + }, + { + "epoch": 0.22, + "learning_rate": 0.000630098294884654, + "loss": 1.3953, + "theoretical_loss": 3.7935112182539887, + "tokens_seen": 726925312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300822467402207, + "loss": 1.4703, + "theoretical_loss": 3.7934769863800475, + "tokens_seen": 726990848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300661985957874, + "loss": 1.3592, + "theoretical_loss": 3.7934427584558286, + "tokens_seen": 727056384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300501504513541, + "loss": 1.4148, + "theoretical_loss": 3.7934085344805206, + "tokens_seen": 727121920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300341023069208, + "loss": 1.4887, + "theoretical_loss": 3.7933743144533114, + "tokens_seen": 727187456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300180541624875, + "loss": 1.3985, + "theoretical_loss": 3.7933400983733905, + "tokens_seen": 727252992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300020060180542, + "loss": 1.4509, + "theoretical_loss": 3.7933058862399465, + "tokens_seen": 727318528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006299859578736209, + "loss": 1.3862, + "theoretical_loss": 3.7932716780521694, + "tokens_seen": 727384064 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 247886, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3108396530151367, + "objective/train/theoretical_loss": 3.793237473809248, + "objective/train/tokens_used": 747909600, + "theoretical_loss": 3.793237473809248, + "tokens_seen": 727449600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006299699097291876, + "loss": 1.3084, + "theoretical_loss": 3.793237473809248, + "tokens_seen": 727449600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006299538615847543, + "loss": 1.3268, + "theoretical_loss": 3.7932032735103722, + "tokens_seen": 727515136 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629937813440321, + "loss": 1.3663, + "theoretical_loss": 3.793169077154732, + "tokens_seen": 727580672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006299217652958877, + "loss": 1.4104, + "theoretical_loss": 3.7931348847415176, + "tokens_seen": 727646208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006299057171514544, + "loss": 1.3822, + "theoretical_loss": 3.79310069626992, + "tokens_seen": 727711744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298896690070211, + "loss": 1.3625, + "theoretical_loss": 3.793066511739129, + "tokens_seen": 727777280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298736208625878, + "loss": 1.5019, + "theoretical_loss": 3.793032331148336, + "tokens_seen": 727842816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298575727181545, + "loss": 1.5795, + "theoretical_loss": 3.7929981544967326, + "tokens_seen": 727908352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298415245737212, + "loss": 1.3908, + "theoretical_loss": 3.7929639817835095, + "tokens_seen": 727973888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298254764292879, + "loss": 1.3007, + "theoretical_loss": 3.7929298130078584, + "tokens_seen": 728039424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298094282848546, + "loss": 1.4696, + "theoretical_loss": 3.792895648168972, + "tokens_seen": 728104960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006297933801404213, + "loss": 1.3193, + "theoretical_loss": 3.792861487266042, + "tokens_seen": 728170496 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629777331995988, + "loss": 1.3398, + "theoretical_loss": 3.7928273302982602, + "tokens_seen": 728236032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006297612838515547, + "loss": 1.4541, + "theoretical_loss": 3.7927931772648202, + "tokens_seen": 728301568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006297452357071214, + "loss": 1.4588, + "theoretical_loss": 3.792759028164914, + "tokens_seen": 728367104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006297291875626881, + "loss": 1.3585, + "theoretical_loss": 3.792724882997735, + "tokens_seen": 728432640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006297131394182548, + "loss": 1.3819, + "theoretical_loss": 3.792690741762477, + "tokens_seen": 728498176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296970912738215, + "loss": 1.2567, + "theoretical_loss": 3.792656604458333, + "tokens_seen": 728563712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296810431293882, + "loss": 1.4528, + "theoretical_loss": 3.7926224710844973, + "tokens_seen": 728629248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296649949849549, + "loss": 1.3446, + "theoretical_loss": 3.7925883416401636, + "tokens_seen": 728694784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296489468405216, + "loss": 1.3966, + "theoretical_loss": 3.792554216124526, + "tokens_seen": 728760320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296328986960883, + "loss": 1.4292, + "theoretical_loss": 3.792520094536779, + "tokens_seen": 728825856 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629616850551655, + "loss": 1.4539, + "theoretical_loss": 3.792485976876118, + "tokens_seen": 728891392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296008024072217, + "loss": 1.4471, + "theoretical_loss": 3.7924518631417374, + "tokens_seen": 728956928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295847542627884, + "loss": 1.5046, + "theoretical_loss": 3.792417753332833, + "tokens_seen": 729022464 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295687061183551, + "loss": 1.4383, + "theoretical_loss": 3.7923836474485997, + "tokens_seen": 729088000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295526579739218, + "loss": 1.3187, + "theoretical_loss": 3.792349545488233, + "tokens_seen": 729153536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295366098294885, + "loss": 1.3993, + "theoretical_loss": 3.79231544745093, + "tokens_seen": 729219072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295205616850552, + "loss": 1.3409, + "theoretical_loss": 3.7922813533358855, + "tokens_seen": 729284608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295045135406219, + "loss": 1.3396, + "theoretical_loss": 3.792247263142297, + "tokens_seen": 729350144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006294884653961886, + "loss": 1.4328, + "theoretical_loss": 3.7922131768693603, + "tokens_seen": 729415680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006294724172517553, + "loss": 1.3611, + "theoretical_loss": 3.792179094516273, + "tokens_seen": 729481216 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629456369107322, + "loss": 1.4281, + "theoretical_loss": 3.792145016082232, + "tokens_seen": 729546752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006294403209628887, + "loss": 1.4321, + "theoretical_loss": 3.7921109415664347, + "tokens_seen": 729612288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006294242728184554, + "loss": 1.3427, + "theoretical_loss": 3.792076870968078, + "tokens_seen": 729677824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006294082246740221, + "loss": 1.3667, + "theoretical_loss": 3.792042804286361, + "tokens_seen": 729743360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293921765295888, + "loss": 1.3824, + "theoretical_loss": 3.792008741520481, + "tokens_seen": 729808896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293761283851555, + "loss": 1.3687, + "theoretical_loss": 3.791974682669636, + "tokens_seen": 729874432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293600802407222, + "loss": 1.306, + "theoretical_loss": 3.791940627733026, + "tokens_seen": 729939968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293440320962888, + "loss": 1.4359, + "theoretical_loss": 3.791906576709848, + "tokens_seen": 730005504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293279839518556, + "loss": 1.412, + "theoretical_loss": 3.791872529599302, + "tokens_seen": 730071040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293119358074222, + "loss": 1.3936, + "theoretical_loss": 3.791838486400587, + "tokens_seen": 730136576 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629295887662989, + "loss": 1.3971, + "theoretical_loss": 3.7918044471129027, + "tokens_seen": 730202112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006292798395185556, + "loss": 1.4084, + "theoretical_loss": 3.7917704117354485, + "tokens_seen": 730267648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006292637913741224, + "loss": 1.4487, + "theoretical_loss": 3.7917363802674244, + "tokens_seen": 730333184 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629247743229689, + "loss": 1.4476, + "theoretical_loss": 3.791702352708031, + "tokens_seen": 730398720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006292316950852558, + "loss": 1.4352, + "theoretical_loss": 3.7916683290564683, + "tokens_seen": 730464256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006292156469408224, + "loss": 1.4082, + "theoretical_loss": 3.7916343093119376, + "tokens_seen": 730529792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291995987963892, + "loss": 1.3462, + "theoretical_loss": 3.791600293473639, + "tokens_seen": 730595328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291835506519558, + "loss": 1.4365, + "theoretical_loss": 3.791566281540774, + "tokens_seen": 730660864 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 249085, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4543607234954834, + "objective/train/theoretical_loss": 3.791532273512544, + "objective/train/tokens_used": 751186400, + "theoretical_loss": 3.791532273512544, + "tokens_seen": 730726400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291675025075226, + "loss": 1.4237, + "theoretical_loss": 3.791532273512544, + "tokens_seen": 730726400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291514543630892, + "loss": 1.4221, + "theoretical_loss": 3.7914982693881503, + "tokens_seen": 730791936 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629135406218656, + "loss": 1.4758, + "theoretical_loss": 3.791464269166795, + "tokens_seen": 730857472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291193580742226, + "loss": 1.4386, + "theoretical_loss": 3.7914302728476805, + "tokens_seen": 730923008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291033099297894, + "loss": 1.4968, + "theoretical_loss": 3.7913962804300088, + "tokens_seen": 730988544 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629087261785356, + "loss": 1.4565, + "theoretical_loss": 3.791362291912982, + "tokens_seen": 731054080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006290712136409228, + "loss": 1.2575, + "theoretical_loss": 3.791328307295804, + "tokens_seen": 731119616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006290551654964895, + "loss": 1.3963, + "theoretical_loss": 3.791294326577677, + "tokens_seen": 731185152 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006290391173520562, + "loss": 1.4078, + "theoretical_loss": 3.7912603497578043, + "tokens_seen": 731250688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006290230692076229, + "loss": 1.4262, + "theoretical_loss": 3.7912263768353895, + "tokens_seen": 731316224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006290070210631896, + "loss": 1.4075, + "theoretical_loss": 3.7911924078096364, + "tokens_seen": 731381760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006289909729187563, + "loss": 1.4069, + "theoretical_loss": 3.791158442679748, + "tokens_seen": 731447296 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628974924774323, + "loss": 1.3423, + "theoretical_loss": 3.7911244814449305, + "tokens_seen": 731512832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006289588766298897, + "loss": 1.5558, + "theoretical_loss": 3.7910905241043866, + "tokens_seen": 731578368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006289428284854564, + "loss": 1.3795, + "theoretical_loss": 3.7910565706573216, + "tokens_seen": 731643904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006289267803410231, + "loss": 1.4941, + "theoretical_loss": 3.7910226211029405, + "tokens_seen": 731709440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006289107321965898, + "loss": 1.4243, + "theoretical_loss": 3.790988675440448, + "tokens_seen": 731774976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006288946840521565, + "loss": 1.4918, + "theoretical_loss": 3.79095473366905, + "tokens_seen": 731840512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006288786359077232, + "loss": 1.4058, + "theoretical_loss": 3.790920795787952, + "tokens_seen": 731906048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006288625877632899, + "loss": 1.3867, + "theoretical_loss": 3.7908868617963587, + "tokens_seen": 731971584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006288465396188566, + "loss": 1.4297, + "theoretical_loss": 3.7908529316934776, + "tokens_seen": 732037120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006288304914744233, + "loss": 1.3749, + "theoretical_loss": 3.7908190054785145, + "tokens_seen": 732102656 + }, + { + "epoch": 0.22, + "learning_rate": 0.00062881444332999, + "loss": 1.2819, + "theoretical_loss": 3.7907850831506753, + "tokens_seen": 732168192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287983951855567, + "loss": 1.485, + "theoretical_loss": 3.7907511647091683, + "tokens_seen": 732233728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287823470411234, + "loss": 1.5243, + "theoretical_loss": 3.7907172501531985, + "tokens_seen": 732299264 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287662988966901, + "loss": 1.4483, + "theoretical_loss": 3.7906833394819746, + "tokens_seen": 732364800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287502507522568, + "loss": 1.3798, + "theoretical_loss": 3.790649432694704, + "tokens_seen": 732430336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287342026078235, + "loss": 1.3989, + "theoretical_loss": 3.7906155297905935, + "tokens_seen": 732495872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287181544633902, + "loss": 1.3249, + "theoretical_loss": 3.7905816307688514, + "tokens_seen": 732561408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287021063189569, + "loss": 1.4464, + "theoretical_loss": 3.790547735628686, + "tokens_seen": 732626944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006286860581745236, + "loss": 1.4306, + "theoretical_loss": 3.790513844369306, + "tokens_seen": 732692480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006286700100300903, + "loss": 1.328, + "theoretical_loss": 3.7904799569899192, + "tokens_seen": 732758016 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628653961885657, + "loss": 1.5002, + "theoretical_loss": 3.790446073489735, + "tokens_seen": 732823552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006286379137412237, + "loss": 1.5499, + "theoretical_loss": 3.7904121938679625, + "tokens_seen": 732889088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006286218655967904, + "loss": 1.4239, + "theoretical_loss": 3.7903783181238104, + "tokens_seen": 732954624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006286058174523571, + "loss": 1.3344, + "theoretical_loss": 3.7903444462564893, + "tokens_seen": 733020160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285897693079238, + "loss": 1.4331, + "theoretical_loss": 3.790310578265208, + "tokens_seen": 733085696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285737211634905, + "loss": 1.4736, + "theoretical_loss": 3.7902767141491776, + "tokens_seen": 733151232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285576730190572, + "loss": 1.472, + "theoretical_loss": 3.790242853907607, + "tokens_seen": 733216768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285416248746239, + "loss": 1.4226, + "theoretical_loss": 3.790208997539707, + "tokens_seen": 733282304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285255767301906, + "loss": 1.446, + "theoretical_loss": 3.7901751450446897, + "tokens_seen": 733347840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285095285857573, + "loss": 1.5121, + "theoretical_loss": 3.790141296421764, + "tokens_seen": 733413376 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628493480441324, + "loss": 1.4117, + "theoretical_loss": 3.790107451670142, + "tokens_seen": 733478912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006284774322968907, + "loss": 1.4495, + "theoretical_loss": 3.790073610789036, + "tokens_seen": 733544448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006284613841524574, + "loss": 1.4667, + "theoretical_loss": 3.7900397737776554, + "tokens_seen": 733609984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006284453360080241, + "loss": 1.4249, + "theoretical_loss": 3.790005940635214, + "tokens_seen": 733675520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006284292878635908, + "loss": 1.4262, + "theoretical_loss": 3.7899721113609237, + "tokens_seen": 733741056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006284132397191575, + "loss": 1.3886, + "theoretical_loss": 3.789938285953996, + "tokens_seen": 733806592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283971915747242, + "loss": 1.3485, + "theoretical_loss": 3.789904464413644, + "tokens_seen": 733872128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283811434302909, + "loss": 1.3421, + "theoretical_loss": 3.7898706467390806, + "tokens_seen": 733937664 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 250229, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4468345642089844, + "objective/train/theoretical_loss": 3.7898368329295176, + "objective/train/tokens_used": 754463200, + "theoretical_loss": 3.7898368329295176, + "tokens_seen": 734003200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283650952858576, + "loss": 1.395, + "theoretical_loss": 3.7898368329295176, + "tokens_seen": 734003200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283490471414243, + "loss": 1.3423, + "theoretical_loss": 3.7898030229841697, + "tokens_seen": 734068736 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628332998996991, + "loss": 1.4906, + "theoretical_loss": 3.78976921690225, + "tokens_seen": 734134272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283169508525577, + "loss": 1.3971, + "theoretical_loss": 3.789735414682972, + "tokens_seen": 734199808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283009027081244, + "loss": 1.4308, + "theoretical_loss": 3.7897016163255497, + "tokens_seen": 734265344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006282848545636911, + "loss": 1.3764, + "theoretical_loss": 3.789667821829197, + "tokens_seen": 734330880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006282688064192578, + "loss": 1.4429, + "theoretical_loss": 3.7896340311931285, + "tokens_seen": 734396416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006282527582748245, + "loss": 1.4776, + "theoretical_loss": 3.789600244416559, + "tokens_seen": 734461952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006282367101303912, + "loss": 1.4843, + "theoretical_loss": 3.789566461498703, + "tokens_seen": 734527488 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628220661985958, + "loss": 1.4393, + "theoretical_loss": 3.789532682438776, + "tokens_seen": 734593024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006282046138415246, + "loss": 1.46, + "theoretical_loss": 3.789498907235993, + "tokens_seen": 734658560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006281885656970914, + "loss": 1.4159, + "theoretical_loss": 3.7894651358895697, + "tokens_seen": 734724096 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628172517552658, + "loss": 1.5819, + "theoretical_loss": 3.789431368398722, + "tokens_seen": 734789632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006281564694082248, + "loss": 1.4192, + "theoretical_loss": 3.789397604762666, + "tokens_seen": 734855168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006281404212637914, + "loss": 1.4651, + "theoretical_loss": 3.789363844980617, + "tokens_seen": 734920704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006281243731193582, + "loss": 1.2948, + "theoretical_loss": 3.7893300890517922, + "tokens_seen": 734986240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006281083249749248, + "loss": 1.3556, + "theoretical_loss": 3.789296336975408, + "tokens_seen": 735051776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006280922768304916, + "loss": 1.3993, + "theoretical_loss": 3.789262588750682, + "tokens_seen": 735117312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006280762286860582, + "loss": 1.383, + "theoretical_loss": 3.7892288443768303, + "tokens_seen": 735182848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006280601805416249, + "loss": 1.3438, + "theoretical_loss": 3.7891951038530713, + "tokens_seen": 735248384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006280441323971916, + "loss": 1.3622, + "theoretical_loss": 3.7891613671786217, + "tokens_seen": 735313920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006280280842527583, + "loss": 1.4872, + "theoretical_loss": 3.7891276343527, + "tokens_seen": 735379456 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628012036108325, + "loss": 1.3421, + "theoretical_loss": 3.789093905374524, + "tokens_seen": 735444992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279959879638917, + "loss": 1.4801, + "theoretical_loss": 3.789060180243312, + "tokens_seen": 735510528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279799398194584, + "loss": 1.3814, + "theoretical_loss": 3.7890264589582827, + "tokens_seen": 735576064 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279638916750251, + "loss": 1.3895, + "theoretical_loss": 3.7889927415186544, + "tokens_seen": 735641600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279478435305917, + "loss": 1.3125, + "theoretical_loss": 3.7889590279236467, + "tokens_seen": 735707136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279317953861585, + "loss": 1.3544, + "theoretical_loss": 3.788925318172478, + "tokens_seen": 735772672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279157472417251, + "loss": 1.4446, + "theoretical_loss": 3.7888916122643685, + "tokens_seen": 735838208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278996990972919, + "loss": 1.3253, + "theoretical_loss": 3.788857910198537, + "tokens_seen": 735903744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278836509528585, + "loss": 1.4505, + "theoretical_loss": 3.7888242119742044, + "tokens_seen": 735969280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278676028084253, + "loss": 1.4354, + "theoretical_loss": 3.7887905175905905, + "tokens_seen": 736034816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278515546639919, + "loss": 1.4241, + "theoretical_loss": 3.7887568270469147, + "tokens_seen": 736100352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278355065195587, + "loss": 1.3215, + "theoretical_loss": 3.788723140342399, + "tokens_seen": 736165888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278194583751253, + "loss": 1.4426, + "theoretical_loss": 3.7886894574762637, + "tokens_seen": 736231424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278034102306921, + "loss": 1.4753, + "theoretical_loss": 3.7886557784477293, + "tokens_seen": 736296960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277873620862587, + "loss": 1.3266, + "theoretical_loss": 3.7886221032560172, + "tokens_seen": 736362496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277713139418255, + "loss": 1.3998, + "theoretical_loss": 3.7885884319003496, + "tokens_seen": 736428032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277552657973921, + "loss": 1.3853, + "theoretical_loss": 3.7885547643799473, + "tokens_seen": 736493568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277392176529589, + "loss": 1.4296, + "theoretical_loss": 3.788521100694033, + "tokens_seen": 736559104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277231695085255, + "loss": 1.3047, + "theoretical_loss": 3.7884874408418288, + "tokens_seen": 736624640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277071213640923, + "loss": 1.3722, + "theoretical_loss": 3.788453784822556, + "tokens_seen": 736690176 + }, + { + "epoch": 0.22, + "learning_rate": 0.000627691073219659, + "loss": 1.456, + "theoretical_loss": 3.788420132635438, + "tokens_seen": 736755712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006276750250752257, + "loss": 1.408, + "theoretical_loss": 3.788386484279698, + "tokens_seen": 736821248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006276589769307924, + "loss": 1.294, + "theoretical_loss": 3.788352839754559, + "tokens_seen": 736886784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006276429287863591, + "loss": 1.4556, + "theoretical_loss": 3.7883191990592433, + "tokens_seen": 736952320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006276268806419258, + "loss": 1.3902, + "theoretical_loss": 3.788285562192976, + "tokens_seen": 737017856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006276108324974925, + "loss": 1.4227, + "theoretical_loss": 3.7882519291549794, + "tokens_seen": 737083392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006275947843530592, + "loss": 1.4074, + "theoretical_loss": 3.788218299944478, + "tokens_seen": 737148928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006275787362086259, + "loss": 1.4197, + "theoretical_loss": 3.7881846745606964, + "tokens_seen": 737214464 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 251121, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3285048007965088, + "objective/train/theoretical_loss": 3.788151053002858, + "objective/train/tokens_used": 757740000, + "theoretical_loss": 3.788151053002858, + "tokens_seen": 737280000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006275626880641926, + "loss": 1.3462, + "theoretical_loss": 3.788151053002858, + "tokens_seen": 737280000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006275466399197593, + "loss": 1.3299, + "theoretical_loss": 3.788117435270189, + "tokens_seen": 737345536 + }, + { + "epoch": 0.22, + "learning_rate": 0.000627530591775326, + "loss": 1.4813, + "theoretical_loss": 3.788083821361913, + "tokens_seen": 737411072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006275145436308927, + "loss": 1.4779, + "theoretical_loss": 3.7880502112772554, + "tokens_seen": 737476608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274984954864594, + "loss": 1.4615, + "theoretical_loss": 3.788016605015442, + "tokens_seen": 737542144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274824473420261, + "loss": 1.3011, + "theoretical_loss": 3.7879830025756975, + "tokens_seen": 737607680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274663991975928, + "loss": 1.4051, + "theoretical_loss": 3.7879494039572483, + "tokens_seen": 737673216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274503510531595, + "loss": 1.5, + "theoretical_loss": 3.787915809159321, + "tokens_seen": 737738752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274343029087262, + "loss": 1.3121, + "theoretical_loss": 3.7878822181811405, + "tokens_seen": 737804288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274182547642929, + "loss": 1.3735, + "theoretical_loss": 3.7878486310219337, + "tokens_seen": 737869824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274022066198596, + "loss": 1.472, + "theoretical_loss": 3.787815047680928, + "tokens_seen": 737935360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006273861584754263, + "loss": 1.3119, + "theoretical_loss": 3.787781468157349, + "tokens_seen": 738000896 + }, + { + "epoch": 0.22, + "learning_rate": 0.000627370110330993, + "loss": 1.3764, + "theoretical_loss": 3.787747892450425, + "tokens_seen": 738066432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006273540621865597, + "loss": 1.4331, + "theoretical_loss": 3.7877143205593837, + "tokens_seen": 738131968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006273380140421264, + "loss": 1.4099, + "theoretical_loss": 3.787680752483451, + "tokens_seen": 738197504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006273219658976931, + "loss": 1.4504, + "theoretical_loss": 3.787647188221856, + "tokens_seen": 738263040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006273059177532598, + "loss": 1.4757, + "theoretical_loss": 3.787613627773826, + "tokens_seen": 738328576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006272898696088265, + "loss": 1.3534, + "theoretical_loss": 3.7875800711385903, + "tokens_seen": 738394112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006272738214643932, + "loss": 1.2828, + "theoretical_loss": 3.7875465183153767, + "tokens_seen": 738459648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006272577733199599, + "loss": 1.5056, + "theoretical_loss": 3.787512969303414, + "tokens_seen": 738525184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006272417251755266, + "loss": 1.4318, + "theoretical_loss": 3.7874794241019307, + "tokens_seen": 738590720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006272256770310933, + "loss": 1.4262, + "theoretical_loss": 3.7874458827101565, + "tokens_seen": 738656256 + }, + { + "epoch": 0.22, + "learning_rate": 0.00062720962888666, + "loss": 1.3815, + "theoretical_loss": 3.7874123451273203, + "tokens_seen": 738721792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271935807422267, + "loss": 1.4049, + "theoretical_loss": 3.787378811352652, + "tokens_seen": 738787328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271775325977934, + "loss": 1.3613, + "theoretical_loss": 3.787345281385382, + "tokens_seen": 738852864 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271614844533601, + "loss": 1.3919, + "theoretical_loss": 3.7873117552247395, + "tokens_seen": 738918400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271454363089268, + "loss": 1.4125, + "theoretical_loss": 3.7872782328699555, + "tokens_seen": 738983936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271293881644935, + "loss": 1.3893, + "theoretical_loss": 3.7872447143202597, + "tokens_seen": 739049472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271133400200602, + "loss": 1.5538, + "theoretical_loss": 3.787211199574883, + "tokens_seen": 739115008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270972918756269, + "loss": 1.3634, + "theoretical_loss": 3.787177688633057, + "tokens_seen": 739180544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270812437311936, + "loss": 1.4892, + "theoretical_loss": 3.7871441814940123, + "tokens_seen": 739246080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270651955867603, + "loss": 1.3793, + "theoretical_loss": 3.7871106781569805, + "tokens_seen": 739311616 + }, + { + "epoch": 0.22, + "learning_rate": 0.000627049147442327, + "loss": 1.5246, + "theoretical_loss": 3.7870771786211925, + "tokens_seen": 739377152 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270330992978937, + "loss": 1.4735, + "theoretical_loss": 3.787043682885882, + "tokens_seen": 739442688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270170511534605, + "loss": 1.4128, + "theoretical_loss": 3.7870101909502787, + "tokens_seen": 739508224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270010030090271, + "loss": 1.3249, + "theoretical_loss": 3.7869767028136163, + "tokens_seen": 739573760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269849548645939, + "loss": 1.4879, + "theoretical_loss": 3.7869432184751277, + "tokens_seen": 739639296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269689067201605, + "loss": 1.3336, + "theoretical_loss": 3.7869097379340446, + "tokens_seen": 739704832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269528585757273, + "loss": 1.4394, + "theoretical_loss": 3.7868762611896, + "tokens_seen": 739770368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269368104312939, + "loss": 1.3731, + "theoretical_loss": 3.786842788241028, + "tokens_seen": 739835904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269207622868607, + "loss": 1.475, + "theoretical_loss": 3.7868093190875616, + "tokens_seen": 739901440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269047141424273, + "loss": 1.3288, + "theoretical_loss": 3.7867758537284337, + "tokens_seen": 739966976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268886659979941, + "loss": 1.5498, + "theoretical_loss": 3.786742392162879, + "tokens_seen": 740032512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268726178535607, + "loss": 1.3945, + "theoretical_loss": 3.7867089343901315, + "tokens_seen": 740098048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268565697091275, + "loss": 1.4395, + "theoretical_loss": 3.786675480409425, + "tokens_seen": 740163584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268405215646941, + "loss": 1.5172, + "theoretical_loss": 3.786642030219994, + "tokens_seen": 740229120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268244734202609, + "loss": 1.426, + "theoretical_loss": 3.7866085838210735, + "tokens_seen": 740294656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268084252758275, + "loss": 1.3196, + "theoretical_loss": 3.786575141211899, + "tokens_seen": 740360192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267923771313943, + "loss": 1.356, + "theoretical_loss": 3.7865417023917054, + "tokens_seen": 740425728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267763289869609, + "loss": 1.4566, + "theoretical_loss": 3.7865082673597277, + "tokens_seen": 740491264 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 252416, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3830698728561401, + "objective/train/theoretical_loss": 3.786474836115201, + "objective/train/tokens_used": 761016800, + "theoretical_loss": 3.786474836115201, + "tokens_seen": 740556800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267602808425277, + "loss": 1.3715, + "theoretical_loss": 3.786474836115201, + "tokens_seen": 740556800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267442326980943, + "loss": 1.3389, + "theoretical_loss": 3.786441408657363, + "tokens_seen": 740622336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267281845536611, + "loss": 1.4551, + "theoretical_loss": 3.7864079849854475, + "tokens_seen": 740687872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267121364092277, + "loss": 1.3781, + "theoretical_loss": 3.7863745650986926, + "tokens_seen": 740753408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006266960882647945, + "loss": 1.4396, + "theoretical_loss": 3.7863411489963346, + "tokens_seen": 740818944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006266800401203611, + "loss": 1.3852, + "theoretical_loss": 3.786307736677609, + "tokens_seen": 740884480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006266639919759278, + "loss": 1.3442, + "theoretical_loss": 3.7862743281417535, + "tokens_seen": 740950016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006266479438314945, + "loss": 1.4706, + "theoretical_loss": 3.786240923388006, + "tokens_seen": 741015552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006266318956870612, + "loss": 1.3943, + "theoretical_loss": 3.7862075224156024, + "tokens_seen": 741081088 + }, + { + "epoch": 0.22, + "learning_rate": 0.000626615847542628, + "loss": 1.465, + "theoretical_loss": 3.7861741252237815, + "tokens_seen": 741146624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006265997993981946, + "loss": 1.4682, + "theoretical_loss": 3.7861407318117815, + "tokens_seen": 741212160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006265837512537613, + "loss": 1.3761, + "theoretical_loss": 3.7861073421788385, + "tokens_seen": 741277696 + }, + { + "epoch": 0.22, + "learning_rate": 0.000626567703109328, + "loss": 1.4734, + "theoretical_loss": 3.786073956324193, + "tokens_seen": 741343232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006265516549648948, + "loss": 1.396, + "theoretical_loss": 3.786040574247082, + "tokens_seen": 741408768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006265356068204614, + "loss": 1.3962, + "theoretical_loss": 3.7860071959467447, + "tokens_seen": 741474304 + }, + { + "epoch": 0.22, + "learning_rate": 0.000626519558676028, + "loss": 1.3869, + "theoretical_loss": 3.785973821422421, + "tokens_seen": 741539840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006265035105315948, + "loss": 1.4752, + "theoretical_loss": 3.7859404506733485, + "tokens_seen": 741605376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006264874623871614, + "loss": 1.4597, + "theoretical_loss": 3.7859070836987674, + "tokens_seen": 741670912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006264714142427282, + "loss": 1.3884, + "theoretical_loss": 3.785873720497917, + "tokens_seen": 741736448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006264553660982948, + "loss": 1.3725, + "theoretical_loss": 3.7858403610700373, + "tokens_seen": 741801984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006264393179538616, + "loss": 1.4943, + "theoretical_loss": 3.7858070054143687, + "tokens_seen": 741867520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006264232698094282, + "loss": 1.4139, + "theoretical_loss": 3.7857736535301507, + "tokens_seen": 741933056 + }, + { + "epoch": 0.22, + "learning_rate": 0.000626407221664995, + "loss": 1.3827, + "theoretical_loss": 3.785740305416624, + "tokens_seen": 741998592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006263911735205616, + "loss": 1.433, + "theoretical_loss": 3.7857069610730303, + "tokens_seen": 742064128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006263751253761284, + "loss": 1.5033, + "theoretical_loss": 3.785673620498609, + "tokens_seen": 742129664 + }, + { + "epoch": 0.22, + "learning_rate": 0.000626359077231695, + "loss": 1.5053, + "theoretical_loss": 3.7856402836926026, + "tokens_seen": 742195200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006263430290872618, + "loss": 1.5469, + "theoretical_loss": 3.785606950654252, + "tokens_seen": 742260736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006263269809428285, + "loss": 1.4765, + "theoretical_loss": 3.785573621382798, + "tokens_seen": 742326272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006263109327983952, + "loss": 1.3257, + "theoretical_loss": 3.7855402958774835, + "tokens_seen": 742391808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006262948846539619, + "loss": 1.3639, + "theoretical_loss": 3.78550697413755, + "tokens_seen": 742457344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006262788365095286, + "loss": 1.4911, + "theoretical_loss": 3.7854736561622397, + "tokens_seen": 742522880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006262627883650953, + "loss": 1.4253, + "theoretical_loss": 3.785440341950795, + "tokens_seen": 742588416 + }, + { + "epoch": 0.23, + "learning_rate": 0.000626246740220662, + "loss": 1.4194, + "theoretical_loss": 3.7854070315024595, + "tokens_seen": 742653952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006262306920762287, + "loss": 1.4468, + "theoretical_loss": 3.7853737248164743, + "tokens_seen": 742719488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006262146439317954, + "loss": 1.4094, + "theoretical_loss": 3.7853404218920845, + "tokens_seen": 742785024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261985957873621, + "loss": 1.367, + "theoretical_loss": 3.7853071227285326, + "tokens_seen": 742850560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261825476429288, + "loss": 1.3086, + "theoretical_loss": 3.785273827325062, + "tokens_seen": 742916096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261664994984955, + "loss": 1.5286, + "theoretical_loss": 3.7852405356809165, + "tokens_seen": 742981632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261504513540622, + "loss": 1.3893, + "theoretical_loss": 3.78520724779534, + "tokens_seen": 743047168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261344032096289, + "loss": 1.367, + "theoretical_loss": 3.785173963667577, + "tokens_seen": 743112704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261183550651956, + "loss": 1.3122, + "theoretical_loss": 3.785140683296872, + "tokens_seen": 743178240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261023069207623, + "loss": 1.4675, + "theoretical_loss": 3.7851074066824695, + "tokens_seen": 743243776 + }, + { + "epoch": 0.23, + "learning_rate": 0.000626086258776329, + "loss": 1.4181, + "theoretical_loss": 3.7850741338236142, + "tokens_seen": 743309312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006260702106318957, + "loss": 1.4501, + "theoretical_loss": 3.7850408647195515, + "tokens_seen": 743374848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006260541624874624, + "loss": 1.4349, + "theoretical_loss": 3.7850075993695267, + "tokens_seen": 743440384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006260381143430291, + "loss": 1.3888, + "theoretical_loss": 3.7849743377727854, + "tokens_seen": 743505920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006260220661985958, + "loss": 1.5065, + "theoretical_loss": 3.784941079928573, + "tokens_seen": 743571456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006260060180541625, + "loss": 1.3948, + "theoretical_loss": 3.7849078258361355, + "tokens_seen": 743636992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006259899699097292, + "loss": 1.3591, + "theoretical_loss": 3.784874575494719, + "tokens_seen": 743702528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006259739217652959, + "loss": 1.3867, + "theoretical_loss": 3.7848413289035703, + "tokens_seen": 743768064 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 253356, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4743261337280273, + "objective/train/theoretical_loss": 3.784808086061936, + "objective/train/tokens_used": 764293600, + "theoretical_loss": 3.784808086061936, + "tokens_seen": 743833600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006259578736208626, + "loss": 1.4374, + "theoretical_loss": 3.784808086061936, + "tokens_seen": 743833600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006259418254764293, + "loss": 1.4514, + "theoretical_loss": 3.7847748469690625, + "tokens_seen": 743899136 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625925777331996, + "loss": 1.4886, + "theoretical_loss": 3.784741611624198, + "tokens_seen": 743964672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006259097291875627, + "loss": 1.3899, + "theoretical_loss": 3.784708380026588, + "tokens_seen": 744030208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006258936810431294, + "loss": 1.4282, + "theoretical_loss": 3.784675152175481, + "tokens_seen": 744095744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006258776328986961, + "loss": 1.4404, + "theoretical_loss": 3.7846419280701245, + "tokens_seen": 744161280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006258615847542628, + "loss": 1.4096, + "theoretical_loss": 3.7846087077097668, + "tokens_seen": 744226816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006258455366098295, + "loss": 1.328, + "theoretical_loss": 3.7845754910936558, + "tokens_seen": 744292352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006258294884653962, + "loss": 1.4444, + "theoretical_loss": 3.784542278221039, + "tokens_seen": 744357888 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625813440320963, + "loss": 1.4253, + "theoretical_loss": 3.784509069091167, + "tokens_seen": 744423424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257973921765296, + "loss": 1.3054, + "theoretical_loss": 3.7844758637032867, + "tokens_seen": 744488960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257813440320964, + "loss": 1.619, + "theoretical_loss": 3.784442662056648, + "tokens_seen": 744554496 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625765295887663, + "loss": 1.4287, + "theoretical_loss": 3.7844094641504995, + "tokens_seen": 744620032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257492477432298, + "loss": 1.4623, + "theoretical_loss": 3.784376269984092, + "tokens_seen": 744685568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257331995987964, + "loss": 1.4089, + "theoretical_loss": 3.784343079556673, + "tokens_seen": 744751104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257171514543632, + "loss": 1.3625, + "theoretical_loss": 3.784309892867494, + "tokens_seen": 744816640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257011033099298, + "loss": 1.411, + "theoretical_loss": 3.784276709915805, + "tokens_seen": 744882176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006256850551654966, + "loss": 1.3846, + "theoretical_loss": 3.7842435307008557, + "tokens_seen": 744947712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006256690070210632, + "loss": 1.4369, + "theoretical_loss": 3.784210355221897, + "tokens_seen": 745013248 + }, + { + "epoch": 0.23, + "learning_rate": 0.00062565295887663, + "loss": 1.4207, + "theoretical_loss": 3.78417718347818, + "tokens_seen": 745078784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006256369107321966, + "loss": 1.3438, + "theoretical_loss": 3.7841440154689545, + "tokens_seen": 745144320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006256208625877634, + "loss": 1.3528, + "theoretical_loss": 3.7841108511934727, + "tokens_seen": 745209856 + }, + { + "epoch": 0.23, + "learning_rate": 0.00062560481444333, + "loss": 1.4787, + "theoretical_loss": 3.7840776906509856, + "tokens_seen": 745275392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255887662988968, + "loss": 1.4328, + "theoretical_loss": 3.784044533840745, + "tokens_seen": 745340928 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255727181544634, + "loss": 1.5011, + "theoretical_loss": 3.784011380762003, + "tokens_seen": 745406464 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255566700100302, + "loss": 1.4873, + "theoretical_loss": 3.7839782314140105, + "tokens_seen": 745472000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255406218655968, + "loss": 1.4528, + "theoretical_loss": 3.78394508579602, + "tokens_seen": 745537536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255245737211636, + "loss": 1.4958, + "theoretical_loss": 3.7839119439072855, + "tokens_seen": 745603072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255085255767302, + "loss": 1.4287, + "theoretical_loss": 3.7838788057470585, + "tokens_seen": 745668608 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625492477432297, + "loss": 1.387, + "theoretical_loss": 3.783845671314592, + "tokens_seen": 745734144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006254764292878636, + "loss": 1.5027, + "theoretical_loss": 3.783812540609139, + "tokens_seen": 745799680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006254603811434304, + "loss": 1.3966, + "theoretical_loss": 3.783779413629953, + "tokens_seen": 745865216 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625444332998997, + "loss": 1.5242, + "theoretical_loss": 3.7837462903762873, + "tokens_seen": 745930752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006254282848545638, + "loss": 1.3962, + "theoretical_loss": 3.7837131708473963, + "tokens_seen": 745996288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006254122367101304, + "loss": 1.5299, + "theoretical_loss": 3.7836800550425336, + "tokens_seen": 746061824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006253961885656972, + "loss": 1.4618, + "theoretical_loss": 3.783646942960953, + "tokens_seen": 746127360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006253801404212638, + "loss": 1.4436, + "theoretical_loss": 3.7836138346019093, + "tokens_seen": 746192896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006253640922768306, + "loss": 1.4595, + "theoretical_loss": 3.783580729964657, + "tokens_seen": 746258432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006253480441323972, + "loss": 1.3886, + "theoretical_loss": 3.783547629048451, + "tokens_seen": 746323968 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625331995987964, + "loss": 1.3628, + "theoretical_loss": 3.7835145318525463, + "tokens_seen": 746389504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006253159478435306, + "loss": 1.378, + "theoretical_loss": 3.7834814383761985, + "tokens_seen": 746455040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252998996990974, + "loss": 1.4002, + "theoretical_loss": 3.7834483486186627, + "tokens_seen": 746520576 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625283851554664, + "loss": 1.3338, + "theoretical_loss": 3.7834152625791946, + "tokens_seen": 746586112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252678034102307, + "loss": 1.4126, + "theoretical_loss": 3.7833821802570506, + "tokens_seen": 746651648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252517552657974, + "loss": 1.4488, + "theoretical_loss": 3.7833491016514857, + "tokens_seen": 746717184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252357071213641, + "loss": 1.3234, + "theoretical_loss": 3.7833160267617574, + "tokens_seen": 746782720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252196589769308, + "loss": 1.3528, + "theoretical_loss": 3.783282955587122, + "tokens_seen": 746848256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252036108324975, + "loss": 1.3483, + "theoretical_loss": 3.7832498881268357, + "tokens_seen": 746913792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251875626880643, + "loss": 1.4388, + "theoretical_loss": 3.7832168243801556, + "tokens_seen": 746979328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251715145436309, + "loss": 1.3172, + "theoretical_loss": 3.7831837643463393, + "tokens_seen": 747044864 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 254556, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5359313488006592, + "objective/train/theoretical_loss": 3.7831507080246443, + "objective/train/tokens_used": 767570400, + "theoretical_loss": 3.7831507080246443, + "tokens_seen": 747110400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251554663991977, + "loss": 1.4609, + "theoretical_loss": 3.7831507080246443, + "tokens_seen": 747110400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251394182547643, + "loss": 1.5006, + "theoretical_loss": 3.783117655414328, + "tokens_seen": 747175936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251233701103309, + "loss": 1.5097, + "theoretical_loss": 3.783084606514648, + "tokens_seen": 747241472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251073219658977, + "loss": 1.4837, + "theoretical_loss": 3.7830515613248625, + "tokens_seen": 747307008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250912738214643, + "loss": 1.5801, + "theoretical_loss": 3.7830185198442297, + "tokens_seen": 747372544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250752256770311, + "loss": 1.3344, + "theoretical_loss": 3.782985482072008, + "tokens_seen": 747438080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250591775325977, + "loss": 1.3575, + "theoretical_loss": 3.782952448007457, + "tokens_seen": 747503616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250431293881645, + "loss": 1.476, + "theoretical_loss": 3.7829194176498335, + "tokens_seen": 747569152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250270812437311, + "loss": 1.4671, + "theoretical_loss": 3.7828863909983994, + "tokens_seen": 747634688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250110330992979, + "loss": 1.3915, + "theoretical_loss": 3.7828533680524115, + "tokens_seen": 747700224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006249949849548645, + "loss": 1.4057, + "theoretical_loss": 3.782820348811131, + "tokens_seen": 747765760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006249789368104313, + "loss": 1.5221, + "theoretical_loss": 3.7827873332738164, + "tokens_seen": 747831296 + }, + { + "epoch": 0.23, + "learning_rate": 0.000624962888665998, + "loss": 1.4371, + "theoretical_loss": 3.782754321439729, + "tokens_seen": 747896832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006249468405215647, + "loss": 1.3435, + "theoretical_loss": 3.7827213133081283, + "tokens_seen": 747962368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006249307923771314, + "loss": 1.3588, + "theoretical_loss": 3.7826883088782743, + "tokens_seen": 748027904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006249147442326981, + "loss": 1.3512, + "theoretical_loss": 3.782655308149428, + "tokens_seen": 748093440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248986960882648, + "loss": 1.3859, + "theoretical_loss": 3.78262231112085, + "tokens_seen": 748158976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248826479438315, + "loss": 1.331, + "theoretical_loss": 3.7825893177918015, + "tokens_seen": 748224512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248665997993982, + "loss": 1.5332, + "theoretical_loss": 3.7825563281615446, + "tokens_seen": 748290048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248505516549649, + "loss": 1.3855, + "theoretical_loss": 3.7825233422293394, + "tokens_seen": 748355584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248345035105316, + "loss": 1.3496, + "theoretical_loss": 3.7824903599944477, + "tokens_seen": 748421120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248184553660983, + "loss": 1.4544, + "theoretical_loss": 3.7824573814561324, + "tokens_seen": 748486656 + }, + { + "epoch": 0.23, + "learning_rate": 0.000624802407221665, + "loss": 1.4627, + "theoretical_loss": 3.782424406613654, + "tokens_seen": 748552192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247863590772317, + "loss": 1.3918, + "theoretical_loss": 3.782391435466277, + "tokens_seen": 748617728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247703109327984, + "loss": 1.486, + "theoretical_loss": 3.7823584680132614, + "tokens_seen": 748683264 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247542627883651, + "loss": 1.4443, + "theoretical_loss": 3.7823255042538717, + "tokens_seen": 748748800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247382146439318, + "loss": 1.4635, + "theoretical_loss": 3.78229254418737, + "tokens_seen": 748814336 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247221664994985, + "loss": 1.4962, + "theoretical_loss": 3.7822595878130203, + "tokens_seen": 748879872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247061183550652, + "loss": 1.3376, + "theoretical_loss": 3.782226635130085, + "tokens_seen": 748945408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006246900702106319, + "loss": 1.3647, + "theoretical_loss": 3.782193686137828, + "tokens_seen": 749010944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006246740220661986, + "loss": 1.5502, + "theoretical_loss": 3.782160740835513, + "tokens_seen": 749076480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006246579739217653, + "loss": 1.4429, + "theoretical_loss": 3.7821277992224043, + "tokens_seen": 749142016 + }, + { + "epoch": 0.23, + "learning_rate": 0.000624641925777332, + "loss": 1.3417, + "theoretical_loss": 3.7820948612977663, + "tokens_seen": 749207552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006246258776328987, + "loss": 1.4858, + "theoretical_loss": 3.7820619270608624, + "tokens_seen": 749273088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006246098294884655, + "loss": 1.3289, + "theoretical_loss": 3.782028996510958, + "tokens_seen": 749338624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245937813440321, + "loss": 1.4173, + "theoretical_loss": 3.781996069647318, + "tokens_seen": 749404160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245777331995989, + "loss": 1.2815, + "theoretical_loss": 3.781963146469207, + "tokens_seen": 749469696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245616850551655, + "loss": 1.3905, + "theoretical_loss": 3.7819302269758905, + "tokens_seen": 749535232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245456369107323, + "loss": 1.4193, + "theoretical_loss": 3.7818973111666345, + "tokens_seen": 749600768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245295887662989, + "loss": 1.4555, + "theoretical_loss": 3.7818643990407037, + "tokens_seen": 749666304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245135406218657, + "loss": 1.4173, + "theoretical_loss": 3.7818314905973645, + "tokens_seen": 749731840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244974924774323, + "loss": 1.4484, + "theoretical_loss": 3.7817985858358827, + "tokens_seen": 749797376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244814443329991, + "loss": 1.4426, + "theoretical_loss": 3.781765684755525, + "tokens_seen": 749862912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244653961885657, + "loss": 1.4372, + "theoretical_loss": 3.7817327873555575, + "tokens_seen": 749928448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244493480441325, + "loss": 1.5416, + "theoretical_loss": 3.7816998936352473, + "tokens_seen": 749993984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244332998996991, + "loss": 1.4446, + "theoretical_loss": 3.7816670035938618, + "tokens_seen": 750059520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244172517552659, + "loss": 1.3866, + "theoretical_loss": 3.7816341172306664, + "tokens_seen": 750125056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244012036108325, + "loss": 1.4924, + "theoretical_loss": 3.781601234544931, + "tokens_seen": 750190592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243851554663993, + "loss": 1.4214, + "theoretical_loss": 3.7815683555359207, + "tokens_seen": 750256128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243691073219659, + "loss": 1.4347, + "theoretical_loss": 3.781535480202905, + "tokens_seen": 750321664 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 255521, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4189245700836182, + "objective/train/theoretical_loss": 3.781502608545151, + "objective/train/tokens_used": 770847200, + "theoretical_loss": 3.781502608545151, + "tokens_seen": 750387200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243530591775327, + "loss": 1.3877, + "theoretical_loss": 3.781502608545151, + "tokens_seen": 750387200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243370110330993, + "loss": 1.4554, + "theoretical_loss": 3.781469740561927, + "tokens_seen": 750452736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243209628886661, + "loss": 1.3596, + "theoretical_loss": 3.7814368762525015, + "tokens_seen": 750518272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243049147442327, + "loss": 1.4392, + "theoretical_loss": 3.7814040156161433, + "tokens_seen": 750583808 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242888665997995, + "loss": 1.4331, + "theoretical_loss": 3.7813711586521217, + "tokens_seen": 750649344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242728184553661, + "loss": 1.4317, + "theoretical_loss": 3.7813383053597045, + "tokens_seen": 750714880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242567703109329, + "loss": 1.4442, + "theoretical_loss": 3.7813054557381616, + "tokens_seen": 750780416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242407221664995, + "loss": 1.3416, + "theoretical_loss": 3.7812726097867624, + "tokens_seen": 750845952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242246740220663, + "loss": 1.4734, + "theoretical_loss": 3.781239767504777, + "tokens_seen": 750911488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242086258776329, + "loss": 1.4555, + "theoretical_loss": 3.781206928891475, + "tokens_seen": 750977024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241925777331997, + "loss": 1.3431, + "theoretical_loss": 3.7811740939461256, + "tokens_seen": 751042560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241765295887663, + "loss": 1.4286, + "theoretical_loss": 3.781141262668, + "tokens_seen": 751108096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241604814443331, + "loss": 1.4231, + "theoretical_loss": 3.7811084350563693, + "tokens_seen": 751173632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241444332998997, + "loss": 1.3431, + "theoretical_loss": 3.781075611110503, + "tokens_seen": 751239168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241283851554665, + "loss": 1.4482, + "theoretical_loss": 3.781042790829672, + "tokens_seen": 751304704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241123370110331, + "loss": 1.4495, + "theoretical_loss": 3.7810099742131484, + "tokens_seen": 751370240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240962888665999, + "loss": 1.4483, + "theoretical_loss": 3.7809771612602034, + "tokens_seen": 751435776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240802407221665, + "loss": 1.514, + "theoretical_loss": 3.780944351970107, + "tokens_seen": 751501312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240641925777333, + "loss": 1.4726, + "theoretical_loss": 3.7809115463421334, + "tokens_seen": 751566848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240481444332999, + "loss": 1.3094, + "theoretical_loss": 3.7808787443755527, + "tokens_seen": 751632384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240320962888667, + "loss": 1.3706, + "theoretical_loss": 3.780845946069638, + "tokens_seen": 751697920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240160481444333, + "loss": 1.5188, + "theoretical_loss": 3.7808131514236614, + "tokens_seen": 751763456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240000000000001, + "loss": 1.387, + "theoretical_loss": 3.7807803604368946, + "tokens_seen": 751828992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239839518555667, + "loss": 1.4186, + "theoretical_loss": 3.780747573108612, + "tokens_seen": 751894528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239679037111335, + "loss": 1.3802, + "theoretical_loss": 3.7807147894380853, + "tokens_seen": 751960064 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239518555667001, + "loss": 1.4785, + "theoretical_loss": 3.7806820094245888, + "tokens_seen": 752025600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239358074222669, + "loss": 1.4123, + "theoretical_loss": 3.780649233067395, + "tokens_seen": 752091136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239197592778335, + "loss": 1.4093, + "theoretical_loss": 3.7806164603657777, + "tokens_seen": 752156672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239037111334003, + "loss": 1.3978, + "theoretical_loss": 3.780583691319011, + "tokens_seen": 752222208 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623887662988967, + "loss": 1.3695, + "theoretical_loss": 3.7805509259263688, + "tokens_seen": 752287744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006238716148445336, + "loss": 1.4512, + "theoretical_loss": 3.7805181641871255, + "tokens_seen": 752353280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006238555667001003, + "loss": 1.5197, + "theoretical_loss": 3.7804854061005555, + "tokens_seen": 752418816 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623839518555667, + "loss": 1.3625, + "theoretical_loss": 3.780452651665933, + "tokens_seen": 752484352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006238234704112337, + "loss": 1.3544, + "theoretical_loss": 3.780419900882533, + "tokens_seen": 752549888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006238074222668004, + "loss": 1.4661, + "theoretical_loss": 3.780387153749632, + "tokens_seen": 752615424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237913741223672, + "loss": 1.389, + "theoretical_loss": 3.780354410266503, + "tokens_seen": 752680960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237753259779338, + "loss": 1.3227, + "theoretical_loss": 3.780321670432423, + "tokens_seen": 752746496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237592778335006, + "loss": 1.3916, + "theoretical_loss": 3.7802889342466672, + "tokens_seen": 752812032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237432296890672, + "loss": 1.4233, + "theoretical_loss": 3.7802562017085117, + "tokens_seen": 752877568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237271815446338, + "loss": 1.3916, + "theoretical_loss": 3.780223472817233, + "tokens_seen": 752943104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237111334002006, + "loss": 1.4423, + "theoretical_loss": 3.7801907475721057, + "tokens_seen": 753008640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006236950852557672, + "loss": 1.4797, + "theoretical_loss": 3.7801580259724084, + "tokens_seen": 753074176 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623679037111334, + "loss": 1.4956, + "theoretical_loss": 3.780125308017417, + "tokens_seen": 753139712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006236629889669006, + "loss": 1.3677, + "theoretical_loss": 3.780092593706408, + "tokens_seen": 753205248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006236469408224674, + "loss": 1.4972, + "theoretical_loss": 3.7800598830386587, + "tokens_seen": 753270784 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623630892678034, + "loss": 1.4059, + "theoretical_loss": 3.780027176013447, + "tokens_seen": 753336320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006236148445336008, + "loss": 1.4172, + "theoretical_loss": 3.7799944726300505, + "tokens_seen": 753401856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235987963891675, + "loss": 1.4357, + "theoretical_loss": 3.7799617728877464, + "tokens_seen": 753467392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235827482447342, + "loss": 1.3688, + "theoretical_loss": 3.7799290767858125, + "tokens_seen": 753532928 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235667001003009, + "loss": 1.4547, + "theoretical_loss": 3.7798963843235276, + "tokens_seen": 753598464 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 256644, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4571774005889893, + "objective/train/theoretical_loss": 3.77986369550017, + "objective/train/tokens_used": 774124000, + "theoretical_loss": 3.77986369550017, + "tokens_seen": 753664000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235506519558676, + "loss": 1.448, + "theoretical_loss": 3.77986369550017, + "tokens_seen": 753664000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235346038114343, + "loss": 1.437, + "theoretical_loss": 3.779831010315018, + "tokens_seen": 753729536 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623518555667001, + "loss": 1.3776, + "theoretical_loss": 3.7797983287673502, + "tokens_seen": 753795072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235025075225677, + "loss": 1.4957, + "theoretical_loss": 3.7797656508564463, + "tokens_seen": 753860608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006234864593781344, + "loss": 1.4569, + "theoretical_loss": 3.7797329765815846, + "tokens_seen": 753926144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006234704112337011, + "loss": 1.4283, + "theoretical_loss": 3.779700305942045, + "tokens_seen": 753991680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006234543630892678, + "loss": 1.3188, + "theoretical_loss": 3.7796676389371076, + "tokens_seen": 754057216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006234383149448345, + "loss": 1.3921, + "theoretical_loss": 3.7796349755660517, + "tokens_seen": 754122752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006234222668004012, + "loss": 1.4486, + "theoretical_loss": 3.7796023158281566, + "tokens_seen": 754188288 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623406218655968, + "loss": 1.3746, + "theoretical_loss": 3.7795696597227035, + "tokens_seen": 754253824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006233901705115346, + "loss": 1.3456, + "theoretical_loss": 3.7795370072489733, + "tokens_seen": 754319360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006233741223671014, + "loss": 1.4396, + "theoretical_loss": 3.7795043584062453, + "tokens_seen": 754384896 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623358074222668, + "loss": 1.4591, + "theoretical_loss": 3.779471713193801, + "tokens_seen": 754450432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006233420260782348, + "loss": 1.4003, + "theoretical_loss": 3.7794390716109216, + "tokens_seen": 754515968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006233259779338014, + "loss": 1.4405, + "theoretical_loss": 3.779406433656888, + "tokens_seen": 754581504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006233099297893682, + "loss": 1.4479, + "theoretical_loss": 3.7793737993309824, + "tokens_seen": 754647040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006232938816449348, + "loss": 1.2737, + "theoretical_loss": 3.7793411686324854, + "tokens_seen": 754712576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006232778335005016, + "loss": 1.4082, + "theoretical_loss": 3.7793085415606793, + "tokens_seen": 754778112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006232617853560682, + "loss": 1.4845, + "theoretical_loss": 3.7792759181148465, + "tokens_seen": 754843648 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623245737211635, + "loss": 1.2884, + "theoretical_loss": 3.7792432982942685, + "tokens_seen": 754909184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006232296890672016, + "loss": 1.4281, + "theoretical_loss": 3.7792106820982285, + "tokens_seen": 754974720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006232136409227684, + "loss": 1.4686, + "theoretical_loss": 3.779178069526009, + "tokens_seen": 755040256 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623197592778335, + "loss": 1.4515, + "theoretical_loss": 3.779145460576893, + "tokens_seen": 755105792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231815446339018, + "loss": 1.4006, + "theoretical_loss": 3.7791128552501636, + "tokens_seen": 755171328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231654964894684, + "loss": 1.5029, + "theoretical_loss": 3.7790802535451036, + "tokens_seen": 755236864 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231494483450352, + "loss": 1.4486, + "theoretical_loss": 3.779047655460997, + "tokens_seen": 755302400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231334002006018, + "loss": 1.4276, + "theoretical_loss": 3.779015060997127, + "tokens_seen": 755367936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231173520561686, + "loss": 1.283, + "theoretical_loss": 3.778982470152778, + "tokens_seen": 755433472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231013039117352, + "loss": 1.5825, + "theoretical_loss": 3.7789498829272343, + "tokens_seen": 755499008 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623085255767302, + "loss": 1.4461, + "theoretical_loss": 3.7789172993197795, + "tokens_seen": 755564544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006230692076228686, + "loss": 1.4566, + "theoretical_loss": 3.7788847193296986, + "tokens_seen": 755630080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006230531594784354, + "loss": 1.3111, + "theoretical_loss": 3.7788521429562767, + "tokens_seen": 755695616 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623037111334002, + "loss": 1.4424, + "theoretical_loss": 3.778819570198798, + "tokens_seen": 755761152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006230210631895688, + "loss": 1.4257, + "theoretical_loss": 3.778787001056547, + "tokens_seen": 755826688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006230050150451354, + "loss": 1.4097, + "theoretical_loss": 3.7787544355288105, + "tokens_seen": 755892224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006229889669007022, + "loss": 1.3617, + "theoretical_loss": 3.7787218736148738, + "tokens_seen": 755957760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006229729187562688, + "loss": 1.436, + "theoretical_loss": 3.778689315314022, + "tokens_seen": 756023296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006229568706118356, + "loss": 1.2949, + "theoretical_loss": 3.778656760625541, + "tokens_seen": 756088832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006229408224674022, + "loss": 1.4906, + "theoretical_loss": 3.7786242095487177, + "tokens_seen": 756154368 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622924774322969, + "loss": 1.4942, + "theoretical_loss": 3.778591662082838, + "tokens_seen": 756219904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006229087261785356, + "loss": 1.3981, + "theoretical_loss": 3.7785591182271885, + "tokens_seen": 756285440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006228926780341024, + "loss": 1.4899, + "theoretical_loss": 3.7785265779810553, + "tokens_seen": 756350976 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622876629889669, + "loss": 1.4776, + "theoretical_loss": 3.778494041343727, + "tokens_seen": 756416512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006228605817452358, + "loss": 1.3764, + "theoretical_loss": 3.778461508314489, + "tokens_seen": 756482048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006228445336008024, + "loss": 1.4456, + "theoretical_loss": 3.7784289788926295, + "tokens_seen": 756547584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006228284854563692, + "loss": 1.4069, + "theoretical_loss": 3.778396453077436, + "tokens_seen": 756613120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006228124373119358, + "loss": 1.4312, + "theoretical_loss": 3.778363930868196, + "tokens_seen": 756678656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006227963891675026, + "loss": 1.4911, + "theoretical_loss": 3.778331412264198, + "tokens_seen": 756744192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006227803410230692, + "loss": 1.4605, + "theoretical_loss": 3.7782988972647296, + "tokens_seen": 756809728 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622764292878636, + "loss": 1.3825, + "theoretical_loss": 3.7782663858690797, + "tokens_seen": 756875264 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 257583, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.471604824066162, + "objective/train/theoretical_loss": 3.778233878076536, + "objective/train/tokens_used": 777400800, + "theoretical_loss": 3.778233878076536, + "tokens_seen": 756940800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006227482447342026, + "loss": 1.4712, + "theoretical_loss": 3.778233878076536, + "tokens_seen": 756940800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006227321965897694, + "loss": 1.38, + "theoretical_loss": 3.7782013738863887, + "tokens_seen": 757006336 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622716148445336, + "loss": 1.2824, + "theoretical_loss": 3.7781688732979255, + "tokens_seen": 757071872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006227001003009028, + "loss": 1.3731, + "theoretical_loss": 3.778136376310436, + "tokens_seen": 757137408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006226840521564694, + "loss": 1.3583, + "theoretical_loss": 3.77810388292321, + "tokens_seen": 757202944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006226680040120362, + "loss": 1.4256, + "theoretical_loss": 3.7780713931355367, + "tokens_seen": 757268480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006226519558676028, + "loss": 1.446, + "theoretical_loss": 3.7780389069467057, + "tokens_seen": 757334016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006226359077231696, + "loss": 1.3744, + "theoretical_loss": 3.778006424356007, + "tokens_seen": 757399552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006226198595787362, + "loss": 1.4748, + "theoretical_loss": 3.7779739453627315, + "tokens_seen": 757465088 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622603811434303, + "loss": 1.304, + "theoretical_loss": 3.7779414699661684, + "tokens_seen": 757530624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006225877632898696, + "loss": 1.3951, + "theoretical_loss": 3.7779089981656093, + "tokens_seen": 757596160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006225717151454364, + "loss": 1.4539, + "theoretical_loss": 3.7778765299603445, + "tokens_seen": 757661696 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622555667001003, + "loss": 1.3827, + "theoretical_loss": 3.777844065349665, + "tokens_seen": 757727232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006225396188565698, + "loss": 1.3948, + "theoretical_loss": 3.7778116043328627, + "tokens_seen": 757792768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006225235707121364, + "loss": 1.4728, + "theoretical_loss": 3.777779146909228, + "tokens_seen": 757858304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006225075225677032, + "loss": 1.4275, + "theoretical_loss": 3.777746693078053, + "tokens_seen": 757923840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224914744232698, + "loss": 1.425, + "theoretical_loss": 3.7777142428386297, + "tokens_seen": 757989376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224754262788365, + "loss": 1.3828, + "theoretical_loss": 3.777681796190249, + "tokens_seen": 758054912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224593781344032, + "loss": 1.4134, + "theoretical_loss": 3.777649353132205, + "tokens_seen": 758120448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224433299899699, + "loss": 1.4292, + "theoretical_loss": 3.7776169136637883, + "tokens_seen": 758185984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224272818455367, + "loss": 1.4421, + "theoretical_loss": 3.7775844777842926, + "tokens_seen": 758251520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224112337011033, + "loss": 1.4663, + "theoretical_loss": 3.7775520454930103, + "tokens_seen": 758317056 + }, + { + "epoch": 0.23, + "learning_rate": 0.00062239518555667, + "loss": 1.3652, + "theoretical_loss": 3.777519616789234, + "tokens_seen": 758382592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006223791374122367, + "loss": 1.3367, + "theoretical_loss": 3.7774871916722574, + "tokens_seen": 758448128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006223630892678035, + "loss": 1.5041, + "theoretical_loss": 3.7774547701413743, + "tokens_seen": 758513664 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006223470411233701, + "loss": 1.3251, + "theoretical_loss": 3.777422352195878, + "tokens_seen": 758579200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006223309929789367, + "loss": 1.3715, + "theoretical_loss": 3.7773899378350615, + "tokens_seen": 758644736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006223149448345035, + "loss": 1.4444, + "theoretical_loss": 3.7773575270582196, + "tokens_seen": 758710272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222988966900701, + "loss": 1.3804, + "theoretical_loss": 3.7773251198646465, + "tokens_seen": 758775808 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222828485456369, + "loss": 1.4785, + "theoretical_loss": 3.777292716253636, + "tokens_seen": 758841344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222668004012035, + "loss": 1.3629, + "theoretical_loss": 3.777260316224484, + "tokens_seen": 758906880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222507522567703, + "loss": 1.4352, + "theoretical_loss": 3.7772279197764838, + "tokens_seen": 758972416 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622234704112337, + "loss": 1.423, + "theoretical_loss": 3.7771955269089315, + "tokens_seen": 759037952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222186559679037, + "loss": 1.4486, + "theoretical_loss": 3.7771631376211214, + "tokens_seen": 759103488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222026078234705, + "loss": 1.4989, + "theoretical_loss": 3.7771307519123494, + "tokens_seen": 759169024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221865596790371, + "loss": 1.4417, + "theoretical_loss": 3.7770983697819114, + "tokens_seen": 759234560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221705115346039, + "loss": 1.4984, + "theoretical_loss": 3.7770659912291027, + "tokens_seen": 759300096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221544633901705, + "loss": 1.4641, + "theoretical_loss": 3.7770336162532194, + "tokens_seen": 759365632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221384152457373, + "loss": 1.3354, + "theoretical_loss": 3.7770012448535577, + "tokens_seen": 759431168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221223671013039, + "loss": 1.4338, + "theoretical_loss": 3.7769688770294145, + "tokens_seen": 759496704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221063189568707, + "loss": 1.3725, + "theoretical_loss": 3.776936512780085, + "tokens_seen": 759562240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220902708124373, + "loss": 1.3946, + "theoretical_loss": 3.776904152104868, + "tokens_seen": 759627776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220742226680041, + "loss": 1.4352, + "theoretical_loss": 3.776871795003059, + "tokens_seen": 759693312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220581745235707, + "loss": 1.5137, + "theoretical_loss": 3.7768394414739554, + "tokens_seen": 759758848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220421263791375, + "loss": 1.4267, + "theoretical_loss": 3.7768070915168552, + "tokens_seen": 759824384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220260782347041, + "loss": 1.4407, + "theoretical_loss": 3.7767747451310556, + "tokens_seen": 759889920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220100300902709, + "loss": 1.5487, + "theoretical_loss": 3.776742402315854, + "tokens_seen": 759955456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219939819458375, + "loss": 1.3201, + "theoretical_loss": 3.7767100630705492, + "tokens_seen": 760020992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219779338014043, + "loss": 1.4038, + "theoretical_loss": 3.776677727394439, + "tokens_seen": 760086528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219618856569709, + "loss": 1.4596, + "theoretical_loss": 3.7766453952868217, + "tokens_seen": 760152064 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 258909, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.467252492904663, + "objective/train/theoretical_loss": 3.776613066746996, + "objective/train/tokens_used": 780677600, + "theoretical_loss": 3.776613066746996, + "tokens_seen": 760217600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219458375125377, + "loss": 1.5356, + "theoretical_loss": 3.776613066746996, + "tokens_seen": 760217600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219297893681043, + "loss": 1.3483, + "theoretical_loss": 3.7765807417742607, + "tokens_seen": 760283136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219137412236711, + "loss": 1.5033, + "theoretical_loss": 3.7765484203679147, + "tokens_seen": 760348672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218976930792377, + "loss": 1.4532, + "theoretical_loss": 3.776516102527258, + "tokens_seen": 760414208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218816449348045, + "loss": 1.4403, + "theoretical_loss": 3.7764837882515874, + "tokens_seen": 760479744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218655967903711, + "loss": 1.4096, + "theoretical_loss": 3.7764514775402054, + "tokens_seen": 760545280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218495486459379, + "loss": 1.471, + "theoretical_loss": 3.776419170392411, + "tokens_seen": 760610816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218335005015045, + "loss": 1.4172, + "theoretical_loss": 3.7763868668075027, + "tokens_seen": 760676352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218174523570713, + "loss": 1.4975, + "theoretical_loss": 3.776354566784783, + "tokens_seen": 760741888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218014042126379, + "loss": 1.5022, + "theoretical_loss": 3.77632227032355, + "tokens_seen": 760807424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217853560682047, + "loss": 1.3702, + "theoretical_loss": 3.7762899774231062, + "tokens_seen": 760872960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217693079237713, + "loss": 1.4161, + "theoretical_loss": 3.776257688082751, + "tokens_seen": 760938496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217532597793381, + "loss": 1.4318, + "theoretical_loss": 3.776225402301786, + "tokens_seen": 761004032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217372116349047, + "loss": 1.3119, + "theoretical_loss": 3.776193120079512, + "tokens_seen": 761069568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217211634904715, + "loss": 1.4322, + "theoretical_loss": 3.7761608414152303, + "tokens_seen": 761135104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217051153460381, + "loss": 1.5481, + "theoretical_loss": 3.776128566308243, + "tokens_seen": 761200640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216890672016049, + "loss": 1.4285, + "theoretical_loss": 3.776096294757852, + "tokens_seen": 761266176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216730190571715, + "loss": 1.4636, + "theoretical_loss": 3.7760640267633576, + "tokens_seen": 761331712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216569709127383, + "loss": 1.3541, + "theoretical_loss": 3.776031762324064, + "tokens_seen": 761397248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216409227683049, + "loss": 1.3975, + "theoretical_loss": 3.775999501439273, + "tokens_seen": 761462784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216248746238717, + "loss": 1.4355, + "theoretical_loss": 3.775967244108286, + "tokens_seen": 761528320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216088264794383, + "loss": 1.486, + "theoretical_loss": 3.775934990330407, + "tokens_seen": 761593856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215927783350051, + "loss": 1.4797, + "theoretical_loss": 3.7759027401049385, + "tokens_seen": 761659392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215767301905717, + "loss": 1.5027, + "theoretical_loss": 3.7758704934311833, + "tokens_seen": 761724928 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215606820461385, + "loss": 1.3778, + "theoretical_loss": 3.7758382503084453, + "tokens_seen": 761790464 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215446339017051, + "loss": 1.3964, + "theoretical_loss": 3.7758060107360274, + "tokens_seen": 761856000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215285857572719, + "loss": 1.4766, + "theoretical_loss": 3.775773774713234, + "tokens_seen": 761921536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215125376128385, + "loss": 1.5921, + "theoretical_loss": 3.775741542239369, + "tokens_seen": 761987072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214964894684053, + "loss": 1.4706, + "theoretical_loss": 3.775709313313736, + "tokens_seen": 762052608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214804413239719, + "loss": 1.4036, + "theoretical_loss": 3.775677087935639, + "tokens_seen": 762118144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214643931795387, + "loss": 1.3538, + "theoretical_loss": 3.7756448661043835, + "tokens_seen": 762183680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214483450351053, + "loss": 1.3597, + "theoretical_loss": 3.7756126478192735, + "tokens_seen": 762249216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214322968906721, + "loss": 1.4321, + "theoretical_loss": 3.775580433079614, + "tokens_seen": 762314752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214162487462387, + "loss": 1.3701, + "theoretical_loss": 3.7755482218847103, + "tokens_seen": 762380288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214002006018055, + "loss": 1.4022, + "theoretical_loss": 3.775516014233868, + "tokens_seen": 762445824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213841524573721, + "loss": 1.4417, + "theoretical_loss": 3.7754838101263912, + "tokens_seen": 762511360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213681043129389, + "loss": 1.3648, + "theoretical_loss": 3.7754516095615873, + "tokens_seen": 762576896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213520561685055, + "loss": 1.375, + "theoretical_loss": 3.775419412538761, + "tokens_seen": 762642432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213360080240723, + "loss": 1.3808, + "theoretical_loss": 3.7753872190572193, + "tokens_seen": 762707968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213199598796389, + "loss": 1.4696, + "theoretical_loss": 3.775355029116268, + "tokens_seen": 762773504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213039117352057, + "loss": 1.465, + "theoretical_loss": 3.7753228427152123, + "tokens_seen": 762839040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212878635907723, + "loss": 1.3666, + "theoretical_loss": 3.7752906598533613, + "tokens_seen": 762904576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212718154463391, + "loss": 1.4051, + "theoretical_loss": 3.77525848053002, + "tokens_seen": 762970112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212557673019057, + "loss": 1.4702, + "theoretical_loss": 3.7752263047444963, + "tokens_seen": 763035648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212397191574725, + "loss": 1.4327, + "theoretical_loss": 3.7751941324960976, + "tokens_seen": 763101184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212236710130391, + "loss": 1.4795, + "theoretical_loss": 3.7751619637841305, + "tokens_seen": 763166720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212076228686059, + "loss": 1.5067, + "theoretical_loss": 3.775129798607903, + "tokens_seen": 763232256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006211915747241725, + "loss": 1.2731, + "theoretical_loss": 3.7750976369667226, + "tokens_seen": 763297792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006211755265797393, + "loss": 1.4211, + "theoretical_loss": 3.7750654788598985, + "tokens_seen": 763363328 + }, + { + "epoch": 0.23, + "learning_rate": 0.000621159478435306, + "loss": 1.3284, + "theoretical_loss": 3.7750333242867375, + "tokens_seen": 763428864 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 259724, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3330668210983276, + "objective/train/theoretical_loss": 3.775001173246549, + "objective/train/tokens_used": 783954400, + "theoretical_loss": 3.775001173246549, + "tokens_seen": 763494400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006211434302908727, + "loss": 1.3791, + "theoretical_loss": 3.775001173246549, + "tokens_seen": 763494400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006211273821464393, + "loss": 1.3849, + "theoretical_loss": 3.774969025738642, + "tokens_seen": 763559936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006211113340020061, + "loss": 1.4685, + "theoretical_loss": 3.7749368817623234, + "tokens_seen": 763625472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210952858575727, + "loss": 1.3842, + "theoretical_loss": 3.774904741316904, + "tokens_seen": 763691008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210792377131395, + "loss": 1.5269, + "theoretical_loss": 3.7748726044016916, + "tokens_seen": 763756544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210631895687062, + "loss": 1.4646, + "theoretical_loss": 3.774840471015997, + "tokens_seen": 763822080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210471414242728, + "loss": 1.3961, + "theoretical_loss": 3.774808341159129, + "tokens_seen": 763887616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210310932798396, + "loss": 1.3616, + "theoretical_loss": 3.7747762148303976, + "tokens_seen": 763953152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210150451354062, + "loss": 1.3605, + "theoretical_loss": 3.7747440920291124, + "tokens_seen": 764018688 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620998996990973, + "loss": 1.4318, + "theoretical_loss": 3.774711972754584, + "tokens_seen": 764084224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006209829488465396, + "loss": 1.4023, + "theoretical_loss": 3.7746798570061224, + "tokens_seen": 764149760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006209669007021064, + "loss": 1.4544, + "theoretical_loss": 3.774647744783038, + "tokens_seen": 764215296 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620950852557673, + "loss": 1.4006, + "theoretical_loss": 3.7746156360846426, + "tokens_seen": 764280832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006209348044132398, + "loss": 1.4134, + "theoretical_loss": 3.7745835309102462, + "tokens_seen": 764346368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006209187562688064, + "loss": 1.4996, + "theoretical_loss": 3.7745514292591595, + "tokens_seen": 764411904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006209027081243732, + "loss": 1.4596, + "theoretical_loss": 3.774519331130695, + "tokens_seen": 764477440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006208866599799398, + "loss": 1.466, + "theoretical_loss": 3.7744872365241635, + "tokens_seen": 764542976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006208706118355066, + "loss": 1.3785, + "theoretical_loss": 3.774455145438877, + "tokens_seen": 764608512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006208545636910732, + "loss": 1.4326, + "theoretical_loss": 3.7744230578741473, + "tokens_seen": 764674048 + }, + { + "epoch": 0.23, + "learning_rate": 0.00062083851554664, + "loss": 1.497, + "theoretical_loss": 3.7743909738292865, + "tokens_seen": 764739584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006208224674022066, + "loss": 1.4737, + "theoretical_loss": 3.7743588933036065, + "tokens_seen": 764805120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006208064192577734, + "loss": 1.3945, + "theoretical_loss": 3.774326816296421, + "tokens_seen": 764870656 + }, + { + "epoch": 0.23, + "learning_rate": 0.00062079037111334, + "loss": 1.39, + "theoretical_loss": 3.774294742807041, + "tokens_seen": 764936192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006207743229689068, + "loss": 1.4299, + "theoretical_loss": 3.774262672834781, + "tokens_seen": 765001728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006207582748244734, + "loss": 1.3708, + "theoretical_loss": 3.7742306063789526, + "tokens_seen": 765067264 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006207422266800402, + "loss": 1.4407, + "theoretical_loss": 3.77419854343887, + "tokens_seen": 765132800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006207261785356068, + "loss": 1.4233, + "theoretical_loss": 3.774166484013846, + "tokens_seen": 765198336 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006207101303911736, + "loss": 1.455, + "theoretical_loss": 3.774134428103195, + "tokens_seen": 765263872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006206940822467402, + "loss": 1.3979, + "theoretical_loss": 3.7741023757062306, + "tokens_seen": 765329408 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620678034102307, + "loss": 1.4455, + "theoretical_loss": 3.7740703268222666, + "tokens_seen": 765394944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006206619859578736, + "loss": 1.4531, + "theoretical_loss": 3.7740382814506175, + "tokens_seen": 765460480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006206459378134404, + "loss": 1.2891, + "theoretical_loss": 3.774006239590597, + "tokens_seen": 765526016 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620629889669007, + "loss": 1.3613, + "theoretical_loss": 3.773974201241521, + "tokens_seen": 765591552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006206138415245738, + "loss": 1.3559, + "theoretical_loss": 3.773942166402703, + "tokens_seen": 765657088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205977933801404, + "loss": 1.4377, + "theoretical_loss": 3.7739101350734585, + "tokens_seen": 765722624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205817452357072, + "loss": 1.3769, + "theoretical_loss": 3.7738781072531027, + "tokens_seen": 765788160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205656970912738, + "loss": 1.443, + "theoretical_loss": 3.7738460829409513, + "tokens_seen": 765853696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205496489468406, + "loss": 1.2463, + "theoretical_loss": 3.773814062136319, + "tokens_seen": 765919232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205336008024072, + "loss": 1.4535, + "theoretical_loss": 3.773782044838523, + "tokens_seen": 765984768 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620517552657974, + "loss": 1.3255, + "theoretical_loss": 3.7737500310468777, + "tokens_seen": 766050304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205015045135406, + "loss": 1.3543, + "theoretical_loss": 3.7737180207607004, + "tokens_seen": 766115840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006204854563691074, + "loss": 1.4537, + "theoretical_loss": 3.7736860139793063, + "tokens_seen": 766181376 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620469408224674, + "loss": 1.428, + "theoretical_loss": 3.773654010702013, + "tokens_seen": 766246912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006204533600802408, + "loss": 1.4351, + "theoretical_loss": 3.773622010928137, + "tokens_seen": 766312448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006204373119358074, + "loss": 1.4944, + "theoretical_loss": 3.7735900146569947, + "tokens_seen": 766377984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006204212637913742, + "loss": 1.5058, + "theoretical_loss": 3.7735580218879035, + "tokens_seen": 766443520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006204052156469408, + "loss": 1.2958, + "theoretical_loss": 3.7735260326201807, + "tokens_seen": 766509056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006203891675025076, + "loss": 1.2789, + "theoretical_loss": 3.773494046853144, + "tokens_seen": 766574592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006203731193580742, + "loss": 1.4017, + "theoretical_loss": 3.773462064586111, + "tokens_seen": 766640128 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620357071213641, + "loss": 1.4233, + "theoretical_loss": 3.773430085818399, + "tokens_seen": 766705664 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 261084, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5048065185546875, + "objective/train/theoretical_loss": 3.7733981105493273, + "objective/train/tokens_used": 787231200, + "theoretical_loss": 3.7733981105493273, + "tokens_seen": 766771200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006203410230692076, + "loss": 1.4697, + "theoretical_loss": 3.7733981105493273, + "tokens_seen": 766771200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006203249749247744, + "loss": 1.3867, + "theoretical_loss": 3.7733661387782123, + "tokens_seen": 766836736 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620308926780341, + "loss": 1.4101, + "theoretical_loss": 3.7733341705043744, + "tokens_seen": 766902272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202928786359078, + "loss": 1.3973, + "theoretical_loss": 3.7733022057271306, + "tokens_seen": 766967808 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202768304914744, + "loss": 1.4544, + "theoretical_loss": 3.7732702444458006, + "tokens_seen": 767033344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202607823470412, + "loss": 1.5491, + "theoretical_loss": 3.773238286659704, + "tokens_seen": 767098880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202447342026078, + "loss": 1.3937, + "theoretical_loss": 3.773206332368159, + "tokens_seen": 767164416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202286860581746, + "loss": 1.3987, + "theoretical_loss": 3.7731743815704846, + "tokens_seen": 767229952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202126379137412, + "loss": 1.3838, + "theoretical_loss": 3.7731424342660014, + "tokens_seen": 767295488 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620196589769308, + "loss": 1.3372, + "theoretical_loss": 3.7731104904540294, + "tokens_seen": 767361024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006201805416248746, + "loss": 1.3957, + "theoretical_loss": 3.7730785501338873, + "tokens_seen": 767426560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006201644934804414, + "loss": 1.4313, + "theoretical_loss": 3.773046613304896, + "tokens_seen": 767492096 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620148445336008, + "loss": 1.3817, + "theoretical_loss": 3.7730146799663764, + "tokens_seen": 767557632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006201323971915748, + "loss": 1.3762, + "theoretical_loss": 3.772982750117648, + "tokens_seen": 767623168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006201163490471414, + "loss": 1.5095, + "theoretical_loss": 3.772950823758032, + "tokens_seen": 767688704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006201003009027082, + "loss": 1.4444, + "theoretical_loss": 3.7729189008868493, + "tokens_seen": 767754240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006200842527582748, + "loss": 1.4159, + "theoretical_loss": 3.7728869815034214, + "tokens_seen": 767819776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006200682046138416, + "loss": 1.5091, + "theoretical_loss": 3.7728550656070685, + "tokens_seen": 767885312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006200521564694082, + "loss": 1.4459, + "theoretical_loss": 3.7728231531971135, + "tokens_seen": 767950848 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620036108324975, + "loss": 1.4411, + "theoretical_loss": 3.772791244272877, + "tokens_seen": 768016384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006200200601805416, + "loss": 1.5223, + "theoretical_loss": 3.7727593388336813, + "tokens_seen": 768081920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006200040120361084, + "loss": 1.4448, + "theoretical_loss": 3.7727274368788484, + "tokens_seen": 768147456 + }, + { + "epoch": 0.23, + "learning_rate": 0.000619987963891675, + "loss": 1.3049, + "theoretical_loss": 3.7726955384077008, + "tokens_seen": 768212992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006199719157472418, + "loss": 1.4686, + "theoretical_loss": 3.7726636434195604, + "tokens_seen": 768278528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006199558676028084, + "loss": 1.3818, + "theoretical_loss": 3.7726317519137504, + "tokens_seen": 768344064 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006199398194583752, + "loss": 1.401, + "theoretical_loss": 3.7725998638895932, + "tokens_seen": 768409600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006199237713139418, + "loss": 1.4325, + "theoretical_loss": 3.772567979346412, + "tokens_seen": 768475136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006199077231695086, + "loss": 1.43, + "theoretical_loss": 3.77253609828353, + "tokens_seen": 768540672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006198916750250752, + "loss": 1.4822, + "theoretical_loss": 3.77250422070027, + "tokens_seen": 768606208 + }, + { + "epoch": 0.23, + "learning_rate": 0.000619875626880642, + "loss": 1.4113, + "theoretical_loss": 3.772472346595957, + "tokens_seen": 768671744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006198595787362086, + "loss": 1.4157, + "theoretical_loss": 3.7724404759699137, + "tokens_seen": 768737280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006198435305917754, + "loss": 1.369, + "theoretical_loss": 3.772408608821464, + "tokens_seen": 768802816 + }, + { + "epoch": 0.23, + "learning_rate": 0.000619827482447342, + "loss": 1.4812, + "theoretical_loss": 3.7723767451499324, + "tokens_seen": 768868352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006198114343029088, + "loss": 1.4944, + "theoretical_loss": 3.772344884954643, + "tokens_seen": 768933888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006197953861584754, + "loss": 1.4708, + "theoretical_loss": 3.77231302823492, + "tokens_seen": 768999424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006197793380140422, + "loss": 1.4866, + "theoretical_loss": 3.772281174990089, + "tokens_seen": 769064960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006197632898696088, + "loss": 1.3638, + "theoretical_loss": 3.772249325219475, + "tokens_seen": 769130496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006197472417251756, + "loss": 1.5073, + "theoretical_loss": 3.7722174789224017, + "tokens_seen": 769196032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006197311935807422, + "loss": 1.3497, + "theoretical_loss": 3.7721856360981953, + "tokens_seen": 769261568 + }, + { + "epoch": 0.23, + "learning_rate": 0.000619715145436309, + "loss": 1.3609, + "theoretical_loss": 3.7721537967461813, + "tokens_seen": 769327104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196990972918757, + "loss": 1.4575, + "theoretical_loss": 3.7721219608656855, + "tokens_seen": 769392640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196830491474424, + "loss": 1.4422, + "theoretical_loss": 3.7720901284560333, + "tokens_seen": 769458176 + }, + { + "epoch": 0.23, + "learning_rate": 0.000619667001003009, + "loss": 1.5332, + "theoretical_loss": 3.7720582995165506, + "tokens_seen": 769523712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196509528585757, + "loss": 1.4122, + "theoretical_loss": 3.772026474046564, + "tokens_seen": 769589248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196349047141425, + "loss": 1.3979, + "theoretical_loss": 3.7719946520454, + "tokens_seen": 769654784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196188565697091, + "loss": 1.4553, + "theoretical_loss": 3.771962833512385, + "tokens_seen": 769720320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196028084252759, + "loss": 1.5624, + "theoretical_loss": 3.7719310184468458, + "tokens_seen": 769785856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195867602808425, + "loss": 1.4628, + "theoretical_loss": 3.771899206848109, + "tokens_seen": 769851392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195707121364093, + "loss": 1.4126, + "theoretical_loss": 3.7718673987155027, + "tokens_seen": 769916928 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195546639919759, + "loss": 1.3705, + "theoretical_loss": 3.771835594048353, + "tokens_seen": 769982464 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 261915, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4069135189056396, + "objective/train/theoretical_loss": 3.7718037928459887, + "objective/train/tokens_used": 790508000, + "theoretical_loss": 3.7718037928459887, + "tokens_seen": 770048000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195386158475427, + "loss": 1.4267, + "theoretical_loss": 3.7718037928459887, + "tokens_seen": 770048000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195225677031093, + "loss": 1.4309, + "theoretical_loss": 3.771771995107737, + "tokens_seen": 770113536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195065195586761, + "loss": 1.5265, + "theoretical_loss": 3.771740200832925, + "tokens_seen": 770179072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194904714142427, + "loss": 1.4222, + "theoretical_loss": 3.771708410020882, + "tokens_seen": 770244608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194744232698095, + "loss": 1.3883, + "theoretical_loss": 3.7716766226709355, + "tokens_seen": 770310144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194583751253761, + "loss": 1.4449, + "theoretical_loss": 3.7716448387824144, + "tokens_seen": 770375680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194423269809429, + "loss": 1.3884, + "theoretical_loss": 3.7716130583546477, + "tokens_seen": 770441216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194262788365095, + "loss": 1.3459, + "theoretical_loss": 3.771581281386963, + "tokens_seen": 770506752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194102306920763, + "loss": 1.369, + "theoretical_loss": 3.7715495078786905, + "tokens_seen": 770572288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193941825476429, + "loss": 1.3907, + "theoretical_loss": 3.771517737829159, + "tokens_seen": 770637824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193781344032097, + "loss": 1.4249, + "theoretical_loss": 3.7714859712376985, + "tokens_seen": 770703360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193620862587763, + "loss": 1.4423, + "theoretical_loss": 3.771454208103637, + "tokens_seen": 770768896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193460381143431, + "loss": 1.4135, + "theoretical_loss": 3.771422448426306, + "tokens_seen": 770834432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193299899699097, + "loss": 1.3792, + "theoretical_loss": 3.7713906922050344, + "tokens_seen": 770899968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193139418254765, + "loss": 1.4996, + "theoretical_loss": 3.771358939439153, + "tokens_seen": 770965504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192978936810431, + "loss": 1.4811, + "theoretical_loss": 3.771327190127992, + "tokens_seen": 771031040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192818455366099, + "loss": 1.4225, + "theoretical_loss": 3.7712954442708813, + "tokens_seen": 771096576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192657973921765, + "loss": 1.3833, + "theoretical_loss": 3.7712637018671527, + "tokens_seen": 771162112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192497492477433, + "loss": 1.3279, + "theoretical_loss": 3.7712319629161364, + "tokens_seen": 771227648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192337011033099, + "loss": 1.4951, + "theoretical_loss": 3.771200227417163, + "tokens_seen": 771293184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192176529588767, + "loss": 1.3993, + "theoretical_loss": 3.771168495369565, + "tokens_seen": 771358720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192016048144433, + "loss": 1.3692, + "theoretical_loss": 3.771136766772673, + "tokens_seen": 771424256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191855566700101, + "loss": 1.4408, + "theoretical_loss": 3.7711050416258187, + "tokens_seen": 771489792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191695085255767, + "loss": 1.3749, + "theoretical_loss": 3.771073319928335, + "tokens_seen": 771555328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191534603811435, + "loss": 1.4492, + "theoretical_loss": 3.7710416016795523, + "tokens_seen": 771620864 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191374122367101, + "loss": 1.3585, + "theoretical_loss": 3.7710098868788036, + "tokens_seen": 771686400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191213640922769, + "loss": 1.3964, + "theoretical_loss": 3.7709781755254212, + "tokens_seen": 771751936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191053159478435, + "loss": 1.3929, + "theoretical_loss": 3.770946467618738, + "tokens_seen": 771817472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190892678034103, + "loss": 1.3236, + "theoretical_loss": 3.7709147631580864, + "tokens_seen": 771883008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190732196589769, + "loss": 1.4498, + "theoretical_loss": 3.7708830621427993, + "tokens_seen": 771948544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190571715145437, + "loss": 1.3861, + "theoretical_loss": 3.77085136457221, + "tokens_seen": 772014080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190411233701103, + "loss": 1.3637, + "theoretical_loss": 3.7708196704456522, + "tokens_seen": 772079616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190250752256771, + "loss": 1.415, + "theoretical_loss": 3.770787979762458, + "tokens_seen": 772145152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190090270812437, + "loss": 1.5101, + "theoretical_loss": 3.770756292521963, + "tokens_seen": 772210688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189929789368105, + "loss": 1.4641, + "theoretical_loss": 3.7707246087235, + "tokens_seen": 772276224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189769307923771, + "loss": 1.4672, + "theoretical_loss": 3.770692928366403, + "tokens_seen": 772341760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189608826479439, + "loss": 1.5139, + "theoretical_loss": 3.7706612514500066, + "tokens_seen": 772407296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189448345035105, + "loss": 1.363, + "theoretical_loss": 3.770629577973645, + "tokens_seen": 772472832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189287863590773, + "loss": 1.5137, + "theoretical_loss": 3.7705979079366534, + "tokens_seen": 772538368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189127382146439, + "loss": 1.3931, + "theoretical_loss": 3.7705662413383654, + "tokens_seen": 772603904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188966900702107, + "loss": 1.3853, + "theoretical_loss": 3.7705345781781174, + "tokens_seen": 772669440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188806419257773, + "loss": 1.4165, + "theoretical_loss": 3.770502918455243, + "tokens_seen": 772734976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188645937813441, + "loss": 1.5651, + "theoretical_loss": 3.770471262169079, + "tokens_seen": 772800512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188485456369107, + "loss": 1.4451, + "theoretical_loss": 3.7704396093189607, + "tokens_seen": 772866048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188324974924775, + "loss": 1.4238, + "theoretical_loss": 3.7704079599042233, + "tokens_seen": 772931584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188164493480441, + "loss": 1.4405, + "theoretical_loss": 3.770376313924203, + "tokens_seen": 772997120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188004012036109, + "loss": 1.3301, + "theoretical_loss": 3.7703446713782354, + "tokens_seen": 773062656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187843530591775, + "loss": 1.4101, + "theoretical_loss": 3.7703130322656575, + "tokens_seen": 773128192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187683049147443, + "loss": 1.3621, + "theoretical_loss": 3.7702813965858053, + "tokens_seen": 773193728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187522567703109, + "loss": 1.4221, + "theoretical_loss": 3.7702497643380157, + "tokens_seen": 773259264 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 263140, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4101719856262207, + "objective/train/theoretical_loss": 3.7702181355216258, + "objective/train/tokens_used": 793784800, + "theoretical_loss": 3.7702181355216258, + "tokens_seen": 773324800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187362086258777, + "loss": 1.37, + "theoretical_loss": 3.7702181355216258, + "tokens_seen": 773324800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187201604814443, + "loss": 1.3152, + "theoretical_loss": 3.770186510135972, + "tokens_seen": 773390336 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187041123370111, + "loss": 1.4912, + "theoretical_loss": 3.7701548881803917, + "tokens_seen": 773455872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186880641925777, + "loss": 1.3692, + "theoretical_loss": 3.7701232696542224, + "tokens_seen": 773521408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186720160481445, + "loss": 1.4688, + "theoretical_loss": 3.7700916545568015, + "tokens_seen": 773586944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186559679037111, + "loss": 1.55, + "theoretical_loss": 3.7700600428874678, + "tokens_seen": 773652480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186399197592779, + "loss": 1.4596, + "theoretical_loss": 3.7700284346455577, + "tokens_seen": 773718016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186238716148445, + "loss": 1.4967, + "theoretical_loss": 3.7699968298304096, + "tokens_seen": 773783552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186078234704113, + "loss": 1.4364, + "theoretical_loss": 3.7699652284413627, + "tokens_seen": 773849088 + }, + { + "epoch": 0.23, + "learning_rate": 0.000618591775325978, + "loss": 1.437, + "theoretical_loss": 3.769933630477755, + "tokens_seen": 773914624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006185757271815447, + "loss": 1.4548, + "theoretical_loss": 3.769902035938925, + "tokens_seen": 773980160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006185596790371114, + "loss": 1.4301, + "theoretical_loss": 3.7698704448242113, + "tokens_seen": 774045696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006185436308926781, + "loss": 1.3961, + "theoretical_loss": 3.769838857132954, + "tokens_seen": 774111232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006185275827482449, + "loss": 1.544, + "theoretical_loss": 3.769807272864491, + "tokens_seen": 774176768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006185115346038115, + "loss": 1.5274, + "theoretical_loss": 3.7697756920181633, + "tokens_seen": 774242304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184954864593783, + "loss": 1.4408, + "theoretical_loss": 3.769744114593309, + "tokens_seen": 774307840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184794383149449, + "loss": 1.4132, + "theoretical_loss": 3.7697125405892686, + "tokens_seen": 774373376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184633901705117, + "loss": 1.4891, + "theoretical_loss": 3.769680970005382, + "tokens_seen": 774438912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184473420260783, + "loss": 1.4711, + "theoretical_loss": 3.769649402840989, + "tokens_seen": 774504448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184312938816451, + "loss": 1.4862, + "theoretical_loss": 3.7696178390954302, + "tokens_seen": 774569984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184152457372117, + "loss": 1.396, + "theoretical_loss": 3.7695862787680463, + "tokens_seen": 774635520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183991975927783, + "loss": 1.4648, + "theoretical_loss": 3.7695547218581775, + "tokens_seen": 774701056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183831494483451, + "loss": 1.4278, + "theoretical_loss": 3.769523168365165, + "tokens_seen": 774766592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183671013039117, + "loss": 1.3675, + "theoretical_loss": 3.7694916182883498, + "tokens_seen": 774832128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183510531594785, + "loss": 1.3466, + "theoretical_loss": 3.7694600716270736, + "tokens_seen": 774897664 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183350050150452, + "loss": 1.5027, + "theoretical_loss": 3.7694285283806774, + "tokens_seen": 774963200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183189568706119, + "loss": 1.3589, + "theoretical_loss": 3.769396988548502, + "tokens_seen": 775028736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183029087261786, + "loss": 1.4402, + "theoretical_loss": 3.7693654521298905, + "tokens_seen": 775094272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006182868605817453, + "loss": 1.4952, + "theoretical_loss": 3.7693339191241844, + "tokens_seen": 775159808 + }, + { + "epoch": 0.23, + "learning_rate": 0.000618270812437312, + "loss": 1.3878, + "theoretical_loss": 3.769302389530726, + "tokens_seen": 775225344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006182547642928786, + "loss": 1.4855, + "theoretical_loss": 3.769270863348857, + "tokens_seen": 775290880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006182387161484454, + "loss": 1.4652, + "theoretical_loss": 3.7692393405779208, + "tokens_seen": 775356416 + }, + { + "epoch": 0.23, + "learning_rate": 0.000618222668004012, + "loss": 1.4373, + "theoretical_loss": 3.7692078212172593, + "tokens_seen": 775421952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006182066198595788, + "loss": 1.4672, + "theoretical_loss": 3.7691763052662157, + "tokens_seen": 775487488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006181905717151454, + "loss": 1.4258, + "theoretical_loss": 3.7691447927241333, + "tokens_seen": 775553024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006181745235707122, + "loss": 1.4884, + "theoretical_loss": 3.769113283590355, + "tokens_seen": 775618560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006181584754262788, + "loss": 1.3293, + "theoretical_loss": 3.7690817778642245, + "tokens_seen": 775684096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006181424272818456, + "loss": 1.3215, + "theoretical_loss": 3.7690502755450854, + "tokens_seen": 775749632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006181263791374122, + "loss": 1.4058, + "theoretical_loss": 3.769018776632281, + "tokens_seen": 775815168 + }, + { + "epoch": 0.24, + "learning_rate": 0.000618110330992979, + "loss": 1.3601, + "theoretical_loss": 3.768987281125156, + "tokens_seen": 775880704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006180942828485456, + "loss": 1.3727, + "theoretical_loss": 3.7689557890230545, + "tokens_seen": 775946240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006180782347041124, + "loss": 1.4911, + "theoretical_loss": 3.7689243003253203, + "tokens_seen": 776011776 + }, + { + "epoch": 0.24, + "learning_rate": 0.000618062186559679, + "loss": 1.4389, + "theoretical_loss": 3.768892815031298, + "tokens_seen": 776077312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006180461384152458, + "loss": 1.4177, + "theoretical_loss": 3.768861333140333, + "tokens_seen": 776142848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006180300902708124, + "loss": 1.4762, + "theoretical_loss": 3.768829854651769, + "tokens_seen": 776208384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006180140421263792, + "loss": 1.478, + "theoretical_loss": 3.7687983795649522, + "tokens_seen": 776273920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006179979939819458, + "loss": 1.341, + "theoretical_loss": 3.768766907879227, + "tokens_seen": 776339456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006179819458375126, + "loss": 1.4228, + "theoretical_loss": 3.7687354395939394, + "tokens_seen": 776404992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006179658976930792, + "loss": 1.4807, + "theoretical_loss": 3.7687039747084348, + "tokens_seen": 776470528 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617949849548646, + "loss": 1.2837, + "theoretical_loss": 3.768672513222059, + "tokens_seen": 776536064 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 264403, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4671874046325684, + "objective/train/theoretical_loss": 3.7686410551341583, + "objective/train/tokens_used": 797061600, + "theoretical_loss": 3.7686410551341583, + "tokens_seen": 776601600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006179338014042126, + "loss": 1.3777, + "theoretical_loss": 3.7686410551341583, + "tokens_seen": 776601600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006179177532597794, + "loss": 1.4223, + "theoretical_loss": 3.768609600444078, + "tokens_seen": 776667136 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617901705115346, + "loss": 1.44, + "theoretical_loss": 3.768578149151165, + "tokens_seen": 776732672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178856569709128, + "loss": 1.3614, + "theoretical_loss": 3.768546701254766, + "tokens_seen": 776798208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178696088264794, + "loss": 1.3948, + "theoretical_loss": 3.7685152567542275, + "tokens_seen": 776863744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178535606820462, + "loss": 1.467, + "theoretical_loss": 3.7684838156488962, + "tokens_seen": 776929280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178375125376128, + "loss": 1.4427, + "theoretical_loss": 3.7684523779381194, + "tokens_seen": 776994816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178214643931796, + "loss": 1.4314, + "theoretical_loss": 3.768420943621244, + "tokens_seen": 777060352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178054162487462, + "loss": 1.4692, + "theoretical_loss": 3.7683895126976177, + "tokens_seen": 777125888 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617789368104313, + "loss": 1.49, + "theoretical_loss": 3.7683580851665885, + "tokens_seen": 777191424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006177733199598796, + "loss": 1.3269, + "theoretical_loss": 3.768326661027503, + "tokens_seen": 777256960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006177572718154464, + "loss": 1.4427, + "theoretical_loss": 3.7682952402797105, + "tokens_seen": 777322496 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617741223671013, + "loss": 1.4641, + "theoretical_loss": 3.7682638229225582, + "tokens_seen": 777388032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006177251755265798, + "loss": 1.3876, + "theoretical_loss": 3.7682324089553947, + "tokens_seen": 777453568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006177091273821464, + "loss": 1.4079, + "theoretical_loss": 3.768200998377569, + "tokens_seen": 777519104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006176930792377132, + "loss": 1.4262, + "theoretical_loss": 3.7681695911884288, + "tokens_seen": 777584640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006176770310932798, + "loss": 1.353, + "theoretical_loss": 3.768138187387324, + "tokens_seen": 777650176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006176609829488466, + "loss": 1.3663, + "theoretical_loss": 3.7681067869736022, + "tokens_seen": 777715712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006176449348044132, + "loss": 1.4153, + "theoretical_loss": 3.768075389946614, + "tokens_seen": 777781248 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061762888665998, + "loss": 1.4123, + "theoretical_loss": 3.7680439963057086, + "tokens_seen": 777846784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006176128385155466, + "loss": 1.5162, + "theoretical_loss": 3.7680126060502355, + "tokens_seen": 777912320 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175967903711134, + "loss": 1.41, + "theoretical_loss": 3.7679812191795437, + "tokens_seen": 777977856 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061758074222668, + "loss": 1.4208, + "theoretical_loss": 3.767949835692984, + "tokens_seen": 778043392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175646940822468, + "loss": 1.4099, + "theoretical_loss": 3.767918455589906, + "tokens_seen": 778108928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175486459378134, + "loss": 1.4533, + "theoretical_loss": 3.76788707886966, + "tokens_seen": 778174464 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175325977933802, + "loss": 1.3446, + "theoretical_loss": 3.7678557055315975, + "tokens_seen": 778240000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175165496489468, + "loss": 1.4287, + "theoretical_loss": 3.767824335575068, + "tokens_seen": 778305536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175005015045136, + "loss": 1.4309, + "theoretical_loss": 3.7677929689994225, + "tokens_seen": 778371072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006174844533600802, + "loss": 1.3957, + "theoretical_loss": 3.7677616058040124, + "tokens_seen": 778436608 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617468405215647, + "loss": 1.3862, + "theoretical_loss": 3.7677302459881883, + "tokens_seen": 778502144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006174523570712136, + "loss": 1.3104, + "theoretical_loss": 3.767698889551302, + "tokens_seen": 778567680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006174363089267804, + "loss": 1.3043, + "theoretical_loss": 3.7676675364927057, + "tokens_seen": 778633216 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617420260782347, + "loss": 1.4242, + "theoretical_loss": 3.76763618681175, + "tokens_seen": 778698752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006174042126379138, + "loss": 1.3909, + "theoretical_loss": 3.7676048405077873, + "tokens_seen": 778764288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006173881644934805, + "loss": 1.2704, + "theoretical_loss": 3.7675734975801696, + "tokens_seen": 778829824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006173721163490472, + "loss": 1.513, + "theoretical_loss": 3.7675421580282498, + "tokens_seen": 778895360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006173560682046139, + "loss": 1.3417, + "theoretical_loss": 3.7675108218513786, + "tokens_seen": 778960896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006173400200601806, + "loss": 1.3729, + "theoretical_loss": 3.7674794890489105, + "tokens_seen": 779026432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006173239719157473, + "loss": 1.3879, + "theoretical_loss": 3.767448159620198, + "tokens_seen": 779091968 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617307923771314, + "loss": 1.3719, + "theoretical_loss": 3.7674168335645932, + "tokens_seen": 779157504 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172918756268807, + "loss": 1.4471, + "theoretical_loss": 3.7673855108814496, + "tokens_seen": 779223040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172758274824474, + "loss": 1.3563, + "theoretical_loss": 3.7673541915701207, + "tokens_seen": 779288576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172597793380141, + "loss": 1.3779, + "theoretical_loss": 3.76732287562996, + "tokens_seen": 779354112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172437311935808, + "loss": 1.3935, + "theoretical_loss": 3.767291563060321, + "tokens_seen": 779419648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172276830491475, + "loss": 1.3951, + "theoretical_loss": 3.7672602538605586, + "tokens_seen": 779485184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172116349047142, + "loss": 1.4309, + "theoretical_loss": 3.7672289480300254, + "tokens_seen": 779550720 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617195586760281, + "loss": 1.4581, + "theoretical_loss": 3.7671976455680767, + "tokens_seen": 779616256 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006171795386158476, + "loss": 1.4006, + "theoretical_loss": 3.767166346474066, + "tokens_seen": 779681792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006171634904714144, + "loss": 1.4974, + "theoretical_loss": 3.7671350507473482, + "tokens_seen": 779747328 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617147442326981, + "loss": 1.4772, + "theoretical_loss": 3.7671037583872784, + "tokens_seen": 779812864 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 265321, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4230022430419922, + "objective/train/theoretical_loss": 3.767072469393211, + "objective/train/tokens_used": 800338400, + "theoretical_loss": 3.767072469393211, + "tokens_seen": 779878400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006171313941825478, + "loss": 1.4604, + "theoretical_loss": 3.767072469393211, + "tokens_seen": 779878400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006171153460381144, + "loss": 1.4413, + "theoretical_loss": 3.767041183764502, + "tokens_seen": 779943936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170992978936812, + "loss": 1.3497, + "theoretical_loss": 3.767009901500506, + "tokens_seen": 780009472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170832497492478, + "loss": 1.477, + "theoretical_loss": 3.766978622600579, + "tokens_seen": 780075008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170672016048146, + "loss": 1.4941, + "theoretical_loss": 3.7669473470640757, + "tokens_seen": 780140544 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170511534603812, + "loss": 1.3191, + "theoretical_loss": 3.7669160748903527, + "tokens_seen": 780206080 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617035105315948, + "loss": 1.3044, + "theoretical_loss": 3.766884806078766, + "tokens_seen": 780271616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170190571715146, + "loss": 1.394, + "theoretical_loss": 3.766853540628672, + "tokens_seen": 780337152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170030090270812, + "loss": 1.3806, + "theoretical_loss": 3.7668222785394256, + "tokens_seen": 780402688 + }, + { + "epoch": 0.24, + "learning_rate": 0.000616986960882648, + "loss": 1.421, + "theoretical_loss": 3.766791019810385, + "tokens_seen": 780468224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006169709127382146, + "loss": 1.3045, + "theoretical_loss": 3.7667597644409065, + "tokens_seen": 780533760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006169548645937814, + "loss": 1.4526, + "theoretical_loss": 3.7667285124303467, + "tokens_seen": 780599296 + }, + { + "epoch": 0.24, + "learning_rate": 0.000616938816449348, + "loss": 1.4164, + "theoretical_loss": 3.7666972637780627, + "tokens_seen": 780664832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006169227683049148, + "loss": 1.4292, + "theoretical_loss": 3.766666018483412, + "tokens_seen": 780730368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006169067201604815, + "loss": 1.4096, + "theoretical_loss": 3.766634776545752, + "tokens_seen": 780795904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168906720160482, + "loss": 1.431, + "theoretical_loss": 3.76660353796444, + "tokens_seen": 780861440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168746238716149, + "loss": 1.3841, + "theoretical_loss": 3.7665723027388336, + "tokens_seen": 780926976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168585757271815, + "loss": 1.3696, + "theoretical_loss": 3.7665410708682914, + "tokens_seen": 780992512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168425275827483, + "loss": 1.3269, + "theoretical_loss": 3.7665098423521712, + "tokens_seen": 781058048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168264794383149, + "loss": 1.2709, + "theoretical_loss": 3.766478617189832, + "tokens_seen": 781123584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168104312938817, + "loss": 1.3507, + "theoretical_loss": 3.766447395380631, + "tokens_seen": 781189120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167943831494483, + "loss": 1.4879, + "theoretical_loss": 3.7664161769239275, + "tokens_seen": 781254656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167783350050151, + "loss": 1.3632, + "theoretical_loss": 3.76638496181908, + "tokens_seen": 781320192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167622868605817, + "loss": 1.3393, + "theoretical_loss": 3.7663537500654485, + "tokens_seen": 781385728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167462387161485, + "loss": 1.5013, + "theoretical_loss": 3.766322541662391, + "tokens_seen": 781451264 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167301905717151, + "loss": 1.3433, + "theoretical_loss": 3.766291336609268, + "tokens_seen": 781516800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167141424272819, + "loss": 1.3584, + "theoretical_loss": 3.766260134905438, + "tokens_seen": 781582336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166980942828485, + "loss": 1.4709, + "theoretical_loss": 3.7662289365502617, + "tokens_seen": 781647872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166820461384153, + "loss": 1.3859, + "theoretical_loss": 3.766197741543098, + "tokens_seen": 781713408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166659979939819, + "loss": 1.3007, + "theoretical_loss": 3.766166549883308, + "tokens_seen": 781778944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166499498495487, + "loss": 1.415, + "theoretical_loss": 3.7661353615702504, + "tokens_seen": 781844480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166339017051153, + "loss": 1.3874, + "theoretical_loss": 3.7661041766032874, + "tokens_seen": 781910016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166178535606821, + "loss": 1.4317, + "theoretical_loss": 3.7660729949817786, + "tokens_seen": 781975552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166018054162487, + "loss": 1.3863, + "theoretical_loss": 3.7660418167050853, + "tokens_seen": 782041088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165857572718155, + "loss": 1.4142, + "theoretical_loss": 3.7660106417725676, + "tokens_seen": 782106624 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165697091273821, + "loss": 1.3271, + "theoretical_loss": 3.765979470183588, + "tokens_seen": 782172160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165536609829489, + "loss": 1.4316, + "theoretical_loss": 3.765948301937506, + "tokens_seen": 782237696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165376128385155, + "loss": 1.3685, + "theoretical_loss": 3.7659171370336844, + "tokens_seen": 782303232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165215646940823, + "loss": 1.283, + "theoretical_loss": 3.765885975471485, + "tokens_seen": 782368768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165055165496489, + "loss": 1.4852, + "theoretical_loss": 3.7658548172502684, + "tokens_seen": 782434304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164894684052157, + "loss": 1.4444, + "theoretical_loss": 3.7658236623693977, + "tokens_seen": 782499840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164734202607823, + "loss": 1.4744, + "theoretical_loss": 3.765792510828234, + "tokens_seen": 782565376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164573721163491, + "loss": 1.3762, + "theoretical_loss": 3.7657613626261415, + "tokens_seen": 782630912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164413239719157, + "loss": 1.3975, + "theoretical_loss": 3.765730217762481, + "tokens_seen": 782696448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164252758274825, + "loss": 1.3729, + "theoretical_loss": 3.765699076236616, + "tokens_seen": 782761984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164092276830491, + "loss": 1.4487, + "theoretical_loss": 3.7656679380479092, + "tokens_seen": 782827520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163931795386159, + "loss": 1.3424, + "theoretical_loss": 3.7656368031957235, + "tokens_seen": 782893056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163771313941825, + "loss": 1.4558, + "theoretical_loss": 3.765605671679422, + "tokens_seen": 782958592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163610832497493, + "loss": 1.3998, + "theoretical_loss": 3.765574543498369, + "tokens_seen": 783024128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163450351053159, + "loss": 1.3661, + "theoretical_loss": 3.765543418651927, + "tokens_seen": 783089664 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 266686, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4450281858444214, + "objective/train/theoretical_loss": 3.765512297139461, + "objective/train/tokens_used": 803615200, + "theoretical_loss": 3.765512297139461, + "tokens_seen": 783155200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163289869608827, + "loss": 1.3858, + "theoretical_loss": 3.765512297139461, + "tokens_seen": 783155200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163129388164493, + "loss": 1.5334, + "theoretical_loss": 3.765481178960333, + "tokens_seen": 783220736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162968906720161, + "loss": 1.3507, + "theoretical_loss": 3.765450064113909, + "tokens_seen": 783286272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162808425275827, + "loss": 1.3889, + "theoretical_loss": 3.7654189525995516, + "tokens_seen": 783351808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162647943831495, + "loss": 1.4549, + "theoretical_loss": 3.7653878444166273, + "tokens_seen": 783417344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162487462387161, + "loss": 1.4686, + "theoretical_loss": 3.765356739564499, + "tokens_seen": 783482880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162326980942829, + "loss": 1.4782, + "theoretical_loss": 3.7653256380425324, + "tokens_seen": 783548416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162166499498495, + "loss": 1.3945, + "theoretical_loss": 3.7652945398500917, + "tokens_seen": 783613952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162006018054163, + "loss": 1.3612, + "theoretical_loss": 3.7652634449865428, + "tokens_seen": 783679488 + }, + { + "epoch": 0.24, + "learning_rate": 0.000616184553660983, + "loss": 1.4655, + "theoretical_loss": 3.7652323534512506, + "tokens_seen": 783745024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006161685055165497, + "loss": 1.416, + "theoretical_loss": 3.765201265243581, + "tokens_seen": 783810560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006161524573721164, + "loss": 1.4005, + "theoretical_loss": 3.765170180362899, + "tokens_seen": 783876096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006161364092276831, + "loss": 1.4177, + "theoretical_loss": 3.7651390988085716, + "tokens_seen": 783941632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006161203610832498, + "loss": 1.3948, + "theoretical_loss": 3.765108020579963, + "tokens_seen": 784007168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006161043129388165, + "loss": 1.4268, + "theoretical_loss": 3.7650769456764417, + "tokens_seen": 784072704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006160882647943832, + "loss": 1.3296, + "theoretical_loss": 3.765045874097372, + "tokens_seen": 784138240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006160722166499499, + "loss": 1.5356, + "theoretical_loss": 3.7650148058421213, + "tokens_seen": 784203776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006160561685055166, + "loss": 1.4808, + "theoretical_loss": 3.7649837409100564, + "tokens_seen": 784269312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006160401203610833, + "loss": 1.4401, + "theoretical_loss": 3.7649526793005443, + "tokens_seen": 784334848 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061602407221665, + "loss": 1.4127, + "theoretical_loss": 3.7649216210129515, + "tokens_seen": 784400384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006160080240722167, + "loss": 1.5829, + "theoretical_loss": 3.764890566046646, + "tokens_seen": 784465920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159919759277834, + "loss": 1.3616, + "theoretical_loss": 3.764859514400994, + "tokens_seen": 784531456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159759277833501, + "loss": 1.495, + "theoretical_loss": 3.7648284660753646, + "tokens_seen": 784596992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159598796389168, + "loss": 1.45, + "theoretical_loss": 3.7647974210691246, + "tokens_seen": 784662528 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159438314944835, + "loss": 1.4143, + "theoretical_loss": 3.7647663793816424, + "tokens_seen": 784728064 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159277833500502, + "loss": 1.4681, + "theoretical_loss": 3.764735341012285, + "tokens_seen": 784793600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159117352056169, + "loss": 1.4675, + "theoretical_loss": 3.7647043059604224, + "tokens_seen": 784859136 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006158956870611836, + "loss": 1.5275, + "theoretical_loss": 3.7646732742254216, + "tokens_seen": 784924672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006158796389167503, + "loss": 1.3571, + "theoretical_loss": 3.764642245806652, + "tokens_seen": 784990208 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615863590772317, + "loss": 1.309, + "theoretical_loss": 3.7646112207034825, + "tokens_seen": 785055744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006158475426278837, + "loss": 1.3369, + "theoretical_loss": 3.764580198915281, + "tokens_seen": 785121280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006158314944834504, + "loss": 1.3798, + "theoretical_loss": 3.764549180441418, + "tokens_seen": 785186816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006158154463390171, + "loss": 1.2188, + "theoretical_loss": 3.7645181652812623, + "tokens_seen": 785252352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157993981945839, + "loss": 1.4434, + "theoretical_loss": 3.7644871534341826, + "tokens_seen": 785317888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157833500501505, + "loss": 1.4183, + "theoretical_loss": 3.7644561448995497, + "tokens_seen": 785383424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157673019057173, + "loss": 1.4196, + "theoretical_loss": 3.764425139676733, + "tokens_seen": 785448960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157512537612839, + "loss": 1.3556, + "theoretical_loss": 3.7643941377651022, + "tokens_seen": 785514496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157352056168507, + "loss": 1.4348, + "theoretical_loss": 3.7643631391640278, + "tokens_seen": 785580032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157191574724173, + "loss": 1.4476, + "theoretical_loss": 3.7643321438728803, + "tokens_seen": 785645568 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615703109327984, + "loss": 1.4187, + "theoretical_loss": 3.7643011518910297, + "tokens_seen": 785711104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156870611835507, + "loss": 1.4248, + "theoretical_loss": 3.764270163217847, + "tokens_seen": 785776640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156710130391175, + "loss": 1.5277, + "theoretical_loss": 3.764239177852703, + "tokens_seen": 785842176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156549648946841, + "loss": 1.3589, + "theoretical_loss": 3.764208195794969, + "tokens_seen": 785907712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156389167502509, + "loss": 1.4421, + "theoretical_loss": 3.7641772170440158, + "tokens_seen": 785973248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156228686058175, + "loss": 1.3881, + "theoretical_loss": 3.7641462415992155, + "tokens_seen": 786038784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156068204613843, + "loss": 1.3179, + "theoretical_loss": 3.7641152694599382, + "tokens_seen": 786104320 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006155907723169509, + "loss": 1.4547, + "theoretical_loss": 3.764084300625557, + "tokens_seen": 786169856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006155747241725176, + "loss": 1.3807, + "theoretical_loss": 3.7640533350954435, + "tokens_seen": 786235392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006155586760280843, + "loss": 1.3404, + "theoretical_loss": 3.7640223728689692, + "tokens_seen": 786300928 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615542627883651, + "loss": 1.4187, + "theoretical_loss": 3.763991413945507, + "tokens_seen": 786366464 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 267627, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3336844444274902, + "objective/train/theoretical_loss": 3.763960458324429, + "objective/train/tokens_used": 806892000, + "theoretical_loss": 3.763960458324429, + "tokens_seen": 786432000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006155265797392177, + "loss": 1.3624, + "theoretical_loss": 3.763960458324429, + "tokens_seen": 786432000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006155105315947844, + "loss": 1.373, + "theoretical_loss": 3.763929506005108, + "tokens_seen": 786497536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154944834503511, + "loss": 1.4639, + "theoretical_loss": 3.7638985569869163, + "tokens_seen": 786563072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154784353059178, + "loss": 1.4227, + "theoretical_loss": 3.7638676112692275, + "tokens_seen": 786628608 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154623871614844, + "loss": 1.4224, + "theoretical_loss": 3.7638366688514138, + "tokens_seen": 786694144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154463390170512, + "loss": 1.3557, + "theoretical_loss": 3.7638057297328493, + "tokens_seen": 786759680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154302908726178, + "loss": 1.4047, + "theoretical_loss": 3.7637747939129067, + "tokens_seen": 786825216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154142427281846, + "loss": 1.4713, + "theoretical_loss": 3.7637438613909606, + "tokens_seen": 786890752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006153981945837512, + "loss": 1.4039, + "theoretical_loss": 3.7637129321663836, + "tokens_seen": 786956288 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615382146439318, + "loss": 1.4927, + "theoretical_loss": 3.7636820062385503, + "tokens_seen": 787021824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006153660982948846, + "loss": 1.3567, + "theoretical_loss": 3.7636510836068355, + "tokens_seen": 787087360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006153500501504514, + "loss": 1.4755, + "theoretical_loss": 3.7636201642706117, + "tokens_seen": 787152896 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615334002006018, + "loss": 1.4107, + "theoretical_loss": 3.7635892482292554, + "tokens_seen": 787218432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006153179538615848, + "loss": 1.3844, + "theoretical_loss": 3.7635583354821396, + "tokens_seen": 787283968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006153019057171514, + "loss": 1.4169, + "theoretical_loss": 3.7635274260286398, + "tokens_seen": 787349504 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006152858575727182, + "loss": 1.5145, + "theoretical_loss": 3.763496519868131, + "tokens_seen": 787415040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006152698094282848, + "loss": 1.3417, + "theoretical_loss": 3.7634656169999885, + "tokens_seen": 787480576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006152537612838516, + "loss": 1.4822, + "theoretical_loss": 3.7634347174235874, + "tokens_seen": 787546112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006152377131394182, + "loss": 1.4409, + "theoretical_loss": 3.7634038211383025, + "tokens_seen": 787611648 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615221664994985, + "loss": 1.4063, + "theoretical_loss": 3.763372928143511, + "tokens_seen": 787677184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006152056168505516, + "loss": 1.5267, + "theoretical_loss": 3.763342038438587, + "tokens_seen": 787742720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006151895687061184, + "loss": 1.4283, + "theoretical_loss": 3.7633111520229083, + "tokens_seen": 787808256 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615173520561685, + "loss": 1.5008, + "theoretical_loss": 3.7632802688958495, + "tokens_seen": 787873792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006151574724172518, + "loss": 1.4748, + "theoretical_loss": 3.7632493890567877, + "tokens_seen": 787939328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006151414242728184, + "loss": 1.3924, + "theoretical_loss": 3.7632185125051, + "tokens_seen": 788004864 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006151253761283852, + "loss": 1.4558, + "theoretical_loss": 3.763187639240161, + "tokens_seen": 788070400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006151093279839518, + "loss": 1.4065, + "theoretical_loss": 3.7631567692613497, + "tokens_seen": 788135936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006150932798395186, + "loss": 1.4183, + "theoretical_loss": 3.763125902568042, + "tokens_seen": 788201472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006150772316950852, + "loss": 1.5066, + "theoretical_loss": 3.763095039159616, + "tokens_seen": 788267008 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615061183550652, + "loss": 1.4107, + "theoretical_loss": 3.7630641790354478, + "tokens_seen": 788332544 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006150451354062186, + "loss": 1.3547, + "theoretical_loss": 3.763033322194916, + "tokens_seen": 788398080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006150290872617854, + "loss": 1.4287, + "theoretical_loss": 3.763002468637398, + "tokens_seen": 788463616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006150130391173521, + "loss": 1.3749, + "theoretical_loss": 3.762971618362271, + "tokens_seen": 788529152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149969909729188, + "loss": 1.3979, + "theoretical_loss": 3.762940771368914, + "tokens_seen": 788594688 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149809428284855, + "loss": 1.4632, + "theoretical_loss": 3.762909927656705, + "tokens_seen": 788660224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149648946840522, + "loss": 1.4155, + "theoretical_loss": 3.7628790872250217, + "tokens_seen": 788725760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149488465396189, + "loss": 1.3951, + "theoretical_loss": 3.762848250073244, + "tokens_seen": 788791296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149327983951856, + "loss": 1.2936, + "theoretical_loss": 3.7628174162007486, + "tokens_seen": 788856832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149167502507523, + "loss": 1.2792, + "theoretical_loss": 3.762786585606916, + "tokens_seen": 788922368 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614900702106319, + "loss": 1.3986, + "theoretical_loss": 3.7627557582911253, + "tokens_seen": 788987904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148846539618857, + "loss": 1.3391, + "theoretical_loss": 3.7627249342527547, + "tokens_seen": 789053440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148686058174524, + "loss": 1.4647, + "theoretical_loss": 3.762694113491184, + "tokens_seen": 789118976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148525576730191, + "loss": 1.4506, + "theoretical_loss": 3.762663296005793, + "tokens_seen": 789184512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148365095285858, + "loss": 1.5645, + "theoretical_loss": 3.7626324817959613, + "tokens_seen": 789250048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148204613841525, + "loss": 1.399, + "theoretical_loss": 3.762601670861069, + "tokens_seen": 789315584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148044132397192, + "loss": 1.3029, + "theoretical_loss": 3.7625708632004953, + "tokens_seen": 789381120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006147883650952859, + "loss": 1.3806, + "theoretical_loss": 3.762540058813622, + "tokens_seen": 789446656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006147723169508526, + "loss": 1.3644, + "theoretical_loss": 3.762509257699828, + "tokens_seen": 789512192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006147562688064193, + "loss": 1.3883, + "theoretical_loss": 3.7624784598584946, + "tokens_seen": 789577728 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614740220661986, + "loss": 1.5156, + "theoretical_loss": 3.7624476652890024, + "tokens_seen": 789643264 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 268814, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2551066875457764, + "objective/train/theoretical_loss": 3.762416873990732, + "objective/train/tokens_used": 810168800, + "theoretical_loss": 3.762416873990732, + "tokens_seen": 789708800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006147241725175527, + "loss": 1.3459, + "theoretical_loss": 3.762416873990732, + "tokens_seen": 789708800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006147081243731194, + "loss": 1.4716, + "theoretical_loss": 3.7623860859630653, + "tokens_seen": 789774336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146920762286861, + "loss": 1.5881, + "theoretical_loss": 3.762355301205383, + "tokens_seen": 789839872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146760280842528, + "loss": 1.4598, + "theoretical_loss": 3.7623245197170663, + "tokens_seen": 789905408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146599799398195, + "loss": 1.2829, + "theoretical_loss": 3.7622937414974977, + "tokens_seen": 789970944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146439317953862, + "loss": 1.3983, + "theoretical_loss": 3.762262966546058, + "tokens_seen": 790036480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146278836509529, + "loss": 1.4769, + "theoretical_loss": 3.762232194862129, + "tokens_seen": 790102016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146118355065196, + "loss": 1.3794, + "theoretical_loss": 3.7622014264450936, + "tokens_seen": 790167552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006145957873620863, + "loss": 1.3926, + "theoretical_loss": 3.7621706612943333, + "tokens_seen": 790233088 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614579739217653, + "loss": 1.4794, + "theoretical_loss": 3.7621398994092314, + "tokens_seen": 790298624 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006145636910732197, + "loss": 1.5054, + "theoretical_loss": 3.76210914078917, + "tokens_seen": 790364160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006145476429287864, + "loss": 1.4222, + "theoretical_loss": 3.762078385433531, + "tokens_seen": 790429696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006145315947843531, + "loss": 1.3232, + "theoretical_loss": 3.7620476333416994, + "tokens_seen": 790495232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006145155466399198, + "loss": 1.4588, + "theoretical_loss": 3.7620168845130566, + "tokens_seen": 790560768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006144994984954865, + "loss": 1.3283, + "theoretical_loss": 3.761986138946986, + "tokens_seen": 790626304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006144834503510532, + "loss": 1.3703, + "theoretical_loss": 3.761955396642872, + "tokens_seen": 790691840 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061446740220662, + "loss": 1.3067, + "theoretical_loss": 3.761924657600097, + "tokens_seen": 790757376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006144513540621866, + "loss": 1.2759, + "theoretical_loss": 3.7618939218180456, + "tokens_seen": 790822912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006144353059177534, + "loss": 1.4833, + "theoretical_loss": 3.7618631892961014, + "tokens_seen": 790888448 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061441925777332, + "loss": 1.4231, + "theoretical_loss": 3.7618324600336486, + "tokens_seen": 790953984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006144032096288868, + "loss": 1.3073, + "theoretical_loss": 3.7618017340300716, + "tokens_seen": 791019520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006143871614844534, + "loss": 1.3236, + "theoretical_loss": 3.761771011284755, + "tokens_seen": 791085056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006143711133400202, + "loss": 1.415, + "theoretical_loss": 3.761740291797082, + "tokens_seen": 791150592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006143550651955868, + "loss": 1.3706, + "theoretical_loss": 3.7617095755664396, + "tokens_seen": 791216128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006143390170511536, + "loss": 1.4312, + "theoretical_loss": 3.761678862592211, + "tokens_seen": 791281664 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006143229689067202, + "loss": 1.4508, + "theoretical_loss": 3.761648152873782, + "tokens_seen": 791347200 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614306920762287, + "loss": 1.3756, + "theoretical_loss": 3.761617446410538, + "tokens_seen": 791412736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006142908726178536, + "loss": 1.4036, + "theoretical_loss": 3.761586743201864, + "tokens_seen": 791478272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006142748244734204, + "loss": 1.3711, + "theoretical_loss": 3.761556043247146, + "tokens_seen": 791543808 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614258776328987, + "loss": 1.4405, + "theoretical_loss": 3.7615253465457696, + "tokens_seen": 791609344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006142427281845538, + "loss": 1.4056, + "theoretical_loss": 3.7614946530971207, + "tokens_seen": 791674880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006142266800401204, + "loss": 1.4189, + "theoretical_loss": 3.7614639629005855, + "tokens_seen": 791740416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006142106318956872, + "loss": 1.4482, + "theoretical_loss": 3.76143327595555, + "tokens_seen": 791805952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141945837512538, + "loss": 1.4486, + "theoretical_loss": 3.761402592261401, + "tokens_seen": 791871488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141785356068205, + "loss": 1.5224, + "theoretical_loss": 3.761371911817525, + "tokens_seen": 791937024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141624874623872, + "loss": 1.3956, + "theoretical_loss": 3.761341234623309, + "tokens_seen": 792002560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141464393179539, + "loss": 1.3544, + "theoretical_loss": 3.7613105606781394, + "tokens_seen": 792068096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141303911735206, + "loss": 1.4256, + "theoretical_loss": 3.7612798899814037, + "tokens_seen": 792133632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141143430290873, + "loss": 1.3331, + "theoretical_loss": 3.7612492225324887, + "tokens_seen": 792199168 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614098294884654, + "loss": 1.4264, + "theoretical_loss": 3.761218558330783, + "tokens_seen": 792264704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140822467402207, + "loss": 1.4669, + "theoretical_loss": 3.761187897375673, + "tokens_seen": 792330240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140661985957874, + "loss": 1.468, + "theoretical_loss": 3.761157239666547, + "tokens_seen": 792395776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140501504513541, + "loss": 1.4272, + "theoretical_loss": 3.761126585202792, + "tokens_seen": 792461312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140341023069207, + "loss": 1.4201, + "theoretical_loss": 3.761095933983798, + "tokens_seen": 792526848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140180541624875, + "loss": 1.4428, + "theoretical_loss": 3.7610652860089515, + "tokens_seen": 792592384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140020060180541, + "loss": 1.3483, + "theoretical_loss": 3.761034641277642, + "tokens_seen": 792657920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139859578736209, + "loss": 1.4491, + "theoretical_loss": 3.7610039997892577, + "tokens_seen": 792723456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139699097291875, + "loss": 1.4242, + "theoretical_loss": 3.7609733615431873, + "tokens_seen": 792788992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139538615847543, + "loss": 1.3877, + "theoretical_loss": 3.76094272653882, + "tokens_seen": 792854528 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139378134403209, + "loss": 1.3439, + "theoretical_loss": 3.760912094775544, + "tokens_seen": 792920064 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 269715, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.299852728843689, + "objective/train/theoretical_loss": 3.7608814662527505, + "objective/train/tokens_used": 813445600, + "theoretical_loss": 3.7608814662527505, + "tokens_seen": 792985600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139217652958877, + "loss": 1.391, + "theoretical_loss": 3.7608814662527505, + "tokens_seen": 792985600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139057171514543, + "loss": 1.3127, + "theoretical_loss": 3.7608508409698267, + "tokens_seen": 793051136 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138896690070211, + "loss": 1.3984, + "theoretical_loss": 3.7608202189261637, + "tokens_seen": 793116672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138736208625877, + "loss": 1.3009, + "theoretical_loss": 3.760789600121151, + "tokens_seen": 793182208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138575727181545, + "loss": 1.4481, + "theoretical_loss": 3.760758984554178, + "tokens_seen": 793247744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138415245737211, + "loss": 1.4492, + "theoretical_loss": 3.7607283722246345, + "tokens_seen": 793313280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138254764292879, + "loss": 1.4088, + "theoretical_loss": 3.760697763131912, + "tokens_seen": 793378816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138094282848546, + "loss": 1.4589, + "theoretical_loss": 3.7606671572754005, + "tokens_seen": 793444352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006137933801404213, + "loss": 1.3007, + "theoretical_loss": 3.7606365546544898, + "tokens_seen": 793509888 + }, + { + "epoch": 0.24, + "learning_rate": 0.000613777331995988, + "loss": 1.3536, + "theoretical_loss": 3.7606059552685713, + "tokens_seen": 793575424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006137612838515547, + "loss": 1.5259, + "theoretical_loss": 3.760575359117036, + "tokens_seen": 793640960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006137452357071214, + "loss": 1.3579, + "theoretical_loss": 3.760544766199275, + "tokens_seen": 793706496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006137291875626881, + "loss": 1.3601, + "theoretical_loss": 3.760514176514679, + "tokens_seen": 793772032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006137131394182548, + "loss": 1.3715, + "theoretical_loss": 3.7604835900626394, + "tokens_seen": 793837568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136970912738215, + "loss": 1.3638, + "theoretical_loss": 3.760453006842549, + "tokens_seen": 793903104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136810431293882, + "loss": 1.5297, + "theoretical_loss": 3.7604224268537982, + "tokens_seen": 793968640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136649949849549, + "loss": 1.4525, + "theoretical_loss": 3.7603918500957794, + "tokens_seen": 794034176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136489468405216, + "loss": 1.4023, + "theoretical_loss": 3.7603612765678847, + "tokens_seen": 794099712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136328986960883, + "loss": 1.4335, + "theoretical_loss": 3.7603307062695066, + "tokens_seen": 794165248 + }, + { + "epoch": 0.24, + "learning_rate": 0.000613616850551655, + "loss": 1.4526, + "theoretical_loss": 3.7603001392000364, + "tokens_seen": 794230784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136008024072217, + "loss": 1.3654, + "theoretical_loss": 3.760269575358868, + "tokens_seen": 794296320 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135847542627884, + "loss": 1.3988, + "theoretical_loss": 3.7602390147453937, + "tokens_seen": 794361856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135687061183551, + "loss": 1.4066, + "theoretical_loss": 3.760208457359006, + "tokens_seen": 794427392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135526579739218, + "loss": 1.3623, + "theoretical_loss": 3.7601779031990983, + "tokens_seen": 794492928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135366098294885, + "loss": 1.4173, + "theoretical_loss": 3.7601473522650637, + "tokens_seen": 794558464 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135205616850552, + "loss": 1.3934, + "theoretical_loss": 3.7601168045562954, + "tokens_seen": 794624000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135045135406219, + "loss": 1.3386, + "theoretical_loss": 3.760086260072188, + "tokens_seen": 794689536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006134884653961886, + "loss": 1.5199, + "theoretical_loss": 3.7600557188121337, + "tokens_seen": 794755072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006134724172517553, + "loss": 1.4429, + "theoretical_loss": 3.7600251807755276, + "tokens_seen": 794820608 + }, + { + "epoch": 0.24, + "learning_rate": 0.000613456369107322, + "loss": 1.4271, + "theoretical_loss": 3.759994645961762, + "tokens_seen": 794886144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006134403209628887, + "loss": 1.4422, + "theoretical_loss": 3.759964114370234, + "tokens_seen": 794951680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006134242728184554, + "loss": 1.3336, + "theoretical_loss": 3.759933586000335, + "tokens_seen": 795017216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006134082246740221, + "loss": 1.4427, + "theoretical_loss": 3.759903060851461, + "tokens_seen": 795082752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133921765295888, + "loss": 1.3202, + "theoretical_loss": 3.759872538923007, + "tokens_seen": 795148288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133761283851555, + "loss": 1.3186, + "theoretical_loss": 3.759842020214367, + "tokens_seen": 795213824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133600802407222, + "loss": 1.332, + "theoretical_loss": 3.7598115047249365, + "tokens_seen": 795279360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133440320962889, + "loss": 1.3651, + "theoretical_loss": 3.759780992454111, + "tokens_seen": 795344896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133279839518556, + "loss": 1.4328, + "theoretical_loss": 3.7597504834012847, + "tokens_seen": 795410432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133119358074223, + "loss": 1.423, + "theoretical_loss": 3.7597199775658545, + "tokens_seen": 795475968 + }, + { + "epoch": 0.24, + "learning_rate": 0.000613295887662989, + "loss": 1.3496, + "theoretical_loss": 3.7596894749472147, + "tokens_seen": 795541504 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006132798395185557, + "loss": 1.4071, + "theoretical_loss": 3.7596589755447622, + "tokens_seen": 795607040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006132637913741224, + "loss": 1.3789, + "theoretical_loss": 3.759628479357893, + "tokens_seen": 795672576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006132477432296891, + "loss": 1.4153, + "theoretical_loss": 3.7595979863860025, + "tokens_seen": 795738112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006132316950852558, + "loss": 1.4625, + "theoretical_loss": 3.759567496628488, + "tokens_seen": 795803648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006132156469408225, + "loss": 1.4996, + "theoretical_loss": 3.759537010084745, + "tokens_seen": 795869184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131995987963892, + "loss": 1.4998, + "theoretical_loss": 3.759506526754171, + "tokens_seen": 795934720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131835506519559, + "loss": 1.3282, + "theoretical_loss": 3.759476046636162, + "tokens_seen": 796000256 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131675025075226, + "loss": 1.3505, + "theoretical_loss": 3.759445569730115, + "tokens_seen": 796065792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131514543630893, + "loss": 1.4525, + "theoretical_loss": 3.7594150960354282, + "tokens_seen": 796131328 + }, + { + "epoch": 0.24, + "learning_rate": 0.000613135406218656, + "loss": 1.4226, + "theoretical_loss": 3.7593846255514984, + "tokens_seen": 796196864 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 270870, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3770694732666016, + "objective/train/theoretical_loss": 3.7593541582777226, + "objective/train/tokens_used": 816722400, + "theoretical_loss": 3.7593541582777226, + "tokens_seen": 796262400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131193580742227, + "loss": 1.3837, + "theoretical_loss": 3.7593541582777226, + "tokens_seen": 796262400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131033099297894, + "loss": 1.2927, + "theoretical_loss": 3.7593236942134993, + "tokens_seen": 796327936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130872617853561, + "loss": 1.3862, + "theoretical_loss": 3.7592932333582247, + "tokens_seen": 796393472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130712136409228, + "loss": 1.4526, + "theoretical_loss": 3.759262775711299, + "tokens_seen": 796459008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130551654964895, + "loss": 1.2557, + "theoretical_loss": 3.759232321272118, + "tokens_seen": 796524544 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130391173520563, + "loss": 1.3715, + "theoretical_loss": 3.7592018700400818, + "tokens_seen": 796590080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130230692076229, + "loss": 1.32, + "theoretical_loss": 3.759171422014588, + "tokens_seen": 796655616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130070210631897, + "loss": 1.4967, + "theoretical_loss": 3.7591409771950355, + "tokens_seen": 796721152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006129909729187563, + "loss": 1.4658, + "theoretical_loss": 3.759110535580823, + "tokens_seen": 796786688 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612974924774323, + "loss": 1.3089, + "theoretical_loss": 3.759080097171349, + "tokens_seen": 796852224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006129588766298897, + "loss": 1.4287, + "theoretical_loss": 3.7590496619660136, + "tokens_seen": 796917760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006129428284854565, + "loss": 1.4344, + "theoretical_loss": 3.759019229964215, + "tokens_seen": 796983296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006129267803410231, + "loss": 1.3833, + "theoretical_loss": 3.758988801165353, + "tokens_seen": 797048832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006129107321965899, + "loss": 1.4713, + "theoretical_loss": 3.7589583755688274, + "tokens_seen": 797114368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128946840521565, + "loss": 1.4555, + "theoretical_loss": 3.7589279531740374, + "tokens_seen": 797179904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128786359077233, + "loss": 1.2965, + "theoretical_loss": 3.7588975339803836, + "tokens_seen": 797245440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128625877632899, + "loss": 1.4055, + "theoretical_loss": 3.7588671179872657, + "tokens_seen": 797310976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128465396188567, + "loss": 1.3899, + "theoretical_loss": 3.7588367051940836, + "tokens_seen": 797376512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128304914744233, + "loss": 1.3753, + "theoretical_loss": 3.7588062956002384, + "tokens_seen": 797442048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128144433299901, + "loss": 1.314, + "theoretical_loss": 3.75877588920513, + "tokens_seen": 797507584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127983951855567, + "loss": 1.4176, + "theoretical_loss": 3.7587454860081597, + "tokens_seen": 797573120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127823470411234, + "loss": 1.5011, + "theoretical_loss": 3.7587150860087277, + "tokens_seen": 797638656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127662988966901, + "loss": 1.4468, + "theoretical_loss": 3.7586846892062358, + "tokens_seen": 797704192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127502507522568, + "loss": 1.3313, + "theoretical_loss": 3.758654295600085, + "tokens_seen": 797769728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127342026078235, + "loss": 1.3622, + "theoretical_loss": 3.758623905189676, + "tokens_seen": 797835264 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127181544633902, + "loss": 1.5191, + "theoretical_loss": 3.758593517974411, + "tokens_seen": 797900800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127021063189569, + "loss": 1.2989, + "theoretical_loss": 3.7585631339536913, + "tokens_seen": 797966336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006126860581745236, + "loss": 1.3811, + "theoretical_loss": 3.7585327531269197, + "tokens_seen": 798031872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006126700100300903, + "loss": 1.4147, + "theoretical_loss": 3.7585023754934963, + "tokens_seen": 798097408 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612653961885657, + "loss": 1.3622, + "theoretical_loss": 3.7584720010528248, + "tokens_seen": 798162944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006126379137412236, + "loss": 1.4507, + "theoretical_loss": 3.7584416298043073, + "tokens_seen": 798228480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006126218655967904, + "loss": 1.3998, + "theoretical_loss": 3.758411261747346, + "tokens_seen": 798294016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006126058174523571, + "loss": 1.3518, + "theoretical_loss": 3.7583808968813437, + "tokens_seen": 798359552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125897693079238, + "loss": 1.3929, + "theoretical_loss": 3.758350535205703, + "tokens_seen": 798425088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125737211634905, + "loss": 1.3411, + "theoretical_loss": 3.7583201767198267, + "tokens_seen": 798490624 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125576730190572, + "loss": 1.3693, + "theoretical_loss": 3.7582898214231184, + "tokens_seen": 798556160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125416248746239, + "loss": 1.3981, + "theoretical_loss": 3.7582594693149813, + "tokens_seen": 798621696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125255767301906, + "loss": 1.3908, + "theoretical_loss": 3.7582291203948195, + "tokens_seen": 798687232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125095285857573, + "loss": 1.4559, + "theoretical_loss": 3.758198774662035, + "tokens_seen": 798752768 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612493480441324, + "loss": 1.3672, + "theoretical_loss": 3.7581684321160322, + "tokens_seen": 798818304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006124774322968907, + "loss": 1.3338, + "theoretical_loss": 3.758138092756216, + "tokens_seen": 798883840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006124613841524574, + "loss": 1.3337, + "theoretical_loss": 3.75810775658199, + "tokens_seen": 798949376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006124453360080241, + "loss": 1.3231, + "theoretical_loss": 3.7580774235927574, + "tokens_seen": 799014912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006124292878635908, + "loss": 1.3307, + "theoretical_loss": 3.7580470937879236, + "tokens_seen": 799080448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006124132397191575, + "loss": 1.3038, + "theoretical_loss": 3.758016767166893, + "tokens_seen": 799145984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123971915747242, + "loss": 1.3889, + "theoretical_loss": 3.757986443729071, + "tokens_seen": 799211520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123811434302909, + "loss": 1.4174, + "theoretical_loss": 3.7579561234738614, + "tokens_seen": 799277056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123650952858576, + "loss": 1.4158, + "theoretical_loss": 3.75792580640067, + "tokens_seen": 799342592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123490471414243, + "loss": 1.4023, + "theoretical_loss": 3.7578954925089008, + "tokens_seen": 799408128 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612332998996991, + "loss": 1.318, + "theoretical_loss": 3.757865181797961, + "tokens_seen": 799473664 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 271751, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4218894243240356, + "objective/train/theoretical_loss": 3.757834874267255, + "objective/train/tokens_used": 819999200, + "theoretical_loss": 3.757834874267255, + "tokens_seen": 799539200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123169508525577, + "loss": 1.4056, + "theoretical_loss": 3.757834874267255, + "tokens_seen": 799539200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123009027081244, + "loss": 1.3216, + "theoretical_loss": 3.757804569916188, + "tokens_seen": 799604736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122848545636911, + "loss": 1.3733, + "theoretical_loss": 3.7577742687441678, + "tokens_seen": 799670272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122688064192578, + "loss": 1.3942, + "theoretical_loss": 3.757743970750598, + "tokens_seen": 799735808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122527582748245, + "loss": 1.3493, + "theoretical_loss": 3.7577136759348866, + "tokens_seen": 799801344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122367101303912, + "loss": 1.4403, + "theoretical_loss": 3.757683384296439, + "tokens_seen": 799866880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122206619859579, + "loss": 1.4301, + "theoretical_loss": 3.7576530958346623, + "tokens_seen": 799932416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122046138415246, + "loss": 1.4529, + "theoretical_loss": 3.7576228105489626, + "tokens_seen": 799997952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006121885656970913, + "loss": 1.4253, + "theoretical_loss": 3.757592528438747, + "tokens_seen": 800063488 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612172517552658, + "loss": 1.4993, + "theoretical_loss": 3.7575622495034224, + "tokens_seen": 800129024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006121564694082247, + "loss": 1.4272, + "theoretical_loss": 3.7575319737423953, + "tokens_seen": 800194560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006121404212637914, + "loss": 1.3275, + "theoretical_loss": 3.757501701155074, + "tokens_seen": 800260096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006121243731193581, + "loss": 1.4945, + "theoretical_loss": 3.757471431740866, + "tokens_seen": 800325632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006121083249749248, + "loss": 1.4018, + "theoretical_loss": 3.757441165499178, + "tokens_seen": 800391168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006120922768304915, + "loss": 1.5022, + "theoretical_loss": 3.7574109024294184, + "tokens_seen": 800456704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006120762286860582, + "loss": 1.3525, + "theoretical_loss": 3.757380642530995, + "tokens_seen": 800522240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006120601805416249, + "loss": 1.4784, + "theoretical_loss": 3.7573503858033157, + "tokens_seen": 800587776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006120441323971916, + "loss": 1.5019, + "theoretical_loss": 3.7573201322457885, + "tokens_seen": 800653312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006120280842527583, + "loss": 1.4038, + "theoretical_loss": 3.757289881857823, + "tokens_seen": 800718848 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612012036108325, + "loss": 1.4068, + "theoretical_loss": 3.7572596346388263, + "tokens_seen": 800784384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119959879638917, + "loss": 1.2788, + "theoretical_loss": 3.7572293905882077, + "tokens_seen": 800849920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119799398194584, + "loss": 1.3467, + "theoretical_loss": 3.7571991497053765, + "tokens_seen": 800915456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119638916750251, + "loss": 1.4457, + "theoretical_loss": 3.757168911989741, + "tokens_seen": 800980992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119478435305918, + "loss": 1.3659, + "theoretical_loss": 3.7571386774407114, + "tokens_seen": 801046528 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119317953861585, + "loss": 1.3445, + "theoretical_loss": 3.757108446057696, + "tokens_seen": 801112064 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119157472417252, + "loss": 1.3314, + "theoretical_loss": 3.7570782178401045, + "tokens_seen": 801177600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118996990972919, + "loss": 1.4833, + "theoretical_loss": 3.757047992787347, + "tokens_seen": 801243136 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118836509528586, + "loss": 1.3912, + "theoretical_loss": 3.757017770898833, + "tokens_seen": 801308672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118676028084253, + "loss": 1.3906, + "theoretical_loss": 3.756987552173973, + "tokens_seen": 801374208 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611851554663992, + "loss": 1.3971, + "theoretical_loss": 3.7569573366121762, + "tokens_seen": 801439744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118355065195587, + "loss": 1.3192, + "theoretical_loss": 3.7569271242128535, + "tokens_seen": 801505280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118194583751254, + "loss": 1.398, + "theoretical_loss": 3.7568969149754157, + "tokens_seen": 801570816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118034102306921, + "loss": 1.4638, + "theoretical_loss": 3.756866708899273, + "tokens_seen": 801636352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006117873620862588, + "loss": 1.4404, + "theoretical_loss": 3.756836505983836, + "tokens_seen": 801701888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006117713139418255, + "loss": 1.3786, + "theoretical_loss": 3.756806306228516, + "tokens_seen": 801767424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006117552657973922, + "loss": 1.3963, + "theoretical_loss": 3.7567761096327237, + "tokens_seen": 801832960 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611739217652959, + "loss": 1.3476, + "theoretical_loss": 3.7567459161958707, + "tokens_seen": 801898496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006117231695085256, + "loss": 1.4406, + "theoretical_loss": 3.7567157259173682, + "tokens_seen": 801964032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006117071213640923, + "loss": 1.3238, + "theoretical_loss": 3.756685538796628, + "tokens_seen": 802029568 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611691073219659, + "loss": 1.337, + "theoretical_loss": 3.756655354833062, + "tokens_seen": 802095104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006116750250752258, + "loss": 1.4085, + "theoretical_loss": 3.756625174026081, + "tokens_seen": 802160640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006116589769307924, + "loss": 1.3654, + "theoretical_loss": 3.756594996375098, + "tokens_seen": 802226176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006116429287863592, + "loss": 1.4613, + "theoretical_loss": 3.7565648218795253, + "tokens_seen": 802291712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006116268806419258, + "loss": 1.3568, + "theoretical_loss": 3.756534650538775, + "tokens_seen": 802357248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006116108324974926, + "loss": 1.348, + "theoretical_loss": 3.7565044823522595, + "tokens_seen": 802422784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006115947843530592, + "loss": 1.363, + "theoretical_loss": 3.7564743173193915, + "tokens_seen": 802488320 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611578736208626, + "loss": 1.4044, + "theoretical_loss": 3.756444155439584, + "tokens_seen": 802553856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006115626880641926, + "loss": 1.5253, + "theoretical_loss": 3.7564139967122494, + "tokens_seen": 802619392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006115466399197594, + "loss": 1.3278, + "theoretical_loss": 3.7563838411368016, + "tokens_seen": 802684928 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611530591775326, + "loss": 1.4076, + "theoretical_loss": 3.756353688712654, + "tokens_seen": 802750464 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 272906, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2556822299957275, + "objective/train/theoretical_loss": 3.7563235394392187, + "objective/train/tokens_used": 823276000, + "theoretical_loss": 3.7563235394392187, + "tokens_seen": 802816000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006115145436308928, + "loss": 1.3715, + "theoretical_loss": 3.7563235394392187, + "tokens_seen": 802816000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114984954864594, + "loss": 1.437, + "theoretical_loss": 3.756293393315911, + "tokens_seen": 802881536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114824473420262, + "loss": 1.4397, + "theoretical_loss": 3.7562632503421436, + "tokens_seen": 802947072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114663991975928, + "loss": 1.4542, + "theoretical_loss": 3.7562331105173308, + "tokens_seen": 803012608 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114503510531596, + "loss": 1.2796, + "theoretical_loss": 3.756202973840886, + "tokens_seen": 803078144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114343029087262, + "loss": 1.4424, + "theoretical_loss": 3.7561728403122254, + "tokens_seen": 803143680 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611418254764293, + "loss": 1.3938, + "theoretical_loss": 3.7561427099307614, + "tokens_seen": 803209216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114022066198596, + "loss": 1.3613, + "theoretical_loss": 3.756112582695909, + "tokens_seen": 803274752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006113861584754263, + "loss": 1.4798, + "theoretical_loss": 3.7560824586070836, + "tokens_seen": 803340288 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611370110330993, + "loss": 1.4933, + "theoretical_loss": 3.756052337663699, + "tokens_seen": 803405824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006113540621865597, + "loss": 1.3328, + "theoretical_loss": 3.756022219865171, + "tokens_seen": 803471360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006113380140421264, + "loss": 1.4218, + "theoretical_loss": 3.7559921052109155, + "tokens_seen": 803536896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006113219658976931, + "loss": 1.4613, + "theoretical_loss": 3.7559619937003457, + "tokens_seen": 803602432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006113059177532598, + "loss": 1.3645, + "theoretical_loss": 3.755931885332879, + "tokens_seen": 803667968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006112898696088265, + "loss": 1.3942, + "theoretical_loss": 3.75590178010793, + "tokens_seen": 803733504 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006112738214643932, + "loss": 1.3609, + "theoretical_loss": 3.7558716780249153, + "tokens_seen": 803799040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006112577733199599, + "loss": 1.3726, + "theoretical_loss": 3.75584157908325, + "tokens_seen": 803864576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006112417251755266, + "loss": 1.3219, + "theoretical_loss": 3.755811483282351, + "tokens_seen": 803930112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006112256770310933, + "loss": 1.3523, + "theoretical_loss": 3.7557813906216344, + "tokens_seen": 803995648 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061120962888666, + "loss": 1.4592, + "theoretical_loss": 3.755751301100516, + "tokens_seen": 804061184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111935807422267, + "loss": 1.4307, + "theoretical_loss": 3.755721214718413, + "tokens_seen": 804126720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111775325977934, + "loss": 1.3931, + "theoretical_loss": 3.755691131474742, + "tokens_seen": 804192256 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111614844533601, + "loss": 1.3656, + "theoretical_loss": 3.7556610513689197, + "tokens_seen": 804257792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111454363089268, + "loss": 1.3319, + "theoretical_loss": 3.755630974400364, + "tokens_seen": 804323328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111293881644935, + "loss": 1.3791, + "theoretical_loss": 3.7556009005684907, + "tokens_seen": 804388864 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111133400200602, + "loss": 1.3199, + "theoretical_loss": 3.7555708298727186, + "tokens_seen": 804454400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110972918756269, + "loss": 1.2016, + "theoretical_loss": 3.7555407623124637, + "tokens_seen": 804519936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110812437311936, + "loss": 1.3712, + "theoretical_loss": 3.755510697887145, + "tokens_seen": 804585472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110651955867603, + "loss": 1.3483, + "theoretical_loss": 3.75548063659618, + "tokens_seen": 804651008 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611049147442327, + "loss": 1.5265, + "theoretical_loss": 3.755450578438986, + "tokens_seen": 804716544 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110330992978937, + "loss": 1.304, + "theoretical_loss": 3.7554205234149824, + "tokens_seen": 804782080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110170511534604, + "loss": 1.3313, + "theoretical_loss": 3.755390471523586, + "tokens_seen": 804847616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110010030090271, + "loss": 1.3138, + "theoretical_loss": 3.755360422764217, + "tokens_seen": 804913152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109849548645938, + "loss": 1.38, + "theoretical_loss": 3.755330377136292, + "tokens_seen": 804978688 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109689067201605, + "loss": 1.3346, + "theoretical_loss": 3.7553003346392315, + "tokens_seen": 805044224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109528585757272, + "loss": 1.3258, + "theoretical_loss": 3.7552702952724535, + "tokens_seen": 805109760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109368104312939, + "loss": 1.411, + "theoretical_loss": 3.755240259035377, + "tokens_seen": 805175296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109207622868606, + "loss": 1.472, + "theoretical_loss": 3.755210225927422, + "tokens_seen": 805240832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109047141424273, + "loss": 1.4201, + "theoretical_loss": 3.7551801959480073, + "tokens_seen": 805306368 + }, + { + "epoch": 0.24, + "learning_rate": 0.000610888665997994, + "loss": 1.3593, + "theoretical_loss": 3.7551501690965523, + "tokens_seen": 805371904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006108726178535607, + "loss": 1.3621, + "theoretical_loss": 3.755120145372477, + "tokens_seen": 805437440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006108565697091274, + "loss": 1.4013, + "theoretical_loss": 3.7550901247752013, + "tokens_seen": 805502976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006108405215646941, + "loss": 1.3432, + "theoretical_loss": 3.755060107304145, + "tokens_seen": 805568512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006108244734202608, + "loss": 1.4648, + "theoretical_loss": 3.7550300929587284, + "tokens_seen": 805634048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006108084252758275, + "loss": 1.3331, + "theoretical_loss": 3.7550000817383715, + "tokens_seen": 805699584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006107923771313942, + "loss": 1.4054, + "theoretical_loss": 3.754970073642496, + "tokens_seen": 805765120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006107763289869609, + "loss": 1.4175, + "theoretical_loss": 3.7549400686705203, + "tokens_seen": 805830656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006107602808425276, + "loss": 1.4496, + "theoretical_loss": 3.754910066821867, + "tokens_seen": 805896192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006107442326980943, + "loss": 1.3061, + "theoretical_loss": 3.7548800680959564, + "tokens_seen": 805961728 + }, + { + "epoch": 0.24, + "learning_rate": 0.000610728184553661, + "loss": 1.3899, + "theoretical_loss": 3.75485007249221, + "tokens_seen": 806027264 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 274160, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.31361985206604, + "objective/train/theoretical_loss": 3.754820080010048, + "objective/train/tokens_used": 826552800, + "theoretical_loss": 3.754820080010048, + "tokens_seen": 806092800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006107121364092277, + "loss": 1.3446, + "theoretical_loss": 3.754820080010048, + "tokens_seen": 806092800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106960882647944, + "loss": 1.4198, + "theoretical_loss": 3.7547900906488927, + "tokens_seen": 806158336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106800401203611, + "loss": 1.3948, + "theoretical_loss": 3.7547601044081658, + "tokens_seen": 806223872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106639919759278, + "loss": 1.3733, + "theoretical_loss": 3.7547301212872886, + "tokens_seen": 806289408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106479438314945, + "loss": 1.41, + "theoretical_loss": 3.7547001412856824, + "tokens_seen": 806354944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106318956870612, + "loss": 1.3092, + "theoretical_loss": 3.7546701644027705, + "tokens_seen": 806420480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106158475426279, + "loss": 1.3469, + "theoretical_loss": 3.754640190637974, + "tokens_seen": 806486016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105997993981946, + "loss": 1.3592, + "theoretical_loss": 3.754610219990716, + "tokens_seen": 806551552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105837512537613, + "loss": 1.426, + "theoretical_loss": 3.7545802524604177, + "tokens_seen": 806617088 + }, + { + "epoch": 0.24, + "learning_rate": 0.000610567703109328, + "loss": 1.4712, + "theoretical_loss": 3.7545502880465036, + "tokens_seen": 806682624 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105516549648947, + "loss": 1.3731, + "theoretical_loss": 3.754520326748395, + "tokens_seen": 806748160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105356068204614, + "loss": 1.3943, + "theoretical_loss": 3.7544903685655155, + "tokens_seen": 806813696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105195586760281, + "loss": 1.2324, + "theoretical_loss": 3.754460413497288, + "tokens_seen": 806879232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105035105315948, + "loss": 1.2384, + "theoretical_loss": 3.7544304615431354, + "tokens_seen": 806944768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006104874623871615, + "loss": 1.3791, + "theoretical_loss": 3.7544005127024818, + "tokens_seen": 807010304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006104714142427282, + "loss": 1.3843, + "theoretical_loss": 3.7543705669747496, + "tokens_seen": 807075840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006104553660982949, + "loss": 1.291, + "theoretical_loss": 3.754340624359364, + "tokens_seen": 807141376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006104393179538616, + "loss": 1.3835, + "theoretical_loss": 3.7543106848557484, + "tokens_seen": 807206912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006104232698094283, + "loss": 1.3209, + "theoretical_loss": 3.754280748463326, + "tokens_seen": 807272448 + }, + { + "epoch": 0.24, + "learning_rate": 0.000610407221664995, + "loss": 1.378, + "theoretical_loss": 3.7542508151815213, + "tokens_seen": 807337984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103911735205617, + "loss": 1.3738, + "theoretical_loss": 3.75422088500976, + "tokens_seen": 807403520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103751253761284, + "loss": 1.4543, + "theoretical_loss": 3.7541909579474644, + "tokens_seen": 807469056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103590772316951, + "loss": 1.4103, + "theoretical_loss": 3.7541610339940603, + "tokens_seen": 807534592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103430290872618, + "loss": 1.4083, + "theoretical_loss": 3.754131113148972, + "tokens_seen": 807600128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103269809428285, + "loss": 1.344, + "theoretical_loss": 3.7541011954116255, + "tokens_seen": 807665664 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103109327983953, + "loss": 1.328, + "theoretical_loss": 3.754071280781445, + "tokens_seen": 807731200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006102948846539619, + "loss": 1.4778, + "theoretical_loss": 3.7540413692578554, + "tokens_seen": 807796736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006102788365095287, + "loss": 1.4331, + "theoretical_loss": 3.7540114608402826, + "tokens_seen": 807862272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006102627883650953, + "loss": 1.3379, + "theoretical_loss": 3.753981555528152, + "tokens_seen": 807927808 + }, + { + "epoch": 0.24, + "learning_rate": 0.000610246740220662, + "loss": 1.4679, + "theoretical_loss": 3.7539516533208896, + "tokens_seen": 807993344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006102306920762287, + "loss": 1.3643, + "theoretical_loss": 3.753921754217921, + "tokens_seen": 808058880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006102146439317955, + "loss": 1.4234, + "theoretical_loss": 3.7538918582186724, + "tokens_seen": 808124416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006101985957873622, + "loss": 1.3992, + "theoretical_loss": 3.75386196532257, + "tokens_seen": 808189952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006101825476429289, + "loss": 1.384, + "theoretical_loss": 3.753832075529039, + "tokens_seen": 808255488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006101664994984956, + "loss": 1.289, + "theoretical_loss": 3.7538021888375077, + "tokens_seen": 808321024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006101504513540623, + "loss": 1.4012, + "theoretical_loss": 3.7537723052474012, + "tokens_seen": 808386560 + }, + { + "epoch": 0.24, + "learning_rate": 0.000610134403209629, + "loss": 1.4271, + "theoretical_loss": 3.753742424758147, + "tokens_seen": 808452096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006101183550651957, + "loss": 1.2788, + "theoretical_loss": 3.7537125473691715, + "tokens_seen": 808517632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006101023069207623, + "loss": 1.3695, + "theoretical_loss": 3.7536826730799024, + "tokens_seen": 808583168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100862587763291, + "loss": 1.416, + "theoretical_loss": 3.753652801889767, + "tokens_seen": 808648704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100702106318957, + "loss": 1.5054, + "theoretical_loss": 3.7536229337981912, + "tokens_seen": 808714240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100541624874625, + "loss": 1.401, + "theoretical_loss": 3.753593068804604, + "tokens_seen": 808779776 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100381143430291, + "loss": 1.3745, + "theoretical_loss": 3.7535632069084333, + "tokens_seen": 808845312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100220661985959, + "loss": 1.3839, + "theoretical_loss": 3.753533348109106, + "tokens_seen": 808910848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100060180541625, + "loss": 1.4323, + "theoretical_loss": 3.7535034924060504, + "tokens_seen": 808976384 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006099899699097292, + "loss": 1.4034, + "theoretical_loss": 3.7534736397986945, + "tokens_seen": 809041920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006099739217652959, + "loss": 1.4289, + "theoretical_loss": 3.753443790286467, + "tokens_seen": 809107456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006099578736208626, + "loss": 1.3386, + "theoretical_loss": 3.7534139438687957, + "tokens_seen": 809172992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006099418254764293, + "loss": 1.3465, + "theoretical_loss": 3.7533841005451096, + "tokens_seen": 809238528 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609925777331996, + "loss": 1.3349, + "theoretical_loss": 3.7533542603148375, + "tokens_seen": 809304064 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 275041, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4928539991378784, + "objective/train/theoretical_loss": 3.7533244231774083, + "objective/train/tokens_used": 829829600, + "theoretical_loss": 3.7533244231774083, + "tokens_seen": 809369600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006099097291875627, + "loss": 1.41, + "theoretical_loss": 3.7533244231774083, + "tokens_seen": 809369600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098936810431294, + "loss": 1.3372, + "theoretical_loss": 3.753294589132251, + "tokens_seen": 809435136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098776328986961, + "loss": 1.3673, + "theoretical_loss": 3.753264758178794, + "tokens_seen": 809500672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098615847542628, + "loss": 1.437, + "theoretical_loss": 3.7532349303164683, + "tokens_seen": 809566208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098455366098295, + "loss": 1.3952, + "theoretical_loss": 3.7532051055447018, + "tokens_seen": 809631744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098294884653962, + "loss": 1.3868, + "theoretical_loss": 3.753175283862925, + "tokens_seen": 809697280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098134403209629, + "loss": 1.4342, + "theoretical_loss": 3.753145465270568, + "tokens_seen": 809762816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097973921765296, + "loss": 1.4018, + "theoretical_loss": 3.753115649767059, + "tokens_seen": 809828352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097813440320963, + "loss": 1.4393, + "theoretical_loss": 3.75308583735183, + "tokens_seen": 809893888 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609765295887663, + "loss": 1.4589, + "theoretical_loss": 3.753056028024311, + "tokens_seen": 809959424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097492477432297, + "loss": 1.4399, + "theoretical_loss": 3.753026221783932, + "tokens_seen": 810024960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097331995987964, + "loss": 1.4689, + "theoretical_loss": 3.752996418630123, + "tokens_seen": 810090496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097171514543631, + "loss": 1.3463, + "theoretical_loss": 3.7529666185623154, + "tokens_seen": 810156032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097011033099298, + "loss": 1.4034, + "theoretical_loss": 3.75293682157994, + "tokens_seen": 810221568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006096850551654965, + "loss": 1.4199, + "theoretical_loss": 3.7529070276824283, + "tokens_seen": 810287104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006096690070210632, + "loss": 1.433, + "theoretical_loss": 3.7528772368692103, + "tokens_seen": 810352640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006096529588766299, + "loss": 1.4171, + "theoretical_loss": 3.7528474491397183, + "tokens_seen": 810418176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006096369107321966, + "loss": 1.3973, + "theoretical_loss": 3.7528176644933833, + "tokens_seen": 810483712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006096208625877633, + "loss": 1.3973, + "theoretical_loss": 3.7527878829296366, + "tokens_seen": 810549248 + }, + { + "epoch": 0.25, + "learning_rate": 0.00060960481444333, + "loss": 1.3531, + "theoretical_loss": 3.752758104447911, + "tokens_seen": 810614784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095887662988967, + "loss": 1.5146, + "theoretical_loss": 3.7527283290476374, + "tokens_seen": 810680320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095727181544634, + "loss": 1.2868, + "theoretical_loss": 3.752698556728248, + "tokens_seen": 810745856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095566700100301, + "loss": 1.2298, + "theoretical_loss": 3.7526687874891755, + "tokens_seen": 810811392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095406218655968, + "loss": 1.4393, + "theoretical_loss": 3.752639021329852, + "tokens_seen": 810876928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095245737211635, + "loss": 1.3986, + "theoretical_loss": 3.75260925824971, + "tokens_seen": 810942464 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095085255767302, + "loss": 1.3188, + "theoretical_loss": 3.752579498248183, + "tokens_seen": 811008000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006094924774322969, + "loss": 1.3937, + "theoretical_loss": 3.752549741324702, + "tokens_seen": 811073536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006094764292878636, + "loss": 1.4371, + "theoretical_loss": 3.752519987478702, + "tokens_seen": 811139072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006094603811434303, + "loss": 1.391, + "theoretical_loss": 3.7524902367096145, + "tokens_seen": 811204608 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609444332998997, + "loss": 1.4319, + "theoretical_loss": 3.7524604890168733, + "tokens_seen": 811270144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006094282848545637, + "loss": 1.3737, + "theoretical_loss": 3.752430744399912, + "tokens_seen": 811335680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006094122367101304, + "loss": 1.4078, + "theoretical_loss": 3.7524010028581634, + "tokens_seen": 811401216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093961885656971, + "loss": 1.3747, + "theoretical_loss": 3.752371264391063, + "tokens_seen": 811466752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093801404212638, + "loss": 1.4404, + "theoretical_loss": 3.752341528998043, + "tokens_seen": 811532288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093640922768305, + "loss": 1.3647, + "theoretical_loss": 3.7523117966785384, + "tokens_seen": 811597824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093480441323972, + "loss": 1.4224, + "theoretical_loss": 3.752282067431983, + "tokens_seen": 811663360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093319959879639, + "loss": 1.3438, + "theoretical_loss": 3.7522523412578104, + "tokens_seen": 811728896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093159478435306, + "loss": 1.3447, + "theoretical_loss": 3.7522226181554563, + "tokens_seen": 811794432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092998996990973, + "loss": 1.2968, + "theoretical_loss": 3.7521928981243544, + "tokens_seen": 811859968 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609283851554664, + "loss": 1.3338, + "theoretical_loss": 3.7521631811639398, + "tokens_seen": 811925504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092678034102307, + "loss": 1.3038, + "theoretical_loss": 3.752133467273648, + "tokens_seen": 811991040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092517552657974, + "loss": 1.3154, + "theoretical_loss": 3.7521037564529127, + "tokens_seen": 812056576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092357071213641, + "loss": 1.3167, + "theoretical_loss": 3.7520740487011706, + "tokens_seen": 812122112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092196589769308, + "loss": 1.3733, + "theoretical_loss": 3.752044344017856, + "tokens_seen": 812187648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092036108324975, + "loss": 1.4768, + "theoretical_loss": 3.752014642402405, + "tokens_seen": 812253184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006091875626880642, + "loss": 1.3319, + "theoretical_loss": 3.751984943854253, + "tokens_seen": 812318720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006091715145436309, + "loss": 1.4028, + "theoretical_loss": 3.751955248372836, + "tokens_seen": 812384256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006091554663991976, + "loss": 1.3679, + "theoretical_loss": 3.75192555595759, + "tokens_seen": 812449792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006091394182547643, + "loss": 1.3209, + "theoretical_loss": 3.751895866607951, + "tokens_seen": 812515328 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609123370110331, + "loss": 1.3726, + "theoretical_loss": 3.7518661803233546, + "tokens_seen": 812580864 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 276340, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2950143814086914, + "objective/train/theoretical_loss": 3.7518364971032385, + "objective/train/tokens_used": 833106400, + "theoretical_loss": 3.7518364971032385, + "tokens_seen": 812646400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006091073219658977, + "loss": 1.3498, + "theoretical_loss": 3.7518364971032385, + "tokens_seen": 812646400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006090912738214644, + "loss": 1.3764, + "theoretical_loss": 3.751806816947038, + "tokens_seen": 812711936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006090752256770311, + "loss": 1.4244, + "theoretical_loss": 3.751777139854191, + "tokens_seen": 812777472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006090591775325978, + "loss": 1.4724, + "theoretical_loss": 3.7517474658241334, + "tokens_seen": 812843008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006090431293881645, + "loss": 1.2368, + "theoretical_loss": 3.7517177948563027, + "tokens_seen": 812908544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006090270812437312, + "loss": 1.4304, + "theoretical_loss": 3.7516881269501354, + "tokens_seen": 812974080 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609011033099298, + "loss": 1.4232, + "theoretical_loss": 3.75165846210507, + "tokens_seen": 813039616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089949849548647, + "loss": 1.4495, + "theoretical_loss": 3.7516288003205434, + "tokens_seen": 813105152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089789368104313, + "loss": 1.3003, + "theoretical_loss": 3.7515991415959933, + "tokens_seen": 813170688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089628886659981, + "loss": 1.4702, + "theoretical_loss": 3.7515694859308564, + "tokens_seen": 813236224 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089468405215648, + "loss": 1.5598, + "theoretical_loss": 3.7515398333245717, + "tokens_seen": 813301760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089307923771315, + "loss": 1.4456, + "theoretical_loss": 3.751510183776577, + "tokens_seen": 813367296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089147442326982, + "loss": 1.5005, + "theoretical_loss": 3.7514805372863105, + "tokens_seen": 813432832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088986960882649, + "loss": 1.3097, + "theoretical_loss": 3.7514508938532107, + "tokens_seen": 813498368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088826479438316, + "loss": 1.4725, + "theoretical_loss": 3.7514212534767157, + "tokens_seen": 813563904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088665997993983, + "loss": 1.4341, + "theoretical_loss": 3.7513916161562646, + "tokens_seen": 813629440 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608850551654965, + "loss": 1.4583, + "theoretical_loss": 3.7513619818912964, + "tokens_seen": 813694976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088345035105317, + "loss": 1.3755, + "theoretical_loss": 3.751332350681249, + "tokens_seen": 813760512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088184553660984, + "loss": 1.4159, + "theoretical_loss": 3.751302722525562, + "tokens_seen": 813826048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088024072216651, + "loss": 1.4529, + "theoretical_loss": 3.7512730974236748, + "tokens_seen": 813891584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087863590772318, + "loss": 1.4448, + "theoretical_loss": 3.751243475375027, + "tokens_seen": 813957120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087703109327985, + "loss": 1.4802, + "theoretical_loss": 3.7512138563790574, + "tokens_seen": 814022656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087542627883652, + "loss": 1.3585, + "theoretical_loss": 3.7511842404352063, + "tokens_seen": 814088192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087382146439319, + "loss": 1.4013, + "theoretical_loss": 3.7511546275429133, + "tokens_seen": 814153728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087221664994986, + "loss": 1.3427, + "theoretical_loss": 3.7511250177016184, + "tokens_seen": 814219264 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087061183550652, + "loss": 1.4272, + "theoretical_loss": 3.7510954109107617, + "tokens_seen": 814284800 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608690070210632, + "loss": 1.3887, + "theoretical_loss": 3.751065807169784, + "tokens_seen": 814350336 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006086740220661986, + "loss": 1.6241, + "theoretical_loss": 3.7510362064781244, + "tokens_seen": 814415872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006086579739217654, + "loss": 1.4205, + "theoretical_loss": 3.7510066088352247, + "tokens_seen": 814481408 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608641925777332, + "loss": 1.3721, + "theoretical_loss": 3.750977014240525, + "tokens_seen": 814546944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006086258776328988, + "loss": 1.4522, + "theoretical_loss": 3.750947422693467, + "tokens_seen": 814612480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006086098294884654, + "loss": 1.3763, + "theoretical_loss": 3.7509178341934906, + "tokens_seen": 814678016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085937813440322, + "loss": 1.4083, + "theoretical_loss": 3.7508882487400372, + "tokens_seen": 814743552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085777331995988, + "loss": 1.4799, + "theoretical_loss": 3.7508586663325483, + "tokens_seen": 814809088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085616850551655, + "loss": 1.4582, + "theoretical_loss": 3.750829086970466, + "tokens_seen": 814874624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085456369107322, + "loss": 1.3615, + "theoretical_loss": 3.750799510653231, + "tokens_seen": 814940160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085295887662989, + "loss": 1.5135, + "theoretical_loss": 3.7507699373802854, + "tokens_seen": 815005696 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085135406218656, + "loss": 1.4188, + "theoretical_loss": 3.750740367151071, + "tokens_seen": 815071232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084974924774323, + "loss": 1.4552, + "theoretical_loss": 3.750710799965029, + "tokens_seen": 815136768 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608481444332999, + "loss": 1.4154, + "theoretical_loss": 3.7506812358216037, + "tokens_seen": 815202304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084653961885657, + "loss": 1.4007, + "theoretical_loss": 3.7506516747202356, + "tokens_seen": 815267840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084493480441324, + "loss": 1.3778, + "theoretical_loss": 3.7506221166603675, + "tokens_seen": 815333376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084332998996991, + "loss": 1.5337, + "theoretical_loss": 3.750592561641443, + "tokens_seen": 815398912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084172517552658, + "loss": 1.44, + "theoretical_loss": 3.750563009662904, + "tokens_seen": 815464448 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084012036108325, + "loss": 1.3755, + "theoretical_loss": 3.750533460724193, + "tokens_seen": 815529984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006083851554663992, + "loss": 1.3554, + "theoretical_loss": 3.750503914824754, + "tokens_seen": 815595520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006083691073219659, + "loss": 1.3796, + "theoretical_loss": 3.75047437196403, + "tokens_seen": 815661056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006083530591775326, + "loss": 1.4956, + "theoretical_loss": 3.750444832141464, + "tokens_seen": 815726592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006083370110330993, + "loss": 1.4646, + "theoretical_loss": 3.7504152953564995, + "tokens_seen": 815792128 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608320962888666, + "loss": 1.2947, + "theoretical_loss": 3.750385761608581, + "tokens_seen": 815857664 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 277338, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.33406662940979, + "objective/train/theoretical_loss": 3.7503562308971516, + "objective/train/tokens_used": 836383200, + "theoretical_loss": 3.7503562308971516, + "tokens_seen": 815923200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006083049147442327, + "loss": 1.3607, + "theoretical_loss": 3.7503562308971516, + "tokens_seen": 815923200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082888665997994, + "loss": 1.5228, + "theoretical_loss": 3.7503267032216554, + "tokens_seen": 815988736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082728184553661, + "loss": 1.2615, + "theoretical_loss": 3.750297178581536, + "tokens_seen": 816054272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082567703109328, + "loss": 1.3789, + "theoretical_loss": 3.7502676569762383, + "tokens_seen": 816119808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082407221664995, + "loss": 1.5057, + "theoretical_loss": 3.7502381384052064, + "tokens_seen": 816185344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082246740220662, + "loss": 1.3762, + "theoretical_loss": 3.750208622867885, + "tokens_seen": 816250880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082086258776329, + "loss": 1.322, + "theoretical_loss": 3.7501791103637188, + "tokens_seen": 816316416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006081925777331996, + "loss": 1.4761, + "theoretical_loss": 3.7501496008921524, + "tokens_seen": 816381952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006081765295887663, + "loss": 1.3314, + "theoretical_loss": 3.750120094452631, + "tokens_seen": 816447488 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608160481444333, + "loss": 1.3851, + "theoretical_loss": 3.750090591044599, + "tokens_seen": 816513024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006081444332998997, + "loss": 1.414, + "theoretical_loss": 3.750061090667503, + "tokens_seen": 816578560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006081283851554664, + "loss": 1.3629, + "theoretical_loss": 3.750031593320787, + "tokens_seen": 816644096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006081123370110331, + "loss": 1.3429, + "theoretical_loss": 3.750002099003898, + "tokens_seen": 816709632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080962888665998, + "loss": 1.379, + "theoretical_loss": 3.7499726077162805, + "tokens_seen": 816775168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080802407221665, + "loss": 1.3893, + "theoretical_loss": 3.749943119457381, + "tokens_seen": 816840704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080641925777332, + "loss": 1.3895, + "theoretical_loss": 3.749913634226645, + "tokens_seen": 816906240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080481444332999, + "loss": 1.3953, + "theoretical_loss": 3.7498841520235193, + "tokens_seen": 816971776 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080320962888666, + "loss": 1.3706, + "theoretical_loss": 3.7498546728474498, + "tokens_seen": 817037312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080160481444333, + "loss": 1.397, + "theoretical_loss": 3.7498251966978824, + "tokens_seen": 817102848 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608, + "loss": 1.3084, + "theoretical_loss": 3.7497957235742647, + "tokens_seen": 817168384 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079839518555667, + "loss": 1.4022, + "theoretical_loss": 3.749766253476043, + "tokens_seen": 817233920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079679037111334, + "loss": 1.3423, + "theoretical_loss": 3.7497367864026634, + "tokens_seen": 817299456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079518555667001, + "loss": 1.4834, + "theoretical_loss": 3.749707322353575, + "tokens_seen": 817364992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079358074222668, + "loss": 1.4279, + "theoretical_loss": 3.7496778613282222, + "tokens_seen": 817430528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079197592778335, + "loss": 1.4755, + "theoretical_loss": 3.749648403326054, + "tokens_seen": 817496064 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079037111334002, + "loss": 1.3799, + "theoretical_loss": 3.7496189483465177, + "tokens_seen": 817561600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006078876629889669, + "loss": 1.4539, + "theoretical_loss": 3.749589496389061, + "tokens_seen": 817627136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006078716148445336, + "loss": 1.3667, + "theoretical_loss": 3.749560047453131, + "tokens_seen": 817692672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006078555667001003, + "loss": 1.4008, + "theoretical_loss": 3.7495306015381766, + "tokens_seen": 817758208 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607839518555667, + "loss": 1.3677, + "theoretical_loss": 3.7495011586436444, + "tokens_seen": 817823744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006078234704112337, + "loss": 1.4381, + "theoretical_loss": 3.7494717187689837, + "tokens_seen": 817889280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006078074222668004, + "loss": 1.419, + "theoretical_loss": 3.7494422819136424, + "tokens_seen": 817954816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006077913741223672, + "loss": 1.4178, + "theoretical_loss": 3.7494128480770694, + "tokens_seen": 818020352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006077753259779338, + "loss": 1.3622, + "theoretical_loss": 3.749383417258713, + "tokens_seen": 818085888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006077592778335006, + "loss": 1.4795, + "theoretical_loss": 3.7493539894580215, + "tokens_seen": 818151424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006077432296890672, + "loss": 1.4322, + "theoretical_loss": 3.749324564674444, + "tokens_seen": 818216960 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607727181544634, + "loss": 1.3771, + "theoretical_loss": 3.74929514290743, + "tokens_seen": 818282496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006077111334002006, + "loss": 1.3654, + "theoretical_loss": 3.749265724156429, + "tokens_seen": 818348032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006076950852557674, + "loss": 1.3776, + "theoretical_loss": 3.7492363084208895, + "tokens_seen": 818413568 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607679037111334, + "loss": 1.4275, + "theoretical_loss": 3.7492068957002607, + "tokens_seen": 818479104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006076629889669008, + "loss": 1.4517, + "theoretical_loss": 3.749177485993993, + "tokens_seen": 818544640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006076469408224674, + "loss": 1.4664, + "theoretical_loss": 3.749148079301537, + "tokens_seen": 818610176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006076308926780342, + "loss": 1.4065, + "theoretical_loss": 3.7491186756223405, + "tokens_seen": 818675712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006076148445336008, + "loss": 1.3951, + "theoretical_loss": 3.749089274955855, + "tokens_seen": 818741248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006075987963891676, + "loss": 1.5266, + "theoretical_loss": 3.74905987730153, + "tokens_seen": 818806784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006075827482447342, + "loss": 1.317, + "theoretical_loss": 3.7490304826588163, + "tokens_seen": 818872320 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607566700100301, + "loss": 1.3463, + "theoretical_loss": 3.7490010910271647, + "tokens_seen": 818937856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006075506519558677, + "loss": 1.4715, + "theoretical_loss": 3.748971702406025, + "tokens_seen": 819003392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006075346038114344, + "loss": 1.3579, + "theoretical_loss": 3.7489423167948486, + "tokens_seen": 819068928 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607518555667001, + "loss": 1.444, + "theoretical_loss": 3.7489129341930862, + "tokens_seen": 819134464 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 278526, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3594393730163574, + "objective/train/theoretical_loss": 3.7488835546001886, + "objective/train/tokens_used": 839660000, + "theoretical_loss": 3.7488835546001886, + "tokens_seen": 819200000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006075025075225678, + "loss": 1.4002, + "theoretical_loss": 3.7488835546001886, + "tokens_seen": 819200000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006074864593781345, + "loss": 1.4299, + "theoretical_loss": 3.7488541780156077, + "tokens_seen": 819265536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006074704112337012, + "loss": 1.4435, + "theoretical_loss": 3.7488248044387946, + "tokens_seen": 819331072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006074543630892679, + "loss": 1.2144, + "theoretical_loss": 3.7487954338692, + "tokens_seen": 819396608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006074383149448346, + "loss": 1.4401, + "theoretical_loss": 3.7487660663062767, + "tokens_seen": 819462144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006074222668004013, + "loss": 1.4884, + "theoretical_loss": 3.7487367017494755, + "tokens_seen": 819527680 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607406218655968, + "loss": 1.3272, + "theoretical_loss": 3.7487073401982487, + "tokens_seen": 819593216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073901705115347, + "loss": 1.4158, + "theoretical_loss": 3.7486779816520492, + "tokens_seen": 819658752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073741223671014, + "loss": 1.3252, + "theoretical_loss": 3.7486486261103282, + "tokens_seen": 819724288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073580742226681, + "loss": 1.4221, + "theoretical_loss": 3.748619273572538, + "tokens_seen": 819789824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073420260782348, + "loss": 1.376, + "theoretical_loss": 3.748589924038132, + "tokens_seen": 819855360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073259779338015, + "loss": 1.3883, + "theoretical_loss": 3.748560577506562, + "tokens_seen": 819920896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073099297893681, + "loss": 1.4445, + "theoretical_loss": 3.748531233977281, + "tokens_seen": 819986432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072938816449349, + "loss": 1.4481, + "theoretical_loss": 3.7485018934497423, + "tokens_seen": 820051968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072778335005015, + "loss": 1.3551, + "theoretical_loss": 3.7484725559233985, + "tokens_seen": 820117504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072617853560683, + "loss": 1.4588, + "theoretical_loss": 3.7484432213977033, + "tokens_seen": 820183040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072457372116349, + "loss": 1.4682, + "theoretical_loss": 3.7484138898721096, + "tokens_seen": 820248576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072296890672017, + "loss": 1.4129, + "theoretical_loss": 3.748384561346071, + "tokens_seen": 820314112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072136409227683, + "loss": 1.4247, + "theoretical_loss": 3.7483552358190417, + "tokens_seen": 820379648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071975927783351, + "loss": 1.3853, + "theoretical_loss": 3.7483259132904747, + "tokens_seen": 820445184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071815446339017, + "loss": 1.3521, + "theoretical_loss": 3.7482965937598243, + "tokens_seen": 820510720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071654964894684, + "loss": 1.4377, + "theoretical_loss": 3.748267277226545, + "tokens_seen": 820576256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071494483450351, + "loss": 1.3544, + "theoretical_loss": 3.74823796369009, + "tokens_seen": 820641792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071334002006018, + "loss": 1.3531, + "theoretical_loss": 3.7482086531499146, + "tokens_seen": 820707328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071173520561685, + "loss": 1.3332, + "theoretical_loss": 3.748179345605473, + "tokens_seen": 820772864 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071013039117352, + "loss": 1.4968, + "theoretical_loss": 3.74815004105622, + "tokens_seen": 820838400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006070852557673019, + "loss": 1.458, + "theoretical_loss": 3.7481207395016103, + "tokens_seen": 820903936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006070692076228686, + "loss": 1.3335, + "theoretical_loss": 3.7480914409410984, + "tokens_seen": 820969472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006070531594784353, + "loss": 1.3834, + "theoretical_loss": 3.7480621453741394, + "tokens_seen": 821035008 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607037111334002, + "loss": 1.4099, + "theoretical_loss": 3.74803285280019, + "tokens_seen": 821100544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006070210631895687, + "loss": 1.3673, + "theoretical_loss": 3.7480035632187034, + "tokens_seen": 821166080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006070050150451354, + "loss": 1.3187, + "theoretical_loss": 3.7479742766291366, + "tokens_seen": 821231616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069889669007021, + "loss": 1.5089, + "theoretical_loss": 3.747944993030945, + "tokens_seen": 821297152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069729187562688, + "loss": 1.4639, + "theoretical_loss": 3.7479157124235836, + "tokens_seen": 821362688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069568706118355, + "loss": 1.4294, + "theoretical_loss": 3.7478864348065093, + "tokens_seen": 821428224 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069408224674022, + "loss": 1.3491, + "theoretical_loss": 3.747857160179178, + "tokens_seen": 821493760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069247743229689, + "loss": 1.3862, + "theoretical_loss": 3.747827888541045, + "tokens_seen": 821559296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069087261785356, + "loss": 1.4502, + "theoretical_loss": 3.7477986198915683, + "tokens_seen": 821624832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006068926780341023, + "loss": 1.3816, + "theoretical_loss": 3.7477693542302033, + "tokens_seen": 821690368 + }, + { + "epoch": 0.25, + "learning_rate": 0.000606876629889669, + "loss": 1.4689, + "theoretical_loss": 3.7477400915564063, + "tokens_seen": 821755904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006068605817452357, + "loss": 1.4246, + "theoretical_loss": 3.747710831869635, + "tokens_seen": 821821440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006068445336008024, + "loss": 1.3817, + "theoretical_loss": 3.747681575169346, + "tokens_seen": 821886976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006068284854563691, + "loss": 1.2922, + "theoretical_loss": 3.747652321454997, + "tokens_seen": 821952512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006068124373119358, + "loss": 1.374, + "theoretical_loss": 3.7476230707260436, + "tokens_seen": 822018048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067963891675025, + "loss": 1.3978, + "theoretical_loss": 3.7475938229819445, + "tokens_seen": 822083584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067803410230692, + "loss": 1.3212, + "theoretical_loss": 3.747564578222157, + "tokens_seen": 822149120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067642928786359, + "loss": 1.395, + "theoretical_loss": 3.747535336446138, + "tokens_seen": 822214656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067482447342026, + "loss": 1.4946, + "theoretical_loss": 3.7475060976533463, + "tokens_seen": 822280192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067321965897693, + "loss": 1.4107, + "theoretical_loss": 3.7474768618432392, + "tokens_seen": 822345728 + }, + { + "epoch": 0.25, + "learning_rate": 0.000606716148445336, + "loss": 1.3553, + "theoretical_loss": 3.747447629015275, + "tokens_seen": 822411264 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 279519, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.31684410572052, + "objective/train/theoretical_loss": 3.747418399168912, + "objective/train/tokens_used": 842936800, + "theoretical_loss": 3.747418399168912, + "tokens_seen": 822476800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067001003009027, + "loss": 1.357, + "theoretical_loss": 3.747418399168912, + "tokens_seen": 822476800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066840521564694, + "loss": 1.4719, + "theoretical_loss": 3.747389172303608, + "tokens_seen": 822542336 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066680040120361, + "loss": 1.3836, + "theoretical_loss": 3.7473599484188225, + "tokens_seen": 822607872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066519558676028, + "loss": 1.4576, + "theoretical_loss": 3.7473307275140133, + "tokens_seen": 822673408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066359077231695, + "loss": 1.3516, + "theoretical_loss": 3.747301509588639, + "tokens_seen": 822738944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066198595787362, + "loss": 1.3676, + "theoretical_loss": 3.7472722946421593, + "tokens_seen": 822804480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066038114343029, + "loss": 1.3243, + "theoretical_loss": 3.747243082674033, + "tokens_seen": 822870016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065877632898697, + "loss": 1.3562, + "theoretical_loss": 3.7472138736837186, + "tokens_seen": 822935552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065717151454363, + "loss": 1.3552, + "theoretical_loss": 3.747184667670677, + "tokens_seen": 823001088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065556670010031, + "loss": 1.4412, + "theoretical_loss": 3.747155464634366, + "tokens_seen": 823066624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065396188565697, + "loss": 1.3834, + "theoretical_loss": 3.747126264574246, + "tokens_seen": 823132160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065235707121365, + "loss": 1.398, + "theoretical_loss": 3.7470970674897766, + "tokens_seen": 823197696 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065075225677031, + "loss": 1.4561, + "theoretical_loss": 3.7470678733804177, + "tokens_seen": 823263232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064914744232699, + "loss": 1.4431, + "theoretical_loss": 3.7470386822456296, + "tokens_seen": 823328768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064754262788365, + "loss": 1.4705, + "theoretical_loss": 3.7470094940848724, + "tokens_seen": 823394304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064593781344033, + "loss": 1.3929, + "theoretical_loss": 3.7469803088976064, + "tokens_seen": 823459840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064433299899699, + "loss": 1.3357, + "theoretical_loss": 3.7469511266832916, + "tokens_seen": 823525376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064272818455367, + "loss": 1.437, + "theoretical_loss": 3.7469219474413893, + "tokens_seen": 823590912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064112337011033, + "loss": 1.4384, + "theoretical_loss": 3.74689277117136, + "tokens_seen": 823656448 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063951855566701, + "loss": 1.4605, + "theoretical_loss": 3.7468635978726645, + "tokens_seen": 823721984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063791374122367, + "loss": 1.442, + "theoretical_loss": 3.746834427544764, + "tokens_seen": 823787520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063630892678035, + "loss": 1.4305, + "theoretical_loss": 3.7468052601871196, + "tokens_seen": 823853056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063470411233701, + "loss": 1.4194, + "theoretical_loss": 3.7467760957991922, + "tokens_seen": 823918592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063309929789369, + "loss": 1.3391, + "theoretical_loss": 3.746746934380444, + "tokens_seen": 823984128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063149448345035, + "loss": 1.5232, + "theoretical_loss": 3.7467177759303363, + "tokens_seen": 824049664 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062988966900703, + "loss": 1.3503, + "theoretical_loss": 3.7466886204483307, + "tokens_seen": 824115200 + }, + { + "epoch": 0.25, + "learning_rate": 0.000606282848545637, + "loss": 1.4009, + "theoretical_loss": 3.746659467933889, + "tokens_seen": 824180736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062668004012037, + "loss": 1.4677, + "theoretical_loss": 3.7466303183864733, + "tokens_seen": 824246272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062507522567703, + "loss": 1.4113, + "theoretical_loss": 3.746601171805546, + "tokens_seen": 824311808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062347041123371, + "loss": 1.4142, + "theoretical_loss": 3.7465720281905694, + "tokens_seen": 824377344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062186559679037, + "loss": 1.3534, + "theoretical_loss": 3.746542887541005, + "tokens_seen": 824442880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062026078234705, + "loss": 1.4531, + "theoretical_loss": 3.746513749856317, + "tokens_seen": 824508416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006061865596790372, + "loss": 1.4338, + "theoretical_loss": 3.7464846151359668, + "tokens_seen": 824573952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006061705115346039, + "loss": 1.3586, + "theoretical_loss": 3.7464554833794175, + "tokens_seen": 824639488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006061544633901706, + "loss": 1.4736, + "theoretical_loss": 3.7464263545861325, + "tokens_seen": 824705024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006061384152457373, + "loss": 1.4254, + "theoretical_loss": 3.7463972287555745, + "tokens_seen": 824770560 + }, + { + "epoch": 0.25, + "learning_rate": 0.000606122367101304, + "loss": 1.4723, + "theoretical_loss": 3.7463681058872074, + "tokens_seen": 824836096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006061063189568707, + "loss": 1.3425, + "theoretical_loss": 3.746338985980494, + "tokens_seen": 824901632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060902708124374, + "loss": 1.2868, + "theoretical_loss": 3.746309869034898, + "tokens_seen": 824967168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060742226680041, + "loss": 1.3821, + "theoretical_loss": 3.746280755049883, + "tokens_seen": 825032704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060581745235708, + "loss": 1.3976, + "theoretical_loss": 3.7462516440249134, + "tokens_seen": 825098240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060421263791375, + "loss": 1.355, + "theoretical_loss": 3.7462225359594523, + "tokens_seen": 825163776 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060260782347042, + "loss": 1.3845, + "theoretical_loss": 3.7461934308529647, + "tokens_seen": 825229312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060100300902709, + "loss": 1.3716, + "theoretical_loss": 3.7461643287049142, + "tokens_seen": 825294848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006059939819458376, + "loss": 1.368, + "theoretical_loss": 3.7461352295147656, + "tokens_seen": 825360384 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006059779338014043, + "loss": 1.4058, + "theoretical_loss": 3.746106133281983, + "tokens_seen": 825425920 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605961885656971, + "loss": 1.3171, + "theoretical_loss": 3.746077040006032, + "tokens_seen": 825491456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006059458375125377, + "loss": 1.4918, + "theoretical_loss": 3.7460479496863757, + "tokens_seen": 825556992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006059297893681044, + "loss": 1.2982, + "theoretical_loss": 3.746018862322481, + "tokens_seen": 825622528 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605913741223671, + "loss": 1.3822, + "theoretical_loss": 3.7459897779138114, + "tokens_seen": 825688064 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 280689, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4213380813598633, + "objective/train/theoretical_loss": 3.745960696459833, + "objective/train/tokens_used": 846213600, + "theoretical_loss": 3.745960696459833, + "tokens_seen": 825753600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058976930792378, + "loss": 1.4046, + "theoretical_loss": 3.745960696459833, + "tokens_seen": 825753600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058816449348044, + "loss": 1.4818, + "theoretical_loss": 3.7459316179600117, + "tokens_seen": 825819136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058655967903712, + "loss": 1.3332, + "theoretical_loss": 3.7459025424138117, + "tokens_seen": 825884672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058495486459378, + "loss": 1.4354, + "theoretical_loss": 3.7458734698206992, + "tokens_seen": 825950208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058335005015046, + "loss": 1.4405, + "theoretical_loss": 3.74584440018014, + "tokens_seen": 826015744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058174523570712, + "loss": 1.3741, + "theoretical_loss": 3.7458153334916, + "tokens_seen": 826081280 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605801404212638, + "loss": 1.4119, + "theoretical_loss": 3.7457862697545456, + "tokens_seen": 826146816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006057853560682046, + "loss": 1.4196, + "theoretical_loss": 3.7457572089684428, + "tokens_seen": 826212352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006057693079237713, + "loss": 1.3461, + "theoretical_loss": 3.7457281511327576, + "tokens_seen": 826277888 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605753259779338, + "loss": 1.405, + "theoretical_loss": 3.7456990962469567, + "tokens_seen": 826343424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006057372116349047, + "loss": 1.4363, + "theoretical_loss": 3.745670044310507, + "tokens_seen": 826408960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006057211634904714, + "loss": 1.4599, + "theoretical_loss": 3.7456409953228746, + "tokens_seen": 826474496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006057051153460381, + "loss": 1.4169, + "theoretical_loss": 3.745611949283527, + "tokens_seen": 826540032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056890672016048, + "loss": 1.4954, + "theoretical_loss": 3.745582906191931, + "tokens_seen": 826605568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056730190571715, + "loss": 1.4297, + "theoretical_loss": 3.7455538660475542, + "tokens_seen": 826671104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056569709127382, + "loss": 1.3785, + "theoretical_loss": 3.7455248288498635, + "tokens_seen": 826736640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056409227683049, + "loss": 1.4381, + "theoretical_loss": 3.745495794598326, + "tokens_seen": 826802176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056248746238716, + "loss": 1.51, + "theoretical_loss": 3.74546676329241, + "tokens_seen": 826867712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056088264794383, + "loss": 1.4123, + "theoretical_loss": 3.7454377349315826, + "tokens_seen": 826933248 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605592778335005, + "loss": 1.4357, + "theoretical_loss": 3.7454087095153126, + "tokens_seen": 826998784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006055767301905717, + "loss": 1.3916, + "theoretical_loss": 3.7453796870430667, + "tokens_seen": 827064320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006055606820461384, + "loss": 1.3999, + "theoretical_loss": 3.7453506675143142, + "tokens_seen": 827129856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006055446339017051, + "loss": 1.426, + "theoretical_loss": 3.7453216509285223, + "tokens_seen": 827195392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006055285857572718, + "loss": 1.4631, + "theoretical_loss": 3.7452926372851607, + "tokens_seen": 827260928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006055125376128385, + "loss": 1.5237, + "theoretical_loss": 3.7452636265836965, + "tokens_seen": 827326464 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054964894684052, + "loss": 1.4424, + "theoretical_loss": 3.7452346188236, + "tokens_seen": 827392000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054804413239719, + "loss": 1.3875, + "theoretical_loss": 3.745205614004339, + "tokens_seen": 827457536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054643931795386, + "loss": 1.4605, + "theoretical_loss": 3.745176612125382, + "tokens_seen": 827523072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054483450351053, + "loss": 1.3546, + "theoretical_loss": 3.7451476131861994, + "tokens_seen": 827588608 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605432296890672, + "loss": 1.303, + "theoretical_loss": 3.74511861718626, + "tokens_seen": 827654144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054162487462387, + "loss": 1.2855, + "theoretical_loss": 3.7450896241250327, + "tokens_seen": 827719680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054002006018054, + "loss": 1.42, + "theoretical_loss": 3.7450606340019874, + "tokens_seen": 827785216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006053841524573722, + "loss": 1.4237, + "theoretical_loss": 3.7450316468165936, + "tokens_seen": 827850752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006053681043129388, + "loss": 1.4083, + "theoretical_loss": 3.7450026625683215, + "tokens_seen": 827916288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006053520561685056, + "loss": 1.4614, + "theoretical_loss": 3.7449736812566403, + "tokens_seen": 827981824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006053360080240722, + "loss": 1.3581, + "theoretical_loss": 3.744944702881021, + "tokens_seen": 828047360 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605319959879639, + "loss": 1.422, + "theoretical_loss": 3.744915727440933, + "tokens_seen": 828112896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006053039117352056, + "loss": 1.5054, + "theoretical_loss": 3.7448867549358473, + "tokens_seen": 828178432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006052878635907724, + "loss": 1.4287, + "theoretical_loss": 3.744857785365234, + "tokens_seen": 828243968 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605271815446339, + "loss": 1.3261, + "theoretical_loss": 3.744828818728563, + "tokens_seen": 828309504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006052557673019058, + "loss": 1.4221, + "theoretical_loss": 3.744799855025307, + "tokens_seen": 828375040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006052397191574724, + "loss": 1.286, + "theoretical_loss": 3.744770894254935, + "tokens_seen": 828440576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006052236710130392, + "loss": 1.3079, + "theoretical_loss": 3.744741936416919, + "tokens_seen": 828506112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006052076228686058, + "loss": 1.318, + "theoretical_loss": 3.74471298151073, + "tokens_seen": 828571648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006051915747241726, + "loss": 1.4599, + "theoretical_loss": 3.744684029535839, + "tokens_seen": 828637184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006051755265797392, + "loss": 1.3454, + "theoretical_loss": 3.744655080491718, + "tokens_seen": 828702720 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605159478435306, + "loss": 1.3969, + "theoretical_loss": 3.744626134377838, + "tokens_seen": 828768256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006051434302908726, + "loss": 1.4725, + "theoretical_loss": 3.7445971911936713, + "tokens_seen": 828833792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006051273821464394, + "loss": 1.4707, + "theoretical_loss": 3.7445682509386895, + "tokens_seen": 828899328 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605111334002006, + "loss": 1.4693, + "theoretical_loss": 3.7445393136123646, + "tokens_seen": 828964864 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 281837, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4682958126068115, + "objective/train/theoretical_loss": 3.7445103792141685, + "objective/train/tokens_used": 849490400, + "theoretical_loss": 3.7445103792141685, + "tokens_seen": 829030400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050952858575728, + "loss": 1.4146, + "theoretical_loss": 3.7445103792141685, + "tokens_seen": 829030400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050792377131394, + "loss": 1.2954, + "theoretical_loss": 3.7444814477435737, + "tokens_seen": 829095936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050631895687062, + "loss": 1.3835, + "theoretical_loss": 3.7444525192000526, + "tokens_seen": 829161472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050471414242728, + "loss": 1.3096, + "theoretical_loss": 3.744423593583078, + "tokens_seen": 829227008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050310932798396, + "loss": 1.518, + "theoretical_loss": 3.744394670892122, + "tokens_seen": 829292544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050150451354062, + "loss": 1.393, + "theoretical_loss": 3.744365751126658, + "tokens_seen": 829358080 + }, + { + "epoch": 0.25, + "learning_rate": 0.000604998996990973, + "loss": 1.3358, + "theoretical_loss": 3.744336834286158, + "tokens_seen": 829423616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006049829488465396, + "loss": 1.4868, + "theoretical_loss": 3.7443079203700966, + "tokens_seen": 829489152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006049669007021064, + "loss": 1.4051, + "theoretical_loss": 3.7442790093779452, + "tokens_seen": 829554688 + }, + { + "epoch": 0.25, + "learning_rate": 0.000604950852557673, + "loss": 1.3656, + "theoretical_loss": 3.744250101309179, + "tokens_seen": 829620224 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006049348044132398, + "loss": 1.4104, + "theoretical_loss": 3.74422119616327, + "tokens_seen": 829685760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006049187562688064, + "loss": 1.3818, + "theoretical_loss": 3.744192293939693, + "tokens_seen": 829751296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006049027081243732, + "loss": 1.4822, + "theoretical_loss": 3.744163394637921, + "tokens_seen": 829816832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006048866599799398, + "loss": 1.412, + "theoretical_loss": 3.744134498257428, + "tokens_seen": 829882368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006048706118355066, + "loss": 1.4419, + "theoretical_loss": 3.7441056047976886, + "tokens_seen": 829947904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006048545636910732, + "loss": 1.3883, + "theoretical_loss": 3.744076714258177, + "tokens_seen": 830013440 + }, + { + "epoch": 0.25, + "learning_rate": 0.00060483851554664, + "loss": 1.5272, + "theoretical_loss": 3.744047826638366, + "tokens_seen": 830078976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006048224674022067, + "loss": 1.415, + "theoretical_loss": 3.7440189419377314, + "tokens_seen": 830144512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006048064192577734, + "loss": 1.4296, + "theoretical_loss": 3.743990060155748, + "tokens_seen": 830210048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00060479037111334, + "loss": 1.4556, + "theoretical_loss": 3.7439611812918896, + "tokens_seen": 830275584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006047743229689068, + "loss": 1.4024, + "theoretical_loss": 3.743932305345632, + "tokens_seen": 830341120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006047582748244735, + "loss": 1.3821, + "theoretical_loss": 3.7439034323164493, + "tokens_seen": 830406656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006047422266800402, + "loss": 1.3547, + "theoretical_loss": 3.7438745622038176, + "tokens_seen": 830472192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006047261785356069, + "loss": 1.3311, + "theoretical_loss": 3.743845695007211, + "tokens_seen": 830537728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006047101303911736, + "loss": 1.3625, + "theoretical_loss": 3.7438168307261055, + "tokens_seen": 830603264 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006046940822467403, + "loss": 1.3913, + "theoretical_loss": 3.7437879693599774, + "tokens_seen": 830668800 + }, + { + "epoch": 0.25, + "learning_rate": 0.000604678034102307, + "loss": 1.4088, + "theoretical_loss": 3.743759110908301, + "tokens_seen": 830734336 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006046619859578737, + "loss": 1.4352, + "theoretical_loss": 3.743730255370553, + "tokens_seen": 830799872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006046459378134404, + "loss": 1.4227, + "theoretical_loss": 3.743701402746209, + "tokens_seen": 830865408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006046298896690071, + "loss": 1.3402, + "theoretical_loss": 3.743672553034745, + "tokens_seen": 830930944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006046138415245738, + "loss": 1.3676, + "theoretical_loss": 3.743643706235638, + "tokens_seen": 830996480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045977933801405, + "loss": 1.3668, + "theoretical_loss": 3.7436148623483634, + "tokens_seen": 831062016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045817452357072, + "loss": 1.5535, + "theoretical_loss": 3.7435860213723977, + "tokens_seen": 831127552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045656970912739, + "loss": 1.4146, + "theoretical_loss": 3.7435571833072183, + "tokens_seen": 831193088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045496489468406, + "loss": 1.3039, + "theoretical_loss": 3.743528348152301, + "tokens_seen": 831258624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045336008024073, + "loss": 1.3983, + "theoretical_loss": 3.743499515907124, + "tokens_seen": 831324160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045175526579739, + "loss": 1.4228, + "theoretical_loss": 3.7434706865711633, + "tokens_seen": 831389696 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045015045135407, + "loss": 1.3759, + "theoretical_loss": 3.7434418601438963, + "tokens_seen": 831455232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044854563691073, + "loss": 1.4089, + "theoretical_loss": 3.7434130366248004, + "tokens_seen": 831520768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044694082246741, + "loss": 1.3858, + "theoretical_loss": 3.743384216013353, + "tokens_seen": 831586304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044533600802407, + "loss": 1.3688, + "theoretical_loss": 3.7433553983090313, + "tokens_seen": 831651840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044373119358075, + "loss": 1.4525, + "theoretical_loss": 3.743326583511313, + "tokens_seen": 831717376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044212637913741, + "loss": 1.4081, + "theoretical_loss": 3.7432977716196767, + "tokens_seen": 831782912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044052156469409, + "loss": 1.4298, + "theoretical_loss": 3.7432689626336, + "tokens_seen": 831848448 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006043891675025075, + "loss": 1.3914, + "theoretical_loss": 3.7432401565525613, + "tokens_seen": 831913984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006043731193580742, + "loss": 1.393, + "theoretical_loss": 3.743211353376038, + "tokens_seen": 831979520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006043570712136409, + "loss": 1.3862, + "theoretical_loss": 3.743182553103509, + "tokens_seen": 832045056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006043410230692076, + "loss": 1.4645, + "theoretical_loss": 3.7431537557344527, + "tokens_seen": 832110592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006043249749247743, + "loss": 1.3851, + "theoretical_loss": 3.7431249612683484, + "tokens_seen": 832176128 + }, + { + "epoch": 0.25, + "learning_rate": 0.000604308926780341, + "loss": 1.4357, + "theoretical_loss": 3.743096169704674, + "tokens_seen": 832241664 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 282632, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4518851041793823, + "objective/train/theoretical_loss": 3.7430673810429083, + "objective/train/tokens_used": 852767200, + "theoretical_loss": 3.7430673810429083, + "tokens_seen": 832307200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042928786359077, + "loss": 1.4014, + "theoretical_loss": 3.7430673810429083, + "tokens_seen": 832307200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042768304914744, + "loss": 1.4634, + "theoretical_loss": 3.743038595282531, + "tokens_seen": 832372736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042607823470411, + "loss": 1.4312, + "theoretical_loss": 3.7430098124230216, + "tokens_seen": 832438272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042447342026078, + "loss": 1.4061, + "theoretical_loss": 3.7429810324638586, + "tokens_seen": 832503808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042286860581745, + "loss": 1.4185, + "theoretical_loss": 3.7429522554045223, + "tokens_seen": 832569344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042126379137412, + "loss": 1.3821, + "theoretical_loss": 3.7429234812444907, + "tokens_seen": 832634880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041965897693079, + "loss": 1.4198, + "theoretical_loss": 3.7428947099832452, + "tokens_seen": 832700416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041805416248747, + "loss": 1.3011, + "theoretical_loss": 3.7428659416202654, + "tokens_seen": 832765952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041644934804413, + "loss": 1.3807, + "theoretical_loss": 3.7428371761550303, + "tokens_seen": 832831488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041484453360081, + "loss": 1.4291, + "theoretical_loss": 3.7428084135870208, + "tokens_seen": 832897024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041323971915747, + "loss": 1.2841, + "theoretical_loss": 3.7427796539157177, + "tokens_seen": 832962560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041163490471415, + "loss": 1.4131, + "theoretical_loss": 3.7427508971406, + "tokens_seen": 833028096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041003009027081, + "loss": 1.3391, + "theoretical_loss": 3.7427221432611493, + "tokens_seen": 833093632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040842527582749, + "loss": 1.4184, + "theoretical_loss": 3.7426933922768457, + "tokens_seen": 833159168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040682046138415, + "loss": 1.4064, + "theoretical_loss": 3.74266464418717, + "tokens_seen": 833224704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040521564694083, + "loss": 1.3462, + "theoretical_loss": 3.7426358989916038, + "tokens_seen": 833290240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040361083249749, + "loss": 1.4502, + "theoretical_loss": 3.742607156689628, + "tokens_seen": 833355776 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040200601805417, + "loss": 1.3462, + "theoretical_loss": 3.742578417280723, + "tokens_seen": 833421312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040040120361083, + "loss": 1.3551, + "theoretical_loss": 3.7425496807643706, + "tokens_seen": 833486848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039879638916751, + "loss": 1.4537, + "theoretical_loss": 3.7425209471400525, + "tokens_seen": 833552384 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039719157472417, + "loss": 1.3427, + "theoretical_loss": 3.7424922164072503, + "tokens_seen": 833617920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039558676028085, + "loss": 1.3983, + "theoretical_loss": 3.7424634885654453, + "tokens_seen": 833683456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039398194583751, + "loss": 1.2434, + "theoretical_loss": 3.7424347636141198, + "tokens_seen": 833748992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039237713139419, + "loss": 1.4972, + "theoretical_loss": 3.742406041552756, + "tokens_seen": 833814528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039077231695085, + "loss": 1.3897, + "theoretical_loss": 3.742377322380835, + "tokens_seen": 833880064 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038916750250753, + "loss": 1.3202, + "theoretical_loss": 3.74234860609784, + "tokens_seen": 833945600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038756268806419, + "loss": 1.3765, + "theoretical_loss": 3.742319892703253, + "tokens_seen": 834011136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038595787362087, + "loss": 1.4412, + "theoretical_loss": 3.7422911821965568, + "tokens_seen": 834076672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038435305917753, + "loss": 1.3687, + "theoretical_loss": 3.7422624745772337, + "tokens_seen": 834142208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038274824473421, + "loss": 1.3772, + "theoretical_loss": 3.742233769844767, + "tokens_seen": 834207744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038114343029087, + "loss": 1.3646, + "theoretical_loss": 3.74220506799864, + "tokens_seen": 834273280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037953861584755, + "loss": 1.4583, + "theoretical_loss": 3.7421763690383347, + "tokens_seen": 834338816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037793380140421, + "loss": 1.4014, + "theoretical_loss": 3.742147672963334, + "tokens_seen": 834404352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037632898696089, + "loss": 1.4687, + "theoretical_loss": 3.742118979773123, + "tokens_seen": 834469888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037472417251755, + "loss": 1.4343, + "theoretical_loss": 3.742090289467184, + "tokens_seen": 834535424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037311935807423, + "loss": 1.3913, + "theoretical_loss": 3.7420616020450006, + "tokens_seen": 834600960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037151454363089, + "loss": 1.4343, + "theoretical_loss": 3.742032917506057, + "tokens_seen": 834666496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036990972918757, + "loss": 1.4094, + "theoretical_loss": 3.742004235849836, + "tokens_seen": 834732032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036830491474423, + "loss": 1.3836, + "theoretical_loss": 3.7419755570758233, + "tokens_seen": 834797568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036670010030091, + "loss": 1.3085, + "theoretical_loss": 3.7419468811835017, + "tokens_seen": 834863104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036509528585757, + "loss": 1.3428, + "theoretical_loss": 3.741918208172356, + "tokens_seen": 834928640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036349047141425, + "loss": 1.461, + "theoretical_loss": 3.74188953804187, + "tokens_seen": 834994176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036188565697091, + "loss": 1.3668, + "theoretical_loss": 3.7418608707915295, + "tokens_seen": 835059712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036028084252759, + "loss": 1.4113, + "theoretical_loss": 3.741832206420818, + "tokens_seen": 835125248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006035867602808425, + "loss": 1.3914, + "theoretical_loss": 3.7418035449292204, + "tokens_seen": 835190784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006035707121364093, + "loss": 1.4598, + "theoretical_loss": 3.741774886316222, + "tokens_seen": 835256320 + }, + { + "epoch": 0.25, + "learning_rate": 0.000603554663991976, + "loss": 1.4639, + "theoretical_loss": 3.741746230581308, + "tokens_seen": 835321856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006035386158475427, + "loss": 1.3883, + "theoretical_loss": 3.7417175777239633, + "tokens_seen": 835387392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006035225677031093, + "loss": 1.4573, + "theoretical_loss": 3.741688927743673, + "tokens_seen": 835452928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006035065195586761, + "loss": 1.5791, + "theoretical_loss": 3.7416602806399233, + "tokens_seen": 835518464 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 283853, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.289686679840088, + "objective/train/theoretical_loss": 3.7416316364121993, + "objective/train/tokens_used": 856044000, + "theoretical_loss": 3.7416316364121993, + "tokens_seen": 835584000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034904714142427, + "loss": 1.394, + "theoretical_loss": 3.7416316364121993, + "tokens_seen": 835584000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034744232698095, + "loss": 1.412, + "theoretical_loss": 3.7416029950599867, + "tokens_seen": 835649536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034583751253762, + "loss": 1.3217, + "theoretical_loss": 3.7415743565827713, + "tokens_seen": 835715072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034423269809429, + "loss": 1.4093, + "theoretical_loss": 3.741545720980039, + "tokens_seen": 835780608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034262788365096, + "loss": 1.3962, + "theoretical_loss": 3.7415170882512765, + "tokens_seen": 835846144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034102306920763, + "loss": 1.4114, + "theoretical_loss": 3.7414884583959696, + "tokens_seen": 835911680 + }, + { + "epoch": 0.25, + "learning_rate": 0.000603394182547643, + "loss": 1.4147, + "theoretical_loss": 3.7414598314136045, + "tokens_seen": 835977216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006033781344032097, + "loss": 1.4463, + "theoretical_loss": 3.741431207303668, + "tokens_seen": 836042752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006033620862587764, + "loss": 1.2938, + "theoretical_loss": 3.7414025860656475, + "tokens_seen": 836108288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006033460381143431, + "loss": 1.6035, + "theoretical_loss": 3.7413739676990283, + "tokens_seen": 836173824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006033299899699098, + "loss": 1.4507, + "theoretical_loss": 3.741345352203298, + "tokens_seen": 836239360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006033139418254765, + "loss": 1.3425, + "theoretical_loss": 3.741316739577944, + "tokens_seen": 836304896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032978936810432, + "loss": 1.4659, + "theoretical_loss": 3.741288129822453, + "tokens_seen": 836370432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032818455366099, + "loss": 1.4809, + "theoretical_loss": 3.7412595229363124, + "tokens_seen": 836435968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032657973921766, + "loss": 1.4753, + "theoretical_loss": 3.74123091891901, + "tokens_seen": 836501504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032497492477433, + "loss": 1.3901, + "theoretical_loss": 3.741202317770033, + "tokens_seen": 836567040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00060323370110331, + "loss": 1.3956, + "theoretical_loss": 3.7411737194888692, + "tokens_seen": 836632576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032176529588767, + "loss": 1.4074, + "theoretical_loss": 3.7411451240750067, + "tokens_seen": 836698112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032016048144434, + "loss": 1.4154, + "theoretical_loss": 3.7411165315279327, + "tokens_seen": 836763648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031855566700101, + "loss": 1.4605, + "theoretical_loss": 3.7410879418471357, + "tokens_seen": 836829184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031695085255768, + "loss": 1.4226, + "theoretical_loss": 3.741059355032104, + "tokens_seen": 836894720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031534603811435, + "loss": 1.4547, + "theoretical_loss": 3.7410307710823263, + "tokens_seen": 836960256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031374122367102, + "loss": 1.4513, + "theoretical_loss": 3.7410021899972907, + "tokens_seen": 837025792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031213640922769, + "loss": 1.3366, + "theoretical_loss": 3.740973611776486, + "tokens_seen": 837091328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031053159478436, + "loss": 1.3743, + "theoretical_loss": 3.740945036419401, + "tokens_seen": 837156864 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006030892678034102, + "loss": 1.4312, + "theoretical_loss": 3.740916463925524, + "tokens_seen": 837222400 + }, + { + "epoch": 0.25, + "learning_rate": 0.000603073219658977, + "loss": 1.5133, + "theoretical_loss": 3.740887894294344, + "tokens_seen": 837287936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006030571715145436, + "loss": 1.4867, + "theoretical_loss": 3.740859327525352, + "tokens_seen": 837353472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006030411233701104, + "loss": 1.423, + "theoretical_loss": 3.740830763618035, + "tokens_seen": 837419008 + }, + { + "epoch": 0.25, + "learning_rate": 0.000603025075225677, + "loss": 1.5043, + "theoretical_loss": 3.7408022025718832, + "tokens_seen": 837484544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006030090270812438, + "loss": 1.34, + "theoretical_loss": 3.740773644386387, + "tokens_seen": 837550080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006029929789368104, + "loss": 1.3819, + "theoretical_loss": 3.7407450890610345, + "tokens_seen": 837615616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006029769307923772, + "loss": 1.4022, + "theoretical_loss": 3.7407165365953166, + "tokens_seen": 837681152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006029608826479438, + "loss": 1.4855, + "theoretical_loss": 3.740687986988723, + "tokens_seen": 837746688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006029448345035106, + "loss": 1.3891, + "theoretical_loss": 3.740659440240744, + "tokens_seen": 837812224 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006029287863590772, + "loss": 1.358, + "theoretical_loss": 3.740630896350869, + "tokens_seen": 837877760 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602912738214644, + "loss": 1.3737, + "theoretical_loss": 3.740602355318589, + "tokens_seen": 837943296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028966900702106, + "loss": 1.3636, + "theoretical_loss": 3.7405738171433947, + "tokens_seen": 838008832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028806419257774, + "loss": 1.4324, + "theoretical_loss": 3.740545281824776, + "tokens_seen": 838074368 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602864593781344, + "loss": 1.3236, + "theoretical_loss": 3.7405167493622242, + "tokens_seen": 838139904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028485456369108, + "loss": 1.3804, + "theoretical_loss": 3.740488219755229, + "tokens_seen": 838205440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028324974924774, + "loss": 1.3289, + "theoretical_loss": 3.740459693003283, + "tokens_seen": 838270976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028164493480442, + "loss": 1.4463, + "theoretical_loss": 3.7404311691058765, + "tokens_seen": 838336512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028004012036108, + "loss": 1.514, + "theoretical_loss": 3.740402648062501, + "tokens_seen": 838402048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006027843530591776, + "loss": 1.4703, + "theoretical_loss": 3.740374129872647, + "tokens_seen": 838467584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006027683049147442, + "loss": 1.4832, + "theoretical_loss": 3.7403456145358076, + "tokens_seen": 838533120 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602752256770311, + "loss": 1.3512, + "theoretical_loss": 3.740317102051473, + "tokens_seen": 838598656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006027362086258776, + "loss": 1.3195, + "theoretical_loss": 3.7402885924191356, + "tokens_seen": 838664192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006027201604814444, + "loss": 1.3966, + "theoretical_loss": 3.740260085638287, + "tokens_seen": 838729728 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602704112337011, + "loss": 1.3818, + "theoretical_loss": 3.7402315817084197, + "tokens_seen": 838795264 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 284714, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.401942253112793, + "objective/train/theoretical_loss": 3.7402030806290254, + "objective/train/tokens_used": 859320800, + "theoretical_loss": 3.7402030806290254, + "tokens_seen": 838860800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026880641925778, + "loss": 1.3878, + "theoretical_loss": 3.7402030806290254, + "tokens_seen": 838860800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026720160481444, + "loss": 1.4426, + "theoretical_loss": 3.7401745823995967, + "tokens_seen": 838926336 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026559679037112, + "loss": 1.4389, + "theoretical_loss": 3.7401460870196255, + "tokens_seen": 838991872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026399197592778, + "loss": 1.4171, + "theoretical_loss": 3.7401175944886047, + "tokens_seen": 839057408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026238716148446, + "loss": 1.3983, + "theoretical_loss": 3.740089104806027, + "tokens_seen": 839122944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026078234704112, + "loss": 1.5401, + "theoretical_loss": 3.7400606179713853, + "tokens_seen": 839188480 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602591775325978, + "loss": 1.4772, + "theoretical_loss": 3.7400321339841724, + "tokens_seen": 839254016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006025757271815446, + "loss": 1.4685, + "theoretical_loss": 3.740003652843882, + "tokens_seen": 839319552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006025596790371114, + "loss": 1.4278, + "theoretical_loss": 3.739975174550006, + "tokens_seen": 839385088 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602543630892678, + "loss": 1.3401, + "theoretical_loss": 3.7399466991020383, + "tokens_seen": 839450624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006025275827482448, + "loss": 1.4309, + "theoretical_loss": 3.739918226499473, + "tokens_seen": 839516160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006025115346038114, + "loss": 1.3303, + "theoretical_loss": 3.7398897567418024, + "tokens_seen": 839581696 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006024954864593782, + "loss": 1.351, + "theoretical_loss": 3.7398612898285215, + "tokens_seen": 839647232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006024794383149448, + "loss": 1.3971, + "theoretical_loss": 3.7398328257591236, + "tokens_seen": 839712768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006024633901705116, + "loss": 1.3685, + "theoretical_loss": 3.7398043645331027, + "tokens_seen": 839778304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006024473420260782, + "loss": 1.4835, + "theoretical_loss": 3.7397759061499523, + "tokens_seen": 839843840 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602431293881645, + "loss": 1.3469, + "theoretical_loss": 3.7397474506091677, + "tokens_seen": 839909376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006024152457372116, + "loss": 1.3865, + "theoretical_loss": 3.7397189979102428, + "tokens_seen": 839974912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023991975927784, + "loss": 1.4223, + "theoretical_loss": 3.739690548052672, + "tokens_seen": 840040448 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602383149448345, + "loss": 1.3895, + "theoretical_loss": 3.7396621010359494, + "tokens_seen": 840105984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023671013039118, + "loss": 1.3644, + "theoretical_loss": 3.739633656859571, + "tokens_seen": 840171520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023510531594784, + "loss": 1.4034, + "theoretical_loss": 3.739605215523031, + "tokens_seen": 840237056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023350050150452, + "loss": 1.4005, + "theoretical_loss": 3.7395767770258246, + "tokens_seen": 840302592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023189568706118, + "loss": 1.4012, + "theoretical_loss": 3.739548341367446, + "tokens_seen": 840368128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023029087261786, + "loss": 1.3468, + "theoretical_loss": 3.7395199085473916, + "tokens_seen": 840433664 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006022868605817452, + "loss": 1.4116, + "theoretical_loss": 3.7394914785651565, + "tokens_seen": 840499200 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602270812437312, + "loss": 1.4516, + "theoretical_loss": 3.739463051420236, + "tokens_seen": 840564736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006022547642928786, + "loss": 1.4447, + "theoretical_loss": 3.739434627112126, + "tokens_seen": 840630272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006022387161484454, + "loss": 1.4481, + "theoretical_loss": 3.7394062056403223, + "tokens_seen": 840695808 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602222668004012, + "loss": 1.4441, + "theoretical_loss": 3.73937778700432, + "tokens_seen": 840761344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006022066198595788, + "loss": 1.4476, + "theoretical_loss": 3.739349371203616, + "tokens_seen": 840826880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006021905717151454, + "loss": 1.3999, + "theoretical_loss": 3.739320958237707, + "tokens_seen": 840892416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006021745235707122, + "loss": 1.375, + "theoretical_loss": 3.7392925481060884, + "tokens_seen": 840957952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006021584754262788, + "loss": 1.4156, + "theoretical_loss": 3.739264140808256, + "tokens_seen": 841023488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006021424272818456, + "loss": 1.3174, + "theoretical_loss": 3.7392357363437077, + "tokens_seen": 841089024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006021263791374122, + "loss": 1.3175, + "theoretical_loss": 3.73920733471194, + "tokens_seen": 841154560 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602110330992979, + "loss": 1.3119, + "theoretical_loss": 3.7391789359124488, + "tokens_seen": 841220096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006020942828485457, + "loss": 1.218, + "theoretical_loss": 3.739150539944732, + "tokens_seen": 841285632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006020782347041124, + "loss": 1.3743, + "theoretical_loss": 3.7391221468082856, + "tokens_seen": 841351168 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602062186559679, + "loss": 1.4401, + "theoretical_loss": 3.7390937565026086, + "tokens_seen": 841416704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006020461384152458, + "loss": 1.3299, + "theoretical_loss": 3.7390653690271964, + "tokens_seen": 841482240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006020300902708125, + "loss": 1.334, + "theoretical_loss": 3.7390369843815474, + "tokens_seen": 841547776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006020140421263792, + "loss": 1.4229, + "theoretical_loss": 3.7390086025651588, + "tokens_seen": 841613312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019979939819459, + "loss": 1.4636, + "theoretical_loss": 3.7389802235775287, + "tokens_seen": 841678848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019819458375126, + "loss": 1.432, + "theoretical_loss": 3.738951847418155, + "tokens_seen": 841744384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019658976930793, + "loss": 1.4492, + "theoretical_loss": 3.738923474086535, + "tokens_seen": 841809920 + }, + { + "epoch": 0.26, + "learning_rate": 0.000601949849548646, + "loss": 1.4188, + "theoretical_loss": 3.7388951035821676, + "tokens_seen": 841875456 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019338014042127, + "loss": 1.3371, + "theoretical_loss": 3.7388667359045504, + "tokens_seen": 841940992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019177532597794, + "loss": 1.3195, + "theoretical_loss": 3.7388383710531823, + "tokens_seen": 842006528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019017051153461, + "loss": 1.3625, + "theoretical_loss": 3.7388100090275618, + "tokens_seen": 842072064 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 285855, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4833232164382935, + "objective/train/theoretical_loss": 3.738781649827186, + "objective/train/tokens_used": 862597600, + "theoretical_loss": 3.738781649827186, + "tokens_seen": 842137600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018856569709128, + "loss": 1.4449, + "theoretical_loss": 3.738781649827186, + "tokens_seen": 842137600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018696088264795, + "loss": 1.4223, + "theoretical_loss": 3.7387532934515564, + "tokens_seen": 842203136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018535606820462, + "loss": 1.5405, + "theoretical_loss": 3.738724939900169, + "tokens_seen": 842268672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018375125376129, + "loss": 1.3723, + "theoretical_loss": 3.738696589172525, + "tokens_seen": 842334208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018214643931796, + "loss": 1.3751, + "theoretical_loss": 3.7386682412681225, + "tokens_seen": 842399744 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018054162487463, + "loss": 1.4065, + "theoretical_loss": 3.73863989618646, + "tokens_seen": 842465280 + }, + { + "epoch": 0.26, + "learning_rate": 0.000601789368104313, + "loss": 1.3013, + "theoretical_loss": 3.738611553927039, + "tokens_seen": 842530816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006017733199598797, + "loss": 1.3471, + "theoretical_loss": 3.738583214489357, + "tokens_seen": 842596352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006017572718154464, + "loss": 1.4147, + "theoretical_loss": 3.7385548778729145, + "tokens_seen": 842661888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006017412236710131, + "loss": 1.3634, + "theoretical_loss": 3.738526544077211, + "tokens_seen": 842727424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006017251755265798, + "loss": 1.4423, + "theoretical_loss": 3.738498213101747, + "tokens_seen": 842792960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006017091273821465, + "loss": 1.4134, + "theoretical_loss": 3.7384698849460216, + "tokens_seen": 842858496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016930792377131, + "loss": 1.3371, + "theoretical_loss": 3.7384415596095355, + "tokens_seen": 842924032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016770310932799, + "loss": 1.362, + "theoretical_loss": 3.7384132370917884, + "tokens_seen": 842989568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016609829488465, + "loss": 1.3966, + "theoretical_loss": 3.738384917392282, + "tokens_seen": 843055104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016449348044133, + "loss": 1.3893, + "theoretical_loss": 3.7383566005105155, + "tokens_seen": 843120640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016288866599799, + "loss": 1.3767, + "theoretical_loss": 3.7383282864459897, + "tokens_seen": 843186176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016128385155467, + "loss": 1.4102, + "theoretical_loss": 3.7382999751982062, + "tokens_seen": 843251712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015967903711133, + "loss": 1.4282, + "theoretical_loss": 3.738271666766665, + "tokens_seen": 843317248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015807422266801, + "loss": 1.4793, + "theoretical_loss": 3.738243361150868, + "tokens_seen": 843382784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015646940822467, + "loss": 1.3901, + "theoretical_loss": 3.7382150583503155, + "tokens_seen": 843448320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015486459378135, + "loss": 1.4639, + "theoretical_loss": 3.7381867583645096, + "tokens_seen": 843513856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015325977933801, + "loss": 1.5218, + "theoretical_loss": 3.738158461192951, + "tokens_seen": 843579392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015165496489469, + "loss": 1.4485, + "theoretical_loss": 3.738130166835141, + "tokens_seen": 843644928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015005015045135, + "loss": 1.3031, + "theoretical_loss": 3.7381018752905826, + "tokens_seen": 843710464 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014844533600803, + "loss": 1.4174, + "theoretical_loss": 3.7380735865587766, + "tokens_seen": 843776000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014684052156469, + "loss": 1.3375, + "theoretical_loss": 3.738045300639225, + "tokens_seen": 843841536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014523570712137, + "loss": 1.4707, + "theoretical_loss": 3.7380170175314302, + "tokens_seen": 843907072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014363089267803, + "loss": 1.3438, + "theoretical_loss": 3.737988737234894, + "tokens_seen": 843972608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014202607823471, + "loss": 1.39, + "theoretical_loss": 3.737960459749119, + "tokens_seen": 844038144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014042126379137, + "loss": 1.4359, + "theoretical_loss": 3.737932185073607, + "tokens_seen": 844103680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013881644934805, + "loss": 1.534, + "theoretical_loss": 3.7379039132078615, + "tokens_seen": 844169216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013721163490471, + "loss": 1.4283, + "theoretical_loss": 3.7378756441513845, + "tokens_seen": 844234752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013560682046139, + "loss": 1.3978, + "theoretical_loss": 3.737847377903679, + "tokens_seen": 844300288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013400200601805, + "loss": 1.4508, + "theoretical_loss": 3.737819114464248, + "tokens_seen": 844365824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013239719157473, + "loss": 1.3086, + "theoretical_loss": 3.7377908538325944, + "tokens_seen": 844431360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013079237713139, + "loss": 1.4145, + "theoretical_loss": 3.7377625960082215, + "tokens_seen": 844496896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012918756268807, + "loss": 1.2873, + "theoretical_loss": 3.7377343409906327, + "tokens_seen": 844562432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012758274824473, + "loss": 1.3702, + "theoretical_loss": 3.7377060887793316, + "tokens_seen": 844627968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012597793380141, + "loss": 1.3439, + "theoretical_loss": 3.7376778393738217, + "tokens_seen": 844693504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012437311935807, + "loss": 1.4976, + "theoretical_loss": 3.737649592773606, + "tokens_seen": 844759040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012276830491475, + "loss": 1.3224, + "theoretical_loss": 3.737621348978189, + "tokens_seen": 844824576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012116349047141, + "loss": 1.3752, + "theoretical_loss": 3.7375931079870743, + "tokens_seen": 844890112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011955867602809, + "loss": 1.4234, + "theoretical_loss": 3.7375648697997663, + "tokens_seen": 844955648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011795386158475, + "loss": 1.3911, + "theoretical_loss": 3.737536634415769, + "tokens_seen": 845021184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011634904714143, + "loss": 1.3739, + "theoretical_loss": 3.737508401834587, + "tokens_seen": 845086720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011474423269809, + "loss": 1.3997, + "theoretical_loss": 3.7374801720557245, + "tokens_seen": 845152256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011313941825477, + "loss": 1.501, + "theoretical_loss": 3.737451945078686, + "tokens_seen": 845217792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011153460381143, + "loss": 1.2698, + "theoretical_loss": 3.737423720902976, + "tokens_seen": 845283328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010992978936811, + "loss": 1.4256, + "theoretical_loss": 3.7373954995281005, + "tokens_seen": 845348864 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 286799, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.428529977798462, + "objective/train/theoretical_loss": 3.7373672809535634, + "objective/train/tokens_used": 865874400, + "theoretical_loss": 3.7373672809535634, + "tokens_seen": 845414400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010832497492477, + "loss": 1.3799, + "theoretical_loss": 3.7373672809535634, + "tokens_seen": 845414400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010672016048145, + "loss": 1.4102, + "theoretical_loss": 3.7373390651788694, + "tokens_seen": 845479936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010511534603811, + "loss": 1.3426, + "theoretical_loss": 3.7373108522035245, + "tokens_seen": 845545472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010351053159479, + "loss": 1.3097, + "theoretical_loss": 3.737282642027034, + "tokens_seen": 845611008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010190571715145, + "loss": 1.2863, + "theoretical_loss": 3.737254434648903, + "tokens_seen": 845676544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010030090270813, + "loss": 1.3013, + "theoretical_loss": 3.7372262300686376, + "tokens_seen": 845742080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009869608826479, + "loss": 1.4221, + "theoretical_loss": 3.737198028285743, + "tokens_seen": 845807616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009709127382147, + "loss": 1.324, + "theoretical_loss": 3.7371698292997255, + "tokens_seen": 845873152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009548645937813, + "loss": 1.3582, + "theoretical_loss": 3.73714163311009, + "tokens_seen": 845938688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009388164493481, + "loss": 1.3429, + "theoretical_loss": 3.737113439716344, + "tokens_seen": 846004224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009227683049147, + "loss": 1.4675, + "theoretical_loss": 3.737085249117993, + "tokens_seen": 846069760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009067201604815, + "loss": 1.4306, + "theoretical_loss": 3.737057061314543, + "tokens_seen": 846135296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006008906720160481, + "loss": 1.5029, + "theoretical_loss": 3.7370288763055015, + "tokens_seen": 846200832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006008746238716149, + "loss": 1.371, + "theoretical_loss": 3.7370006940903737, + "tokens_seen": 846266368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006008585757271815, + "loss": 1.4102, + "theoretical_loss": 3.7369725146686674, + "tokens_seen": 846331904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006008425275827483, + "loss": 1.4441, + "theoretical_loss": 3.7369443380398897, + "tokens_seen": 846397440 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600826479438315, + "loss": 1.4409, + "theoretical_loss": 3.736916164203546, + "tokens_seen": 846462976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006008104312938817, + "loss": 1.6028, + "theoretical_loss": 3.7368879931591454, + "tokens_seen": 846528512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007943831494483, + "loss": 1.3932, + "theoretical_loss": 3.736859824906193, + "tokens_seen": 846594048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007783350050151, + "loss": 1.2522, + "theoretical_loss": 3.736831659444198, + "tokens_seen": 846659584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007622868605817, + "loss": 1.3381, + "theoretical_loss": 3.7368034967726667, + "tokens_seen": 846725120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007462387161485, + "loss": 1.3857, + "theoretical_loss": 3.736775336891107, + "tokens_seen": 846790656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007301905717151, + "loss": 1.3398, + "theoretical_loss": 3.736747179799027, + "tokens_seen": 846856192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007141424272819, + "loss": 1.3893, + "theoretical_loss": 3.736719025495934, + "tokens_seen": 846921728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006980942828486, + "loss": 1.4846, + "theoretical_loss": 3.7366908739813365, + "tokens_seen": 846987264 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006820461384153, + "loss": 1.3505, + "theoretical_loss": 3.736662725254742, + "tokens_seen": 847052800 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600665997993982, + "loss": 1.3939, + "theoretical_loss": 3.736634579315659, + "tokens_seen": 847118336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006499498495487, + "loss": 1.321, + "theoretical_loss": 3.736606436163596, + "tokens_seen": 847183872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006339017051154, + "loss": 1.4235, + "theoretical_loss": 3.7365782957980613, + "tokens_seen": 847249408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006178535606821, + "loss": 1.4006, + "theoretical_loss": 3.736550158218563, + "tokens_seen": 847314944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006018054162488, + "loss": 1.4018, + "theoretical_loss": 3.736522023424611, + "tokens_seen": 847380480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005857572718155, + "loss": 1.3999, + "theoretical_loss": 3.736493891415713, + "tokens_seen": 847446016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005697091273823, + "loss": 1.3375, + "theoretical_loss": 3.7364657621913784, + "tokens_seen": 847511552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005536609829489, + "loss": 1.2714, + "theoretical_loss": 3.7364376357511166, + "tokens_seen": 847577088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005376128385157, + "loss": 1.355, + "theoretical_loss": 3.7364095120944363, + "tokens_seen": 847642624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005215646940823, + "loss": 1.2644, + "theoretical_loss": 3.736381391220847, + "tokens_seen": 847708160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005055165496491, + "loss": 1.469, + "theoretical_loss": 3.736353273129858, + "tokens_seen": 847773696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004894684052157, + "loss": 1.397, + "theoretical_loss": 3.7363251578209797, + "tokens_seen": 847839232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004734202607825, + "loss": 1.4887, + "theoretical_loss": 3.7362970452937208, + "tokens_seen": 847904768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004573721163491, + "loss": 1.4575, + "theoretical_loss": 3.7362689355475913, + "tokens_seen": 847970304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004413239719158, + "loss": 1.3535, + "theoretical_loss": 3.7362408285821016, + "tokens_seen": 848035840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004252758274825, + "loss": 1.4311, + "theoretical_loss": 3.7362127243967613, + "tokens_seen": 848101376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004092276830492, + "loss": 1.4185, + "theoretical_loss": 3.7361846229910816, + "tokens_seen": 848166912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006003931795386159, + "loss": 1.2694, + "theoretical_loss": 3.736156524364571, + "tokens_seen": 848232448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006003771313941826, + "loss": 1.3745, + "theoretical_loss": 3.736128428516742, + "tokens_seen": 848297984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006003610832497493, + "loss": 1.4684, + "theoretical_loss": 3.736100335447104, + "tokens_seen": 848363520 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600345035105316, + "loss": 1.3414, + "theoretical_loss": 3.736072245155168, + "tokens_seen": 848429056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006003289869608827, + "loss": 1.4138, + "theoretical_loss": 3.7360441576404444, + "tokens_seen": 848494592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006003129388164494, + "loss": 1.4227, + "theoretical_loss": 3.7360160729024443, + "tokens_seen": 848560128 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600296890672016, + "loss": 1.4908, + "theoretical_loss": 3.7359879909406795, + "tokens_seen": 848625664 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 287940, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3254786729812622, + "objective/train/theoretical_loss": 3.7359599117546605, + "objective/train/tokens_used": 869151200, + "theoretical_loss": 3.7359599117546605, + "tokens_seen": 848691200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002808425275828, + "loss": 1.35, + "theoretical_loss": 3.7359599117546605, + "tokens_seen": 848691200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002647943831494, + "loss": 1.4168, + "theoretical_loss": 3.735931835343899, + "tokens_seen": 848756736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002487462387162, + "loss": 1.5306, + "theoretical_loss": 3.7359037617079056, + "tokens_seen": 848822272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002326980942828, + "loss": 1.3725, + "theoretical_loss": 3.735875690846193, + "tokens_seen": 848887808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002166499498496, + "loss": 1.3881, + "theoretical_loss": 3.735847622758272, + "tokens_seen": 848953344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002006018054162, + "loss": 1.4268, + "theoretical_loss": 3.7358195574436555, + "tokens_seen": 849018880 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600184553660983, + "loss": 1.3647, + "theoretical_loss": 3.7357914949018545, + "tokens_seen": 849084416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006001685055165496, + "loss": 1.4098, + "theoretical_loss": 3.735763435132381, + "tokens_seen": 849149952 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006001524573721164, + "loss": 1.2927, + "theoretical_loss": 3.735735378134748, + "tokens_seen": 849215488 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600136409227683, + "loss": 1.304, + "theoretical_loss": 3.735707323908467, + "tokens_seen": 849281024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006001203610832498, + "loss": 1.3165, + "theoretical_loss": 3.7356792724530505, + "tokens_seen": 849346560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006001043129388164, + "loss": 1.3498, + "theoretical_loss": 3.735651223768012, + "tokens_seen": 849412096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006000882647943832, + "loss": 1.3796, + "theoretical_loss": 3.735623177852863, + "tokens_seen": 849477632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006000722166499498, + "loss": 1.3351, + "theoretical_loss": 3.7355951347071166, + "tokens_seen": 849543168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006000561685055166, + "loss": 1.4175, + "theoretical_loss": 3.7355670943302863, + "tokens_seen": 849608704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006000401203610832, + "loss": 1.4171, + "theoretical_loss": 3.735539056721885, + "tokens_seen": 849674240 + }, + { + "epoch": 0.26, + "learning_rate": 0.00060002407221665, + "loss": 1.3831, + "theoretical_loss": 3.7355110218814254, + "tokens_seen": 849739776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006000080240722166, + "loss": 1.324, + "theoretical_loss": 3.7354829898084208, + "tokens_seen": 849805312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005999919759277834, + "loss": 1.4369, + "theoretical_loss": 3.735454960502385, + "tokens_seen": 849870848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00059997592778335, + "loss": 1.4079, + "theoretical_loss": 3.735426933962831, + "tokens_seen": 849936384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005999598796389168, + "loss": 1.4077, + "theoretical_loss": 3.7353989101892733, + "tokens_seen": 850001920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005999438314944834, + "loss": 1.4385, + "theoretical_loss": 3.7353708891812256, + "tokens_seen": 850067456 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005999277833500502, + "loss": 1.5027, + "theoretical_loss": 3.735342870938201, + "tokens_seen": 850132992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005999117352056168, + "loss": 1.3286, + "theoretical_loss": 3.7353148554597144, + "tokens_seen": 850198528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005998956870611836, + "loss": 1.4426, + "theoretical_loss": 3.7352868427452792, + "tokens_seen": 850264064 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005998796389167502, + "loss": 1.3627, + "theoretical_loss": 3.7352588327944103, + "tokens_seen": 850329600 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599863590772317, + "loss": 1.423, + "theoretical_loss": 3.735230825606622, + "tokens_seen": 850395136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005998475426278836, + "loss": 1.4013, + "theoretical_loss": 3.735202821181428, + "tokens_seen": 850460672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005998314944834504, + "loss": 1.5009, + "theoretical_loss": 3.7351748195183436, + "tokens_seen": 850526208 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599815446339017, + "loss": 1.4742, + "theoretical_loss": 3.735146820616884, + "tokens_seen": 850591744 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997993981945838, + "loss": 1.4065, + "theoretical_loss": 3.735118824476564, + "tokens_seen": 850657280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997833500501504, + "loss": 1.4179, + "theoretical_loss": 3.7350908310968975, + "tokens_seen": 850722816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997673019057172, + "loss": 1.4582, + "theoretical_loss": 3.735062840477401, + "tokens_seen": 850788352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997512537612838, + "loss": 1.4548, + "theoretical_loss": 3.735034852617589, + "tokens_seen": 850853888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997352056168506, + "loss": 1.3788, + "theoretical_loss": 3.7350068675169776, + "tokens_seen": 850919424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997191574724172, + "loss": 1.4552, + "theoretical_loss": 3.734978885175081, + "tokens_seen": 850984960 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599703109327984, + "loss": 1.4249, + "theoretical_loss": 3.734950905591416, + "tokens_seen": 851050496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005996870611835506, + "loss": 1.3944, + "theoretical_loss": 3.7349229287654984, + "tokens_seen": 851116032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005996710130391174, + "loss": 1.3482, + "theoretical_loss": 3.734894954696843, + "tokens_seen": 851181568 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599654964894684, + "loss": 1.4952, + "theoretical_loss": 3.7348669833849666, + "tokens_seen": 851247104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005996389167502508, + "loss": 1.3647, + "theoretical_loss": 3.734839014829385, + "tokens_seen": 851312640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005996228686058174, + "loss": 1.3801, + "theoretical_loss": 3.7348110490296147, + "tokens_seen": 851378176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005996068204613842, + "loss": 1.4457, + "theoretical_loss": 3.734783085985172, + "tokens_seen": 851443712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005995907723169508, + "loss": 1.3893, + "theoretical_loss": 3.7347551256955738, + "tokens_seen": 851509248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005995747241725176, + "loss": 1.3184, + "theoretical_loss": 3.734727168160336, + "tokens_seen": 851574784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005995586760280842, + "loss": 1.4529, + "theoretical_loss": 3.734699213378975, + "tokens_seen": 851640320 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599542627883651, + "loss": 1.4379, + "theoretical_loss": 3.734671261351009, + "tokens_seen": 851705856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005995265797392176, + "loss": 1.3981, + "theoretical_loss": 3.7346433120759537, + "tokens_seen": 851771392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005995105315947844, + "loss": 1.422, + "theoretical_loss": 3.734615365553327, + "tokens_seen": 851836928 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599494483450351, + "loss": 1.4512, + "theoretical_loss": 3.7345874217826456, + "tokens_seen": 851902464 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 288857, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3938164710998535, + "objective/train/theoretical_loss": 3.734559480763427, + "objective/train/tokens_used": 872428000, + "theoretical_loss": 3.734559480763427, + "tokens_seen": 851968000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005994784353059178, + "loss": 1.3835, + "theoretical_loss": 3.734559480763427, + "tokens_seen": 851968000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005994623871614844, + "loss": 1.3932, + "theoretical_loss": 3.734531542495189, + "tokens_seen": 852033536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005994463390170512, + "loss": 1.2591, + "theoretical_loss": 3.7345036069774484, + "tokens_seen": 852099072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005994302908726178, + "loss": 1.3624, + "theoretical_loss": 3.734475674209724, + "tokens_seen": 852164608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005994142427281846, + "loss": 1.4323, + "theoretical_loss": 3.7344477441915322, + "tokens_seen": 852230144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993981945837514, + "loss": 1.4479, + "theoretical_loss": 3.7344198169223928, + "tokens_seen": 852295680 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599382146439318, + "loss": 1.4421, + "theoretical_loss": 3.734391892401822, + "tokens_seen": 852361216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993660982948848, + "loss": 1.3885, + "theoretical_loss": 3.7343639706293392, + "tokens_seen": 852426752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993500501504514, + "loss": 1.3702, + "theoretical_loss": 3.734336051604462, + "tokens_seen": 852492288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993340020060182, + "loss": 1.3556, + "theoretical_loss": 3.7343081353267094, + "tokens_seen": 852557824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993179538615848, + "loss": 1.427, + "theoretical_loss": 3.7342802217955997, + "tokens_seen": 852623360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993019057171516, + "loss": 1.4236, + "theoretical_loss": 3.7342523110106516, + "tokens_seen": 852688896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005992858575727182, + "loss": 1.3168, + "theoretical_loss": 3.7342244029713845, + "tokens_seen": 852754432 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599269809428285, + "loss": 1.4055, + "theoretical_loss": 3.7341964976773157, + "tokens_seen": 852819968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005992537612838516, + "loss": 1.3059, + "theoretical_loss": 3.734168595127966, + "tokens_seen": 852885504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005992377131394184, + "loss": 1.3196, + "theoretical_loss": 3.7341406953228535, + "tokens_seen": 852951040 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599221664994985, + "loss": 1.3316, + "theoretical_loss": 3.7341127982614974, + "tokens_seen": 853016576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005992056168505518, + "loss": 1.3972, + "theoretical_loss": 3.734084903943418, + "tokens_seen": 853082112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005991895687061184, + "loss": 1.4237, + "theoretical_loss": 3.734057012368134, + "tokens_seen": 853147648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005991735205616852, + "loss": 1.4201, + "theoretical_loss": 3.7340291235351657, + "tokens_seen": 853213184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005991574724172518, + "loss": 1.4324, + "theoretical_loss": 3.734001237444032, + "tokens_seen": 853278720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005991414242728186, + "loss": 1.4876, + "theoretical_loss": 3.7339733540942537, + "tokens_seen": 853344256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005991253761283852, + "loss": 1.2994, + "theoretical_loss": 3.7339454734853508, + "tokens_seen": 853409792 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599109327983952, + "loss": 1.342, + "theoretical_loss": 3.733917595616842, + "tokens_seen": 853475328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005990932798395186, + "loss": 1.4272, + "theoretical_loss": 3.7338897204882495, + "tokens_seen": 853540864 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005990772316950854, + "loss": 1.3983, + "theoretical_loss": 3.7338618480990924, + "tokens_seen": 853606400 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599061183550652, + "loss": 1.43, + "theoretical_loss": 3.7338339784488914, + "tokens_seen": 853671936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005990451354062187, + "loss": 1.3391, + "theoretical_loss": 3.7338061115371675, + "tokens_seen": 853737472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005990290872617854, + "loss": 1.2993, + "theoretical_loss": 3.7337782473634413, + "tokens_seen": 853803008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005990130391173521, + "loss": 1.3782, + "theoretical_loss": 3.733750385927234, + "tokens_seen": 853868544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989969909729188, + "loss": 1.4878, + "theoretical_loss": 3.733722527228065, + "tokens_seen": 853934080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989809428284855, + "loss": 1.4, + "theoretical_loss": 3.7336946712654573, + "tokens_seen": 853999616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989648946840522, + "loss": 1.3102, + "theoretical_loss": 3.733666818038931, + "tokens_seen": 854065152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989488465396189, + "loss": 1.2489, + "theoretical_loss": 3.733638967548008, + "tokens_seen": 854130688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989327983951856, + "loss": 1.4477, + "theoretical_loss": 3.733611119792209, + "tokens_seen": 854196224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989167502507523, + "loss": 1.3739, + "theoretical_loss": 3.733583274771057, + "tokens_seen": 854261760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989007021063189, + "loss": 1.4298, + "theoretical_loss": 3.733555432484072, + "tokens_seen": 854327296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988846539618857, + "loss": 1.3673, + "theoretical_loss": 3.733527592930777, + "tokens_seen": 854392832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988686058174523, + "loss": 1.2787, + "theoretical_loss": 3.7334997561106933, + "tokens_seen": 854458368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988525576730191, + "loss": 1.4309, + "theoretical_loss": 3.7334719220233437, + "tokens_seen": 854523904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988365095285857, + "loss": 1.3878, + "theoretical_loss": 3.7334440906682493, + "tokens_seen": 854589440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988204613841525, + "loss": 1.4067, + "theoretical_loss": 3.7334162620449334, + "tokens_seen": 854654976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988044132397191, + "loss": 1.2822, + "theoretical_loss": 3.733388436152918, + "tokens_seen": 854720512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987883650952859, + "loss": 1.4486, + "theoretical_loss": 3.733360612991725, + "tokens_seen": 854786048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987723169508525, + "loss": 1.5065, + "theoretical_loss": 3.733332792560878, + "tokens_seen": 854851584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987562688064193, + "loss": 1.3977, + "theoretical_loss": 3.7333049748599, + "tokens_seen": 854917120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987402206619859, + "loss": 1.4028, + "theoretical_loss": 3.7332771598883134, + "tokens_seen": 854982656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987241725175527, + "loss": 1.4019, + "theoretical_loss": 3.7332493476456405, + "tokens_seen": 855048192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987081243731193, + "loss": 1.4379, + "theoretical_loss": 3.7332215381314056, + "tokens_seen": 855113728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986920762286861, + "loss": 1.4441, + "theoretical_loss": 3.733193731345131, + "tokens_seen": 855179264 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 290023, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.482955813407898, + "objective/train/theoretical_loss": 3.733165927286341, + "objective/train/tokens_used": 875704800, + "theoretical_loss": 3.733165927286341, + "tokens_seen": 855244800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986760280842527, + "loss": 1.3843, + "theoretical_loss": 3.733165927286341, + "tokens_seen": 855244800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986599799398195, + "loss": 1.3889, + "theoretical_loss": 3.7331381259545586, + "tokens_seen": 855310336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986439317953861, + "loss": 1.4774, + "theoretical_loss": 3.7331103273493076, + "tokens_seen": 855375872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986278836509529, + "loss": 1.4163, + "theoretical_loss": 3.733082531470111, + "tokens_seen": 855441408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986118355065195, + "loss": 1.4104, + "theoretical_loss": 3.7330547383164943, + "tokens_seen": 855506944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985957873620863, + "loss": 1.3666, + "theoretical_loss": 3.7330269478879794, + "tokens_seen": 855572480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985797392176529, + "loss": 1.4714, + "theoretical_loss": 3.732999160184092, + "tokens_seen": 855638016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985636910732197, + "loss": 1.3387, + "theoretical_loss": 3.7329713752043556, + "tokens_seen": 855703552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985476429287863, + "loss": 1.378, + "theoretical_loss": 3.7329435929482946, + "tokens_seen": 855769088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985315947843531, + "loss": 1.4485, + "theoretical_loss": 3.732915813415434, + "tokens_seen": 855834624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985155466399197, + "loss": 1.32, + "theoretical_loss": 3.732888036605298, + "tokens_seen": 855900160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984994984954865, + "loss": 1.4106, + "theoretical_loss": 3.7328602625174105, + "tokens_seen": 855965696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984834503510531, + "loss": 1.4481, + "theoretical_loss": 3.7328324911512976, + "tokens_seen": 856031232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984674022066199, + "loss": 1.2989, + "theoretical_loss": 3.7328047225064838, + "tokens_seen": 856096768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984513540621865, + "loss": 1.3665, + "theoretical_loss": 3.7327769565824935, + "tokens_seen": 856162304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984353059177533, + "loss": 1.2125, + "theoretical_loss": 3.7327491933788526, + "tokens_seen": 856227840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984192577733199, + "loss": 1.3547, + "theoretical_loss": 3.7327214328950857, + "tokens_seen": 856293376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984032096288867, + "loss": 1.3171, + "theoretical_loss": 3.7326936751307196, + "tokens_seen": 856358912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983871614844533, + "loss": 1.4804, + "theoretical_loss": 3.732665920085278, + "tokens_seen": 856424448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983711133400201, + "loss": 1.3623, + "theoretical_loss": 3.732638167758288, + "tokens_seen": 856489984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983550651955867, + "loss": 1.4546, + "theoretical_loss": 3.7326104181492745, + "tokens_seen": 856555520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983390170511535, + "loss": 1.4384, + "theoretical_loss": 3.732582671257764, + "tokens_seen": 856621056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983229689067201, + "loss": 1.3613, + "theoretical_loss": 3.732554927083282, + "tokens_seen": 856686592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983069207622869, + "loss": 1.5608, + "theoretical_loss": 3.7325271856253552, + "tokens_seen": 856752128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982908726178535, + "loss": 1.4797, + "theoretical_loss": 3.732499446883509, + "tokens_seen": 856817664 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982748244734203, + "loss": 1.3227, + "theoretical_loss": 3.7324717108572703, + "tokens_seen": 856883200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982587763289869, + "loss": 1.3517, + "theoretical_loss": 3.7324439775461657, + "tokens_seen": 856948736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982427281845537, + "loss": 1.4239, + "theoretical_loss": 3.7324162469497217, + "tokens_seen": 857014272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982266800401203, + "loss": 1.4292, + "theoretical_loss": 3.732388519067465, + "tokens_seen": 857079808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982106318956871, + "loss": 1.4278, + "theoretical_loss": 3.732360793898922, + "tokens_seen": 857145344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981945837512539, + "loss": 1.3697, + "theoretical_loss": 3.7323330714436205, + "tokens_seen": 857210880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981785356068205, + "loss": 1.4008, + "theoretical_loss": 3.7323053517010862, + "tokens_seen": 857276416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981624874623873, + "loss": 1.3641, + "theoretical_loss": 3.732277634670848, + "tokens_seen": 857341952 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981464393179539, + "loss": 1.3369, + "theoretical_loss": 3.732249920352432, + "tokens_seen": 857407488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981303911735207, + "loss": 1.4052, + "theoretical_loss": 3.732222208745366, + "tokens_seen": 857473024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981143430290873, + "loss": 1.3744, + "theoretical_loss": 3.732194499849178, + "tokens_seen": 857538560 + }, + { + "epoch": 0.26, + "learning_rate": 0.000598098294884654, + "loss": 1.3696, + "theoretical_loss": 3.732166793663395, + "tokens_seen": 857604096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980822467402207, + "loss": 1.3836, + "theoretical_loss": 3.7321390901875455, + "tokens_seen": 857669632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980661985957875, + "loss": 1.3862, + "theoretical_loss": 3.7321113894211564, + "tokens_seen": 857735168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980501504513541, + "loss": 1.4014, + "theoretical_loss": 3.732083691363756, + "tokens_seen": 857800704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980341023069209, + "loss": 1.3548, + "theoretical_loss": 3.732055996014873, + "tokens_seen": 857866240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980180541624875, + "loss": 1.4254, + "theoretical_loss": 3.732028303374035, + "tokens_seen": 857931776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980020060180543, + "loss": 1.4273, + "theoretical_loss": 3.7320006134407713, + "tokens_seen": 857997312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979859578736209, + "loss": 1.3563, + "theoretical_loss": 3.7319729262146093, + "tokens_seen": 858062848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979699097291877, + "loss": 1.3577, + "theoretical_loss": 3.7319452416950782, + "tokens_seen": 858128384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979538615847543, + "loss": 1.2903, + "theoretical_loss": 3.731917559881707, + "tokens_seen": 858193920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979378134403211, + "loss": 1.4991, + "theoretical_loss": 3.7318898807740237, + "tokens_seen": 858259456 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979217652958877, + "loss": 1.3234, + "theoretical_loss": 3.7318622043715584, + "tokens_seen": 858324992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979057171514545, + "loss": 1.4198, + "theoretical_loss": 3.731834530673839, + "tokens_seen": 858390528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978896690070211, + "loss": 1.3654, + "theoretical_loss": 3.731806859680395, + "tokens_seen": 858456064 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 291206, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.360671043395996, + "objective/train/theoretical_loss": 3.7317791913907565, + "objective/train/tokens_used": 878981600, + "theoretical_loss": 3.7317791913907565, + "tokens_seen": 858521600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978736208625879, + "loss": 1.3988, + "theoretical_loss": 3.7317791913907565, + "tokens_seen": 858521600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978575727181545, + "loss": 1.4183, + "theoretical_loss": 3.7317515258044525, + "tokens_seen": 858587136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978415245737213, + "loss": 1.4504, + "theoretical_loss": 3.731723862921012, + "tokens_seen": 858652672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978254764292879, + "loss": 1.3, + "theoretical_loss": 3.7316962027399656, + "tokens_seen": 858718208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978094282848547, + "loss": 1.3513, + "theoretical_loss": 3.731668545260842, + "tokens_seen": 858783744 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977933801404213, + "loss": 1.3975, + "theoretical_loss": 3.731640890483172, + "tokens_seen": 858849280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977773319959881, + "loss": 1.3566, + "theoretical_loss": 3.7316132384064855, + "tokens_seen": 858914816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977612838515547, + "loss": 1.2503, + "theoretical_loss": 3.7315855890303125, + "tokens_seen": 858980352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977452357071215, + "loss": 1.3104, + "theoretical_loss": 3.7315579423541836, + "tokens_seen": 859045888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977291875626881, + "loss": 1.3923, + "theoretical_loss": 3.731530298377628, + "tokens_seen": 859111424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977131394182549, + "loss": 1.3938, + "theoretical_loss": 3.7315026571001777, + "tokens_seen": 859176960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976970912738215, + "loss": 1.3879, + "theoretical_loss": 3.731475018521362, + "tokens_seen": 859242496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976810431293883, + "loss": 1.4689, + "theoretical_loss": 3.731447382640713, + "tokens_seen": 859308032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976649949849549, + "loss": 1.3008, + "theoretical_loss": 3.73141974945776, + "tokens_seen": 859373568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976489468405216, + "loss": 1.399, + "theoretical_loss": 3.7313921189720354, + "tokens_seen": 859439104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976328986960883, + "loss": 1.2753, + "theoretical_loss": 3.7313644911830695, + "tokens_seen": 859504640 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597616850551655, + "loss": 1.3152, + "theoretical_loss": 3.731336866090394, + "tokens_seen": 859570176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976008024072217, + "loss": 1.4584, + "theoretical_loss": 3.7313092436935396, + "tokens_seen": 859635712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975847542627884, + "loss": 1.4569, + "theoretical_loss": 3.7312816239920377, + "tokens_seen": 859701248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975687061183551, + "loss": 1.3976, + "theoretical_loss": 3.731254006985421, + "tokens_seen": 859766784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975526579739218, + "loss": 1.3175, + "theoretical_loss": 3.7312263926732196, + "tokens_seen": 859832320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975366098294885, + "loss": 1.3636, + "theoretical_loss": 3.731198781054966, + "tokens_seen": 859897856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975205616850552, + "loss": 1.3564, + "theoretical_loss": 3.7311711721301926, + "tokens_seen": 859963392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975045135406218, + "loss": 1.348, + "theoretical_loss": 3.7311435658984307, + "tokens_seen": 860028928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005974884653961886, + "loss": 1.4549, + "theoretical_loss": 3.7311159623592127, + "tokens_seen": 860094464 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005974724172517552, + "loss": 1.3301, + "theoretical_loss": 3.7310883615120707, + "tokens_seen": 860160000 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597456369107322, + "loss": 1.385, + "theoretical_loss": 3.7310607633565374, + "tokens_seen": 860225536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005974403209628886, + "loss": 1.3361, + "theoretical_loss": 3.7310331678921456, + "tokens_seen": 860291072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005974242728184554, + "loss": 1.4569, + "theoretical_loss": 3.731005575118427, + "tokens_seen": 860356608 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597408224674022, + "loss": 1.4702, + "theoretical_loss": 3.7309779850349143, + "tokens_seen": 860422144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973921765295888, + "loss": 1.3233, + "theoretical_loss": 3.730950397641141, + "tokens_seen": 860487680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973761283851554, + "loss": 1.3819, + "theoretical_loss": 3.73092281293664, + "tokens_seen": 860553216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973600802407222, + "loss": 1.3594, + "theoretical_loss": 3.730895230920944, + "tokens_seen": 860618752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973440320962888, + "loss": 1.2954, + "theoretical_loss": 3.730867651593586, + "tokens_seen": 860684288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973279839518556, + "loss": 1.3937, + "theoretical_loss": 3.7308400749541004, + "tokens_seen": 860749824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973119358074222, + "loss": 1.4639, + "theoretical_loss": 3.7308125010020197, + "tokens_seen": 860815360 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597295887662989, + "loss": 1.4161, + "theoretical_loss": 3.7307849297368776, + "tokens_seen": 860880896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005972798395185556, + "loss": 1.4161, + "theoretical_loss": 3.730757361158208, + "tokens_seen": 860946432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005972637913741224, + "loss": 1.4708, + "theoretical_loss": 3.7307297952655443, + "tokens_seen": 861011968 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597247743229689, + "loss": 1.2609, + "theoretical_loss": 3.7307022320584204, + "tokens_seen": 861077504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005972316950852558, + "loss": 1.522, + "theoretical_loss": 3.730674671536371, + "tokens_seen": 861143040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005972156469408224, + "loss": 1.511, + "theoretical_loss": 3.7306471136989288, + "tokens_seen": 861208576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971995987963892, + "loss": 1.3947, + "theoretical_loss": 3.7306195585456297, + "tokens_seen": 861274112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971835506519558, + "loss": 1.3479, + "theoretical_loss": 3.730592006076007, + "tokens_seen": 861339648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971675025075226, + "loss": 1.2669, + "theoretical_loss": 3.730564456289595, + "tokens_seen": 861405184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971514543630892, + "loss": 1.394, + "theoretical_loss": 3.730536909185929, + "tokens_seen": 861470720 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597135406218656, + "loss": 1.4651, + "theoretical_loss": 3.7305093647645435, + "tokens_seen": 861536256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971193580742226, + "loss": 1.3565, + "theoretical_loss": 3.730481823024973, + "tokens_seen": 861601792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971033099297894, + "loss": 1.4322, + "theoretical_loss": 3.730454283966753, + "tokens_seen": 861667328 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597087261785356, + "loss": 1.302, + "theoretical_loss": 3.7304267475894184, + "tokens_seen": 861732864 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 292159, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5102205276489258, + "objective/train/theoretical_loss": 3.730399213892504, + "objective/train/tokens_used": 882258400, + "theoretical_loss": 3.730399213892504, + "tokens_seen": 861798400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005970712136409228, + "loss": 1.4731, + "theoretical_loss": 3.730399213892504, + "tokens_seen": 861798400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005970551654964894, + "loss": 1.4135, + "theoretical_loss": 3.730371682875545, + "tokens_seen": 861863936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005970391173520562, + "loss": 1.4878, + "theoretical_loss": 3.730344154538077, + "tokens_seen": 861929472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005970230692076228, + "loss": 1.3914, + "theoretical_loss": 3.7303166288796357, + "tokens_seen": 861995008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005970070210631896, + "loss": 1.5092, + "theoretical_loss": 3.7302891058997565, + "tokens_seen": 862060544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005969909729187563, + "loss": 1.4024, + "theoretical_loss": 3.730261585597976, + "tokens_seen": 862126080 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596974924774323, + "loss": 1.399, + "theoretical_loss": 3.7302340679738286, + "tokens_seen": 862191616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005969588766298897, + "loss": 1.445, + "theoretical_loss": 3.730206553026851, + "tokens_seen": 862257152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005969428284854564, + "loss": 1.5203, + "theoretical_loss": 3.730179040756579, + "tokens_seen": 862322688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005969267803410231, + "loss": 1.322, + "theoretical_loss": 3.7301515311625497, + "tokens_seen": 862388224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005969107321965898, + "loss": 1.3981, + "theoretical_loss": 3.730124024244298, + "tokens_seen": 862453760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005968946840521565, + "loss": 1.4271, + "theoretical_loss": 3.730096520001362, + "tokens_seen": 862519296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005968786359077232, + "loss": 1.477, + "theoretical_loss": 3.730069018433277, + "tokens_seen": 862584832 + }, + { + "epoch": 0.26, + "learning_rate": 0.00059686258776329, + "loss": 1.3415, + "theoretical_loss": 3.73004151953958, + "tokens_seen": 862650368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005968465396188566, + "loss": 1.508, + "theoretical_loss": 3.730014023319808, + "tokens_seen": 862715904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005968304914744233, + "loss": 1.3229, + "theoretical_loss": 3.7299865297734973, + "tokens_seen": 862781440 + }, + { + "epoch": 0.26, + "learning_rate": 0.00059681444332999, + "loss": 1.4359, + "theoretical_loss": 3.7299590389001858, + "tokens_seen": 862846976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967983951855568, + "loss": 1.3911, + "theoretical_loss": 3.72993155069941, + "tokens_seen": 862912512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967823470411234, + "loss": 1.581, + "theoretical_loss": 3.7299040651707074, + "tokens_seen": 862978048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967662988966902, + "loss": 1.3578, + "theoretical_loss": 3.7298765823136155, + "tokens_seen": 863043584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967502507522568, + "loss": 1.3588, + "theoretical_loss": 3.7298491021276714, + "tokens_seen": 863109120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967342026078236, + "loss": 1.3742, + "theoretical_loss": 3.7298216246124127, + "tokens_seen": 863174656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967181544633902, + "loss": 1.4532, + "theoretical_loss": 3.729794149767377, + "tokens_seen": 863240192 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596702106318957, + "loss": 1.32, + "theoretical_loss": 3.729766677592103, + "tokens_seen": 863305728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005966860581745236, + "loss": 1.3781, + "theoretical_loss": 3.7297392080861282, + "tokens_seen": 863371264 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005966700100300904, + "loss": 1.3734, + "theoretical_loss": 3.7297117412489897, + "tokens_seen": 863436800 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596653961885657, + "loss": 1.3839, + "theoretical_loss": 3.729684277080227, + "tokens_seen": 863502336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005966379137412238, + "loss": 1.4134, + "theoretical_loss": 3.7296568155793777, + "tokens_seen": 863567872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005966218655967904, + "loss": 1.3296, + "theoretical_loss": 3.7296293567459804, + "tokens_seen": 863633408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005966058174523572, + "loss": 1.4878, + "theoretical_loss": 3.7296019005795733, + "tokens_seen": 863698944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005965897693079238, + "loss": 1.4308, + "theoretical_loss": 3.7295744470796954, + "tokens_seen": 863764480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005965737211634906, + "loss": 1.4164, + "theoretical_loss": 3.729546996245885, + "tokens_seen": 863830016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005965576730190572, + "loss": 1.395, + "theoretical_loss": 3.7295195480776817, + "tokens_seen": 863895552 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596541624874624, + "loss": 1.3631, + "theoretical_loss": 3.729492102574624, + "tokens_seen": 863961088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005965255767301906, + "loss": 1.4525, + "theoretical_loss": 3.7294646597362506, + "tokens_seen": 864026624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005965095285857574, + "loss": 1.4787, + "theoretical_loss": 3.729437219562101, + "tokens_seen": 864092160 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596493480441324, + "loss": 1.4133, + "theoretical_loss": 3.7294097820517154, + "tokens_seen": 864157696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005964774322968908, + "loss": 1.5079, + "theoretical_loss": 3.7293823472046315, + "tokens_seen": 864223232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005964613841524574, + "loss": 1.3559, + "theoretical_loss": 3.729354915020391, + "tokens_seen": 864288768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005964453360080242, + "loss": 1.5025, + "theoretical_loss": 3.729327485498531, + "tokens_seen": 864354304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005964292878635908, + "loss": 1.3629, + "theoretical_loss": 3.7293000586385934, + "tokens_seen": 864419840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005964132397191576, + "loss": 1.4256, + "theoretical_loss": 3.7292726344401173, + "tokens_seen": 864485376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005963971915747242, + "loss": 1.5274, + "theoretical_loss": 3.7292452129026428, + "tokens_seen": 864550912 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596381143430291, + "loss": 1.4937, + "theoretical_loss": 3.7292177940257094, + "tokens_seen": 864616448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005963650952858576, + "loss": 1.3922, + "theoretical_loss": 3.7291903778088584, + "tokens_seen": 864681984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005963490471414244, + "loss": 1.4654, + "theoretical_loss": 3.7291629642516293, + "tokens_seen": 864747520 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596332998996991, + "loss": 1.3205, + "theoretical_loss": 3.7291355533535633, + "tokens_seen": 864813056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005963169508525578, + "loss": 1.3455, + "theoretical_loss": 3.7291081451142003, + "tokens_seen": 864878592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005963009027081244, + "loss": 1.5219, + "theoretical_loss": 3.729080739533081, + "tokens_seen": 864944128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962848545636912, + "loss": 1.3509, + "theoretical_loss": 3.7290533366097467, + "tokens_seen": 865009664 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 293335, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5877540111541748, + "objective/train/theoretical_loss": 3.7290259363437377, + "objective/train/tokens_used": 885535200, + "theoretical_loss": 3.7290259363437377, + "tokens_seen": 865075200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962688064192578, + "loss": 1.4697, + "theoretical_loss": 3.7290259363437377, + "tokens_seen": 865075200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962527582748246, + "loss": 1.4238, + "theoretical_loss": 3.7289985387345963, + "tokens_seen": 865140736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962367101303912, + "loss": 1.4721, + "theoretical_loss": 3.7289711437818616, + "tokens_seen": 865206272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962206619859579, + "loss": 1.4989, + "theoretical_loss": 3.7289437514850765, + "tokens_seen": 865271808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962046138415246, + "loss": 1.3594, + "theoretical_loss": 3.728916361843782, + "tokens_seen": 865337344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005961885656970913, + "loss": 1.3321, + "theoretical_loss": 3.728888974857519, + "tokens_seen": 865402880 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596172517552658, + "loss": 1.3389, + "theoretical_loss": 3.7288615905258298, + "tokens_seen": 865468416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005961564694082247, + "loss": 1.4009, + "theoretical_loss": 3.728834208848256, + "tokens_seen": 865533952 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005961404212637914, + "loss": 1.439, + "theoretical_loss": 3.7288068298243386, + "tokens_seen": 865599488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005961243731193581, + "loss": 1.3532, + "theoretical_loss": 3.7287794534536207, + "tokens_seen": 865665024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005961083249749248, + "loss": 1.3144, + "theoretical_loss": 3.728752079735644, + "tokens_seen": 865730560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960922768304915, + "loss": 1.448, + "theoretical_loss": 3.728724708669951, + "tokens_seen": 865796096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960762286860581, + "loss": 1.3527, + "theoretical_loss": 3.7286973402560823, + "tokens_seen": 865861632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960601805416249, + "loss": 1.3912, + "theoretical_loss": 3.7286699744935827, + "tokens_seen": 865927168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960441323971915, + "loss": 1.4209, + "theoretical_loss": 3.728642611381993, + "tokens_seen": 865992704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960280842527583, + "loss": 1.3568, + "theoretical_loss": 3.728615250920856, + "tokens_seen": 866058240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960120361083249, + "loss": 1.4931, + "theoretical_loss": 3.728587893109715, + "tokens_seen": 866123776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959959879638917, + "loss": 1.3623, + "theoretical_loss": 3.7285605379481126, + "tokens_seen": 866189312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959799398194583, + "loss": 1.3508, + "theoretical_loss": 3.728533185435592, + "tokens_seen": 866254848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959638916750251, + "loss": 1.3635, + "theoretical_loss": 3.7285058355716965, + "tokens_seen": 866320384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959478435305917, + "loss": 1.4739, + "theoretical_loss": 3.728478488355968, + "tokens_seen": 866385920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959317953861585, + "loss": 1.4582, + "theoretical_loss": 3.728451143787951, + "tokens_seen": 866451456 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959157472417251, + "loss": 1.4172, + "theoretical_loss": 3.7284238018671885, + "tokens_seen": 866516992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958996990972919, + "loss": 1.2999, + "theoretical_loss": 3.728396462593224, + "tokens_seen": 866582528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958836509528585, + "loss": 1.4225, + "theoretical_loss": 3.7283691259656018, + "tokens_seen": 866648064 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958676028084253, + "loss": 1.3977, + "theoretical_loss": 3.7283417919838646, + "tokens_seen": 866713600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958515546639919, + "loss": 1.3359, + "theoretical_loss": 3.7283144606475576, + "tokens_seen": 866779136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958355065195587, + "loss": 1.5323, + "theoretical_loss": 3.7282871319562227, + "tokens_seen": 866844672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958194583751253, + "loss": 1.304, + "theoretical_loss": 3.7282598059094063, + "tokens_seen": 866910208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958034102306921, + "loss": 1.3826, + "theoretical_loss": 3.728232482506651, + "tokens_seen": 866975744 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957873620862588, + "loss": 1.361, + "theoretical_loss": 3.7282051617475016, + "tokens_seen": 867041280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957713139418255, + "loss": 1.4337, + "theoretical_loss": 3.728177843631503, + "tokens_seen": 867106816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957552657973922, + "loss": 1.413, + "theoretical_loss": 3.7281505281581993, + "tokens_seen": 867172352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957392176529589, + "loss": 1.4236, + "theoretical_loss": 3.7281232153271353, + "tokens_seen": 867237888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957231695085256, + "loss": 1.4413, + "theoretical_loss": 3.728095905137856, + "tokens_seen": 867303424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957071213640923, + "loss": 1.4082, + "theoretical_loss": 3.728068597589905, + "tokens_seen": 867368960 + }, + { + "epoch": 0.26, + "learning_rate": 0.000595691073219659, + "loss": 1.4053, + "theoretical_loss": 3.7280412926828292, + "tokens_seen": 867434496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005956750250752257, + "loss": 1.2886, + "theoretical_loss": 3.7280139904161724, + "tokens_seen": 867500032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005956589769307924, + "loss": 1.3978, + "theoretical_loss": 3.7279866907894803, + "tokens_seen": 867565568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005956429287863591, + "loss": 1.3115, + "theoretical_loss": 3.7279593938022986, + "tokens_seen": 867631104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005956268806419258, + "loss": 1.4112, + "theoretical_loss": 3.7279320994541716, + "tokens_seen": 867696640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005956108324974925, + "loss": 1.37, + "theoretical_loss": 3.727904807744646, + "tokens_seen": 867762176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005955947843530592, + "loss": 1.4193, + "theoretical_loss": 3.7278775186732664, + "tokens_seen": 867827712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005955787362086259, + "loss": 1.4642, + "theoretical_loss": 3.7278502322395797, + "tokens_seen": 867893248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005955626880641926, + "loss": 1.4219, + "theoretical_loss": 3.727822948443132, + "tokens_seen": 867958784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005955466399197593, + "loss": 1.4288, + "theoretical_loss": 3.727795667283468, + "tokens_seen": 868024320 + }, + { + "epoch": 0.26, + "learning_rate": 0.000595530591775326, + "loss": 1.4387, + "theoretical_loss": 3.7277683887601345, + "tokens_seen": 868089856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005955145436308927, + "loss": 1.4614, + "theoretical_loss": 3.7277411128726774, + "tokens_seen": 868155392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954984954864594, + "loss": 1.3479, + "theoretical_loss": 3.7277138396206437, + "tokens_seen": 868220928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954824473420261, + "loss": 1.3631, + "theoretical_loss": 3.7276865690035796, + "tokens_seen": 868286464 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 294369, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3449276685714722, + "objective/train/theoretical_loss": 3.7276593010210313, + "objective/train/tokens_used": 888812000, + "theoretical_loss": 3.7276593010210313, + "tokens_seen": 868352000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954663991975928, + "loss": 1.3866, + "theoretical_loss": 3.7276593010210313, + "tokens_seen": 868352000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954503510531595, + "loss": 1.4186, + "theoretical_loss": 3.727632035672546, + "tokens_seen": 868417536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954343029087263, + "loss": 1.4225, + "theoretical_loss": 3.7276047729576707, + "tokens_seen": 868483072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954182547642929, + "loss": 1.4115, + "theoretical_loss": 3.727577512875951, + "tokens_seen": 868548608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954022066198597, + "loss": 1.5025, + "theoretical_loss": 3.7275502554269355, + "tokens_seen": 868614144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005953861584754263, + "loss": 1.2588, + "theoretical_loss": 3.7275230006101703, + "tokens_seen": 868679680 + }, + { + "epoch": 0.26, + "learning_rate": 0.000595370110330993, + "loss": 1.3555, + "theoretical_loss": 3.7274957484252034, + "tokens_seen": 868745216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005953540621865597, + "loss": 1.4491, + "theoretical_loss": 3.7274684988715814, + "tokens_seen": 868810752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005953380140421265, + "loss": 1.3474, + "theoretical_loss": 3.7274412519488522, + "tokens_seen": 868876288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005953219658976931, + "loss": 1.4697, + "theoretical_loss": 3.727414007656564, + "tokens_seen": 868941824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005953059177532599, + "loss": 1.3817, + "theoretical_loss": 3.7273867659942628, + "tokens_seen": 869007360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952898696088265, + "loss": 1.3573, + "theoretical_loss": 3.7273595269614974, + "tokens_seen": 869072896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952738214643933, + "loss": 1.291, + "theoretical_loss": 3.727332290557816, + "tokens_seen": 869138432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952577733199599, + "loss": 1.3947, + "theoretical_loss": 3.7273050567827664, + "tokens_seen": 869203968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952417251755267, + "loss": 1.3047, + "theoretical_loss": 3.7272778256358974, + "tokens_seen": 869269504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952256770310933, + "loss": 1.4723, + "theoretical_loss": 3.7272505971167558, + "tokens_seen": 869335040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952096288866601, + "loss": 1.3844, + "theoretical_loss": 3.7272233712248903, + "tokens_seen": 869400576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951935807422267, + "loss": 1.322, + "theoretical_loss": 3.7271961479598508, + "tokens_seen": 869466112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951775325977935, + "loss": 1.3923, + "theoretical_loss": 3.727168927321184, + "tokens_seen": 869531648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951614844533601, + "loss": 1.3031, + "theoretical_loss": 3.72714170930844, + "tokens_seen": 869597184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951454363089269, + "loss": 1.4718, + "theoretical_loss": 3.7271144939211673, + "tokens_seen": 869662720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951293881644935, + "loss": 1.4241, + "theoretical_loss": 3.727087281158914, + "tokens_seen": 869728256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951133400200603, + "loss": 1.329, + "theoretical_loss": 3.7270600710212305, + "tokens_seen": 869793792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950972918756269, + "loss": 1.3704, + "theoretical_loss": 3.7270328635076644, + "tokens_seen": 869859328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950812437311937, + "loss": 1.4418, + "theoretical_loss": 3.7270056586177662, + "tokens_seen": 869924864 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950651955867603, + "loss": 1.4354, + "theoretical_loss": 3.726978456351085, + "tokens_seen": 869990400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950491474423271, + "loss": 1.5375, + "theoretical_loss": 3.72695125670717, + "tokens_seen": 870055936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950330992978937, + "loss": 1.5777, + "theoretical_loss": 3.7269240596855706, + "tokens_seen": 870121472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950170511534605, + "loss": 1.3397, + "theoretical_loss": 3.7268968652858367, + "tokens_seen": 870187008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950010030090271, + "loss": 1.431, + "theoretical_loss": 3.726869673507519, + "tokens_seen": 870252544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949849548645939, + "loss": 1.3722, + "theoretical_loss": 3.726842484350166, + "tokens_seen": 870318080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949689067201605, + "loss": 1.3484, + "theoretical_loss": 3.726815297813329, + "tokens_seen": 870383616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949528585757273, + "loss": 1.4931, + "theoretical_loss": 3.7267881138965566, + "tokens_seen": 870449152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949368104312939, + "loss": 1.4237, + "theoretical_loss": 3.726760932599401, + "tokens_seen": 870514688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949207622868607, + "loss": 1.443, + "theoretical_loss": 3.726733753921411, + "tokens_seen": 870580224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949047141424273, + "loss": 1.3554, + "theoretical_loss": 3.7267065778621378, + "tokens_seen": 870645760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948886659979941, + "loss": 1.4397, + "theoretical_loss": 3.726679404421131, + "tokens_seen": 870711296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948726178535607, + "loss": 1.3659, + "theoretical_loss": 3.7266522335979433, + "tokens_seen": 870776832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948565697091275, + "loss": 1.5223, + "theoretical_loss": 3.726625065392124, + "tokens_seen": 870842368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948405215646941, + "loss": 1.5717, + "theoretical_loss": 3.726597899803224, + "tokens_seen": 870907904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948244734202608, + "loss": 1.3426, + "theoretical_loss": 3.726570736830795, + "tokens_seen": 870973440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948084252758275, + "loss": 1.3967, + "theoretical_loss": 3.7265435764743877, + "tokens_seen": 871038976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005947923771313942, + "loss": 1.3258, + "theoretical_loss": 3.7265164187335538, + "tokens_seen": 871104512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005947763289869609, + "loss": 1.3116, + "theoretical_loss": 3.7264892636078444, + "tokens_seen": 871170048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005947602808425276, + "loss": 1.45, + "theoretical_loss": 3.7264621110968106, + "tokens_seen": 871235584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005947442326980943, + "loss": 1.4683, + "theoretical_loss": 3.726434961200005, + "tokens_seen": 871301120 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594728184553661, + "loss": 1.4142, + "theoretical_loss": 3.7264078139169783, + "tokens_seen": 871366656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005947121364092277, + "loss": 1.3235, + "theoretical_loss": 3.7263806692472823, + "tokens_seen": 871432192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005946960882647944, + "loss": 1.3342, + "theoretical_loss": 3.7263535271904695, + "tokens_seen": 871497728 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594680040120361, + "loss": 1.3977, + "theoretical_loss": 3.726326387746092, + "tokens_seen": 871563264 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 295704, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3763213157653809, + "objective/train/theoretical_loss": 3.7262992509137014, + "objective/train/tokens_used": 892088800, + "theoretical_loss": 3.7262992509137014, + "tokens_seen": 871628800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005946639919759278, + "loss": 1.3689, + "theoretical_loss": 3.7262992509137014, + "tokens_seen": 871628800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005946479438314944, + "loss": 1.4538, + "theoretical_loss": 3.72627211669285, + "tokens_seen": 871694336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005946318956870612, + "loss": 1.3805, + "theoretical_loss": 3.7262449850830905, + "tokens_seen": 871759872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005946158475426278, + "loss": 1.3534, + "theoretical_loss": 3.7262178560839754, + "tokens_seen": 871825408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945997993981946, + "loss": 1.4327, + "theoretical_loss": 3.726190729695057, + "tokens_seen": 871890944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945837512537613, + "loss": 1.4202, + "theoretical_loss": 3.7261636059158887, + "tokens_seen": 871956480 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594567703109328, + "loss": 1.4297, + "theoretical_loss": 3.726136484746022, + "tokens_seen": 872022016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945516549648947, + "loss": 1.4908, + "theoretical_loss": 3.7261093661850104, + "tokens_seen": 872087552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945356068204614, + "loss": 1.4314, + "theoretical_loss": 3.7260822502324076, + "tokens_seen": 872153088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945195586760281, + "loss": 1.4478, + "theoretical_loss": 3.726055136887766, + "tokens_seen": 872218624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945035105315948, + "loss": 1.5178, + "theoretical_loss": 3.7260280261506393, + "tokens_seen": 872284160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005944874623871615, + "loss": 1.3651, + "theoretical_loss": 3.7260009180205804, + "tokens_seen": 872349696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005944714142427282, + "loss": 1.4939, + "theoretical_loss": 3.7259738124971427, + "tokens_seen": 872415232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005944553660982949, + "loss": 1.4669, + "theoretical_loss": 3.7259467095798806, + "tokens_seen": 872480768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005944393179538616, + "loss": 1.365, + "theoretical_loss": 3.7259196092683466, + "tokens_seen": 872546304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005944232698094283, + "loss": 1.3216, + "theoretical_loss": 3.7258925115620953, + "tokens_seen": 872611840 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594407221664995, + "loss": 1.307, + "theoretical_loss": 3.725865416460681, + "tokens_seen": 872677376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943911735205617, + "loss": 1.3002, + "theoretical_loss": 3.7258383239636563, + "tokens_seen": 872742912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943751253761284, + "loss": 1.299, + "theoretical_loss": 3.7258112340705765, + "tokens_seen": 872808448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943590772316951, + "loss": 1.3581, + "theoretical_loss": 3.7257841467809953, + "tokens_seen": 872873984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943430290872618, + "loss": 1.2732, + "theoretical_loss": 3.725757062094467, + "tokens_seen": 872939520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943269809428285, + "loss": 1.375, + "theoretical_loss": 3.7257299800105472, + "tokens_seen": 873005056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943109327983952, + "loss": 1.4402, + "theoretical_loss": 3.7257029005287894, + "tokens_seen": 873070592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005942948846539619, + "loss": 1.3421, + "theoretical_loss": 3.7256758236487477, + "tokens_seen": 873136128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005942788365095286, + "loss": 1.4131, + "theoretical_loss": 3.725648749369978, + "tokens_seen": 873201664 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005942627883650953, + "loss": 1.4883, + "theoretical_loss": 3.725621677692035, + "tokens_seen": 873267200 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594246740220662, + "loss": 1.4669, + "theoretical_loss": 3.725594608614473, + "tokens_seen": 873332736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005942306920762287, + "loss": 1.4359, + "theoretical_loss": 3.725567542136848, + "tokens_seen": 873398272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005942146439317954, + "loss": 1.4512, + "theoretical_loss": 3.725540478258715, + "tokens_seen": 873463808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941985957873621, + "loss": 1.3902, + "theoretical_loss": 3.725513416979629, + "tokens_seen": 873529344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941825476429288, + "loss": 1.4411, + "theoretical_loss": 3.725486358299145, + "tokens_seen": 873594880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941664994984955, + "loss": 1.3979, + "theoretical_loss": 3.7254593022168203, + "tokens_seen": 873660416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941504513540622, + "loss": 1.3819, + "theoretical_loss": 3.7254322487322087, + "tokens_seen": 873725952 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594134403209629, + "loss": 1.4271, + "theoretical_loss": 3.7254051978448666, + "tokens_seen": 873791488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941183550651956, + "loss": 1.271, + "theoretical_loss": 3.7253781495543503, + "tokens_seen": 873857024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941023069207623, + "loss": 1.3593, + "theoretical_loss": 3.7253511038602154, + "tokens_seen": 873922560 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594086258776329, + "loss": 1.4129, + "theoretical_loss": 3.7253240607620177, + "tokens_seen": 873988096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005940702106318958, + "loss": 1.3292, + "theoretical_loss": 3.7252970202593136, + "tokens_seen": 874053632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005940541624874624, + "loss": 1.2952, + "theoretical_loss": 3.7252699823516604, + "tokens_seen": 874119168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005940381143430292, + "loss": 1.4277, + "theoretical_loss": 3.7252429470386135, + "tokens_seen": 874184704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005940220661985958, + "loss": 1.3726, + "theoretical_loss": 3.7252159143197288, + "tokens_seen": 874250240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005940060180541626, + "loss": 1.3579, + "theoretical_loss": 3.7251888841945644, + "tokens_seen": 874315776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005939899699097292, + "loss": 1.4098, + "theoretical_loss": 3.7251618566626767, + "tokens_seen": 874381312 + }, + { + "epoch": 0.26, + "learning_rate": 0.000593973921765296, + "loss": 1.2776, + "theoretical_loss": 3.7251348317236217, + "tokens_seen": 874446848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005939578736208626, + "loss": 1.5063, + "theoretical_loss": 3.725107809376957, + "tokens_seen": 874512384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005939418254764294, + "loss": 1.4131, + "theoretical_loss": 3.7250807896222398, + "tokens_seen": 874577920 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593925777331996, + "loss": 1.3592, + "theoretical_loss": 3.725053772459027, + "tokens_seen": 874643456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005939097291875628, + "loss": 1.4288, + "theoretical_loss": 3.7250267578868757, + "tokens_seen": 874708992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005938936810431294, + "loss": 1.5116, + "theoretical_loss": 3.724999745905344, + "tokens_seen": 874774528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005938776328986962, + "loss": 1.4114, + "theoretical_loss": 3.724972736513989, + "tokens_seen": 874840064 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 296615, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4356341361999512, + "objective/train/theoretical_loss": 3.7249457297123687, + "objective/train/tokens_used": 895365600, + "theoretical_loss": 3.7249457297123687, + "tokens_seen": 874905600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005938615847542628, + "loss": 1.3512, + "theoretical_loss": 3.7249457297123687, + "tokens_seen": 874905600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005938455366098296, + "loss": 1.4639, + "theoretical_loss": 3.7249187255000393, + "tokens_seen": 874971136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005938294884653962, + "loss": 1.4029, + "theoretical_loss": 3.724891723876561, + "tokens_seen": 875036672 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593813440320963, + "loss": 1.4275, + "theoretical_loss": 3.7248647248414897, + "tokens_seen": 875102208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937973921765296, + "loss": 1.2328, + "theoretical_loss": 3.7248377283943848, + "tokens_seen": 875167744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937813440320964, + "loss": 1.3946, + "theoretical_loss": 3.7248107345348034, + "tokens_seen": 875233280 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593765295887663, + "loss": 1.3459, + "theoretical_loss": 3.7247837432623054, + "tokens_seen": 875298816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937492477432298, + "loss": 1.501, + "theoretical_loss": 3.724756754576447, + "tokens_seen": 875364352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937331995987964, + "loss": 1.4329, + "theoretical_loss": 3.7247297684767884, + "tokens_seen": 875429888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937171514543632, + "loss": 1.4151, + "theoretical_loss": 3.7247027849628873, + "tokens_seen": 875495424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937011033099298, + "loss": 1.4494, + "theoretical_loss": 3.7246758040343027, + "tokens_seen": 875560960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005936850551654966, + "loss": 1.211, + "theoretical_loss": 3.724648825690594, + "tokens_seen": 875626496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005936690070210632, + "loss": 1.489, + "theoretical_loss": 3.724621849931319, + "tokens_seen": 875692032 + }, + { + "epoch": 0.27, + "learning_rate": 0.00059365295887663, + "loss": 1.4049, + "theoretical_loss": 3.7245948767560373, + "tokens_seen": 875757568 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005936369107321966, + "loss": 1.3986, + "theoretical_loss": 3.724567906164308, + "tokens_seen": 875823104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005936208625877634, + "loss": 1.4051, + "theoretical_loss": 3.72454093815569, + "tokens_seen": 875888640 + }, + { + "epoch": 0.27, + "learning_rate": 0.00059360481444333, + "loss": 1.4094, + "theoretical_loss": 3.724513972729744, + "tokens_seen": 875954176 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935887662988968, + "loss": 1.3648, + "theoretical_loss": 3.724487009886028, + "tokens_seen": 876019712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935727181544634, + "loss": 1.3693, + "theoretical_loss": 3.724460049624102, + "tokens_seen": 876085248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935566700100302, + "loss": 1.3855, + "theoretical_loss": 3.7244330919435256, + "tokens_seen": 876150784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935406218655968, + "loss": 1.3825, + "theoretical_loss": 3.724406136843859, + "tokens_seen": 876216320 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935245737211636, + "loss": 1.4952, + "theoretical_loss": 3.7243791843246616, + "tokens_seen": 876281856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935085255767302, + "loss": 1.4827, + "theoretical_loss": 3.724352234385494, + "tokens_seen": 876347392 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593492477432297, + "loss": 1.3579, + "theoretical_loss": 3.7243252870259154, + "tokens_seen": 876412928 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005934764292878636, + "loss": 1.3541, + "theoretical_loss": 3.7242983422454867, + "tokens_seen": 876478464 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005934603811434304, + "loss": 1.4119, + "theoretical_loss": 3.724271400043768, + "tokens_seen": 876544000 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593444332998997, + "loss": 1.3913, + "theoretical_loss": 3.72424446042032, + "tokens_seen": 876609536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005934282848545637, + "loss": 1.4509, + "theoretical_loss": 3.724217523374703, + "tokens_seen": 876675072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005934122367101304, + "loss": 1.4332, + "theoretical_loss": 3.7241905889064784, + "tokens_seen": 876740608 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933961885656971, + "loss": 1.4808, + "theoretical_loss": 3.7241636570152052, + "tokens_seen": 876806144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933801404212638, + "loss": 1.4096, + "theoretical_loss": 3.724136727700446, + "tokens_seen": 876871680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933640922768305, + "loss": 1.3544, + "theoretical_loss": 3.7241098009617604, + "tokens_seen": 876937216 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933480441323972, + "loss": 1.3773, + "theoretical_loss": 3.724082876798711, + "tokens_seen": 877002752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933319959879639, + "loss": 1.4223, + "theoretical_loss": 3.724055955210858, + "tokens_seen": 877068288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933159478435306, + "loss": 1.3957, + "theoretical_loss": 3.7240290361977624, + "tokens_seen": 877133824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933159478435306, + "loss": 1.3962, + "theoretical_loss": 3.7240021197589863, + "tokens_seen": 877199360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932998996990973, + "loss": 1.4461, + "theoretical_loss": 3.723975205894091, + "tokens_seen": 877264896 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593283851554664, + "loss": 1.4704, + "theoretical_loss": 3.7239482946026383, + "tokens_seen": 877330432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932678034102307, + "loss": 1.3398, + "theoretical_loss": 3.7239213858841893, + "tokens_seen": 877395968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932517552657974, + "loss": 1.3765, + "theoretical_loss": 3.723894479738307, + "tokens_seen": 877461504 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932357071213641, + "loss": 1.4135, + "theoretical_loss": 3.723867576164552, + "tokens_seen": 877527040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932196589769308, + "loss": 1.2494, + "theoretical_loss": 3.723840675162487, + "tokens_seen": 877592576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932036108324975, + "loss": 1.3955, + "theoretical_loss": 3.7238137767316744, + "tokens_seen": 877658112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005931875626880642, + "loss": 1.382, + "theoretical_loss": 3.723786880871676, + "tokens_seen": 877723648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005931715145436309, + "loss": 1.3312, + "theoretical_loss": 3.7237599875820546, + "tokens_seen": 877789184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005931554663991976, + "loss": 1.3627, + "theoretical_loss": 3.723733096862372, + "tokens_seen": 877854720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005931394182547643, + "loss": 1.308, + "theoretical_loss": 3.7237062087121915, + "tokens_seen": 877920256 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593123370110331, + "loss": 1.3959, + "theoretical_loss": 3.7236793231310754, + "tokens_seen": 877985792 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005931073219658977, + "loss": 1.4604, + "theoretical_loss": 3.7236524401185864, + "tokens_seen": 878051328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930912738214644, + "loss": 1.4636, + "theoretical_loss": 3.7236255596742884, + "tokens_seen": 878116864 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 297634, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4732657670974731, + "objective/train/theoretical_loss": 3.7235986817977427, + "objective/train/tokens_used": 898642400, + "theoretical_loss": 3.7235986817977427, + "tokens_seen": 878182400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930752256770311, + "loss": 1.4573, + "theoretical_loss": 3.7235986817977427, + "tokens_seen": 878182400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930591775325978, + "loss": 1.3969, + "theoretical_loss": 3.723571806488514, + "tokens_seen": 878247936 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930431293881645, + "loss": 1.2044, + "theoretical_loss": 3.7235449337461644, + "tokens_seen": 878313472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930270812437312, + "loss": 1.4234, + "theoretical_loss": 3.723518063570258, + "tokens_seen": 878379008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930110330992979, + "loss": 1.3543, + "theoretical_loss": 3.7234911959603574, + "tokens_seen": 878444544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005929949849548646, + "loss": 1.3475, + "theoretical_loss": 3.7234643309160274, + "tokens_seen": 878510080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005929789368104313, + "loss": 1.4172, + "theoretical_loss": 3.7234374684368303, + "tokens_seen": 878575616 + }, + { + "epoch": 0.27, + "learning_rate": 0.000592962888665998, + "loss": 1.4079, + "theoretical_loss": 3.7234106085223306, + "tokens_seen": 878641152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005929468405215647, + "loss": 1.3668, + "theoretical_loss": 3.7233837511720926, + "tokens_seen": 878706688 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005929307923771314, + "loss": 1.3957, + "theoretical_loss": 3.7233568963856793, + "tokens_seen": 878772224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005929147442326981, + "loss": 1.3877, + "theoretical_loss": 3.7233300441626556, + "tokens_seen": 878837760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928986960882648, + "loss": 1.4901, + "theoretical_loss": 3.7233031945025847, + "tokens_seen": 878903296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928826479438315, + "loss": 1.3192, + "theoretical_loss": 3.723276347405032, + "tokens_seen": 878968832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928665997993982, + "loss": 1.2972, + "theoretical_loss": 3.723249502869561, + "tokens_seen": 879034368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928505516549649, + "loss": 1.3476, + "theoretical_loss": 3.723222660895737, + "tokens_seen": 879099904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928345035105316, + "loss": 1.2606, + "theoretical_loss": 3.7231958214831242, + "tokens_seen": 879165440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928184553660983, + "loss": 1.4602, + "theoretical_loss": 3.723168984631287, + "tokens_seen": 879230976 + }, + { + "epoch": 0.27, + "learning_rate": 0.000592802407221665, + "loss": 1.3551, + "theoretical_loss": 3.7231421503397906, + "tokens_seen": 879296512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927863590772317, + "loss": 1.3874, + "theoretical_loss": 3.7231153186082, + "tokens_seen": 879362048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927703109327984, + "loss": 1.5372, + "theoretical_loss": 3.7230884894360807, + "tokens_seen": 879427584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927542627883651, + "loss": 1.4817, + "theoretical_loss": 3.723061662822997, + "tokens_seen": 879493120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927382146439318, + "loss": 1.4971, + "theoretical_loss": 3.7230348387685144, + "tokens_seen": 879558656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927221664994985, + "loss": 1.4103, + "theoretical_loss": 3.723008017272198, + "tokens_seen": 879624192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927061183550653, + "loss": 1.2916, + "theoretical_loss": 3.7229811983336134, + "tokens_seen": 879689728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005926900702106319, + "loss": 1.392, + "theoretical_loss": 3.7229543819523268, + "tokens_seen": 879755264 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005926740220661987, + "loss": 1.3366, + "theoretical_loss": 3.7229275681279033, + "tokens_seen": 879820800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005926579739217653, + "loss": 1.4538, + "theoretical_loss": 3.722900756859909, + "tokens_seen": 879886336 + }, + { + "epoch": 0.27, + "learning_rate": 0.000592641925777332, + "loss": 1.2782, + "theoretical_loss": 3.722873948147909, + "tokens_seen": 879951872 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005926258776328987, + "loss": 1.3477, + "theoretical_loss": 3.72284714199147, + "tokens_seen": 880017408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005926098294884655, + "loss": 1.3581, + "theoretical_loss": 3.722820338390158, + "tokens_seen": 880082944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925937813440321, + "loss": 1.4581, + "theoretical_loss": 3.7227935373435397, + "tokens_seen": 880148480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925777331995989, + "loss": 1.251, + "theoretical_loss": 3.7227667388511807, + "tokens_seen": 880214016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925616850551655, + "loss": 1.387, + "theoretical_loss": 3.7227399429126473, + "tokens_seen": 880279552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925456369107323, + "loss": 1.382, + "theoretical_loss": 3.7227131495275065, + "tokens_seen": 880345088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925295887662989, + "loss": 1.3781, + "theoretical_loss": 3.7226863586953245, + "tokens_seen": 880410624 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925135406218657, + "loss": 1.5287, + "theoretical_loss": 3.7226595704156686, + "tokens_seen": 880476160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924974924774323, + "loss": 1.3997, + "theoretical_loss": 3.7226327846881055, + "tokens_seen": 880541696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924814443329991, + "loss": 1.357, + "theoretical_loss": 3.722606001512202, + "tokens_seen": 880607232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924653961885657, + "loss": 1.3955, + "theoretical_loss": 3.722579220887525, + "tokens_seen": 880672768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924493480441325, + "loss": 1.5141, + "theoretical_loss": 3.722552442813641, + "tokens_seen": 880738304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924332998996991, + "loss": 1.3306, + "theoretical_loss": 3.7225256672901192, + "tokens_seen": 880803840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924172517552659, + "loss": 1.4259, + "theoretical_loss": 3.7224988943165256, + "tokens_seen": 880869376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924012036108325, + "loss": 1.4479, + "theoretical_loss": 3.7224721238924277, + "tokens_seen": 880934912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923851554663993, + "loss": 1.4694, + "theoretical_loss": 3.722445356017393, + "tokens_seen": 881000448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923691073219659, + "loss": 1.3357, + "theoretical_loss": 3.7224185906909897, + "tokens_seen": 881065984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923530591775327, + "loss": 1.3031, + "theoretical_loss": 3.7223918279127854, + "tokens_seen": 881131520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923370110330993, + "loss": 1.4759, + "theoretical_loss": 3.7223650676823485, + "tokens_seen": 881197056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923209628886661, + "loss": 1.435, + "theoretical_loss": 3.7223383099992455, + "tokens_seen": 881262592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923049147442327, + "loss": 1.4392, + "theoretical_loss": 3.7223115548630465, + "tokens_seen": 881328128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922888665997995, + "loss": 1.3413, + "theoretical_loss": 3.722284802273318, + "tokens_seen": 881393664 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 298414, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.473338007926941, + "objective/train/theoretical_loss": 3.722258052229629, + "objective/train/tokens_used": 901919200, + "theoretical_loss": 3.722258052229629, + "tokens_seen": 881459200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922728184553661, + "loss": 1.4058, + "theoretical_loss": 3.722258052229629, + "tokens_seen": 881459200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922567703109329, + "loss": 1.3992, + "theoretical_loss": 3.7222313047315483, + "tokens_seen": 881524736 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922407221664995, + "loss": 1.5223, + "theoretical_loss": 3.722204559778644, + "tokens_seen": 881590272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922246740220663, + "loss": 1.2933, + "theoretical_loss": 3.7221778173704845, + "tokens_seen": 881655808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922086258776329, + "loss": 1.4027, + "theoretical_loss": 3.722151077506639, + "tokens_seen": 881721344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921925777331997, + "loss": 1.3407, + "theoretical_loss": 3.722124340186676, + "tokens_seen": 881786880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921765295887663, + "loss": 1.368, + "theoretical_loss": 3.7220976054101653, + "tokens_seen": 881852416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921604814443331, + "loss": 1.4898, + "theoretical_loss": 3.7220708731766745, + "tokens_seen": 881917952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921444332998997, + "loss": 1.395, + "theoretical_loss": 3.722044143485774, + "tokens_seen": 881983488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921283851554665, + "loss": 1.3127, + "theoretical_loss": 3.7220174163370325, + "tokens_seen": 882049024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921123370110331, + "loss": 1.4073, + "theoretical_loss": 3.72199069173002, + "tokens_seen": 882114560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920962888665999, + "loss": 1.2712, + "theoretical_loss": 3.721963969664305, + "tokens_seen": 882180096 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920802407221665, + "loss": 1.4562, + "theoretical_loss": 3.721937250139457, + "tokens_seen": 882245632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920641925777333, + "loss": 1.4463, + "theoretical_loss": 3.721910533155047, + "tokens_seen": 882311168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920481444332999, + "loss": 1.4729, + "theoretical_loss": 3.721883818710644, + "tokens_seen": 882376704 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920320962888666, + "loss": 1.4927, + "theoretical_loss": 3.7218571068058175, + "tokens_seen": 882442240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920160481444333, + "loss": 1.2466, + "theoretical_loss": 3.7218303974401383, + "tokens_seen": 882507776 + }, + { + "epoch": 0.27, + "learning_rate": 0.000592, + "loss": 1.4052, + "theoretical_loss": 3.7218036906131755, + "tokens_seen": 882573312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919839518555667, + "loss": 1.3998, + "theoretical_loss": 3.7217769863245005, + "tokens_seen": 882638848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919679037111334, + "loss": 1.4173, + "theoretical_loss": 3.721750284573683, + "tokens_seen": 882704384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919518555667001, + "loss": 1.3993, + "theoretical_loss": 3.7217235853602935, + "tokens_seen": 882769920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919358074222668, + "loss": 1.3656, + "theoretical_loss": 3.721696888683902, + "tokens_seen": 882835456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919197592778335, + "loss": 1.4411, + "theoretical_loss": 3.72167019454408, + "tokens_seen": 882900992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919037111334002, + "loss": 1.4962, + "theoretical_loss": 3.7216435029403976, + "tokens_seen": 882966528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005918876629889669, + "loss": 1.4093, + "theoretical_loss": 3.721616813872426, + "tokens_seen": 883032064 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005918716148445336, + "loss": 1.4233, + "theoretical_loss": 3.721590127339736, + "tokens_seen": 883097600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005918555667001003, + "loss": 1.3883, + "theoretical_loss": 3.721563443341899, + "tokens_seen": 883163136 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591839518555667, + "loss": 1.4037, + "theoretical_loss": 3.721536761878485, + "tokens_seen": 883228672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005918234704112337, + "loss": 1.4487, + "theoretical_loss": 3.7215100829490666, + "tokens_seen": 883294208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005918074222668004, + "loss": 1.2825, + "theoretical_loss": 3.7214834065532147, + "tokens_seen": 883359744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917913741223671, + "loss": 1.4454, + "theoretical_loss": 3.7214567326905006, + "tokens_seen": 883425280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917753259779338, + "loss": 1.4214, + "theoretical_loss": 3.721430061360496, + "tokens_seen": 883490816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917592778335005, + "loss": 1.3491, + "theoretical_loss": 3.721403392562772, + "tokens_seen": 883556352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917432296890672, + "loss": 1.4518, + "theoretical_loss": 3.7213767262969015, + "tokens_seen": 883621888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917271815446339, + "loss": 1.389, + "theoretical_loss": 3.721350062562456, + "tokens_seen": 883687424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917111334002006, + "loss": 1.3153, + "theoretical_loss": 3.7213234013590064, + "tokens_seen": 883752960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005916950852557673, + "loss": 1.3632, + "theoretical_loss": 3.7212967426861265, + "tokens_seen": 883818496 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591679037111334, + "loss": 1.3569, + "theoretical_loss": 3.721270086543387, + "tokens_seen": 883884032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005916629889669007, + "loss": 1.2478, + "theoretical_loss": 3.721243432930361, + "tokens_seen": 883949568 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005916469408224674, + "loss": 1.3319, + "theoretical_loss": 3.721216781846621, + "tokens_seen": 884015104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005916308926780341, + "loss": 1.2826, + "theoretical_loss": 3.721190133291739, + "tokens_seen": 884080640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005916148445336008, + "loss": 1.393, + "theoretical_loss": 3.7211634872652875, + "tokens_seen": 884146176 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915987963891675, + "loss": 1.4237, + "theoretical_loss": 3.72113684376684, + "tokens_seen": 884211712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915827482447342, + "loss": 1.4712, + "theoretical_loss": 3.721110202795969, + "tokens_seen": 884277248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915667001003009, + "loss": 1.4201, + "theoretical_loss": 3.721083564352247, + "tokens_seen": 884342784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915506519558676, + "loss": 1.4385, + "theoretical_loss": 3.7210569284352473, + "tokens_seen": 884408320 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915346038114343, + "loss": 1.3228, + "theoretical_loss": 3.7210302950445433, + "tokens_seen": 884473856 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591518555667001, + "loss": 1.4452, + "theoretical_loss": 3.7210036641797073, + "tokens_seen": 884539392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915025075225677, + "loss": 1.3413, + "theoretical_loss": 3.7209770358403143, + "tokens_seen": 884604928 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005914864593781344, + "loss": 1.3588, + "theoretical_loss": 3.720950410025936, + "tokens_seen": 884670464 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 299625, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3763837814331055, + "objective/train/theoretical_loss": 3.7209237867361473, + "objective/train/tokens_used": 905196000, + "theoretical_loss": 3.7209237867361473, + "tokens_seen": 884736000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005914704112337011, + "loss": 1.3689, + "theoretical_loss": 3.7209237867361473, + "tokens_seen": 884736000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005914543630892678, + "loss": 1.3451, + "theoretical_loss": 3.720897165970521, + "tokens_seen": 884801536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005914383149448345, + "loss": 1.2868, + "theoretical_loss": 3.7208705477286306, + "tokens_seen": 884867072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005914222668004012, + "loss": 1.3519, + "theoretical_loss": 3.720843932010051, + "tokens_seen": 884932608 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591406218655968, + "loss": 1.3974, + "theoretical_loss": 3.720817318814355, + "tokens_seen": 884998144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005913901705115346, + "loss": 1.3589, + "theoretical_loss": 3.720790708141118, + "tokens_seen": 885063680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005913741223671013, + "loss": 1.3256, + "theoretical_loss": 3.7207640999899128, + "tokens_seen": 885129216 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591358074222668, + "loss": 1.3769, + "theoretical_loss": 3.720737494360314, + "tokens_seen": 885194752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005913420260782348, + "loss": 1.3317, + "theoretical_loss": 3.720710891251897, + "tokens_seen": 885260288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005913259779338014, + "loss": 1.3677, + "theoretical_loss": 3.7206842906642352, + "tokens_seen": 885325824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005913099297893682, + "loss": 1.3947, + "theoretical_loss": 3.7206576925969035, + "tokens_seen": 885391360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005912938816449348, + "loss": 1.4569, + "theoretical_loss": 3.720631097049477, + "tokens_seen": 885456896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005912778335005016, + "loss": 1.2061, + "theoretical_loss": 3.720604504021529, + "tokens_seen": 885522432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005912617853560682, + "loss": 1.4317, + "theoretical_loss": 3.7205779135126367, + "tokens_seen": 885587968 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591245737211635, + "loss": 1.3402, + "theoretical_loss": 3.720551325522373, + "tokens_seen": 885653504 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005912296890672016, + "loss": 1.3553, + "theoretical_loss": 3.720524740050314, + "tokens_seen": 885719040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005912136409227684, + "loss": 1.3534, + "theoretical_loss": 3.720498157096035, + "tokens_seen": 885784576 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591197592778335, + "loss": 1.3756, + "theoretical_loss": 3.7204715766591105, + "tokens_seen": 885850112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911815446339018, + "loss": 1.2983, + "theoretical_loss": 3.720444998739117, + "tokens_seen": 885915648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911654964894684, + "loss": 1.3486, + "theoretical_loss": 3.720418423335629, + "tokens_seen": 885981184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911494483450352, + "loss": 1.4364, + "theoretical_loss": 3.7203918504482227, + "tokens_seen": 886046720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911334002006018, + "loss": 1.4148, + "theoretical_loss": 3.7203652800764737, + "tokens_seen": 886112256 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911173520561686, + "loss": 1.4583, + "theoretical_loss": 3.7203387122199576, + "tokens_seen": 886177792 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911013039117352, + "loss": 1.3716, + "theoretical_loss": 3.7203121468782507, + "tokens_seen": 886243328 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591085255767302, + "loss": 1.3507, + "theoretical_loss": 3.720285584050928, + "tokens_seen": 886308864 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005910692076228686, + "loss": 1.3487, + "theoretical_loss": 3.7202590237375674, + "tokens_seen": 886374400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005910531594784354, + "loss": 1.4123, + "theoretical_loss": 3.7202324659377437, + "tokens_seen": 886439936 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591037111334002, + "loss": 1.301, + "theoretical_loss": 3.7202059106510337, + "tokens_seen": 886505472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005910210631895688, + "loss": 1.3052, + "theoretical_loss": 3.720179357877014, + "tokens_seen": 886571008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005910050150451354, + "loss": 1.2118, + "theoretical_loss": 3.7201528076152606, + "tokens_seen": 886636544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005909889669007022, + "loss": 1.2721, + "theoretical_loss": 3.7201262598653506, + "tokens_seen": 886702080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005909729187562689, + "loss": 1.4885, + "theoretical_loss": 3.720099714626861, + "tokens_seen": 886767616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005909568706118356, + "loss": 1.3028, + "theoretical_loss": 3.720073171899368, + "tokens_seen": 886833152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005909408224674023, + "loss": 1.3956, + "theoretical_loss": 3.720046631682448, + "tokens_seen": 886898688 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590924774322969, + "loss": 1.2754, + "theoretical_loss": 3.7200200939756796, + "tokens_seen": 886964224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005909087261785357, + "loss": 1.2936, + "theoretical_loss": 3.719993558778639, + "tokens_seen": 887029760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908926780341024, + "loss": 1.4283, + "theoretical_loss": 3.7199670260909037, + "tokens_seen": 887095296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908766298896691, + "loss": 1.3382, + "theoretical_loss": 3.7199404959120512, + "tokens_seen": 887160832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908605817452358, + "loss": 1.3836, + "theoretical_loss": 3.7199139682416584, + "tokens_seen": 887226368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908445336008025, + "loss": 1.4363, + "theoretical_loss": 3.719887443079303, + "tokens_seen": 887291904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908284854563692, + "loss": 1.3388, + "theoretical_loss": 3.719860920424563, + "tokens_seen": 887357440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908124373119359, + "loss": 1.2925, + "theoretical_loss": 3.7198344002770165, + "tokens_seen": 887422976 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005907963891675026, + "loss": 1.306, + "theoretical_loss": 3.719807882636241, + "tokens_seen": 887488512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005907803410230693, + "loss": 1.3648, + "theoretical_loss": 3.719781367501813, + "tokens_seen": 887554048 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590764292878636, + "loss": 1.3699, + "theoretical_loss": 3.719754854873313, + "tokens_seen": 887619584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005907482447342026, + "loss": 1.3634, + "theoretical_loss": 3.719728344750318, + "tokens_seen": 887685120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005907321965897694, + "loss": 1.4477, + "theoretical_loss": 3.7197018371324067, + "tokens_seen": 887750656 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590716148445336, + "loss": 1.3321, + "theoretical_loss": 3.7196753320191567, + "tokens_seen": 887816192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005907001003009028, + "loss": 1.4212, + "theoretical_loss": 3.7196488294101466, + "tokens_seen": 887881728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906840521564694, + "loss": 1.3911, + "theoretical_loss": 3.719622329304956, + "tokens_seen": 887947264 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 300533, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4114042520523071, + "objective/train/theoretical_loss": 3.7195958317031623, + "objective/train/tokens_used": 908472800, + "theoretical_loss": 3.7195958317031623, + "tokens_seen": 888012800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906680040120362, + "loss": 1.313, + "theoretical_loss": 3.7195958317031623, + "tokens_seen": 888012800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906519558676028, + "loss": 1.3689, + "theoretical_loss": 3.719569336604345, + "tokens_seen": 888078336 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906359077231695, + "loss": 1.4185, + "theoretical_loss": 3.7195428440080827, + "tokens_seen": 888143872 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906198595787362, + "loss": 1.2331, + "theoretical_loss": 3.7195163539139546, + "tokens_seen": 888209408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906038114343029, + "loss": 1.3944, + "theoretical_loss": 3.71948986632154, + "tokens_seen": 888274944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005905877632898696, + "loss": 1.323, + "theoretical_loss": 3.719463381230418, + "tokens_seen": 888340480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005905717151454363, + "loss": 1.3213, + "theoretical_loss": 3.7194368986401676, + "tokens_seen": 888406016 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590555667001003, + "loss": 1.4546, + "theoretical_loss": 3.719410418550368, + "tokens_seen": 888471552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005905396188565697, + "loss": 1.3573, + "theoretical_loss": 3.7193839409605993, + "tokens_seen": 888537088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005905235707121364, + "loss": 1.3619, + "theoretical_loss": 3.7193574658704405, + "tokens_seen": 888602624 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005905075225677031, + "loss": 1.3303, + "theoretical_loss": 3.719330993279472, + "tokens_seen": 888668160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904914744232698, + "loss": 1.3685, + "theoretical_loss": 3.7193045231872732, + "tokens_seen": 888733696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904754262788365, + "loss": 1.4181, + "theoretical_loss": 3.7192780555934237, + "tokens_seen": 888799232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904593781344032, + "loss": 1.4243, + "theoretical_loss": 3.719251590497504, + "tokens_seen": 888864768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904433299899699, + "loss": 1.3591, + "theoretical_loss": 3.7192251278990938, + "tokens_seen": 888930304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904272818455366, + "loss": 1.3494, + "theoretical_loss": 3.7191986677977735, + "tokens_seen": 888995840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904112337011033, + "loss": 1.3963, + "theoretical_loss": 3.7191722101931237, + "tokens_seen": 889061376 + }, + { + "epoch": 0.27, + "learning_rate": 0.00059039518555667, + "loss": 1.2319, + "theoretical_loss": 3.7191457550847247, + "tokens_seen": 889126912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005903791374122367, + "loss": 1.4435, + "theoretical_loss": 3.7191193024721567, + "tokens_seen": 889192448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005903630892678034, + "loss": 1.2296, + "theoretical_loss": 3.719092852355, + "tokens_seen": 889257984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005903470411233701, + "loss": 1.395, + "theoretical_loss": 3.7190664047328363, + "tokens_seen": 889323520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005903309929789368, + "loss": 1.3008, + "theoretical_loss": 3.719039959605246, + "tokens_seen": 889389056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005903149448345035, + "loss": 1.3368, + "theoretical_loss": 3.7190135169718097, + "tokens_seen": 889454592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902988966900702, + "loss": 1.4134, + "theoretical_loss": 3.718987076832109, + "tokens_seen": 889520128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902828485456369, + "loss": 1.2957, + "theoretical_loss": 3.718960639185724, + "tokens_seen": 889585664 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902668004012036, + "loss": 1.4356, + "theoretical_loss": 3.7189342040322373, + "tokens_seen": 889651200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902507522567703, + "loss": 1.3634, + "theoretical_loss": 3.718907771371229, + "tokens_seen": 889716736 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590234704112337, + "loss": 1.3358, + "theoretical_loss": 3.718881341202281, + "tokens_seen": 889782272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902186559679037, + "loss": 1.2821, + "theoretical_loss": 3.7188549135249755, + "tokens_seen": 889847808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902026078234704, + "loss": 1.3268, + "theoretical_loss": 3.718828488338893, + "tokens_seen": 889913344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901865596790371, + "loss": 1.4549, + "theoretical_loss": 3.718802065643616, + "tokens_seen": 889978880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901705115346038, + "loss": 1.3998, + "theoretical_loss": 3.7187756454387255, + "tokens_seen": 890044416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901544633901705, + "loss": 1.4264, + "theoretical_loss": 3.7187492277238046, + "tokens_seen": 890109952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901384152457372, + "loss": 1.3757, + "theoretical_loss": 3.7187228124984344, + "tokens_seen": 890175488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901223671013039, + "loss": 1.3704, + "theoretical_loss": 3.7186963997621976, + "tokens_seen": 890241024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901063189568706, + "loss": 1.4204, + "theoretical_loss": 3.7186699895146758, + "tokens_seen": 890306560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005900902708124373, + "loss": 1.286, + "theoretical_loss": 3.718643581755452, + "tokens_seen": 890372096 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590074222668004, + "loss": 1.3411, + "theoretical_loss": 3.7186171764841083, + "tokens_seen": 890437632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005900581745235707, + "loss": 1.4316, + "theoretical_loss": 3.7185907737002273, + "tokens_seen": 890503168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005900421263791374, + "loss": 1.3444, + "theoretical_loss": 3.7185643734033915, + "tokens_seen": 890568704 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005900260782347041, + "loss": 1.3862, + "theoretical_loss": 3.7185379755931836, + "tokens_seen": 890634240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005900100300902708, + "loss": 1.3811, + "theoretical_loss": 3.718511580269187, + "tokens_seen": 890699776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005899939819458375, + "loss": 1.4249, + "theoretical_loss": 3.718485187430984, + "tokens_seen": 890765312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005899779338014042, + "loss": 1.3565, + "theoretical_loss": 3.7184587970781573, + "tokens_seen": 890830848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005899618856569709, + "loss": 1.3189, + "theoretical_loss": 3.7184324092102914, + "tokens_seen": 890896384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005899458375125377, + "loss": 1.4689, + "theoretical_loss": 3.7184060238269683, + "tokens_seen": 890961920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005899297893681043, + "loss": 1.3887, + "theoretical_loss": 3.718379640927772, + "tokens_seen": 891027456 + }, + { + "epoch": 0.27, + "learning_rate": 0.000589913741223671, + "loss": 1.3704, + "theoretical_loss": 3.718353260512286, + "tokens_seen": 891092992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898976930792377, + "loss": 1.4669, + "theoretical_loss": 3.718326882580093, + "tokens_seen": 891158528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898816449348045, + "loss": 1.2718, + "theoretical_loss": 3.7183005071307775, + "tokens_seen": 891224064 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 301711, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2756049633026123, + "objective/train/theoretical_loss": 3.7182741341639227, + "objective/train/tokens_used": 911749600, + "theoretical_loss": 3.7182741341639227, + "tokens_seen": 891289600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898655967903711, + "loss": 1.286, + "theoretical_loss": 3.7182741341639227, + "tokens_seen": 891289600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898495486459379, + "loss": 1.4418, + "theoretical_loss": 3.7182477636791127, + "tokens_seen": 891355136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898335005015045, + "loss": 1.331, + "theoretical_loss": 3.718221395675932, + "tokens_seen": 891420672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898174523570713, + "loss": 1.3085, + "theoretical_loss": 3.7181950301539635, + "tokens_seen": 891486208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898014042126379, + "loss": 1.3143, + "theoretical_loss": 3.718168667112792, + "tokens_seen": 891551744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897853560682047, + "loss": 1.2849, + "theoretical_loss": 3.718142306552002, + "tokens_seen": 891617280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897693079237714, + "loss": 1.2788, + "theoretical_loss": 3.718115948471177, + "tokens_seen": 891682816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897532597793381, + "loss": 1.4999, + "theoretical_loss": 3.718089592869902, + "tokens_seen": 891748352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897372116349048, + "loss": 1.2384, + "theoretical_loss": 3.718063239747762, + "tokens_seen": 891813888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897211634904715, + "loss": 1.4044, + "theoretical_loss": 3.718036889104341, + "tokens_seen": 891879424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897051153460382, + "loss": 1.3401, + "theoretical_loss": 3.7180105409392237, + "tokens_seen": 891944960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005896890672016049, + "loss": 1.3459, + "theoretical_loss": 3.7179841952519945, + "tokens_seen": 892010496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005896730190571716, + "loss": 1.283, + "theoretical_loss": 3.7179578520422396, + "tokens_seen": 892076032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005896569709127383, + "loss": 1.372, + "theoretical_loss": 3.7179315113095437, + "tokens_seen": 892141568 + }, + { + "epoch": 0.27, + "learning_rate": 0.000589640922768305, + "loss": 1.4019, + "theoretical_loss": 3.7179051730534916, + "tokens_seen": 892207104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005896248746238717, + "loss": 1.3742, + "theoretical_loss": 3.717878837273668, + "tokens_seen": 892272640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005896088264794384, + "loss": 1.3471, + "theoretical_loss": 3.7178525039696595, + "tokens_seen": 892338176 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895927783350051, + "loss": 1.351, + "theoretical_loss": 3.7178261731410505, + "tokens_seen": 892403712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895767301905718, + "loss": 1.3865, + "theoretical_loss": 3.717799844787427, + "tokens_seen": 892469248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895606820461385, + "loss": 1.4294, + "theoretical_loss": 3.7177735189083743, + "tokens_seen": 892534784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895446339017052, + "loss": 1.3942, + "theoretical_loss": 3.717747195503479, + "tokens_seen": 892600320 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895285857572719, + "loss": 1.4086, + "theoretical_loss": 3.7177208745723256, + "tokens_seen": 892665856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895125376128386, + "loss": 1.3606, + "theoretical_loss": 3.717694556114502, + "tokens_seen": 892731392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894964894684053, + "loss": 1.2228, + "theoretical_loss": 3.717668240129592, + "tokens_seen": 892796928 + }, + { + "epoch": 0.27, + "learning_rate": 0.000589480441323972, + "loss": 1.3704, + "theoretical_loss": 3.717641926617183, + "tokens_seen": 892862464 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894643931795387, + "loss": 1.3442, + "theoretical_loss": 3.717615615576861, + "tokens_seen": 892928000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894483450351054, + "loss": 1.312, + "theoretical_loss": 3.717589307008213, + "tokens_seen": 892993536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894322968906721, + "loss": 1.365, + "theoretical_loss": 3.717563000910824, + "tokens_seen": 893059072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894162487462388, + "loss": 1.3723, + "theoretical_loss": 3.717536697284282, + "tokens_seen": 893124608 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894002006018055, + "loss": 1.3776, + "theoretical_loss": 3.717510396128173, + "tokens_seen": 893190144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893841524573722, + "loss": 1.4324, + "theoretical_loss": 3.7174840974420826, + "tokens_seen": 893255680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893681043129389, + "loss": 1.4634, + "theoretical_loss": 3.7174578012256, + "tokens_seen": 893321216 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893520561685055, + "loss": 1.2339, + "theoretical_loss": 3.7174315074783104, + "tokens_seen": 893386752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893360080240723, + "loss": 1.4133, + "theoretical_loss": 3.7174052161998015, + "tokens_seen": 893452288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893199598796389, + "loss": 1.2415, + "theoretical_loss": 3.7173789273896602, + "tokens_seen": 893517824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893039117352057, + "loss": 1.465, + "theoretical_loss": 3.717352641047474, + "tokens_seen": 893583360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892878635907723, + "loss": 1.3275, + "theoretical_loss": 3.7173263571728303, + "tokens_seen": 893648896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892718154463391, + "loss": 1.328, + "theoretical_loss": 3.7173000757653156, + "tokens_seen": 893714432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892557673019057, + "loss": 1.3894, + "theoretical_loss": 3.717273796824519, + "tokens_seen": 893779968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892397191574725, + "loss": 1.4125, + "theoretical_loss": 3.7172475203500266, + "tokens_seen": 893845504 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892236710130391, + "loss": 1.3008, + "theoretical_loss": 3.717221246341427, + "tokens_seen": 893911040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892076228686058, + "loss": 1.4317, + "theoretical_loss": 3.717194974798308, + "tokens_seen": 893976576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005891915747241725, + "loss": 1.4452, + "theoretical_loss": 3.7171687057202574, + "tokens_seen": 894042112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005891755265797392, + "loss": 1.323, + "theoretical_loss": 3.717142439106863, + "tokens_seen": 894107648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005891594784353059, + "loss": 1.4508, + "theoretical_loss": 3.717116174957713, + "tokens_seen": 894173184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005891434302908726, + "loss": 1.42, + "theoretical_loss": 3.717089913272396, + "tokens_seen": 894238720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005891273821464393, + "loss": 1.3354, + "theoretical_loss": 3.7170636540504995, + "tokens_seen": 894304256 + }, + { + "epoch": 0.27, + "learning_rate": 0.000589111334002006, + "loss": 1.4287, + "theoretical_loss": 3.7170373972916133, + "tokens_seen": 894369792 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890952858575727, + "loss": 1.3652, + "theoretical_loss": 3.7170111429953248, + "tokens_seen": 894435328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890792377131394, + "loss": 1.3814, + "theoretical_loss": 3.716984891161223, + "tokens_seen": 894500864 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 302661, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2354652881622314, + "objective/train/theoretical_loss": 3.7169586417888967, + "objective/train/tokens_used": 915026400, + "theoretical_loss": 3.7169586417888967, + "tokens_seen": 894566400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890631895687061, + "loss": 1.2496, + "theoretical_loss": 3.7169586417888967, + "tokens_seen": 894566400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890471414242728, + "loss": 1.2781, + "theoretical_loss": 3.7169323948779343, + "tokens_seen": 894631936 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890310932798395, + "loss": 1.3616, + "theoretical_loss": 3.716906150427925, + "tokens_seen": 894697472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890150451354062, + "loss": 1.4008, + "theoretical_loss": 3.7168799084384583, + "tokens_seen": 894763008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889989969909729, + "loss": 1.4406, + "theoretical_loss": 3.7168536689091223, + "tokens_seen": 894828544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889829488465396, + "loss": 1.2974, + "theoretical_loss": 3.716827431839507, + "tokens_seen": 894894080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889669007021063, + "loss": 1.4378, + "theoretical_loss": 3.7168011972292017, + "tokens_seen": 894959616 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588950852557673, + "loss": 1.4663, + "theoretical_loss": 3.7167749650777955, + "tokens_seen": 895025152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889348044132397, + "loss": 1.3886, + "theoretical_loss": 3.7167487353848774, + "tokens_seen": 895090688 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889187562688064, + "loss": 1.258, + "theoretical_loss": 3.7167225081500384, + "tokens_seen": 895156224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889027081243731, + "loss": 1.302, + "theoretical_loss": 3.716696283372867, + "tokens_seen": 895221760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888866599799398, + "loss": 1.3362, + "theoretical_loss": 3.716670061052954, + "tokens_seen": 895287296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888706118355065, + "loss": 1.3319, + "theoretical_loss": 3.7166438411898883, + "tokens_seen": 895352832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888545636910732, + "loss": 1.3314, + "theoretical_loss": 3.7166176237832604, + "tokens_seen": 895418368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888385155466399, + "loss": 1.3141, + "theoretical_loss": 3.7165914088326604, + "tokens_seen": 895483904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888224674022066, + "loss": 1.3686, + "theoretical_loss": 3.7165651963376787, + "tokens_seen": 895549440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888064192577733, + "loss": 1.2977, + "theoretical_loss": 3.716538986297905, + "tokens_seen": 895614976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00058879037111334, + "loss": 1.4045, + "theoretical_loss": 3.7165127787129304, + "tokens_seen": 895680512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005887743229689067, + "loss": 1.3782, + "theoretical_loss": 3.716486573582345, + "tokens_seen": 895746048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005887582748244734, + "loss": 1.2311, + "theoretical_loss": 3.7164603709057387, + "tokens_seen": 895811584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005887422266800401, + "loss": 1.5138, + "theoretical_loss": 3.716434170682704, + "tokens_seen": 895877120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005887261785356068, + "loss": 1.326, + "theoretical_loss": 3.7164079729128296, + "tokens_seen": 895942656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005887101303911735, + "loss": 1.4816, + "theoretical_loss": 3.7163817775957084, + "tokens_seen": 896008192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005886940822467402, + "loss": 1.4378, + "theoretical_loss": 3.71635558473093, + "tokens_seen": 896073728 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588678034102307, + "loss": 1.3225, + "theoretical_loss": 3.7163293943180857, + "tokens_seen": 896139264 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005886619859578736, + "loss": 1.3436, + "theoretical_loss": 3.7163032063567676, + "tokens_seen": 896204800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005886459378134403, + "loss": 1.4952, + "theoretical_loss": 3.7162770208465656, + "tokens_seen": 896270336 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588629889669007, + "loss": 1.3227, + "theoretical_loss": 3.716250837787072, + "tokens_seen": 896335872 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005886138415245737, + "loss": 1.4207, + "theoretical_loss": 3.716224657177878, + "tokens_seen": 896401408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885977933801404, + "loss": 1.311, + "theoretical_loss": 3.716198479018575, + "tokens_seen": 896466944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885817452357072, + "loss": 1.5235, + "theoretical_loss": 3.7161723033087553, + "tokens_seen": 896532480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885656970912739, + "loss": 1.451, + "theoretical_loss": 3.71614613004801, + "tokens_seen": 896598016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885496489468406, + "loss": 1.3184, + "theoretical_loss": 3.716119959235931, + "tokens_seen": 896663552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885336008024073, + "loss": 1.257, + "theoretical_loss": 3.716093790872111, + "tokens_seen": 896729088 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588517552657974, + "loss": 1.3325, + "theoretical_loss": 3.716067624956141, + "tokens_seen": 896794624 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885015045135407, + "loss": 1.3376, + "theoretical_loss": 3.716041461487614, + "tokens_seen": 896860160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884854563691074, + "loss": 1.4556, + "theoretical_loss": 3.716015300466122, + "tokens_seen": 896925696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884694082246741, + "loss": 1.4852, + "theoretical_loss": 3.7159891418912574, + "tokens_seen": 896991232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884533600802408, + "loss": 1.4474, + "theoretical_loss": 3.715962985762612, + "tokens_seen": 897056768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884373119358075, + "loss": 1.3913, + "theoretical_loss": 3.7159368320797794, + "tokens_seen": 897122304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884212637913742, + "loss": 1.3786, + "theoretical_loss": 3.7159106808423523, + "tokens_seen": 897187840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884052156469409, + "loss": 1.3073, + "theoretical_loss": 3.7158845320499223, + "tokens_seen": 897253376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005883891675025076, + "loss": 1.3805, + "theoretical_loss": 3.7158583857020826, + "tokens_seen": 897318912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005883731193580743, + "loss": 1.5188, + "theoretical_loss": 3.7158322417984273, + "tokens_seen": 897384448 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588357071213641, + "loss": 1.3454, + "theoretical_loss": 3.715806100338548, + "tokens_seen": 897449984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005883410230692077, + "loss": 1.3985, + "theoretical_loss": 3.7157799613220384, + "tokens_seen": 897515520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005883249749247744, + "loss": 1.3109, + "theoretical_loss": 3.715753824748492, + "tokens_seen": 897581056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005883089267803411, + "loss": 1.3792, + "theoretical_loss": 3.715727690617502, + "tokens_seen": 897646592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882928786359078, + "loss": 1.4112, + "theoretical_loss": 3.7157015589286617, + "tokens_seen": 897712128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882768304914745, + "loss": 1.5598, + "theoretical_loss": 3.715675429681564, + "tokens_seen": 897777664 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 303919, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4079523086547852, + "objective/train/theoretical_loss": 3.7156493028758035, + "objective/train/tokens_used": 918303200, + "theoretical_loss": 3.7156493028758035, + "tokens_seen": 897843200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882607823470412, + "loss": 1.4222, + "theoretical_loss": 3.7156493028758035, + "tokens_seen": 897843200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882447342026079, + "loss": 1.462, + "theoretical_loss": 3.715623178510974, + "tokens_seen": 897908736 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882286860581746, + "loss": 1.3918, + "theoretical_loss": 3.7155970565866685, + "tokens_seen": 897974272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882126379137413, + "loss": 1.4074, + "theoretical_loss": 3.7155709371024814, + "tokens_seen": 898039808 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588196589769308, + "loss": 1.3259, + "theoretical_loss": 3.715544820058007, + "tokens_seen": 898105344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881805416248747, + "loss": 1.387, + "theoretical_loss": 3.7155187054528387, + "tokens_seen": 898170880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881644934804414, + "loss": 1.4242, + "theoretical_loss": 3.715492593286571, + "tokens_seen": 898236416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881484453360081, + "loss": 1.3918, + "theoretical_loss": 3.715466483558799, + "tokens_seen": 898301952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881323971915748, + "loss": 1.343, + "theoretical_loss": 3.7154403762691155, + "tokens_seen": 898367488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881163490471415, + "loss": 1.3612, + "theoretical_loss": 3.7154142714171168, + "tokens_seen": 898433024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881003009027082, + "loss": 1.4501, + "theoretical_loss": 3.715388169002396, + "tokens_seen": 898498560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005880842527582749, + "loss": 1.3329, + "theoretical_loss": 3.715362069024548, + "tokens_seen": 898564096 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005880682046138416, + "loss": 1.245, + "theoretical_loss": 3.7153359714831686, + "tokens_seen": 898629632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005880521564694083, + "loss": 1.3412, + "theoretical_loss": 3.7153098763778516, + "tokens_seen": 898695168 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588036108324975, + "loss": 1.3866, + "theoretical_loss": 3.715283783708193, + "tokens_seen": 898760704 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005880200601805417, + "loss": 1.4898, + "theoretical_loss": 3.715257693473787, + "tokens_seen": 898826240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005880040120361084, + "loss": 1.3591, + "theoretical_loss": 3.715231605674229, + "tokens_seen": 898891776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879879638916751, + "loss": 1.4781, + "theoretical_loss": 3.7152055203091146, + "tokens_seen": 898957312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879719157472418, + "loss": 1.2317, + "theoretical_loss": 3.7151794373780387, + "tokens_seen": 899022848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879558676028084, + "loss": 1.4104, + "theoretical_loss": 3.715153356880597, + "tokens_seen": 899088384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879398194583752, + "loss": 1.3016, + "theoretical_loss": 3.715127278816386, + "tokens_seen": 899153920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879237713139418, + "loss": 1.4785, + "theoretical_loss": 3.7151012031849993, + "tokens_seen": 899219456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879077231695086, + "loss": 1.3736, + "theoretical_loss": 3.715075129986034, + "tokens_seen": 899284992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005878916750250752, + "loss": 1.3649, + "theoretical_loss": 3.715049059219086, + "tokens_seen": 899350528 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587875626880642, + "loss": 1.3917, + "theoretical_loss": 3.7150229908837513, + "tokens_seen": 899416064 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005878595787362086, + "loss": 1.3189, + "theoretical_loss": 3.7149969249796255, + "tokens_seen": 899481600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005878435305917754, + "loss": 1.3525, + "theoretical_loss": 3.7149708615063046, + "tokens_seen": 899547136 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587827482447342, + "loss": 1.3852, + "theoretical_loss": 3.7149448004633854, + "tokens_seen": 899612672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005878114343029087, + "loss": 1.3628, + "theoretical_loss": 3.714918741850464, + "tokens_seen": 899678208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877953861584754, + "loss": 1.3601, + "theoretical_loss": 3.7148926856671367, + "tokens_seen": 899743744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877793380140421, + "loss": 1.4436, + "theoretical_loss": 3.7148666319130004, + "tokens_seen": 899809280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877632898696088, + "loss": 1.4471, + "theoretical_loss": 3.7148405805876514, + "tokens_seen": 899874816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877472417251755, + "loss": 1.3154, + "theoretical_loss": 3.7148145316906866, + "tokens_seen": 899940352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877311935807422, + "loss": 1.3245, + "theoretical_loss": 3.7147884852217024, + "tokens_seen": 900005888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877151454363089, + "loss": 1.4425, + "theoretical_loss": 3.714762441180296, + "tokens_seen": 900071424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876990972918756, + "loss": 1.38, + "theoretical_loss": 3.714736399566065, + "tokens_seen": 900136960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876830491474423, + "loss": 1.1755, + "theoretical_loss": 3.7147103603786054, + "tokens_seen": 900202496 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587667001003009, + "loss": 1.2811, + "theoretical_loss": 3.7146843236175155, + "tokens_seen": 900268032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876509528585757, + "loss": 1.4444, + "theoretical_loss": 3.714658289282392, + "tokens_seen": 900333568 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876349047141424, + "loss": 1.4442, + "theoretical_loss": 3.7146322573728323, + "tokens_seen": 900399104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876188565697091, + "loss": 1.4458, + "theoretical_loss": 3.7146062278884338, + "tokens_seen": 900464640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876028084252758, + "loss": 1.304, + "theoretical_loss": 3.7145802008287943, + "tokens_seen": 900530176 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005875867602808425, + "loss": 1.3643, + "theoretical_loss": 3.7145541761935115, + "tokens_seen": 900595712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005875707121364092, + "loss": 1.4021, + "theoretical_loss": 3.714528153982183, + "tokens_seen": 900661248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005875546639919759, + "loss": 1.4293, + "theoretical_loss": 3.714502134194407, + "tokens_seen": 900726784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005875386158475426, + "loss": 1.396, + "theoretical_loss": 3.7144761168297817, + "tokens_seen": 900792320 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005875225677031093, + "loss": 1.3798, + "theoretical_loss": 3.714450101887904, + "tokens_seen": 900857856 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587506519558676, + "loss": 1.3311, + "theoretical_loss": 3.7144240893683733, + "tokens_seen": 900923392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874904714142427, + "loss": 1.3374, + "theoretical_loss": 3.7143980792707874, + "tokens_seen": 900988928 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874744232698094, + "loss": 1.4061, + "theoretical_loss": 3.7143720715947444, + "tokens_seen": 901054464 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 305176, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4514453411102295, + "objective/train/theoretical_loss": 3.7143460663398433, + "objective/train/tokens_used": 921580000, + "theoretical_loss": 3.7143460663398433, + "tokens_seen": 901120000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874583751253761, + "loss": 1.4118, + "theoretical_loss": 3.7143460663398433, + "tokens_seen": 901120000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874423269809428, + "loss": 1.4363, + "theoretical_loss": 3.714320063505682, + "tokens_seen": 901185536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874262788365095, + "loss": 1.2861, + "theoretical_loss": 3.7142940630918595, + "tokens_seen": 901251072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874102306920762, + "loss": 1.5371, + "theoretical_loss": 3.7142680650979747, + "tokens_seen": 901316608 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005873941825476429, + "loss": 1.3465, + "theoretical_loss": 3.7142420695236265, + "tokens_seen": 901382144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005873781344032096, + "loss": 1.3457, + "theoretical_loss": 3.7142160763684133, + "tokens_seen": 901447680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005873620862587764, + "loss": 1.3535, + "theoretical_loss": 3.7141900856319343, + "tokens_seen": 901513216 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587346038114343, + "loss": 1.4932, + "theoretical_loss": 3.714164097313789, + "tokens_seen": 901578752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005873299899699098, + "loss": 1.343, + "theoretical_loss": 3.7141381114135763, + "tokens_seen": 901644288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005873139418254764, + "loss": 1.3572, + "theoretical_loss": 3.7141121279308957, + "tokens_seen": 901709824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872978936810432, + "loss": 1.3743, + "theoretical_loss": 3.7140861468653465, + "tokens_seen": 901775360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872818455366098, + "loss": 1.4433, + "theoretical_loss": 3.7140601682165277, + "tokens_seen": 901840896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872657973921766, + "loss": 1.393, + "theoretical_loss": 3.71403419198404, + "tokens_seen": 901906432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872497492477432, + "loss": 1.2557, + "theoretical_loss": 3.714008218167482, + "tokens_seen": 901971968 + }, + { + "epoch": 0.27, + "learning_rate": 0.00058723370110331, + "loss": 1.4866, + "theoretical_loss": 3.7139822467664545, + "tokens_seen": 902037504 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872176529588767, + "loss": 1.3731, + "theoretical_loss": 3.713956277780557, + "tokens_seen": 902103040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872016048144434, + "loss": 1.4039, + "theoretical_loss": 3.7139303112093884, + "tokens_seen": 902168576 + }, + { + "epoch": 0.27, + "learning_rate": 0.00058718555667001, + "loss": 1.2867, + "theoretical_loss": 3.7139043470525506, + "tokens_seen": 902234112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005871695085255768, + "loss": 1.3927, + "theoretical_loss": 3.713878385309643, + "tokens_seen": 902299648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005871534603811435, + "loss": 1.4256, + "theoretical_loss": 3.7138524259802654, + "tokens_seen": 902365184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005871374122367102, + "loss": 1.4969, + "theoretical_loss": 3.7138264690640184, + "tokens_seen": 902430720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005871213640922769, + "loss": 1.3782, + "theoretical_loss": 3.7138005145605026, + "tokens_seen": 902496256 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005871053159478436, + "loss": 1.3181, + "theoretical_loss": 3.713774562469319, + "tokens_seen": 902561792 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005870892678034103, + "loss": 1.4391, + "theoretical_loss": 3.7137486127900674, + "tokens_seen": 902627328 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587073219658977, + "loss": 1.4097, + "theoretical_loss": 3.713722665522349, + "tokens_seen": 902692864 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005870571715145437, + "loss": 1.4201, + "theoretical_loss": 3.713696720665765, + "tokens_seen": 902758400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005870411233701104, + "loss": 1.349, + "theoretical_loss": 3.7136707782199156, + "tokens_seen": 902823936 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005870250752256771, + "loss": 1.3211, + "theoretical_loss": 3.7136448381844023, + "tokens_seen": 902889472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005870090270812438, + "loss": 1.4117, + "theoretical_loss": 3.713618900558826, + "tokens_seen": 902955008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005869929789368105, + "loss": 1.4085, + "theoretical_loss": 3.7135929653427877, + "tokens_seen": 903020544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005869769307923772, + "loss": 1.3658, + "theoretical_loss": 3.7135670325358894, + "tokens_seen": 903086080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005869608826479439, + "loss": 1.5581, + "theoretical_loss": 3.7135411021377323, + "tokens_seen": 903151616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005869448345035106, + "loss": 1.4429, + "theoretical_loss": 3.713515174147917, + "tokens_seen": 903217152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005869287863590773, + "loss": 1.3243, + "theoretical_loss": 3.7134892485660465, + "tokens_seen": 903282688 + }, + { + "epoch": 0.27, + "learning_rate": 0.000586912738214644, + "loss": 1.4235, + "theoretical_loss": 3.7134633253917215, + "tokens_seen": 903348224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868966900702107, + "loss": 1.4358, + "theoretical_loss": 3.713437404624544, + "tokens_seen": 903413760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868806419257774, + "loss": 1.4272, + "theoretical_loss": 3.7134114862641163, + "tokens_seen": 903479296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868645937813441, + "loss": 1.3875, + "theoretical_loss": 3.71338557031004, + "tokens_seen": 903544832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868485456369108, + "loss": 1.3641, + "theoretical_loss": 3.713359656761917, + "tokens_seen": 903610368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868324974924775, + "loss": 1.4771, + "theoretical_loss": 3.71333374561935, + "tokens_seen": 903675904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868164493480442, + "loss": 1.3326, + "theoretical_loss": 3.713307836881941, + "tokens_seen": 903741440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868004012036109, + "loss": 1.4815, + "theoretical_loss": 3.713281930549292, + "tokens_seen": 903806976 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005867843530591776, + "loss": 1.2739, + "theoretical_loss": 3.7132560266210057, + "tokens_seen": 903872512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005867683049147443, + "loss": 1.3471, + "theoretical_loss": 3.713230125096685, + "tokens_seen": 903938048 + }, + { + "epoch": 0.27, + "learning_rate": 0.000586752256770311, + "loss": 1.4101, + "theoretical_loss": 3.7132042259759324, + "tokens_seen": 904003584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005867362086258777, + "loss": 1.4148, + "theoretical_loss": 3.71317832925835, + "tokens_seen": 904069120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005867201604814444, + "loss": 1.4313, + "theoretical_loss": 3.7131524349435416, + "tokens_seen": 904134656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005867041123370111, + "loss": 1.398, + "theoretical_loss": 3.7131265430311093, + "tokens_seen": 904200192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866880641925778, + "loss": 1.3387, + "theoretical_loss": 3.713100653520657, + "tokens_seen": 904265728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866720160481445, + "loss": 1.521, + "theoretical_loss": 3.7130747664117862, + "tokens_seen": 904331264 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 306066, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.375870943069458, + "objective/train/theoretical_loss": 3.7130488817041023, + "objective/train/tokens_used": 924856800, + "theoretical_loss": 3.7130488817041023, + "tokens_seen": 904396800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866559679037112, + "loss": 1.4468, + "theoretical_loss": 3.7130488817041023, + "tokens_seen": 904396800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866399197592779, + "loss": 1.3345, + "theoretical_loss": 3.7130229993972073, + "tokens_seen": 904462336 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866238716148446, + "loss": 1.3374, + "theoretical_loss": 3.712997119490704, + "tokens_seen": 904527872 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866078234704113, + "loss": 1.4349, + "theoretical_loss": 3.7129712419841976, + "tokens_seen": 904593408 + }, + { + "epoch": 0.27, + "learning_rate": 0.000586591775325978, + "loss": 1.3844, + "theoretical_loss": 3.7129453668772903, + "tokens_seen": 904658944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005865757271815447, + "loss": 1.3579, + "theoretical_loss": 3.7129194941695864, + "tokens_seen": 904724480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005865596790371113, + "loss": 1.3183, + "theoretical_loss": 3.7128936238606896, + "tokens_seen": 904790016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005865436308926781, + "loss": 1.3435, + "theoretical_loss": 3.712867755950203, + "tokens_seen": 904855552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005865275827482447, + "loss": 1.4187, + "theoretical_loss": 3.7128418904377316, + "tokens_seen": 904921088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005865115346038115, + "loss": 1.4611, + "theoretical_loss": 3.712816027322879, + "tokens_seen": 904986624 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864954864593781, + "loss": 1.2744, + "theoretical_loss": 3.71279016660525, + "tokens_seen": 905052160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864794383149449, + "loss": 1.3814, + "theoretical_loss": 3.712764308284448, + "tokens_seen": 905117696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864633901705115, + "loss": 1.4546, + "theoretical_loss": 3.7127384523600773, + "tokens_seen": 905183232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864473420260783, + "loss": 1.2955, + "theoretical_loss": 3.712712598831743, + "tokens_seen": 905248768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864312938816449, + "loss": 1.3816, + "theoretical_loss": 3.712686747699049, + "tokens_seen": 905314304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864152457372116, + "loss": 1.3562, + "theoretical_loss": 3.7126608989616, + "tokens_seen": 905379840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863991975927783, + "loss": 1.3533, + "theoretical_loss": 3.7126350526190013, + "tokens_seen": 905445376 + }, + { + "epoch": 0.27, + "learning_rate": 0.000586383149448345, + "loss": 1.4271, + "theoretical_loss": 3.712609208670857, + "tokens_seen": 905510912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863671013039117, + "loss": 1.3787, + "theoretical_loss": 3.712583367116773, + "tokens_seen": 905576448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863510531594784, + "loss": 1.3765, + "theoretical_loss": 3.7125575279563527, + "tokens_seen": 905641984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863350050150451, + "loss": 1.5112, + "theoretical_loss": 3.7125316911892026, + "tokens_seen": 905707520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863189568706118, + "loss": 1.3779, + "theoretical_loss": 3.7125058568149267, + "tokens_seen": 905773056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863029087261785, + "loss": 1.3798, + "theoretical_loss": 3.7124800248331318, + "tokens_seen": 905838592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005862868605817452, + "loss": 1.4332, + "theoretical_loss": 3.712454195243422, + "tokens_seen": 905904128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005862708124373119, + "loss": 1.3842, + "theoretical_loss": 3.712428368045403, + "tokens_seen": 905969664 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005862547642928786, + "loss": 1.3734, + "theoretical_loss": 3.7124025432386807, + "tokens_seen": 906035200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005862387161484453, + "loss": 1.358, + "theoretical_loss": 3.712376720822861, + "tokens_seen": 906100736 + }, + { + "epoch": 0.27, + "learning_rate": 0.000586222668004012, + "loss": 1.3914, + "theoretical_loss": 3.7123509007975484, + "tokens_seen": 906166272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005862066198595787, + "loss": 1.4996, + "theoretical_loss": 3.7123250831623498, + "tokens_seen": 906231808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861905717151454, + "loss": 1.3434, + "theoretical_loss": 3.712299267916871, + "tokens_seen": 906297344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861745235707121, + "loss": 1.4045, + "theoretical_loss": 3.7122734550607177, + "tokens_seen": 906362880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861584754262789, + "loss": 1.4185, + "theoretical_loss": 3.712247644593497, + "tokens_seen": 906428416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861424272818455, + "loss": 1.3847, + "theoretical_loss": 3.7122218365148134, + "tokens_seen": 906493952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861263791374123, + "loss": 1.3732, + "theoretical_loss": 3.7121960308242743, + "tokens_seen": 906559488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861103309929789, + "loss": 1.2714, + "theoretical_loss": 3.7121702275214865, + "tokens_seen": 906625024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860942828485457, + "loss": 1.3374, + "theoretical_loss": 3.7121444266060553, + "tokens_seen": 906690560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860782347041123, + "loss": 1.3808, + "theoretical_loss": 3.712118628077588, + "tokens_seen": 906756096 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860621865596791, + "loss": 1.4362, + "theoretical_loss": 3.712092831935691, + "tokens_seen": 906821632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860461384152457, + "loss": 1.3248, + "theoretical_loss": 3.712067038179972, + "tokens_seen": 906887168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860300902708125, + "loss": 1.485, + "theoretical_loss": 3.7120412468100366, + "tokens_seen": 906952704 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860140421263791, + "loss": 1.3724, + "theoretical_loss": 3.7120154578254922, + "tokens_seen": 907018240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005859979939819459, + "loss": 1.4268, + "theoretical_loss": 3.711989671225946, + "tokens_seen": 907083776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005859819458375125, + "loss": 1.4211, + "theoretical_loss": 3.711963887011005, + "tokens_seen": 907149312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005859658976930793, + "loss": 1.4342, + "theoretical_loss": 3.7119381051802764, + "tokens_seen": 907214848 + }, + { + "epoch": 0.27, + "learning_rate": 0.000585949849548646, + "loss": 1.3711, + "theoretical_loss": 3.7119123257333677, + "tokens_seen": 907280384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005859338014042127, + "loss": 1.3671, + "theoretical_loss": 3.711886548669886, + "tokens_seen": 907345920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005859177532597793, + "loss": 1.3794, + "theoretical_loss": 3.7118607739894394, + "tokens_seen": 907411456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005859017051153461, + "loss": 1.3348, + "theoretical_loss": 3.711835001691635, + "tokens_seen": 907476992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858856569709127, + "loss": 1.4623, + "theoretical_loss": 3.7118092317760807, + "tokens_seen": 907542528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858696088264795, + "loss": 1.4309, + "theoretical_loss": 3.7117834642423846, + "tokens_seen": 907608064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 307328, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.252403736114502, + "objective/train/theoretical_loss": 3.7117576990901537, + "objective/train/tokens_used": 928133600, + "theoretical_loss": 3.7117576990901537, + "tokens_seen": 907673600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858535606820462, + "loss": 1.3462, + "theoretical_loss": 3.7117576990901537, + "tokens_seen": 907673600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858375125376129, + "loss": 1.3259, + "theoretical_loss": 3.711731936318997, + "tokens_seen": 907739136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858214643931796, + "loss": 1.3657, + "theoretical_loss": 3.711706175928522, + "tokens_seen": 907804672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858054162487463, + "loss": 1.538, + "theoretical_loss": 3.7116804179183367, + "tokens_seen": 907870208 + }, + { + "epoch": 0.28, + "learning_rate": 0.000585789368104313, + "loss": 1.3548, + "theoretical_loss": 3.7116546622880495, + "tokens_seen": 907935744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005857733199598797, + "loss": 1.3871, + "theoretical_loss": 3.711628909037269, + "tokens_seen": 908001280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005857572718154464, + "loss": 1.3435, + "theoretical_loss": 3.7116031581656035, + "tokens_seen": 908066816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005857412236710131, + "loss": 1.3695, + "theoretical_loss": 3.7115774096726617, + "tokens_seen": 908132352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005857251755265798, + "loss": 1.3902, + "theoretical_loss": 3.711551663558052, + "tokens_seen": 908197888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005857091273821465, + "loss": 1.2973, + "theoretical_loss": 3.7115259198213835, + "tokens_seen": 908263424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005856930792377132, + "loss": 1.5021, + "theoretical_loss": 3.7115001784622637, + "tokens_seen": 908328960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005856770310932799, + "loss": 1.2715, + "theoretical_loss": 3.7114744394803036, + "tokens_seen": 908394496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005856609829488466, + "loss": 1.3926, + "theoretical_loss": 3.711448702875111, + "tokens_seen": 908460032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005856449348044133, + "loss": 1.3528, + "theoretical_loss": 3.7114229686462945, + "tokens_seen": 908525568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00058562888665998, + "loss": 1.3323, + "theoretical_loss": 3.711397236793464, + "tokens_seen": 908591104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005856128385155467, + "loss": 1.3253, + "theoretical_loss": 3.711371507316229, + "tokens_seen": 908656640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855967903711134, + "loss": 1.2965, + "theoretical_loss": 3.7113457802141983, + "tokens_seen": 908722176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855807422266801, + "loss": 1.3901, + "theoretical_loss": 3.7113200554869814, + "tokens_seen": 908787712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855646940822468, + "loss": 1.2374, + "theoretical_loss": 3.7112943331341883, + "tokens_seen": 908853248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855486459378135, + "loss": 1.3409, + "theoretical_loss": 3.711268613155428, + "tokens_seen": 908918784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855325977933802, + "loss": 1.4335, + "theoretical_loss": 3.7112428955503116, + "tokens_seen": 908984320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855165496489469, + "loss": 1.4171, + "theoretical_loss": 3.711217180318447, + "tokens_seen": 909049856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855005015045136, + "loss": 1.3504, + "theoretical_loss": 3.7111914674594457, + "tokens_seen": 909115392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005854844533600803, + "loss": 1.3422, + "theoretical_loss": 3.7111657569729166, + "tokens_seen": 909180928 + }, + { + "epoch": 0.28, + "learning_rate": 0.000585468405215647, + "loss": 1.3639, + "theoretical_loss": 3.7111400488584705, + "tokens_seen": 909246464 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005854523570712137, + "loss": 1.3825, + "theoretical_loss": 3.7111143431157174, + "tokens_seen": 909312000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005854363089267804, + "loss": 1.4059, + "theoretical_loss": 3.7110886397442675, + "tokens_seen": 909377536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005854202607823471, + "loss": 1.3736, + "theoretical_loss": 3.7110629387437317, + "tokens_seen": 909443072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005854042126379138, + "loss": 1.3197, + "theoretical_loss": 3.7110372401137193, + "tokens_seen": 909508608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005853881644934805, + "loss": 1.4951, + "theoretical_loss": 3.711011543853842, + "tokens_seen": 909574144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005853721163490472, + "loss": 1.3901, + "theoretical_loss": 3.71098584996371, + "tokens_seen": 909639680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005853560682046139, + "loss": 1.4033, + "theoretical_loss": 3.7109601584429344, + "tokens_seen": 909705216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005853400200601806, + "loss": 1.3312, + "theoretical_loss": 3.7109344692911255, + "tokens_seen": 909770752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005853239719157473, + "loss": 1.4203, + "theoretical_loss": 3.710908782507895, + "tokens_seen": 909836288 + }, + { + "epoch": 0.28, + "learning_rate": 0.000585307923771314, + "loss": 1.4514, + "theoretical_loss": 3.710883098092853, + "tokens_seen": 909901824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852918756268807, + "loss": 1.4127, + "theoretical_loss": 3.7108574160456107, + "tokens_seen": 909967360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852758274824474, + "loss": 1.4031, + "theoretical_loss": 3.7108317363657797, + "tokens_seen": 910032896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852597793380141, + "loss": 1.407, + "theoretical_loss": 3.710806059052972, + "tokens_seen": 910098432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852437311935808, + "loss": 1.3883, + "theoretical_loss": 3.7107803841067977, + "tokens_seen": 910163968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852276830491475, + "loss": 1.3155, + "theoretical_loss": 3.710754711526869, + "tokens_seen": 910229504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852116349047142, + "loss": 1.343, + "theoretical_loss": 3.7107290413127973, + "tokens_seen": 910295040 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005851955867602809, + "loss": 1.3359, + "theoretical_loss": 3.7107033734641943, + "tokens_seen": 910360576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005851795386158476, + "loss": 1.356, + "theoretical_loss": 3.710677707980672, + "tokens_seen": 910426112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005851634904714142, + "loss": 1.3019, + "theoretical_loss": 3.710652044861842, + "tokens_seen": 910491648 + }, + { + "epoch": 0.28, + "learning_rate": 0.000585147442326981, + "loss": 1.3407, + "theoretical_loss": 3.7106263841073153, + "tokens_seen": 910557184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005851313941825476, + "loss": 1.4251, + "theoretical_loss": 3.7106007257167057, + "tokens_seen": 910622720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005851153460381144, + "loss": 1.4142, + "theoretical_loss": 3.7105750696896247, + "tokens_seen": 910688256 + }, + { + "epoch": 0.28, + "learning_rate": 0.000585099297893681, + "loss": 1.4038, + "theoretical_loss": 3.710549416025684, + "tokens_seen": 910753792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850832497492478, + "loss": 1.4423, + "theoretical_loss": 3.7105237647244964, + "tokens_seen": 910819328 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850672016048144, + "loss": 1.4541, + "theoretical_loss": 3.710498115785674, + "tokens_seen": 910884864 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 308239, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.30680251121521, + "objective/train/theoretical_loss": 3.7104724692088293, + "objective/train/tokens_used": 931410400, + "theoretical_loss": 3.7104724692088293, + "tokens_seen": 910950400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850511534603812, + "loss": 1.386, + "theoretical_loss": 3.7104724692088293, + "tokens_seen": 910950400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850351053159478, + "loss": 1.4251, + "theoretical_loss": 3.7104468249935754, + "tokens_seen": 911015936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850190571715145, + "loss": 1.4013, + "theoretical_loss": 3.710421183139524, + "tokens_seen": 911081472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850030090270812, + "loss": 1.2597, + "theoretical_loss": 3.710395543646289, + "tokens_seen": 911147008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005849869608826479, + "loss": 1.4452, + "theoretical_loss": 3.710369906513483, + "tokens_seen": 911212544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005849709127382146, + "loss": 1.4181, + "theoretical_loss": 3.710344271740718, + "tokens_seen": 911278080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005849548645937814, + "loss": 1.4182, + "theoretical_loss": 3.710318639327608, + "tokens_seen": 911343616 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584938816449348, + "loss": 1.3231, + "theoretical_loss": 3.7102930092737663, + "tokens_seen": 911409152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005849227683049148, + "loss": 1.3125, + "theoretical_loss": 3.710267381578805, + "tokens_seen": 911474688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005849067201604814, + "loss": 1.3279, + "theoretical_loss": 3.710241756242339, + "tokens_seen": 911540224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005848906720160482, + "loss": 1.3827, + "theoretical_loss": 3.7102161332639794, + "tokens_seen": 911605760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005848746238716148, + "loss": 1.2973, + "theoretical_loss": 3.7101905126433423, + "tokens_seen": 911671296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005848585757271816, + "loss": 1.4018, + "theoretical_loss": 3.7101648943800396, + "tokens_seen": 911736832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005848425275827482, + "loss": 1.4474, + "theoretical_loss": 3.7101392784736857, + "tokens_seen": 911802368 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584826479438315, + "loss": 1.3856, + "theoretical_loss": 3.7101136649238944, + "tokens_seen": 911867904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005848104312938816, + "loss": 1.435, + "theoretical_loss": 3.7100880537302787, + "tokens_seen": 911933440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005847943831494484, + "loss": 1.3914, + "theoretical_loss": 3.7100624448924533, + "tokens_seen": 911998976 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584778335005015, + "loss": 1.378, + "theoretical_loss": 3.710036838410032, + "tokens_seen": 912064512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005847622868605818, + "loss": 1.3083, + "theoretical_loss": 3.7100112342826295, + "tokens_seen": 912130048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005847462387161484, + "loss": 1.4158, + "theoretical_loss": 3.7099856325098592, + "tokens_seen": 912195584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005847301905717152, + "loss": 1.3268, + "theoretical_loss": 3.7099600330913356, + "tokens_seen": 912261120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005847141424272818, + "loss": 1.412, + "theoretical_loss": 3.709934436026674, + "tokens_seen": 912326656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005846980942828486, + "loss": 1.3536, + "theoretical_loss": 3.709908841315488, + "tokens_seen": 912392192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005846820461384152, + "loss": 1.4512, + "theoretical_loss": 3.709883248957392, + "tokens_seen": 912457728 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584665997993982, + "loss": 1.3793, + "theoretical_loss": 3.709857658952001, + "tokens_seen": 912523264 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005846499498495486, + "loss": 1.4338, + "theoretical_loss": 3.7098320712989303, + "tokens_seen": 912588800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005846339017051154, + "loss": 1.3351, + "theoretical_loss": 3.7098064859977944, + "tokens_seen": 912654336 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584617853560682, + "loss": 1.379, + "theoretical_loss": 3.7097809030482076, + "tokens_seen": 912719872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005846018054162488, + "loss": 1.3192, + "theoretical_loss": 3.709755322449786, + "tokens_seen": 912785408 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005845857572718154, + "loss": 1.3225, + "theoretical_loss": 3.7097297442021437, + "tokens_seen": 912850944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005845697091273822, + "loss": 1.3692, + "theoretical_loss": 3.7097041683048966, + "tokens_seen": 912916480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005845536609829488, + "loss": 1.3323, + "theoretical_loss": 3.7096785947576603, + "tokens_seen": 912982016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005845376128385156, + "loss": 1.4171, + "theoretical_loss": 3.7096530235600493, + "tokens_seen": 913047552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005845215646940822, + "loss": 1.4956, + "theoretical_loss": 3.70962745471168, + "tokens_seen": 913113088 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584505516549649, + "loss": 1.3194, + "theoretical_loss": 3.709601888212167, + "tokens_seen": 913178624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005844894684052156, + "loss": 1.3993, + "theoretical_loss": 3.7095763240611275, + "tokens_seen": 913244160 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005844734202607824, + "loss": 1.4061, + "theoretical_loss": 3.7095507622581754, + "tokens_seen": 913309696 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584457372116349, + "loss": 1.3649, + "theoretical_loss": 3.7095252028029275, + "tokens_seen": 913375232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005844413239719158, + "loss": 1.3078, + "theoretical_loss": 3.7094996456950002, + "tokens_seen": 913440768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005844252758274825, + "loss": 1.419, + "theoretical_loss": 3.709474090934009, + "tokens_seen": 913506304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005844092276830492, + "loss": 1.4194, + "theoretical_loss": 3.70944853851957, + "tokens_seen": 913571840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005843931795386159, + "loss": 1.3092, + "theoretical_loss": 3.7094229884512995, + "tokens_seen": 913637376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005843771313941826, + "loss": 1.36, + "theoretical_loss": 3.7093974407288135, + "tokens_seen": 913702912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005843610832497493, + "loss": 1.4087, + "theoretical_loss": 3.7093718953517287, + "tokens_seen": 913768448 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584345035105316, + "loss": 1.3081, + "theoretical_loss": 3.7093463523196624, + "tokens_seen": 913833984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005843289869608827, + "loss": 1.4695, + "theoretical_loss": 3.70932081163223, + "tokens_seen": 913899520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005843129388164494, + "loss": 1.3757, + "theoretical_loss": 3.709295273289048, + "tokens_seen": 913965056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842968906720161, + "loss": 1.3217, + "theoretical_loss": 3.709269737289734, + "tokens_seen": 914030592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842808425275828, + "loss": 1.4032, + "theoretical_loss": 3.7092442036339044, + "tokens_seen": 914096128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842647943831495, + "loss": 1.3886, + "theoretical_loss": 3.7092186723211773, + "tokens_seen": 914161664 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 309446, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3680208921432495, + "objective/train/theoretical_loss": 3.7091931433511673, + "objective/train/tokens_used": 934687200, + "theoretical_loss": 3.7091931433511673, + "tokens_seen": 914227200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842487462387162, + "loss": 1.3914, + "theoretical_loss": 3.7091931433511673, + "tokens_seen": 914227200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842326980942829, + "loss": 1.4397, + "theoretical_loss": 3.7091676167234944, + "tokens_seen": 914292736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842166499498496, + "loss": 1.4257, + "theoretical_loss": 3.7091420924377734, + "tokens_seen": 914358272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842006018054163, + "loss": 1.4036, + "theoretical_loss": 3.709116570493623, + "tokens_seen": 914423808 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584184553660983, + "loss": 1.3334, + "theoretical_loss": 3.70909105089066, + "tokens_seen": 914489344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005841685055165497, + "loss": 1.3621, + "theoretical_loss": 3.709065533628502, + "tokens_seen": 914554880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005841524573721164, + "loss": 1.3287, + "theoretical_loss": 3.7090400187067667, + "tokens_seen": 914620416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005841364092276831, + "loss": 1.3724, + "theoretical_loss": 3.709014506125072, + "tokens_seen": 914685952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005841203610832498, + "loss": 1.3539, + "theoretical_loss": 3.708988995883035, + "tokens_seen": 914751488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005841043129388165, + "loss": 1.4639, + "theoretical_loss": 3.7089634879802738, + "tokens_seen": 914817024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005840882647943832, + "loss": 1.4849, + "theoretical_loss": 3.7089379824164066, + "tokens_seen": 914882560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005840722166499499, + "loss": 1.3807, + "theoretical_loss": 3.708912479191051, + "tokens_seen": 914948096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005840561685055166, + "loss": 1.4203, + "theoretical_loss": 3.708886978303825, + "tokens_seen": 915013632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005840401203610833, + "loss": 1.447, + "theoretical_loss": 3.708861479754348, + "tokens_seen": 915079168 + }, + { + "epoch": 0.28, + "learning_rate": 0.00058402407221665, + "loss": 1.3096, + "theoretical_loss": 3.708835983542237, + "tokens_seen": 915144704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005840080240722167, + "loss": 1.375, + "theoretical_loss": 3.708810489667111, + "tokens_seen": 915210240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839919759277834, + "loss": 1.3331, + "theoretical_loss": 3.7087849981285883, + "tokens_seen": 915275776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839759277833501, + "loss": 1.307, + "theoretical_loss": 3.7087595089262875, + "tokens_seen": 915341312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839598796389168, + "loss": 1.395, + "theoretical_loss": 3.7087340220598266, + "tokens_seen": 915406848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839438314944835, + "loss": 1.3897, + "theoretical_loss": 3.7087085375288256, + "tokens_seen": 915472384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839277833500502, + "loss": 1.4486, + "theoretical_loss": 3.708683055332902, + "tokens_seen": 915537920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839117352056169, + "loss": 1.4165, + "theoretical_loss": 3.7086575754716757, + "tokens_seen": 915603456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005838956870611836, + "loss": 1.4194, + "theoretical_loss": 3.7086320979447653, + "tokens_seen": 915668992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005838796389167503, + "loss": 1.4557, + "theoretical_loss": 3.70860662275179, + "tokens_seen": 915734528 + }, + { + "epoch": 0.28, + "learning_rate": 0.000583863590772317, + "loss": 1.3824, + "theoretical_loss": 3.7085811498923693, + "tokens_seen": 915800064 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005838475426278837, + "loss": 1.4194, + "theoretical_loss": 3.7085556793661216, + "tokens_seen": 915865600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005838314944834504, + "loss": 1.4242, + "theoretical_loss": 3.708530211172667, + "tokens_seen": 915931136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005838154463390171, + "loss": 1.3427, + "theoretical_loss": 3.7085047453116244, + "tokens_seen": 915996672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837993981945838, + "loss": 1.3287, + "theoretical_loss": 3.7084792817826147, + "tokens_seen": 916062208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837833500501505, + "loss": 1.4027, + "theoretical_loss": 3.7084538205852553, + "tokens_seen": 916127744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837673019057172, + "loss": 1.3169, + "theoretical_loss": 3.708428361719168, + "tokens_seen": 916193280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837512537612839, + "loss": 1.3977, + "theoretical_loss": 3.7084029051839713, + "tokens_seen": 916258816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837352056168505, + "loss": 1.4781, + "theoretical_loss": 3.7083774509792855, + "tokens_seen": 916324352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837191574724173, + "loss": 1.4445, + "theoretical_loss": 3.708351999104731, + "tokens_seen": 916389888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837031093279839, + "loss": 1.4419, + "theoretical_loss": 3.7083265495599274, + "tokens_seen": 916455424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836870611835507, + "loss": 1.4046, + "theoretical_loss": 3.708301102344495, + "tokens_seen": 916520960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836710130391173, + "loss": 1.383, + "theoretical_loss": 3.7082756574580538, + "tokens_seen": 916586496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836549648946841, + "loss": 1.3516, + "theoretical_loss": 3.7082502149002243, + "tokens_seen": 916652032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836389167502507, + "loss": 1.2977, + "theoretical_loss": 3.7082247746706276, + "tokens_seen": 916717568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836228686058175, + "loss": 1.3344, + "theoretical_loss": 3.7081993367688835, + "tokens_seen": 916783104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836068204613841, + "loss": 1.4308, + "theoretical_loss": 3.708173901194612, + "tokens_seen": 916848640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835907723169509, + "loss": 1.4316, + "theoretical_loss": 3.708148467947435, + "tokens_seen": 916914176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835747241725175, + "loss": 1.3856, + "theoretical_loss": 3.708123037026973, + "tokens_seen": 916979712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835586760280843, + "loss": 1.3514, + "theoretical_loss": 3.7080976084328467, + "tokens_seen": 917045248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835426278836509, + "loss": 1.2815, + "theoretical_loss": 3.7080721821646767, + "tokens_seen": 917110784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835265797392177, + "loss": 1.3831, + "theoretical_loss": 3.7080467582220855, + "tokens_seen": 917176320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835105315947843, + "loss": 1.4292, + "theoretical_loss": 3.708021336604692, + "tokens_seen": 917241856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834944834503511, + "loss": 1.4708, + "theoretical_loss": 3.707995917312119, + "tokens_seen": 917307392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834784353059177, + "loss": 1.4295, + "theoretical_loss": 3.7079705003439876, + "tokens_seen": 917372928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834623871614845, + "loss": 1.3694, + "theoretical_loss": 3.7079450856999188, + "tokens_seen": 917438464 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 310499, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3221814632415771, + "objective/train/theoretical_loss": 3.7079196733795348, + "objective/train/tokens_used": 937964000, + "theoretical_loss": 3.7079196733795348, + "tokens_seen": 917504000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834463390170511, + "loss": 1.3723, + "theoretical_loss": 3.7079196733795348, + "tokens_seen": 917504000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834302908726179, + "loss": 1.3693, + "theoretical_loss": 3.7078942633824563, + "tokens_seen": 917569536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834142427281845, + "loss": 1.3665, + "theoretical_loss": 3.7078688557083055, + "tokens_seen": 917635072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833981945837513, + "loss": 1.4111, + "theoretical_loss": 3.7078434503567044, + "tokens_seen": 917700608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833821464393179, + "loss": 1.415, + "theoretical_loss": 3.707818047327274, + "tokens_seen": 917766144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833660982948847, + "loss": 1.4638, + "theoretical_loss": 3.7077926466196374, + "tokens_seen": 917831680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833500501504513, + "loss": 1.3468, + "theoretical_loss": 3.7077672482334156, + "tokens_seen": 917897216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833340020060181, + "loss": 1.2463, + "theoretical_loss": 3.7077418521682315, + "tokens_seen": 917962752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833179538615847, + "loss": 1.2948, + "theoretical_loss": 3.7077164584237066, + "tokens_seen": 918028288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833019057171515, + "loss": 1.4244, + "theoretical_loss": 3.7076910669994634, + "tokens_seen": 918093824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005832858575727181, + "loss": 1.4251, + "theoretical_loss": 3.7076656778951245, + "tokens_seen": 918159360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005832698094282849, + "loss": 1.468, + "theoretical_loss": 3.7076402911103123, + "tokens_seen": 918224896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005832537612838515, + "loss": 1.4648, + "theoretical_loss": 3.7076149066446495, + "tokens_seen": 918290432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005832377131394183, + "loss": 1.3711, + "theoretical_loss": 3.707589524497759, + "tokens_seen": 918355968 + }, + { + "epoch": 0.28, + "learning_rate": 0.000583221664994985, + "loss": 1.3111, + "theoretical_loss": 3.707564144669263, + "tokens_seen": 918421504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005832056168505517, + "loss": 1.3687, + "theoretical_loss": 3.707538767158784, + "tokens_seen": 918487040 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831895687061183, + "loss": 1.3913, + "theoretical_loss": 3.7075133919659455, + "tokens_seen": 918552576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831735205616851, + "loss": 1.3898, + "theoretical_loss": 3.7074880190903707, + "tokens_seen": 918618112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831574724172517, + "loss": 1.3975, + "theoretical_loss": 3.707462648531682, + "tokens_seen": 918683648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831414242728185, + "loss": 1.4459, + "theoretical_loss": 3.7074372802895033, + "tokens_seen": 918749184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831253761283851, + "loss": 1.4212, + "theoretical_loss": 3.7074119143634574, + "tokens_seen": 918814720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831093279839519, + "loss": 1.3909, + "theoretical_loss": 3.707386550753168, + "tokens_seen": 918880256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005830932798395186, + "loss": 1.3345, + "theoretical_loss": 3.7073611894582585, + "tokens_seen": 918945792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005830772316950853, + "loss": 1.3417, + "theoretical_loss": 3.707335830478352, + "tokens_seen": 919011328 + }, + { + "epoch": 0.28, + "learning_rate": 0.000583061183550652, + "loss": 1.398, + "theoretical_loss": 3.707310473813072, + "tokens_seen": 919076864 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005830451354062187, + "loss": 1.3413, + "theoretical_loss": 3.707285119462043, + "tokens_seen": 919142400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005830290872617854, + "loss": 1.4258, + "theoretical_loss": 3.7072597674248886, + "tokens_seen": 919207936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005830130391173521, + "loss": 1.3826, + "theoretical_loss": 3.707234417701232, + "tokens_seen": 919273472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829969909729188, + "loss": 1.4574, + "theoretical_loss": 3.707209070290698, + "tokens_seen": 919339008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829809428284855, + "loss": 1.4397, + "theoretical_loss": 3.70718372519291, + "tokens_seen": 919404544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829648946840522, + "loss": 1.3371, + "theoretical_loss": 3.707158382407493, + "tokens_seen": 919470080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829488465396189, + "loss": 1.4102, + "theoretical_loss": 3.7071330419340702, + "tokens_seen": 919535616 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829327983951856, + "loss": 1.4334, + "theoretical_loss": 3.707107703772267, + "tokens_seen": 919601152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829167502507523, + "loss": 1.4841, + "theoretical_loss": 3.7070823679217066, + "tokens_seen": 919666688 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582900702106319, + "loss": 1.439, + "theoretical_loss": 3.7070570343820144, + "tokens_seen": 919732224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828846539618857, + "loss": 1.3562, + "theoretical_loss": 3.707031703152815, + "tokens_seen": 919797760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828686058174524, + "loss": 1.4295, + "theoretical_loss": 3.7070063742337322, + "tokens_seen": 919863296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828525576730191, + "loss": 1.3745, + "theoretical_loss": 3.706981047624392, + "tokens_seen": 919928832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828365095285858, + "loss": 1.3858, + "theoretical_loss": 3.706955723324418, + "tokens_seen": 919994368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828204613841525, + "loss": 1.3942, + "theoretical_loss": 3.706930401333436, + "tokens_seen": 920059904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828044132397192, + "loss": 1.3518, + "theoretical_loss": 3.7069050816510707, + "tokens_seen": 920125440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005827883650952859, + "loss": 1.4327, + "theoretical_loss": 3.7068797642769478, + "tokens_seen": 920190976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005827723169508526, + "loss": 1.4311, + "theoretical_loss": 3.7068544492106916, + "tokens_seen": 920256512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005827562688064193, + "loss": 1.4935, + "theoretical_loss": 3.706829136451928, + "tokens_seen": 920322048 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582740220661986, + "loss": 1.3996, + "theoretical_loss": 3.7068038260002814, + "tokens_seen": 920387584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005827241725175527, + "loss": 1.3055, + "theoretical_loss": 3.706778517855379, + "tokens_seen": 920453120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005827081243731194, + "loss": 1.3009, + "theoretical_loss": 3.706753212016845, + "tokens_seen": 920518656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826920762286861, + "loss": 1.3855, + "theoretical_loss": 3.7067279084843054, + "tokens_seen": 920584192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826760280842528, + "loss": 1.4394, + "theoretical_loss": 3.7067026072573857, + "tokens_seen": 920649728 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826599799398195, + "loss": 1.3881, + "theoretical_loss": 3.7066773083357125, + "tokens_seen": 920715264 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 311765, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3808051347732544, + "objective/train/theoretical_loss": 3.7066520117189103, + "objective/train/tokens_used": 941240800, + "theoretical_loss": 3.7066520117189103, + "tokens_seen": 920780800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826439317953862, + "loss": 1.3491, + "theoretical_loss": 3.7066520117189103, + "tokens_seen": 920780800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826278836509529, + "loss": 1.421, + "theoretical_loss": 3.706626717406607, + "tokens_seen": 920846336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826118355065196, + "loss": 1.2673, + "theoretical_loss": 3.7066014253984267, + "tokens_seen": 920911872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825957873620863, + "loss": 1.4163, + "theoretical_loss": 3.7065761356939966, + "tokens_seen": 920977408 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825797392176531, + "loss": 1.415, + "theoretical_loss": 3.7065508482929435, + "tokens_seen": 921042944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825636910732197, + "loss": 1.3242, + "theoretical_loss": 3.7065255631948926, + "tokens_seen": 921108480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825476429287865, + "loss": 1.3071, + "theoretical_loss": 3.706500280399471, + "tokens_seen": 921174016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825315947843531, + "loss": 1.3735, + "theoretical_loss": 3.7064749999063045, + "tokens_seen": 921239552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825155466399199, + "loss": 1.3342, + "theoretical_loss": 3.7064497217150207, + "tokens_seen": 921305088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824994984954865, + "loss": 1.3412, + "theoretical_loss": 3.7064244458252453, + "tokens_seen": 921370624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824834503510532, + "loss": 1.4254, + "theoretical_loss": 3.706399172236606, + "tokens_seen": 921436160 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824674022066199, + "loss": 1.4699, + "theoretical_loss": 3.706373900948729, + "tokens_seen": 921501696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824513540621866, + "loss": 1.3505, + "theoretical_loss": 3.7063486319612418, + "tokens_seen": 921567232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824353059177533, + "loss": 1.451, + "theoretical_loss": 3.7063233652737706, + "tokens_seen": 921632768 + }, + { + "epoch": 0.28, + "learning_rate": 0.00058241925777332, + "loss": 1.3264, + "theoretical_loss": 3.7062981008859435, + "tokens_seen": 921698304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824032096288867, + "loss": 1.3487, + "theoretical_loss": 3.7062728387973873, + "tokens_seen": 921763840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823871614844534, + "loss": 1.3578, + "theoretical_loss": 3.7062475790077283, + "tokens_seen": 921829376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823711133400201, + "loss": 1.3563, + "theoretical_loss": 3.706222321516596, + "tokens_seen": 921894912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823550651955868, + "loss": 1.4439, + "theoretical_loss": 3.7061970663236155, + "tokens_seen": 921960448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823390170511534, + "loss": 1.3973, + "theoretical_loss": 3.7061718134284165, + "tokens_seen": 922025984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823229689067202, + "loss": 1.2689, + "theoretical_loss": 3.706146562830625, + "tokens_seen": 922091520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823069207622868, + "loss": 1.3997, + "theoretical_loss": 3.70612131452987, + "tokens_seen": 922157056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005822908726178536, + "loss": 1.4652, + "theoretical_loss": 3.7060960685257784, + "tokens_seen": 922222592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005822748244734202, + "loss": 1.339, + "theoretical_loss": 3.7060708248179783, + "tokens_seen": 922288128 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582258776328987, + "loss": 1.4247, + "theoretical_loss": 3.7060455834060977, + "tokens_seen": 922353664 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005822427281845536, + "loss": 1.4533, + "theoretical_loss": 3.7060203442897652, + "tokens_seen": 922419200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005822266800401204, + "loss": 1.3602, + "theoretical_loss": 3.7059951074686084, + "tokens_seen": 922484736 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582210631895687, + "loss": 1.4916, + "theoretical_loss": 3.705969872942256, + "tokens_seen": 922550272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821945837512538, + "loss": 1.4019, + "theoretical_loss": 3.705944640710335, + "tokens_seen": 922615808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821785356068204, + "loss": 1.4527, + "theoretical_loss": 3.7059194107724758, + "tokens_seen": 922681344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821624874623872, + "loss": 1.3166, + "theoretical_loss": 3.7058941831283057, + "tokens_seen": 922746880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821464393179538, + "loss": 1.3701, + "theoretical_loss": 3.7058689577774535, + "tokens_seen": 922812416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821303911735206, + "loss": 1.4912, + "theoretical_loss": 3.705843734719548, + "tokens_seen": 922877952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821143430290872, + "loss": 1.3856, + "theoretical_loss": 3.7058185139542177, + "tokens_seen": 922943488 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582098294884654, + "loss": 1.3559, + "theoretical_loss": 3.705793295481092, + "tokens_seen": 923009024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005820822467402206, + "loss": 1.2921, + "theoretical_loss": 3.7057680792997996, + "tokens_seen": 923074560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005820661985957874, + "loss": 1.4317, + "theoretical_loss": 3.7057428654099693, + "tokens_seen": 923140096 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582050150451354, + "loss": 1.3484, + "theoretical_loss": 3.70571765381123, + "tokens_seen": 923205632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005820341023069208, + "loss": 1.4788, + "theoretical_loss": 3.705692444503211, + "tokens_seen": 923271168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005820180541624874, + "loss": 1.3357, + "theoretical_loss": 3.7056672374855424, + "tokens_seen": 923336704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005820020060180542, + "loss": 1.2888, + "theoretical_loss": 3.7056420327578534, + "tokens_seen": 923402240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005819859578736208, + "loss": 1.355, + "theoretical_loss": 3.705616830319773, + "tokens_seen": 923467776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005819699097291876, + "loss": 1.3329, + "theoretical_loss": 3.7055916301709297, + "tokens_seen": 923533312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005819538615847542, + "loss": 1.3257, + "theoretical_loss": 3.705566432310955, + "tokens_seen": 923598848 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581937813440321, + "loss": 1.3043, + "theoretical_loss": 3.705541236739478, + "tokens_seen": 923664384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005819217652958876, + "loss": 1.4197, + "theoretical_loss": 3.7055160434561283, + "tokens_seen": 923729920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005819057171514544, + "loss": 1.372, + "theoretical_loss": 3.7054908524605357, + "tokens_seen": 923795456 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581889669007021, + "loss": 1.3131, + "theoretical_loss": 3.70546566375233, + "tokens_seen": 923860992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005818736208625878, + "loss": 1.4444, + "theoretical_loss": 3.705440477331142, + "tokens_seen": 923926528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005818575727181544, + "loss": 1.3753, + "theoretical_loss": 3.7054152931966016, + "tokens_seen": 923992064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 312607, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3203315734863281, + "objective/train/theoretical_loss": 3.705390111348339, + "objective/train/tokens_used": 944517600, + "theoretical_loss": 3.705390111348339, + "tokens_seen": 924057600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005818415245737212, + "loss": 1.3776, + "theoretical_loss": 3.705390111348339, + "tokens_seen": 924057600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005818254764292878, + "loss": 1.4551, + "theoretical_loss": 3.705364931785984, + "tokens_seen": 924123136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005818094282848546, + "loss": 1.1539, + "theoretical_loss": 3.7053397545091675, + "tokens_seen": 924188672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005817933801404212, + "loss": 1.3143, + "theoretical_loss": 3.7053145795175197, + "tokens_seen": 924254208 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581777331995988, + "loss": 1.2291, + "theoretical_loss": 3.705289406810672, + "tokens_seen": 924319744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005817612838515546, + "loss": 1.372, + "theoretical_loss": 3.7052642363882535, + "tokens_seen": 924385280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005817452357071214, + "loss": 1.2999, + "theoretical_loss": 3.7052390682498966, + "tokens_seen": 924450816 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581729187562688, + "loss": 1.386, + "theoretical_loss": 3.705213902395231, + "tokens_seen": 924516352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005817131394182548, + "loss": 1.441, + "theoretical_loss": 3.705188738823889, + "tokens_seen": 924581888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816970912738215, + "loss": 1.3598, + "theoretical_loss": 3.7051635775354996, + "tokens_seen": 924647424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816810431293882, + "loss": 1.2233, + "theoretical_loss": 3.705138418529695, + "tokens_seen": 924712960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816649949849549, + "loss": 1.2805, + "theoretical_loss": 3.7051132618061073, + "tokens_seen": 924778496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816489468405216, + "loss": 1.4946, + "theoretical_loss": 3.705088107364366, + "tokens_seen": 924844032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816328986960883, + "loss": 1.3883, + "theoretical_loss": 3.7050629552041032, + "tokens_seen": 924909568 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581616850551655, + "loss": 1.3847, + "theoretical_loss": 3.7050378053249506, + "tokens_seen": 924975104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816008024072217, + "loss": 1.2696, + "theoretical_loss": 3.70501265772654, + "tokens_seen": 925040640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815847542627884, + "loss": 1.3819, + "theoretical_loss": 3.7049875124085014, + "tokens_seen": 925106176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815687061183551, + "loss": 1.4409, + "theoretical_loss": 3.7049623693704685, + "tokens_seen": 925171712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815526579739218, + "loss": 1.4033, + "theoretical_loss": 3.7049372286120716, + "tokens_seen": 925237248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815366098294885, + "loss": 1.3403, + "theoretical_loss": 3.704912090132944, + "tokens_seen": 925302784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815205616850552, + "loss": 1.3072, + "theoretical_loss": 3.704886953932716, + "tokens_seen": 925368320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815045135406219, + "loss": 1.3655, + "theoretical_loss": 3.704861820011021, + "tokens_seen": 925433856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005814884653961886, + "loss": 1.3922, + "theoretical_loss": 3.70483668836749, + "tokens_seen": 925499392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005814724172517553, + "loss": 1.3912, + "theoretical_loss": 3.7048115590017563, + "tokens_seen": 925564928 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581456369107322, + "loss": 1.397, + "theoretical_loss": 3.704786431913451, + "tokens_seen": 925630464 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005814403209628887, + "loss": 1.3192, + "theoretical_loss": 3.7047613071022076, + "tokens_seen": 925696000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005814242728184554, + "loss": 1.4069, + "theoretical_loss": 3.7047361845676585, + "tokens_seen": 925761536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005814082246740221, + "loss": 1.4717, + "theoretical_loss": 3.704711064309435, + "tokens_seen": 925827072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005813921765295888, + "loss": 1.3609, + "theoretical_loss": 3.704685946327171, + "tokens_seen": 925892608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005813761283851556, + "loss": 1.3113, + "theoretical_loss": 3.704660830620499, + "tokens_seen": 925958144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005813600802407222, + "loss": 1.4137, + "theoretical_loss": 3.704635717189051, + "tokens_seen": 926023680 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581344032096289, + "loss": 1.3567, + "theoretical_loss": 3.7046106060324613, + "tokens_seen": 926089216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005813279839518556, + "loss": 1.4285, + "theoretical_loss": 3.7045854971503616, + "tokens_seen": 926154752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005813119358074224, + "loss": 1.3764, + "theoretical_loss": 3.704560390542385, + "tokens_seen": 926220288 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581295887662989, + "loss": 1.2782, + "theoretical_loss": 3.7045352862081655, + "tokens_seen": 926285824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005812798395185558, + "loss": 1.3771, + "theoretical_loss": 3.7045101841473365, + "tokens_seen": 926351360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005812637913741224, + "loss": 1.293, + "theoretical_loss": 3.70448508435953, + "tokens_seen": 926416896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005812477432296892, + "loss": 1.4121, + "theoretical_loss": 3.7044599868443804, + "tokens_seen": 926482432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005812316950852558, + "loss": 1.4473, + "theoretical_loss": 3.7044348916015206, + "tokens_seen": 926547968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005812156469408226, + "loss": 1.2784, + "theoretical_loss": 3.704409798630585, + "tokens_seen": 926613504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005811995987963892, + "loss": 1.5596, + "theoretical_loss": 3.7043847079312058, + "tokens_seen": 926679040 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581183550651956, + "loss": 1.51, + "theoretical_loss": 3.704359619503018, + "tokens_seen": 926744576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005811675025075226, + "loss": 1.4248, + "theoretical_loss": 3.704334533345656, + "tokens_seen": 926810112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005811514543630894, + "loss": 1.3905, + "theoretical_loss": 3.704309449458752, + "tokens_seen": 926875648 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581135406218656, + "loss": 1.3438, + "theoretical_loss": 3.7042843678419404, + "tokens_seen": 926941184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005811193580742228, + "loss": 1.406, + "theoretical_loss": 3.7042592884948564, + "tokens_seen": 927006720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005811033099297894, + "loss": 1.3171, + "theoretical_loss": 3.704234211417133, + "tokens_seen": 927072256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810872617853561, + "loss": 1.3694, + "theoretical_loss": 3.704209136608405, + "tokens_seen": 927137792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810712136409228, + "loss": 1.317, + "theoretical_loss": 3.704184064068307, + "tokens_seen": 927203328 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810551654964895, + "loss": 1.3885, + "theoretical_loss": 3.704158993796473, + "tokens_seen": 927268864 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 313936, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.348827600479126, + "objective/train/theoretical_loss": 3.7041339257925374, + "objective/train/tokens_used": 947794400, + "theoretical_loss": 3.7041339257925374, + "tokens_seen": 927334400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810391173520562, + "loss": 1.4109, + "theoretical_loss": 3.7041339257925374, + "tokens_seen": 927334400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810230692076229, + "loss": 1.3908, + "theoretical_loss": 3.7041088600561345, + "tokens_seen": 927399936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810070210631896, + "loss": 1.3938, + "theoretical_loss": 3.7040837965869002, + "tokens_seen": 927465472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005809909729187563, + "loss": 1.3591, + "theoretical_loss": 3.704058735384468, + "tokens_seen": 927531008 + }, + { + "epoch": 0.28, + "learning_rate": 0.000580974924774323, + "loss": 1.3256, + "theoretical_loss": 3.704033676448473, + "tokens_seen": 927596544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005809588766298897, + "loss": 1.4747, + "theoretical_loss": 3.7040086197785502, + "tokens_seen": 927662080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005809428284854563, + "loss": 1.3971, + "theoretical_loss": 3.7039835653743354, + "tokens_seen": 927727616 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005809267803410231, + "loss": 1.3978, + "theoretical_loss": 3.703958513235463, + "tokens_seen": 927793152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005809107321965897, + "loss": 1.405, + "theoretical_loss": 3.703933463361568, + "tokens_seen": 927858688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808946840521565, + "loss": 1.5449, + "theoretical_loss": 3.7039084157522857, + "tokens_seen": 927924224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808786359077231, + "loss": 1.4407, + "theoretical_loss": 3.703883370407252, + "tokens_seen": 927989760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808625877632899, + "loss": 1.2564, + "theoretical_loss": 3.7038583273261017, + "tokens_seen": 928055296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808465396188565, + "loss": 1.3291, + "theoretical_loss": 3.7038332865084707, + "tokens_seen": 928120832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808304914744233, + "loss": 1.3966, + "theoretical_loss": 3.703808247953994, + "tokens_seen": 928186368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808144433299899, + "loss": 1.452, + "theoretical_loss": 3.7037832116623086, + "tokens_seen": 928251904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807983951855567, + "loss": 1.4109, + "theoretical_loss": 3.7037581776330493, + "tokens_seen": 928317440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807823470411233, + "loss": 1.2942, + "theoretical_loss": 3.7037331458658516, + "tokens_seen": 928382976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807662988966901, + "loss": 1.3621, + "theoretical_loss": 3.7037081163603522, + "tokens_seen": 928448512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807502507522567, + "loss": 1.4988, + "theoretical_loss": 3.703683089116187, + "tokens_seen": 928514048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807342026078235, + "loss": 1.5123, + "theoretical_loss": 3.7036580641329917, + "tokens_seen": 928579584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807181544633901, + "loss": 1.3268, + "theoretical_loss": 3.703633041410403, + "tokens_seen": 928645120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807021063189569, + "loss": 1.3235, + "theoretical_loss": 3.7036080209480566, + "tokens_seen": 928710656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806860581745235, + "loss": 1.361, + "theoretical_loss": 3.7035830027455896, + "tokens_seen": 928776192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806700100300903, + "loss": 1.3305, + "theoretical_loss": 3.7035579868026383, + "tokens_seen": 928841728 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806539618856569, + "loss": 1.3372, + "theoretical_loss": 3.703532973118839, + "tokens_seen": 928907264 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806379137412237, + "loss": 1.3979, + "theoretical_loss": 3.703507961693828, + "tokens_seen": 928972800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806218655967903, + "loss": 1.3843, + "theoretical_loss": 3.703482952527242, + "tokens_seen": 929038336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806058174523571, + "loss": 1.3754, + "theoretical_loss": 3.7034579456187187, + "tokens_seen": 929103872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805897693079237, + "loss": 1.3553, + "theoretical_loss": 3.7034329409678937, + "tokens_seen": 929169408 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805737211634905, + "loss": 1.4483, + "theoretical_loss": 3.703407938574405, + "tokens_seen": 929234944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805576730190571, + "loss": 1.3585, + "theoretical_loss": 3.703382938437889, + "tokens_seen": 929300480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805416248746239, + "loss": 1.4125, + "theoretical_loss": 3.703357940557983, + "tokens_seen": 929366016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805255767301905, + "loss": 1.3327, + "theoretical_loss": 3.7033329449343246, + "tokens_seen": 929431552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805095285857573, + "loss": 1.3395, + "theoretical_loss": 3.7033079515665506, + "tokens_seen": 929497088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804934804413239, + "loss": 1.5735, + "theoretical_loss": 3.703282960454299, + "tokens_seen": 929562624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804774322968907, + "loss": 1.3129, + "theoretical_loss": 3.703257971597206, + "tokens_seen": 929628160 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804613841524573, + "loss": 1.3756, + "theoretical_loss": 3.70323298499491, + "tokens_seen": 929693696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804453360080241, + "loss": 1.3092, + "theoretical_loss": 3.7032080006470487, + "tokens_seen": 929759232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804292878635907, + "loss": 1.3523, + "theoretical_loss": 3.7031830185532595, + "tokens_seen": 929824768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804132397191575, + "loss": 1.3675, + "theoretical_loss": 3.7031580387131804, + "tokens_seen": 929890304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803971915747241, + "loss": 1.3851, + "theoretical_loss": 3.7031330611264495, + "tokens_seen": 929955840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803811434302909, + "loss": 1.2445, + "theoretical_loss": 3.703108085792704, + "tokens_seen": 930021376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803650952858576, + "loss": 1.3681, + "theoretical_loss": 3.7030831127115826, + "tokens_seen": 930086912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803490471414243, + "loss": 1.3383, + "theoretical_loss": 3.7030581418827233, + "tokens_seen": 930152448 + }, + { + "epoch": 0.28, + "learning_rate": 0.000580332998996991, + "loss": 1.3631, + "theoretical_loss": 3.703033173305764, + "tokens_seen": 930217984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803169508525577, + "loss": 1.3712, + "theoretical_loss": 3.703008206980343, + "tokens_seen": 930283520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803009027081244, + "loss": 1.328, + "theoretical_loss": 3.7029832429060994, + "tokens_seen": 930349056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802848545636911, + "loss": 1.3491, + "theoretical_loss": 3.7029582810826707, + "tokens_seen": 930414592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802688064192578, + "loss": 1.4885, + "theoretical_loss": 3.702933321509696, + "tokens_seen": 930480128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802527582748245, + "loss": 1.3232, + "theoretical_loss": 3.702908364186814, + "tokens_seen": 930545664 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 314827, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3260948657989502, + "objective/train/theoretical_loss": 3.702883409113663, + "objective/train/tokens_used": 951071200, + "theoretical_loss": 3.702883409113663, + "tokens_seen": 930611200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802367101303912, + "loss": 1.3089, + "theoretical_loss": 3.702883409113663, + "tokens_seen": 930611200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802206619859579, + "loss": 1.3488, + "theoretical_loss": 3.7028584562898823, + "tokens_seen": 930676736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802046138415246, + "loss": 1.4493, + "theoretical_loss": 3.7028335057151103, + "tokens_seen": 930742272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801885656970913, + "loss": 1.4082, + "theoretical_loss": 3.7028085573889857, + "tokens_seen": 930807808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801725175526581, + "loss": 1.4522, + "theoretical_loss": 3.7027836113111485, + "tokens_seen": 930873344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801564694082247, + "loss": 1.4337, + "theoretical_loss": 3.7027586674812376, + "tokens_seen": 930938880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801404212637915, + "loss": 1.3837, + "theoretical_loss": 3.7027337258988915, + "tokens_seen": 931004416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801243731193581, + "loss": 1.3573, + "theoretical_loss": 3.7027087865637496, + "tokens_seen": 931069952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801083249749249, + "loss": 1.3928, + "theoretical_loss": 3.702683849475452, + "tokens_seen": 931135488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800922768304915, + "loss": 1.49, + "theoretical_loss": 3.7026589146336377, + "tokens_seen": 931201024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800762286860583, + "loss": 1.4633, + "theoretical_loss": 3.702633982037946, + "tokens_seen": 931266560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800601805416249, + "loss": 1.417, + "theoretical_loss": 3.7026090516880172, + "tokens_seen": 931332096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800441323971917, + "loss": 1.3736, + "theoretical_loss": 3.7025841235834904, + "tokens_seen": 931397632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800280842527583, + "loss": 1.4305, + "theoretical_loss": 3.702559197724006, + "tokens_seen": 931463168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800120361083251, + "loss": 1.3007, + "theoretical_loss": 3.702534274109203, + "tokens_seen": 931528704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799959879638917, + "loss": 1.2282, + "theoretical_loss": 3.702509352738722, + "tokens_seen": 931594240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799799398194585, + "loss": 1.3787, + "theoretical_loss": 3.7024844336122027, + "tokens_seen": 931659776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799638916750251, + "loss": 1.4832, + "theoretical_loss": 3.7024595167292853, + "tokens_seen": 931725312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799478435305919, + "loss": 1.3689, + "theoretical_loss": 3.7024346020896104, + "tokens_seen": 931790848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799317953861585, + "loss": 1.3569, + "theoretical_loss": 3.7024096896928183, + "tokens_seen": 931856384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799157472417253, + "loss": 1.4301, + "theoretical_loss": 3.7023847795385487, + "tokens_seen": 931921920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798996990972919, + "loss": 1.3889, + "theoretical_loss": 3.7023598716264425, + "tokens_seen": 931987456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798836509528587, + "loss": 1.3287, + "theoretical_loss": 3.7023349659561395, + "tokens_seen": 932052992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798676028084253, + "loss": 1.409, + "theoretical_loss": 3.702310062527282, + "tokens_seen": 932118528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798515546639921, + "loss": 1.4233, + "theoretical_loss": 3.7022851613395096, + "tokens_seen": 932184064 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798355065195587, + "loss": 1.3944, + "theoretical_loss": 3.7022602623924623, + "tokens_seen": 932249600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798194583751255, + "loss": 1.3632, + "theoretical_loss": 3.702235365685783, + "tokens_seen": 932315136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798034102306921, + "loss": 1.3414, + "theoretical_loss": 3.702210471219111, + "tokens_seen": 932380672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797873620862589, + "loss": 1.4716, + "theoretical_loss": 3.7021855789920872, + "tokens_seen": 932446208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797713139418255, + "loss": 1.4241, + "theoretical_loss": 3.702160689004354, + "tokens_seen": 932511744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797552657973923, + "loss": 1.4795, + "theoretical_loss": 3.702135801255552, + "tokens_seen": 932577280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797392176529589, + "loss": 1.3867, + "theoretical_loss": 3.7021109157453225, + "tokens_seen": 932642816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797231695085257, + "loss": 1.2557, + "theoretical_loss": 3.7020860324733063, + "tokens_seen": 932708352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797071213640923, + "loss": 1.2278, + "theoretical_loss": 3.7020611514391457, + "tokens_seen": 932773888 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579691073219659, + "loss": 1.4045, + "theoretical_loss": 3.7020362726424816, + "tokens_seen": 932839424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005796750250752257, + "loss": 1.3367, + "theoretical_loss": 3.7020113960829564, + "tokens_seen": 932904960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005796589769307924, + "loss": 1.4045, + "theoretical_loss": 3.701986521760211, + "tokens_seen": 932970496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005796429287863591, + "loss": 1.2608, + "theoretical_loss": 3.701961649673888, + "tokens_seen": 933036032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005796268806419258, + "loss": 1.3224, + "theoretical_loss": 3.701936779823628, + "tokens_seen": 933101568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005796108324974925, + "loss": 1.4404, + "theoretical_loss": 3.7019119122090736, + "tokens_seen": 933167104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005795947843530592, + "loss": 1.4872, + "theoretical_loss": 3.7018870468298672, + "tokens_seen": 933232640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005795787362086259, + "loss": 1.3217, + "theoretical_loss": 3.701862183685651, + "tokens_seen": 933298176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005795626880641926, + "loss": 1.2918, + "theoretical_loss": 3.701837322776066, + "tokens_seen": 933363712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005795466399197592, + "loss": 1.3156, + "theoretical_loss": 3.7018124641007564, + "tokens_seen": 933429248 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579530591775326, + "loss": 1.3608, + "theoretical_loss": 3.7017876076593623, + "tokens_seen": 933494784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005795145436308926, + "loss": 1.3975, + "theoretical_loss": 3.701762753451528, + "tokens_seen": 933560320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794984954864594, + "loss": 1.3239, + "theoretical_loss": 3.701737901476895, + "tokens_seen": 933625856 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579482447342026, + "loss": 1.3837, + "theoretical_loss": 3.7017130517351067, + "tokens_seen": 933691392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794663991975928, + "loss": 1.2296, + "theoretical_loss": 3.701688204225804, + "tokens_seen": 933756928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794503510531594, + "loss": 1.4696, + "theoretical_loss": 3.7016633589486325, + "tokens_seen": 933822464 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 316104, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.325061559677124, + "objective/train/theoretical_loss": 3.7016385159032326, + "objective/train/tokens_used": 954348000, + "theoretical_loss": 3.7016385159032326, + "tokens_seen": 933888000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794343029087262, + "loss": 1.3622, + "theoretical_loss": 3.7016385159032326, + "tokens_seen": 933888000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794182547642928, + "loss": 1.3294, + "theoretical_loss": 3.7016136750892485, + "tokens_seen": 933953536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794022066198596, + "loss": 1.3868, + "theoretical_loss": 3.701588836506323, + "tokens_seen": 934019072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005793861584754262, + "loss": 1.3594, + "theoretical_loss": 3.7015640001540984, + "tokens_seen": 934084608 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579370110330993, + "loss": 1.3129, + "theoretical_loss": 3.701539166032219, + "tokens_seen": 934150144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005793540621865596, + "loss": 1.4787, + "theoretical_loss": 3.7015143341403274, + "tokens_seen": 934215680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005793380140421264, + "loss": 1.3054, + "theoretical_loss": 3.7014895044780673, + "tokens_seen": 934281216 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579321965897693, + "loss": 1.5103, + "theoretical_loss": 3.701464677045082, + "tokens_seen": 934346752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005793059177532598, + "loss": 1.2755, + "theoretical_loss": 3.7014398518410148, + "tokens_seen": 934412288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005792898696088264, + "loss": 1.3949, + "theoretical_loss": 3.7014150288655094, + "tokens_seen": 934477824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005792738214643932, + "loss": 1.3603, + "theoretical_loss": 3.7013902081182097, + "tokens_seen": 934543360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005792577733199598, + "loss": 1.2299, + "theoretical_loss": 3.7013653895987595, + "tokens_seen": 934608896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005792417251755266, + "loss": 1.3886, + "theoretical_loss": 3.701340573306802, + "tokens_seen": 934674432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005792256770310932, + "loss": 1.3569, + "theoretical_loss": 3.7013157592419814, + "tokens_seen": 934739968 + }, + { + "epoch": 0.28, + "learning_rate": 0.00057920962888666, + "loss": 1.2818, + "theoretical_loss": 3.701290947403942, + "tokens_seen": 934805504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005791935807422266, + "loss": 1.2883, + "theoretical_loss": 3.701266137792328, + "tokens_seen": 934871040 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005791775325977934, + "loss": 1.3721, + "theoretical_loss": 3.7012413304067833, + "tokens_seen": 934936576 + }, + { + "epoch": 0.28, + "learning_rate": 0.00057916148445336, + "loss": 1.4051, + "theoretical_loss": 3.701216525246952, + "tokens_seen": 935002112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005791454363089268, + "loss": 1.4096, + "theoretical_loss": 3.701191722312478, + "tokens_seen": 935067648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005791293881644934, + "loss": 1.4638, + "theoretical_loss": 3.701166921603007, + "tokens_seen": 935133184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005791133400200602, + "loss": 1.3949, + "theoretical_loss": 3.7011421231181822, + "tokens_seen": 935198720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005790972918756268, + "loss": 1.3449, + "theoretical_loss": 3.7011173268576494, + "tokens_seen": 935264256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005790812437311936, + "loss": 1.3907, + "theoretical_loss": 3.701092532821052, + "tokens_seen": 935329792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005790651955867602, + "loss": 1.409, + "theoretical_loss": 3.7010677410080355, + "tokens_seen": 935395328 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579049147442327, + "loss": 1.4154, + "theoretical_loss": 3.7010429514182444, + "tokens_seen": 935460864 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005790330992978936, + "loss": 1.2992, + "theoretical_loss": 3.701018164051324, + "tokens_seen": 935526400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005790170511534604, + "loss": 1.4031, + "theoretical_loss": 3.7009933789069187, + "tokens_seen": 935591936 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579001003009027, + "loss": 1.4881, + "theoretical_loss": 3.7009685959846736, + "tokens_seen": 935657472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005789849548645938, + "loss": 1.3425, + "theoretical_loss": 3.7009438152842344, + "tokens_seen": 935723008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005789689067201606, + "loss": 1.3699, + "theoretical_loss": 3.7009190368052463, + "tokens_seen": 935788544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005789528585757272, + "loss": 1.3374, + "theoretical_loss": 3.7008942605473543, + "tokens_seen": 935854080 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578936810431294, + "loss": 1.4309, + "theoretical_loss": 3.7008694865102036, + "tokens_seen": 935919616 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005789207622868606, + "loss": 1.4752, + "theoretical_loss": 3.7008447146934396, + "tokens_seen": 935985152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005789047141424274, + "loss": 1.3088, + "theoretical_loss": 3.7008199450967085, + "tokens_seen": 936050688 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578888665997994, + "loss": 1.3702, + "theoretical_loss": 3.7007951777196553, + "tokens_seen": 936116224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005788726178535608, + "loss": 1.3588, + "theoretical_loss": 3.7007704125619263, + "tokens_seen": 936181760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005788565697091274, + "loss": 1.354, + "theoretical_loss": 3.7007456496231668, + "tokens_seen": 936247296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005788405215646942, + "loss": 1.3702, + "theoretical_loss": 3.7007208889030228, + "tokens_seen": 936312832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005788244734202608, + "loss": 1.4917, + "theoretical_loss": 3.7006961304011403, + "tokens_seen": 936378368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005788084252758276, + "loss": 1.4233, + "theoretical_loss": 3.7006713741171655, + "tokens_seen": 936443904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005787923771313942, + "loss": 1.5235, + "theoretical_loss": 3.700646620050744, + "tokens_seen": 936509440 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578776328986961, + "loss": 1.3346, + "theoretical_loss": 3.700621868201523, + "tokens_seen": 936574976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005787602808425276, + "loss": 1.3589, + "theoretical_loss": 3.7005971185691475, + "tokens_seen": 936640512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005787442326980944, + "loss": 1.3806, + "theoretical_loss": 3.700572371153265, + "tokens_seen": 936706048 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578728184553661, + "loss": 1.3391, + "theoretical_loss": 3.7005476259535213, + "tokens_seen": 936771584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005787121364092278, + "loss": 1.3121, + "theoretical_loss": 3.700522882969563, + "tokens_seen": 936837120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005786960882647944, + "loss": 1.3904, + "theoretical_loss": 3.7004981422010372, + "tokens_seen": 936902656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005786800401203612, + "loss": 1.3613, + "theoretical_loss": 3.7004734036475897, + "tokens_seen": 936968192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005786639919759278, + "loss": 1.3834, + "theoretical_loss": 3.7004486673088675, + "tokens_seen": 937033728 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005786479438314946, + "loss": 1.381, + "theoretical_loss": 3.7004239331845183, + "tokens_seen": 937099264 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 317140, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3840632438659668, + "objective/train/theoretical_loss": 3.700399201274188, + "objective/train/tokens_used": 957624800, + "theoretical_loss": 3.700399201274188, + "tokens_seen": 937164800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005786318956870612, + "loss": 1.3405, + "theoretical_loss": 3.700399201274188, + "tokens_seen": 937164800 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578615847542628, + "loss": 1.3924, + "theoretical_loss": 3.700374471577524, + "tokens_seen": 937230336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785997993981946, + "loss": 1.431, + "theoretical_loss": 3.700349744094174, + "tokens_seen": 937295872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785837512537614, + "loss": 1.3422, + "theoretical_loss": 3.7003250188237837, + "tokens_seen": 937361408 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578567703109328, + "loss": 1.3686, + "theoretical_loss": 3.700300295766002, + "tokens_seen": 937426944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785516549648948, + "loss": 1.3968, + "theoretical_loss": 3.700275574920475, + "tokens_seen": 937492480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785356068204614, + "loss": 1.3153, + "theoretical_loss": 3.700250856286851, + "tokens_seen": 937558016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785195586760282, + "loss": 1.4538, + "theoretical_loss": 3.7002261398647764, + "tokens_seen": 937623552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785035105315948, + "loss": 1.4646, + "theoretical_loss": 3.7002014256539004, + "tokens_seen": 937689088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005784874623871616, + "loss": 1.4331, + "theoretical_loss": 3.7001767136538692, + "tokens_seen": 937754624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005784714142427282, + "loss": 1.3257, + "theoretical_loss": 3.700152003864331, + "tokens_seen": 937820160 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578455366098295, + "loss": 1.3505, + "theoretical_loss": 3.700127296284934, + "tokens_seen": 937885696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005784393179538616, + "loss": 1.3649, + "theoretical_loss": 3.700102590915326, + "tokens_seen": 937951232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005784232698094284, + "loss": 1.4132, + "theoretical_loss": 3.700077887755154, + "tokens_seen": 938016768 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578407221664995, + "loss": 1.3479, + "theoretical_loss": 3.7000531868040674, + "tokens_seen": 938082304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783911735205618, + "loss": 1.4686, + "theoretical_loss": 3.7000284880617134, + "tokens_seen": 938147840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783751253761284, + "loss": 1.4947, + "theoretical_loss": 3.700003791527741, + "tokens_seen": 938213376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783590772316952, + "loss": 1.354, + "theoretical_loss": 3.699979097201798, + "tokens_seen": 938278912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783430290872618, + "loss": 1.329, + "theoretical_loss": 3.699954405083533, + "tokens_seen": 938344448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783269809428286, + "loss": 1.3881, + "theoretical_loss": 3.699929715172594, + "tokens_seen": 938409984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783109327983952, + "loss": 1.4842, + "theoretical_loss": 3.69990502746863, + "tokens_seen": 938475520 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578294884653962, + "loss": 1.2677, + "theoretical_loss": 3.69988034197129, + "tokens_seen": 938541056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005782788365095286, + "loss": 1.3845, + "theoretical_loss": 3.6998556586802214, + "tokens_seen": 938606592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005782627883650953, + "loss": 1.2799, + "theoretical_loss": 3.6998309775950746, + "tokens_seen": 938672128 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578246740220662, + "loss": 1.4618, + "theoretical_loss": 3.699806298715497, + "tokens_seen": 938737664 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005782306920762287, + "loss": 1.4908, + "theoretical_loss": 3.6997816220411384, + "tokens_seen": 938803200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005782146439317954, + "loss": 1.2352, + "theoretical_loss": 3.6997569475716476, + "tokens_seen": 938868736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781985957873621, + "loss": 1.3963, + "theoretical_loss": 3.6997322753066744, + "tokens_seen": 938934272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781825476429288, + "loss": 1.3798, + "theoretical_loss": 3.699707605245867, + "tokens_seen": 938999808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781664994984955, + "loss": 1.3078, + "theoretical_loss": 3.699682937388875, + "tokens_seen": 939065344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781504513540621, + "loss": 1.4004, + "theoretical_loss": 3.6996582717353474, + "tokens_seen": 939130880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781344032096289, + "loss": 1.4052, + "theoretical_loss": 3.699633608284935, + "tokens_seen": 939196416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781183550651955, + "loss": 1.4474, + "theoretical_loss": 3.699608947037285, + "tokens_seen": 939261952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781023069207623, + "loss": 1.4475, + "theoretical_loss": 3.699584287992049, + "tokens_seen": 939327488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780862587763289, + "loss": 1.3526, + "theoretical_loss": 3.699559631148876, + "tokens_seen": 939393024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780702106318957, + "loss": 1.3568, + "theoretical_loss": 3.6995349765074157, + "tokens_seen": 939458560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780541624874623, + "loss": 1.3594, + "theoretical_loss": 3.699510324067318, + "tokens_seen": 939524096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780381143430291, + "loss": 1.2978, + "theoretical_loss": 3.6994856738282325, + "tokens_seen": 939589632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780220661985957, + "loss": 1.4097, + "theoretical_loss": 3.6994610257898097, + "tokens_seen": 939655168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780060180541625, + "loss": 1.3657, + "theoretical_loss": 3.6994363799516994, + "tokens_seen": 939720704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779899699097291, + "loss": 1.5101, + "theoretical_loss": 3.699411736313552, + "tokens_seen": 939786240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779739217652959, + "loss": 1.488, + "theoretical_loss": 3.699387094875017, + "tokens_seen": 939851776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779578736208625, + "loss": 1.266, + "theoretical_loss": 3.6993624556357454, + "tokens_seen": 939917312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779418254764293, + "loss": 1.2609, + "theoretical_loss": 3.6993378185953874, + "tokens_seen": 939982848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779257773319959, + "loss": 1.3096, + "theoretical_loss": 3.6993131837535937, + "tokens_seen": 940048384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779097291875627, + "loss": 1.4811, + "theoretical_loss": 3.6992885511100146, + "tokens_seen": 940113920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005778936810431293, + "loss": 1.3487, + "theoretical_loss": 3.6992639206643005, + "tokens_seen": 940179456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005778776328986961, + "loss": 1.4474, + "theoretical_loss": 3.6992392924161024, + "tokens_seen": 940244992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005778615847542627, + "loss": 1.4966, + "theoretical_loss": 3.699214666365071, + "tokens_seen": 940310528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005778455366098295, + "loss": 1.381, + "theoretical_loss": 3.6991900425108577, + "tokens_seen": 940376064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 318390, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4625651836395264, + "objective/train/theoretical_loss": 3.6991654208531126, + "objective/train/tokens_used": 960901600, + "theoretical_loss": 3.6991654208531126, + "tokens_seen": 940441600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005778294884653961, + "loss": 1.3777, + "theoretical_loss": 3.6991654208531126, + "tokens_seen": 940441600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005778134403209629, + "loss": 1.3918, + "theoretical_loss": 3.6991408013914873, + "tokens_seen": 940507136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777973921765295, + "loss": 1.4778, + "theoretical_loss": 3.6991161841256326, + "tokens_seen": 940572672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777813440320963, + "loss": 1.3639, + "theoretical_loss": 3.6990915690551995, + "tokens_seen": 940638208 + }, + { + "epoch": 0.29, + "learning_rate": 0.000577765295887663, + "loss": 1.3475, + "theoretical_loss": 3.6990669561798404, + "tokens_seen": 940703744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777492477432297, + "loss": 1.3163, + "theoretical_loss": 3.699042345499205, + "tokens_seen": 940769280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777331995987965, + "loss": 1.3902, + "theoretical_loss": 3.6990177370129462, + "tokens_seen": 940834816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777171514543631, + "loss": 1.4047, + "theoretical_loss": 3.698993130720714, + "tokens_seen": 940900352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777011033099299, + "loss": 1.2753, + "theoretical_loss": 3.6989685266221617, + "tokens_seen": 940965888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776850551654965, + "loss": 1.4524, + "theoretical_loss": 3.69894392471694, + "tokens_seen": 941031424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776690070210633, + "loss": 1.3627, + "theoretical_loss": 3.6989193250047006, + "tokens_seen": 941096960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776529588766299, + "loss": 1.3037, + "theoretical_loss": 3.6988947274850954, + "tokens_seen": 941162496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776369107321967, + "loss": 1.4261, + "theoretical_loss": 3.6988701321577766, + "tokens_seen": 941228032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776208625877633, + "loss": 1.298, + "theoretical_loss": 3.698845539022396, + "tokens_seen": 941293568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776048144433301, + "loss": 1.5898, + "theoretical_loss": 3.6988209480786054, + "tokens_seen": 941359104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775887662988967, + "loss": 1.4488, + "theoretical_loss": 3.6987963593260567, + "tokens_seen": 941424640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775727181544635, + "loss": 1.4654, + "theoretical_loss": 3.6987717727644034, + "tokens_seen": 941490176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775566700100301, + "loss": 1.4613, + "theoretical_loss": 3.698747188393297, + "tokens_seen": 941555712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775406218655969, + "loss": 1.284, + "theoretical_loss": 3.6987226062123897, + "tokens_seen": 941621248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775245737211635, + "loss": 1.4079, + "theoretical_loss": 3.698698026221334, + "tokens_seen": 941686784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775085255767303, + "loss": 1.4136, + "theoretical_loss": 3.698673448419783, + "tokens_seen": 941752320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774924774322969, + "loss": 1.504, + "theoretical_loss": 3.6986488728073885, + "tokens_seen": 941817856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774764292878637, + "loss": 1.3827, + "theoretical_loss": 3.6986242993838037, + "tokens_seen": 941883392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774603811434303, + "loss": 1.4615, + "theoretical_loss": 3.6985997281486815, + "tokens_seen": 941948928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774443329989971, + "loss": 1.3283, + "theoretical_loss": 3.6985751591016744, + "tokens_seen": 942014464 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774282848545637, + "loss": 1.2975, + "theoretical_loss": 3.6985505922424355, + "tokens_seen": 942080000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774122367101305, + "loss": 1.4891, + "theoretical_loss": 3.698526027570618, + "tokens_seen": 942145536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773961885656971, + "loss": 1.311, + "theoretical_loss": 3.698501465085875, + "tokens_seen": 942211072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773801404212639, + "loss": 1.4593, + "theoretical_loss": 3.6984769047878587, + "tokens_seen": 942276608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773640922768305, + "loss": 1.4576, + "theoretical_loss": 3.6984523466762234, + "tokens_seen": 942342144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773480441323973, + "loss": 1.3528, + "theoretical_loss": 3.698427790750623, + "tokens_seen": 942407680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773319959879639, + "loss": 1.3787, + "theoretical_loss": 3.698403237010709, + "tokens_seen": 942473216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773159478435307, + "loss": 1.406, + "theoretical_loss": 3.6983786854561362, + "tokens_seen": 942538752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772998996990973, + "loss": 1.3402, + "theoretical_loss": 3.6983541360865586, + "tokens_seen": 942604288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772838515546641, + "loss": 1.265, + "theoretical_loss": 3.6983295889016286, + "tokens_seen": 942669824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772678034102307, + "loss": 1.3356, + "theoretical_loss": 3.6983050439010006, + "tokens_seen": 942735360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772517552657975, + "loss": 1.2517, + "theoretical_loss": 3.6982805010843283, + "tokens_seen": 942800896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772357071213641, + "loss": 1.3905, + "theoretical_loss": 3.6982559604512657, + "tokens_seen": 942866432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772196589769309, + "loss": 1.3475, + "theoretical_loss": 3.6982314220014665, + "tokens_seen": 942931968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772036108324975, + "loss": 1.3674, + "theoretical_loss": 3.698206885734585, + "tokens_seen": 942997504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771875626880643, + "loss": 1.3298, + "theoretical_loss": 3.6981823516502756, + "tokens_seen": 943063040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771715145436309, + "loss": 1.3344, + "theoretical_loss": 3.698157819748192, + "tokens_seen": 943128576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771554663991977, + "loss": 1.3038, + "theoretical_loss": 3.6981332900279886, + "tokens_seen": 943194112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771394182547643, + "loss": 1.4594, + "theoretical_loss": 3.698108762489319, + "tokens_seen": 943259648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771233701103311, + "loss": 1.537, + "theoretical_loss": 3.6980842371318396, + "tokens_seen": 943325184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771073219658977, + "loss": 1.3607, + "theoretical_loss": 3.698059713955203, + "tokens_seen": 943390720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770912738214645, + "loss": 1.3189, + "theoretical_loss": 3.6980351929590647, + "tokens_seen": 943456256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770752256770311, + "loss": 1.3077, + "theoretical_loss": 3.6980106741430796, + "tokens_seen": 943521792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770591775325979, + "loss": 1.3808, + "theoretical_loss": 3.697986157506902, + "tokens_seen": 943587328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770431293881645, + "loss": 1.3744, + "theoretical_loss": 3.697961643050186, + "tokens_seen": 943652864 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 319507, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.49009370803833, + "objective/train/theoretical_loss": 3.697937130772588, + "objective/train/tokens_used": 964178400, + "theoretical_loss": 3.697937130772588, + "tokens_seen": 943718400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770270812437313, + "loss": 1.4312, + "theoretical_loss": 3.697937130772588, + "tokens_seen": 943718400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770110330992979, + "loss": 1.2883, + "theoretical_loss": 3.697912620673762, + "tokens_seen": 943783936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769949849548647, + "loss": 1.4021, + "theoretical_loss": 3.6978881127533634, + "tokens_seen": 943849472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769789368104313, + "loss": 1.428, + "theoretical_loss": 3.6978636070110475, + "tokens_seen": 943915008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769628886659981, + "loss": 1.2986, + "theoretical_loss": 3.6978391034464693, + "tokens_seen": 943980544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769468405215647, + "loss": 1.3775, + "theoretical_loss": 3.6978146020592835, + "tokens_seen": 944046080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769307923771315, + "loss": 1.3471, + "theoretical_loss": 3.697790102849147, + "tokens_seen": 944111616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769147442326981, + "loss": 1.3715, + "theoretical_loss": 3.6977656058157136, + "tokens_seen": 944177152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768986960882649, + "loss": 1.4267, + "theoretical_loss": 3.69774111095864, + "tokens_seen": 944242688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768826479438315, + "loss": 1.2568, + "theoretical_loss": 3.697716618277582, + "tokens_seen": 944308224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768665997993982, + "loss": 1.3657, + "theoretical_loss": 3.6976921277721937, + "tokens_seen": 944373760 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768505516549649, + "loss": 1.3924, + "theoretical_loss": 3.697667639442132, + "tokens_seen": 944439296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768345035105316, + "loss": 1.4021, + "theoretical_loss": 3.6976431532870535, + "tokens_seen": 944504832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768184553660983, + "loss": 1.3073, + "theoretical_loss": 3.697618669306612, + "tokens_seen": 944570368 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576802407221665, + "loss": 1.3843, + "theoretical_loss": 3.697594187500466, + "tokens_seen": 944635904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767863590772317, + "loss": 1.4449, + "theoretical_loss": 3.6975697078682694, + "tokens_seen": 944701440 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767703109327984, + "loss": 1.3983, + "theoretical_loss": 3.69754523040968, + "tokens_seen": 944766976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767542627883651, + "loss": 1.3271, + "theoretical_loss": 3.697520755124353, + "tokens_seen": 944832512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767382146439318, + "loss": 1.3503, + "theoretical_loss": 3.6974962820119455, + "tokens_seen": 944898048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767221664994984, + "loss": 1.2448, + "theoretical_loss": 3.697471811072113, + "tokens_seen": 944963584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767061183550652, + "loss": 1.3467, + "theoretical_loss": 3.697447342304513, + "tokens_seen": 945029120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005766900702106318, + "loss": 1.2963, + "theoretical_loss": 3.697422875708801, + "tokens_seen": 945094656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005766740220661986, + "loss": 1.3276, + "theoretical_loss": 3.6973984112846345, + "tokens_seen": 945160192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005766579739217652, + "loss": 1.3833, + "theoretical_loss": 3.69737394903167, + "tokens_seen": 945225728 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576641925777332, + "loss": 1.365, + "theoretical_loss": 3.697349488949564, + "tokens_seen": 945291264 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005766258776328986, + "loss": 1.3639, + "theoretical_loss": 3.6973250310379733, + "tokens_seen": 945356800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005766098294884654, + "loss": 1.3873, + "theoretical_loss": 3.697300575296555, + "tokens_seen": 945422336 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576593781344032, + "loss": 1.4619, + "theoretical_loss": 3.6972761217249666, + "tokens_seen": 945487872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005765777331995988, + "loss": 1.4046, + "theoretical_loss": 3.6972516703228644, + "tokens_seen": 945553408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005765616850551655, + "loss": 1.4257, + "theoretical_loss": 3.697227221089906, + "tokens_seen": 945618944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005765456369107322, + "loss": 1.4034, + "theoretical_loss": 3.697202774025749, + "tokens_seen": 945684480 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576529588766299, + "loss": 1.3286, + "theoretical_loss": 3.6971783291300504, + "tokens_seen": 945750016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005765135406218656, + "loss": 1.3103, + "theoretical_loss": 3.6971538864024676, + "tokens_seen": 945815552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764974924774323, + "loss": 1.3545, + "theoretical_loss": 3.697129445842658, + "tokens_seen": 945881088 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576481444332999, + "loss": 1.3565, + "theoretical_loss": 3.697105007450279, + "tokens_seen": 945946624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764653961885658, + "loss": 1.3578, + "theoretical_loss": 3.697080571224989, + "tokens_seen": 946012160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764493480441324, + "loss": 1.4201, + "theoretical_loss": 3.6970561371664443, + "tokens_seen": 946077696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764332998996992, + "loss": 1.4193, + "theoretical_loss": 3.6970317052743042, + "tokens_seen": 946143232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764172517552658, + "loss": 1.4432, + "theoretical_loss": 3.697007275548226, + "tokens_seen": 946208768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764012036108326, + "loss": 1.4115, + "theoretical_loss": 3.6969828479878677, + "tokens_seen": 946274304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005763851554663992, + "loss": 1.4443, + "theoretical_loss": 3.6969584225928873, + "tokens_seen": 946339840 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576369107321966, + "loss": 1.3909, + "theoretical_loss": 3.696933999362943, + "tokens_seen": 946405376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005763530591775326, + "loss": 1.3736, + "theoretical_loss": 3.696909578297692, + "tokens_seen": 946470912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005763370110330994, + "loss": 1.5252, + "theoretical_loss": 3.6968851593967944, + "tokens_seen": 946536448 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576320962888666, + "loss": 1.4132, + "theoretical_loss": 3.696860742659908, + "tokens_seen": 946601984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005763049147442328, + "loss": 1.3822, + "theoretical_loss": 3.6968363280866896, + "tokens_seen": 946667520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005762888665997994, + "loss": 1.4366, + "theoretical_loss": 3.6968119156768, + "tokens_seen": 946733056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005762728184553662, + "loss": 1.3858, + "theoretical_loss": 3.696787505429896, + "tokens_seen": 946798592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005762567703109328, + "loss": 1.3448, + "theoretical_loss": 3.6967630973456376, + "tokens_seen": 946864128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005762407221664996, + "loss": 1.387, + "theoretical_loss": 3.6967386914236826, + "tokens_seen": 946929664 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 320452, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4507883787155151, + "objective/train/theoretical_loss": 3.6967142876636903, + "objective/train/tokens_used": 967455200, + "theoretical_loss": 3.6967142876636903, + "tokens_seen": 946995200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005762246740220662, + "loss": 1.4276, + "theoretical_loss": 3.6967142876636903, + "tokens_seen": 946995200 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576208625877633, + "loss": 1.3087, + "theoretical_loss": 3.696689886065319, + "tokens_seen": 947060736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005761925777331996, + "loss": 1.3931, + "theoretical_loss": 3.6966654866282282, + "tokens_seen": 947126272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005761765295887664, + "loss": 1.4385, + "theoretical_loss": 3.6966410893520774, + "tokens_seen": 947191808 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576160481444333, + "loss": 1.3713, + "theoretical_loss": 3.696616694236524, + "tokens_seen": 947257344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005761444332998998, + "loss": 1.3719, + "theoretical_loss": 3.696592301281229, + "tokens_seen": 947322880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005761283851554664, + "loss": 1.2812, + "theoretical_loss": 3.6965679104858515, + "tokens_seen": 947388416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005761123370110332, + "loss": 1.4943, + "theoretical_loss": 3.6965435218500495, + "tokens_seen": 947453952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760962888665998, + "loss": 1.4148, + "theoretical_loss": 3.6965191353734843, + "tokens_seen": 947519488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760802407221666, + "loss": 1.2766, + "theoretical_loss": 3.6964947510558135, + "tokens_seen": 947585024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760641925777332, + "loss": 1.4846, + "theoretical_loss": 3.6964703688966978, + "tokens_seen": 947650560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760481444333, + "loss": 1.4069, + "theoretical_loss": 3.696445988895797, + "tokens_seen": 947716096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760320962888666, + "loss": 1.3377, + "theoretical_loss": 3.69642161105277, + "tokens_seen": 947781632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760160481444334, + "loss": 1.289, + "theoretical_loss": 3.6963972353672774, + "tokens_seen": 947847168 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576, + "loss": 1.3997, + "theoretical_loss": 3.696372861838979, + "tokens_seen": 947912704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759839518555668, + "loss": 1.448, + "theoretical_loss": 3.696348490467534, + "tokens_seen": 947978240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759679037111334, + "loss": 1.4665, + "theoretical_loss": 3.6963241212526032, + "tokens_seen": 948043776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759518555667002, + "loss": 1.2979, + "theoretical_loss": 3.6962997541938467, + "tokens_seen": 948109312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759358074222668, + "loss": 1.3509, + "theoretical_loss": 3.696275389290925, + "tokens_seen": 948174848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759197592778336, + "loss": 1.3388, + "theoretical_loss": 3.6962510265434974, + "tokens_seen": 948240384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759037111334002, + "loss": 1.3306, + "theoretical_loss": 3.6962266659512246, + "tokens_seen": 948305920 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575887662988967, + "loss": 1.4484, + "theoretical_loss": 3.6962023075137673, + "tokens_seen": 948371456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005758716148445336, + "loss": 1.4334, + "theoretical_loss": 3.696177951230786, + "tokens_seen": 948436992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005758555667001004, + "loss": 1.3966, + "theoretical_loss": 3.6961535971019415, + "tokens_seen": 948502528 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575839518555667, + "loss": 1.2923, + "theoretical_loss": 3.696129245126894, + "tokens_seen": 948568064 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005758234704112338, + "loss": 1.4143, + "theoretical_loss": 3.6961048953053046, + "tokens_seen": 948633600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005758074222668004, + "loss": 1.3425, + "theoretical_loss": 3.6960805476368335, + "tokens_seen": 948699136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005757913741223672, + "loss": 1.4563, + "theoretical_loss": 3.696056202121142, + "tokens_seen": 948764672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005757753259779338, + "loss": 1.3792, + "theoretical_loss": 3.696031858757891, + "tokens_seen": 948830208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005757592778335006, + "loss": 1.3231, + "theoretical_loss": 3.696007517546742, + "tokens_seen": 948895744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005757432296890672, + "loss": 1.3163, + "theoretical_loss": 3.6959831784873565, + "tokens_seen": 948961280 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575727181544634, + "loss": 1.4687, + "theoretical_loss": 3.695958841579394, + "tokens_seen": 949026816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005757111334002006, + "loss": 1.3181, + "theoretical_loss": 3.6959345068225167, + "tokens_seen": 949092352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005756950852557674, + "loss": 1.3718, + "theoretical_loss": 3.6959101742163867, + "tokens_seen": 949157888 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575679037111334, + "loss": 1.3565, + "theoretical_loss": 3.695885843760664, + "tokens_seen": 949223424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005756629889669008, + "loss": 1.4187, + "theoretical_loss": 3.695861515455011, + "tokens_seen": 949288960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005756469408224674, + "loss": 1.372, + "theoretical_loss": 3.6958371892990893, + "tokens_seen": 949354496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005756308926780342, + "loss": 1.3937, + "theoretical_loss": 3.695812865292561, + "tokens_seen": 949420032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005756148445336008, + "loss": 1.4296, + "theoretical_loss": 3.695788543435086, + "tokens_seen": 949485568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005755987963891676, + "loss": 1.3169, + "theoretical_loss": 3.695764223726328, + "tokens_seen": 949551104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005755827482447342, + "loss": 1.3959, + "theoretical_loss": 3.695739906165948, + "tokens_seen": 949616640 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575566700100301, + "loss": 1.3078, + "theoretical_loss": 3.6957155907536086, + "tokens_seen": 949682176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005755506519558676, + "loss": 1.4033, + "theoretical_loss": 3.6956912774889714, + "tokens_seen": 949747712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005755346038114344, + "loss": 1.3494, + "theoretical_loss": 3.6956669663716983, + "tokens_seen": 949813248 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575518555667001, + "loss": 1.3703, + "theoretical_loss": 3.695642657401452, + "tokens_seen": 949878784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005755025075225678, + "loss": 1.4269, + "theoretical_loss": 3.6956183505778943, + "tokens_seen": 949944320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754864593781344, + "loss": 1.4431, + "theoretical_loss": 3.6955940459006884, + "tokens_seen": 950009856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754704112337011, + "loss": 1.3288, + "theoretical_loss": 3.6955697433694956, + "tokens_seen": 950075392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754543630892678, + "loss": 1.3045, + "theoretical_loss": 3.6955454429839785, + "tokens_seen": 950140928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754383149448345, + "loss": 1.3202, + "theoretical_loss": 3.695521144743801, + "tokens_seen": 950206464 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 321619, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4422682523727417, + "objective/train/theoretical_loss": 3.695496848648624, + "objective/train/tokens_used": 970732000, + "theoretical_loss": 3.695496848648624, + "tokens_seen": 950272000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754222668004012, + "loss": 1.5001, + "theoretical_loss": 3.695496848648624, + "tokens_seen": 950272000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754062186559679, + "loss": 1.3796, + "theoretical_loss": 3.695472554698112, + "tokens_seen": 950337536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005753901705115346, + "loss": 1.3869, + "theoretical_loss": 3.695448262891926, + "tokens_seen": 950403072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005753741223671013, + "loss": 1.4017, + "theoretical_loss": 3.69542397322973, + "tokens_seen": 950468608 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575358074222668, + "loss": 1.4283, + "theoretical_loss": 3.6953996857111875, + "tokens_seen": 950534144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005753420260782347, + "loss": 1.3539, + "theoretical_loss": 3.69537540033596, + "tokens_seen": 950599680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005753259779338014, + "loss": 1.3158, + "theoretical_loss": 3.6953511171037112, + "tokens_seen": 950665216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005753099297893681, + "loss": 1.3968, + "theoretical_loss": 3.6953268360141047, + "tokens_seen": 950730752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752938816449348, + "loss": 1.3244, + "theoretical_loss": 3.695302557066804, + "tokens_seen": 950796288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752778335005015, + "loss": 1.4501, + "theoretical_loss": 3.695278280261472, + "tokens_seen": 950861824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752617853560682, + "loss": 1.3713, + "theoretical_loss": 3.6952540055977714, + "tokens_seen": 950927360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752457372116349, + "loss": 1.3846, + "theoretical_loss": 3.695229733075367, + "tokens_seen": 950992896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752296890672016, + "loss": 1.3798, + "theoretical_loss": 3.695205462693922, + "tokens_seen": 951058432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752136409227683, + "loss": 1.3508, + "theoretical_loss": 3.695181194453099, + "tokens_seen": 951123968 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575197592778335, + "loss": 1.4068, + "theoretical_loss": 3.6951569283525627, + "tokens_seen": 951189504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751815446339017, + "loss": 1.3751, + "theoretical_loss": 3.695132664391977, + "tokens_seen": 951255040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751654964894684, + "loss": 1.4757, + "theoretical_loss": 3.6951084025710053, + "tokens_seen": 951320576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751494483450351, + "loss": 1.3342, + "theoretical_loss": 3.695084142889312, + "tokens_seen": 951386112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751334002006018, + "loss": 1.452, + "theoretical_loss": 3.695059885346561, + "tokens_seen": 951451648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751173520561685, + "loss": 1.3871, + "theoretical_loss": 3.6950356299424163, + "tokens_seen": 951517184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751013039117353, + "loss": 1.499, + "theoretical_loss": 3.6950113766765416, + "tokens_seen": 951582720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005750852557673019, + "loss": 1.3029, + "theoretical_loss": 3.694987125548602, + "tokens_seen": 951648256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005750692076228687, + "loss": 1.3716, + "theoretical_loss": 3.694962876558261, + "tokens_seen": 951713792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005750531594784353, + "loss": 1.3362, + "theoretical_loss": 3.6949386297051836, + "tokens_seen": 951779328 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575037111334002, + "loss": 1.4624, + "theoretical_loss": 3.6949143849890342, + "tokens_seen": 951844864 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005750210631895687, + "loss": 1.3921, + "theoretical_loss": 3.6948901424094776, + "tokens_seen": 951910400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005750050150451355, + "loss": 1.3522, + "theoretical_loss": 3.6948659019661783, + "tokens_seen": 951975936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749889669007021, + "loss": 1.4732, + "theoretical_loss": 3.6948416636588, + "tokens_seen": 952041472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749729187562689, + "loss": 1.4277, + "theoretical_loss": 3.694817427487009, + "tokens_seen": 952107008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749568706118355, + "loss": 1.2995, + "theoretical_loss": 3.6947931934504687, + "tokens_seen": 952172544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749408224674023, + "loss": 1.4295, + "theoretical_loss": 3.6947689615488457, + "tokens_seen": 952238080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749247743229689, + "loss": 1.4033, + "theoretical_loss": 3.6947447317818036, + "tokens_seen": 952303616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749087261785357, + "loss": 1.3872, + "theoretical_loss": 3.694720504149008, + "tokens_seen": 952369152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748926780341023, + "loss": 1.4375, + "theoretical_loss": 3.694696278650124, + "tokens_seen": 952434688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748766298896691, + "loss": 1.3641, + "theoretical_loss": 3.6946720552848173, + "tokens_seen": 952500224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748605817452357, + "loss": 1.379, + "theoretical_loss": 3.694647834052753, + "tokens_seen": 952565760 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748445336008025, + "loss": 1.2778, + "theoretical_loss": 3.6946236149535956, + "tokens_seen": 952631296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748284854563691, + "loss": 1.3413, + "theoretical_loss": 3.6945993979870115, + "tokens_seen": 952696832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748124373119359, + "loss": 1.4252, + "theoretical_loss": 3.694575183152666, + "tokens_seen": 952762368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747963891675025, + "loss": 1.3981, + "theoretical_loss": 3.6945509704502246, + "tokens_seen": 952827904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747803410230693, + "loss": 1.3961, + "theoretical_loss": 3.6945267598793534, + "tokens_seen": 952893440 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747642928786359, + "loss": 1.3123, + "theoretical_loss": 3.694502551439718, + "tokens_seen": 952958976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747482447342027, + "loss": 1.396, + "theoretical_loss": 3.6944783451309835, + "tokens_seen": 953024512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747321965897693, + "loss": 1.4178, + "theoretical_loss": 3.694454140952817, + "tokens_seen": 953090048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747161484453361, + "loss": 1.3638, + "theoretical_loss": 3.694429938904884, + "tokens_seen": 953155584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747001003009027, + "loss": 1.5011, + "theoretical_loss": 3.69440573898685, + "tokens_seen": 953221120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746840521564695, + "loss": 1.3855, + "theoretical_loss": 3.694381541198382, + "tokens_seen": 953286656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746680040120361, + "loss": 1.3978, + "theoretical_loss": 3.694357345539146, + "tokens_seen": 953352192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746519558676029, + "loss": 1.4071, + "theoretical_loss": 3.6943331520088076, + "tokens_seen": 953417728 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746359077231695, + "loss": 1.4986, + "theoretical_loss": 3.694308960607034, + "tokens_seen": 953483264 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 322577, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3857512474060059, + "objective/train/theoretical_loss": 3.694284771333491, + "objective/train/tokens_used": 974008800, + "theoretical_loss": 3.694284771333491, + "tokens_seen": 953548800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746198595787363, + "loss": 1.3747, + "theoretical_loss": 3.694284771333491, + "tokens_seen": 953548800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746038114343029, + "loss": 1.4324, + "theoretical_loss": 3.6942605841878464, + "tokens_seen": 953614336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745877632898697, + "loss": 1.35, + "theoretical_loss": 3.694236399169765, + "tokens_seen": 953679872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745717151454363, + "loss": 1.3603, + "theoretical_loss": 3.6942122162789146, + "tokens_seen": 953745408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745556670010031, + "loss": 1.444, + "theoretical_loss": 3.694188035514962, + "tokens_seen": 953810944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745396188565697, + "loss": 1.3485, + "theoretical_loss": 3.694163856877573, + "tokens_seen": 953876480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745235707121365, + "loss": 1.2706, + "theoretical_loss": 3.6941396803664155, + "tokens_seen": 953942016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745075225677031, + "loss": 1.4315, + "theoretical_loss": 3.6941155059811566, + "tokens_seen": 954007552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744914744232699, + "loss": 1.406, + "theoretical_loss": 3.6940913337214627, + "tokens_seen": 954073088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744754262788365, + "loss": 1.3743, + "theoretical_loss": 3.6940671635870013, + "tokens_seen": 954138624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744593781344033, + "loss": 1.3819, + "theoretical_loss": 3.6940429955774388, + "tokens_seen": 954204160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744433299899699, + "loss": 1.4316, + "theoretical_loss": 3.6940188296924434, + "tokens_seen": 954269696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744272818455367, + "loss": 1.4471, + "theoretical_loss": 3.6939946659316822, + "tokens_seen": 954335232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744112337011033, + "loss": 1.3403, + "theoretical_loss": 3.693970504294823, + "tokens_seen": 954400768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743951855566701, + "loss": 1.4192, + "theoretical_loss": 3.6939463447815317, + "tokens_seen": 954466304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743791374122367, + "loss": 1.4291, + "theoretical_loss": 3.693922187391478, + "tokens_seen": 954531840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743630892678035, + "loss": 1.3475, + "theoretical_loss": 3.6938980321243284, + "tokens_seen": 954597376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743470411233701, + "loss": 1.3311, + "theoretical_loss": 3.6938738789797507, + "tokens_seen": 954662912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743309929789369, + "loss": 1.3657, + "theoretical_loss": 3.6938497279574127, + "tokens_seen": 954728448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743149448345035, + "loss": 1.2317, + "theoretical_loss": 3.6938255790569823, + "tokens_seen": 954793984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742988966900703, + "loss": 1.476, + "theoretical_loss": 3.6938014322781276, + "tokens_seen": 954859520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742828485456369, + "loss": 1.2893, + "theoretical_loss": 3.6937772876205166, + "tokens_seen": 954925056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742668004012037, + "loss": 1.3469, + "theoretical_loss": 3.6937531450838175, + "tokens_seen": 954990592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742507522567703, + "loss": 1.3121, + "theoretical_loss": 3.6937290046676976, + "tokens_seen": 955056128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742347041123371, + "loss": 1.4125, + "theoretical_loss": 3.693704866371826, + "tokens_seen": 955121664 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742186559679037, + "loss": 1.3416, + "theoretical_loss": 3.6936807301958714, + "tokens_seen": 955187200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742026078234705, + "loss": 1.4748, + "theoretical_loss": 3.693656596139501, + "tokens_seen": 955252736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741865596790371, + "loss": 1.3533, + "theoretical_loss": 3.693632464202384, + "tokens_seen": 955318272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741705115346039, + "loss": 1.417, + "theoretical_loss": 3.693608334384189, + "tokens_seen": 955383808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741544633901705, + "loss": 1.3945, + "theoretical_loss": 3.6935842066845845, + "tokens_seen": 955449344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741384152457373, + "loss": 1.3676, + "theoretical_loss": 3.6935600811032394, + "tokens_seen": 955514880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741223671013039, + "loss": 1.4023, + "theoretical_loss": 3.6935359576398215, + "tokens_seen": 955580416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741063189568707, + "loss": 1.3785, + "theoretical_loss": 3.6935118362940007, + "tokens_seen": 955645952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005740902708124373, + "loss": 1.5164, + "theoretical_loss": 3.6934877170654454, + "tokens_seen": 955711488 + }, + { + "epoch": 0.29, + "learning_rate": 0.000574074222668004, + "loss": 1.3221, + "theoretical_loss": 3.693463599953825, + "tokens_seen": 955777024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005740581745235707, + "loss": 1.4135, + "theoretical_loss": 3.693439484958808, + "tokens_seen": 955842560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005740421263791374, + "loss": 1.512, + "theoretical_loss": 3.693415372080064, + "tokens_seen": 955908096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005740260782347041, + "loss": 1.3623, + "theoretical_loss": 3.693391261317262, + "tokens_seen": 955973632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005740100300902708, + "loss": 1.4184, + "theoretical_loss": 3.693367152670071, + "tokens_seen": 956039168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005739939819458375, + "loss": 1.486, + "theoretical_loss": 3.6933430461381613, + "tokens_seen": 956104704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005739779338014042, + "loss": 1.3866, + "theoretical_loss": 3.693318941721201, + "tokens_seen": 956170240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005739618856569709, + "loss": 1.4363, + "theoretical_loss": 3.693294839418861, + "tokens_seen": 956235776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005739458375125376, + "loss": 1.4961, + "theoretical_loss": 3.69327073923081, + "tokens_seen": 956301312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005739297893681043, + "loss": 1.2741, + "theoretical_loss": 3.6932466411567173, + "tokens_seen": 956366848 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573913741223671, + "loss": 1.3535, + "theoretical_loss": 3.693222545196254, + "tokens_seen": 956432384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738976930792377, + "loss": 1.3106, + "theoretical_loss": 3.693198451349089, + "tokens_seen": 956497920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738816449348044, + "loss": 1.3628, + "theoretical_loss": 3.6931743596148916, + "tokens_seen": 956563456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738655967903711, + "loss": 1.408, + "theoretical_loss": 3.6931502699933336, + "tokens_seen": 956628992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738495486459378, + "loss": 1.2204, + "theoretical_loss": 3.693126182484083, + "tokens_seen": 956694528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738335005015045, + "loss": 1.3629, + "theoretical_loss": 3.693102097086811, + "tokens_seen": 956760064 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 323899, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3709452152252197, + "objective/train/theoretical_loss": 3.693078013801187, + "objective/train/tokens_used": 977285600, + "theoretical_loss": 3.693078013801187, + "tokens_seen": 956825600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738174523570712, + "loss": 1.3689, + "theoretical_loss": 3.693078013801187, + "tokens_seen": 956825600 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573801404212638, + "loss": 1.3385, + "theoretical_loss": 3.6930539326268828, + "tokens_seen": 956891136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005737853560682046, + "loss": 1.407, + "theoretical_loss": 3.6930298535635666, + "tokens_seen": 956956672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005737693079237713, + "loss": 1.3102, + "theoretical_loss": 3.693005776610911, + "tokens_seen": 957022208 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573753259779338, + "loss": 1.3474, + "theoretical_loss": 3.692981701768585, + "tokens_seen": 957087744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005737372116349047, + "loss": 1.3031, + "theoretical_loss": 3.6929576290362593, + "tokens_seen": 957153280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005737211634904714, + "loss": 1.3257, + "theoretical_loss": 3.6929335584136047, + "tokens_seen": 957218816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005737051153460382, + "loss": 1.41, + "theoretical_loss": 3.692909489900292, + "tokens_seen": 957284352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005736890672016048, + "loss": 1.4056, + "theoretical_loss": 3.6928854234959925, + "tokens_seen": 957349888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005736730190571716, + "loss": 1.2758, + "theoretical_loss": 3.692861359200376, + "tokens_seen": 957415424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005736569709127382, + "loss": 1.3864, + "theoretical_loss": 3.6928372970131136, + "tokens_seen": 957480960 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573640922768305, + "loss": 1.3342, + "theoretical_loss": 3.692813236933877, + "tokens_seen": 957546496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005736248746238716, + "loss": 1.3311, + "theoretical_loss": 3.692789178962337, + "tokens_seen": 957612032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005736088264794384, + "loss": 1.3913, + "theoretical_loss": 3.692765123098164, + "tokens_seen": 957677568 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573592778335005, + "loss": 1.3976, + "theoretical_loss": 3.69274106934103, + "tokens_seen": 957743104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005735767301905718, + "loss": 1.5343, + "theoretical_loss": 3.6927170176906063, + "tokens_seen": 957808640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005735606820461384, + "loss": 1.3732, + "theoretical_loss": 3.692692968146564, + "tokens_seen": 957874176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005735446339017052, + "loss": 1.3071, + "theoretical_loss": 3.692668920708574, + "tokens_seen": 957939712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005735285857572718, + "loss": 1.3795, + "theoretical_loss": 3.692644875376309, + "tokens_seen": 958005248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005735125376128386, + "loss": 1.4814, + "theoretical_loss": 3.6926208321494394, + "tokens_seen": 958070784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005734964894684052, + "loss": 1.4087, + "theoretical_loss": 3.692596791027638, + "tokens_seen": 958136320 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573480441323972, + "loss": 1.3891, + "theoretical_loss": 3.6925727520105753, + "tokens_seen": 958201856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005734643931795386, + "loss": 1.2506, + "theoretical_loss": 3.692548715097924, + "tokens_seen": 958267392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005734483450351054, + "loss": 1.3148, + "theoretical_loss": 3.6925246802893557, + "tokens_seen": 958332928 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573432296890672, + "loss": 1.4035, + "theoretical_loss": 3.6925006475845423, + "tokens_seen": 958398464 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005734162487462388, + "loss": 1.226, + "theoretical_loss": 3.692476616983156, + "tokens_seen": 958464000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005734002006018054, + "loss": 1.4648, + "theoretical_loss": 3.6924525884848687, + "tokens_seen": 958529536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005733841524573722, + "loss": 1.4966, + "theoretical_loss": 3.6924285620893524, + "tokens_seen": 958595072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005733681043129388, + "loss": 1.3732, + "theoretical_loss": 3.69240453779628, + "tokens_seen": 958660608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005733520561685056, + "loss": 1.4946, + "theoretical_loss": 3.692380515605323, + "tokens_seen": 958726144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005733360080240722, + "loss": 1.3418, + "theoretical_loss": 3.6923564955161545, + "tokens_seen": 958791680 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573319959879639, + "loss": 1.3368, + "theoretical_loss": 3.6923324775284465, + "tokens_seen": 958857216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005733039117352056, + "loss": 1.5341, + "theoretical_loss": 3.692308461641872, + "tokens_seen": 958922752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005732878635907724, + "loss": 1.3288, + "theoretical_loss": 3.6922844478561028, + "tokens_seen": 958988288 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573271815446339, + "loss": 1.3637, + "theoretical_loss": 3.6922604361708125, + "tokens_seen": 959053824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005732557673019058, + "loss": 1.4385, + "theoretical_loss": 3.692236426585673, + "tokens_seen": 959119360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005732397191574724, + "loss": 1.2887, + "theoretical_loss": 3.6922124191003576, + "tokens_seen": 959184896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005732236710130392, + "loss": 1.3529, + "theoretical_loss": 3.6921884137145393, + "tokens_seen": 959250432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005732076228686058, + "loss": 1.394, + "theoretical_loss": 3.692164410427891, + "tokens_seen": 959315968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005731915747241726, + "loss": 1.3602, + "theoretical_loss": 3.6921404092400856, + "tokens_seen": 959381504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005731755265797392, + "loss": 1.407, + "theoretical_loss": 3.6921164101507964, + "tokens_seen": 959447040 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573159478435306, + "loss": 1.4307, + "theoretical_loss": 3.6920924131596964, + "tokens_seen": 959512576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005731434302908726, + "loss": 1.4011, + "theoretical_loss": 3.692068418266459, + "tokens_seen": 959578112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005731273821464394, + "loss": 1.4794, + "theoretical_loss": 3.6920444254707574, + "tokens_seen": 959643648 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573111334002006, + "loss": 1.4011, + "theoretical_loss": 3.6920204347722656, + "tokens_seen": 959709184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730952858575728, + "loss": 1.3967, + "theoretical_loss": 3.6919964461706565, + "tokens_seen": 959774720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730792377131394, + "loss": 1.4577, + "theoretical_loss": 3.691972459665603, + "tokens_seen": 959840256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730631895687062, + "loss": 1.4282, + "theoretical_loss": 3.6919484752567806, + "tokens_seen": 959905792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730471414242728, + "loss": 1.2874, + "theoretical_loss": 3.691924492943861, + "tokens_seen": 959971328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730310932798396, + "loss": 1.3386, + "theoretical_loss": 3.6919005127265194, + "tokens_seen": 960036864 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 325104, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.465695858001709, + "objective/train/theoretical_loss": 3.6918765346044298, + "objective/train/tokens_used": 980562400, + "theoretical_loss": 3.6918765346044298, + "tokens_seen": 960102400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730150451354062, + "loss": 1.4385, + "theoretical_loss": 3.6918765346044298, + "tokens_seen": 960102400 + }, + { + "epoch": 0.29, + "learning_rate": 0.000572998996990973, + "loss": 1.4227, + "theoretical_loss": 3.6918525585772644, + "tokens_seen": 960167936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729829488465396, + "loss": 1.429, + "theoretical_loss": 3.691828584644699, + "tokens_seen": 960233472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729669007021064, + "loss": 1.3622, + "theoretical_loss": 3.6918046128064064, + "tokens_seen": 960299008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729508525576731, + "loss": 1.5271, + "theoretical_loss": 3.691780643062062, + "tokens_seen": 960364544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729348044132398, + "loss": 1.4355, + "theoretical_loss": 3.6917566754113387, + "tokens_seen": 960430080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729187562688065, + "loss": 1.3899, + "theoretical_loss": 3.6917327098539117, + "tokens_seen": 960495616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729027081243732, + "loss": 1.3185, + "theoretical_loss": 3.6917087463894553, + "tokens_seen": 960561152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005728866599799399, + "loss": 1.2467, + "theoretical_loss": 3.6916847850176433, + "tokens_seen": 960626688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005728706118355066, + "loss": 1.41, + "theoretical_loss": 3.6916608257381505, + "tokens_seen": 960692224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005728545636910733, + "loss": 1.3998, + "theoretical_loss": 3.6916368685506518, + "tokens_seen": 960757760 + }, + { + "epoch": 0.29, + "learning_rate": 0.00057283851554664, + "loss": 1.3782, + "theoretical_loss": 3.691612913454822, + "tokens_seen": 960823296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005728224674022067, + "loss": 1.5504, + "theoretical_loss": 3.6915889604503356, + "tokens_seen": 960888832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005728064192577734, + "loss": 1.3247, + "theoretical_loss": 3.6915650095368666, + "tokens_seen": 960954368 + }, + { + "epoch": 0.29, + "learning_rate": 0.00057279037111334, + "loss": 1.3303, + "theoretical_loss": 3.6915410607140915, + "tokens_seen": 961019904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005727743229689068, + "loss": 1.4098, + "theoretical_loss": 3.6915171139816834, + "tokens_seen": 961085440 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005727582748244734, + "loss": 1.3843, + "theoretical_loss": 3.691493169339319, + "tokens_seen": 961150976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005727422266800402, + "loss": 1.3701, + "theoretical_loss": 3.691469226786672, + "tokens_seen": 961216512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005727261785356068, + "loss": 1.3938, + "theoretical_loss": 3.6914452863234186, + "tokens_seen": 961282048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005727101303911736, + "loss": 1.3824, + "theoretical_loss": 3.691421347949234, + "tokens_seen": 961347584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005726940822467402, + "loss": 1.4114, + "theoretical_loss": 3.691397411663792, + "tokens_seen": 961413120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005726780341023069, + "loss": 1.3189, + "theoretical_loss": 3.6913734774667706, + "tokens_seen": 961478656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005726619859578736, + "loss": 1.3585, + "theoretical_loss": 3.691349545357843, + "tokens_seen": 961544192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005726459378134403, + "loss": 1.3614, + "theoretical_loss": 3.6913256153366856, + "tokens_seen": 961609728 + }, + { + "epoch": 0.29, + "learning_rate": 0.000572629889669007, + "loss": 1.3773, + "theoretical_loss": 3.691301687402974, + "tokens_seen": 961675264 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005726138415245737, + "loss": 1.3317, + "theoretical_loss": 3.6912777615563837, + "tokens_seen": 961740800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725977933801404, + "loss": 1.2783, + "theoretical_loss": 3.69125383779659, + "tokens_seen": 961806336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725817452357071, + "loss": 1.3975, + "theoretical_loss": 3.69122991612327, + "tokens_seen": 961871872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725656970912738, + "loss": 1.3962, + "theoretical_loss": 3.691205996536099, + "tokens_seen": 961937408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725496489468405, + "loss": 1.395, + "theoretical_loss": 3.6911820790347525, + "tokens_seen": 962002944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725336008024072, + "loss": 1.3541, + "theoretical_loss": 3.6911581636189066, + "tokens_seen": 962068480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725175526579739, + "loss": 1.3784, + "theoretical_loss": 3.691134250288238, + "tokens_seen": 962134016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725015045135406, + "loss": 1.3566, + "theoretical_loss": 3.6911103390424227, + "tokens_seen": 962199552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005724854563691073, + "loss": 1.2805, + "theoretical_loss": 3.6910864298811363, + "tokens_seen": 962265088 + }, + { + "epoch": 0.29, + "learning_rate": 0.000572469408224674, + "loss": 1.3285, + "theoretical_loss": 3.6910625228040557, + "tokens_seen": 962330624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005724533600802407, + "loss": 1.4194, + "theoretical_loss": 3.6910386178108574, + "tokens_seen": 962396160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005724373119358074, + "loss": 1.3384, + "theoretical_loss": 3.691014714901218, + "tokens_seen": 962461696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005724212637913741, + "loss": 1.4406, + "theoretical_loss": 3.6909908140748136, + "tokens_seen": 962527232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005724052156469408, + "loss": 1.383, + "theoretical_loss": 3.6909669153313205, + "tokens_seen": 962592768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005723891675025075, + "loss": 1.2981, + "theoretical_loss": 3.690943018670416, + "tokens_seen": 962658304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005723731193580742, + "loss": 1.3386, + "theoretical_loss": 3.690919124091777, + "tokens_seen": 962723840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005723570712136409, + "loss": 1.2929, + "theoretical_loss": 3.69089523159508, + "tokens_seen": 962789376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005723410230692077, + "loss": 1.4045, + "theoretical_loss": 3.690871341180001, + "tokens_seen": 962854912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005723249749247743, + "loss": 1.3683, + "theoretical_loss": 3.690847452846219, + "tokens_seen": 962920448 + }, + { + "epoch": 0.29, + "learning_rate": 0.000572308926780341, + "loss": 1.3836, + "theoretical_loss": 3.6908235665934095, + "tokens_seen": 962985984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722928786359077, + "loss": 1.3284, + "theoretical_loss": 3.6907996824212503, + "tokens_seen": 963051520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722768304914745, + "loss": 1.4417, + "theoretical_loss": 3.690775800329418, + "tokens_seen": 963117056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722607823470411, + "loss": 1.4012, + "theoretical_loss": 3.6907519203175907, + "tokens_seen": 963182592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722447342026079, + "loss": 1.3698, + "theoretical_loss": 3.6907280423854445, + "tokens_seen": 963248128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722286860581745, + "loss": 1.3087, + "theoretical_loss": 3.690704166532658, + "tokens_seen": 963313664 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 326047, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.373966932296753, + "objective/train/theoretical_loss": 3.690680292758908, + "objective/train/tokens_used": 983839200, + "theoretical_loss": 3.690680292758908, + "tokens_seen": 963379200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722126379137413, + "loss": 1.437, + "theoretical_loss": 3.690680292758908, + "tokens_seen": 963379200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721965897693079, + "loss": 1.4679, + "theoretical_loss": 3.690656421063873, + "tokens_seen": 963444736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721805416248747, + "loss": 1.3647, + "theoretical_loss": 3.690632551447229, + "tokens_seen": 963510272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721644934804413, + "loss": 1.3025, + "theoretical_loss": 3.690608683908655, + "tokens_seen": 963575808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721484453360081, + "loss": 1.2803, + "theoretical_loss": 3.6905848184478285, + "tokens_seen": 963641344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721323971915747, + "loss": 1.3523, + "theoretical_loss": 3.690560955064427, + "tokens_seen": 963706880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721163490471415, + "loss": 1.4114, + "theoretical_loss": 3.690537093758129, + "tokens_seen": 963772416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721003009027081, + "loss": 1.4266, + "theoretical_loss": 3.690513234528612, + "tokens_seen": 963837952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720842527582749, + "loss": 1.3681, + "theoretical_loss": 3.690489377375554, + "tokens_seen": 963903488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720682046138415, + "loss": 1.3101, + "theoretical_loss": 3.6904655222986333, + "tokens_seen": 963969024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720521564694083, + "loss": 1.3685, + "theoretical_loss": 3.690441669297528, + "tokens_seen": 964034560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720361083249749, + "loss": 1.4999, + "theoretical_loss": 3.6904178183719165, + "tokens_seen": 964100096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720200601805417, + "loss": 1.3722, + "theoretical_loss": 3.690393969521477, + "tokens_seen": 964165632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720040120361083, + "loss": 1.4375, + "theoretical_loss": 3.6903701227458883, + "tokens_seen": 964231168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719879638916751, + "loss": 1.3005, + "theoretical_loss": 3.6903462780448284, + "tokens_seen": 964296704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719719157472417, + "loss": 1.296, + "theoretical_loss": 3.690322435417976, + "tokens_seen": 964362240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719558676028085, + "loss": 1.2849, + "theoretical_loss": 3.69029859486501, + "tokens_seen": 964427776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719398194583751, + "loss": 1.314, + "theoretical_loss": 3.690274756385609, + "tokens_seen": 964493312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719237713139419, + "loss": 1.4088, + "theoretical_loss": 3.6902509199794507, + "tokens_seen": 964558848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719077231695085, + "loss": 1.4873, + "theoretical_loss": 3.6902270856462156, + "tokens_seen": 964624384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718916750250753, + "loss": 1.3435, + "theoretical_loss": 3.690203253385582, + "tokens_seen": 964689920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718756268806419, + "loss": 1.4194, + "theoretical_loss": 3.690179423197229, + "tokens_seen": 964755456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718595787362087, + "loss": 1.4869, + "theoretical_loss": 3.690155595080835, + "tokens_seen": 964820992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718435305917753, + "loss": 1.3384, + "theoretical_loss": 3.6901317690360793, + "tokens_seen": 964886528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718274824473421, + "loss": 1.4252, + "theoretical_loss": 3.6901079450626417, + "tokens_seen": 964952064 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718114343029087, + "loss": 1.4402, + "theoretical_loss": 3.6900841231602017, + "tokens_seen": 965017600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005717953861584755, + "loss": 1.3564, + "theoretical_loss": 3.690060303328438, + "tokens_seen": 965083136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005717793380140421, + "loss": 1.3953, + "theoretical_loss": 3.6900364855670293, + "tokens_seen": 965148672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005717632898696089, + "loss": 1.3565, + "theoretical_loss": 3.6900126698756566, + "tokens_seen": 965214208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005717472417251756, + "loss": 1.3693, + "theoretical_loss": 3.689988856253998, + "tokens_seen": 965279744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005717311935807423, + "loss": 1.3556, + "theoretical_loss": 3.6899650447017347, + "tokens_seen": 965345280 + }, + { + "epoch": 0.29, + "learning_rate": 0.000571715145436309, + "loss": 1.403, + "theoretical_loss": 3.6899412352185452, + "tokens_seen": 965410816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716990972918757, + "loss": 1.2627, + "theoretical_loss": 3.68991742780411, + "tokens_seen": 965476352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716830491474424, + "loss": 1.3862, + "theoretical_loss": 3.689893622458108, + "tokens_seen": 965541888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716670010030091, + "loss": 1.406, + "theoretical_loss": 3.6898698191802195, + "tokens_seen": 965607424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716509528585758, + "loss": 1.3838, + "theoretical_loss": 3.689846017970125, + "tokens_seen": 965672960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716349047141425, + "loss": 1.404, + "theoretical_loss": 3.689822218827505, + "tokens_seen": 965738496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716188565697092, + "loss": 1.254, + "theoretical_loss": 3.6897984217520383, + "tokens_seen": 965804032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716028084252759, + "loss": 1.4042, + "theoretical_loss": 3.6897746267434055, + "tokens_seen": 965869568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005715867602808426, + "loss": 1.3868, + "theoretical_loss": 3.6897508338012877, + "tokens_seen": 965935104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005715707121364093, + "loss": 1.1966, + "theoretical_loss": 3.689727042925364, + "tokens_seen": 966000640 + }, + { + "epoch": 0.29, + "learning_rate": 0.000571554663991976, + "loss": 1.4443, + "theoretical_loss": 3.689703254115315, + "tokens_seen": 966066176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005715386158475427, + "loss": 1.4936, + "theoretical_loss": 3.689679467370823, + "tokens_seen": 966131712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005715225677031094, + "loss": 1.3745, + "theoretical_loss": 3.6896556826915656, + "tokens_seen": 966197248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005715065195586761, + "loss": 1.3299, + "theoretical_loss": 3.689631900077226, + "tokens_seen": 966262784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714904714142428, + "loss": 1.5032, + "theoretical_loss": 3.6896081195274837, + "tokens_seen": 966328320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714744232698095, + "loss": 1.3185, + "theoretical_loss": 3.6895843410420195, + "tokens_seen": 966393856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714583751253762, + "loss": 1.4487, + "theoretical_loss": 3.689560564620514, + "tokens_seen": 966459392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714423269809429, + "loss": 1.3979, + "theoretical_loss": 3.689536790262649, + "tokens_seen": 966524928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714262788365096, + "loss": 1.343, + "theoretical_loss": 3.6895130179681046, + "tokens_seen": 966590464 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 327277, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.415788173675537, + "objective/train/theoretical_loss": 3.689489247736563, + "objective/train/tokens_used": 987116000, + "theoretical_loss": 3.689489247736563, + "tokens_seen": 966656000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714102306920763, + "loss": 1.3681, + "theoretical_loss": 3.689489247736563, + "tokens_seen": 966656000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713941825476429, + "loss": 1.3016, + "theoretical_loss": 3.689465479567703, + "tokens_seen": 966721536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713781344032097, + "loss": 1.4658, + "theoretical_loss": 3.6894417134612087, + "tokens_seen": 966787072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713620862587763, + "loss": 1.4065, + "theoretical_loss": 3.6894179494167596, + "tokens_seen": 966852608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713460381143431, + "loss": 1.3852, + "theoretical_loss": 3.6893941874340372, + "tokens_seen": 966918144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713299899699097, + "loss": 1.349, + "theoretical_loss": 3.6893704275127233, + "tokens_seen": 966983680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713139418254765, + "loss": 1.3741, + "theoretical_loss": 3.6893466696524992, + "tokens_seen": 967049216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712978936810431, + "loss": 1.3975, + "theoretical_loss": 3.689322913853047, + "tokens_seen": 967114752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712818455366099, + "loss": 1.4219, + "theoretical_loss": 3.689299160114047, + "tokens_seen": 967180288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712657973921765, + "loss": 1.4576, + "theoretical_loss": 3.689275408435182, + "tokens_seen": 967245824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712497492477432, + "loss": 1.4267, + "theoretical_loss": 3.6892516588161333, + "tokens_seen": 967311360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712337011033099, + "loss": 1.3833, + "theoretical_loss": 3.689227911256583, + "tokens_seen": 967376896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712176529588766, + "loss": 1.3428, + "theoretical_loss": 3.6892041657562125, + "tokens_seen": 967442432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712016048144433, + "loss": 1.3452, + "theoretical_loss": 3.689180422314705, + "tokens_seen": 967507968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00057118555667001, + "loss": 1.3975, + "theoretical_loss": 3.689156680931741, + "tokens_seen": 967573504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005711695085255767, + "loss": 1.3263, + "theoretical_loss": 3.6891329416070033, + "tokens_seen": 967639040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005711534603811434, + "loss": 1.3677, + "theoretical_loss": 3.6891092043401743, + "tokens_seen": 967704576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005711374122367101, + "loss": 1.332, + "theoretical_loss": 3.6890854691309354, + "tokens_seen": 967770112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005711213640922768, + "loss": 1.3547, + "theoretical_loss": 3.6890617359789704, + "tokens_seen": 967835648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005711053159478435, + "loss": 1.3289, + "theoretical_loss": 3.689038004883961, + "tokens_seen": 967901184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005710892678034102, + "loss": 1.3085, + "theoretical_loss": 3.6890142758455884, + "tokens_seen": 967966720 + }, + { + "epoch": 0.29, + "learning_rate": 0.000571073219658977, + "loss": 1.3303, + "theoretical_loss": 3.6889905488635373, + "tokens_seen": 968032256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005710571715145436, + "loss": 1.3869, + "theoretical_loss": 3.6889668239374886, + "tokens_seen": 968097792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005710411233701103, + "loss": 1.3357, + "theoretical_loss": 3.6889431010671254, + "tokens_seen": 968163328 + }, + { + "epoch": 0.29, + "learning_rate": 0.000571025075225677, + "loss": 1.3994, + "theoretical_loss": 3.6889193802521314, + "tokens_seen": 968228864 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005710090270812437, + "loss": 1.3353, + "theoretical_loss": 3.6888956614921886, + "tokens_seen": 968294400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005709929789368104, + "loss": 1.4054, + "theoretical_loss": 3.68887194478698, + "tokens_seen": 968359936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005709769307923772, + "loss": 1.2776, + "theoretical_loss": 3.6888482301361885, + "tokens_seen": 968425472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005709608826479438, + "loss": 1.3476, + "theoretical_loss": 3.688824517539497, + "tokens_seen": 968491008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005709448345035106, + "loss": 1.3195, + "theoretical_loss": 3.688800806996589, + "tokens_seen": 968556544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005709287863590772, + "loss": 1.3728, + "theoretical_loss": 3.6887770985071473, + "tokens_seen": 968622080 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570912738214644, + "loss": 1.4356, + "theoretical_loss": 3.688753392070855, + "tokens_seen": 968687616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708966900702106, + "loss": 1.3998, + "theoretical_loss": 3.6887296876873963, + "tokens_seen": 968753152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708806419257774, + "loss": 1.3481, + "theoretical_loss": 3.688705985356454, + "tokens_seen": 968818688 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570864593781344, + "loss": 1.3533, + "theoretical_loss": 3.688682285077711, + "tokens_seen": 968884224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708485456369108, + "loss": 1.4222, + "theoretical_loss": 3.6886585868508517, + "tokens_seen": 968949760 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708324974924774, + "loss": 1.3566, + "theoretical_loss": 3.688634890675559, + "tokens_seen": 969015296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708164493480442, + "loss": 1.4013, + "theoretical_loss": 3.688611196551517, + "tokens_seen": 969080832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708004012036108, + "loss": 1.4087, + "theoretical_loss": 3.68858750447841, + "tokens_seen": 969146368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005707843530591776, + "loss": 1.3192, + "theoretical_loss": 3.6885638144559207, + "tokens_seen": 969211904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005707683049147442, + "loss": 1.3206, + "theoretical_loss": 3.6885401264837334, + "tokens_seen": 969277440 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570752256770311, + "loss": 1.4492, + "theoretical_loss": 3.688516440561532, + "tokens_seen": 969342976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005707362086258776, + "loss": 1.311, + "theoretical_loss": 3.6884927566890005, + "tokens_seen": 969408512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005707201604814444, + "loss": 1.2671, + "theoretical_loss": 3.6884690748658233, + "tokens_seen": 969474048 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570704112337011, + "loss": 1.4067, + "theoretical_loss": 3.6884453950916845, + "tokens_seen": 969539584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706880641925778, + "loss": 1.3854, + "theoretical_loss": 3.6884217173662677, + "tokens_seen": 969605120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706720160481444, + "loss": 1.486, + "theoretical_loss": 3.6883980416892577, + "tokens_seen": 969670656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706559679037112, + "loss": 1.3749, + "theoretical_loss": 3.688374368060339, + "tokens_seen": 969736192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706399197592778, + "loss": 1.2727, + "theoretical_loss": 3.688350696479196, + "tokens_seen": 969801728 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706238716148446, + "loss": 1.3913, + "theoretical_loss": 3.6883270269455126, + "tokens_seen": 969867264 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 328315, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.420799970626831, + "objective/train/theoretical_loss": 3.688303359458974, + "objective/train/tokens_used": 990392800, + "theoretical_loss": 3.688303359458974, + "tokens_seen": 969932800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706078234704112, + "loss": 1.4175, + "theoretical_loss": 3.688303359458974, + "tokens_seen": 969932800 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570591775325978, + "loss": 1.4095, + "theoretical_loss": 3.6882796940192644, + "tokens_seen": 969998336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005705757271815446, + "loss": 1.3945, + "theoretical_loss": 3.6882560306260688, + "tokens_seen": 970063872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005705596790371114, + "loss": 1.41, + "theoretical_loss": 3.6882323692790724, + "tokens_seen": 970129408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005705436308926781, + "loss": 1.345, + "theoretical_loss": 3.688208709977959, + "tokens_seen": 970194944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005705275827482448, + "loss": 1.3141, + "theoretical_loss": 3.6881850527224147, + "tokens_seen": 970260480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005705115346038115, + "loss": 1.508, + "theoretical_loss": 3.6881613975121232, + "tokens_seen": 970326016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005704954864593782, + "loss": 1.3958, + "theoretical_loss": 3.6881377443467707, + "tokens_seen": 970391552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005704794383149449, + "loss": 1.3226, + "theoretical_loss": 3.6881140932260417, + "tokens_seen": 970457088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005704633901705116, + "loss": 1.3344, + "theoretical_loss": 3.688090444149622, + "tokens_seen": 970522624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005704473420260783, + "loss": 1.3384, + "theoretical_loss": 3.6880667971171963, + "tokens_seen": 970588160 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570431293881645, + "loss": 1.3955, + "theoretical_loss": 3.68804315212845, + "tokens_seen": 970653696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005704152457372117, + "loss": 1.4069, + "theoretical_loss": 3.6880195091830688, + "tokens_seen": 970719232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703991975927784, + "loss": 1.3192, + "theoretical_loss": 3.687995868280738, + "tokens_seen": 970784768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703831494483451, + "loss": 1.4352, + "theoretical_loss": 3.687972229421143, + "tokens_seen": 970850304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703671013039118, + "loss": 1.3864, + "theoretical_loss": 3.6879485926039695, + "tokens_seen": 970915840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703510531594785, + "loss": 1.3829, + "theoretical_loss": 3.687924957828903, + "tokens_seen": 970981376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703350050150452, + "loss": 1.3642, + "theoretical_loss": 3.6879013250956305, + "tokens_seen": 971046912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703189568706119, + "loss": 1.3393, + "theoretical_loss": 3.687877694403836, + "tokens_seen": 971112448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703029087261786, + "loss": 1.2679, + "theoretical_loss": 3.687854065753206, + "tokens_seen": 971177984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005702868605817453, + "loss": 1.3317, + "theoretical_loss": 3.6878304391434273, + "tokens_seen": 971243520 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570270812437312, + "loss": 1.356, + "theoretical_loss": 3.687806814574185, + "tokens_seen": 971309056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005702547642928787, + "loss": 1.3752, + "theoretical_loss": 3.687783192045166, + "tokens_seen": 971374592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005702387161484454, + "loss": 1.4202, + "theoretical_loss": 3.6877595715560556, + "tokens_seen": 971440128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005702226680040121, + "loss": 1.4366, + "theoretical_loss": 3.6877359531065412, + "tokens_seen": 971505664 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005702066198595788, + "loss": 1.3614, + "theoretical_loss": 3.6877123366963076, + "tokens_seen": 971571200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005701905717151455, + "loss": 1.4171, + "theoretical_loss": 3.6876887223250425, + "tokens_seen": 971636736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005701745235707122, + "loss": 1.2972, + "theoretical_loss": 3.687665109992431, + "tokens_seen": 971702272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005701584754262789, + "loss": 1.3607, + "theoretical_loss": 3.6876414996981612, + "tokens_seen": 971767808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005701424272818456, + "loss": 1.3152, + "theoretical_loss": 3.687617891441919, + "tokens_seen": 971833344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005701263791374123, + "loss": 1.4115, + "theoretical_loss": 3.6875942852233905, + "tokens_seen": 971898880 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570110330992979, + "loss": 1.5083, + "theoretical_loss": 3.6875706810422635, + "tokens_seen": 971964416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700942828485457, + "loss": 1.4294, + "theoretical_loss": 3.687547078898224, + "tokens_seen": 972029952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700782347041124, + "loss": 1.3117, + "theoretical_loss": 3.6875234787909585, + "tokens_seen": 972095488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700621865596791, + "loss": 1.463, + "theoretical_loss": 3.6874998807201553, + "tokens_seen": 972161024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700461384152458, + "loss": 1.4098, + "theoretical_loss": 3.6874762846855003, + "tokens_seen": 972226560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700300902708125, + "loss": 1.3988, + "theoretical_loss": 3.687452690686681, + "tokens_seen": 972292096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700140421263792, + "loss": 1.3265, + "theoretical_loss": 3.687429098723385, + "tokens_seen": 972357632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005699979939819458, + "loss": 1.3901, + "theoretical_loss": 3.687405508795298, + "tokens_seen": 972423168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005699819458375126, + "loss": 1.4919, + "theoretical_loss": 3.6873819209021086, + "tokens_seen": 972488704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005699658976930792, + "loss": 1.3806, + "theoretical_loss": 3.687358335043504, + "tokens_seen": 972554240 + }, + { + "epoch": 0.29, + "learning_rate": 0.000569949849548646, + "loss": 1.3869, + "theoretical_loss": 3.687334751219171, + "tokens_seen": 972619776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005699338014042126, + "loss": 1.3302, + "theoretical_loss": 3.6873111694287983, + "tokens_seen": 972685312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005699177532597794, + "loss": 1.371, + "theoretical_loss": 3.6872875896720725, + "tokens_seen": 972750848 + }, + { + "epoch": 0.29, + "learning_rate": 0.000569901705115346, + "loss": 1.379, + "theoretical_loss": 3.687264011948681, + "tokens_seen": 972816384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698856569709128, + "loss": 1.4398, + "theoretical_loss": 3.687240436258312, + "tokens_seen": 972881920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698696088264794, + "loss": 1.3119, + "theoretical_loss": 3.6872168626006534, + "tokens_seen": 972947456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698535606820461, + "loss": 1.3912, + "theoretical_loss": 3.687193290975393, + "tokens_seen": 973012992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698375125376128, + "loss": 1.4236, + "theoretical_loss": 3.6871697213822188, + "tokens_seen": 973078528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698214643931795, + "loss": 1.3956, + "theoretical_loss": 3.6871461538208177, + "tokens_seen": 973144064 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 329553, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5348623991012573, + "objective/train/theoretical_loss": 3.6871225882908787, + "objective/train/tokens_used": 993669600, + "theoretical_loss": 3.6871225882908787, + "tokens_seen": 973209600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698054162487462, + "loss": 1.45, + "theoretical_loss": 3.6871225882908787, + "tokens_seen": 973209600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005697893681043129, + "loss": 1.3545, + "theoretical_loss": 3.68709902479209, + "tokens_seen": 973275136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005697733199598796, + "loss": 1.4072, + "theoretical_loss": 3.687075463324139, + "tokens_seen": 973340672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005697572718154463, + "loss": 1.4714, + "theoretical_loss": 3.6870519038867156, + "tokens_seen": 973406208 + }, + { + "epoch": 0.29, + "learning_rate": 0.000569741223671013, + "loss": 1.4299, + "theoretical_loss": 3.687028346479506, + "tokens_seen": 973471744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005697251755265797, + "loss": 1.4439, + "theoretical_loss": 3.6870047911022, + "tokens_seen": 973537280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005697091273821464, + "loss": 1.4044, + "theoretical_loss": 3.6869812377544857, + "tokens_seen": 973602816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696930792377131, + "loss": 1.3847, + "theoretical_loss": 3.6869576864360516, + "tokens_seen": 973668352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696770310932798, + "loss": 1.5151, + "theoretical_loss": 3.6869341371465865, + "tokens_seen": 973733888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696609829488465, + "loss": 1.3391, + "theoretical_loss": 3.686910589885778, + "tokens_seen": 973799424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696449348044132, + "loss": 1.339, + "theoretical_loss": 3.686887044653316, + "tokens_seen": 973864960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696288866599799, + "loss": 1.5226, + "theoretical_loss": 3.6868635014488893, + "tokens_seen": 973930496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696128385155467, + "loss": 1.3032, + "theoretical_loss": 3.686839960272186, + "tokens_seen": 973996032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695967903711133, + "loss": 1.4417, + "theoretical_loss": 3.686816421122896, + "tokens_seen": 974061568 + }, + { + "epoch": 0.3, + "learning_rate": 0.00056958074222668, + "loss": 1.4024, + "theoretical_loss": 3.686792884000708, + "tokens_seen": 974127104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695646940822467, + "loss": 1.2908, + "theoretical_loss": 3.68676934890531, + "tokens_seen": 974192640 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695486459378135, + "loss": 1.3637, + "theoretical_loss": 3.6867458158363924, + "tokens_seen": 974258176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695325977933801, + "loss": 1.294, + "theoretical_loss": 3.6867222847936443, + "tokens_seen": 974323712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695165496489469, + "loss": 1.4052, + "theoretical_loss": 3.686698755776754, + "tokens_seen": 974389248 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695005015045135, + "loss": 1.5044, + "theoretical_loss": 3.686675228785412, + "tokens_seen": 974454784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694844533600803, + "loss": 1.2905, + "theoretical_loss": 3.686651703819307, + "tokens_seen": 974520320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694684052156469, + "loss": 1.3481, + "theoretical_loss": 3.686628180878129, + "tokens_seen": 974585856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694523570712137, + "loss": 1.408, + "theoretical_loss": 3.686604659961567, + "tokens_seen": 974651392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694363089267803, + "loss": 1.4492, + "theoretical_loss": 3.6865811410693117, + "tokens_seen": 974716928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694202607823471, + "loss": 1.3604, + "theoretical_loss": 3.6865576242010514, + "tokens_seen": 974782464 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694042126379137, + "loss": 1.3904, + "theoretical_loss": 3.6865341093564767, + "tokens_seen": 974848000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005693881644934805, + "loss": 1.2957, + "theoretical_loss": 3.6865105965352765, + "tokens_seen": 974913536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005693721163490471, + "loss": 1.3714, + "theoretical_loss": 3.6864870857371423, + "tokens_seen": 974979072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005693560682046139, + "loss": 1.3326, + "theoretical_loss": 3.6864635769617626, + "tokens_seen": 975044608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005693400200601806, + "loss": 1.5433, + "theoretical_loss": 3.6864400702088282, + "tokens_seen": 975110144 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005693239719157473, + "loss": 1.3617, + "theoretical_loss": 3.686416565478029, + "tokens_seen": 975175680 + }, + { + "epoch": 0.3, + "learning_rate": 0.000569307923771314, + "loss": 1.2906, + "theoretical_loss": 3.6863930627690555, + "tokens_seen": 975241216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692918756268807, + "loss": 1.4415, + "theoretical_loss": 3.6863695620815973, + "tokens_seen": 975306752 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692758274824474, + "loss": 1.504, + "theoretical_loss": 3.686346063415345, + "tokens_seen": 975372288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692597793380141, + "loss": 1.3098, + "theoretical_loss": 3.686322566769989, + "tokens_seen": 975437824 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692437311935808, + "loss": 1.2811, + "theoretical_loss": 3.6862990721452196, + "tokens_seen": 975503360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692276830491475, + "loss": 1.3407, + "theoretical_loss": 3.6862755795407276, + "tokens_seen": 975568896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692116349047142, + "loss": 1.3383, + "theoretical_loss": 3.686252088956203, + "tokens_seen": 975634432 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005691955867602809, + "loss": 1.4899, + "theoretical_loss": 3.6862286003913374, + "tokens_seen": 975699968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005691795386158476, + "loss": 1.3076, + "theoretical_loss": 3.686205113845821, + "tokens_seen": 975765504 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005691634904714143, + "loss": 1.4034, + "theoretical_loss": 3.686181629319344, + "tokens_seen": 975831040 + }, + { + "epoch": 0.3, + "learning_rate": 0.000569147442326981, + "loss": 1.3919, + "theoretical_loss": 3.6861581468115983, + "tokens_seen": 975896576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005691313941825477, + "loss": 1.3594, + "theoretical_loss": 3.6861346663222743, + "tokens_seen": 975962112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005691153460381144, + "loss": 1.2964, + "theoretical_loss": 3.686111187851063, + "tokens_seen": 976027648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690992978936811, + "loss": 1.5076, + "theoretical_loss": 3.686087711397655, + "tokens_seen": 976093184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690832497492478, + "loss": 1.3465, + "theoretical_loss": 3.6860642369617427, + "tokens_seen": 976158720 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690672016048145, + "loss": 1.3509, + "theoretical_loss": 3.686040764543016, + "tokens_seen": 976224256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690511534603812, + "loss": 1.2808, + "theoretical_loss": 3.686017294141167, + "tokens_seen": 976289792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690351053159479, + "loss": 1.3149, + "theoretical_loss": 3.6859938257558866, + "tokens_seen": 976355328 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690190571715146, + "loss": 1.3197, + "theoretical_loss": 3.6859703593868662, + "tokens_seen": 976420864 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 330391, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5027896165847778, + "objective/train/theoretical_loss": 3.6859468950337977, + "objective/train/tokens_used": 996946400, + "theoretical_loss": 3.6859468950337977, + "tokens_seen": 976486400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690030090270813, + "loss": 1.4294, + "theoretical_loss": 3.6859468950337977, + "tokens_seen": 976486400 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568986960882648, + "loss": 1.4013, + "theoretical_loss": 3.685923432696372, + "tokens_seen": 976551936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005689709127382147, + "loss": 1.4, + "theoretical_loss": 3.6858999723742807, + "tokens_seen": 976617472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005689548645937814, + "loss": 1.3802, + "theoretical_loss": 3.685876514067216, + "tokens_seen": 976683008 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005689388164493481, + "loss": 1.4382, + "theoretical_loss": 3.685853057774869, + "tokens_seen": 976748544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005689227683049148, + "loss": 1.3721, + "theoretical_loss": 3.6858296034969324, + "tokens_seen": 976814080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005689067201604815, + "loss": 1.4985, + "theoretical_loss": 3.6858061512330975, + "tokens_seen": 976879616 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005688906720160482, + "loss": 1.4508, + "theoretical_loss": 3.685782700983056, + "tokens_seen": 976945152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005688746238716149, + "loss": 1.3566, + "theoretical_loss": 3.6857592527465006, + "tokens_seen": 977010688 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005688585757271816, + "loss": 1.4136, + "theoretical_loss": 3.6857358065231227, + "tokens_seen": 977076224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005688425275827483, + "loss": 1.4871, + "theoretical_loss": 3.685712362312615, + "tokens_seen": 977141760 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568826479438315, + "loss": 1.2774, + "theoretical_loss": 3.6856889201146688, + "tokens_seen": 977207296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005688104312938817, + "loss": 1.3291, + "theoretical_loss": 3.6856654799289776, + "tokens_seen": 977272832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687943831494484, + "loss": 1.4611, + "theoretical_loss": 3.6856420417552336, + "tokens_seen": 977338368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687783350050151, + "loss": 1.3763, + "theoretical_loss": 3.685618605593128, + "tokens_seen": 977403904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687622868605818, + "loss": 1.4502, + "theoretical_loss": 3.6855951714423543, + "tokens_seen": 977469440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687462387161485, + "loss": 1.35, + "theoretical_loss": 3.6855717393026044, + "tokens_seen": 977534976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687301905717152, + "loss": 1.3829, + "theoretical_loss": 3.685548309173572, + "tokens_seen": 977600512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687141424272819, + "loss": 1.3952, + "theoretical_loss": 3.6855248810549486, + "tokens_seen": 977666048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686980942828486, + "loss": 1.4016, + "theoretical_loss": 3.6855014549464276, + "tokens_seen": 977731584 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686820461384153, + "loss": 1.4011, + "theoretical_loss": 3.685478030847702, + "tokens_seen": 977797120 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568665997993982, + "loss": 1.3656, + "theoretical_loss": 3.685454608758464, + "tokens_seen": 977862656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686499498495487, + "loss": 1.4097, + "theoretical_loss": 3.6854311886784066, + "tokens_seen": 977928192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686339017051154, + "loss": 1.4644, + "theoretical_loss": 3.685407770607224, + "tokens_seen": 977993728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686178535606821, + "loss": 1.353, + "theoretical_loss": 3.685384354544608, + "tokens_seen": 978059264 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686018054162487, + "loss": 1.3915, + "theoretical_loss": 3.685360940490252, + "tokens_seen": 978124800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685857572718155, + "loss": 1.3349, + "theoretical_loss": 3.685337528443849, + "tokens_seen": 978190336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685697091273821, + "loss": 1.3566, + "theoretical_loss": 3.6853141184050937, + "tokens_seen": 978255872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685536609829489, + "loss": 1.4095, + "theoretical_loss": 3.6852907103736774, + "tokens_seen": 978321408 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685376128385155, + "loss": 1.403, + "theoretical_loss": 3.6852673043492956, + "tokens_seen": 978386944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685215646940823, + "loss": 1.2823, + "theoretical_loss": 3.68524390033164, + "tokens_seen": 978452480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685055165496489, + "loss": 1.3668, + "theoretical_loss": 3.6852204983204055, + "tokens_seen": 978518016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005684894684052157, + "loss": 1.3809, + "theoretical_loss": 3.6851970983152844, + "tokens_seen": 978583552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005684734202607823, + "loss": 1.3704, + "theoretical_loss": 3.685173700315972, + "tokens_seen": 978649088 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568457372116349, + "loss": 1.3677, + "theoretical_loss": 3.6851503043221605, + "tokens_seen": 978714624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005684413239719157, + "loss": 1.4072, + "theoretical_loss": 3.685126910333545, + "tokens_seen": 978780160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005684252758274824, + "loss": 1.3737, + "theoretical_loss": 3.6851035183498184, + "tokens_seen": 978845696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005684092276830491, + "loss": 1.3461, + "theoretical_loss": 3.6850801283706756, + "tokens_seen": 978911232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005683931795386158, + "loss": 1.3565, + "theoretical_loss": 3.68505674039581, + "tokens_seen": 978976768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005683771313941825, + "loss": 1.3611, + "theoretical_loss": 3.6850333544249154, + "tokens_seen": 979042304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005683610832497492, + "loss": 1.4217, + "theoretical_loss": 3.6850099704576866, + "tokens_seen": 979107840 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568345035105316, + "loss": 1.332, + "theoretical_loss": 3.684986588493818, + "tokens_seen": 979173376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005683289869608826, + "loss": 1.2867, + "theoretical_loss": 3.684963208533003, + "tokens_seen": 979238912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005683129388164493, + "loss": 1.4204, + "theoretical_loss": 3.684939830574937, + "tokens_seen": 979304448 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568296890672016, + "loss": 1.3366, + "theoretical_loss": 3.6849164546193136, + "tokens_seen": 979369984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682808425275827, + "loss": 1.3858, + "theoretical_loss": 3.6848930806658275, + "tokens_seen": 979435520 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682647943831494, + "loss": 1.4454, + "theoretical_loss": 3.684869708714174, + "tokens_seen": 979501056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682487462387162, + "loss": 1.3815, + "theoretical_loss": 3.684846338764047, + "tokens_seen": 979566592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682326980942828, + "loss": 1.3057, + "theoretical_loss": 3.6848229708151408, + "tokens_seen": 979632128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682166499498496, + "loss": 1.3947, + "theoretical_loss": 3.684799604867151, + "tokens_seen": 979697664 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 331668, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3969978094100952, + "objective/train/theoretical_loss": 3.6847762409197724, + "objective/train/tokens_used": 1000223200, + "theoretical_loss": 3.6847762409197724, + "tokens_seen": 979763200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682006018054162, + "loss": 1.2919, + "theoretical_loss": 3.6847762409197724, + "tokens_seen": 979763200 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568184553660983, + "loss": 1.3791, + "theoretical_loss": 3.6847528789726995, + "tokens_seen": 979828736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005681685055165496, + "loss": 1.3665, + "theoretical_loss": 3.6847295190256273, + "tokens_seen": 979894272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005681524573721164, + "loss": 1.3441, + "theoretical_loss": 3.684706161078251, + "tokens_seen": 979959808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005681364092276831, + "loss": 1.3108, + "theoretical_loss": 3.6846828051302656, + "tokens_seen": 980025344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005681203610832498, + "loss": 1.3302, + "theoretical_loss": 3.6846594511813664, + "tokens_seen": 980090880 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005681043129388165, + "loss": 1.4192, + "theoretical_loss": 3.6846360992312484, + "tokens_seen": 980156416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005680882647943832, + "loss": 1.2893, + "theoretical_loss": 3.6846127492796077, + "tokens_seen": 980221952 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005680722166499499, + "loss": 1.3537, + "theoretical_loss": 3.6845894013261384, + "tokens_seen": 980287488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005680561685055166, + "loss": 1.3803, + "theoretical_loss": 3.684566055370537, + "tokens_seen": 980353024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005680401203610833, + "loss": 1.3628, + "theoretical_loss": 3.6845427114124982, + "tokens_seen": 980418560 + }, + { + "epoch": 0.3, + "learning_rate": 0.00056802407221665, + "loss": 1.4395, + "theoretical_loss": 3.684519369451718, + "tokens_seen": 980484096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005680080240722167, + "loss": 1.3303, + "theoretical_loss": 3.6844960294878923, + "tokens_seen": 980549632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679919759277834, + "loss": 1.2955, + "theoretical_loss": 3.684472691520716, + "tokens_seen": 980615168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679759277833501, + "loss": 1.4059, + "theoretical_loss": 3.684449355549886, + "tokens_seen": 980680704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679598796389168, + "loss": 1.3102, + "theoretical_loss": 3.6844260215750975, + "tokens_seen": 980746240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679438314944835, + "loss": 1.4076, + "theoretical_loss": 3.684402689596046, + "tokens_seen": 980811776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679277833500502, + "loss": 1.2741, + "theoretical_loss": 3.6843793596124277, + "tokens_seen": 980877312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679117352056169, + "loss": 1.3524, + "theoretical_loss": 3.684356031623939, + "tokens_seen": 980942848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005678956870611836, + "loss": 1.3458, + "theoretical_loss": 3.6843327056302764, + "tokens_seen": 981008384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005678796389167503, + "loss": 1.5072, + "theoretical_loss": 3.6843093816311354, + "tokens_seen": 981073920 + }, + { + "epoch": 0.3, + "learning_rate": 0.000567863590772317, + "loss": 1.3678, + "theoretical_loss": 3.6842860596262117, + "tokens_seen": 981139456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005678475426278837, + "loss": 1.4274, + "theoretical_loss": 3.6842627396152023, + "tokens_seen": 981204992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005678314944834504, + "loss": 1.3302, + "theoretical_loss": 3.684239421597804, + "tokens_seen": 981270528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005678154463390171, + "loss": 1.3208, + "theoretical_loss": 3.6842161055737126, + "tokens_seen": 981336064 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677993981945838, + "loss": 1.2496, + "theoretical_loss": 3.6841927915426247, + "tokens_seen": 981401600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677833500501505, + "loss": 1.4826, + "theoretical_loss": 3.684169479504237, + "tokens_seen": 981467136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677673019057172, + "loss": 1.3356, + "theoretical_loss": 3.684146169458246, + "tokens_seen": 981532672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677512537612839, + "loss": 1.3267, + "theoretical_loss": 3.6841228614043486, + "tokens_seen": 981598208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677352056168506, + "loss": 1.3417, + "theoretical_loss": 3.6840995553422413, + "tokens_seen": 981663744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677191574724173, + "loss": 1.2657, + "theoretical_loss": 3.6840762512716214, + "tokens_seen": 981729280 + }, + { + "epoch": 0.3, + "learning_rate": 0.000567703109327984, + "loss": 1.4828, + "theoretical_loss": 3.6840529491921847, + "tokens_seen": 981794816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676870611835507, + "loss": 1.4698, + "theoretical_loss": 3.68402964910363, + "tokens_seen": 981860352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676710130391174, + "loss": 1.4686, + "theoretical_loss": 3.6840063510056527, + "tokens_seen": 981925888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676549648946841, + "loss": 1.416, + "theoretical_loss": 3.6839830548979506, + "tokens_seen": 981991424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676389167502508, + "loss": 1.2738, + "theoretical_loss": 3.683959760780221, + "tokens_seen": 982056960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676228686058175, + "loss": 1.3382, + "theoretical_loss": 3.68393646865216, + "tokens_seen": 982122496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676068204613842, + "loss": 1.3057, + "theoretical_loss": 3.6839131785134667, + "tokens_seen": 982188032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005675907723169509, + "loss": 1.4602, + "theoretical_loss": 3.683889890363837, + "tokens_seen": 982253568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005675747241725176, + "loss": 1.3691, + "theoretical_loss": 3.6838666042029695, + "tokens_seen": 982319104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005675586760280843, + "loss": 1.3261, + "theoretical_loss": 3.6838433200305607, + "tokens_seen": 982384640 + }, + { + "epoch": 0.3, + "learning_rate": 0.000567542627883651, + "loss": 1.3717, + "theoretical_loss": 3.683820037846308, + "tokens_seen": 982450176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005675265797392177, + "loss": 1.297, + "theoretical_loss": 3.6837967576499104, + "tokens_seen": 982515712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005675105315947844, + "loss": 1.3108, + "theoretical_loss": 3.6837734794410646, + "tokens_seen": 982581248 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005674944834503511, + "loss": 1.3577, + "theoretical_loss": 3.6837502032194687, + "tokens_seen": 982646784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005674784353059178, + "loss": 1.3823, + "theoretical_loss": 3.68372692898482, + "tokens_seen": 982712320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005674623871614845, + "loss": 1.3033, + "theoretical_loss": 3.6837036567368173, + "tokens_seen": 982777856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005674463390170512, + "loss": 1.3632, + "theoretical_loss": 3.6836803864751575, + "tokens_seen": 982843392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005674302908726179, + "loss": 1.2767, + "theoretical_loss": 3.6836571181995392, + "tokens_seen": 982908928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005674142427281846, + "loss": 1.3832, + "theoretical_loss": 3.6836338519096605, + "tokens_seen": 982974464 + }, + { + "debugging/Compilability": 0.0, + "debugging/distinct-1-grams": 0.7061842754091193, + "debugging/entropy-1-grams": 4.557400890096247, + "debugging/length": 580.2, + "debugging/num_segments": 5, + "debugging/score": 0.06456836857494044, + "debugging/score_std": 0.019067822519855632, + "epoch": 0.3, + "objective/train/docs_used": 332958, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4429404735565186, + "objective/train/theoretical_loss": 3.68361058760522, + "objective/train/tokens_used": 1003500000, + "theoretical_loss": 3.68361058760522, + "tokens_seen": 983040000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005673981945837513, + "loss": 1.3946, + "theoretical_loss": 3.68361058760522, + "tokens_seen": 983040000 + }, + { + "epoch": 0.3, + "learning_rate": 0.000567382146439318, + "loss": 1.2636, + "theoretical_loss": 3.683587325285915, + "tokens_seen": 983105536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005673660982948847, + "loss": 1.402, + "theoretical_loss": 3.6835640649514443, + "tokens_seen": 983171072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005673500501504514, + "loss": 1.3532, + "theoretical_loss": 3.6835408066015063, + "tokens_seen": 983236608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005673340020060181, + "loss": 1.3186, + "theoretical_loss": 3.6835175502357993, + "tokens_seen": 983302144 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005673179538615848, + "loss": 1.3792, + "theoretical_loss": 3.6834942958540218, + "tokens_seen": 983367680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005673019057171515, + "loss": 1.4187, + "theoretical_loss": 3.683471043455872, + "tokens_seen": 983433216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005672858575727182, + "loss": 1.3978, + "theoretical_loss": 3.6834477930410494, + "tokens_seen": 983498752 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005672698094282849, + "loss": 1.277, + "theoretical_loss": 3.683424544609252, + "tokens_seen": 983564288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005672537612838516, + "loss": 1.2866, + "theoretical_loss": 3.6834012981601787, + "tokens_seen": 983629824 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005672377131394183, + "loss": 1.4135, + "theoretical_loss": 3.683378053693528, + "tokens_seen": 983695360 + }, + { + "epoch": 0.3, + "learning_rate": 0.000567221664994985, + "loss": 1.3519, + "theoretical_loss": 3.683354811209, + "tokens_seen": 983760896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005672056168505516, + "loss": 1.4097, + "theoretical_loss": 3.6833315707062924, + "tokens_seen": 983826432 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005671895687061184, + "loss": 1.4026, + "theoretical_loss": 3.6833083321851046, + "tokens_seen": 983891968 + }, + { + "epoch": 0.3, + "learning_rate": 0.000567173520561685, + "loss": 1.4019, + "theoretical_loss": 3.6832850956451355, + "tokens_seen": 983957504 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005671574724172518, + "loss": 1.344, + "theoretical_loss": 3.6832618610860846, + "tokens_seen": 984023040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005671414242728184, + "loss": 1.3308, + "theoretical_loss": 3.6832386285076506, + "tokens_seen": 984088576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005671253761283852, + "loss": 1.3346, + "theoretical_loss": 3.6832153979095335, + "tokens_seen": 984154112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005671093279839518, + "loss": 1.3408, + "theoretical_loss": 3.6831921692914324, + "tokens_seen": 984219648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005670932798395186, + "loss": 1.4718, + "theoretical_loss": 3.6831689426530465, + "tokens_seen": 984285184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005670772316950852, + "loss": 1.365, + "theoretical_loss": 3.683145717994075, + "tokens_seen": 984350720 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005670611835506519, + "loss": 1.2906, + "theoretical_loss": 3.683122495314218, + "tokens_seen": 984416256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005670451354062186, + "loss": 1.3021, + "theoretical_loss": 3.683099274613175, + "tokens_seen": 984481792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005670290872617853, + "loss": 1.4236, + "theoretical_loss": 3.6830760558906457, + "tokens_seen": 984547328 + }, + { + "epoch": 0.3, + "learning_rate": 0.000567013039117352, + "loss": 1.377, + "theoretical_loss": 3.6830528391463293, + "tokens_seen": 984612864 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005669969909729187, + "loss": 1.3042, + "theoretical_loss": 3.6830296243799263, + "tokens_seen": 984678400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005669809428284854, + "loss": 1.3744, + "theoretical_loss": 3.6830064115911365, + "tokens_seen": 984743936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005669648946840522, + "loss": 1.3447, + "theoretical_loss": 3.6829832007796597, + "tokens_seen": 984809472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005669488465396188, + "loss": 1.4428, + "theoretical_loss": 3.6829599919451956, + "tokens_seen": 984875008 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005669327983951856, + "loss": 1.331, + "theoretical_loss": 3.682936785087444, + "tokens_seen": 984940544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005669167502507522, + "loss": 1.393, + "theoretical_loss": 3.6829135802061064, + "tokens_seen": 985006080 + }, + { + "epoch": 0.3, + "learning_rate": 0.000566900702106319, + "loss": 1.3394, + "theoretical_loss": 3.6828903773008816, + "tokens_seen": 985071616 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005668846539618856, + "loss": 1.4251, + "theoretical_loss": 3.6828671763714707, + "tokens_seen": 985137152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005668686058174524, + "loss": 1.4133, + "theoretical_loss": 3.6828439774175736, + "tokens_seen": 985202688 + }, + { + "epoch": 0.3, + "learning_rate": 0.000566852557673019, + "loss": 1.433, + "theoretical_loss": 3.682820780438891, + "tokens_seen": 985268224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005668365095285858, + "loss": 1.4331, + "theoretical_loss": 3.6827975854351225, + "tokens_seen": 985333760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005668204613841525, + "loss": 1.3828, + "theoretical_loss": 3.68277439240597, + "tokens_seen": 985399296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005668044132397192, + "loss": 1.3815, + "theoretical_loss": 3.682751201351133, + "tokens_seen": 985464832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005667883650952859, + "loss": 1.3695, + "theoretical_loss": 3.6827280122703128, + "tokens_seen": 985530368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005667723169508526, + "loss": 1.3144, + "theoretical_loss": 3.68270482516321, + "tokens_seen": 985595904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005667562688064193, + "loss": 1.4188, + "theoretical_loss": 3.682681640029525, + "tokens_seen": 985661440 + }, + { + "epoch": 0.3, + "learning_rate": 0.000566740220661986, + "loss": 1.3489, + "theoretical_loss": 3.6826584568689587, + "tokens_seen": 985726976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005667241725175527, + "loss": 1.4191, + "theoretical_loss": 3.6826352756812124, + "tokens_seen": 985792512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005667081243731194, + "loss": 1.3455, + "theoretical_loss": 3.6826120964659874, + "tokens_seen": 985858048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005666920762286861, + "loss": 1.3907, + "theoretical_loss": 3.682588919222984, + "tokens_seen": 985923584 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005666760280842528, + "loss": 1.4821, + "theoretical_loss": 3.682565743951903, + "tokens_seen": 985989120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005666599799398195, + "loss": 1.3521, + "theoretical_loss": 3.682542570652447, + "tokens_seen": 986054656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005666439317953862, + "loss": 1.489, + "theoretical_loss": 3.6825193993243155, + "tokens_seen": 986120192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005666278836509529, + "loss": 1.3119, + "theoretical_loss": 3.682496229967211, + "tokens_seen": 986185728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005666118355065196, + "loss": 1.3445, + "theoretical_loss": 3.6824730625808346, + "tokens_seen": 986251264 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 333817, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.331803798675537, + "objective/train/theoretical_loss": 3.6824498971648882, + "objective/train/tokens_used": 1006776800, + "theoretical_loss": 3.6824498971648882, + "tokens_seen": 986316800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005665957873620863, + "loss": 1.3384, + "theoretical_loss": 3.6824498971648882, + "tokens_seen": 986316800 + }, + { + "epoch": 0.3, + "learning_rate": 0.000566579739217653, + "loss": 1.4972, + "theoretical_loss": 3.682426733719072, + "tokens_seen": 986382336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005665636910732197, + "loss": 1.4241, + "theoretical_loss": 3.6824035722430892, + "tokens_seen": 986447872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005665476429287864, + "loss": 1.3828, + "theoretical_loss": 3.6823804127366397, + "tokens_seen": 986513408 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005665315947843531, + "loss": 1.369, + "theoretical_loss": 3.6823572551994266, + "tokens_seen": 986578944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005665155466399198, + "loss": 1.3784, + "theoretical_loss": 3.6823340996311513, + "tokens_seen": 986644480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005664994984954865, + "loss": 1.3649, + "theoretical_loss": 3.682310946031515, + "tokens_seen": 986710016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005664834503510532, + "loss": 1.4104, + "theoretical_loss": 3.6822877944002204, + "tokens_seen": 986775552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005664674022066199, + "loss": 1.3924, + "theoretical_loss": 3.682264644736969, + "tokens_seen": 986841088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005664513540621866, + "loss": 1.3395, + "theoretical_loss": 3.6822414970414634, + "tokens_seen": 986906624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005664353059177533, + "loss": 1.355, + "theoretical_loss": 3.682218351313405, + "tokens_seen": 986972160 + }, + { + "epoch": 0.3, + "learning_rate": 0.00056641925777332, + "loss": 1.3924, + "theoretical_loss": 3.6821952075524957, + "tokens_seen": 987037696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005664032096288867, + "loss": 1.4015, + "theoretical_loss": 3.6821720657584387, + "tokens_seen": 987103232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005663871614844534, + "loss": 1.404, + "theoretical_loss": 3.6821489259309357, + "tokens_seen": 987168768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005663711133400201, + "loss": 1.2869, + "theoretical_loss": 3.682125788069689, + "tokens_seen": 987234304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005663550651955868, + "loss": 1.4381, + "theoretical_loss": 3.682102652174401, + "tokens_seen": 987299840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005663390170511535, + "loss": 1.4067, + "theoretical_loss": 3.682079518244775, + "tokens_seen": 987365376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005663229689067202, + "loss": 1.3896, + "theoretical_loss": 3.6820563862805127, + "tokens_seen": 987430912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005663069207622869, + "loss": 1.3642, + "theoretical_loss": 3.6820332562813167, + "tokens_seen": 987496448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005662908726178536, + "loss": 1.4163, + "theoretical_loss": 3.6820101282468896, + "tokens_seen": 987561984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005662748244734203, + "loss": 1.3182, + "theoretical_loss": 3.6819870021769345, + "tokens_seen": 987627520 + }, + { + "epoch": 0.3, + "learning_rate": 0.000566258776328987, + "loss": 1.3719, + "theoretical_loss": 3.6819638780711546, + "tokens_seen": 987693056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005662427281845537, + "loss": 1.3108, + "theoretical_loss": 3.6819407559292516, + "tokens_seen": 987758592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005662266800401204, + "loss": 1.3835, + "theoretical_loss": 3.6819176357509296, + "tokens_seen": 987824128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005662106318956871, + "loss": 1.2571, + "theoretical_loss": 3.681894517535891, + "tokens_seen": 987889664 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005661945837512538, + "loss": 1.2335, + "theoretical_loss": 3.6818714012838383, + "tokens_seen": 987955200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005661785356068205, + "loss": 1.3365, + "theoretical_loss": 3.6818482869944757, + "tokens_seen": 988020736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005661624874623872, + "loss": 1.3487, + "theoretical_loss": 3.681825174667506, + "tokens_seen": 988086272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005661464393179539, + "loss": 1.3836, + "theoretical_loss": 3.6818020643026323, + "tokens_seen": 988151808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005661303911735206, + "loss": 1.3399, + "theoretical_loss": 3.681778955899558, + "tokens_seen": 988217344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005661143430290873, + "loss": 1.3863, + "theoretical_loss": 3.6817558494579865, + "tokens_seen": 988282880 + }, + { + "epoch": 0.3, + "learning_rate": 0.000566098294884654, + "loss": 1.4341, + "theoretical_loss": 3.681732744977621, + "tokens_seen": 988348416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005660822467402207, + "loss": 1.3173, + "theoretical_loss": 3.681709642458166, + "tokens_seen": 988413952 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005660661985957874, + "loss": 1.2942, + "theoretical_loss": 3.6816865418993237, + "tokens_seen": 988479488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005660501504513541, + "loss": 1.3678, + "theoretical_loss": 3.681663443300798, + "tokens_seen": 988545024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005660341023069208, + "loss": 1.2993, + "theoretical_loss": 3.6816403466622942, + "tokens_seen": 988610560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005660180541624875, + "loss": 1.322, + "theoretical_loss": 3.681617251983514, + "tokens_seen": 988676096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005660020060180542, + "loss": 1.3447, + "theoretical_loss": 3.681594159264162, + "tokens_seen": 988741632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005659859578736209, + "loss": 1.3304, + "theoretical_loss": 3.681571068503942, + "tokens_seen": 988807168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005659699097291876, + "loss": 1.3252, + "theoretical_loss": 3.6815479797025588, + "tokens_seen": 988872704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005659538615847543, + "loss": 1.3822, + "theoretical_loss": 3.6815248928597155, + "tokens_seen": 988938240 + }, + { + "epoch": 0.3, + "learning_rate": 0.000565937813440321, + "loss": 1.3233, + "theoretical_loss": 3.681501807975116, + "tokens_seen": 989003776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005659217652958877, + "loss": 1.3444, + "theoretical_loss": 3.681478725048465, + "tokens_seen": 989069312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005659057171514544, + "loss": 1.3964, + "theoretical_loss": 3.6814556440794672, + "tokens_seen": 989134848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005658896690070211, + "loss": 1.3694, + "theoretical_loss": 3.681432565067826, + "tokens_seen": 989200384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005658736208625878, + "loss": 1.407, + "theoretical_loss": 3.681409488013246, + "tokens_seen": 989265920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005658575727181545, + "loss": 1.225, + "theoretical_loss": 3.6813864129154314, + "tokens_seen": 989331456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005658415245737212, + "loss": 1.4289, + "theoretical_loss": 3.6813633397740873, + "tokens_seen": 989396992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005658254764292879, + "loss": 1.3986, + "theoretical_loss": 3.681340268588918, + "tokens_seen": 989462528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005658094282848546, + "loss": 1.367, + "theoretical_loss": 3.6813171993596274, + "tokens_seen": 989528064 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 334953, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3690037727355957, + "objective/train/theoretical_loss": 3.681294132085921, + "objective/train/tokens_used": 1010053600, + "theoretical_loss": 3.681294132085921, + "tokens_seen": 989593600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005657933801404213, + "loss": 1.3338, + "theoretical_loss": 3.681294132085921, + "tokens_seen": 989593600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005657773319959879, + "loss": 1.4075, + "theoretical_loss": 3.6812710667675033, + "tokens_seen": 989659136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005657612838515547, + "loss": 1.3119, + "theoretical_loss": 3.6812480034040793, + "tokens_seen": 989724672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005657452357071213, + "loss": 1.3045, + "theoretical_loss": 3.681224941995353, + "tokens_seen": 989790208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005657291875626881, + "loss": 1.3002, + "theoretical_loss": 3.681201882541031, + "tokens_seen": 989855744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005657131394182547, + "loss": 1.3496, + "theoretical_loss": 3.681178825040816, + "tokens_seen": 989921280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005656970912738215, + "loss": 1.3146, + "theoretical_loss": 3.6811557694944153, + "tokens_seen": 989986816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005656810431293881, + "loss": 1.3166, + "theoretical_loss": 3.6811327159015326, + "tokens_seen": 990052352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005656649949849549, + "loss": 1.3282, + "theoretical_loss": 3.6811096642618732, + "tokens_seen": 990117888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005656489468405215, + "loss": 1.4342, + "theoretical_loss": 3.6810866145751433, + "tokens_seen": 990183424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005656328986960883, + "loss": 1.2805, + "theoretical_loss": 3.6810635668410474, + "tokens_seen": 990248960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005656168505516549, + "loss": 1.4442, + "theoretical_loss": 3.6810405210592907, + "tokens_seen": 990314496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005656008024072217, + "loss": 1.3321, + "theoretical_loss": 3.681017477229579, + "tokens_seen": 990380032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005655847542627883, + "loss": 1.306, + "theoretical_loss": 3.680994435351618, + "tokens_seen": 990445568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005655687061183551, + "loss": 1.286, + "theoretical_loss": 3.6809713954251126, + "tokens_seen": 990511104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005655526579739217, + "loss": 1.3902, + "theoretical_loss": 3.6809483574497692, + "tokens_seen": 990576640 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005655366098294885, + "loss": 1.3559, + "theoretical_loss": 3.6809253214252933, + "tokens_seen": 990642176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005655205616850551, + "loss": 1.3252, + "theoretical_loss": 3.6809022873513904, + "tokens_seen": 990707712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005655045135406219, + "loss": 1.351, + "theoretical_loss": 3.680879255227766, + "tokens_seen": 990773248 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005654884653961886, + "loss": 1.3565, + "theoretical_loss": 3.680856225054127, + "tokens_seen": 990838784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005654724172517553, + "loss": 1.388, + "theoretical_loss": 3.680833196830178, + "tokens_seen": 990904320 + }, + { + "epoch": 0.3, + "learning_rate": 0.000565456369107322, + "loss": 1.3968, + "theoretical_loss": 3.6808101705556266, + "tokens_seen": 990969856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005654403209628887, + "loss": 1.3266, + "theoretical_loss": 3.6807871462301778, + "tokens_seen": 991035392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005654242728184554, + "loss": 1.3703, + "theoretical_loss": 3.6807641238535376, + "tokens_seen": 991100928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005654082246740221, + "loss": 1.2335, + "theoretical_loss": 3.6807411034254125, + "tokens_seen": 991166464 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005653921765295888, + "loss": 1.4172, + "theoretical_loss": 3.6807180849455094, + "tokens_seen": 991232000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005653761283851555, + "loss": 1.334, + "theoretical_loss": 3.680695068413534, + "tokens_seen": 991297536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005653600802407222, + "loss": 1.3617, + "theoretical_loss": 3.680672053829192, + "tokens_seen": 991363072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005653440320962889, + "loss": 1.3674, + "theoretical_loss": 3.6806490411921917, + "tokens_seen": 991428608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005653279839518556, + "loss": 1.3773, + "theoretical_loss": 3.6806260305022382, + "tokens_seen": 991494144 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005653119358074223, + "loss": 1.3921, + "theoretical_loss": 3.680603021759038, + "tokens_seen": 991559680 + }, + { + "epoch": 0.3, + "learning_rate": 0.000565295887662989, + "loss": 1.4225, + "theoretical_loss": 3.680580014962299, + "tokens_seen": 991625216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005652798395185557, + "loss": 1.3689, + "theoretical_loss": 3.680557010111726, + "tokens_seen": 991690752 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005652637913741224, + "loss": 1.3267, + "theoretical_loss": 3.6805340072070276, + "tokens_seen": 991756288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005652477432296891, + "loss": 1.2849, + "theoretical_loss": 3.68051100624791, + "tokens_seen": 991821824 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005652316950852558, + "loss": 1.2479, + "theoretical_loss": 3.6804880072340795, + "tokens_seen": 991887360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005652156469408225, + "loss": 1.283, + "theoretical_loss": 3.680465010165244, + "tokens_seen": 991952896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005651995987963892, + "loss": 1.3904, + "theoretical_loss": 3.68044201504111, + "tokens_seen": 992018432 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005651835506519559, + "loss": 1.3015, + "theoretical_loss": 3.680419021861385, + "tokens_seen": 992083968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005651675025075226, + "loss": 1.3007, + "theoretical_loss": 3.6803960306257752, + "tokens_seen": 992149504 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005651514543630893, + "loss": 1.4119, + "theoretical_loss": 3.680373041333989, + "tokens_seen": 992215040 + }, + { + "epoch": 0.3, + "learning_rate": 0.000565135406218656, + "loss": 1.3537, + "theoretical_loss": 3.680350053985733, + "tokens_seen": 992280576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005651193580742227, + "loss": 1.3171, + "theoretical_loss": 3.6803270685807146, + "tokens_seen": 992346112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005651033099297894, + "loss": 1.292, + "theoretical_loss": 3.680304085118641, + "tokens_seen": 992411648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005650872617853561, + "loss": 1.3236, + "theoretical_loss": 3.6802811035992207, + "tokens_seen": 992477184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005650712136409228, + "loss": 1.3134, + "theoretical_loss": 3.68025812402216, + "tokens_seen": 992542720 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005650551654964895, + "loss": 1.3056, + "theoretical_loss": 3.680235146387167, + "tokens_seen": 992608256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005650391173520562, + "loss": 1.3597, + "theoretical_loss": 3.6802121706939497, + "tokens_seen": 992673792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005650230692076229, + "loss": 1.4204, + "theoretical_loss": 3.680189196942216, + "tokens_seen": 992739328 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005650070210631896, + "loss": 1.3497, + "theoretical_loss": 3.6801662251316722, + "tokens_seen": 992804864 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 335764, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3931187391281128, + "objective/train/theoretical_loss": 3.6801432552620277, + "objective/train/tokens_used": 1013330400, + "theoretical_loss": 3.6801432552620277, + "tokens_seen": 992870400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005649909729187563, + "loss": 1.4383, + "theoretical_loss": 3.6801432552620277, + "tokens_seen": 992870400 + }, + { + "epoch": 0.3, + "learning_rate": 0.000564974924774323, + "loss": 1.422, + "theoretical_loss": 3.68012028733299, + "tokens_seen": 992935936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005649588766298897, + "loss": 1.4616, + "theoretical_loss": 3.6800973213442667, + "tokens_seen": 993001472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005649428284854564, + "loss": 1.2986, + "theoretical_loss": 3.680074357295566, + "tokens_seen": 993067008 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005649267803410231, + "loss": 1.4569, + "theoretical_loss": 3.680051395186596, + "tokens_seen": 993132544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005649107321965898, + "loss": 1.2824, + "theoretical_loss": 3.680028435017065, + "tokens_seen": 993198080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005648946840521565, + "loss": 1.2645, + "theoretical_loss": 3.6800054767866817, + "tokens_seen": 993263616 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005648786359077232, + "loss": 1.2743, + "theoretical_loss": 3.6799825204951535, + "tokens_seen": 993329152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005648625877632899, + "loss": 1.3661, + "theoretical_loss": 3.6799595661421893, + "tokens_seen": 993394688 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005648465396188566, + "loss": 1.3528, + "theoretical_loss": 3.679936613727498, + "tokens_seen": 993460224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005648304914744233, + "loss": 1.3753, + "theoretical_loss": 3.6799136632507867, + "tokens_seen": 993525760 + }, + { + "epoch": 0.3, + "learning_rate": 0.00056481444332999, + "loss": 1.3209, + "theoretical_loss": 3.6798907147117648, + "tokens_seen": 993591296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005647983951855567, + "loss": 1.2764, + "theoretical_loss": 3.6798677681101406, + "tokens_seen": 993656832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005647823470411234, + "loss": 1.2637, + "theoretical_loss": 3.6798448234456234, + "tokens_seen": 993722368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005647662988966901, + "loss": 1.3528, + "theoretical_loss": 3.6798218807179213, + "tokens_seen": 993787904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005647502507522568, + "loss": 1.3155, + "theoretical_loss": 3.6797989399267435, + "tokens_seen": 993853440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005647342026078235, + "loss": 1.2588, + "theoretical_loss": 3.679776001071799, + "tokens_seen": 993918976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005647181544633902, + "loss": 1.2973, + "theoretical_loss": 3.679753064152796, + "tokens_seen": 993984512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005647021063189569, + "loss": 1.3545, + "theoretical_loss": 3.679730129169444, + "tokens_seen": 994050048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005646860581745236, + "loss": 1.2825, + "theoretical_loss": 3.679707196121452, + "tokens_seen": 994115584 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005646700100300903, + "loss": 1.372, + "theoretical_loss": 3.6796842650085293, + "tokens_seen": 994181120 + }, + { + "epoch": 0.3, + "learning_rate": 0.000564653961885657, + "loss": 1.3828, + "theoretical_loss": 3.6796613358303842, + "tokens_seen": 994246656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005646379137412237, + "loss": 1.3242, + "theoretical_loss": 3.6796384085867277, + "tokens_seen": 994312192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005646218655967904, + "loss": 1.3462, + "theoretical_loss": 3.679615483277267, + "tokens_seen": 994377728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005646058174523571, + "loss": 1.3769, + "theoretical_loss": 3.679592559901713, + "tokens_seen": 994443264 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005645897693079238, + "loss": 1.3495, + "theoretical_loss": 3.6795696384597747, + "tokens_seen": 994508800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005645737211634905, + "loss": 1.3529, + "theoretical_loss": 3.679546718951161, + "tokens_seen": 994574336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005645576730190573, + "loss": 1.3009, + "theoretical_loss": 3.6795238013755824, + "tokens_seen": 994639872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005645416248746239, + "loss": 1.3876, + "theoretical_loss": 3.6795008857327476, + "tokens_seen": 994705408 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005645255767301906, + "loss": 1.3747, + "theoretical_loss": 3.6794779720223674, + "tokens_seen": 994770944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005645095285857573, + "loss": 1.3314, + "theoretical_loss": 3.6794550602441505, + "tokens_seen": 994836480 + }, + { + "epoch": 0.3, + "learning_rate": 0.000564493480441324, + "loss": 1.3172, + "theoretical_loss": 3.6794321503978074, + "tokens_seen": 994902016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005644774322968907, + "loss": 1.3119, + "theoretical_loss": 3.679409242483047, + "tokens_seen": 994967552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005644613841524574, + "loss": 1.3156, + "theoretical_loss": 3.6793863364995802, + "tokens_seen": 995033088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005644453360080241, + "loss": 1.382, + "theoretical_loss": 3.6793634324471167, + "tokens_seen": 995098624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005644292878635908, + "loss": 1.4278, + "theoretical_loss": 3.6793405303253675, + "tokens_seen": 995164160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005644132397191575, + "loss": 1.3369, + "theoretical_loss": 3.6793176301340407, + "tokens_seen": 995229696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005643971915747242, + "loss": 1.3507, + "theoretical_loss": 3.6792947318728473, + "tokens_seen": 995295232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005643811434302908, + "loss": 1.3575, + "theoretical_loss": 3.679271835541499, + "tokens_seen": 995360768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005643650952858576, + "loss": 1.3269, + "theoretical_loss": 3.6792489411397042, + "tokens_seen": 995426304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005643490471414242, + "loss": 1.3141, + "theoretical_loss": 3.6792260486671737, + "tokens_seen": 995491840 + }, + { + "epoch": 0.3, + "learning_rate": 0.000564332998996991, + "loss": 1.291, + "theoretical_loss": 3.6792031581236184, + "tokens_seen": 995557376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005643169508525576, + "loss": 1.3886, + "theoretical_loss": 3.6791802695087483, + "tokens_seen": 995622912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005643009027081244, + "loss": 1.3119, + "theoretical_loss": 3.6791573828222752, + "tokens_seen": 995688448 + }, + { + "epoch": 0.3, + "learning_rate": 0.000564284854563691, + "loss": 1.2696, + "theoretical_loss": 3.679134498063908, + "tokens_seen": 995753984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005642688064192578, + "loss": 1.276, + "theoretical_loss": 3.6791116152333583, + "tokens_seen": 995819520 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005642527582748244, + "loss": 1.3491, + "theoretical_loss": 3.6790887343303367, + "tokens_seen": 995885056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005642367101303912, + "loss": 1.3067, + "theoretical_loss": 3.679065855354554, + "tokens_seen": 995950592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005642206619859578, + "loss": 1.3119, + "theoretical_loss": 3.6790429783057212, + "tokens_seen": 996016128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005642046138415246, + "loss": 1.2713, + "theoretical_loss": 3.679020103183549, + "tokens_seen": 996081664 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 336954, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1566932201385498, + "objective/train/theoretical_loss": 3.6789972299877487, + "objective/train/tokens_used": 1016607200, + "theoretical_loss": 3.6789972299877487, + "tokens_seen": 996147200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005641885656970912, + "loss": 1.2733, + "theoretical_loss": 3.6789972299877487, + "tokens_seen": 996147200 + }, + { + "epoch": 0.3, + "learning_rate": 0.000564172517552658, + "loss": 1.4281, + "theoretical_loss": 3.678974358718031, + "tokens_seen": 996212736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005641564694082246, + "loss": 1.4024, + "theoretical_loss": 3.678951489374107, + "tokens_seen": 996278272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005641404212637914, + "loss": 1.4178, + "theoretical_loss": 3.6789286219556887, + "tokens_seen": 996343808 + }, + { + "epoch": 0.3, + "learning_rate": 0.000564124373119358, + "loss": 1.3535, + "theoretical_loss": 3.678905756462486, + "tokens_seen": 996409344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005641083249749248, + "loss": 1.4022, + "theoretical_loss": 3.6788828928942108, + "tokens_seen": 996474880 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005640922768304915, + "loss": 1.3485, + "theoretical_loss": 3.6788600312505753, + "tokens_seen": 996540416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005640762286860582, + "loss": 1.3762, + "theoretical_loss": 3.6788371715312898, + "tokens_seen": 996605952 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005640601805416249, + "loss": 1.2325, + "theoretical_loss": 3.6788143137360665, + "tokens_seen": 996671488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005640441323971916, + "loss": 1.4216, + "theoretical_loss": 3.678791457864616, + "tokens_seen": 996737024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005640280842527583, + "loss": 1.36, + "theoretical_loss": 3.678768603916651, + "tokens_seen": 996802560 + }, + { + "epoch": 0.3, + "learning_rate": 0.000564012036108325, + "loss": 1.2724, + "theoretical_loss": 3.678745751891883, + "tokens_seen": 996868096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005639959879638917, + "loss": 1.3032, + "theoretical_loss": 3.678722901790023, + "tokens_seen": 996933632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005639799398194584, + "loss": 1.3816, + "theoretical_loss": 3.6787000536107834, + "tokens_seen": 996999168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005639638916750251, + "loss": 1.3505, + "theoretical_loss": 3.6786772073538763, + "tokens_seen": 997064704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005639478435305918, + "loss": 1.3334, + "theoretical_loss": 3.678654363019013, + "tokens_seen": 997130240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005639317953861585, + "loss": 1.2945, + "theoretical_loss": 3.6786315206059057, + "tokens_seen": 997195776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005639157472417252, + "loss": 1.2427, + "theoretical_loss": 3.678608680114267, + "tokens_seen": 997261312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005638996990972919, + "loss": 1.2997, + "theoretical_loss": 3.678585841543808, + "tokens_seen": 997326848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005638836509528586, + "loss": 1.3328, + "theoretical_loss": 3.678563004894242, + "tokens_seen": 997392384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005638676028084253, + "loss": 1.498, + "theoretical_loss": 3.67854017016528, + "tokens_seen": 997457920 + }, + { + "epoch": 0.3, + "learning_rate": 0.000563851554663992, + "loss": 1.4053, + "theoretical_loss": 3.6785173373566353, + "tokens_seen": 997523456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005638355065195587, + "loss": 1.2954, + "theoretical_loss": 3.6784945064680197, + "tokens_seen": 997588992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005638194583751254, + "loss": 1.3759, + "theoretical_loss": 3.6784716774991457, + "tokens_seen": 997654528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005638034102306921, + "loss": 1.4184, + "theoretical_loss": 3.6784488504497266, + "tokens_seen": 997720064 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005637873620862588, + "loss": 1.3446, + "theoretical_loss": 3.6784260253194736, + "tokens_seen": 997785600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005637713139418255, + "loss": 1.3536, + "theoretical_loss": 3.6784032021081003, + "tokens_seen": 997851136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005637552657973922, + "loss": 1.2899, + "theoretical_loss": 3.6783803808153195, + "tokens_seen": 997916672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005637392176529589, + "loss": 1.3224, + "theoretical_loss": 3.678357561440843, + "tokens_seen": 997982208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005637231695085256, + "loss": 1.4113, + "theoretical_loss": 3.678334743984384, + "tokens_seen": 998047744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005637071213640923, + "loss": 1.3151, + "theoretical_loss": 3.6783119284456554, + "tokens_seen": 998113280 + }, + { + "epoch": 0.3, + "learning_rate": 0.000563691073219659, + "loss": 1.3028, + "theoretical_loss": 3.6782891148243704, + "tokens_seen": 998178816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005636750250752257, + "loss": 1.3659, + "theoretical_loss": 3.6782663031202416, + "tokens_seen": 998244352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005636589769307924, + "loss": 1.3531, + "theoretical_loss": 3.6782434933329826, + "tokens_seen": 998309888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005636429287863591, + "loss": 1.4025, + "theoretical_loss": 3.678220685462305, + "tokens_seen": 998375424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005636268806419258, + "loss": 1.2639, + "theoretical_loss": 3.678197879507924, + "tokens_seen": 998440960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005636108324974925, + "loss": 1.4739, + "theoretical_loss": 3.678175075469552, + "tokens_seen": 998506496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005635947843530592, + "loss": 1.3225, + "theoretical_loss": 3.678152273346902, + "tokens_seen": 998572032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005635787362086259, + "loss": 1.2327, + "theoretical_loss": 3.6781294731396876, + "tokens_seen": 998637568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005635626880641926, + "loss": 1.4357, + "theoretical_loss": 3.678106674847622, + "tokens_seen": 998703104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005635466399197593, + "loss": 1.3422, + "theoretical_loss": 3.678083878470418, + "tokens_seen": 998768640 + }, + { + "epoch": 0.3, + "learning_rate": 0.000563530591775326, + "loss": 1.3486, + "theoretical_loss": 3.6780610840077914, + "tokens_seen": 998834176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005635145436308927, + "loss": 1.2121, + "theoretical_loss": 3.6780382914594534, + "tokens_seen": 998899712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005634984954864594, + "loss": 1.2469, + "theoretical_loss": 3.678015500825119, + "tokens_seen": 998965248 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005634824473420261, + "loss": 1.3119, + "theoretical_loss": 3.6779927121045013, + "tokens_seen": 999030784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005634663991975928, + "loss": 1.3937, + "theoretical_loss": 3.6779699252973144, + "tokens_seen": 999096320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005634503510531595, + "loss": 1.3069, + "theoretical_loss": 3.6779471404032718, + "tokens_seen": 999161856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005634343029087262, + "loss": 1.3422, + "theoretical_loss": 3.6779243574220883, + "tokens_seen": 999227392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005634182547642929, + "loss": 1.3918, + "theoretical_loss": 3.677901576353477, + "tokens_seen": 999292928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005634022066198596, + "loss": 1.3897, + "theoretical_loss": 3.6778787971971516, + "tokens_seen": 999358464 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 337911, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3854376077651978, + "objective/train/theoretical_loss": 3.677856019952827, + "objective/train/tokens_used": 1019884000, + "theoretical_loss": 3.677856019952827, + "tokens_seen": 999424000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005633861584754263, + "loss": 1.3928, + "theoretical_loss": 3.677856019952827, + "tokens_seen": 999424000 + }, + { + "epoch": 0.3, + "learning_rate": 0.000563370110330993, + "loss": 1.4098, + "theoretical_loss": 3.677833244620218, + "tokens_seen": 999489536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005633540621865598, + "loss": 1.3254, + "theoretical_loss": 3.677810471199037, + "tokens_seen": 999555072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005633380140421264, + "loss": 1.3597, + "theoretical_loss": 3.6777876996889995, + "tokens_seen": 999620608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005633219658976932, + "loss": 1.3813, + "theoretical_loss": 3.677764930089819, + "tokens_seen": 999686144 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005633059177532598, + "loss": 1.4033, + "theoretical_loss": 3.677742162401211, + "tokens_seen": 999751680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005632898696088266, + "loss": 1.3265, + "theoretical_loss": 3.6777193966228894, + "tokens_seen": 999817216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005632738214643932, + "loss": 1.221, + "theoretical_loss": 3.677696632754569, + "tokens_seen": 999882752 + }, + { + "epoch": 0.3, + "learning_rate": 0.00056325777331996, + "loss": 1.3621, + "theoretical_loss": 3.6776738707959633, + "tokens_seen": 999948288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005632417251755266, + "loss": 1.4019, + "theoretical_loss": 3.677651110746788, + "tokens_seen": 1000013824 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005632256770310934, + "loss": 1.3801, + "theoretical_loss": 3.6776283526067575, + "tokens_seen": 1000079360 + }, + { + "epoch": 0.3, + "learning_rate": 0.00056320962888666, + "loss": 1.23, + "theoretical_loss": 3.677605596375587, + "tokens_seen": 1000144896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005631935807422268, + "loss": 1.3374, + "theoretical_loss": 3.6775828420529906, + "tokens_seen": 1000210432 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005631775325977934, + "loss": 1.2955, + "theoretical_loss": 3.6775600896386837, + "tokens_seen": 1000275968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005631614844533602, + "loss": 1.3149, + "theoretical_loss": 3.677537339132381, + "tokens_seen": 1000341504 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005631454363089268, + "loss": 1.3515, + "theoretical_loss": 3.6775145905337974, + "tokens_seen": 1000407040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005631293881644935, + "loss": 1.3775, + "theoretical_loss": 3.6774918438426485, + "tokens_seen": 1000472576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005631133400200602, + "loss": 1.3265, + "theoretical_loss": 3.6774690990586487, + "tokens_seen": 1000538112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005630972918756269, + "loss": 1.2553, + "theoretical_loss": 3.677446356181514, + "tokens_seen": 1000603648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005630812437311936, + "loss": 1.2805, + "theoretical_loss": 3.6774236152109587, + "tokens_seen": 1000669184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005630651955867603, + "loss": 1.4398, + "theoretical_loss": 3.6774008761466987, + "tokens_seen": 1000734720 + }, + { + "epoch": 0.3, + "learning_rate": 0.000563049147442327, + "loss": 1.3433, + "theoretical_loss": 3.67737813898845, + "tokens_seen": 1000800256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005630330992978937, + "loss": 1.3674, + "theoretical_loss": 3.677355403735927, + "tokens_seen": 1000865792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005630170511534604, + "loss": 1.3955, + "theoretical_loss": 3.6773326703888456, + "tokens_seen": 1000931328 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005630010030090271, + "loss": 1.4065, + "theoretical_loss": 3.6773099389469213, + "tokens_seen": 1000996864 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005629849548645937, + "loss": 1.4244, + "theoretical_loss": 3.6772872094098696, + "tokens_seen": 1001062400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005629689067201605, + "loss": 1.3244, + "theoretical_loss": 3.6772644817774065, + "tokens_seen": 1001127936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005629528585757271, + "loss": 1.2779, + "theoretical_loss": 3.6772417560492476, + "tokens_seen": 1001193472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005629368104312939, + "loss": 1.388, + "theoretical_loss": 3.677219032225109, + "tokens_seen": 1001259008 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005629207622868605, + "loss": 1.3168, + "theoretical_loss": 3.677196310304706, + "tokens_seen": 1001324544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005629047141424273, + "loss": 1.3445, + "theoretical_loss": 3.6771735902877545, + "tokens_seen": 1001390080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005628886659979939, + "loss": 1.4859, + "theoretical_loss": 3.6771508721739714, + "tokens_seen": 1001455616 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005628726178535607, + "loss": 1.2793, + "theoretical_loss": 3.6771281559630715, + "tokens_seen": 1001521152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005628565697091273, + "loss": 1.2923, + "theoretical_loss": 3.677105441654772, + "tokens_seen": 1001586688 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005628405215646941, + "loss": 1.2933, + "theoretical_loss": 3.6770827292487884, + "tokens_seen": 1001652224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005628244734202607, + "loss": 1.2476, + "theoretical_loss": 3.677060018744837, + "tokens_seen": 1001717760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005628084252758275, + "loss": 1.3712, + "theoretical_loss": 3.6770373101426346, + "tokens_seen": 1001783296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005627923771313941, + "loss": 1.3297, + "theoretical_loss": 3.677014603441897, + "tokens_seen": 1001848832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005627763289869609, + "loss": 1.4308, + "theoretical_loss": 3.6769918986423407, + "tokens_seen": 1001914368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005627602808425276, + "loss": 1.4035, + "theoretical_loss": 3.6769691957436823, + "tokens_seen": 1001979904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005627442326980943, + "loss": 1.4258, + "theoretical_loss": 3.6769464947456383, + "tokens_seen": 1002045440 + }, + { + "epoch": 0.3, + "learning_rate": 0.000562728184553661, + "loss": 1.3288, + "theoretical_loss": 3.6769237956479257, + "tokens_seen": 1002110976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005627121364092277, + "loss": 1.2861, + "theoretical_loss": 3.6769010984502604, + "tokens_seen": 1002176512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005626960882647944, + "loss": 1.3088, + "theoretical_loss": 3.67687840315236, + "tokens_seen": 1002242048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005626800401203611, + "loss": 1.1965, + "theoretical_loss": 3.67685570975394, + "tokens_seen": 1002307584 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005626639919759278, + "loss": 1.3618, + "theoretical_loss": 3.676833018254719, + "tokens_seen": 1002373120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005626479438314945, + "loss": 1.3594, + "theoretical_loss": 3.676810328654412, + "tokens_seen": 1002438656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005626318956870612, + "loss": 1.3654, + "theoretical_loss": 3.676787640952737, + "tokens_seen": 1002504192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005626158475426279, + "loss": 1.3873, + "theoretical_loss": 3.6767649551494115, + "tokens_seen": 1002569728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005625997993981946, + "loss": 1.2778, + "theoretical_loss": 3.6767422712441515, + "tokens_seen": 1002635264 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 339058, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4166014194488525, + "objective/train/theoretical_loss": 3.676719589236675, + "objective/train/tokens_used": 1023160800, + "theoretical_loss": 3.676719589236675, + "tokens_seen": 1002700800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005625837512537613, + "loss": 1.3904, + "theoretical_loss": 3.676719589236675, + "tokens_seen": 1002700800 + }, + { + "epoch": 0.3, + "learning_rate": 0.000562567703109328, + "loss": 1.2092, + "theoretical_loss": 3.676696909126698, + "tokens_seen": 1002766336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005625516549648947, + "loss": 1.32, + "theoretical_loss": 3.6766742309139397, + "tokens_seen": 1002831872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005625356068204614, + "loss": 1.4426, + "theoretical_loss": 3.6766515545981155, + "tokens_seen": 1002897408 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005625195586760281, + "loss": 1.3065, + "theoretical_loss": 3.676628880178944, + "tokens_seen": 1002962944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005625035105315948, + "loss": 1.3856, + "theoretical_loss": 3.676606207656142, + "tokens_seen": 1003028480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005624874623871615, + "loss": 1.4278, + "theoretical_loss": 3.6765835370294275, + "tokens_seen": 1003094016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005624714142427282, + "loss": 1.4277, + "theoretical_loss": 3.676560868298518, + "tokens_seen": 1003159552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005624553660982949, + "loss": 1.2708, + "theoretical_loss": 3.6765382014631305, + "tokens_seen": 1003225088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005624393179538616, + "loss": 1.279, + "theoretical_loss": 3.6765155365229836, + "tokens_seen": 1003290624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005624232698094283, + "loss": 1.3475, + "theoretical_loss": 3.6764928734777946, + "tokens_seen": 1003356160 + }, + { + "epoch": 0.3, + "learning_rate": 0.000562407221664995, + "loss": 1.4427, + "theoretical_loss": 3.676470212327281, + "tokens_seen": 1003421696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005623911735205617, + "loss": 1.4315, + "theoretical_loss": 3.6764475530711613, + "tokens_seen": 1003487232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005623751253761284, + "loss": 1.3445, + "theoretical_loss": 3.676424895709153, + "tokens_seen": 1003552768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005623590772316951, + "loss": 1.286, + "theoretical_loss": 3.676402240240974, + "tokens_seen": 1003618304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005623430290872618, + "loss": 1.3735, + "theoretical_loss": 3.676379586666343, + "tokens_seen": 1003683840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005623269809428285, + "loss": 1.4462, + "theoretical_loss": 3.6763569349849776, + "tokens_seen": 1003749376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005623109327983952, + "loss": 1.3635, + "theoretical_loss": 3.676334285196596, + "tokens_seen": 1003814912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005622948846539619, + "loss": 1.4446, + "theoretical_loss": 3.676311637300916, + "tokens_seen": 1003880448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005622788365095286, + "loss": 1.3577, + "theoretical_loss": 3.676288991297657, + "tokens_seen": 1003945984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005622627883650953, + "loss": 1.3886, + "theoretical_loss": 3.6762663471865364, + "tokens_seen": 1004011520 + }, + { + "epoch": 0.3, + "learning_rate": 0.000562246740220662, + "loss": 1.2953, + "theoretical_loss": 3.676243704967273, + "tokens_seen": 1004077056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005622306920762287, + "loss": 1.3162, + "theoretical_loss": 3.676221064639585, + "tokens_seen": 1004142592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005622146439317954, + "loss": 1.3495, + "theoretical_loss": 3.6761984262031917, + "tokens_seen": 1004208128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005621985957873621, + "loss": 1.4397, + "theoretical_loss": 3.676175789657811, + "tokens_seen": 1004273664 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005621825476429288, + "loss": 1.2939, + "theoretical_loss": 3.676153155003161, + "tokens_seen": 1004339200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005621664994984955, + "loss": 1.3325, + "theoretical_loss": 3.676130522238962, + "tokens_seen": 1004404736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005621504513540623, + "loss": 1.2272, + "theoretical_loss": 3.6761078913649317, + "tokens_seen": 1004470272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005621344032096289, + "loss": 1.4954, + "theoretical_loss": 3.6760852623807887, + "tokens_seen": 1004535808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005621183550651957, + "loss": 1.3522, + "theoretical_loss": 3.676062635286253, + "tokens_seen": 1004601344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005621023069207623, + "loss": 1.4191, + "theoretical_loss": 3.676040010081042, + "tokens_seen": 1004666880 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005620862587763291, + "loss": 1.2587, + "theoretical_loss": 3.6760173867648764, + "tokens_seen": 1004732416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005620702106318957, + "loss": 1.2593, + "theoretical_loss": 3.6759947653374745, + "tokens_seen": 1004797952 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005620541624874625, + "loss": 1.3543, + "theoretical_loss": 3.6759721457985552, + "tokens_seen": 1004863488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005620381143430291, + "loss": 1.3352, + "theoretical_loss": 3.6759495281478376, + "tokens_seen": 1004929024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005620220661985959, + "loss": 1.3456, + "theoretical_loss": 3.6759269123850413, + "tokens_seen": 1004994560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005620060180541625, + "loss": 1.365, + "theoretical_loss": 3.6759042985098858, + "tokens_seen": 1005060096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005619899699097293, + "loss": 1.4014, + "theoretical_loss": 3.6758816865220902, + "tokens_seen": 1005125632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005619739217652959, + "loss": 1.4408, + "theoretical_loss": 3.675859076421374, + "tokens_seen": 1005191168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005619578736208627, + "loss": 1.4228, + "theoretical_loss": 3.6758364682074567, + "tokens_seen": 1005256704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005619418254764293, + "loss": 1.3138, + "theoretical_loss": 3.6758138618800578, + "tokens_seen": 1005322240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005619257773319961, + "loss": 1.2984, + "theoretical_loss": 3.675791257438897, + "tokens_seen": 1005387776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005619097291875627, + "loss": 1.3153, + "theoretical_loss": 3.675768654883694, + "tokens_seen": 1005453312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005618936810431295, + "loss": 1.3233, + "theoretical_loss": 3.6757460542141684, + "tokens_seen": 1005518848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005618776328986961, + "loss": 1.3306, + "theoretical_loss": 3.67572345543004, + "tokens_seen": 1005584384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005618615847542629, + "loss": 1.3512, + "theoretical_loss": 3.675700858531029, + "tokens_seen": 1005649920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005618455366098295, + "loss": 1.4213, + "theoretical_loss": 3.675678263516855, + "tokens_seen": 1005715456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005618294884653963, + "loss": 1.3265, + "theoretical_loss": 3.675655670387238, + "tokens_seen": 1005780992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005618134403209629, + "loss": 1.2062, + "theoretical_loss": 3.675633079141898, + "tokens_seen": 1005846528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005617973921765297, + "loss": 1.327, + "theoretical_loss": 3.6756104897805546, + "tokens_seen": 1005912064 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 339945, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3827526569366455, + "objective/train/theoretical_loss": 3.6755879023029294, + "objective/train/tokens_used": 1026437600, + "theoretical_loss": 3.6755879023029294, + "tokens_seen": 1005977600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005617813440320963, + "loss": 1.2794, + "theoretical_loss": 3.6755879023029294, + "tokens_seen": 1005977600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005617652958876631, + "loss": 1.3544, + "theoretical_loss": 3.675565316708741, + "tokens_seen": 1006043136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005617492477432297, + "loss": 1.3751, + "theoretical_loss": 3.6755427329977106, + "tokens_seen": 1006108672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005617331995987964, + "loss": 1.4148, + "theoretical_loss": 3.6755201511695588, + "tokens_seen": 1006174208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005617171514543631, + "loss": 1.2918, + "theoretical_loss": 3.675497571224005, + "tokens_seen": 1006239744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005617011033099298, + "loss": 1.3348, + "theoretical_loss": 3.675474993160771, + "tokens_seen": 1006305280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005616850551654965, + "loss": 1.3271, + "theoretical_loss": 3.6754524169795753, + "tokens_seen": 1006370816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005616690070210632, + "loss": 1.404, + "theoretical_loss": 3.67542984268014, + "tokens_seen": 1006436352 + }, + { + "epoch": 0.31, + "learning_rate": 0.00056165295887663, + "loss": 1.4002, + "theoretical_loss": 3.675407270262186, + "tokens_seen": 1006501888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005616369107321966, + "loss": 1.4938, + "theoretical_loss": 3.675384699725433, + "tokens_seen": 1006567424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005616208625877633, + "loss": 1.3689, + "theoretical_loss": 3.675362131069602, + "tokens_seen": 1006632960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00056160481444333, + "loss": 1.2742, + "theoretical_loss": 3.675339564294414, + "tokens_seen": 1006698496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005615887662988966, + "loss": 1.4429, + "theoretical_loss": 3.6753169993995902, + "tokens_seen": 1006764032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005615727181544634, + "loss": 1.2831, + "theoretical_loss": 3.675294436384851, + "tokens_seen": 1006829568 + }, + { + "epoch": 0.31, + "learning_rate": 0.00056155667001003, + "loss": 1.3226, + "theoretical_loss": 3.6752718752499174, + "tokens_seen": 1006895104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005615406218655968, + "loss": 1.2452, + "theoretical_loss": 3.6752493159945105, + "tokens_seen": 1006960640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005615245737211634, + "loss": 1.4371, + "theoretical_loss": 3.675226758618352, + "tokens_seen": 1007026176 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005615085255767302, + "loss": 1.399, + "theoretical_loss": 3.6752042031211625, + "tokens_seen": 1007091712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005614924774322968, + "loss": 1.379, + "theoretical_loss": 3.6751816495026626, + "tokens_seen": 1007157248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005614764292878636, + "loss": 1.4198, + "theoretical_loss": 3.6751590977625748, + "tokens_seen": 1007222784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005614603811434302, + "loss": 1.338, + "theoretical_loss": 3.67513654790062, + "tokens_seen": 1007288320 + }, + { + "epoch": 0.31, + "learning_rate": 0.000561444332998997, + "loss": 1.2114, + "theoretical_loss": 3.67511399991652, + "tokens_seen": 1007353856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005614282848545636, + "loss": 1.4396, + "theoretical_loss": 3.675091453809995, + "tokens_seen": 1007419392 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005614122367101304, + "loss": 1.3514, + "theoretical_loss": 3.6750689095807676, + "tokens_seen": 1007484928 + }, + { + "epoch": 0.31, + "learning_rate": 0.000561396188565697, + "loss": 1.26, + "theoretical_loss": 3.6750463672285596, + "tokens_seen": 1007550464 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005613801404212638, + "loss": 1.4395, + "theoretical_loss": 3.6750238267530917, + "tokens_seen": 1007616000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005613640922768305, + "loss": 1.3453, + "theoretical_loss": 3.6750012881540863, + "tokens_seen": 1007681536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005613480441323972, + "loss": 1.3833, + "theoretical_loss": 3.674978751431265, + "tokens_seen": 1007747072 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005613319959879639, + "loss": 1.5011, + "theoretical_loss": 3.674956216584349, + "tokens_seen": 1007812608 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005613159478435306, + "loss": 1.3377, + "theoretical_loss": 3.6749336836130615, + "tokens_seen": 1007878144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005612998996990973, + "loss": 1.3894, + "theoretical_loss": 3.674911152517123, + "tokens_seen": 1007943680 + }, + { + "epoch": 0.31, + "learning_rate": 0.000561283851554664, + "loss": 1.4529, + "theoretical_loss": 3.674888623296257, + "tokens_seen": 1008009216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005612678034102307, + "loss": 1.274, + "theoretical_loss": 3.6748660959501844, + "tokens_seen": 1008074752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005612517552657974, + "loss": 1.2635, + "theoretical_loss": 3.674843570478627, + "tokens_seen": 1008140288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005612357071213641, + "loss": 1.4344, + "theoretical_loss": 3.6748210468813087, + "tokens_seen": 1008205824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005612196589769308, + "loss": 1.5382, + "theoretical_loss": 3.6747985251579505, + "tokens_seen": 1008271360 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005612036108324975, + "loss": 1.2626, + "theoretical_loss": 3.6747760053082743, + "tokens_seen": 1008336896 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005611875626880642, + "loss": 1.4066, + "theoretical_loss": 3.674753487332004, + "tokens_seen": 1008402432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005611715145436309, + "loss": 1.3055, + "theoretical_loss": 3.6747309712288603, + "tokens_seen": 1008467968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005611554663991976, + "loss": 1.3769, + "theoretical_loss": 3.674708456998567, + "tokens_seen": 1008533504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005611394182547643, + "loss": 1.4245, + "theoretical_loss": 3.6746859446408457, + "tokens_seen": 1008599040 + }, + { + "epoch": 0.31, + "learning_rate": 0.000561123370110331, + "loss": 1.4843, + "theoretical_loss": 3.674663434155419, + "tokens_seen": 1008664576 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005611073219658977, + "loss": 1.306, + "theoretical_loss": 3.6746409255420103, + "tokens_seen": 1008730112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005610912738214644, + "loss": 1.2671, + "theoretical_loss": 3.674618418800342, + "tokens_seen": 1008795648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005610752256770311, + "loss": 1.3045, + "theoretical_loss": 3.674595913930137, + "tokens_seen": 1008861184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005610591775325978, + "loss": 1.3975, + "theoretical_loss": 3.6745734109311177, + "tokens_seen": 1008926720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005610431293881645, + "loss": 1.4402, + "theoretical_loss": 3.674550909803007, + "tokens_seen": 1008992256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005610270812437312, + "loss": 1.451, + "theoretical_loss": 3.6745284105455287, + "tokens_seen": 1009057792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005610110330992979, + "loss": 1.5097, + "theoretical_loss": 3.674505913158405, + "tokens_seen": 1009123328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005609949849548646, + "loss": 1.2531, + "theoretical_loss": 3.6744834176413588, + "tokens_seen": 1009188864 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 341239, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2152767181396484, + "objective/train/theoretical_loss": 3.6744609239941135, + "objective/train/tokens_used": 1029714400, + "theoretical_loss": 3.6744609239941135, + "tokens_seen": 1009254400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005609789368104313, + "loss": 1.2052, + "theoretical_loss": 3.6744609239941135, + "tokens_seen": 1009254400 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560962888665998, + "loss": 1.3659, + "theoretical_loss": 3.6744384322163928, + "tokens_seen": 1009319936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005609468405215648, + "loss": 1.3328, + "theoretical_loss": 3.674415942307919, + "tokens_seen": 1009385472 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005609307923771314, + "loss": 1.2977, + "theoretical_loss": 3.674393454268416, + "tokens_seen": 1009451008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005609147442326982, + "loss": 1.3546, + "theoretical_loss": 3.674370968097607, + "tokens_seen": 1009516544 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005608986960882648, + "loss": 1.4168, + "theoretical_loss": 3.6743484837952165, + "tokens_seen": 1009582080 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005608826479438316, + "loss": 1.3549, + "theoretical_loss": 3.674326001360966, + "tokens_seen": 1009647616 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005608665997993982, + "loss": 1.4207, + "theoretical_loss": 3.67430352079458, + "tokens_seen": 1009713152 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560850551654965, + "loss": 1.2511, + "theoretical_loss": 3.6742810420957825, + "tokens_seen": 1009778688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005608345035105316, + "loss": 1.4117, + "theoretical_loss": 3.6742585652642967, + "tokens_seen": 1009844224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005608184553660984, + "loss": 1.3136, + "theoretical_loss": 3.6742360902998454, + "tokens_seen": 1009909760 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560802407221665, + "loss": 1.3801, + "theoretical_loss": 3.6742136172021542, + "tokens_seen": 1009975296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005607863590772318, + "loss": 1.3655, + "theoretical_loss": 3.6741911459709464, + "tokens_seen": 1010040832 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005607703109327984, + "loss": 1.3055, + "theoretical_loss": 3.674168676605945, + "tokens_seen": 1010106368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005607542627883652, + "loss": 1.3593, + "theoretical_loss": 3.674146209106875, + "tokens_seen": 1010171904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005607382146439318, + "loss": 1.3674, + "theoretical_loss": 3.6741237434734595, + "tokens_seen": 1010237440 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005607221664994986, + "loss": 1.3816, + "theoretical_loss": 3.6741012797054227, + "tokens_seen": 1010302976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005607061183550652, + "loss": 1.3807, + "theoretical_loss": 3.6740788178024895, + "tokens_seen": 1010368512 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560690070210632, + "loss": 1.4319, + "theoretical_loss": 3.6740563577643837, + "tokens_seen": 1010434048 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005606740220661986, + "loss": 1.3671, + "theoretical_loss": 3.674033899590829, + "tokens_seen": 1010499584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005606579739217654, + "loss": 1.3143, + "theoretical_loss": 3.6740114432815503, + "tokens_seen": 1010565120 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560641925777332, + "loss": 1.3913, + "theoretical_loss": 3.6739889888362716, + "tokens_seen": 1010630656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005606258776328988, + "loss": 1.3219, + "theoretical_loss": 3.6739665362547176, + "tokens_seen": 1010696192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005606098294884654, + "loss": 1.2968, + "theoretical_loss": 3.673944085536613, + "tokens_seen": 1010761728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005605937813440322, + "loss": 1.3181, + "theoretical_loss": 3.6739216366816816, + "tokens_seen": 1010827264 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005605777331995988, + "loss": 1.4921, + "theoretical_loss": 3.6738991896896485, + "tokens_seen": 1010892800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005605616850551656, + "loss": 1.2868, + "theoretical_loss": 3.6738767445602383, + "tokens_seen": 1010958336 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005605456369107322, + "loss": 1.2886, + "theoretical_loss": 3.6738543012931757, + "tokens_seen": 1011023872 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560529588766299, + "loss": 1.3538, + "theoretical_loss": 3.6738318598881854, + "tokens_seen": 1011089408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005605135406218656, + "loss": 1.354, + "theoretical_loss": 3.673809420344993, + "tokens_seen": 1011154944 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005604974924774324, + "loss": 1.3059, + "theoretical_loss": 3.6737869826633216, + "tokens_seen": 1011220480 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560481444332999, + "loss": 1.2208, + "theoretical_loss": 3.6737645468428974, + "tokens_seen": 1011286016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005604653961885658, + "loss": 1.358, + "theoretical_loss": 3.6737421128834455, + "tokens_seen": 1011351552 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005604493480441324, + "loss": 1.3231, + "theoretical_loss": 3.673719680784691, + "tokens_seen": 1011417088 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005604332998996992, + "loss": 1.4374, + "theoretical_loss": 3.673697250546358, + "tokens_seen": 1011482624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005604172517552658, + "loss": 1.4058, + "theoretical_loss": 3.6736748221681728, + "tokens_seen": 1011548160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005604012036108326, + "loss": 1.2774, + "theoretical_loss": 3.67365239564986, + "tokens_seen": 1011613696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005603851554663992, + "loss": 1.4357, + "theoretical_loss": 3.6736299709911453, + "tokens_seen": 1011679232 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560369107321966, + "loss": 1.431, + "theoretical_loss": 3.673607548191754, + "tokens_seen": 1011744768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005603530591775326, + "loss": 1.3777, + "theoretical_loss": 3.6735851272514113, + "tokens_seen": 1011810304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005603370110330994, + "loss": 1.2492, + "theoretical_loss": 3.6735627081698423, + "tokens_seen": 1011875840 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560320962888666, + "loss": 1.3058, + "theoretical_loss": 3.6735402909467734, + "tokens_seen": 1011941376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005603049147442327, + "loss": 1.3889, + "theoretical_loss": 3.6735178755819295, + "tokens_seen": 1012006912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005602888665997994, + "loss": 1.441, + "theoretical_loss": 3.673495462075037, + "tokens_seen": 1012072448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005602728184553661, + "loss": 1.2577, + "theoretical_loss": 3.6734730504258204, + "tokens_seen": 1012137984 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005602567703109328, + "loss": 1.4346, + "theoretical_loss": 3.6734506406340066, + "tokens_seen": 1012203520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005602407221664995, + "loss": 1.2212, + "theoretical_loss": 3.673428232699321, + "tokens_seen": 1012269056 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005602246740220663, + "loss": 1.379, + "theoretical_loss": 3.6734058266214893, + "tokens_seen": 1012334592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005602086258776329, + "loss": 1.4254, + "theoretical_loss": 3.6733834224002377, + "tokens_seen": 1012400128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005601925777331995, + "loss": 1.2654, + "theoretical_loss": 3.6733610200352924, + "tokens_seen": 1012465664 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 342041, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2200353145599365, + "objective/train/theoretical_loss": 3.673338619526379, + "objective/train/tokens_used": 1032991200, + "theoretical_loss": 3.673338619526379, + "tokens_seen": 1012531200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005601765295887663, + "loss": 1.2664, + "theoretical_loss": 3.673338619526379, + "tokens_seen": 1012531200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005601604814443329, + "loss": 1.309, + "theoretical_loss": 3.6733162208732235, + "tokens_seen": 1012596736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005601444332998997, + "loss": 1.3172, + "theoretical_loss": 3.6732938240755524, + "tokens_seen": 1012662272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005601283851554663, + "loss": 1.283, + "theoretical_loss": 3.673271429133092, + "tokens_seen": 1012727808 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005601123370110331, + "loss": 1.192, + "theoretical_loss": 3.6732490360455685, + "tokens_seen": 1012793344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005600962888665997, + "loss": 1.3267, + "theoretical_loss": 3.6732266448127078, + "tokens_seen": 1012858880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005600802407221665, + "loss": 1.221, + "theoretical_loss": 3.6732042554342375, + "tokens_seen": 1012924416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005600641925777331, + "loss": 1.3396, + "theoretical_loss": 3.673181867909882, + "tokens_seen": 1012989952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005600481444332999, + "loss": 1.3371, + "theoretical_loss": 3.6731594822393703, + "tokens_seen": 1013055488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005600320962888665, + "loss": 1.3225, + "theoretical_loss": 3.673137098422427, + "tokens_seen": 1013121024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005600160481444333, + "loss": 1.3604, + "theoretical_loss": 3.6731147164587803, + "tokens_seen": 1013186560 + }, + { + "epoch": 0.31, + "learning_rate": 0.00056, + "loss": 1.262, + "theoretical_loss": 3.6730923363481556, + "tokens_seen": 1013252096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005599839518555667, + "loss": 1.3062, + "theoretical_loss": 3.67306995809028, + "tokens_seen": 1013317632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005599679037111334, + "loss": 1.3287, + "theoretical_loss": 3.6730475816848807, + "tokens_seen": 1013383168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005599518555667001, + "loss": 1.4042, + "theoretical_loss": 3.6730252071316842, + "tokens_seen": 1013448704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005599358074222668, + "loss": 1.3411, + "theoretical_loss": 3.6730028344304175, + "tokens_seen": 1013514240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005599197592778335, + "loss": 1.3035, + "theoretical_loss": 3.6729804635808074, + "tokens_seen": 1013579776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005599037111334002, + "loss": 1.4123, + "theoretical_loss": 3.6729580945825813, + "tokens_seen": 1013645312 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005598876629889669, + "loss": 1.2369, + "theoretical_loss": 3.6729357274354664, + "tokens_seen": 1013710848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005598716148445336, + "loss": 1.399, + "theoretical_loss": 3.672913362139189, + "tokens_seen": 1013776384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005598555667001003, + "loss": 1.3412, + "theoretical_loss": 3.6728909986934775, + "tokens_seen": 1013841920 + }, + { + "epoch": 0.31, + "learning_rate": 0.000559839518555667, + "loss": 1.3273, + "theoretical_loss": 3.6728686370980586, + "tokens_seen": 1013907456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005598234704112337, + "loss": 1.4403, + "theoretical_loss": 3.6728462773526593, + "tokens_seen": 1013972992 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005598074222668004, + "loss": 1.3912, + "theoretical_loss": 3.6728239194570076, + "tokens_seen": 1014038528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005597913741223671, + "loss": 1.3262, + "theoretical_loss": 3.67280156341083, + "tokens_seen": 1014104064 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005597753259779338, + "loss": 1.2509, + "theoretical_loss": 3.6727792092138554, + "tokens_seen": 1014169600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005597592778335005, + "loss": 1.3865, + "theoretical_loss": 3.6727568568658104, + "tokens_seen": 1014235136 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005597432296890673, + "loss": 1.2653, + "theoretical_loss": 3.6727345063664227, + "tokens_seen": 1014300672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005597271815446339, + "loss": 1.3199, + "theoretical_loss": 3.67271215771542, + "tokens_seen": 1014366208 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005597111334002007, + "loss": 1.3486, + "theoretical_loss": 3.67268981091253, + "tokens_seen": 1014431744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005596950852557673, + "loss": 1.3965, + "theoretical_loss": 3.672667465957481, + "tokens_seen": 1014497280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005596790371113341, + "loss": 1.3149, + "theoretical_loss": 3.6726451228500006, + "tokens_seen": 1014562816 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005596629889669007, + "loss": 1.2876, + "theoretical_loss": 3.6726227815898165, + "tokens_seen": 1014628352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005596469408224675, + "loss": 1.4492, + "theoretical_loss": 3.6726004421766567, + "tokens_seen": 1014693888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005596308926780341, + "loss": 1.3578, + "theoretical_loss": 3.672578104610249, + "tokens_seen": 1014759424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005596148445336009, + "loss": 1.2204, + "theoretical_loss": 3.672555768890322, + "tokens_seen": 1014824960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005595987963891675, + "loss": 1.3516, + "theoretical_loss": 3.6725334350166037, + "tokens_seen": 1014890496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005595827482447343, + "loss": 1.3082, + "theoretical_loss": 3.672511102988822, + "tokens_seen": 1014956032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005595667001003009, + "loss": 1.3985, + "theoretical_loss": 3.6724887728067053, + "tokens_seen": 1015021568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005595506519558677, + "loss": 1.3852, + "theoretical_loss": 3.672466444469982, + "tokens_seen": 1015087104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005595346038114343, + "loss": 1.251, + "theoretical_loss": 3.6724441179783804, + "tokens_seen": 1015152640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005595185556670011, + "loss": 1.3817, + "theoretical_loss": 3.672421793331629, + "tokens_seen": 1015218176 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005595025075225677, + "loss": 1.3173, + "theoretical_loss": 3.672399470529456, + "tokens_seen": 1015283712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005594864593781345, + "loss": 1.4168, + "theoretical_loss": 3.6723771495715907, + "tokens_seen": 1015349248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005594704112337011, + "loss": 1.3295, + "theoretical_loss": 3.6723548304577607, + "tokens_seen": 1015414784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005594543630892679, + "loss": 1.4062, + "theoretical_loss": 3.672332513187695, + "tokens_seen": 1015480320 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005594383149448345, + "loss": 1.3011, + "theoretical_loss": 3.6723101977611226, + "tokens_seen": 1015545856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005594222668004013, + "loss": 1.4394, + "theoretical_loss": 3.672287884177772, + "tokens_seen": 1015611392 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005594062186559679, + "loss": 1.348, + "theoretical_loss": 3.6722655724373725, + "tokens_seen": 1015676928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005593901705115347, + "loss": 1.3012, + "theoretical_loss": 3.672243262539652, + "tokens_seen": 1015742464 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 343196, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3883546590805054, + "objective/train/theoretical_loss": 3.6722209544843407, + "objective/train/tokens_used": 1036268000, + "theoretical_loss": 3.6722209544843407, + "tokens_seen": 1015808000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005593741223671013, + "loss": 1.3079, + "theoretical_loss": 3.6722209544843407, + "tokens_seen": 1015808000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005593580742226681, + "loss": 1.3882, + "theoretical_loss": 3.6721986482711664, + "tokens_seen": 1015873536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005593420260782347, + "loss": 1.4233, + "theoretical_loss": 3.6721763438998587, + "tokens_seen": 1015939072 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005593259779338015, + "loss": 1.2578, + "theoretical_loss": 3.672154041370147, + "tokens_seen": 1016004608 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005593099297893681, + "loss": 1.2674, + "theoretical_loss": 3.6721317406817597, + "tokens_seen": 1016070144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005592938816449349, + "loss": 1.3096, + "theoretical_loss": 3.6721094418344276, + "tokens_seen": 1016135680 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005592778335005015, + "loss": 1.3513, + "theoretical_loss": 3.672087144827878, + "tokens_seen": 1016201216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005592617853560683, + "loss": 1.338, + "theoretical_loss": 3.672064849661841, + "tokens_seen": 1016266752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005592457372116349, + "loss": 1.32, + "theoretical_loss": 3.672042556336047, + "tokens_seen": 1016332288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005592296890672017, + "loss": 1.3809, + "theoretical_loss": 3.6720202648502234, + "tokens_seen": 1016397824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005592136409227683, + "loss": 1.386, + "theoretical_loss": 3.6719979752041016, + "tokens_seen": 1016463360 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005591975927783351, + "loss": 1.2795, + "theoretical_loss": 3.6719756873974103, + "tokens_seen": 1016528896 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005591815446339017, + "loss": 1.3663, + "theoretical_loss": 3.6719534014298794, + "tokens_seen": 1016594432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005591654964894685, + "loss": 1.3, + "theoretical_loss": 3.671931117301239, + "tokens_seen": 1016659968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005591494483450351, + "loss": 1.459, + "theoretical_loss": 3.6719088350112177, + "tokens_seen": 1016725504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005591334002006019, + "loss": 1.3767, + "theoretical_loss": 3.6718865545595456, + "tokens_seen": 1016791040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005591173520561685, + "loss": 1.355, + "theoretical_loss": 3.6718642759459534, + "tokens_seen": 1016856576 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005591013039117353, + "loss": 1.4298, + "theoretical_loss": 3.6718419991701703, + "tokens_seen": 1016922112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005590852557673019, + "loss": 1.3993, + "theoretical_loss": 3.671819724231927, + "tokens_seen": 1016987648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005590692076228687, + "loss": 1.3459, + "theoretical_loss": 3.671797451130953, + "tokens_seen": 1017053184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005590531594784353, + "loss": 1.2541, + "theoretical_loss": 3.6717751798669775, + "tokens_seen": 1017118720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005590371113340021, + "loss": 1.4025, + "theoretical_loss": 3.671752910439732, + "tokens_seen": 1017184256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005590210631895687, + "loss": 1.4173, + "theoretical_loss": 3.6717306428489467, + "tokens_seen": 1017249792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005590050150451355, + "loss": 1.3877, + "theoretical_loss": 3.671708377094351, + "tokens_seen": 1017315328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005589889669007021, + "loss": 1.3171, + "theoretical_loss": 3.6716861131756753, + "tokens_seen": 1017380864 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005589729187562689, + "loss": 1.2392, + "theoretical_loss": 3.671663851092651, + "tokens_seen": 1017446400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005589568706118355, + "loss": 1.4565, + "theoretical_loss": 3.6716415908450073, + "tokens_seen": 1017511936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005589408224674023, + "loss": 1.4564, + "theoretical_loss": 3.671619332432475, + "tokens_seen": 1017577472 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558924774322969, + "loss": 1.3598, + "theoretical_loss": 3.671597075854785, + "tokens_seen": 1017643008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005589087261785356, + "loss": 1.3916, + "theoretical_loss": 3.671574821111668, + "tokens_seen": 1017708544 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005588926780341023, + "loss": 1.298, + "theoretical_loss": 3.6715525682028547, + "tokens_seen": 1017774080 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558876629889669, + "loss": 1.3317, + "theoretical_loss": 3.6715303171280746, + "tokens_seen": 1017839616 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005588605817452358, + "loss": 1.4106, + "theoretical_loss": 3.6715080678870597, + "tokens_seen": 1017905152 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005588445336008024, + "loss": 1.4136, + "theoretical_loss": 3.671485820479541, + "tokens_seen": 1017970688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005588284854563692, + "loss": 1.407, + "theoretical_loss": 3.671463574905248, + "tokens_seen": 1018036224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005588124373119358, + "loss": 1.2389, + "theoretical_loss": 3.671441331163913, + "tokens_seen": 1018101760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005587963891675026, + "loss": 1.2802, + "theoretical_loss": 3.6714190892552665, + "tokens_seen": 1018167296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005587803410230692, + "loss": 1.2805, + "theoretical_loss": 3.6713968491790396, + "tokens_seen": 1018232832 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005587642928786358, + "loss": 1.3192, + "theoretical_loss": 3.671374610934963, + "tokens_seen": 1018298368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005587482447342026, + "loss": 1.4082, + "theoretical_loss": 3.671352374522769, + "tokens_seen": 1018363904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005587321965897692, + "loss": 1.3361, + "theoretical_loss": 3.671330139942188, + "tokens_seen": 1018429440 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558716148445336, + "loss": 1.3375, + "theoretical_loss": 3.6713079071929506, + "tokens_seen": 1018494976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005587001003009026, + "loss": 1.303, + "theoretical_loss": 3.671285676274789, + "tokens_seen": 1018560512 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005586840521564694, + "loss": 1.2425, + "theoretical_loss": 3.671263447187435, + "tokens_seen": 1018626048 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558668004012036, + "loss": 1.2442, + "theoretical_loss": 3.6712412199306192, + "tokens_seen": 1018691584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005586519558676028, + "loss": 1.3234, + "theoretical_loss": 3.6712189945040734, + "tokens_seen": 1018757120 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005586359077231695, + "loss": 1.3611, + "theoretical_loss": 3.6711967709075295, + "tokens_seen": 1018822656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005586198595787362, + "loss": 1.3453, + "theoretical_loss": 3.6711745491407184, + "tokens_seen": 1018888192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005586038114343029, + "loss": 1.3404, + "theoretical_loss": 3.6711523292033723, + "tokens_seen": 1018953728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005585877632898696, + "loss": 1.431, + "theoretical_loss": 3.671130111095223, + "tokens_seen": 1019019264 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 343997, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2892177104949951, + "objective/train/theoretical_loss": 3.671107894816002, + "objective/train/tokens_used": 1039544800, + "theoretical_loss": 3.671107894816002, + "tokens_seen": 1019084800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005585717151454363, + "loss": 1.2274, + "theoretical_loss": 3.671107894816002, + "tokens_seen": 1019084800 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558555667001003, + "loss": 1.3134, + "theoretical_loss": 3.671085680365441, + "tokens_seen": 1019150336 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005585396188565698, + "loss": 1.3146, + "theoretical_loss": 3.6710634677432727, + "tokens_seen": 1019215872 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005585235707121364, + "loss": 1.3461, + "theoretical_loss": 3.6710412569492283, + "tokens_seen": 1019281408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005585075225677032, + "loss": 1.3037, + "theoretical_loss": 3.67101904798304, + "tokens_seen": 1019346944 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005584914744232698, + "loss": 1.2787, + "theoretical_loss": 3.67099684084444, + "tokens_seen": 1019412480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005584754262788366, + "loss": 1.3193, + "theoretical_loss": 3.6709746355331605, + "tokens_seen": 1019478016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005584593781344032, + "loss": 1.4775, + "theoretical_loss": 3.6709524320489333, + "tokens_seen": 1019543552 + }, + { + "epoch": 0.31, + "learning_rate": 0.00055844332998997, + "loss": 1.31, + "theoretical_loss": 3.6709302303914906, + "tokens_seen": 1019609088 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005584272818455366, + "loss": 1.3576, + "theoretical_loss": 3.6709080305605655, + "tokens_seen": 1019674624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005584112337011034, + "loss": 1.3663, + "theoretical_loss": 3.6708858325558893, + "tokens_seen": 1019740160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00055839518555667, + "loss": 1.2277, + "theoretical_loss": 3.670863636377195, + "tokens_seen": 1019805696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005583791374122368, + "loss": 1.4093, + "theoretical_loss": 3.6708414420242157, + "tokens_seen": 1019871232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005583630892678034, + "loss": 1.3397, + "theoretical_loss": 3.6708192494966827, + "tokens_seen": 1019936768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005583470411233702, + "loss": 1.4206, + "theoretical_loss": 3.670797058794329, + "tokens_seen": 1020002304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005583309929789368, + "loss": 1.3876, + "theoretical_loss": 3.6707748699168876, + "tokens_seen": 1020067840 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005583149448345036, + "loss": 1.4703, + "theoretical_loss": 3.670752682864091, + "tokens_seen": 1020133376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005582988966900702, + "loss": 1.3421, + "theoretical_loss": 3.670730497635671, + "tokens_seen": 1020198912 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558282848545637, + "loss": 1.4444, + "theoretical_loss": 3.670708314231362, + "tokens_seen": 1020264448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005582668004012036, + "loss": 1.3564, + "theoretical_loss": 3.6706861326508964, + "tokens_seen": 1020329984 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005582507522567704, + "loss": 1.3278, + "theoretical_loss": 3.6706639528940066, + "tokens_seen": 1020395520 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558234704112337, + "loss": 1.3452, + "theoretical_loss": 3.6706417749604263, + "tokens_seen": 1020461056 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005582186559679038, + "loss": 1.3457, + "theoretical_loss": 3.6706195988498873, + "tokens_seen": 1020526592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005582026078234704, + "loss": 1.4581, + "theoretical_loss": 3.670597424562124, + "tokens_seen": 1020592128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005581865596790372, + "loss": 1.2731, + "theoretical_loss": 3.670575252096869, + "tokens_seen": 1020657664 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005581705115346038, + "loss": 1.3628, + "theoretical_loss": 3.670553081453855, + "tokens_seen": 1020723200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005581544633901706, + "loss": 1.3148, + "theoretical_loss": 3.6705309126328163, + "tokens_seen": 1020788736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005581384152457372, + "loss": 1.3534, + "theoretical_loss": 3.6705087456334855, + "tokens_seen": 1020854272 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558122367101304, + "loss": 1.4729, + "theoretical_loss": 3.6704865804555964, + "tokens_seen": 1020919808 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005581063189568706, + "loss": 1.4037, + "theoretical_loss": 3.6704644170988816, + "tokens_seen": 1020985344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005580902708124374, + "loss": 1.2846, + "theoretical_loss": 3.6704422555630756, + "tokens_seen": 1021050880 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558074222668004, + "loss": 1.2909, + "theoretical_loss": 3.6704200958479114, + "tokens_seen": 1021116416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005580581745235708, + "loss": 1.5132, + "theoretical_loss": 3.6703979379531226, + "tokens_seen": 1021181952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005580421263791374, + "loss": 1.3148, + "theoretical_loss": 3.6703757818784433, + "tokens_seen": 1021247488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005580260782347042, + "loss": 1.3601, + "theoretical_loss": 3.6703536276236064, + "tokens_seen": 1021313024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005580100300902708, + "loss": 1.299, + "theoretical_loss": 3.6703314751883465, + "tokens_seen": 1021378560 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005579939819458376, + "loss": 1.3861, + "theoretical_loss": 3.670309324572396, + "tokens_seen": 1021444096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005579779338014042, + "loss": 1.3621, + "theoretical_loss": 3.670287175775491, + "tokens_seen": 1021509632 + }, + { + "epoch": 0.31, + "learning_rate": 0.000557961885656971, + "loss": 1.3271, + "theoretical_loss": 3.6702650287973637, + "tokens_seen": 1021575168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005579458375125376, + "loss": 1.2815, + "theoretical_loss": 3.6702428836377488, + "tokens_seen": 1021640704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005579297893681044, + "loss": 1.4229, + "theoretical_loss": 3.67022074029638, + "tokens_seen": 1021706240 + }, + { + "epoch": 0.31, + "learning_rate": 0.000557913741223671, + "loss": 1.2939, + "theoretical_loss": 3.6701985987729913, + "tokens_seen": 1021771776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005578976930792378, + "loss": 1.4508, + "theoretical_loss": 3.670176459067317, + "tokens_seen": 1021837312 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005578816449348044, + "loss": 1.353, + "theoretical_loss": 3.6701543211790923, + "tokens_seen": 1021902848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005578655967903712, + "loss": 1.2911, + "theoretical_loss": 3.6701321851080495, + "tokens_seen": 1021968384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005578495486459378, + "loss": 1.3805, + "theoretical_loss": 3.670110050853925, + "tokens_seen": 1022033920 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005578335005015046, + "loss": 1.2826, + "theoretical_loss": 3.670087918416451, + "tokens_seen": 1022099456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005578174523570712, + "loss": 1.391, + "theoretical_loss": 3.670065787795364, + "tokens_seen": 1022164992 + }, + { + "epoch": 0.31, + "learning_rate": 0.000557801404212638, + "loss": 1.3321, + "theoretical_loss": 3.670043658990397, + "tokens_seen": 1022230528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005577853560682046, + "loss": 1.4022, + "theoretical_loss": 3.670021532001286, + "tokens_seen": 1022296064 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 345138, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3124229907989502, + "objective/train/theoretical_loss": 3.6699994068277637, + "objective/train/tokens_used": 1042821600, + "theoretical_loss": 3.6699994068277637, + "tokens_seen": 1022361600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005577693079237714, + "loss": 1.3542, + "theoretical_loss": 3.6699994068277637, + "tokens_seen": 1022361600 + }, + { + "epoch": 0.31, + "learning_rate": 0.000557753259779338, + "loss": 1.326, + "theoretical_loss": 3.6699772834695663, + "tokens_seen": 1022427136 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005577372116349048, + "loss": 1.4202, + "theoretical_loss": 3.6699551619264277, + "tokens_seen": 1022492672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005577211634904714, + "loss": 1.3727, + "theoretical_loss": 3.6699330421980836, + "tokens_seen": 1022558208 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005577051153460382, + "loss": 1.2421, + "theoretical_loss": 3.6699109242842676, + "tokens_seen": 1022623744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005576890672016048, + "loss": 1.3208, + "theoretical_loss": 3.669888808184715, + "tokens_seen": 1022689280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005576730190571716, + "loss": 1.3568, + "theoretical_loss": 3.6698666938991615, + "tokens_seen": 1022754816 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005576569709127382, + "loss": 1.3501, + "theoretical_loss": 3.6698445814273413, + "tokens_seen": 1022820352 + }, + { + "epoch": 0.31, + "learning_rate": 0.000557640922768305, + "loss": 1.3411, + "theoretical_loss": 3.669822470768989, + "tokens_seen": 1022885888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005576248746238716, + "loss": 1.4406, + "theoretical_loss": 3.669800361923841, + "tokens_seen": 1022951424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005576088264794384, + "loss": 1.2884, + "theoretical_loss": 3.669778254891632, + "tokens_seen": 1023016960 + }, + { + "epoch": 0.31, + "learning_rate": 0.000557592778335005, + "loss": 1.3368, + "theoretical_loss": 3.6697561496720965, + "tokens_seen": 1023082496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005575767301905718, + "loss": 1.4343, + "theoretical_loss": 3.6697340462649706, + "tokens_seen": 1023148032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005575606820461384, + "loss": 1.4026, + "theoretical_loss": 3.669711944669989, + "tokens_seen": 1023213568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005575446339017052, + "loss": 1.3179, + "theoretical_loss": 3.669689844886887, + "tokens_seen": 1023279104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005575285857572718, + "loss": 1.2564, + "theoretical_loss": 3.669667746915401, + "tokens_seen": 1023344640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005575125376128385, + "loss": 1.4114, + "theoretical_loss": 3.6696456507552657, + "tokens_seen": 1023410176 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005574964894684052, + "loss": 1.3473, + "theoretical_loss": 3.6696235564062167, + "tokens_seen": 1023475712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005574804413239719, + "loss": 1.292, + "theoretical_loss": 3.6696014638679904, + "tokens_seen": 1023541248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005574643931795387, + "loss": 1.3808, + "theoretical_loss": 3.669579373140321, + "tokens_seen": 1023606784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005574483450351053, + "loss": 1.4791, + "theoretical_loss": 3.669557284222946, + "tokens_seen": 1023672320 + }, + { + "epoch": 0.31, + "learning_rate": 0.000557432296890672, + "loss": 1.3604, + "theoretical_loss": 3.669535197115599, + "tokens_seen": 1023737856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005574162487462387, + "loss": 1.4341, + "theoretical_loss": 3.6695131118180173, + "tokens_seen": 1023803392 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005574002006018055, + "loss": 1.3428, + "theoretical_loss": 3.6694910283299373, + "tokens_seen": 1023868928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005573841524573721, + "loss": 1.4249, + "theoretical_loss": 3.6694689466510937, + "tokens_seen": 1023934464 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005573681043129387, + "loss": 1.3218, + "theoretical_loss": 3.6694468667812226, + "tokens_seen": 1024000000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005573520561685055, + "loss": 1.3406, + "theoretical_loss": 3.66942478872006, + "tokens_seen": 1024065536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005573360080240723, + "loss": 1.4468, + "theoretical_loss": 3.669402712467343, + "tokens_seen": 1024131072 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005573199598796389, + "loss": 1.2285, + "theoretical_loss": 3.6693806380228073, + "tokens_seen": 1024196608 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005573039117352057, + "loss": 1.238, + "theoretical_loss": 3.6693585653861884, + "tokens_seen": 1024262144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005572878635907723, + "loss": 1.471, + "theoretical_loss": 3.669336494557223, + "tokens_seen": 1024327680 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005572718154463391, + "loss": 1.4066, + "theoretical_loss": 3.6693144255356485, + "tokens_seen": 1024393216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005572557673019057, + "loss": 1.3275, + "theoretical_loss": 3.6692923583211994, + "tokens_seen": 1024458752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005572397191574725, + "loss": 1.3698, + "theoretical_loss": 3.6692702929136134, + "tokens_seen": 1024524288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005572236710130391, + "loss": 1.3979, + "theoretical_loss": 3.6692482293126263, + "tokens_seen": 1024589824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005572076228686059, + "loss": 1.2864, + "theoretical_loss": 3.669226167517975, + "tokens_seen": 1024655360 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005571915747241725, + "loss": 1.3593, + "theoretical_loss": 3.6692041075293966, + "tokens_seen": 1024720896 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005571755265797393, + "loss": 1.3714, + "theoretical_loss": 3.6691820493466265, + "tokens_seen": 1024786432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005571594784353059, + "loss": 1.3756, + "theoretical_loss": 3.6691599929694023, + "tokens_seen": 1024851968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005571434302908727, + "loss": 1.3525, + "theoretical_loss": 3.6691379383974607, + "tokens_seen": 1024917504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005571273821464393, + "loss": 1.2336, + "theoretical_loss": 3.6691158856305384, + "tokens_seen": 1024983040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005571113340020061, + "loss": 1.3104, + "theoretical_loss": 3.6690938346683724, + "tokens_seen": 1025048576 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005570952858575727, + "loss": 1.3472, + "theoretical_loss": 3.6690717855106993, + "tokens_seen": 1025114112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005570792377131395, + "loss": 1.3492, + "theoretical_loss": 3.6690497381572564, + "tokens_seen": 1025179648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005570631895687061, + "loss": 1.4162, + "theoretical_loss": 3.6690276926077807, + "tokens_seen": 1025245184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005570471414242729, + "loss": 1.2798, + "theoretical_loss": 3.669005648862009, + "tokens_seen": 1025310720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005570310932798395, + "loss": 1.3499, + "theoretical_loss": 3.668983606919679, + "tokens_seen": 1025376256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005570150451354063, + "loss": 1.4369, + "theoretical_loss": 3.668961566780527, + "tokens_seen": 1025441792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005569989969909729, + "loss": 1.4385, + "theoretical_loss": 3.6689395284442914, + "tokens_seen": 1025507328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005569829488465397, + "loss": 1.4733, + "theoretical_loss": 3.668917491910709, + "tokens_seen": 1025572864 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 345966, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3591279983520508, + "objective/train/theoretical_loss": 3.6688954571795165, + "objective/train/tokens_used": 1046098400, + "theoretical_loss": 3.6688954571795165, + "tokens_seen": 1025638400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005569669007021063, + "loss": 1.3544, + "theoretical_loss": 3.6688954571795165, + "tokens_seen": 1025638400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005569508525576731, + "loss": 1.2817, + "theoretical_loss": 3.6688734242504526, + "tokens_seen": 1025703936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005569348044132397, + "loss": 1.442, + "theoretical_loss": 3.668851393123254, + "tokens_seen": 1025769472 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005569187562688065, + "loss": 1.3131, + "theoretical_loss": 3.6688293637976583, + "tokens_seen": 1025835008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005569027081243731, + "loss": 1.3058, + "theoretical_loss": 3.668807336273403, + "tokens_seen": 1025900544 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005568866599799399, + "loss": 1.3676, + "theoretical_loss": 3.668785310550226, + "tokens_seen": 1025966080 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005568706118355065, + "loss": 1.438, + "theoretical_loss": 3.6687632866278657, + "tokens_seen": 1026031616 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005568545636910733, + "loss": 1.3054, + "theoretical_loss": 3.668741264506058, + "tokens_seen": 1026097152 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005568385155466399, + "loss": 1.432, + "theoretical_loss": 3.668719244184542, + "tokens_seen": 1026162688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005568224674022067, + "loss": 1.3624, + "theoretical_loss": 3.668697225663056, + "tokens_seen": 1026228224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005568064192577733, + "loss": 1.431, + "theoretical_loss": 3.6686752089413366, + "tokens_seen": 1026293760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005567903711133401, + "loss": 1.3012, + "theoretical_loss": 3.6686531940191225, + "tokens_seen": 1026359296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005567743229689067, + "loss": 1.4499, + "theoretical_loss": 3.668631180896152, + "tokens_seen": 1026424832 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005567582748244735, + "loss": 1.3879, + "theoretical_loss": 3.6686091695721625, + "tokens_seen": 1026490368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005567422266800401, + "loss": 1.311, + "theoretical_loss": 3.6685871600468927, + "tokens_seen": 1026555904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005567261785356069, + "loss": 1.3203, + "theoretical_loss": 3.6685651523200806, + "tokens_seen": 1026621440 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005567101303911735, + "loss": 1.3811, + "theoretical_loss": 3.6685431463914644, + "tokens_seen": 1026686976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005566940822467403, + "loss": 1.3512, + "theoretical_loss": 3.668521142260783, + "tokens_seen": 1026752512 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005566780341023069, + "loss": 1.4013, + "theoretical_loss": 3.668499139927773, + "tokens_seen": 1026818048 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005566619859578737, + "loss": 1.2922, + "theoretical_loss": 3.668477139392175, + "tokens_seen": 1026883584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005566459378134403, + "loss": 1.3866, + "theoretical_loss": 3.668455140653726, + "tokens_seen": 1026949120 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005566298896690071, + "loss": 1.3262, + "theoretical_loss": 3.668433143712165, + "tokens_seen": 1027014656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005566138415245737, + "loss": 1.4521, + "theoretical_loss": 3.6684111485672313, + "tokens_seen": 1027080192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005565977933801405, + "loss": 1.3321, + "theoretical_loss": 3.6683891552186623, + "tokens_seen": 1027145728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005565817452357071, + "loss": 1.3675, + "theoretical_loss": 3.668367163666197, + "tokens_seen": 1027211264 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005565656970912739, + "loss": 1.3501, + "theoretical_loss": 3.6683451739095747, + "tokens_seen": 1027276800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005565496489468405, + "loss": 1.3871, + "theoretical_loss": 3.668323185948534, + "tokens_seen": 1027342336 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005565336008024073, + "loss": 1.3615, + "theoretical_loss": 3.668301199782813, + "tokens_seen": 1027407872 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005565175526579739, + "loss": 1.3568, + "theoretical_loss": 3.6682792154121517, + "tokens_seen": 1027473408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005565015045135407, + "loss": 1.3667, + "theoretical_loss": 3.6682572328362886, + "tokens_seen": 1027538944 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005564854563691073, + "loss": 1.3261, + "theoretical_loss": 3.6682352520549624, + "tokens_seen": 1027604480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005564694082246741, + "loss": 1.2886, + "theoretical_loss": 3.668213273067913, + "tokens_seen": 1027670016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005564533600802407, + "loss": 1.4568, + "theoretical_loss": 3.668191295874878, + "tokens_seen": 1027735552 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005564373119358075, + "loss": 1.3701, + "theoretical_loss": 3.6681693204755987, + "tokens_seen": 1027801088 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005564212637913741, + "loss": 1.4025, + "theoretical_loss": 3.6681473468698123, + "tokens_seen": 1027866624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005564052156469409, + "loss": 1.3558, + "theoretical_loss": 3.6681253750572593, + "tokens_seen": 1027932160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005563891675025075, + "loss": 1.3646, + "theoretical_loss": 3.668103405037679, + "tokens_seen": 1027997696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005563731193580743, + "loss": 1.3556, + "theoretical_loss": 3.66808143681081, + "tokens_seen": 1028063232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005563570712136409, + "loss": 1.2993, + "theoretical_loss": 3.668059470376393, + "tokens_seen": 1028128768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005563410230692077, + "loss": 1.3762, + "theoretical_loss": 3.6680375057341665, + "tokens_seen": 1028194304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005563249749247743, + "loss": 1.401, + "theoretical_loss": 3.66801554288387, + "tokens_seen": 1028259840 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005563089267803411, + "loss": 1.2452, + "theoretical_loss": 3.6679935818252436, + "tokens_seen": 1028325376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005562928786359077, + "loss": 1.3394, + "theoretical_loss": 3.667971622558027, + "tokens_seen": 1028390912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005562768304914745, + "loss": 1.4385, + "theoretical_loss": 3.6679496650819594, + "tokens_seen": 1028456448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005562607823470411, + "loss": 1.3868, + "theoretical_loss": 3.6679277093967815, + "tokens_seen": 1028521984 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005562447342026079, + "loss": 1.2272, + "theoretical_loss": 3.6679057555022325, + "tokens_seen": 1028587520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005562286860581745, + "loss": 1.233, + "theoretical_loss": 3.667883803398052, + "tokens_seen": 1028653056 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005562126379137413, + "loss": 1.3053, + "theoretical_loss": 3.6678618530839806, + "tokens_seen": 1028718592 + }, + { + "epoch": 0.31, + "learning_rate": 0.000556196589769308, + "loss": 1.2998, + "theoretical_loss": 3.6678399045597576, + "tokens_seen": 1028784128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005561805416248747, + "loss": 1.1746, + "theoretical_loss": 3.6678179578251235, + "tokens_seen": 1028849664 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 347142, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3573336601257324, + "objective/train/theoretical_loss": 3.667796012879819, + "objective/train/tokens_used": 1049375200, + "theoretical_loss": 3.667796012879819, + "tokens_seen": 1028915200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005561644934804413, + "loss": 1.3217, + "theoretical_loss": 3.667796012879819, + "tokens_seen": 1028915200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005561484453360081, + "loss": 1.4169, + "theoretical_loss": 3.667774069723583, + "tokens_seen": 1028980736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005561323971915747, + "loss": 1.2895, + "theoretical_loss": 3.667752128356156, + "tokens_seen": 1029046272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005561163490471414, + "loss": 1.3405, + "theoretical_loss": 3.6677301887772793, + "tokens_seen": 1029111808 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005561003009027082, + "loss": 1.3382, + "theoretical_loss": 3.667708250986692, + "tokens_seen": 1029177344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005560842527582748, + "loss": 1.3336, + "theoretical_loss": 3.6676863149841354, + "tokens_seen": 1029242880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005560682046138416, + "loss": 1.333, + "theoretical_loss": 3.6676643807693496, + "tokens_seen": 1029308416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005560521564694082, + "loss": 1.3367, + "theoretical_loss": 3.6676424483420744, + "tokens_seen": 1029373952 + }, + { + "epoch": 0.31, + "learning_rate": 0.000556036108324975, + "loss": 1.3724, + "theoretical_loss": 3.6676205177020513, + "tokens_seen": 1029439488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005560200601805416, + "loss": 1.2695, + "theoretical_loss": 3.6675985888490206, + "tokens_seen": 1029505024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005560040120361084, + "loss": 1.3062, + "theoretical_loss": 3.667576661782723, + "tokens_seen": 1029570560 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555987963891675, + "loss": 1.3483, + "theoretical_loss": 3.6675547365028986, + "tokens_seen": 1029636096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005559719157472418, + "loss": 1.2031, + "theoretical_loss": 3.6675328130092892, + "tokens_seen": 1029701632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005559558676028084, + "loss": 1.4142, + "theoretical_loss": 3.6675108913016348, + "tokens_seen": 1029767168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005559398194583752, + "loss": 1.261, + "theoretical_loss": 3.6674889713796763, + "tokens_seen": 1029832704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005559237713139418, + "loss": 1.3173, + "theoretical_loss": 3.667467053243155, + "tokens_seen": 1029898240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005559077231695086, + "loss": 1.4307, + "theoretical_loss": 3.667445136891812, + "tokens_seen": 1029963776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005558916750250752, + "loss": 1.2764, + "theoretical_loss": 3.6674232223253878, + "tokens_seen": 1030029312 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555875626880642, + "loss": 1.3415, + "theoretical_loss": 3.6674013095436235, + "tokens_seen": 1030094848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005558595787362086, + "loss": 1.3558, + "theoretical_loss": 3.6673793985462604, + "tokens_seen": 1030160384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005558435305917754, + "loss": 1.3806, + "theoretical_loss": 3.6673574893330403, + "tokens_seen": 1030225920 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555827482447342, + "loss": 1.2824, + "theoretical_loss": 3.6673355819037035, + "tokens_seen": 1030291456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005558114343029088, + "loss": 1.4197, + "theoretical_loss": 3.6673136762579914, + "tokens_seen": 1030356992 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005557953861584754, + "loss": 1.3481, + "theoretical_loss": 3.6672917723956457, + "tokens_seen": 1030422528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005557793380140422, + "loss": 1.3848, + "theoretical_loss": 3.6672698703164084, + "tokens_seen": 1030488064 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005557632898696088, + "loss": 1.2509, + "theoretical_loss": 3.6672479700200196, + "tokens_seen": 1030553600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005557472417251756, + "loss": 1.4003, + "theoretical_loss": 3.6672260715062217, + "tokens_seen": 1030619136 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005557311935807422, + "loss": 1.3734, + "theoretical_loss": 3.667204174774756, + "tokens_seen": 1030684672 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555715145436309, + "loss": 1.4423, + "theoretical_loss": 3.6671822798253637, + "tokens_seen": 1030750208 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005556990972918756, + "loss": 1.3854, + "theoretical_loss": 3.6671603866577875, + "tokens_seen": 1030815744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005556830491474424, + "loss": 1.4254, + "theoretical_loss": 3.6671384952717685, + "tokens_seen": 1030881280 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555667001003009, + "loss": 1.4466, + "theoretical_loss": 3.6671166056670486, + "tokens_seen": 1030946816 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005556509528585758, + "loss": 1.2626, + "theoretical_loss": 3.6670947178433693, + "tokens_seen": 1031012352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005556349047141424, + "loss": 1.3451, + "theoretical_loss": 3.667072831800473, + "tokens_seen": 1031077888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005556188565697092, + "loss": 1.4075, + "theoretical_loss": 3.6670509475381015, + "tokens_seen": 1031143424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005556028084252758, + "loss": 1.2638, + "theoretical_loss": 3.6670290650559965, + "tokens_seen": 1031208960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005555867602808426, + "loss": 1.3928, + "theoretical_loss": 3.6670071843539005, + "tokens_seen": 1031274496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005555707121364092, + "loss": 1.3528, + "theoretical_loss": 3.666985305431555, + "tokens_seen": 1031340032 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555554663991976, + "loss": 1.3304, + "theoretical_loss": 3.666963428288703, + "tokens_seen": 1031405568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005555386158475426, + "loss": 1.2793, + "theoretical_loss": 3.666941552925086, + "tokens_seen": 1031471104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005555225677031094, + "loss": 1.2503, + "theoretical_loss": 3.6669196793404466, + "tokens_seen": 1031536640 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555506519558676, + "loss": 1.2333, + "theoretical_loss": 3.666897807534527, + "tokens_seen": 1031602176 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005554904714142428, + "loss": 1.3717, + "theoretical_loss": 3.6668759375070694, + "tokens_seen": 1031667712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005554744232698094, + "loss": 1.3454, + "theoretical_loss": 3.6668540692578167, + "tokens_seen": 1031733248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005554583751253762, + "loss": 1.3405, + "theoretical_loss": 3.6668322027865106, + "tokens_seen": 1031798784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005554423269809428, + "loss": 1.3685, + "theoretical_loss": 3.666810338092895, + "tokens_seen": 1031864320 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005554262788365096, + "loss": 1.4335, + "theoretical_loss": 3.666788475176711, + "tokens_seen": 1031929856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005554102306920762, + "loss": 1.3048, + "theoretical_loss": 3.6667666140377015, + "tokens_seen": 1031995392 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555394182547643, + "loss": 1.388, + "theoretical_loss": 3.6667447546756105, + "tokens_seen": 1032060928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005553781344032096, + "loss": 1.2973, + "theoretical_loss": 3.666722897090179, + "tokens_seen": 1032126464 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 348090, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3344188928604126, + "objective/train/theoretical_loss": 3.6667010412811507, + "objective/train/tokens_used": 1052652000, + "theoretical_loss": 3.6667010412811507, + "tokens_seen": 1032192000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005553620862587764, + "loss": 1.2576, + "theoretical_loss": 3.6667010412811507, + "tokens_seen": 1032192000 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555346038114343, + "loss": 1.31, + "theoretical_loss": 3.6666791872482687, + "tokens_seen": 1032257536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005553299899699098, + "loss": 1.3013, + "theoretical_loss": 3.6666573349912754, + "tokens_seen": 1032323072 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005553139418254764, + "loss": 1.4247, + "theoretical_loss": 3.6666354845099143, + "tokens_seen": 1032388608 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005552978936810432, + "loss": 1.2885, + "theoretical_loss": 3.6666136358039276, + "tokens_seen": 1032454144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005552818455366098, + "loss": 1.4111, + "theoretical_loss": 3.666591788873059, + "tokens_seen": 1032519680 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005552657973921766, + "loss": 1.2379, + "theoretical_loss": 3.666569943717051, + "tokens_seen": 1032585216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005552497492477432, + "loss": 1.3001, + "theoretical_loss": 3.666548100335648, + "tokens_seen": 1032650752 + }, + { + "epoch": 0.31, + "learning_rate": 0.00055523370110331, + "loss": 1.3514, + "theoretical_loss": 3.6665262587285916, + "tokens_seen": 1032716288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005552176529588766, + "loss": 1.3702, + "theoretical_loss": 3.6665044188956264, + "tokens_seen": 1032781824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005552016048144434, + "loss": 1.3201, + "theoretical_loss": 3.6664825808364956, + "tokens_seen": 1032847360 + }, + { + "epoch": 0.31, + "learning_rate": 0.00055518555667001, + "loss": 1.3788, + "theoretical_loss": 3.6664607445509416, + "tokens_seen": 1032912896 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005551695085255768, + "loss": 1.4063, + "theoretical_loss": 3.6664389100387087, + "tokens_seen": 1032978432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005551534603811434, + "loss": 1.4044, + "theoretical_loss": 3.66641707729954, + "tokens_seen": 1033043968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005551374122367102, + "loss": 1.3637, + "theoretical_loss": 3.6663952463331797, + "tokens_seen": 1033109504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005551213640922768, + "loss": 1.3576, + "theoretical_loss": 3.6663734171393707, + "tokens_seen": 1033175040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005551053159478436, + "loss": 1.2985, + "theoretical_loss": 3.666351589717857, + "tokens_seen": 1033240576 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005550892678034102, + "loss": 1.3548, + "theoretical_loss": 3.666329764068382, + "tokens_seen": 1033306112 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555073219658977, + "loss": 1.3824, + "theoretical_loss": 3.6663079401906895, + "tokens_seen": 1033371648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005550571715145436, + "loss": 1.3748, + "theoretical_loss": 3.666286118084524, + "tokens_seen": 1033437184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005550411233701104, + "loss": 1.5217, + "theoretical_loss": 3.6662642977496285, + "tokens_seen": 1033502720 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555025075225677, + "loss": 1.3761, + "theoretical_loss": 3.6662424791857475, + "tokens_seen": 1033568256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005550090270812438, + "loss": 1.267, + "theoretical_loss": 3.666220662392625, + "tokens_seen": 1033633792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005549929789368104, + "loss": 1.4167, + "theoretical_loss": 3.6661988473700045, + "tokens_seen": 1033699328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005549769307923772, + "loss": 1.4184, + "theoretical_loss": 3.66617703411763, + "tokens_seen": 1033764864 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005549608826479438, + "loss": 1.5055, + "theoretical_loss": 3.6661552226352465, + "tokens_seen": 1033830400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005549448345035106, + "loss": 1.4087, + "theoretical_loss": 3.6661334129225978, + "tokens_seen": 1033895936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005549287863590774, + "loss": 1.3579, + "theoretical_loss": 3.6661116049794273, + "tokens_seen": 1033961472 + }, + { + "epoch": 0.31, + "learning_rate": 0.000554912738214644, + "loss": 1.3957, + "theoretical_loss": 3.666089798805481, + "tokens_seen": 1034027008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005548966900702108, + "loss": 1.2851, + "theoretical_loss": 3.6660679944005015, + "tokens_seen": 1034092544 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005548806419257774, + "loss": 1.3388, + "theoretical_loss": 3.6660461917642344, + "tokens_seen": 1034158080 + }, + { + "epoch": 0.31, + "learning_rate": 0.000554864593781344, + "loss": 1.4184, + "theoretical_loss": 3.666024390896424, + "tokens_seen": 1034223616 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005548485456369108, + "loss": 1.369, + "theoretical_loss": 3.666002591796814, + "tokens_seen": 1034289152 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005548324974924774, + "loss": 1.3376, + "theoretical_loss": 3.665980794465149, + "tokens_seen": 1034354688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005548164493480442, + "loss": 1.3469, + "theoretical_loss": 3.6659589989011754, + "tokens_seen": 1034420224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005548004012036108, + "loss": 1.3072, + "theoretical_loss": 3.665937205104636, + "tokens_seen": 1034485760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005547843530591776, + "loss": 1.3264, + "theoretical_loss": 3.665915413075276, + "tokens_seen": 1034551296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005547683049147442, + "loss": 1.469, + "theoretical_loss": 3.66589362281284, + "tokens_seen": 1034616832 + }, + { + "epoch": 0.31, + "learning_rate": 0.000554752256770311, + "loss": 1.3863, + "theoretical_loss": 3.6658718343170738, + "tokens_seen": 1034682368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005547362086258777, + "loss": 1.3203, + "theoretical_loss": 3.665850047587721, + "tokens_seen": 1034747904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005547201604814443, + "loss": 1.4613, + "theoretical_loss": 3.6658282626245278, + "tokens_seen": 1034813440 + }, + { + "epoch": 0.31, + "learning_rate": 0.000554704112337011, + "loss": 1.3122, + "theoretical_loss": 3.6658064794272383, + "tokens_seen": 1034878976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005546880641925777, + "loss": 1.3626, + "theoretical_loss": 3.6657846979955973, + "tokens_seen": 1034944512 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005546720160481445, + "loss": 1.3732, + "theoretical_loss": 3.6657629183293508, + "tokens_seen": 1035010048 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005546559679037111, + "loss": 1.4149, + "theoretical_loss": 3.665741140428243, + "tokens_seen": 1035075584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005546399197592779, + "loss": 1.3765, + "theoretical_loss": 3.66571936429202, + "tokens_seen": 1035141120 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005546238716148445, + "loss": 1.3002, + "theoretical_loss": 3.6656975899204265, + "tokens_seen": 1035206656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005546078234704113, + "loss": 1.3946, + "theoretical_loss": 3.665675817313208, + "tokens_seen": 1035272192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005545917753259779, + "loss": 1.4278, + "theoretical_loss": 3.6656540464701095, + "tokens_seen": 1035337728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005545757271815447, + "loss": 1.4373, + "theoretical_loss": 3.665632277390877, + "tokens_seen": 1035403264 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 349230, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5953205823898315, + "objective/train/theoretical_loss": 3.665610510075256, + "objective/train/tokens_used": 1055928800, + "theoretical_loss": 3.665610510075256, + "tokens_seen": 1035468800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005545596790371113, + "loss": 1.5163, + "theoretical_loss": 3.665610510075256, + "tokens_seen": 1035468800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005545436308926781, + "loss": 1.3309, + "theoretical_loss": 3.6655887445229913, + "tokens_seen": 1035534336 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005545275827482447, + "loss": 1.3277, + "theoretical_loss": 3.6655669807338285, + "tokens_seen": 1035599872 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005545115346038115, + "loss": 1.296, + "theoretical_loss": 3.665545218707514, + "tokens_seen": 1035665408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005544954864593781, + "loss": 1.3078, + "theoretical_loss": 3.6655234584437926, + "tokens_seen": 1035730944 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005544794383149449, + "loss": 1.4387, + "theoretical_loss": 3.665501699942411, + "tokens_seen": 1035796480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005544633901705115, + "loss": 1.3512, + "theoretical_loss": 3.665479943203114, + "tokens_seen": 1035862016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005544473420260783, + "loss": 1.2364, + "theoretical_loss": 3.6654581882256485, + "tokens_seen": 1035927552 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005544312938816449, + "loss": 1.2795, + "theoretical_loss": 3.6654364350097595, + "tokens_seen": 1035993088 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005544152457372117, + "loss": 1.3323, + "theoretical_loss": 3.665414683555193, + "tokens_seen": 1036058624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005543991975927783, + "loss": 1.3176, + "theoretical_loss": 3.665392933861695, + "tokens_seen": 1036124160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005543831494483451, + "loss": 1.3466, + "theoretical_loss": 3.665371185929012, + "tokens_seen": 1036189696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005543671013039117, + "loss": 1.2921, + "theoretical_loss": 3.6653494397568895, + "tokens_seen": 1036255232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005543510531594785, + "loss": 1.3309, + "theoretical_loss": 3.6653276953450744, + "tokens_seen": 1036320768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005543350050150451, + "loss": 1.3214, + "theoretical_loss": 3.6653059526933123, + "tokens_seen": 1036386304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005543189568706119, + "loss": 1.342, + "theoretical_loss": 3.6652842118013496, + "tokens_seen": 1036451840 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005543029087261785, + "loss": 1.342, + "theoretical_loss": 3.6652624726689327, + "tokens_seen": 1036517376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005542868605817453, + "loss": 1.4497, + "theoretical_loss": 3.6652407352958076, + "tokens_seen": 1036582912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005542708124373119, + "loss": 1.3414, + "theoretical_loss": 3.665218999681721, + "tokens_seen": 1036648448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005542547642928787, + "loss": 1.4255, + "theoretical_loss": 3.6651972658264196, + "tokens_seen": 1036713984 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005542387161484453, + "loss": 1.3207, + "theoretical_loss": 3.66517553372965, + "tokens_seen": 1036779520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005542226680040121, + "loss": 1.302, + "theoretical_loss": 3.6651538033911577, + "tokens_seen": 1036845056 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005542066198595787, + "loss": 1.3584, + "theoretical_loss": 3.6651320748106904, + "tokens_seen": 1036910592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005541905717151455, + "loss": 1.2684, + "theoretical_loss": 3.6651103479879943, + "tokens_seen": 1036976128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005541745235707121, + "loss": 1.2881, + "theoretical_loss": 3.665088622922817, + "tokens_seen": 1037041664 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005541584754262789, + "loss": 1.4304, + "theoretical_loss": 3.665066899614904, + "tokens_seen": 1037107200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005541424272818455, + "loss": 1.363, + "theoretical_loss": 3.665045178064002, + "tokens_seen": 1037172736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005541263791374123, + "loss": 1.2961, + "theoretical_loss": 3.6650234582698595, + "tokens_seen": 1037238272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005541103309929789, + "loss": 1.3131, + "theoretical_loss": 3.665001740232222, + "tokens_seen": 1037303808 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005540942828485457, + "loss": 1.3062, + "theoretical_loss": 3.664980023950837, + "tokens_seen": 1037369344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005540782347041123, + "loss": 1.3362, + "theoretical_loss": 3.664958309425452, + "tokens_seen": 1037434880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005540621865596791, + "loss": 1.3453, + "theoretical_loss": 3.664936596655813, + "tokens_seen": 1037500416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005540461384152457, + "loss": 1.4069, + "theoretical_loss": 3.664914885641668, + "tokens_seen": 1037565952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005540300902708125, + "loss": 1.3398, + "theoretical_loss": 3.664893176382764, + "tokens_seen": 1037631488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005540140421263791, + "loss": 1.4761, + "theoretical_loss": 3.6648714688788484, + "tokens_seen": 1037697024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005539979939819459, + "loss": 1.3642, + "theoretical_loss": 3.664849763129668, + "tokens_seen": 1037762560 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005539819458375125, + "loss": 1.3796, + "theoretical_loss": 3.6648280591349707, + "tokens_seen": 1037828096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005539658976930793, + "loss": 1.3271, + "theoretical_loss": 3.6648063568945033, + "tokens_seen": 1037893632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005539498495486459, + "loss": 1.3468, + "theoretical_loss": 3.6647846564080138, + "tokens_seen": 1037959168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005539338014042127, + "loss": 1.2307, + "theoretical_loss": 3.66476295767525, + "tokens_seen": 1038024704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005539177532597793, + "loss": 1.2928, + "theoretical_loss": 3.6647412606959584, + "tokens_seen": 1038090240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005539017051153461, + "loss": 1.29, + "theoretical_loss": 3.664719565469887, + "tokens_seen": 1038155776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005538856569709127, + "loss": 1.3733, + "theoretical_loss": 3.6646978719967844, + "tokens_seen": 1038221312 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005538696088264795, + "loss": 1.1986, + "theoretical_loss": 3.664676180276397, + "tokens_seen": 1038286848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005538535606820461, + "loss": 1.3375, + "theoretical_loss": 3.6646544903084735, + "tokens_seen": 1038352384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005538375125376129, + "loss": 1.2587, + "theoretical_loss": 3.664632802092761, + "tokens_seen": 1038417920 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005538214643931795, + "loss": 1.2895, + "theoretical_loss": 3.6646111156290084, + "tokens_seen": 1038483456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005538054162487463, + "loss": 1.2788, + "theoretical_loss": 3.664589430916963, + "tokens_seen": 1038548992 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005537893681043129, + "loss": 1.2994, + "theoretical_loss": 3.6645677479563723, + "tokens_seen": 1038614528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005537733199598797, + "loss": 1.3, + "theoretical_loss": 3.6645460667469854, + "tokens_seen": 1038680064 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 350251, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3899097442626953, + "objective/train/theoretical_loss": 3.66452438728855, + "objective/train/tokens_used": 1059205600, + "theoretical_loss": 3.66452438728855, + "tokens_seen": 1038745600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005537572718154463, + "loss": 1.332, + "theoretical_loss": 3.66452438728855, + "tokens_seen": 1038745600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005537412236710131, + "loss": 1.3857, + "theoretical_loss": 3.6645027095808134, + "tokens_seen": 1038811136 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005537251755265798, + "loss": 1.2491, + "theoretical_loss": 3.6644810336235243, + "tokens_seen": 1038876672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005537091273821465, + "loss": 1.3745, + "theoretical_loss": 3.6644593594164316, + "tokens_seen": 1038942208 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005536930792377132, + "loss": 1.3818, + "theoretical_loss": 3.6644376869592827, + "tokens_seen": 1039007744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005536770310932799, + "loss": 1.2186, + "theoretical_loss": 3.6644160162518267, + "tokens_seen": 1039073280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005536609829488466, + "loss": 1.3415, + "theoretical_loss": 3.664394347293811, + "tokens_seen": 1039138816 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005536449348044133, + "loss": 1.358, + "theoretical_loss": 3.664372680084986, + "tokens_seen": 1039204352 + }, + { + "epoch": 0.31, + "learning_rate": 0.00055362888665998, + "loss": 1.3312, + "theoretical_loss": 3.6643510146250975, + "tokens_seen": 1039269888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005536128385155467, + "loss": 1.2917, + "theoretical_loss": 3.664329350913896, + "tokens_seen": 1039335424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005535967903711134, + "loss": 1.2901, + "theoretical_loss": 3.6643076889511295, + "tokens_seen": 1039400960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005535807422266801, + "loss": 1.3579, + "theoretical_loss": 3.6642860287365466, + "tokens_seen": 1039466496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005535646940822469, + "loss": 1.3679, + "theoretical_loss": 3.6642643702698963, + "tokens_seen": 1039532032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005535486459378135, + "loss": 1.3271, + "theoretical_loss": 3.6642427135509275, + "tokens_seen": 1039597568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005535325977933803, + "loss": 1.3528, + "theoretical_loss": 3.664221058579388, + "tokens_seen": 1039663104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005535165496489469, + "loss": 1.3484, + "theoretical_loss": 3.6641994053550277, + "tokens_seen": 1039728640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005535005015045137, + "loss": 1.3224, + "theoretical_loss": 3.664177753877595, + "tokens_seen": 1039794176 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005534844533600803, + "loss": 1.2996, + "theoretical_loss": 3.664156104146839, + "tokens_seen": 1039859712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005534684052156471, + "loss": 1.2883, + "theoretical_loss": 3.664134456162509, + "tokens_seen": 1039925248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005534523570712137, + "loss": 1.4707, + "theoretical_loss": 3.6641128099243536, + "tokens_seen": 1039990784 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005534363089267803, + "loss": 1.3435, + "theoretical_loss": 3.664091165432122, + "tokens_seen": 1040056320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005534202607823471, + "loss": 1.2943, + "theoretical_loss": 3.6640695226855646, + "tokens_seen": 1040121856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005534042126379137, + "loss": 1.4396, + "theoretical_loss": 3.6640478816844286, + "tokens_seen": 1040187392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005533881644934805, + "loss": 1.3399, + "theoretical_loss": 3.6640262424284646, + "tokens_seen": 1040252928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005533721163490472, + "loss": 1.3788, + "theoretical_loss": 3.6640046049174213, + "tokens_seen": 1040318464 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005533560682046139, + "loss": 1.3841, + "theoretical_loss": 3.663982969151049, + "tokens_seen": 1040384000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005533400200601806, + "loss": 1.3785, + "theoretical_loss": 3.6639613351290956, + "tokens_seen": 1040449536 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005533239719157473, + "loss": 1.4335, + "theoretical_loss": 3.6639397028513123, + "tokens_seen": 1040515072 + }, + { + "epoch": 0.32, + "learning_rate": 0.000553307923771314, + "loss": 1.2857, + "theoretical_loss": 3.663918072317448, + "tokens_seen": 1040580608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005532918756268806, + "loss": 1.2753, + "theoretical_loss": 3.663896443527251, + "tokens_seen": 1040646144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005532758274824474, + "loss": 1.3353, + "theoretical_loss": 3.663874816480473, + "tokens_seen": 1040711680 + }, + { + "epoch": 0.32, + "learning_rate": 0.000553259779338014, + "loss": 1.3405, + "theoretical_loss": 3.6638531911768624, + "tokens_seen": 1040777216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005532437311935808, + "loss": 1.3946, + "theoretical_loss": 3.6638315676161697, + "tokens_seen": 1040842752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005532276830491474, + "loss": 1.3927, + "theoretical_loss": 3.6638099457981443, + "tokens_seen": 1040908288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005532116349047142, + "loss": 1.3701, + "theoretical_loss": 3.6637883257225354, + "tokens_seen": 1040973824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005531955867602808, + "loss": 1.3353, + "theoretical_loss": 3.6637667073890943, + "tokens_seen": 1041039360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005531795386158476, + "loss": 1.3539, + "theoretical_loss": 3.66374509079757, + "tokens_seen": 1041104896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005531634904714142, + "loss": 1.3463, + "theoretical_loss": 3.6637234759477124, + "tokens_seen": 1041170432 + }, + { + "epoch": 0.32, + "learning_rate": 0.000553147442326981, + "loss": 1.4955, + "theoretical_loss": 3.6637018628392726, + "tokens_seen": 1041235968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005531313941825476, + "loss": 1.2463, + "theoretical_loss": 3.663680251471999, + "tokens_seen": 1041301504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005531153460381144, + "loss": 1.3755, + "theoretical_loss": 3.6636586418456436, + "tokens_seen": 1041367040 + }, + { + "epoch": 0.32, + "learning_rate": 0.000553099297893681, + "loss": 1.4584, + "theoretical_loss": 3.6636370339599553, + "tokens_seen": 1041432576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005530832497492478, + "loss": 1.3345, + "theoretical_loss": 3.663615427814685, + "tokens_seen": 1041498112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005530672016048144, + "loss": 1.3675, + "theoretical_loss": 3.663593823409583, + "tokens_seen": 1041563648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005530511534603812, + "loss": 1.327, + "theoretical_loss": 3.6635722207444, + "tokens_seen": 1041629184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005530351053159478, + "loss": 1.4008, + "theoretical_loss": 3.663550619818885, + "tokens_seen": 1041694720 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005530190571715146, + "loss": 1.3456, + "theoretical_loss": 3.6635290206327897, + "tokens_seen": 1041760256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005530030090270812, + "loss": 1.4695, + "theoretical_loss": 3.663507423185864, + "tokens_seen": 1041825792 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552986960882648, + "loss": 1.2368, + "theoretical_loss": 3.6634858274778597, + "tokens_seen": 1041891328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005529709127382146, + "loss": 1.3337, + "theoretical_loss": 3.6634642335085257, + "tokens_seen": 1041956864 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 351434, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3981451988220215, + "objective/train/theoretical_loss": 3.663442641277614, + "objective/train/tokens_used": 1062482400, + "theoretical_loss": 3.663442641277614, + "tokens_seen": 1042022400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005529548645937814, + "loss": 1.404, + "theoretical_loss": 3.663442641277614, + "tokens_seen": 1042022400 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552938816449348, + "loss": 1.2685, + "theoretical_loss": 3.6634210507848746, + "tokens_seen": 1042087936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005529227683049148, + "loss": 1.2916, + "theoretical_loss": 3.6633994620300587, + "tokens_seen": 1042153472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005529067201604814, + "loss": 1.3839, + "theoretical_loss": 3.663377875012917, + "tokens_seen": 1042219008 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005528906720160482, + "loss": 1.2788, + "theoretical_loss": 3.6633562897331995, + "tokens_seen": 1042284544 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005528746238716148, + "loss": 1.3901, + "theoretical_loss": 3.6633347061906587, + "tokens_seen": 1042350080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005528585757271816, + "loss": 1.3639, + "theoretical_loss": 3.663313124385045, + "tokens_seen": 1042415616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005528425275827482, + "loss": 1.4525, + "theoretical_loss": 3.6632915443161087, + "tokens_seen": 1042481152 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552826479438315, + "loss": 1.3389, + "theoretical_loss": 3.663269965983601, + "tokens_seen": 1042546688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005528104312938816, + "loss": 1.2627, + "theoretical_loss": 3.6632483893872747, + "tokens_seen": 1042612224 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005527943831494484, + "loss": 1.267, + "theoretical_loss": 3.6632268145268796, + "tokens_seen": 1042677760 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552778335005015, + "loss": 1.3791, + "theoretical_loss": 3.6632052414021663, + "tokens_seen": 1042743296 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005527622868605818, + "loss": 1.2854, + "theoretical_loss": 3.6631836700128875, + "tokens_seen": 1042808832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005527462387161484, + "loss": 1.3592, + "theoretical_loss": 3.663162100358794, + "tokens_seen": 1042874368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005527301905717152, + "loss": 1.5055, + "theoretical_loss": 3.6631405324396367, + "tokens_seen": 1042939904 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005527141424272818, + "loss": 1.3553, + "theoretical_loss": 3.663118966255168, + "tokens_seen": 1043005440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005526980942828486, + "loss": 1.4006, + "theoretical_loss": 3.6630974018051385, + "tokens_seen": 1043070976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005526820461384152, + "loss": 1.3105, + "theoretical_loss": 3.6630758390893003, + "tokens_seen": 1043136512 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552665997993982, + "loss": 1.2154, + "theoretical_loss": 3.663054278107405, + "tokens_seen": 1043202048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005526499498495486, + "loss": 1.3248, + "theoretical_loss": 3.6630327188592036, + "tokens_seen": 1043267584 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005526339017051154, + "loss": 1.2735, + "theoretical_loss": 3.663011161344449, + "tokens_seen": 1043333120 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552617853560682, + "loss": 1.3892, + "theoretical_loss": 3.662989605562892, + "tokens_seen": 1043398656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005526018054162488, + "loss": 1.3775, + "theoretical_loss": 3.6629680515142846, + "tokens_seen": 1043464192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005525857572718154, + "loss": 1.4082, + "theoretical_loss": 3.6629464991983784, + "tokens_seen": 1043529728 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005525697091273822, + "loss": 1.413, + "theoretical_loss": 3.6629249486149256, + "tokens_seen": 1043595264 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005525536609829488, + "loss": 1.49, + "theoretical_loss": 3.662903399763678, + "tokens_seen": 1043660800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005525376128385156, + "loss": 1.2857, + "theoretical_loss": 3.6628818526443885, + "tokens_seen": 1043726336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005525215646940823, + "loss": 1.3294, + "theoretical_loss": 3.6628603072568082, + "tokens_seen": 1043791872 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552505516549649, + "loss": 1.3684, + "theoretical_loss": 3.662838763600689, + "tokens_seen": 1043857408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005524894684052157, + "loss": 1.4218, + "theoretical_loss": 3.6628172216757844, + "tokens_seen": 1043922944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005524734202607824, + "loss": 1.4024, + "theoretical_loss": 3.662795681481845, + "tokens_seen": 1043988480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005524573721163491, + "loss": 1.5027, + "theoretical_loss": 3.662774143018624, + "tokens_seen": 1044054016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005524413239719158, + "loss": 1.3652, + "theoretical_loss": 3.6627526062858733, + "tokens_seen": 1044119552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005524252758274825, + "loss": 1.3955, + "theoretical_loss": 3.6627310712833454, + "tokens_seen": 1044185088 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005524092276830492, + "loss": 1.2968, + "theoretical_loss": 3.6627095380107932, + "tokens_seen": 1044250624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005523931795386159, + "loss": 1.385, + "theoretical_loss": 3.6626880064679685, + "tokens_seen": 1044316160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005523771313941826, + "loss": 1.424, + "theoretical_loss": 3.662666476654624, + "tokens_seen": 1044381696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005523610832497493, + "loss": 1.3254, + "theoretical_loss": 3.6626449485705126, + "tokens_seen": 1044447232 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552345035105316, + "loss": 1.3258, + "theoretical_loss": 3.662623422215386, + "tokens_seen": 1044512768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005523289869608827, + "loss": 1.3716, + "theoretical_loss": 3.6626018975889982, + "tokens_seen": 1044578304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005523129388164494, + "loss": 1.2937, + "theoretical_loss": 3.6625803746911014, + "tokens_seen": 1044643840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005522968906720161, + "loss": 1.3599, + "theoretical_loss": 3.6625588535214475, + "tokens_seen": 1044709376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005522808425275828, + "loss": 1.4456, + "theoretical_loss": 3.6625373340797904, + "tokens_seen": 1044774912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005522647943831495, + "loss": 1.3919, + "theoretical_loss": 3.6625158163658824, + "tokens_seen": 1044840448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005522487462387162, + "loss": 1.3677, + "theoretical_loss": 3.6624943003794765, + "tokens_seen": 1044905984 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552232698094283, + "loss": 1.3915, + "theoretical_loss": 3.6624727861203263, + "tokens_seen": 1044971520 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005522166499498496, + "loss": 1.3365, + "theoretical_loss": 3.6624512735881845, + "tokens_seen": 1045037056 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005522006018054164, + "loss": 1.3189, + "theoretical_loss": 3.6624297627828035, + "tokens_seen": 1045102592 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552184553660983, + "loss": 1.4014, + "theoretical_loss": 3.662408253703937, + "tokens_seen": 1045168128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005521685055165498, + "loss": 1.3043, + "theoretical_loss": 3.6623867463513387, + "tokens_seen": 1045233664 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 352278, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2615886926651, + "objective/train/theoretical_loss": 3.662365240724761, + "objective/train/tokens_used": 1065759200, + "theoretical_loss": 3.662365240724761, + "tokens_seen": 1045299200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005521524573721164, + "loss": 1.3378, + "theoretical_loss": 3.662365240724761, + "tokens_seen": 1045299200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005521364092276832, + "loss": 1.4455, + "theoretical_loss": 3.6623437368239573, + "tokens_seen": 1045364736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005521203610832498, + "loss": 1.3465, + "theoretical_loss": 3.6623222346486806, + "tokens_seen": 1045430272 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005521043129388166, + "loss": 1.4097, + "theoretical_loss": 3.662300734198685, + "tokens_seen": 1045495808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005520882647943832, + "loss": 1.4539, + "theoretical_loss": 3.6622792354737244, + "tokens_seen": 1045561344 + }, + { + "epoch": 0.32, + "learning_rate": 0.00055207221664995, + "loss": 1.3983, + "theoretical_loss": 3.662257738473551, + "tokens_seen": 1045626880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005520561685055166, + "loss": 1.4315, + "theoretical_loss": 3.6622362431979187, + "tokens_seen": 1045692416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005520401203610832, + "loss": 1.4431, + "theoretical_loss": 3.662214749646582, + "tokens_seen": 1045757952 + }, + { + "epoch": 0.32, + "learning_rate": 0.00055202407221665, + "loss": 1.3975, + "theoretical_loss": 3.6621932578192933, + "tokens_seen": 1045823488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005520080240722167, + "loss": 1.4283, + "theoretical_loss": 3.6621717677158068, + "tokens_seen": 1045889024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005519919759277834, + "loss": 1.2036, + "theoretical_loss": 3.6621502793358762, + "tokens_seen": 1045954560 + }, + { + "epoch": 0.32, + "learning_rate": 0.00055197592778335, + "loss": 1.2878, + "theoretical_loss": 3.6621287926792556, + "tokens_seen": 1046020096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005519598796389168, + "loss": 1.4843, + "theoretical_loss": 3.6621073077456985, + "tokens_seen": 1046085632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005519438314944835, + "loss": 1.3248, + "theoretical_loss": 3.662085824534959, + "tokens_seen": 1046151168 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005519277833500502, + "loss": 1.2949, + "theoretical_loss": 3.6620643430467905, + "tokens_seen": 1046216704 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005519117352056169, + "loss": 1.3293, + "theoretical_loss": 3.6620428632809476, + "tokens_seen": 1046282240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005518956870611835, + "loss": 1.3417, + "theoretical_loss": 3.6620213852371846, + "tokens_seen": 1046347776 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005518796389167503, + "loss": 1.1959, + "theoretical_loss": 3.661999908915254, + "tokens_seen": 1046413312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005518635907723169, + "loss": 1.2977, + "theoretical_loss": 3.6619784343149124, + "tokens_seen": 1046478848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005518475426278837, + "loss": 1.2672, + "theoretical_loss": 3.6619569614359118, + "tokens_seen": 1046544384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005518314944834503, + "loss": 1.3203, + "theoretical_loss": 3.6619354902780072, + "tokens_seen": 1046609920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005518154463390171, + "loss": 1.3206, + "theoretical_loss": 3.6619140208409533, + "tokens_seen": 1046675456 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005517993981945837, + "loss": 1.3258, + "theoretical_loss": 3.6618925531245043, + "tokens_seen": 1046740992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005517833500501505, + "loss": 1.2913, + "theoretical_loss": 3.6618710871284144, + "tokens_seen": 1046806528 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005517673019057171, + "loss": 1.3044, + "theoretical_loss": 3.661849622852438, + "tokens_seen": 1046872064 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005517512537612839, + "loss": 1.3536, + "theoretical_loss": 3.6618281602963294, + "tokens_seen": 1046937600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005517352056168505, + "loss": 1.3636, + "theoretical_loss": 3.6618066994598437, + "tokens_seen": 1047003136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005517191574724173, + "loss": 1.204, + "theoretical_loss": 3.661785240342735, + "tokens_seen": 1047068672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005517031093279839, + "loss": 1.4292, + "theoretical_loss": 3.6617637829447585, + "tokens_seen": 1047134208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005516870611835507, + "loss": 1.4248, + "theoretical_loss": 3.661742327265668, + "tokens_seen": 1047199744 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005516710130391173, + "loss": 1.2418, + "theoretical_loss": 3.6617208733052196, + "tokens_seen": 1047265280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005516549648946841, + "loss": 1.3601, + "theoretical_loss": 3.6616994210631666, + "tokens_seen": 1047330816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005516389167502507, + "loss": 1.3372, + "theoretical_loss": 3.661677970539265, + "tokens_seen": 1047396352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005516228686058175, + "loss": 1.2883, + "theoretical_loss": 3.661656521733269, + "tokens_seen": 1047461888 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005516068204613841, + "loss": 1.3536, + "theoretical_loss": 3.6616350746449338, + "tokens_seen": 1047527424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005515907723169509, + "loss": 1.4302, + "theoretical_loss": 3.6616136292740142, + "tokens_seen": 1047592960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005515747241725175, + "loss": 1.3126, + "theoretical_loss": 3.6615921856202656, + "tokens_seen": 1047658496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005515586760280843, + "loss": 1.371, + "theoretical_loss": 3.661570743683443, + "tokens_seen": 1047724032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005515426278836509, + "loss": 1.3739, + "theoretical_loss": 3.6615493034633015, + "tokens_seen": 1047789568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005515265797392177, + "loss": 1.3689, + "theoretical_loss": 3.6615278649595964, + "tokens_seen": 1047855104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005515105315947843, + "loss": 1.303, + "theoretical_loss": 3.661506428172083, + "tokens_seen": 1047920640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005514944834503511, + "loss": 1.3732, + "theoretical_loss": 3.661484993100516, + "tokens_seen": 1047986176 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005514784353059177, + "loss": 1.3382, + "theoretical_loss": 3.661463559744651, + "tokens_seen": 1048051712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005514623871614845, + "loss": 1.2, + "theoretical_loss": 3.6614421281042437, + "tokens_seen": 1048117248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005514463390170511, + "loss": 1.2869, + "theoretical_loss": 3.66142069817905, + "tokens_seen": 1048182784 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005514302908726179, + "loss": 1.3309, + "theoretical_loss": 3.6613992699688245, + "tokens_seen": 1048248320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005514142427281845, + "loss": 1.3837, + "theoretical_loss": 3.6613778434733226, + "tokens_seen": 1048313856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005513981945837513, + "loss": 1.4049, + "theoretical_loss": 3.661356418692301, + "tokens_seen": 1048379392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005513821464393179, + "loss": 1.3956, + "theoretical_loss": 3.6613349956255146, + "tokens_seen": 1048444928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005513660982948847, + "loss": 1.4081, + "theoretical_loss": 3.661313574272719, + "tokens_seen": 1048510464 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 353441, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3471620082855225, + "objective/train/theoretical_loss": 3.6612921546336703, + "objective/train/tokens_used": 1069036000, + "theoretical_loss": 3.6612921546336703, + "tokens_seen": 1048576000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005513500501504514, + "loss": 1.3995, + "theoretical_loss": 3.6612921546336703, + "tokens_seen": 1048576000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005513340020060181, + "loss": 1.3516, + "theoretical_loss": 3.6612707367081243, + "tokens_seen": 1048641536 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005513179538615848, + "loss": 1.3101, + "theoretical_loss": 3.6612493204958367, + "tokens_seen": 1048707072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005513019057171515, + "loss": 1.305, + "theoretical_loss": 3.6612279059965633, + "tokens_seen": 1048772608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005512858575727182, + "loss": 1.2781, + "theoretical_loss": 3.6612064932100603, + "tokens_seen": 1048838144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005512698094282849, + "loss": 1.3912, + "theoretical_loss": 3.6611850821360834, + "tokens_seen": 1048903680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005512537612838516, + "loss": 1.3579, + "theoretical_loss": 3.6611636727743893, + "tokens_seen": 1048969216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005512377131394183, + "loss": 1.3985, + "theoretical_loss": 3.661142265124733, + "tokens_seen": 1049034752 + }, + { + "epoch": 0.32, + "learning_rate": 0.000551221664994985, + "loss": 1.2568, + "theoretical_loss": 3.661120859186872, + "tokens_seen": 1049100288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005512056168505517, + "loss": 1.2912, + "theoretical_loss": 3.6610994549605613, + "tokens_seen": 1049165824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005511895687061184, + "loss": 1.3483, + "theoretical_loss": 3.661078052445558, + "tokens_seen": 1049231360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005511735205616851, + "loss": 1.4482, + "theoretical_loss": 3.6610566516416174, + "tokens_seen": 1049296896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005511574724172518, + "loss": 1.1986, + "theoretical_loss": 3.661035252548497, + "tokens_seen": 1049362432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005511414242728185, + "loss": 1.3118, + "theoretical_loss": 3.661013855165953, + "tokens_seen": 1049427968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005511253761283852, + "loss": 1.3358, + "theoretical_loss": 3.660992459493741, + "tokens_seen": 1049493504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005511093279839519, + "loss": 1.4138, + "theoretical_loss": 3.6609710655316183, + "tokens_seen": 1049559040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005510932798395186, + "loss": 1.2601, + "theoretical_loss": 3.660949673279341, + "tokens_seen": 1049624576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005510772316950853, + "loss": 1.2612, + "theoretical_loss": 3.6609282827366663, + "tokens_seen": 1049690112 + }, + { + "epoch": 0.32, + "learning_rate": 0.000551061183550652, + "loss": 1.373, + "theoretical_loss": 3.66090689390335, + "tokens_seen": 1049755648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005510451354062187, + "loss": 1.2842, + "theoretical_loss": 3.660885506779149, + "tokens_seen": 1049821184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005510290872617854, + "loss": 1.3445, + "theoretical_loss": 3.660864121363821, + "tokens_seen": 1049886720 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005510130391173521, + "loss": 1.2281, + "theoretical_loss": 3.6608427376571218, + "tokens_seen": 1049952256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005509969909729188, + "loss": 1.2281, + "theoretical_loss": 3.6608213556588085, + "tokens_seen": 1050017792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005509809428284855, + "loss": 1.377, + "theoretical_loss": 3.660799975368638, + "tokens_seen": 1050083328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005509648946840522, + "loss": 1.4085, + "theoretical_loss": 3.6607785967863675, + "tokens_seen": 1050148864 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005509488465396189, + "loss": 1.3558, + "theoretical_loss": 3.6607572199117535, + "tokens_seen": 1050214400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005509327983951856, + "loss": 1.3707, + "theoretical_loss": 3.6607358447445533, + "tokens_seen": 1050279936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005509167502507523, + "loss": 1.3686, + "theoretical_loss": 3.6607144712845243, + "tokens_seen": 1050345472 + }, + { + "epoch": 0.32, + "learning_rate": 0.000550900702106319, + "loss": 1.3544, + "theoretical_loss": 3.6606930995314233, + "tokens_seen": 1050411008 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005508846539618857, + "loss": 1.2874, + "theoretical_loss": 3.6606717294850077, + "tokens_seen": 1050476544 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005508686058174524, + "loss": 1.3522, + "theoretical_loss": 3.6606503611450343, + "tokens_seen": 1050542080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005508525576730191, + "loss": 1.4388, + "theoretical_loss": 3.6606289945112613, + "tokens_seen": 1050607616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005508365095285859, + "loss": 1.3087, + "theoretical_loss": 3.660607629583445, + "tokens_seen": 1050673152 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005508204613841525, + "loss": 1.2838, + "theoretical_loss": 3.6605862663613435, + "tokens_seen": 1050738688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005508044132397193, + "loss": 1.3144, + "theoretical_loss": 3.6605649048447138, + "tokens_seen": 1050804224 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005507883650952859, + "loss": 1.3996, + "theoretical_loss": 3.660543545033314, + "tokens_seen": 1050869760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005507723169508527, + "loss": 1.2333, + "theoretical_loss": 3.660522186926901, + "tokens_seen": 1050935296 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005507562688064193, + "loss": 1.3352, + "theoretical_loss": 3.6605008305252325, + "tokens_seen": 1051000832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005507402206619861, + "loss": 1.2704, + "theoretical_loss": 3.6604794758280663, + "tokens_seen": 1051066368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005507241725175527, + "loss": 1.3452, + "theoretical_loss": 3.66045812283516, + "tokens_seen": 1051131904 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005507081243731195, + "loss": 1.2633, + "theoretical_loss": 3.660436771546272, + "tokens_seen": 1051197440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005506920762286861, + "loss": 1.313, + "theoretical_loss": 3.660415421961159, + "tokens_seen": 1051262976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005506760280842529, + "loss": 1.2397, + "theoretical_loss": 3.6603940740795795, + "tokens_seen": 1051328512 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005506599799398195, + "loss": 1.3666, + "theoretical_loss": 3.660372727901291, + "tokens_seen": 1051394048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005506439317953861, + "loss": 1.2966, + "theoretical_loss": 3.660351383426052, + "tokens_seen": 1051459584 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005506278836509529, + "loss": 1.3406, + "theoretical_loss": 3.6603300406536206, + "tokens_seen": 1051525120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005506118355065196, + "loss": 1.3207, + "theoretical_loss": 3.660308699583754, + "tokens_seen": 1051590656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005505957873620863, + "loss": 1.243, + "theoretical_loss": 3.660287360216211, + "tokens_seen": 1051656192 + }, + { + "epoch": 0.32, + "learning_rate": 0.000550579739217653, + "loss": 1.2685, + "theoretical_loss": 3.660266022550749, + "tokens_seen": 1051721728 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005505636910732197, + "loss": 1.3218, + "theoretical_loss": 3.6602446865871263, + "tokens_seen": 1051787264 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 354381, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3571959733963013, + "objective/train/theoretical_loss": 3.660223352325102, + "objective/train/tokens_used": 1072312800, + "theoretical_loss": 3.660223352325102, + "tokens_seen": 1051852800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005505476429287864, + "loss": 1.4772, + "theoretical_loss": 3.660223352325102, + "tokens_seen": 1051852800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005505315947843531, + "loss": 1.4361, + "theoretical_loss": 3.660202019764434, + "tokens_seen": 1051918336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005505155466399198, + "loss": 1.2858, + "theoretical_loss": 3.6601806889048802, + "tokens_seen": 1051983872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005504994984954864, + "loss": 1.3927, + "theoretical_loss": 3.6601593597461997, + "tokens_seen": 1052049408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005504834503510532, + "loss": 1.3095, + "theoretical_loss": 3.6601380322881503, + "tokens_seen": 1052114944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005504674022066198, + "loss": 1.3242, + "theoretical_loss": 3.6601167065304905, + "tokens_seen": 1052180480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005504513540621866, + "loss": 1.4475, + "theoretical_loss": 3.6600953824729796, + "tokens_seen": 1052246016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005504353059177532, + "loss": 1.3673, + "theoretical_loss": 3.660074060115375, + "tokens_seen": 1052311552 + }, + { + "epoch": 0.32, + "learning_rate": 0.00055041925777332, + "loss": 1.2225, + "theoretical_loss": 3.660052739457436, + "tokens_seen": 1052377088 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005504032096288866, + "loss": 1.2726, + "theoretical_loss": 3.660031420498922, + "tokens_seen": 1052442624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005503871614844534, + "loss": 1.3033, + "theoretical_loss": 3.66001010323959, + "tokens_seen": 1052508160 + }, + { + "epoch": 0.32, + "learning_rate": 0.00055037111334002, + "loss": 1.3181, + "theoretical_loss": 3.6599887876792003, + "tokens_seen": 1052573696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005503550651955868, + "loss": 1.3174, + "theoretical_loss": 3.6599674738175114, + "tokens_seen": 1052639232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005503390170511534, + "loss": 1.2422, + "theoretical_loss": 3.6599461616542817, + "tokens_seen": 1052704768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005503229689067202, + "loss": 1.2816, + "theoretical_loss": 3.6599248511892704, + "tokens_seen": 1052770304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005503069207622868, + "loss": 1.4043, + "theoretical_loss": 3.659903542422237, + "tokens_seen": 1052835840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005502908726178536, + "loss": 1.295, + "theoretical_loss": 3.6598822353529386, + "tokens_seen": 1052901376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005502748244734202, + "loss": 1.3358, + "theoretical_loss": 3.6598609299811367, + "tokens_seen": 1052966912 + }, + { + "epoch": 0.32, + "learning_rate": 0.000550258776328987, + "loss": 1.3238, + "theoretical_loss": 3.6598396263065895, + "tokens_seen": 1053032448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005502427281845536, + "loss": 1.2124, + "theoretical_loss": 3.6598183243290556, + "tokens_seen": 1053097984 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005502266800401204, + "loss": 1.3889, + "theoretical_loss": 3.659797024048295, + "tokens_seen": 1053163520 + }, + { + "epoch": 0.32, + "learning_rate": 0.000550210631895687, + "loss": 1.411, + "theoretical_loss": 3.6597757254640664, + "tokens_seen": 1053229056 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005501945837512538, + "loss": 1.2472, + "theoretical_loss": 3.6597544285761296, + "tokens_seen": 1053294592 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005501785356068204, + "loss": 1.3524, + "theoretical_loss": 3.6597331333842438, + "tokens_seen": 1053360128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005501624874623872, + "loss": 1.3964, + "theoretical_loss": 3.6597118398881685, + "tokens_seen": 1053425664 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005501464393179539, + "loss": 1.3295, + "theoretical_loss": 3.6596905480876627, + "tokens_seen": 1053491200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005501303911735206, + "loss": 1.2603, + "theoretical_loss": 3.6596692579824865, + "tokens_seen": 1053556736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005501143430290873, + "loss": 1.2604, + "theoretical_loss": 3.659647969572399, + "tokens_seen": 1053622272 + }, + { + "epoch": 0.32, + "learning_rate": 0.000550098294884654, + "loss": 1.4479, + "theoretical_loss": 3.6596266828571604, + "tokens_seen": 1053687808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005500822467402207, + "loss": 1.3373, + "theoretical_loss": 3.65960539783653, + "tokens_seen": 1053753344 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005500661985957874, + "loss": 1.4261, + "theoretical_loss": 3.659584114510267, + "tokens_seen": 1053818880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005500501504513541, + "loss": 1.3478, + "theoretical_loss": 3.6595628328781324, + "tokens_seen": 1053884416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005500341023069208, + "loss": 1.3394, + "theoretical_loss": 3.659541552939885, + "tokens_seen": 1053949952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005500180541624875, + "loss": 1.3559, + "theoretical_loss": 3.659520274695285, + "tokens_seen": 1054015488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005500020060180542, + "loss": 1.5336, + "theoretical_loss": 3.659498998144092, + "tokens_seen": 1054081024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005499859578736209, + "loss": 1.3408, + "theoretical_loss": 3.6594777232860665, + "tokens_seen": 1054146560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005499699097291876, + "loss": 1.3231, + "theoretical_loss": 3.6594564501209685, + "tokens_seen": 1054212096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005499538615847543, + "loss": 1.3286, + "theoretical_loss": 3.6594351786485575, + "tokens_seen": 1054277632 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549937813440321, + "loss": 1.2911, + "theoretical_loss": 3.659413908868594, + "tokens_seen": 1054343168 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005499217652958877, + "loss": 1.3796, + "theoretical_loss": 3.6593926407808377, + "tokens_seen": 1054408704 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005499057171514544, + "loss": 1.4812, + "theoretical_loss": 3.65937137438505, + "tokens_seen": 1054474240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005498896690070211, + "loss": 1.3864, + "theoretical_loss": 3.6593501096809895, + "tokens_seen": 1054539776 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005498736208625878, + "loss": 1.3371, + "theoretical_loss": 3.6593288466684175, + "tokens_seen": 1054605312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005498575727181545, + "loss": 1.2628, + "theoretical_loss": 3.6593075853470944, + "tokens_seen": 1054670848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005498415245737212, + "loss": 1.3075, + "theoretical_loss": 3.6592863257167805, + "tokens_seen": 1054736384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005498254764292879, + "loss": 1.2905, + "theoretical_loss": 3.659265067777236, + "tokens_seen": 1054801920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005498094282848546, + "loss": 1.3658, + "theoretical_loss": 3.6592438115282215, + "tokens_seen": 1054867456 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005497933801404213, + "loss": 1.4136, + "theoretical_loss": 3.6592225569694974, + "tokens_seen": 1054932992 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549777331995988, + "loss": 1.3177, + "theoretical_loss": 3.659201304100825, + "tokens_seen": 1054998528 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005497612838515547, + "loss": 1.3653, + "theoretical_loss": 3.659180052921964, + "tokens_seen": 1055064064 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 355696, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4703857898712158, + "objective/train/theoretical_loss": 3.659158803432675, + "objective/train/tokens_used": 1075589600, + "theoretical_loss": 3.659158803432675, + "tokens_seen": 1055129600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005497452357071214, + "loss": 1.3876, + "theoretical_loss": 3.659158803432675, + "tokens_seen": 1055129600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005497291875626881, + "loss": 1.3574, + "theoretical_loss": 3.6591375556327197, + "tokens_seen": 1055195136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005497131394182548, + "loss": 1.3676, + "theoretical_loss": 3.659116309521859, + "tokens_seen": 1055260672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005496970912738215, + "loss": 1.4873, + "theoretical_loss": 3.6590950650998524, + "tokens_seen": 1055326208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005496810431293882, + "loss": 1.4917, + "theoretical_loss": 3.6590738223664614, + "tokens_seen": 1055391744 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005496649949849549, + "loss": 1.4306, + "theoretical_loss": 3.659052581321448, + "tokens_seen": 1055457280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005496489468405216, + "loss": 1.3966, + "theoretical_loss": 3.6590313419645715, + "tokens_seen": 1055522816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005496328986960883, + "loss": 1.3749, + "theoretical_loss": 3.659010104295594, + "tokens_seen": 1055588352 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549616850551655, + "loss": 1.3479, + "theoretical_loss": 3.6589888683142764, + "tokens_seen": 1055653888 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005496008024072217, + "loss": 1.4363, + "theoretical_loss": 3.6589676340203794, + "tokens_seen": 1055719424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005495847542627884, + "loss": 1.3794, + "theoretical_loss": 3.6589464014136643, + "tokens_seen": 1055784960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005495687061183551, + "loss": 1.2471, + "theoretical_loss": 3.6589251704938928, + "tokens_seen": 1055850496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005495526579739218, + "loss": 1.3435, + "theoretical_loss": 3.658903941260826, + "tokens_seen": 1055916032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005495526579739218, + "loss": 1.4349, + "theoretical_loss": 3.6588827137142252, + "tokens_seen": 1055981568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005495526579739218, + "loss": 1.3724, + "theoretical_loss": 3.6588614878538515, + "tokens_seen": 1056047104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005495366098294885, + "loss": 1.2603, + "theoretical_loss": 3.658840263679467, + "tokens_seen": 1056112640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005495205616850552, + "loss": 1.3944, + "theoretical_loss": 3.6588190411908323, + "tokens_seen": 1056178176 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549504513540622, + "loss": 1.5014, + "theoretical_loss": 3.658797820387709, + "tokens_seen": 1056243712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005494884653961886, + "loss": 1.4993, + "theoretical_loss": 3.6587766012698593, + "tokens_seen": 1056309248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005494724172517554, + "loss": 1.2745, + "theoretical_loss": 3.658755383837045, + "tokens_seen": 1056374784 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549456369107322, + "loss": 1.4437, + "theoretical_loss": 3.658734168089026, + "tokens_seen": 1056440320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005494403209628888, + "loss": 1.4329, + "theoretical_loss": 3.6587129540255665, + "tokens_seen": 1056505856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005494242728184554, + "loss": 1.399, + "theoretical_loss": 3.6586917416464266, + "tokens_seen": 1056571392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005494082246740222, + "loss": 1.3012, + "theoretical_loss": 3.658670530951368, + "tokens_seen": 1056636928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005493921765295888, + "loss": 1.3914, + "theoretical_loss": 3.658649321940154, + "tokens_seen": 1056702464 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005493761283851556, + "loss": 1.2222, + "theoretical_loss": 3.6586281146125454, + "tokens_seen": 1056768000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005493600802407222, + "loss": 1.2534, + "theoretical_loss": 3.658606908968304, + "tokens_seen": 1056833536 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549344032096289, + "loss": 1.3097, + "theoretical_loss": 3.6585857050071926, + "tokens_seen": 1056899072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005493279839518556, + "loss": 1.3939, + "theoretical_loss": 3.6585645027289724, + "tokens_seen": 1056964608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005493119358074224, + "loss": 1.36, + "theoretical_loss": 3.658543302133406, + "tokens_seen": 1057030144 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549295887662989, + "loss": 1.3646, + "theoretical_loss": 3.658522103220256, + "tokens_seen": 1057095680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005492798395185558, + "loss": 1.4481, + "theoretical_loss": 3.6585009059892832, + "tokens_seen": 1057161216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005492637913741224, + "loss": 1.3917, + "theoretical_loss": 3.6584797104402513, + "tokens_seen": 1057226752 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549247743229689, + "loss": 1.4084, + "theoretical_loss": 3.6584585165729218, + "tokens_seen": 1057292288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005492316950852558, + "loss": 1.4677, + "theoretical_loss": 3.6584373243870574, + "tokens_seen": 1057357824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005492156469408225, + "loss": 1.3319, + "theoretical_loss": 3.6584161338824206, + "tokens_seen": 1057423360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005491995987963892, + "loss": 1.3983, + "theoretical_loss": 3.658394945058773, + "tokens_seen": 1057488896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005491835506519559, + "loss": 1.4069, + "theoretical_loss": 3.6583737579158777, + "tokens_seen": 1057554432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005491675025075226, + "loss": 1.348, + "theoretical_loss": 3.6583525724534978, + "tokens_seen": 1057619968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005491514543630893, + "loss": 1.3386, + "theoretical_loss": 3.6583313886713946, + "tokens_seen": 1057685504 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549135406218656, + "loss": 1.2386, + "theoretical_loss": 3.6583102065693316, + "tokens_seen": 1057751040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005491193580742227, + "loss": 1.3445, + "theoretical_loss": 3.6582890261470715, + "tokens_seen": 1057816576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005491033099297893, + "loss": 1.4633, + "theoretical_loss": 3.658267847404377, + "tokens_seen": 1057882112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005490872617853561, + "loss": 1.3658, + "theoretical_loss": 3.6582466703410104, + "tokens_seen": 1057947648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005490712136409227, + "loss": 1.3889, + "theoretical_loss": 3.658225494956735, + "tokens_seen": 1058013184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005490551654964895, + "loss": 1.3035, + "theoretical_loss": 3.658204321251313, + "tokens_seen": 1058078720 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005490391173520561, + "loss": 1.3589, + "theoretical_loss": 3.6581831492245085, + "tokens_seen": 1058144256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005490230692076229, + "loss": 1.4039, + "theoretical_loss": 3.658161978876083, + "tokens_seen": 1058209792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005490070210631895, + "loss": 1.2996, + "theoretical_loss": 3.6581408102058015, + "tokens_seen": 1058275328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005489909729187563, + "loss": 1.3559, + "theoretical_loss": 3.658119643213425, + "tokens_seen": 1058340864 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 356680, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2355961799621582, + "objective/train/theoretical_loss": 3.658098477898718, + "objective/train/tokens_used": 1078866400, + "theoretical_loss": 3.658098477898718, + "tokens_seen": 1058406400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005489749247743229, + "loss": 1.2988, + "theoretical_loss": 3.658098477898718, + "tokens_seen": 1058406400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005489588766298897, + "loss": 1.4177, + "theoretical_loss": 3.6580773142614422, + "tokens_seen": 1058471936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005489428284854564, + "loss": 1.3128, + "theoretical_loss": 3.658056152301363, + "tokens_seen": 1058537472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005489267803410231, + "loss": 1.3534, + "theoretical_loss": 3.6580349920182416, + "tokens_seen": 1058603008 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005489107321965898, + "loss": 1.3942, + "theoretical_loss": 3.658013833411842, + "tokens_seen": 1058668544 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005488946840521565, + "loss": 1.396, + "theoretical_loss": 3.6579926764819284, + "tokens_seen": 1058734080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005488786359077232, + "loss": 1.4598, + "theoretical_loss": 3.657971521228263, + "tokens_seen": 1058799616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005488625877632899, + "loss": 1.3345, + "theoretical_loss": 3.65795036765061, + "tokens_seen": 1058865152 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005488465396188566, + "loss": 1.5156, + "theoretical_loss": 3.6579292157487324, + "tokens_seen": 1058930688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005488304914744233, + "loss": 1.3158, + "theoretical_loss": 3.6579080655223937, + "tokens_seen": 1058996224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00054881444332999, + "loss": 1.3787, + "theoretical_loss": 3.657886916971359, + "tokens_seen": 1059061760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005487983951855567, + "loss": 1.4935, + "theoretical_loss": 3.6578657700953894, + "tokens_seen": 1059127296 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005487823470411234, + "loss": 1.4037, + "theoretical_loss": 3.6578446248942504, + "tokens_seen": 1059192832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005487662988966901, + "loss": 1.2548, + "theoretical_loss": 3.6578234813677053, + "tokens_seen": 1059258368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005487502507522568, + "loss": 1.3403, + "theoretical_loss": 3.6578023395155173, + "tokens_seen": 1059323904 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005487342026078235, + "loss": 1.3848, + "theoretical_loss": 3.6577811993374514, + "tokens_seen": 1059389440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005487181544633902, + "loss": 1.4138, + "theoretical_loss": 3.6577600608332705, + "tokens_seen": 1059454976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005487021063189569, + "loss": 1.2685, + "theoretical_loss": 3.6577389240027394, + "tokens_seen": 1059520512 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005486860581745236, + "loss": 1.4151, + "theoretical_loss": 3.657717788845621, + "tokens_seen": 1059586048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005486700100300903, + "loss": 1.4643, + "theoretical_loss": 3.65769665536168, + "tokens_seen": 1059651584 + }, + { + "epoch": 0.32, + "learning_rate": 0.000548653961885657, + "loss": 1.2092, + "theoretical_loss": 3.6576755235506804, + "tokens_seen": 1059717120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005486379137412237, + "loss": 1.2998, + "theoretical_loss": 3.6576543934123866, + "tokens_seen": 1059782656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005486218655967904, + "loss": 1.3523, + "theoretical_loss": 3.657633264946562, + "tokens_seen": 1059848192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005486058174523571, + "loss": 1.3499, + "theoretical_loss": 3.6576121381529716, + "tokens_seen": 1059913728 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005485897693079238, + "loss": 1.2299, + "theoretical_loss": 3.657591013031379, + "tokens_seen": 1059979264 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005485737211634905, + "loss": 1.3489, + "theoretical_loss": 3.6575698895815494, + "tokens_seen": 1060044800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005485576730190572, + "loss": 1.437, + "theoretical_loss": 3.657548767803246, + "tokens_seen": 1060110336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005485416248746239, + "loss": 1.3298, + "theoretical_loss": 3.6575276476962335, + "tokens_seen": 1060175872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005485255767301906, + "loss": 1.3867, + "theoretical_loss": 3.6575065292602775, + "tokens_seen": 1060241408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005485095285857573, + "loss": 1.3321, + "theoretical_loss": 3.6574854124951415, + "tokens_seen": 1060306944 + }, + { + "epoch": 0.32, + "learning_rate": 0.000548493480441324, + "loss": 1.2883, + "theoretical_loss": 3.65746429740059, + "tokens_seen": 1060372480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005484774322968907, + "loss": 1.2418, + "theoretical_loss": 3.6574431839763877, + "tokens_seen": 1060438016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005484613841524574, + "loss": 1.3314, + "theoretical_loss": 3.6574220722222996, + "tokens_seen": 1060503552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005484453360080241, + "loss": 1.4457, + "theoretical_loss": 3.65740096213809, + "tokens_seen": 1060569088 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005484292878635908, + "loss": 1.4252, + "theoretical_loss": 3.657379853723524, + "tokens_seen": 1060634624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005484132397191575, + "loss": 1.2749, + "theoretical_loss": 3.657358746978366, + "tokens_seen": 1060700160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005483971915747242, + "loss": 1.3305, + "theoretical_loss": 3.6573376419023815, + "tokens_seen": 1060765696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005483811434302909, + "loss": 1.4029, + "theoretical_loss": 3.6573165384953343, + "tokens_seen": 1060831232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005483650952858576, + "loss": 1.3269, + "theoretical_loss": 3.6572954367569905, + "tokens_seen": 1060896768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005483490471414243, + "loss": 1.3708, + "theoretical_loss": 3.657274336687114, + "tokens_seen": 1060962304 + }, + { + "epoch": 0.32, + "learning_rate": 0.000548332998996991, + "loss": 1.3537, + "theoretical_loss": 3.6572532382854708, + "tokens_seen": 1061027840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005483169508525577, + "loss": 1.4125, + "theoretical_loss": 3.6572321415518254, + "tokens_seen": 1061093376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005483009027081244, + "loss": 1.3706, + "theoretical_loss": 3.657211046485943, + "tokens_seen": 1061158912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005482848545636911, + "loss": 1.2991, + "theoretical_loss": 3.6571899530875887, + "tokens_seen": 1061224448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005482688064192578, + "loss": 1.2968, + "theoretical_loss": 3.657168861356528, + "tokens_seen": 1061289984 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005482527582748245, + "loss": 1.3236, + "theoretical_loss": 3.657147771292526, + "tokens_seen": 1061355520 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005482367101303912, + "loss": 1.3278, + "theoretical_loss": 3.657126682895348, + "tokens_seen": 1061421056 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005482206619859579, + "loss": 1.4331, + "theoretical_loss": 3.65710559616476, + "tokens_seen": 1061486592 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005482046138415246, + "loss": 1.3522, + "theoretical_loss": 3.6570845111005257, + "tokens_seen": 1061552128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005481885656970913, + "loss": 1.4438, + "theoretical_loss": 3.657063427702412, + "tokens_seen": 1061617664 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 357916, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2271572351455688, + "objective/train/theoretical_loss": 3.6570423459701846, + "objective/train/tokens_used": 1082143200, + "theoretical_loss": 3.6570423459701846, + "tokens_seen": 1061683200 + }, + { + "epoch": 0.32, + "learning_rate": 0.000548172517552658, + "loss": 1.37, + "theoretical_loss": 3.6570423459701846, + "tokens_seen": 1061683200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005481564694082247, + "loss": 1.4331, + "theoretical_loss": 3.657021265903608, + "tokens_seen": 1061748736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005481404212637914, + "loss": 1.3292, + "theoretical_loss": 3.6570001875024483, + "tokens_seen": 1061814272 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005481243731193581, + "loss": 1.2978, + "theoretical_loss": 3.6569791107664713, + "tokens_seen": 1061879808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005481083249749249, + "loss": 1.3009, + "theoretical_loss": 3.6569580356954425, + "tokens_seen": 1061945344 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005480922768304915, + "loss": 1.4697, + "theoretical_loss": 3.6569369622891283, + "tokens_seen": 1062010880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005480762286860583, + "loss": 1.3022, + "theoretical_loss": 3.6569158905472934, + "tokens_seen": 1062076416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005480601805416249, + "loss": 1.3102, + "theoretical_loss": 3.656894820469704, + "tokens_seen": 1062141952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005480441323971917, + "loss": 1.3952, + "theoretical_loss": 3.656873752056126, + "tokens_seen": 1062207488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005480280842527583, + "loss": 1.4186, + "theoretical_loss": 3.6568526853063252, + "tokens_seen": 1062273024 + }, + { + "epoch": 0.32, + "learning_rate": 0.000548012036108325, + "loss": 1.3164, + "theoretical_loss": 3.6568316202200686, + "tokens_seen": 1062338560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005479959879638917, + "loss": 1.3217, + "theoretical_loss": 3.656810556797121, + "tokens_seen": 1062404096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005479799398194585, + "loss": 1.2744, + "theoretical_loss": 3.6567894950372493, + "tokens_seen": 1062469632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005479638916750251, + "loss": 1.2897, + "theoretical_loss": 3.6567684349402185, + "tokens_seen": 1062535168 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005479478435305919, + "loss": 1.2689, + "theoretical_loss": 3.6567473765057956, + "tokens_seen": 1062600704 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005479317953861585, + "loss": 1.4276, + "theoretical_loss": 3.656726319733747, + "tokens_seen": 1062666240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005479157472417253, + "loss": 1.2482, + "theoretical_loss": 3.656705264623839, + "tokens_seen": 1062731776 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005478996990972919, + "loss": 1.401, + "theoretical_loss": 3.656684211175837, + "tokens_seen": 1062797312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005478836509528587, + "loss": 1.2575, + "theoretical_loss": 3.6566631593895083, + "tokens_seen": 1062862848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005478676028084253, + "loss": 1.4389, + "theoretical_loss": 3.656642109264619, + "tokens_seen": 1062928384 + }, + { + "epoch": 0.32, + "learning_rate": 0.000547851554663992, + "loss": 1.4014, + "theoretical_loss": 3.6566210608009353, + "tokens_seen": 1062993920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005478355065195587, + "loss": 1.3772, + "theoretical_loss": 3.6566000139982235, + "tokens_seen": 1063059456 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005478194583751254, + "loss": 1.3588, + "theoretical_loss": 3.656578968856251, + "tokens_seen": 1063124992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005478034102306921, + "loss": 1.4303, + "theoretical_loss": 3.656557925374784, + "tokens_seen": 1063190528 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005477873620862588, + "loss": 1.4474, + "theoretical_loss": 3.6565368835535885, + "tokens_seen": 1063256064 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005477713139418255, + "loss": 1.2801, + "theoretical_loss": 3.656515843392432, + "tokens_seen": 1063321600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005477552657973922, + "loss": 1.4602, + "theoretical_loss": 3.6564948048910813, + "tokens_seen": 1063387136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005477392176529589, + "loss": 1.2746, + "theoretical_loss": 3.656473768049303, + "tokens_seen": 1063452672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005477231695085256, + "loss": 1.301, + "theoretical_loss": 3.6564527328668635, + "tokens_seen": 1063518208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005477071213640923, + "loss": 1.3755, + "theoretical_loss": 3.6564316993435297, + "tokens_seen": 1063583744 + }, + { + "epoch": 0.32, + "learning_rate": 0.000547691073219659, + "loss": 1.399, + "theoretical_loss": 3.656410667479069, + "tokens_seen": 1063649280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005476750250752257, + "loss": 1.3038, + "theoretical_loss": 3.6563896372732483, + "tokens_seen": 1063714816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005476589769307924, + "loss": 1.3797, + "theoretical_loss": 3.6563686087258347, + "tokens_seen": 1063780352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005476429287863591, + "loss": 1.3812, + "theoretical_loss": 3.6563475818365943, + "tokens_seen": 1063845888 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005476268806419258, + "loss": 1.4176, + "theoretical_loss": 3.6563265566052956, + "tokens_seen": 1063911424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005476108324974925, + "loss": 1.3154, + "theoretical_loss": 3.6563055330317047, + "tokens_seen": 1063976960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005475947843530592, + "loss": 1.3561, + "theoretical_loss": 3.6562845111155893, + "tokens_seen": 1064042496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005475787362086259, + "loss": 1.2732, + "theoretical_loss": 3.656263490856717, + "tokens_seen": 1064108032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005475626880641926, + "loss": 1.3845, + "theoretical_loss": 3.6562424722548545, + "tokens_seen": 1064173568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005475466399197593, + "loss": 1.3015, + "theoretical_loss": 3.656221455309769, + "tokens_seen": 1064239104 + }, + { + "epoch": 0.32, + "learning_rate": 0.000547530591775326, + "loss": 1.28, + "theoretical_loss": 3.6562004400212285, + "tokens_seen": 1064304640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005475145436308927, + "loss": 1.3322, + "theoretical_loss": 3.6561794263890004, + "tokens_seen": 1064370176 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005474984954864594, + "loss": 1.2637, + "theoretical_loss": 3.656158414412851, + "tokens_seen": 1064435712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005474824473420261, + "loss": 1.3027, + "theoretical_loss": 3.65613740409255, + "tokens_seen": 1064501248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005474663991975928, + "loss": 1.3667, + "theoretical_loss": 3.656116395427863, + "tokens_seen": 1064566784 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005474503510531595, + "loss": 1.3948, + "theoretical_loss": 3.6560953884185583, + "tokens_seen": 1064632320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005474343029087262, + "loss": 1.3762, + "theoretical_loss": 3.6560743830644036, + "tokens_seen": 1064697856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005474182547642929, + "loss": 1.3312, + "theoretical_loss": 3.656053379365167, + "tokens_seen": 1064763392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005474022066198596, + "loss": 1.3581, + "theoretical_loss": 3.656032377320616, + "tokens_seen": 1064828928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005473861584754263, + "loss": 1.353, + "theoretical_loss": 3.6560113769305183, + "tokens_seen": 1064894464 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 359144, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4166908264160156, + "objective/train/theoretical_loss": 3.655990378194642, + "objective/train/tokens_used": 1085420000, + "theoretical_loss": 3.655990378194642, + "tokens_seen": 1064960000 + }, + { + "epoch": 0.32, + "learning_rate": 0.000547370110330993, + "loss": 1.3862, + "theoretical_loss": 3.655990378194642, + "tokens_seen": 1064960000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005473540621865597, + "loss": 1.4587, + "theoretical_loss": 3.655969381112754, + "tokens_seen": 1065025536 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005473380140421264, + "loss": 1.4051, + "theoretical_loss": 3.655948385684624, + "tokens_seen": 1065091072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005473219658976931, + "loss": 1.3726, + "theoretical_loss": 3.6559273919100184, + "tokens_seen": 1065156608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005473059177532598, + "loss": 1.3313, + "theoretical_loss": 3.6559063997887065, + "tokens_seen": 1065222144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005472898696088265, + "loss": 1.3552, + "theoretical_loss": 3.6558854093204554, + "tokens_seen": 1065287680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005472738214643932, + "loss": 1.3286, + "theoretical_loss": 3.655864420505034, + "tokens_seen": 1065353216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005472577733199599, + "loss": 1.32, + "theoretical_loss": 3.65584343334221, + "tokens_seen": 1065418752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005472417251755266, + "loss": 1.3726, + "theoretical_loss": 3.655822447831752, + "tokens_seen": 1065484288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005472256770310933, + "loss": 1.4471, + "theoretical_loss": 3.655801463973428, + "tokens_seen": 1065549824 + }, + { + "epoch": 0.32, + "learning_rate": 0.00054720962888666, + "loss": 1.4042, + "theoretical_loss": 3.6557804817670063, + "tokens_seen": 1065615360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005471935807422267, + "loss": 1.3077, + "theoretical_loss": 3.6557595012122555, + "tokens_seen": 1065680896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005471775325977934, + "loss": 1.295, + "theoretical_loss": 3.655738522308944, + "tokens_seen": 1065746432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005471614844533601, + "loss": 1.3996, + "theoretical_loss": 3.65571754505684, + "tokens_seen": 1065811968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005471454363089268, + "loss": 1.2835, + "theoretical_loss": 3.6556965694557126, + "tokens_seen": 1065877504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005471293881644935, + "loss": 1.3876, + "theoretical_loss": 3.6556755955053295, + "tokens_seen": 1065943040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005471133400200602, + "loss": 1.3348, + "theoretical_loss": 3.65565462320546, + "tokens_seen": 1066008576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005470972918756269, + "loss": 1.3571, + "theoretical_loss": 3.6556336525558724, + "tokens_seen": 1066074112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005470812437311936, + "loss": 1.2934, + "theoretical_loss": 3.6556126835563356, + "tokens_seen": 1066139648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005470651955867603, + "loss": 1.3368, + "theoretical_loss": 3.6555917162066187, + "tokens_seen": 1066205184 + }, + { + "epoch": 0.32, + "learning_rate": 0.000547049147442327, + "loss": 1.3447, + "theoretical_loss": 3.65557075050649, + "tokens_seen": 1066270720 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005470330992978937, + "loss": 1.332, + "theoretical_loss": 3.6555497864557176, + "tokens_seen": 1066336256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005470170511534604, + "loss": 1.3861, + "theoretical_loss": 3.655528824054072, + "tokens_seen": 1066401792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005470010030090271, + "loss": 1.2749, + "theoretical_loss": 3.655507863301321, + "tokens_seen": 1066467328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005469849548645938, + "loss": 1.2881, + "theoretical_loss": 3.655486904197234, + "tokens_seen": 1066532864 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005469689067201605, + "loss": 1.3396, + "theoretical_loss": 3.65546594674158, + "tokens_seen": 1066598400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005469528585757272, + "loss": 1.3189, + "theoretical_loss": 3.6554449909341287, + "tokens_seen": 1066663936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005469368104312939, + "loss": 1.4198, + "theoretical_loss": 3.6554240367746473, + "tokens_seen": 1066729472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005469207622868606, + "loss": 1.2967, + "theoretical_loss": 3.6554030842629075, + "tokens_seen": 1066795008 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005469047141424273, + "loss": 1.2496, + "theoretical_loss": 3.6553821333986765, + "tokens_seen": 1066860544 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546888665997994, + "loss": 1.2873, + "theoretical_loss": 3.6553611841817246, + "tokens_seen": 1066926080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005468726178535607, + "loss": 1.3809, + "theoretical_loss": 3.6553402366118206, + "tokens_seen": 1066991616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005468565697091274, + "loss": 1.3995, + "theoretical_loss": 3.6553192906887344, + "tokens_seen": 1067057152 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005468405215646941, + "loss": 1.359, + "theoretical_loss": 3.655298346412235, + "tokens_seen": 1067122688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005468244734202608, + "loss": 1.3591, + "theoretical_loss": 3.655277403782092, + "tokens_seen": 1067188224 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005468084252758275, + "loss": 1.4017, + "theoretical_loss": 3.6552564627980746, + "tokens_seen": 1067253760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005467923771313942, + "loss": 1.3742, + "theoretical_loss": 3.655235523459953, + "tokens_seen": 1067319296 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546776328986961, + "loss": 1.3002, + "theoretical_loss": 3.655214585767496, + "tokens_seen": 1067384832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005467602808425276, + "loss": 1.2625, + "theoretical_loss": 3.6551936497204736, + "tokens_seen": 1067450368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005467442326980943, + "loss": 1.2739, + "theoretical_loss": 3.6551727153186553, + "tokens_seen": 1067515904 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546728184553661, + "loss": 1.2936, + "theoretical_loss": 3.6551517825618114, + "tokens_seen": 1067581440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005467121364092278, + "loss": 1.4605, + "theoretical_loss": 3.6551308514497114, + "tokens_seen": 1067646976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005466960882647944, + "loss": 1.3799, + "theoretical_loss": 3.6551099219821244, + "tokens_seen": 1067712512 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005466800401203612, + "loss": 1.3642, + "theoretical_loss": 3.655088994158821, + "tokens_seen": 1067778048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005466639919759278, + "loss": 1.3985, + "theoretical_loss": 3.6550680679795713, + "tokens_seen": 1067843584 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005466479438314946, + "loss": 1.4229, + "theoretical_loss": 3.655047143444145, + "tokens_seen": 1067909120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005466318956870612, + "loss": 1.3782, + "theoretical_loss": 3.655026220552312, + "tokens_seen": 1067974656 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546615847542628, + "loss": 1.338, + "theoretical_loss": 3.655005299303842, + "tokens_seen": 1068040192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005465997993981946, + "loss": 1.2879, + "theoretical_loss": 3.6549843796985053, + "tokens_seen": 1068105728 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005465837512537614, + "loss": 1.3942, + "theoretical_loss": 3.654963461736073, + "tokens_seen": 1068171264 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 360072, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2868728637695312, + "objective/train/theoretical_loss": 3.654942545416314, + "objective/train/tokens_used": 1088696800, + "theoretical_loss": 3.654942545416314, + "tokens_seen": 1068236800 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546567703109328, + "loss": 1.3204, + "theoretical_loss": 3.654942545416314, + "tokens_seen": 1068236800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005465516549648948, + "loss": 1.3386, + "theoretical_loss": 3.6549216307389996, + "tokens_seen": 1068302336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005465356068204614, + "loss": 1.4649, + "theoretical_loss": 3.6549007177038986, + "tokens_seen": 1068367872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005465195586760282, + "loss": 1.3254, + "theoretical_loss": 3.654879806310783, + "tokens_seen": 1068433408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005465035105315948, + "loss": 1.416, + "theoretical_loss": 3.654858896559422, + "tokens_seen": 1068498944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005464874623871616, + "loss": 1.3876, + "theoretical_loss": 3.654837988449587, + "tokens_seen": 1068564480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005464714142427282, + "loss": 1.3284, + "theoretical_loss": 3.6548170819810477, + "tokens_seen": 1068630016 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546455366098295, + "loss": 1.3933, + "theoretical_loss": 3.6547961771535746, + "tokens_seen": 1068695552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005464393179538616, + "loss": 1.3981, + "theoretical_loss": 3.654775273966939, + "tokens_seen": 1068761088 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005464232698094283, + "loss": 1.2899, + "theoretical_loss": 3.6547543724209106, + "tokens_seen": 1068826624 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546407221664995, + "loss": 1.3313, + "theoretical_loss": 3.654733472515261, + "tokens_seen": 1068892160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005463911735205617, + "loss": 1.3814, + "theoretical_loss": 3.6547125742497597, + "tokens_seen": 1068957696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005463751253761284, + "loss": 1.3791, + "theoretical_loss": 3.6546916776241787, + "tokens_seen": 1069023232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005463590772316951, + "loss": 1.3717, + "theoretical_loss": 3.654670782638288, + "tokens_seen": 1069088768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005463430290872618, + "loss": 1.4064, + "theoretical_loss": 3.654649889291858, + "tokens_seen": 1069154304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005463269809428285, + "loss": 1.3779, + "theoretical_loss": 3.654628997584661, + "tokens_seen": 1069219840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005463109327983952, + "loss": 1.4312, + "theoretical_loss": 3.6546081075164665, + "tokens_seen": 1069285376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005462948846539619, + "loss": 1.3432, + "theoretical_loss": 3.6545872190870465, + "tokens_seen": 1069350912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005462788365095286, + "loss": 1.3097, + "theoretical_loss": 3.6545663322961714, + "tokens_seen": 1069416448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005462627883650953, + "loss": 1.5095, + "theoretical_loss": 3.6545454471436125, + "tokens_seen": 1069481984 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546246740220662, + "loss": 1.4114, + "theoretical_loss": 3.654524563629141, + "tokens_seen": 1069547520 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005462306920762287, + "loss": 1.4444, + "theoretical_loss": 3.6545036817525283, + "tokens_seen": 1069613056 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005462146439317954, + "loss": 1.3725, + "theoretical_loss": 3.654482801513545, + "tokens_seen": 1069678592 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005461985957873621, + "loss": 1.3284, + "theoretical_loss": 3.654461922911963, + "tokens_seen": 1069744128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005461825476429288, + "loss": 1.4145, + "theoretical_loss": 3.6544410459475527, + "tokens_seen": 1069809664 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005461664994984955, + "loss": 1.233, + "theoretical_loss": 3.654420170620086, + "tokens_seen": 1069875200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005461504513540622, + "loss": 1.242, + "theoretical_loss": 3.654399296929335, + "tokens_seen": 1069940736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005461344032096289, + "loss": 1.352, + "theoretical_loss": 3.6543784248750697, + "tokens_seen": 1070006272 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005461183550651956, + "loss": 1.3607, + "theoretical_loss": 3.6543575544570626, + "tokens_seen": 1070071808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005461023069207623, + "loss": 1.4777, + "theoretical_loss": 3.6543366856750845, + "tokens_seen": 1070137344 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546086258776329, + "loss": 1.2618, + "theoretical_loss": 3.6543158185289077, + "tokens_seen": 1070202880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005460702106318957, + "loss": 1.3509, + "theoretical_loss": 3.6542949530183035, + "tokens_seen": 1070268416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005460541624874624, + "loss": 1.3639, + "theoretical_loss": 3.654274089143043, + "tokens_seen": 1070333952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005460381143430291, + "loss": 1.4166, + "theoretical_loss": 3.654253226902899, + "tokens_seen": 1070399488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005460220661985958, + "loss": 1.3486, + "theoretical_loss": 3.6542323662976424, + "tokens_seen": 1070465024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005460060180541625, + "loss": 1.4473, + "theoretical_loss": 3.654211507327046, + "tokens_seen": 1070530560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005459899699097292, + "loss": 1.2746, + "theoretical_loss": 3.6541906499908805, + "tokens_seen": 1070596096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005459739217652959, + "loss": 1.3722, + "theoretical_loss": 3.6541697942889177, + "tokens_seen": 1070661632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005459578736208626, + "loss": 1.4536, + "theoretical_loss": 3.654148940220931, + "tokens_seen": 1070727168 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005459418254764293, + "loss": 1.3258, + "theoretical_loss": 3.654128087786691, + "tokens_seen": 1070792704 + }, + { + "epoch": 0.32, + "learning_rate": 0.000545925777331996, + "loss": 1.3775, + "theoretical_loss": 3.65410723698597, + "tokens_seen": 1070858240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005459097291875627, + "loss": 1.3203, + "theoretical_loss": 3.654086387818541, + "tokens_seen": 1070923776 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005458936810431294, + "loss": 1.3357, + "theoretical_loss": 3.654065540284175, + "tokens_seen": 1070989312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005458776328986961, + "loss": 1.4537, + "theoretical_loss": 3.6540446943826446, + "tokens_seen": 1071054848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005458615847542628, + "loss": 1.3457, + "theoretical_loss": 3.654023850113722, + "tokens_seen": 1071120384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005458455366098295, + "loss": 1.4527, + "theoretical_loss": 3.654003007477179, + "tokens_seen": 1071185920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005458294884653962, + "loss": 1.3514, + "theoretical_loss": 3.6539821664727894, + "tokens_seen": 1071251456 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005458134403209629, + "loss": 1.3989, + "theoretical_loss": 3.6539613271003235, + "tokens_seen": 1071316992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005457973921765296, + "loss": 1.5363, + "theoretical_loss": 3.653940489359555, + "tokens_seen": 1071382528 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005457813440320963, + "loss": 1.473, + "theoretical_loss": 3.6539196532502562, + "tokens_seen": 1071448064 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 361363, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2865943908691406, + "objective/train/theoretical_loss": 3.6538988187721997, + "objective/train/tokens_used": 1091973600, + "theoretical_loss": 3.6538988187721997, + "tokens_seen": 1071513600 + }, + { + "epoch": 0.32, + "learning_rate": 0.000545765295887663, + "loss": 1.3613, + "theoretical_loss": 3.6538988187721997, + "tokens_seen": 1071513600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005457492477432297, + "loss": 1.326, + "theoretical_loss": 3.653877985925157, + "tokens_seen": 1071579136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005457331995987964, + "loss": 1.456, + "theoretical_loss": 3.653857154708902, + "tokens_seen": 1071644672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005457171514543631, + "loss": 1.3492, + "theoretical_loss": 3.6538363251232067, + "tokens_seen": 1071710208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005457011033099298, + "loss": 1.3966, + "theoretical_loss": 3.6538154971678436, + "tokens_seen": 1071775744 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005456850551654965, + "loss": 1.2911, + "theoretical_loss": 3.6537946708425864, + "tokens_seen": 1071841280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005456690070210632, + "loss": 1.1557, + "theoretical_loss": 3.653773846147206, + "tokens_seen": 1071906816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005456529588766299, + "loss": 1.3184, + "theoretical_loss": 3.653753023081477, + "tokens_seen": 1071972352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005456369107321966, + "loss": 1.3235, + "theoretical_loss": 3.6537322016451714, + "tokens_seen": 1072037888 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005456208625877633, + "loss": 1.3949, + "theoretical_loss": 3.653711381838062, + "tokens_seen": 1072103424 + }, + { + "epoch": 0.32, + "learning_rate": 0.00054560481444333, + "loss": 1.3414, + "theoretical_loss": 3.6536905636599224, + "tokens_seen": 1072168960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005455887662988967, + "loss": 1.3246, + "theoretical_loss": 3.6536697471105253, + "tokens_seen": 1072234496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005455727181544634, + "loss": 1.4048, + "theoretical_loss": 3.6536489321896433, + "tokens_seen": 1072300032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005455566700100301, + "loss": 1.2653, + "theoretical_loss": 3.65362811889705, + "tokens_seen": 1072365568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005455406218655968, + "loss": 1.3582, + "theoretical_loss": 3.653607307232518, + "tokens_seen": 1072431104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005455245737211635, + "loss": 1.3284, + "theoretical_loss": 3.653586497195821, + "tokens_seen": 1072496640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005455085255767302, + "loss": 1.4269, + "theoretical_loss": 3.653565688786732, + "tokens_seen": 1072562176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005454924774322969, + "loss": 1.33, + "theoretical_loss": 3.6535448820050247, + "tokens_seen": 1072627712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005454764292878636, + "loss": 1.3119, + "theoretical_loss": 3.6535240768504718, + "tokens_seen": 1072693248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005454603811434303, + "loss": 1.3163, + "theoretical_loss": 3.6535032733228467, + "tokens_seen": 1072758784 + }, + { + "epoch": 0.33, + "learning_rate": 0.000545444332998997, + "loss": 1.3202, + "theoretical_loss": 3.6534824714219227, + "tokens_seen": 1072824320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005454282848545637, + "loss": 1.3219, + "theoretical_loss": 3.6534616711474737, + "tokens_seen": 1072889856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005454122367101304, + "loss": 1.294, + "theoretical_loss": 3.653440872499273, + "tokens_seen": 1072955392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005453961885656971, + "loss": 1.3378, + "theoretical_loss": 3.6534200754770945, + "tokens_seen": 1073020928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005453801404212638, + "loss": 1.3848, + "theoretical_loss": 3.653399280080711, + "tokens_seen": 1073086464 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005453640922768305, + "loss": 1.4149, + "theoretical_loss": 3.6533784863098964, + "tokens_seen": 1073152000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005453480441323973, + "loss": 1.4493, + "theoretical_loss": 3.6533576941644244, + "tokens_seen": 1073217536 + }, + { + "epoch": 0.33, + "learning_rate": 0.000545331995987964, + "loss": 1.4422, + "theoretical_loss": 3.6533369036440693, + "tokens_seen": 1073283072 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005453159478435307, + "loss": 1.344, + "theoretical_loss": 3.6533161147486037, + "tokens_seen": 1073348608 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005452998996990974, + "loss": 1.3856, + "theoretical_loss": 3.6532953274778026, + "tokens_seen": 1073414144 + }, + { + "epoch": 0.33, + "learning_rate": 0.000545283851554664, + "loss": 1.4246, + "theoretical_loss": 3.653274541831439, + "tokens_seen": 1073479680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005452678034102308, + "loss": 1.4139, + "theoretical_loss": 3.6532537578092876, + "tokens_seen": 1073545216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005452517552657975, + "loss": 1.2834, + "theoretical_loss": 3.6532329754111217, + "tokens_seen": 1073610752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005452357071213642, + "loss": 1.3038, + "theoretical_loss": 3.6532121946367146, + "tokens_seen": 1073676288 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005452196589769309, + "loss": 1.4628, + "theoretical_loss": 3.653191415485842, + "tokens_seen": 1073741824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005452036108324976, + "loss": 1.3071, + "theoretical_loss": 3.6531706379582776, + "tokens_seen": 1073807360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005451875626880643, + "loss": 1.3948, + "theoretical_loss": 3.653149862053794, + "tokens_seen": 1073872896 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005451715145436309, + "loss": 1.2693, + "theoretical_loss": 3.653129087772167, + "tokens_seen": 1073938432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005451554663991977, + "loss": 1.3543, + "theoretical_loss": 3.65310831511317, + "tokens_seen": 1074003968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005451394182547643, + "loss": 1.3167, + "theoretical_loss": 3.6530875440765778, + "tokens_seen": 1074069504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005451233701103311, + "loss": 1.2411, + "theoretical_loss": 3.6530667746621646, + "tokens_seen": 1074135040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005451073219658977, + "loss": 1.3381, + "theoretical_loss": 3.6530460068697037, + "tokens_seen": 1074200576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005450912738214645, + "loss": 1.3835, + "theoretical_loss": 3.6530252406989714, + "tokens_seen": 1074266112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005450752256770311, + "loss": 1.4331, + "theoretical_loss": 3.65300447614974, + "tokens_seen": 1074331648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005450591775325979, + "loss": 1.4747, + "theoretical_loss": 3.6529837132217864, + "tokens_seen": 1074397184 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005450431293881645, + "loss": 1.3274, + "theoretical_loss": 3.6529629519148825, + "tokens_seen": 1074462720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005450270812437312, + "loss": 1.3775, + "theoretical_loss": 3.6529421922288052, + "tokens_seen": 1074528256 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005450110330992979, + "loss": 1.4209, + "theoretical_loss": 3.6529214341633276, + "tokens_seen": 1074593792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005449949849548646, + "loss": 1.3885, + "theoretical_loss": 3.6529006777182254, + "tokens_seen": 1074659328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005449789368104313, + "loss": 1.3243, + "theoretical_loss": 3.652879922893272, + "tokens_seen": 1074724864 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 362330, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3024756908416748, + "objective/train/theoretical_loss": 3.6528591696882433, + "objective/train/tokens_used": 1095250400, + "theoretical_loss": 3.6528591696882433, + "tokens_seen": 1074790400 + }, + { + "epoch": 0.33, + "learning_rate": 0.000544962888665998, + "loss": 1.3094, + "theoretical_loss": 3.6528591696882433, + "tokens_seen": 1074790400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005449468405215647, + "loss": 1.4161, + "theoretical_loss": 3.652838418102914, + "tokens_seen": 1074855936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005449307923771314, + "loss": 1.3407, + "theoretical_loss": 3.6528176681370583, + "tokens_seen": 1074921472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005449147442326981, + "loss": 1.3344, + "theoretical_loss": 3.6527969197904513, + "tokens_seen": 1074987008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005448986960882648, + "loss": 1.3573, + "theoretical_loss": 3.6527761730628687, + "tokens_seen": 1075052544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005448826479438315, + "loss": 1.3483, + "theoretical_loss": 3.6527554279540846, + "tokens_seen": 1075118080 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005448665997993982, + "loss": 1.3747, + "theoretical_loss": 3.6527346844638746, + "tokens_seen": 1075183616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005448505516549649, + "loss": 1.3018, + "theoretical_loss": 3.6527139425920128, + "tokens_seen": 1075249152 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005448345035105316, + "loss": 1.3957, + "theoretical_loss": 3.652693202338275, + "tokens_seen": 1075314688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005448184553660983, + "loss": 1.3263, + "theoretical_loss": 3.6526724637024373, + "tokens_seen": 1075380224 + }, + { + "epoch": 0.33, + "learning_rate": 0.000544802407221665, + "loss": 1.3125, + "theoretical_loss": 3.6526517266842733, + "tokens_seen": 1075445760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005447863590772317, + "loss": 1.3413, + "theoretical_loss": 3.652630991283559, + "tokens_seen": 1075511296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005447703109327984, + "loss": 1.2878, + "theoretical_loss": 3.6526102575000694, + "tokens_seen": 1075576832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005447542627883651, + "loss": 1.3885, + "theoretical_loss": 3.6525895253335805, + "tokens_seen": 1075642368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005447382146439318, + "loss": 1.2449, + "theoretical_loss": 3.6525687947838668, + "tokens_seen": 1075707904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005447221664994985, + "loss": 1.33, + "theoretical_loss": 3.652548065850705, + "tokens_seen": 1075773440 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005447061183550652, + "loss": 1.2818, + "theoretical_loss": 3.6525273385338686, + "tokens_seen": 1075838976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005446900702106319, + "loss": 1.2698, + "theoretical_loss": 3.6525066128331347, + "tokens_seen": 1075904512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005446740220661986, + "loss": 1.4027, + "theoretical_loss": 3.6524858887482785, + "tokens_seen": 1075970048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005446579739217653, + "loss": 1.323, + "theoretical_loss": 3.6524651662790752, + "tokens_seen": 1076035584 + }, + { + "epoch": 0.33, + "learning_rate": 0.000544641925777332, + "loss": 1.4093, + "theoretical_loss": 3.652444445425301, + "tokens_seen": 1076101120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005446258776328987, + "loss": 1.2921, + "theoretical_loss": 3.652423726186732, + "tokens_seen": 1076166656 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005446098294884654, + "loss": 1.3405, + "theoretical_loss": 3.6524030085631427, + "tokens_seen": 1076232192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005445937813440321, + "loss": 1.4306, + "theoretical_loss": 3.652382292554309, + "tokens_seen": 1076297728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005445777331995988, + "loss": 1.4591, + "theoretical_loss": 3.652361578160008, + "tokens_seen": 1076363264 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005445616850551655, + "loss": 1.3253, + "theoretical_loss": 3.6523408653800145, + "tokens_seen": 1076428800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005445456369107322, + "loss": 1.2952, + "theoretical_loss": 3.652320154214105, + "tokens_seen": 1076494336 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005445295887662989, + "loss": 1.3806, + "theoretical_loss": 3.652299444662055, + "tokens_seen": 1076559872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005445135406218656, + "loss": 1.4121, + "theoretical_loss": 3.6522787367236407, + "tokens_seen": 1076625408 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005444974924774323, + "loss": 1.4716, + "theoretical_loss": 3.652258030398638, + "tokens_seen": 1076690944 + }, + { + "epoch": 0.33, + "learning_rate": 0.000544481444332999, + "loss": 1.2995, + "theoretical_loss": 3.6522373256868237, + "tokens_seen": 1076756480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005444653961885657, + "loss": 1.3696, + "theoretical_loss": 3.652216622587973, + "tokens_seen": 1076822016 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005444493480441324, + "loss": 1.3486, + "theoretical_loss": 3.6521959211018626, + "tokens_seen": 1076887552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005444332998996991, + "loss": 1.322, + "theoretical_loss": 3.6521752212282688, + "tokens_seen": 1076953088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005444172517552658, + "loss": 1.3572, + "theoretical_loss": 3.652154522966968, + "tokens_seen": 1077018624 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005444012036108325, + "loss": 1.3745, + "theoretical_loss": 3.652133826317735, + "tokens_seen": 1077084160 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005443851554663992, + "loss": 1.5146, + "theoretical_loss": 3.6521131312803483, + "tokens_seen": 1077149696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005443691073219659, + "loss": 1.4918, + "theoretical_loss": 3.652092437854584, + "tokens_seen": 1077215232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005443530591775326, + "loss": 1.3465, + "theoretical_loss": 3.652071746040217, + "tokens_seen": 1077280768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005443370110330993, + "loss": 1.2983, + "theoretical_loss": 3.652051055837025, + "tokens_seen": 1077346304 + }, + { + "epoch": 0.33, + "learning_rate": 0.000544320962888666, + "loss": 1.2927, + "theoretical_loss": 3.6520303672447847, + "tokens_seen": 1077411840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005443049147442327, + "loss": 1.4392, + "theoretical_loss": 3.652009680263272, + "tokens_seen": 1077477376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005442888665997994, + "loss": 1.4125, + "theoretical_loss": 3.651988994892264, + "tokens_seen": 1077542912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005442728184553661, + "loss": 1.3548, + "theoretical_loss": 3.6519683111315375, + "tokens_seen": 1077608448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005442567703109328, + "loss": 1.4242, + "theoretical_loss": 3.651947628980869, + "tokens_seen": 1077673984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005442407221664995, + "loss": 1.3011, + "theoretical_loss": 3.6519269484400345, + "tokens_seen": 1077739520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005442246740220662, + "loss": 1.2613, + "theoretical_loss": 3.651906269508812, + "tokens_seen": 1077805056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005442086258776329, + "loss": 1.311, + "theoretical_loss": 3.651885592186978, + "tokens_seen": 1077870592 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005441925777331996, + "loss": 1.3905, + "theoretical_loss": 3.6518649164743096, + "tokens_seen": 1077936128 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005441765295887663, + "loss": 1.383, + "theoretical_loss": 3.651844242370583, + "tokens_seen": 1078001664 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 363687, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3248112201690674, + "objective/train/theoretical_loss": 3.651823569875576, + "objective/train/tokens_used": 1098527200, + "theoretical_loss": 3.651823569875576, + "tokens_seen": 1078067200 + }, + { + "epoch": 0.33, + "learning_rate": 0.000544160481444333, + "loss": 1.3867, + "theoretical_loss": 3.651823569875576, + "tokens_seen": 1078067200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005441444332998997, + "loss": 1.3997, + "theoretical_loss": 3.6518028989890654, + "tokens_seen": 1078132736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005441283851554665, + "loss": 1.3344, + "theoretical_loss": 3.651782229710828, + "tokens_seen": 1078198272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005441123370110331, + "loss": 1.2955, + "theoretical_loss": 3.6517615620406407, + "tokens_seen": 1078263808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005440962888665999, + "loss": 1.4162, + "theoretical_loss": 3.6517408959782816, + "tokens_seen": 1078329344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005440802407221665, + "loss": 1.3732, + "theoretical_loss": 3.651720231523527, + "tokens_seen": 1078394880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005440641925777333, + "loss": 1.4056, + "theoretical_loss": 3.651699568676155, + "tokens_seen": 1078460416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005440481444333, + "loss": 1.3838, + "theoretical_loss": 3.651678907435942, + "tokens_seen": 1078525952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005440320962888667, + "loss": 1.3309, + "theoretical_loss": 3.6516582478026662, + "tokens_seen": 1078591488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005440160481444333, + "loss": 1.3331, + "theoretical_loss": 3.6516375897761044, + "tokens_seen": 1078657024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005440000000000001, + "loss": 1.2792, + "theoretical_loss": 3.651616933356034, + "tokens_seen": 1078722560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005439839518555668, + "loss": 1.4044, + "theoretical_loss": 3.6515962785422333, + "tokens_seen": 1078788096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005439679037111335, + "loss": 1.4031, + "theoretical_loss": 3.6515756253344787, + "tokens_seen": 1078853632 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005439518555667002, + "loss": 1.3347, + "theoretical_loss": 3.651554973732549, + "tokens_seen": 1078919168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005439358074222669, + "loss": 1.4163, + "theoretical_loss": 3.6515343237362203, + "tokens_seen": 1078984704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005439197592778336, + "loss": 1.39, + "theoretical_loss": 3.6515136753452717, + "tokens_seen": 1079050240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005439037111334003, + "loss": 1.3692, + "theoretical_loss": 3.65149302855948, + "tokens_seen": 1079115776 + }, + { + "epoch": 0.33, + "learning_rate": 0.000543887662988967, + "loss": 1.2648, + "theoretical_loss": 3.651472383378623, + "tokens_seen": 1079181312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005438716148445337, + "loss": 1.4106, + "theoretical_loss": 3.651451739802479, + "tokens_seen": 1079246848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005438555667001004, + "loss": 1.4743, + "theoretical_loss": 3.6514310978308258, + "tokens_seen": 1079312384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005438395185556671, + "loss": 1.3613, + "theoretical_loss": 3.6514104574634407, + "tokens_seen": 1079377920 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005438234704112338, + "loss": 1.4059, + "theoretical_loss": 3.6513898187001024, + "tokens_seen": 1079443456 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005438074222668005, + "loss": 1.357, + "theoretical_loss": 3.6513691815405878, + "tokens_seen": 1079508992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005437913741223672, + "loss": 1.3831, + "theoretical_loss": 3.6513485459846757, + "tokens_seen": 1079574528 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005437753259779338, + "loss": 1.2636, + "theoretical_loss": 3.6513279120321442, + "tokens_seen": 1079640064 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005437592778335006, + "loss": 1.3083, + "theoretical_loss": 3.6513072796827712, + "tokens_seen": 1079705600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005437432296890672, + "loss": 1.4416, + "theoretical_loss": 3.651286648936335, + "tokens_seen": 1079771136 + }, + { + "epoch": 0.33, + "learning_rate": 0.000543727181544634, + "loss": 1.3512, + "theoretical_loss": 3.6512660197926134, + "tokens_seen": 1079836672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005437111334002006, + "loss": 1.3073, + "theoretical_loss": 3.6512453922513854, + "tokens_seen": 1079902208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005436950852557674, + "loss": 1.2654, + "theoretical_loss": 3.6512247663124278, + "tokens_seen": 1079967744 + }, + { + "epoch": 0.33, + "learning_rate": 0.000543679037111334, + "loss": 1.3332, + "theoretical_loss": 3.6512041419755206, + "tokens_seen": 1080033280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005436629889669008, + "loss": 1.374, + "theoretical_loss": 3.6511835192404414, + "tokens_seen": 1080098816 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005436469408224674, + "loss": 1.395, + "theoretical_loss": 3.6511628981069686, + "tokens_seen": 1080164352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005436308926780341, + "loss": 1.4138, + "theoretical_loss": 3.6511422785748806, + "tokens_seen": 1080229888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005436148445336008, + "loss": 1.4597, + "theoretical_loss": 3.651121660643956, + "tokens_seen": 1080295424 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005435987963891675, + "loss": 1.4631, + "theoretical_loss": 3.651101044313973, + "tokens_seen": 1080360960 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005435827482447342, + "loss": 1.4754, + "theoretical_loss": 3.651080429584711, + "tokens_seen": 1080426496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005435667001003009, + "loss": 1.3145, + "theoretical_loss": 3.651059816455948, + "tokens_seen": 1080492032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005435506519558676, + "loss": 1.4438, + "theoretical_loss": 3.6510392049274625, + "tokens_seen": 1080557568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005435346038114343, + "loss": 1.3457, + "theoretical_loss": 3.651018594999034, + "tokens_seen": 1080623104 + }, + { + "epoch": 0.33, + "learning_rate": 0.000543518555667001, + "loss": 1.413, + "theoretical_loss": 3.6509979866704407, + "tokens_seen": 1080688640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005435025075225677, + "loss": 1.4507, + "theoretical_loss": 3.6509773799414615, + "tokens_seen": 1080754176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005434864593781344, + "loss": 1.3434, + "theoretical_loss": 3.6509567748118754, + "tokens_seen": 1080819712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005434704112337011, + "loss": 1.2866, + "theoretical_loss": 3.650936171281461, + "tokens_seen": 1080885248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005434543630892678, + "loss": 1.324, + "theoretical_loss": 3.6509155693499977, + "tokens_seen": 1080950784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005434383149448345, + "loss": 1.3252, + "theoretical_loss": 3.650894969017264, + "tokens_seen": 1081016320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005434222668004012, + "loss": 1.4008, + "theoretical_loss": 3.650874370283039, + "tokens_seen": 1081081856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005434062186559679, + "loss": 1.2546, + "theoretical_loss": 3.650853773147102, + "tokens_seen": 1081147392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005433901705115346, + "loss": 1.3324, + "theoretical_loss": 3.6508331776092318, + "tokens_seen": 1081212928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005433741223671013, + "loss": 1.3447, + "theoretical_loss": 3.6508125836692082, + "tokens_seen": 1081278464 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 364592, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2894819974899292, + "objective/train/theoretical_loss": 3.6507919913268094, + "objective/train/tokens_used": 1101804000, + "theoretical_loss": 3.6507919913268094, + "tokens_seen": 1081344000 + }, + { + "epoch": 0.33, + "learning_rate": 0.000543358074222668, + "loss": 1.316, + "theoretical_loss": 3.6507919913268094, + "tokens_seen": 1081344000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005433420260782347, + "loss": 1.2995, + "theoretical_loss": 3.6507714005818155, + "tokens_seen": 1081409536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005433259779338014, + "loss": 1.418, + "theoretical_loss": 3.6507508114340057, + "tokens_seen": 1081475072 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005433099297893681, + "loss": 1.3716, + "theoretical_loss": 3.6507302238831585, + "tokens_seen": 1081540608 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005432938816449348, + "loss": 1.4082, + "theoretical_loss": 3.6507096379290545, + "tokens_seen": 1081606144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005432778335005015, + "loss": 1.3643, + "theoretical_loss": 3.6506890535714724, + "tokens_seen": 1081671680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005432617853560682, + "loss": 1.2969, + "theoretical_loss": 3.6506684708101913, + "tokens_seen": 1081737216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005432457372116349, + "loss": 1.3222, + "theoretical_loss": 3.6506478896449917, + "tokens_seen": 1081802752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005432296890672016, + "loss": 1.3993, + "theoretical_loss": 3.6506273100756528, + "tokens_seen": 1081868288 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005432136409227683, + "loss": 1.387, + "theoretical_loss": 3.6506067321019535, + "tokens_seen": 1081933824 + }, + { + "epoch": 0.33, + "learning_rate": 0.000543197592778335, + "loss": 1.3076, + "theoretical_loss": 3.6505861557236745, + "tokens_seen": 1081999360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005431815446339017, + "loss": 1.2477, + "theoretical_loss": 3.6505655809405946, + "tokens_seen": 1082064896 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005431654964894684, + "loss": 1.2312, + "theoretical_loss": 3.6505450077524944, + "tokens_seen": 1082130432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005431494483450351, + "loss": 1.4247, + "theoretical_loss": 3.6505244361591527, + "tokens_seen": 1082195968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005431334002006018, + "loss": 1.3597, + "theoretical_loss": 3.65050386616035, + "tokens_seen": 1082261504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005431173520561685, + "loss": 1.2222, + "theoretical_loss": 3.6504832977558657, + "tokens_seen": 1082327040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005431013039117352, + "loss": 1.407, + "theoretical_loss": 3.6504627309454802, + "tokens_seen": 1082392576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005430852557673019, + "loss": 1.3942, + "theoretical_loss": 3.6504421657289727, + "tokens_seen": 1082458112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005430692076228686, + "loss": 1.3663, + "theoretical_loss": 3.6504216021061247, + "tokens_seen": 1082523648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005430531594784353, + "loss": 1.4238, + "theoretical_loss": 3.6504010400767144, + "tokens_seen": 1082589184 + }, + { + "epoch": 0.33, + "learning_rate": 0.000543037111334002, + "loss": 1.2975, + "theoretical_loss": 3.650380479640523, + "tokens_seen": 1082654720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005430210631895687, + "loss": 1.3741, + "theoretical_loss": 3.65035992079733, + "tokens_seen": 1082720256 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005430050150451354, + "loss": 1.3612, + "theoretical_loss": 3.650339363546916, + "tokens_seen": 1082785792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005429889669007021, + "loss": 1.3039, + "theoretical_loss": 3.6503188078890614, + "tokens_seen": 1082851328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005429729187562688, + "loss": 1.4418, + "theoretical_loss": 3.650298253823546, + "tokens_seen": 1082916864 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005429568706118355, + "loss": 1.4219, + "theoretical_loss": 3.6502777013501504, + "tokens_seen": 1082982400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005429408224674022, + "loss": 1.3086, + "theoretical_loss": 3.6502571504686543, + "tokens_seen": 1083047936 + }, + { + "epoch": 0.33, + "learning_rate": 0.000542924774322969, + "loss": 1.3986, + "theoretical_loss": 3.6502366011788387, + "tokens_seen": 1083113472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005429087261785356, + "loss": 1.4054, + "theoretical_loss": 3.650216053480484, + "tokens_seen": 1083179008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005428926780341024, + "loss": 1.3911, + "theoretical_loss": 3.650195507373371, + "tokens_seen": 1083244544 + }, + { + "epoch": 0.33, + "learning_rate": 0.000542876629889669, + "loss": 1.3547, + "theoretical_loss": 3.650174962857279, + "tokens_seen": 1083310080 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005428605817452358, + "loss": 1.37, + "theoretical_loss": 3.6501544199319897, + "tokens_seen": 1083375616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005428445336008024, + "loss": 1.3649, + "theoretical_loss": 3.650133878597283, + "tokens_seen": 1083441152 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005428284854563692, + "loss": 1.3181, + "theoretical_loss": 3.6501133388529405, + "tokens_seen": 1083506688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005428124373119358, + "loss": 1.3656, + "theoretical_loss": 3.650092800698742, + "tokens_seen": 1083572224 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005427963891675026, + "loss": 1.4077, + "theoretical_loss": 3.6500722641344683, + "tokens_seen": 1083637760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005427803410230692, + "loss": 1.3426, + "theoretical_loss": 3.6500517291599004, + "tokens_seen": 1083703296 + }, + { + "epoch": 0.33, + "learning_rate": 0.000542764292878636, + "loss": 1.3751, + "theoretical_loss": 3.6500311957748193, + "tokens_seen": 1083768832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005427482447342026, + "loss": 1.335, + "theoretical_loss": 3.6500106639790055, + "tokens_seen": 1083834368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005427321965897694, + "loss": 1.5347, + "theoretical_loss": 3.6499901337722402, + "tokens_seen": 1083899904 + }, + { + "epoch": 0.33, + "learning_rate": 0.000542716148445336, + "loss": 1.3838, + "theoretical_loss": 3.649969605154304, + "tokens_seen": 1083965440 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005427001003009028, + "loss": 1.2416, + "theoretical_loss": 3.649949078124978, + "tokens_seen": 1084030976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005426840521564694, + "loss": 1.3768, + "theoretical_loss": 3.6499285526840435, + "tokens_seen": 1084096512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005426680040120362, + "loss": 1.3936, + "theoretical_loss": 3.6499080288312813, + "tokens_seen": 1084162048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005426519558676028, + "loss": 1.4992, + "theoretical_loss": 3.649887506566473, + "tokens_seen": 1084227584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005426359077231696, + "loss": 1.4021, + "theoretical_loss": 3.6498669858893993, + "tokens_seen": 1084293120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005426198595787363, + "loss": 1.2887, + "theoretical_loss": 3.6498464667998416, + "tokens_seen": 1084358656 + }, + { + "epoch": 0.33, + "learning_rate": 0.000542603811434303, + "loss": 1.4855, + "theoretical_loss": 3.6498259492975804, + "tokens_seen": 1084424192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005425877632898697, + "loss": 1.3107, + "theoretical_loss": 3.649805433382398, + "tokens_seen": 1084489728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005425717151454364, + "loss": 1.3343, + "theoretical_loss": 3.649784919054076, + "tokens_seen": 1084555264 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 365785, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3670212030410767, + "objective/train/theoretical_loss": 3.649764406312394, + "objective/train/tokens_used": 1105080800, + "theoretical_loss": 3.649764406312394, + "tokens_seen": 1084620800 + }, + { + "epoch": 0.33, + "learning_rate": 0.000542555667001003, + "loss": 1.4863, + "theoretical_loss": 3.649764406312394, + "tokens_seen": 1084620800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005425396188565698, + "loss": 1.4798, + "theoretical_loss": 3.649743895157135, + "tokens_seen": 1084686336 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005425235707121365, + "loss": 1.2752, + "theoretical_loss": 3.6497233855880804, + "tokens_seen": 1084751872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005425075225677032, + "loss": 1.4032, + "theoretical_loss": 3.6497028776050113, + "tokens_seen": 1084817408 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005424914744232699, + "loss": 1.3299, + "theoretical_loss": 3.6496823712077093, + "tokens_seen": 1084882944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005424754262788366, + "loss": 1.339, + "theoretical_loss": 3.649661866395957, + "tokens_seen": 1084948480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005424593781344033, + "loss": 1.394, + "theoretical_loss": 3.6496413631695335, + "tokens_seen": 1085014016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00054244332998997, + "loss": 1.3726, + "theoretical_loss": 3.6496208615282226, + "tokens_seen": 1085079552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005424272818455367, + "loss": 1.3767, + "theoretical_loss": 3.6496003614718058, + "tokens_seen": 1085145088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005424112337011034, + "loss": 1.3156, + "theoretical_loss": 3.6495798630000644, + "tokens_seen": 1085210624 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005423951855566701, + "loss": 1.4943, + "theoretical_loss": 3.64955936611278, + "tokens_seen": 1085276160 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005423791374122367, + "loss": 1.3143, + "theoretical_loss": 3.649538870809735, + "tokens_seen": 1085341696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005423630892678035, + "loss": 1.3866, + "theoretical_loss": 3.649518377090712, + "tokens_seen": 1085407232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005423470411233701, + "loss": 1.2915, + "theoretical_loss": 3.6494978849554913, + "tokens_seen": 1085472768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005423309929789369, + "loss": 1.2183, + "theoretical_loss": 3.6494773944038554, + "tokens_seen": 1085538304 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005423149448345035, + "loss": 1.4131, + "theoretical_loss": 3.6494569054355877, + "tokens_seen": 1085603840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005422988966900703, + "loss": 1.3191, + "theoretical_loss": 3.649436418050468, + "tokens_seen": 1085669376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005422828485456369, + "loss": 1.468, + "theoretical_loss": 3.64941593224828, + "tokens_seen": 1085734912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005422668004012037, + "loss": 1.3757, + "theoretical_loss": 3.6493954480288058, + "tokens_seen": 1085800448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005422507522567703, + "loss": 1.3375, + "theoretical_loss": 3.649374965391827, + "tokens_seen": 1085865984 + }, + { + "epoch": 0.33, + "learning_rate": 0.000542234704112337, + "loss": 1.3805, + "theoretical_loss": 3.649354484337126, + "tokens_seen": 1085931520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005422186559679037, + "loss": 1.397, + "theoretical_loss": 3.6493340048644853, + "tokens_seen": 1085997056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005422026078234704, + "loss": 1.321, + "theoretical_loss": 3.649313526973687, + "tokens_seen": 1086062592 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005421865596790371, + "loss": 1.3058, + "theoretical_loss": 3.649293050664514, + "tokens_seen": 1086128128 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005421705115346038, + "loss": 1.3293, + "theoretical_loss": 3.649272575936748, + "tokens_seen": 1086193664 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005421544633901705, + "loss": 1.2778, + "theoretical_loss": 3.6492521027901716, + "tokens_seen": 1086259200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005421384152457372, + "loss": 1.3685, + "theoretical_loss": 3.649231631224568, + "tokens_seen": 1086324736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005421223671013039, + "loss": 1.3574, + "theoretical_loss": 3.649211161239719, + "tokens_seen": 1086390272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005421063189568706, + "loss": 1.3703, + "theoretical_loss": 3.6491906928354076, + "tokens_seen": 1086455808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005420902708124373, + "loss": 1.3048, + "theoretical_loss": 3.649170226011416, + "tokens_seen": 1086521344 + }, + { + "epoch": 0.33, + "learning_rate": 0.000542074222668004, + "loss": 1.3437, + "theoretical_loss": 3.6491497607675276, + "tokens_seen": 1086586880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005420581745235707, + "loss": 1.3581, + "theoretical_loss": 3.6491292971035243, + "tokens_seen": 1086652416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005420421263791374, + "loss": 1.2778, + "theoretical_loss": 3.649108835019189, + "tokens_seen": 1086717952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005420260782347041, + "loss": 1.366, + "theoretical_loss": 3.6490883745143052, + "tokens_seen": 1086783488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005420100300902708, + "loss": 1.4187, + "theoretical_loss": 3.649067915588655, + "tokens_seen": 1086849024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005419939819458375, + "loss": 1.3827, + "theoretical_loss": 3.6490474582420216, + "tokens_seen": 1086914560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005419779338014042, + "loss": 1.4968, + "theoretical_loss": 3.649027002474188, + "tokens_seen": 1086980096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005419618856569709, + "loss": 1.368, + "theoretical_loss": 3.6490065482849374, + "tokens_seen": 1087045632 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005419458375125376, + "loss": 1.4541, + "theoretical_loss": 3.648986095674052, + "tokens_seen": 1087111168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005419297893681043, + "loss": 1.2993, + "theoretical_loss": 3.6489656446413155, + "tokens_seen": 1087176704 + }, + { + "epoch": 0.33, + "learning_rate": 0.000541913741223671, + "loss": 1.3196, + "theoretical_loss": 3.648945195186511, + "tokens_seen": 1087242240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005418976930792377, + "loss": 1.3327, + "theoretical_loss": 3.6489247473094215, + "tokens_seen": 1087307776 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005418816449348044, + "loss": 1.417, + "theoretical_loss": 3.6489043010098303, + "tokens_seen": 1087373312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005418655967903711, + "loss": 1.333, + "theoretical_loss": 3.6488838562875205, + "tokens_seen": 1087438848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005418495486459378, + "loss": 1.3239, + "theoretical_loss": 3.6488634131422755, + "tokens_seen": 1087504384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005418335005015045, + "loss": 1.3577, + "theoretical_loss": 3.6488429715738784, + "tokens_seen": 1087569920 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005418174523570712, + "loss": 1.313, + "theoretical_loss": 3.6488225315821126, + "tokens_seen": 1087635456 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005418014042126379, + "loss": 1.4152, + "theoretical_loss": 3.648802093166762, + "tokens_seen": 1087700992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005417853560682046, + "loss": 1.5566, + "theoretical_loss": 3.6487816563276096, + "tokens_seen": 1087766528 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005417693079237713, + "loss": 1.3243, + "theoretical_loss": 3.648761221064439, + "tokens_seen": 1087832064 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 366774, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3897125720977783, + "objective/train/theoretical_loss": 3.6487407873770334, + "objective/train/tokens_used": 1108357600, + "theoretical_loss": 3.6487407873770334, + "tokens_seen": 1087897600 + }, + { + "epoch": 0.33, + "learning_rate": 0.000541753259779338, + "loss": 1.4037, + "theoretical_loss": 3.6487407873770334, + "tokens_seen": 1087897600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005417372116349047, + "loss": 1.3203, + "theoretical_loss": 3.6487203552651772, + "tokens_seen": 1087963136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005417211634904715, + "loss": 1.4418, + "theoretical_loss": 3.648699924728653, + "tokens_seen": 1088028672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005417051153460381, + "loss": 1.3909, + "theoretical_loss": 3.6486794957672455, + "tokens_seen": 1088094208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005416890672016049, + "loss": 1.3976, + "theoretical_loss": 3.648659068380737, + "tokens_seen": 1088159744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005416730190571715, + "loss": 1.374, + "theoretical_loss": 3.648638642568913, + "tokens_seen": 1088225280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005416569709127383, + "loss": 1.3495, + "theoretical_loss": 3.6486182183315563, + "tokens_seen": 1088290816 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005416409227683049, + "loss": 1.3232, + "theoretical_loss": 3.6485977956684503, + "tokens_seen": 1088356352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005416248746238717, + "loss": 1.3278, + "theoretical_loss": 3.64857737457938, + "tokens_seen": 1088421888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005416088264794383, + "loss": 1.2627, + "theoretical_loss": 3.6485569550641284, + "tokens_seen": 1088487424 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005415927783350051, + "loss": 1.3575, + "theoretical_loss": 3.64853653712248, + "tokens_seen": 1088552960 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005415767301905717, + "loss": 1.599, + "theoretical_loss": 3.6485161207542185, + "tokens_seen": 1088618496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005415606820461385, + "loss": 1.3982, + "theoretical_loss": 3.648495705959128, + "tokens_seen": 1088684032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005415446339017051, + "loss": 1.334, + "theoretical_loss": 3.648475292736993, + "tokens_seen": 1088749568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005415285857572719, + "loss": 1.407, + "theoretical_loss": 3.6484548810875967, + "tokens_seen": 1088815104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005415125376128385, + "loss": 1.2509, + "theoretical_loss": 3.6484344710107246, + "tokens_seen": 1088880640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005414964894684053, + "loss": 1.387, + "theoretical_loss": 3.6484140625061596, + "tokens_seen": 1088946176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005414804413239719, + "loss": 1.3232, + "theoretical_loss": 3.648393655573687, + "tokens_seen": 1089011712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005414643931795387, + "loss": 1.4861, + "theoretical_loss": 3.6483732502130906, + "tokens_seen": 1089077248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005414483450351053, + "loss": 1.3551, + "theoretical_loss": 3.6483528464241544, + "tokens_seen": 1089142784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005414322968906721, + "loss": 1.3327, + "theoretical_loss": 3.6483324442066634, + "tokens_seen": 1089208320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005414162487462387, + "loss": 1.4498, + "theoretical_loss": 3.6483120435604017, + "tokens_seen": 1089273856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005414002006018055, + "loss": 1.4741, + "theoretical_loss": 3.648291644485154, + "tokens_seen": 1089339392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005413841524573721, + "loss": 1.3898, + "theoretical_loss": 3.6482712469807046, + "tokens_seen": 1089404928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005413681043129389, + "loss": 1.3997, + "theoretical_loss": 3.6482508510468383, + "tokens_seen": 1089470464 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005413520561685055, + "loss": 1.4476, + "theoretical_loss": 3.6482304566833395, + "tokens_seen": 1089536000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005413360080240723, + "loss": 1.3293, + "theoretical_loss": 3.6482100638899926, + "tokens_seen": 1089601536 + }, + { + "epoch": 0.33, + "learning_rate": 0.000541319959879639, + "loss": 1.4213, + "theoretical_loss": 3.6481896726665832, + "tokens_seen": 1089667072 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005413039117352057, + "loss": 1.3317, + "theoretical_loss": 3.648169283012895, + "tokens_seen": 1089732608 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005412878635907723, + "loss": 1.3798, + "theoretical_loss": 3.6481488949287133, + "tokens_seen": 1089798144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005412718154463391, + "loss": 1.3283, + "theoretical_loss": 3.648128508413823, + "tokens_seen": 1089863680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005412557673019058, + "loss": 1.3927, + "theoretical_loss": 3.648108123468009, + "tokens_seen": 1089929216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005412397191574725, + "loss": 1.4057, + "theoretical_loss": 3.648087740091055, + "tokens_seen": 1089994752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005412236710130392, + "loss": 1.3371, + "theoretical_loss": 3.6480673582827476, + "tokens_seen": 1090060288 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005412076228686059, + "loss": 1.2532, + "theoretical_loss": 3.6480469780428706, + "tokens_seen": 1090125824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005411915747241726, + "loss": 1.3346, + "theoretical_loss": 3.64802659937121, + "tokens_seen": 1090191360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005411755265797393, + "loss": 1.4336, + "theoretical_loss": 3.6480062222675502, + "tokens_seen": 1090256896 + }, + { + "epoch": 0.33, + "learning_rate": 0.000541159478435306, + "loss": 1.4444, + "theoretical_loss": 3.647985846731676, + "tokens_seen": 1090322432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005411434302908727, + "loss": 1.3397, + "theoretical_loss": 3.647965472763374, + "tokens_seen": 1090387968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005411273821464394, + "loss": 1.3925, + "theoretical_loss": 3.6479451003624277, + "tokens_seen": 1090453504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005411113340020061, + "loss": 1.3722, + "theoretical_loss": 3.647924729528624, + "tokens_seen": 1090519040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005410952858575728, + "loss": 1.3825, + "theoretical_loss": 3.6479043602617462, + "tokens_seen": 1090584576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005410792377131395, + "loss": 1.2827, + "theoretical_loss": 3.6478839925615807, + "tokens_seen": 1090650112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005410631895687062, + "loss": 1.2401, + "theoretical_loss": 3.647863626427913, + "tokens_seen": 1090715648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005410471414242729, + "loss": 1.4024, + "theoretical_loss": 3.6478432618605288, + "tokens_seen": 1090781184 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005410310932798396, + "loss": 1.4545, + "theoretical_loss": 3.647822898859213, + "tokens_seen": 1090846720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005410150451354063, + "loss": 1.347, + "theoretical_loss": 3.647802537423751, + "tokens_seen": 1090912256 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540998996990973, + "loss": 1.3257, + "theoretical_loss": 3.6477821775539283, + "tokens_seen": 1090977792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005409829488465397, + "loss": 1.3556, + "theoretical_loss": 3.647761819249531, + "tokens_seen": 1091043328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005409669007021064, + "loss": 1.3661, + "theoretical_loss": 3.647741462510344, + "tokens_seen": 1091108864 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 368007, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3992815017700195, + "objective/train/theoretical_loss": 3.647721107336154, + "objective/train/tokens_used": 1111634400, + "theoretical_loss": 3.647721107336154, + "tokens_seen": 1091174400 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540950852557673, + "loss": 1.5353, + "theoretical_loss": 3.647721107336154, + "tokens_seen": 1091174400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005409348044132398, + "loss": 1.3725, + "theoretical_loss": 3.647700753726746, + "tokens_seen": 1091239936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005409187562688064, + "loss": 1.4593, + "theoretical_loss": 3.6476804016819053, + "tokens_seen": 1091305472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005409027081243732, + "loss": 1.2989, + "theoretical_loss": 3.6476600512014183, + "tokens_seen": 1091371008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005408866599799398, + "loss": 1.2775, + "theoretical_loss": 3.647639702285071, + "tokens_seen": 1091436544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005408706118355066, + "loss": 1.4484, + "theoretical_loss": 3.647619354932649, + "tokens_seen": 1091502080 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005408545636910732, + "loss": 1.3696, + "theoretical_loss": 3.647599009143938, + "tokens_seen": 1091567616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005408385155466399, + "loss": 1.3474, + "theoretical_loss": 3.6475786649187247, + "tokens_seen": 1091633152 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005408224674022066, + "loss": 1.4259, + "theoretical_loss": 3.6475583222567938, + "tokens_seen": 1091698688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005408064192577733, + "loss": 1.3465, + "theoretical_loss": 3.6475379811579325, + "tokens_seen": 1091764224 + }, + { + "epoch": 0.33, + "learning_rate": 0.00054079037111334, + "loss": 1.5082, + "theoretical_loss": 3.647517641621927, + "tokens_seen": 1091829760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005407743229689067, + "loss": 1.3053, + "theoretical_loss": 3.647497303648562, + "tokens_seen": 1091895296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005407582748244734, + "loss": 1.3067, + "theoretical_loss": 3.647476967237625, + "tokens_seen": 1091960832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005407422266800401, + "loss": 1.4078, + "theoretical_loss": 3.6474566323889017, + "tokens_seen": 1092026368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005407261785356068, + "loss": 1.3382, + "theoretical_loss": 3.6474362991021785, + "tokens_seen": 1092091904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005407101303911735, + "loss": 1.3884, + "theoretical_loss": 3.647415967377242, + "tokens_seen": 1092157440 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005406940822467402, + "loss": 1.3208, + "theoretical_loss": 3.6473956372138776, + "tokens_seen": 1092222976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005406780341023069, + "loss": 1.2752, + "theoretical_loss": 3.6473753086118723, + "tokens_seen": 1092288512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005406619859578736, + "loss": 1.2989, + "theoretical_loss": 3.647354981571013, + "tokens_seen": 1092354048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005406459378134403, + "loss": 1.2932, + "theoretical_loss": 3.6473346560910853, + "tokens_seen": 1092419584 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540629889669007, + "loss": 1.3318, + "theoretical_loss": 3.6473143321718755, + "tokens_seen": 1092485120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005406138415245737, + "loss": 1.37, + "theoretical_loss": 3.647294009813171, + "tokens_seen": 1092550656 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005405977933801404, + "loss": 1.2931, + "theoretical_loss": 3.6472736890147583, + "tokens_seen": 1092616192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005405817452357071, + "loss": 1.3279, + "theoretical_loss": 3.647253369776424, + "tokens_seen": 1092681728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005405656970912738, + "loss": 1.4076, + "theoretical_loss": 3.6472330520979543, + "tokens_seen": 1092747264 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005405496489468405, + "loss": 1.3562, + "theoretical_loss": 3.647212735979136, + "tokens_seen": 1092812800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005405336008024072, + "loss": 1.3469, + "theoretical_loss": 3.647192421419756, + "tokens_seen": 1092878336 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540517552657974, + "loss": 1.4398, + "theoretical_loss": 3.6471721084196016, + "tokens_seen": 1092943872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005405015045135406, + "loss": 1.2875, + "theoretical_loss": 3.6471517969784584, + "tokens_seen": 1093009408 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005404854563691074, + "loss": 1.2645, + "theoretical_loss": 3.6471314870961145, + "tokens_seen": 1093074944 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540469408224674, + "loss": 1.3876, + "theoretical_loss": 3.6471111787723567, + "tokens_seen": 1093140480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005404533600802408, + "loss": 1.381, + "theoretical_loss": 3.647090872006971, + "tokens_seen": 1093206016 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005404373119358074, + "loss": 1.4007, + "theoretical_loss": 3.647070566799745, + "tokens_seen": 1093271552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005404212637913742, + "loss": 1.3374, + "theoretical_loss": 3.6470502631504664, + "tokens_seen": 1093337088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005404052156469408, + "loss": 1.3547, + "theoretical_loss": 3.647029961058921, + "tokens_seen": 1093402624 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005403891675025076, + "loss": 1.2061, + "theoretical_loss": 3.6470096605248976, + "tokens_seen": 1093468160 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005403731193580742, + "loss": 1.3681, + "theoretical_loss": 3.646989361548181, + "tokens_seen": 1093533696 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540357071213641, + "loss": 1.4661, + "theoretical_loss": 3.646969064128561, + "tokens_seen": 1093599232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005403410230692076, + "loss": 1.251, + "theoretical_loss": 3.646948768265823, + "tokens_seen": 1093664768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005403249749247744, + "loss": 1.3535, + "theoretical_loss": 3.646928473959755, + "tokens_seen": 1093730304 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540308926780341, + "loss": 1.4352, + "theoretical_loss": 3.6469081812101445, + "tokens_seen": 1093795840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005402928786359078, + "loss": 1.4161, + "theoretical_loss": 3.6468878900167785, + "tokens_seen": 1093861376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005402768304914744, + "loss": 1.4134, + "theoretical_loss": 3.6468676003794447, + "tokens_seen": 1093926912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005402607823470412, + "loss": 1.3992, + "theoretical_loss": 3.6468473122979304, + "tokens_seen": 1093992448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005402447342026078, + "loss": 1.3122, + "theoretical_loss": 3.6468270257720232, + "tokens_seen": 1094057984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005402286860581746, + "loss": 1.2352, + "theoretical_loss": 3.6468067408015106, + "tokens_seen": 1094123520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005402126379137412, + "loss": 1.537, + "theoretical_loss": 3.64678645738618, + "tokens_seen": 1094189056 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540196589769308, + "loss": 1.4234, + "theoretical_loss": 3.6467661755258196, + "tokens_seen": 1094254592 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005401805416248746, + "loss": 1.3623, + "theoretical_loss": 3.6467458952202163, + "tokens_seen": 1094320128 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005401644934804414, + "loss": 1.2615, + "theoretical_loss": 3.6467256164691584, + "tokens_seen": 1094385664 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 368931, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4394853115081787, + "objective/train/theoretical_loss": 3.646705339272434, + "objective/train/tokens_used": 1114911200, + "theoretical_loss": 3.646705339272434, + "tokens_seen": 1094451200 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540148445336008, + "loss": 1.3719, + "theoretical_loss": 3.646705339272434, + "tokens_seen": 1094451200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005401323971915748, + "loss": 1.42, + "theoretical_loss": 3.64668506362983, + "tokens_seen": 1094516736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005401163490471414, + "loss": 1.4435, + "theoretical_loss": 3.646664789541134, + "tokens_seen": 1094582272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005401003009027082, + "loss": 1.3787, + "theoretical_loss": 3.6466445170061355, + "tokens_seen": 1094647808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005400842527582748, + "loss": 1.4719, + "theoretical_loss": 3.646624246024621, + "tokens_seen": 1094713344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005400682046138416, + "loss": 1.3855, + "theoretical_loss": 3.6466039765963787, + "tokens_seen": 1094778880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005400521564694082, + "loss": 1.3523, + "theoretical_loss": 3.646583708721197, + "tokens_seen": 1094844416 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540036108324975, + "loss": 1.4296, + "theoretical_loss": 3.646563442398864, + "tokens_seen": 1094909952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005400200601805416, + "loss": 1.4298, + "theoretical_loss": 3.6465431776291677, + "tokens_seen": 1094975488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005400040120361084, + "loss": 1.4982, + "theoretical_loss": 3.6465229144118956, + "tokens_seen": 1095041024 + }, + { + "epoch": 0.33, + "learning_rate": 0.000539987963891675, + "loss": 1.4781, + "theoretical_loss": 3.6465026527468365, + "tokens_seen": 1095106560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005399719157472418, + "loss": 1.4457, + "theoretical_loss": 3.6464823926337786, + "tokens_seen": 1095172096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005399558676028084, + "loss": 1.2957, + "theoretical_loss": 3.64646213407251, + "tokens_seen": 1095237632 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005399398194583752, + "loss": 1.4198, + "theoretical_loss": 3.6464418770628186, + "tokens_seen": 1095303168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005399237713139418, + "loss": 1.3637, + "theoretical_loss": 3.646421621604494, + "tokens_seen": 1095368704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005399077231695086, + "loss": 1.3435, + "theoretical_loss": 3.6464013676973233, + "tokens_seen": 1095434240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005398916750250752, + "loss": 1.374, + "theoretical_loss": 3.6463811153410948, + "tokens_seen": 1095499776 + }, + { + "epoch": 0.33, + "learning_rate": 0.000539875626880642, + "loss": 1.3455, + "theoretical_loss": 3.6463608645355983, + "tokens_seen": 1095565312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005398595787362087, + "loss": 1.3548, + "theoretical_loss": 3.6463406152806215, + "tokens_seen": 1095630848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005398435305917754, + "loss": 1.4417, + "theoretical_loss": 3.646320367575953, + "tokens_seen": 1095696384 + }, + { + "epoch": 0.33, + "learning_rate": 0.000539827482447342, + "loss": 1.2332, + "theoretical_loss": 3.6463001214213815, + "tokens_seen": 1095761920 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005398114343029088, + "loss": 1.359, + "theoretical_loss": 3.646279876816695, + "tokens_seen": 1095827456 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005397953861584755, + "loss": 1.2995, + "theoretical_loss": 3.646259633761683, + "tokens_seen": 1095892992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005397793380140422, + "loss": 1.3454, + "theoretical_loss": 3.646239392256134, + "tokens_seen": 1095958528 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005397632898696089, + "loss": 1.3796, + "theoretical_loss": 3.646219152299836, + "tokens_seen": 1096024064 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005397472417251756, + "loss": 1.3976, + "theoretical_loss": 3.6461989138925794, + "tokens_seen": 1096089600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005397311935807423, + "loss": 1.4253, + "theoretical_loss": 3.6461786770341518, + "tokens_seen": 1096155136 + }, + { + "epoch": 0.33, + "learning_rate": 0.000539715145436309, + "loss": 1.3519, + "theoretical_loss": 3.6461584417243422, + "tokens_seen": 1096220672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005396990972918757, + "loss": 1.4295, + "theoretical_loss": 3.64613820796294, + "tokens_seen": 1096286208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005396830491474424, + "loss": 1.3605, + "theoretical_loss": 3.6461179757497337, + "tokens_seen": 1096351744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005396670010030091, + "loss": 1.4344, + "theoretical_loss": 3.6460977450845125, + "tokens_seen": 1096417280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005396509528585758, + "loss": 1.2537, + "theoretical_loss": 3.646077515967065, + "tokens_seen": 1096482816 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005396349047141425, + "loss": 1.3941, + "theoretical_loss": 3.6460572883971816, + "tokens_seen": 1096548352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005396188565697092, + "loss": 1.4557, + "theoretical_loss": 3.6460370623746496, + "tokens_seen": 1096613888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005396028084252759, + "loss": 1.3558, + "theoretical_loss": 3.6460168378992597, + "tokens_seen": 1096679424 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005395867602808426, + "loss": 1.3014, + "theoretical_loss": 3.6459966149708, + "tokens_seen": 1096744960 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005395707121364093, + "loss": 1.4605, + "theoretical_loss": 3.6459763935890606, + "tokens_seen": 1096810496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005395546639919759, + "loss": 1.3429, + "theoretical_loss": 3.6459561737538304, + "tokens_seen": 1096876032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005395386158475427, + "loss": 1.4886, + "theoretical_loss": 3.645935955464899, + "tokens_seen": 1096941568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005395225677031093, + "loss": 1.4814, + "theoretical_loss": 3.645915738722055, + "tokens_seen": 1097007104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005395065195586761, + "loss": 1.3228, + "theoretical_loss": 3.645895523525089, + "tokens_seen": 1097072640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005394904714142427, + "loss": 1.2483, + "theoretical_loss": 3.6458753098737895, + "tokens_seen": 1097138176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005394744232698095, + "loss": 1.3837, + "theoretical_loss": 3.6458550977679467, + "tokens_seen": 1097203712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005394583751253761, + "loss": 1.3484, + "theoretical_loss": 3.645834887207349, + "tokens_seen": 1097269248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005394423269809429, + "loss": 1.3385, + "theoretical_loss": 3.6458146781917877, + "tokens_seen": 1097334784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005394262788365095, + "loss": 1.2497, + "theoretical_loss": 3.645794470721051, + "tokens_seen": 1097400320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005394102306920762, + "loss": 1.3259, + "theoretical_loss": 3.6457742647949294, + "tokens_seen": 1097465856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005393941825476429, + "loss": 1.407, + "theoretical_loss": 3.6457540604132115, + "tokens_seen": 1097531392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005393781344032096, + "loss": 1.3469, + "theoretical_loss": 3.645733857575688, + "tokens_seen": 1097596928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005393620862587763, + "loss": 1.3935, + "theoretical_loss": 3.6457136562821493, + "tokens_seen": 1097662464 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 370145, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3979780673980713, + "objective/train/theoretical_loss": 3.645693456532384, + "objective/train/tokens_used": 1118188000, + "theoretical_loss": 3.645693456532384, + "tokens_seen": 1097728000 + }, + { + "epoch": 0.33, + "learning_rate": 0.000539346038114343, + "loss": 1.3858, + "theoretical_loss": 3.645693456532384, + "tokens_seen": 1097728000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005393299899699097, + "loss": 1.3624, + "theoretical_loss": 3.645673258326182, + "tokens_seen": 1097793536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005393139418254765, + "loss": 1.4369, + "theoretical_loss": 3.6456530616633334, + "tokens_seen": 1097859072 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005392978936810431, + "loss": 1.479, + "theoretical_loss": 3.645632866543629, + "tokens_seen": 1097924608 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005392818455366099, + "loss": 1.4302, + "theoretical_loss": 3.645612672966858, + "tokens_seen": 1097990144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005392657973921765, + "loss": 1.401, + "theoretical_loss": 3.6455924809328106, + "tokens_seen": 1098055680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005392497492477433, + "loss": 1.3846, + "theoretical_loss": 3.645572290441277, + "tokens_seen": 1098121216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005392337011033099, + "loss": 1.4016, + "theoretical_loss": 3.645552101492047, + "tokens_seen": 1098186752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005392176529588767, + "loss": 1.3517, + "theoretical_loss": 3.645531914084911, + "tokens_seen": 1098252288 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005392016048144433, + "loss": 1.4179, + "theoretical_loss": 3.6455117282196587, + "tokens_seen": 1098317824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005391855566700101, + "loss": 1.3205, + "theoretical_loss": 3.6454915438960813, + "tokens_seen": 1098383360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005391695085255767, + "loss": 1.3199, + "theoretical_loss": 3.6454713611139686, + "tokens_seen": 1098448896 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005391534603811435, + "loss": 1.3495, + "theoretical_loss": 3.6454511798731106, + "tokens_seen": 1098514432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005391374122367101, + "loss": 1.2753, + "theoretical_loss": 3.645431000173298, + "tokens_seen": 1098579968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005391213640922769, + "loss": 1.4133, + "theoretical_loss": 3.6454108220143215, + "tokens_seen": 1098645504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005391053159478435, + "loss": 1.388, + "theoretical_loss": 3.6453906453959704, + "tokens_seen": 1098711040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005390892678034103, + "loss": 1.4133, + "theoretical_loss": 3.6453704703180367, + "tokens_seen": 1098776576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005390732196589769, + "loss": 1.45, + "theoretical_loss": 3.6453502967803098, + "tokens_seen": 1098842112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005390571715145437, + "loss": 1.3928, + "theoretical_loss": 3.645330124782581, + "tokens_seen": 1098907648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005390411233701103, + "loss": 1.3588, + "theoretical_loss": 3.6453099543246403, + "tokens_seen": 1098973184 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005390250752256771, + "loss": 1.4345, + "theoretical_loss": 3.6452897854062787, + "tokens_seen": 1099038720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005390090270812437, + "loss": 1.3978, + "theoretical_loss": 3.645269618027287, + "tokens_seen": 1099104256 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005389929789368105, + "loss": 1.3343, + "theoretical_loss": 3.6452494521874557, + "tokens_seen": 1099169792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005389769307923771, + "loss": 1.4268, + "theoretical_loss": 3.6452292878865755, + "tokens_seen": 1099235328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005389608826479439, + "loss": 1.3604, + "theoretical_loss": 3.6452091251244374, + "tokens_seen": 1099300864 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005389448345035105, + "loss": 1.3272, + "theoretical_loss": 3.645188963900832, + "tokens_seen": 1099366400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005389287863590773, + "loss": 1.4702, + "theoretical_loss": 3.645168804215551, + "tokens_seen": 1099431936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005389127382146439, + "loss": 1.3084, + "theoretical_loss": 3.6451486460683844, + "tokens_seen": 1099497472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005388966900702107, + "loss": 1.386, + "theoretical_loss": 3.645128489459123, + "tokens_seen": 1099563008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005388806419257773, + "loss": 1.2313, + "theoretical_loss": 3.645108334387559, + "tokens_seen": 1099628544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005388645937813441, + "loss": 1.4147, + "theoretical_loss": 3.6450881808534827, + "tokens_seen": 1099694080 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005388485456369107, + "loss": 1.5094, + "theoretical_loss": 3.6450680288566852, + "tokens_seen": 1099759616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005388324974924775, + "loss": 1.2891, + "theoretical_loss": 3.6450478783969573, + "tokens_seen": 1099825152 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005388164493480441, + "loss": 1.3042, + "theoretical_loss": 3.6450277294740916, + "tokens_seen": 1099890688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005388004012036109, + "loss": 1.4374, + "theoretical_loss": 3.645007582087877, + "tokens_seen": 1099956224 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005387843530591775, + "loss": 1.3827, + "theoretical_loss": 3.6449874362381074, + "tokens_seen": 1100021760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005387683049147443, + "loss": 1.3079, + "theoretical_loss": 3.644967291924572, + "tokens_seen": 1100087296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005387522567703109, + "loss": 1.3284, + "theoretical_loss": 3.6449471491470633, + "tokens_seen": 1100152832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005387362086258777, + "loss": 1.3271, + "theoretical_loss": 3.6449270079053724, + "tokens_seen": 1100218368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005387201604814443, + "loss": 1.4039, + "theoretical_loss": 3.64490686819929, + "tokens_seen": 1100283904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005387041123370111, + "loss": 1.3387, + "theoretical_loss": 3.6448867300286087, + "tokens_seen": 1100349440 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005386880641925777, + "loss": 1.3554, + "theoretical_loss": 3.64486659339312, + "tokens_seen": 1100414976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005386720160481445, + "loss": 1.4049, + "theoretical_loss": 3.6448464582926143, + "tokens_seen": 1100480512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005386559679037111, + "loss": 1.3117, + "theoretical_loss": 3.6448263247268846, + "tokens_seen": 1100546048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005386399197592779, + "loss": 1.3727, + "theoretical_loss": 3.6448061926957207, + "tokens_seen": 1100611584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005386238716148445, + "loss": 1.3975, + "theoretical_loss": 3.644786062198916, + "tokens_seen": 1100677120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005386078234704113, + "loss": 1.3578, + "theoretical_loss": 3.6447659332362616, + "tokens_seen": 1100742656 + }, + { + "epoch": 0.33, + "learning_rate": 0.000538591775325978, + "loss": 1.3708, + "theoretical_loss": 3.6447458058075495, + "tokens_seen": 1100808192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005385757271815447, + "loss": 1.4256, + "theoretical_loss": 3.644725679912571, + "tokens_seen": 1100873728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005385596790371113, + "loss": 1.4461, + "theoretical_loss": 3.644705555551118, + "tokens_seen": 1100939264 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 370963, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3684258460998535, + "objective/train/theoretical_loss": 3.6446854327229827, + "objective/train/tokens_used": 1121464800, + "theoretical_loss": 3.6446854327229827, + "tokens_seen": 1101004800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005385436308926781, + "loss": 1.451, + "theoretical_loss": 3.6446854327229827, + "tokens_seen": 1101004800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005385275827482447, + "loss": 1.3557, + "theoretical_loss": 3.6446653114279566, + "tokens_seen": 1101070336 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005385115346038115, + "loss": 1.3526, + "theoretical_loss": 3.6446451916658322, + "tokens_seen": 1101135872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005384954864593782, + "loss": 1.4003, + "theoretical_loss": 3.6446250734364014, + "tokens_seen": 1101201408 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005384794383149449, + "loss": 1.4314, + "theoretical_loss": 3.644604956739456, + "tokens_seen": 1101266944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005384633901705116, + "loss": 1.3957, + "theoretical_loss": 3.6445848415747877, + "tokens_seen": 1101332480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005384473420260783, + "loss": 1.5251, + "theoretical_loss": 3.644564727942189, + "tokens_seen": 1101398016 + }, + { + "epoch": 0.33, + "learning_rate": 0.000538431293881645, + "loss": 1.4191, + "theoretical_loss": 3.6445446158414523, + "tokens_seen": 1101463552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005384152457372117, + "loss": 1.3306, + "theoretical_loss": 3.6445245052723694, + "tokens_seen": 1101529088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005383991975927784, + "loss": 1.3011, + "theoretical_loss": 3.644504396234733, + "tokens_seen": 1101594624 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005383831494483451, + "loss": 1.356, + "theoretical_loss": 3.6444842887283353, + "tokens_seen": 1101660160 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005383671013039118, + "loss": 1.3998, + "theoretical_loss": 3.644464182752968, + "tokens_seen": 1101725696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005383510531594785, + "loss": 1.2681, + "theoretical_loss": 3.644444078308424, + "tokens_seen": 1101791232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005383350050150452, + "loss": 1.3577, + "theoretical_loss": 3.644423975394496, + "tokens_seen": 1101856768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005383189568706119, + "loss": 1.3414, + "theoretical_loss": 3.6444038740109757, + "tokens_seen": 1101922304 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005383029087261786, + "loss": 1.394, + "theoretical_loss": 3.6443837741576557, + "tokens_seen": 1101987840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005382868605817453, + "loss": 1.3517, + "theoretical_loss": 3.644363675834329, + "tokens_seen": 1102053376 + }, + { + "epoch": 0.33, + "learning_rate": 0.000538270812437312, + "loss": 1.3735, + "theoretical_loss": 3.644343579040788, + "tokens_seen": 1102118912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005382547642928787, + "loss": 1.2879, + "theoretical_loss": 3.6443234837768252, + "tokens_seen": 1102184448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005382387161484454, + "loss": 1.2937, + "theoretical_loss": 3.644303390042233, + "tokens_seen": 1102249984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005382226680040121, + "loss": 1.2894, + "theoretical_loss": 3.6442832978368047, + "tokens_seen": 1102315520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005382066198595788, + "loss": 1.2497, + "theoretical_loss": 3.6442632071603325, + "tokens_seen": 1102381056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005381905717151455, + "loss": 1.3784, + "theoretical_loss": 3.6442431180126094, + "tokens_seen": 1102446592 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005381745235707122, + "loss": 1.3875, + "theoretical_loss": 3.6442230303934284, + "tokens_seen": 1102512128 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005381584754262788, + "loss": 1.252, + "theoretical_loss": 3.6442029443025814, + "tokens_seen": 1102577664 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005381424272818456, + "loss": 1.3298, + "theoretical_loss": 3.6441828597398622, + "tokens_seen": 1102643200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005381263791374122, + "loss": 1.3961, + "theoretical_loss": 3.6441627767050635, + "tokens_seen": 1102708736 + }, + { + "epoch": 0.33, + "learning_rate": 0.000538110330992979, + "loss": 1.3554, + "theoretical_loss": 3.6441426951979787, + "tokens_seen": 1102774272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005380942828485456, + "loss": 1.3804, + "theoretical_loss": 3.6441226152184, + "tokens_seen": 1102839808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005380782347041124, + "loss": 1.4734, + "theoretical_loss": 3.6441025367661206, + "tokens_seen": 1102905344 + }, + { + "epoch": 0.33, + "learning_rate": 0.000538062186559679, + "loss": 1.3292, + "theoretical_loss": 3.6440824598409347, + "tokens_seen": 1102970880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005380461384152458, + "loss": 1.3139, + "theoretical_loss": 3.644062384442634, + "tokens_seen": 1103036416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005380300902708124, + "loss": 1.2785, + "theoretical_loss": 3.6440423105710122, + "tokens_seen": 1103101952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005380140421263792, + "loss": 1.4209, + "theoretical_loss": 3.6440222382258622, + "tokens_seen": 1103167488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005379979939819458, + "loss": 1.4102, + "theoretical_loss": 3.644002167406978, + "tokens_seen": 1103233024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005379819458375126, + "loss": 1.3151, + "theoretical_loss": 3.643982098114152, + "tokens_seen": 1103298560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005379658976930792, + "loss": 1.4163, + "theoretical_loss": 3.6439620303471782, + "tokens_seen": 1103364096 + }, + { + "epoch": 0.33, + "learning_rate": 0.000537949849548646, + "loss": 1.3813, + "theoretical_loss": 3.64394196410585, + "tokens_seen": 1103429632 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005379338014042126, + "loss": 1.4278, + "theoretical_loss": 3.6439218993899605, + "tokens_seen": 1103495168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005379177532597794, + "loss": 1.43, + "theoretical_loss": 3.643901836199303, + "tokens_seen": 1103560704 + }, + { + "epoch": 0.33, + "learning_rate": 0.000537901705115346, + "loss": 1.3511, + "theoretical_loss": 3.643881774533672, + "tokens_seen": 1103626240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005378856569709128, + "loss": 1.3673, + "theoretical_loss": 3.6438617143928598, + "tokens_seen": 1103691776 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005378696088264794, + "loss": 1.4885, + "theoretical_loss": 3.6438416557766597, + "tokens_seen": 1103757312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005378535606820462, + "loss": 1.5448, + "theoretical_loss": 3.643821598684867, + "tokens_seen": 1103822848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005378375125376128, + "loss": 1.304, + "theoretical_loss": 3.6438015431172737, + "tokens_seen": 1103888384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005378214643931796, + "loss": 1.3643, + "theoretical_loss": 3.6437814890736746, + "tokens_seen": 1103953920 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005378054162487462, + "loss": 1.3302, + "theoretical_loss": 3.643761436553863, + "tokens_seen": 1104019456 + }, + { + "epoch": 0.33, + "learning_rate": 0.000537789368104313, + "loss": 1.3877, + "theoretical_loss": 3.6437413855576324, + "tokens_seen": 1104084992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005377733199598796, + "loss": 1.3457, + "theoretical_loss": 3.643721336084777, + "tokens_seen": 1104150528 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005377572718154464, + "loss": 1.434, + "theoretical_loss": 3.6437012881350905, + "tokens_seen": 1104216064 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 372202, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2808032035827637, + "objective/train/theoretical_loss": 3.643681241708367, + "objective/train/tokens_used": 1124741600, + "theoretical_loss": 3.643681241708367, + "tokens_seen": 1104281600 + }, + { + "epoch": 0.33, + "learning_rate": 0.000537741223671013, + "loss": 1.3161, + "theoretical_loss": 3.643681241708367, + "tokens_seen": 1104281600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005377251755265798, + "loss": 1.3353, + "theoretical_loss": 3.6436611968044, + "tokens_seen": 1104347136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005377091273821464, + "loss": 1.4238, + "theoretical_loss": 3.6436411534229842, + "tokens_seen": 1104412672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005376930792377132, + "loss": 1.2942, + "theoretical_loss": 3.6436211115639128, + "tokens_seen": 1104478208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005376770310932798, + "loss": 1.2785, + "theoretical_loss": 3.64360107122698, + "tokens_seen": 1104543744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005376609829488466, + "loss": 1.4836, + "theoretical_loss": 3.643581032411981, + "tokens_seen": 1104609280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005376449348044132, + "loss": 1.3562, + "theoretical_loss": 3.6435609951187082, + "tokens_seen": 1104674816 + }, + { + "epoch": 0.33, + "learning_rate": 0.00053762888665998, + "loss": 1.4225, + "theoretical_loss": 3.643540959346957, + "tokens_seen": 1104740352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005376128385155466, + "loss": 1.3835, + "theoretical_loss": 3.643520925096521, + "tokens_seen": 1104805888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005375967903711134, + "loss": 1.4524, + "theoretical_loss": 3.6435008923671948, + "tokens_seen": 1104871424 + }, + { + "epoch": 0.33, + "learning_rate": 0.00053758074222668, + "loss": 1.4, + "theoretical_loss": 3.6434808611587726, + "tokens_seen": 1104936960 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005375646940822468, + "loss": 1.3913, + "theoretical_loss": 3.6434608314710486, + "tokens_seen": 1105002496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005375486459378134, + "loss": 1.3233, + "theoretical_loss": 3.6434408033038173, + "tokens_seen": 1105068032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005375325977933802, + "loss": 1.3725, + "theoretical_loss": 3.6434207766568734, + "tokens_seen": 1105133568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005375165496489468, + "loss": 1.4318, + "theoretical_loss": 3.643400751530011, + "tokens_seen": 1105199104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005375005015045136, + "loss": 1.3133, + "theoretical_loss": 3.6433807279230246, + "tokens_seen": 1105264640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005374844533600802, + "loss": 1.4323, + "theoretical_loss": 3.6433607058357085, + "tokens_seen": 1105330176 + }, + { + "epoch": 0.33, + "learning_rate": 0.000537468405215647, + "loss": 1.3564, + "theoretical_loss": 3.643340685267858, + "tokens_seen": 1105395712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005374523570712136, + "loss": 1.3261, + "theoretical_loss": 3.6433206662192665, + "tokens_seen": 1105461248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005374363089267804, + "loss": 1.393, + "theoretical_loss": 3.6433006486897304, + "tokens_seen": 1105526784 + }, + { + "epoch": 0.34, + "learning_rate": 0.000537420260782347, + "loss": 1.2428, + "theoretical_loss": 3.6432806326790423, + "tokens_seen": 1105592320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005374042126379138, + "loss": 1.407, + "theoretical_loss": 3.643260618186999, + "tokens_seen": 1105657856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005373881644934804, + "loss": 1.3188, + "theoretical_loss": 3.643240605213394, + "tokens_seen": 1105723392 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005373721163490472, + "loss": 1.4036, + "theoretical_loss": 3.6432205937580227, + "tokens_seen": 1105788928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005373560682046138, + "loss": 1.4072, + "theoretical_loss": 3.643200583820679, + "tokens_seen": 1105854464 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005373400200601806, + "loss": 1.3953, + "theoretical_loss": 3.6431805754011592, + "tokens_seen": 1105920000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005373239719157472, + "loss": 1.382, + "theoretical_loss": 3.6431605684992574, + "tokens_seen": 1105985536 + }, + { + "epoch": 0.34, + "learning_rate": 0.000537307923771314, + "loss": 1.4688, + "theoretical_loss": 3.6431405631147684, + "tokens_seen": 1106051072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005372918756268806, + "loss": 1.3794, + "theoretical_loss": 3.6431205592474876, + "tokens_seen": 1106116608 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005372758274824474, + "loss": 1.361, + "theoretical_loss": 3.64310055689721, + "tokens_seen": 1106182144 + }, + { + "epoch": 0.34, + "learning_rate": 0.000537259779338014, + "loss": 1.3768, + "theoretical_loss": 3.6430805560637305, + "tokens_seen": 1106247680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005372437311935808, + "loss": 1.3212, + "theoretical_loss": 3.643060556746845, + "tokens_seen": 1106313216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005372276830491474, + "loss": 1.2949, + "theoretical_loss": 3.643040558946347, + "tokens_seen": 1106378752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005372116349047142, + "loss": 1.3277, + "theoretical_loss": 3.643020562662034, + "tokens_seen": 1106444288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005371955867602808, + "loss": 1.4497, + "theoretical_loss": 3.643000567893699, + "tokens_seen": 1106509824 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005371795386158476, + "loss": 1.3987, + "theoretical_loss": 3.642980574641139, + "tokens_seen": 1106575360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005371634904714142, + "loss": 1.3366, + "theoretical_loss": 3.6429605829041476, + "tokens_seen": 1106640896 + }, + { + "epoch": 0.34, + "learning_rate": 0.000537147442326981, + "loss": 1.4619, + "theoretical_loss": 3.642940592682522, + "tokens_seen": 1106706432 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005371313941825477, + "loss": 1.4139, + "theoretical_loss": 3.642920603976057, + "tokens_seen": 1106771968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005371153460381144, + "loss": 1.3719, + "theoretical_loss": 3.6429006167845475, + "tokens_seen": 1106837504 + }, + { + "epoch": 0.34, + "learning_rate": 0.000537099297893681, + "loss": 1.3862, + "theoretical_loss": 3.642880631107789, + "tokens_seen": 1106903040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005370832497492478, + "loss": 1.4111, + "theoretical_loss": 3.642860646945578, + "tokens_seen": 1106968576 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005370672016048145, + "loss": 1.2819, + "theoretical_loss": 3.64284066429771, + "tokens_seen": 1107034112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005370511534603812, + "loss": 1.2746, + "theoretical_loss": 3.642820683163979, + "tokens_seen": 1107099648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005370351053159479, + "loss": 1.3164, + "theoretical_loss": 3.6428007035441823, + "tokens_seen": 1107165184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005370190571715146, + "loss": 1.3168, + "theoretical_loss": 3.6427807254381146, + "tokens_seen": 1107230720 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005370030090270813, + "loss": 1.3052, + "theoretical_loss": 3.642760748845572, + "tokens_seen": 1107296256 + }, + { + "epoch": 0.34, + "learning_rate": 0.000536986960882648, + "loss": 1.4019, + "theoretical_loss": 3.6427407737663504, + "tokens_seen": 1107361792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005369709127382147, + "loss": 1.3857, + "theoretical_loss": 3.642720800200246, + "tokens_seen": 1107427328 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005369548645937814, + "loss": 1.3194, + "theoretical_loss": 3.6427008281470536, + "tokens_seen": 1107492864 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 373185, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4264893531799316, + "objective/train/theoretical_loss": 3.64268085760657, + "objective/train/tokens_used": 1128018400, + "theoretical_loss": 3.64268085760657, + "tokens_seen": 1107558400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005369388164493481, + "loss": 1.3387, + "theoretical_loss": 3.64268085760657, + "tokens_seen": 1107558400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005369227683049148, + "loss": 1.2995, + "theoretical_loss": 3.6426608885785905, + "tokens_seen": 1107623936 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005369067201604815, + "loss": 1.3299, + "theoretical_loss": 3.6426409210629114, + "tokens_seen": 1107689472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005368906720160482, + "loss": 1.4579, + "theoretical_loss": 3.6426209550593285, + "tokens_seen": 1107755008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005368746238716149, + "loss": 1.325, + "theoretical_loss": 3.6426009905676384, + "tokens_seen": 1107820544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005368585757271816, + "loss": 1.4244, + "theoretical_loss": 3.642581027587636, + "tokens_seen": 1107886080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005368425275827483, + "loss": 1.4572, + "theoretical_loss": 3.6425610661191192, + "tokens_seen": 1107951616 + }, + { + "epoch": 0.34, + "learning_rate": 0.000536826479438315, + "loss": 1.3712, + "theoretical_loss": 3.6425411061618824, + "tokens_seen": 1108017152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005368104312938817, + "loss": 1.4417, + "theoretical_loss": 3.6425211477157227, + "tokens_seen": 1108082688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005367943831494484, + "loss": 1.4092, + "theoretical_loss": 3.6425011907804365, + "tokens_seen": 1108148224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005367783350050151, + "loss": 1.4101, + "theoretical_loss": 3.64248123535582, + "tokens_seen": 1108213760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005367622868605817, + "loss": 1.3931, + "theoretical_loss": 3.6424612814416686, + "tokens_seen": 1108279296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005367462387161485, + "loss": 1.3125, + "theoretical_loss": 3.64244132903778, + "tokens_seen": 1108344832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005367301905717151, + "loss": 1.3324, + "theoretical_loss": 3.6424213781439496, + "tokens_seen": 1108410368 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005367141424272819, + "loss": 1.3561, + "theoretical_loss": 3.642401428759974, + "tokens_seen": 1108475904 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005366980942828485, + "loss": 1.3003, + "theoretical_loss": 3.6423814808856507, + "tokens_seen": 1108541440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005366820461384153, + "loss": 1.299, + "theoretical_loss": 3.6423615345207745, + "tokens_seen": 1108606976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005366659979939819, + "loss": 1.502, + "theoretical_loss": 3.6423415896651434, + "tokens_seen": 1108672512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005366499498495487, + "loss": 1.2873, + "theoretical_loss": 3.6423216463185533, + "tokens_seen": 1108738048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005366339017051153, + "loss": 1.2303, + "theoretical_loss": 3.642301704480801, + "tokens_seen": 1108803584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005366178535606821, + "loss": 1.4252, + "theoretical_loss": 3.642281764151683, + "tokens_seen": 1108869120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005366018054162487, + "loss": 1.3477, + "theoretical_loss": 3.642261825330997, + "tokens_seen": 1108934656 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005365857572718155, + "loss": 1.4279, + "theoretical_loss": 3.642241888018538, + "tokens_seen": 1109000192 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005365697091273821, + "loss": 1.4184, + "theoretical_loss": 3.642221952214104, + "tokens_seen": 1109065728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005365536609829489, + "loss": 1.4077, + "theoretical_loss": 3.6422020179174917, + "tokens_seen": 1109131264 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005365376128385155, + "loss": 1.333, + "theoretical_loss": 3.642182085128498, + "tokens_seen": 1109196800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005365215646940823, + "loss": 1.3556, + "theoretical_loss": 3.642162153846919, + "tokens_seen": 1109262336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005365055165496489, + "loss": 1.4242, + "theoretical_loss": 3.642142224072553, + "tokens_seen": 1109327872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005364894684052157, + "loss": 1.432, + "theoretical_loss": 3.642122295805197, + "tokens_seen": 1109393408 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005364734202607823, + "loss": 1.3803, + "theoretical_loss": 3.642102369044646, + "tokens_seen": 1109458944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005364573721163491, + "loss": 1.3278, + "theoretical_loss": 3.642082443790699, + "tokens_seen": 1109524480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005364413239719157, + "loss": 1.3702, + "theoretical_loss": 3.6420625200431527, + "tokens_seen": 1109590016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005364252758274825, + "loss": 1.3172, + "theoretical_loss": 3.6420425978018036, + "tokens_seen": 1109655552 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005364092276830491, + "loss": 1.3497, + "theoretical_loss": 3.64202267706645, + "tokens_seen": 1109721088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005363931795386159, + "loss": 1.3812, + "theoretical_loss": 3.642002757836888, + "tokens_seen": 1109786624 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005363771313941825, + "loss": 1.4259, + "theoretical_loss": 3.6419828401129153, + "tokens_seen": 1109852160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005363610832497493, + "loss": 1.367, + "theoretical_loss": 3.6419629238943294, + "tokens_seen": 1109917696 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005363450351053159, + "loss": 1.3267, + "theoretical_loss": 3.6419430091809275, + "tokens_seen": 1109983232 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005363289869608827, + "loss": 1.3759, + "theoretical_loss": 3.641923095972507, + "tokens_seen": 1110048768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005363129388164493, + "loss": 1.4187, + "theoretical_loss": 3.641903184268865, + "tokens_seen": 1110114304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005362968906720161, + "loss": 1.3049, + "theoretical_loss": 3.6418832740698, + "tokens_seen": 1110179840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005362808425275827, + "loss": 1.3543, + "theoretical_loss": 3.6418633653751087, + "tokens_seen": 1110245376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005362647943831495, + "loss": 1.3798, + "theoretical_loss": 3.6418434581845887, + "tokens_seen": 1110310912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005362487462387161, + "loss": 1.3423, + "theoretical_loss": 3.641823552498037, + "tokens_seen": 1110376448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005362326980942829, + "loss": 1.3781, + "theoretical_loss": 3.6418036483152525, + "tokens_seen": 1110441984 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005362166499498495, + "loss": 1.4235, + "theoretical_loss": 3.641783745636032, + "tokens_seen": 1110507520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005362006018054163, + "loss": 1.3896, + "theoretical_loss": 3.641763844460173, + "tokens_seen": 1110573056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005361845536609829, + "loss": 1.4963, + "theoretical_loss": 3.641743944787474, + "tokens_seen": 1110638592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005361685055165497, + "loss": 1.4341, + "theoretical_loss": 3.6417240466177327, + "tokens_seen": 1110704128 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005361524573721163, + "loss": 1.4107, + "theoretical_loss": 3.641704149950746, + "tokens_seen": 1110769664 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 374395, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.658529281616211, + "objective/train/theoretical_loss": 3.6416842547863126, + "objective/train/tokens_used": 1131295200, + "theoretical_loss": 3.6416842547863126, + "tokens_seen": 1110835200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005361364092276831, + "loss": 1.5326, + "theoretical_loss": 3.6416842547863126, + "tokens_seen": 1110835200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005361203610832497, + "loss": 1.4379, + "theoretical_loss": 3.6416643611242305, + "tokens_seen": 1110900736 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005361043129388165, + "loss": 1.4246, + "theoretical_loss": 3.641644468964297, + "tokens_seen": 1110966272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005360882647943831, + "loss": 1.3521, + "theoretical_loss": 3.6416245783063106, + "tokens_seen": 1111031808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005360722166499499, + "loss": 1.4709, + "theoretical_loss": 3.6416046891500686, + "tokens_seen": 1111097344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005360561685055165, + "loss": 1.2381, + "theoretical_loss": 3.64158480149537, + "tokens_seen": 1111162880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005360401203610833, + "loss": 1.4248, + "theoretical_loss": 3.641564915342012, + "tokens_seen": 1111228416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005360240722166499, + "loss": 1.377, + "theoretical_loss": 3.6415450306897936, + "tokens_seen": 1111293952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005360080240722167, + "loss": 1.3799, + "theoretical_loss": 3.6415251475385126, + "tokens_seen": 1111359488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005359919759277833, + "loss": 1.4776, + "theoretical_loss": 3.6415052658879667, + "tokens_seen": 1111425024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005359759277833501, + "loss": 1.3046, + "theoretical_loss": 3.6414853857379548, + "tokens_seen": 1111490560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005359598796389167, + "loss": 1.3972, + "theoretical_loss": 3.641465507088275, + "tokens_seen": 1111556096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005359438314944835, + "loss": 1.3837, + "theoretical_loss": 3.6414456299387257, + "tokens_seen": 1111621632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005359277833500501, + "loss": 1.4221, + "theoretical_loss": 3.641425754289105, + "tokens_seen": 1111687168 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005359117352056169, + "loss": 1.348, + "theoretical_loss": 3.641405880139211, + "tokens_seen": 1111752704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005358956870611835, + "loss": 1.3488, + "theoretical_loss": 3.641386007488843, + "tokens_seen": 1111818240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005358796389167503, + "loss": 1.3655, + "theoretical_loss": 3.6413661363377994, + "tokens_seen": 1111883776 + }, + { + "epoch": 0.34, + "learning_rate": 0.000535863590772317, + "loss": 1.3767, + "theoretical_loss": 3.6413462666858782, + "tokens_seen": 1111949312 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005358475426278837, + "loss": 1.3838, + "theoretical_loss": 3.641326398532878, + "tokens_seen": 1112014848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005358314944834503, + "loss": 1.3424, + "theoretical_loss": 3.6413065318785978, + "tokens_seen": 1112080384 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005358154463390171, + "loss": 1.3995, + "theoretical_loss": 3.6412866667228356, + "tokens_seen": 1112145920 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005357993981945837, + "loss": 1.3884, + "theoretical_loss": 3.641266803065391, + "tokens_seen": 1112211456 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005357833500501505, + "loss": 1.5106, + "theoretical_loss": 3.6412469409060617, + "tokens_seen": 1112276992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005357673019057172, + "loss": 1.2812, + "theoretical_loss": 3.641227080244647, + "tokens_seen": 1112342528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005357512537612839, + "loss": 1.3918, + "theoretical_loss": 3.6412072210809456, + "tokens_seen": 1112408064 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005357352056168507, + "loss": 1.3502, + "theoretical_loss": 3.6411873634147565, + "tokens_seen": 1112473600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005357191574724173, + "loss": 1.4278, + "theoretical_loss": 3.6411675072458785, + "tokens_seen": 1112539136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005357031093279841, + "loss": 1.3277, + "theoretical_loss": 3.64114765257411, + "tokens_seen": 1112604672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005356870611835507, + "loss": 1.3654, + "theoretical_loss": 3.641127799399251, + "tokens_seen": 1112670208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005356710130391175, + "loss": 1.3517, + "theoretical_loss": 3.641107947721099, + "tokens_seen": 1112735744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005356549648946841, + "loss": 1.3816, + "theoretical_loss": 3.641088097539454, + "tokens_seen": 1112801280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005356389167502509, + "loss": 1.293, + "theoretical_loss": 3.6410682488541157, + "tokens_seen": 1112866816 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005356228686058175, + "loss": 1.2678, + "theoretical_loss": 3.641048401664882, + "tokens_seen": 1112932352 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005356068204613843, + "loss": 1.4152, + "theoretical_loss": 3.6410285559715523, + "tokens_seen": 1112997888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005355907723169509, + "loss": 1.377, + "theoretical_loss": 3.6410087117739263, + "tokens_seen": 1113063424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005355747241725177, + "loss": 1.333, + "theoretical_loss": 3.6409888690718026, + "tokens_seen": 1113128960 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005355586760280843, + "loss": 1.3061, + "theoretical_loss": 3.640969027864981, + "tokens_seen": 1113194496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005355426278836511, + "loss": 1.4545, + "theoretical_loss": 3.6409491881532605, + "tokens_seen": 1113260032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005355265797392177, + "loss": 1.356, + "theoretical_loss": 3.64092934993644, + "tokens_seen": 1113325568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005355105315947845, + "loss": 1.3286, + "theoretical_loss": 3.64090951321432, + "tokens_seen": 1113391104 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005354944834503511, + "loss": 1.4135, + "theoretical_loss": 3.6408896779866993, + "tokens_seen": 1113456640 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005354784353059178, + "loss": 1.3987, + "theoretical_loss": 3.640869844253377, + "tokens_seen": 1113522176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005354623871614845, + "loss": 1.3355, + "theoretical_loss": 3.6408500120141527, + "tokens_seen": 1113587712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005354463390170512, + "loss": 1.4287, + "theoretical_loss": 3.640830181268826, + "tokens_seen": 1113653248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005354302908726179, + "loss": 1.465, + "theoretical_loss": 3.6408103520171973, + "tokens_seen": 1113718784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005354142427281846, + "loss": 1.3813, + "theoretical_loss": 3.6407905242590646, + "tokens_seen": 1113784320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005353981945837513, + "loss": 1.3069, + "theoretical_loss": 3.6407706979942294, + "tokens_seen": 1113849856 + }, + { + "epoch": 0.34, + "learning_rate": 0.000535382146439318, + "loss": 1.3965, + "theoretical_loss": 3.6407508732224896, + "tokens_seen": 1113915392 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005353660982948846, + "loss": 1.4686, + "theoretical_loss": 3.6407310499436463, + "tokens_seen": 1113980928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005353500501504514, + "loss": 1.4068, + "theoretical_loss": 3.6407112281574987, + "tokens_seen": 1114046464 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 375251, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4065958261489868, + "objective/train/theoretical_loss": 3.640691407863846, + "objective/train/tokens_used": 1134572000, + "theoretical_loss": 3.640691407863846, + "tokens_seen": 1114112000 + }, + { + "epoch": 0.34, + "learning_rate": 0.000535334002006018, + "loss": 1.3989, + "theoretical_loss": 3.640691407863846, + "tokens_seen": 1114112000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005353179538615848, + "loss": 1.4307, + "theoretical_loss": 3.6406715890624897, + "tokens_seen": 1114177536 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005353019057171514, + "loss": 1.4326, + "theoretical_loss": 3.6406517717532276, + "tokens_seen": 1114243072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005352858575727182, + "loss": 1.2901, + "theoretical_loss": 3.640631955935861, + "tokens_seen": 1114308608 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005352698094282848, + "loss": 1.4398, + "theoretical_loss": 3.6406121416101893, + "tokens_seen": 1114374144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005352537612838516, + "loss": 1.4284, + "theoretical_loss": 3.6405923287760134, + "tokens_seen": 1114439680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005352377131394182, + "loss": 1.2453, + "theoretical_loss": 3.6405725174331316, + "tokens_seen": 1114505216 + }, + { + "epoch": 0.34, + "learning_rate": 0.000535221664994985, + "loss": 1.2765, + "theoretical_loss": 3.640552707581346, + "tokens_seen": 1114570752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005352056168505516, + "loss": 1.3611, + "theoretical_loss": 3.640532899220455, + "tokens_seen": 1114636288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005351895687061184, + "loss": 1.3387, + "theoretical_loss": 3.6405130923502593, + "tokens_seen": 1114701824 + }, + { + "epoch": 0.34, + "learning_rate": 0.000535173520561685, + "loss": 1.3431, + "theoretical_loss": 3.6404932869705595, + "tokens_seen": 1114767360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005351574724172518, + "loss": 1.4032, + "theoretical_loss": 3.6404734830811556, + "tokens_seen": 1114832896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005351414242728184, + "loss": 1.3052, + "theoretical_loss": 3.6404536806818477, + "tokens_seen": 1114898432 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005351253761283852, + "loss": 1.314, + "theoretical_loss": 3.640433879772436, + "tokens_seen": 1114963968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005351093279839518, + "loss": 1.2596, + "theoretical_loss": 3.6404140803527216, + "tokens_seen": 1115029504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005350932798395186, + "loss": 1.3923, + "theoretical_loss": 3.640394282422504, + "tokens_seen": 1115095040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005350772316950852, + "loss": 1.3095, + "theoretical_loss": 3.6403744859815843, + "tokens_seen": 1115160576 + }, + { + "epoch": 0.34, + "learning_rate": 0.000535061183550652, + "loss": 1.3673, + "theoretical_loss": 3.6403546910297617, + "tokens_seen": 1115226112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005350451354062186, + "loss": 1.229, + "theoretical_loss": 3.6403348975668384, + "tokens_seen": 1115291648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005350290872617854, + "loss": 1.4101, + "theoretical_loss": 3.640315105592614, + "tokens_seen": 1115357184 + }, + { + "epoch": 0.34, + "learning_rate": 0.000535013039117352, + "loss": 1.3442, + "theoretical_loss": 3.640295315106889, + "tokens_seen": 1115422720 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005349969909729188, + "loss": 1.3302, + "theoretical_loss": 3.640275526109464, + "tokens_seen": 1115488256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005349809428284854, + "loss": 1.3207, + "theoretical_loss": 3.64025573860014, + "tokens_seen": 1115553792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005349648946840522, + "loss": 1.4234, + "theoretical_loss": 3.640235952578718, + "tokens_seen": 1115619328 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005349488465396188, + "loss": 1.3565, + "theoretical_loss": 3.640216168044997, + "tokens_seen": 1115684864 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005349327983951856, + "loss": 1.3293, + "theoretical_loss": 3.64019638499878, + "tokens_seen": 1115750400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005349167502507522, + "loss": 1.3408, + "theoretical_loss": 3.640176603439867, + "tokens_seen": 1115815936 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534900702106319, + "loss": 1.3538, + "theoretical_loss": 3.640156823368058, + "tokens_seen": 1115881472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005348846539618856, + "loss": 1.3844, + "theoretical_loss": 3.6401370447831547, + "tokens_seen": 1115947008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005348686058174524, + "loss": 1.3133, + "theoretical_loss": 3.640117267684958, + "tokens_seen": 1116012544 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534852557673019, + "loss": 1.4505, + "theoretical_loss": 3.6400974920732683, + "tokens_seen": 1116078080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005348365095285858, + "loss": 1.381, + "theoretical_loss": 3.6400777179478876, + "tokens_seen": 1116143616 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005348204613841524, + "loss": 1.3998, + "theoretical_loss": 3.640057945308616, + "tokens_seen": 1116209152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005348044132397192, + "loss": 1.3332, + "theoretical_loss": 3.6400381741552548, + "tokens_seen": 1116274688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005347883650952858, + "loss": 1.3156, + "theoretical_loss": 3.6400184044876056, + "tokens_seen": 1116340224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005347723169508526, + "loss": 1.3118, + "theoretical_loss": 3.6399986363054686, + "tokens_seen": 1116405760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005347562688064192, + "loss": 1.3324, + "theoretical_loss": 3.6399788696086457, + "tokens_seen": 1116471296 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534740220661986, + "loss": 1.2942, + "theoretical_loss": 3.639959104396938, + "tokens_seen": 1116536832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005347241725175526, + "loss": 1.4229, + "theoretical_loss": 3.6399393406701464, + "tokens_seen": 1116602368 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005347081243731194, + "loss": 1.3852, + "theoretical_loss": 3.639919578428073, + "tokens_seen": 1116667904 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534692076228686, + "loss": 1.3425, + "theoretical_loss": 3.639899817670518, + "tokens_seen": 1116733440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005346760280842528, + "loss": 1.4459, + "theoretical_loss": 3.639880058397284, + "tokens_seen": 1116798976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005346599799398194, + "loss": 1.2834, + "theoretical_loss": 3.639860300608172, + "tokens_seen": 1116864512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005346439317953862, + "loss": 1.2596, + "theoretical_loss": 3.6398405443029827, + "tokens_seen": 1116930048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005346278836509528, + "loss": 1.3818, + "theoretical_loss": 3.6398207894815178, + "tokens_seen": 1116995584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005346118355065196, + "loss": 1.3449, + "theoretical_loss": 3.63980103614358, + "tokens_seen": 1117061120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005345957873620862, + "loss": 1.4162, + "theoretical_loss": 3.6397812842889694, + "tokens_seen": 1117126656 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534579739217653, + "loss": 1.3391, + "theoretical_loss": 3.639761533917489, + "tokens_seen": 1117192192 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005345636910732196, + "loss": 1.3396, + "theoretical_loss": 3.639741785028939, + "tokens_seen": 1117257728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005345476429287864, + "loss": 1.3862, + "theoretical_loss": 3.6397220376231223, + "tokens_seen": 1117323264 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 376505, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3129369020462036, + "objective/train/theoretical_loss": 3.63970229169984, + "objective/train/tokens_used": 1137848800, + "theoretical_loss": 3.63970229169984, + "tokens_seen": 1117388800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005345315947843532, + "loss": 1.3009, + "theoretical_loss": 3.63970229169984, + "tokens_seen": 1117388800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005345155466399198, + "loss": 1.4542, + "theoretical_loss": 3.639682547258894, + "tokens_seen": 1117454336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005344994984954866, + "loss": 1.4217, + "theoretical_loss": 3.6396628043000856, + "tokens_seen": 1117519872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005344834503510532, + "loss": 1.3604, + "theoretical_loss": 3.6396430628232173, + "tokens_seen": 1117585408 + }, + { + "epoch": 0.34, + "learning_rate": 0.00053446740220662, + "loss": 1.3649, + "theoretical_loss": 3.6396233228280908, + "tokens_seen": 1117650944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005344513540621866, + "loss": 1.4095, + "theoretical_loss": 3.639603584314508, + "tokens_seen": 1117716480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005344353059177534, + "loss": 1.3334, + "theoretical_loss": 3.639583847282271, + "tokens_seen": 1117782016 + }, + { + "epoch": 0.34, + "learning_rate": 0.00053441925777332, + "loss": 1.3851, + "theoretical_loss": 3.6395641117311817, + "tokens_seen": 1117847552 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005344032096288868, + "loss": 1.2414, + "theoretical_loss": 3.6395443776610414, + "tokens_seen": 1117913088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005343871614844534, + "loss": 1.3993, + "theoretical_loss": 3.6395246450716527, + "tokens_seen": 1117978624 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005343711133400202, + "loss": 1.3598, + "theoretical_loss": 3.6395049139628184, + "tokens_seen": 1118044160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005343550651955868, + "loss": 1.415, + "theoretical_loss": 3.63948518433434, + "tokens_seen": 1118109696 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005343390170511536, + "loss": 1.4148, + "theoretical_loss": 3.639465456186019, + "tokens_seen": 1118175232 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005343229689067202, + "loss": 1.3158, + "theoretical_loss": 3.639445729517659, + "tokens_seen": 1118240768 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534306920762287, + "loss": 1.2983, + "theoretical_loss": 3.6394260043290614, + "tokens_seen": 1118306304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005342908726178536, + "loss": 1.33, + "theoretical_loss": 3.6394062806200287, + "tokens_seen": 1118371840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005342748244734204, + "loss": 1.4273, + "theoretical_loss": 3.639386558390363, + "tokens_seen": 1118437376 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534258776328987, + "loss": 1.343, + "theoretical_loss": 3.6393668376398667, + "tokens_seen": 1118502912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005342427281845538, + "loss": 1.3797, + "theoretical_loss": 3.6393471183683426, + "tokens_seen": 1118568448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005342266800401204, + "loss": 1.3075, + "theoretical_loss": 3.639327400575593, + "tokens_seen": 1118633984 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005342106318956872, + "loss": 1.3961, + "theoretical_loss": 3.6393076842614196, + "tokens_seen": 1118699520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005341945837512538, + "loss": 1.2817, + "theoretical_loss": 3.6392879694256264, + "tokens_seen": 1118765056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005341785356068206, + "loss": 1.3376, + "theoretical_loss": 3.6392682560680143, + "tokens_seen": 1118830592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005341624874623872, + "loss": 1.2789, + "theoretical_loss": 3.6392485441883875, + "tokens_seen": 1118896128 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534146439317954, + "loss": 1.3712, + "theoretical_loss": 3.6392288337865475, + "tokens_seen": 1118961664 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005341303911735206, + "loss": 1.3982, + "theoretical_loss": 3.6392091248622975, + "tokens_seen": 1119027200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005341143430290874, + "loss": 1.4322, + "theoretical_loss": 3.6391894174154396, + "tokens_seen": 1119092736 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534098294884654, + "loss": 1.4059, + "theoretical_loss": 3.6391697114457773, + "tokens_seen": 1119158272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005340822467402207, + "loss": 1.338, + "theoretical_loss": 3.639150006953113, + "tokens_seen": 1119223808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005340661985957874, + "loss": 1.2945, + "theoretical_loss": 3.63913030393725, + "tokens_seen": 1119289344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005340501504513541, + "loss": 1.2693, + "theoretical_loss": 3.6391106023979902, + "tokens_seen": 1119354880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005340341023069208, + "loss": 1.4471, + "theoretical_loss": 3.6390909023351368, + "tokens_seen": 1119420416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005340180541624875, + "loss": 1.3414, + "theoretical_loss": 3.639071203748494, + "tokens_seen": 1119485952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005340020060180542, + "loss": 1.243, + "theoretical_loss": 3.6390515066378626, + "tokens_seen": 1119551488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005339859578736209, + "loss": 1.2848, + "theoretical_loss": 3.6390318110030475, + "tokens_seen": 1119617024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005339699097291876, + "loss": 1.3206, + "theoretical_loss": 3.6390121168438507, + "tokens_seen": 1119682560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005339538615847543, + "loss": 1.3479, + "theoretical_loss": 3.638992424160076, + "tokens_seen": 1119748096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005339378134403209, + "loss": 1.4657, + "theoretical_loss": 3.6389727329515256, + "tokens_seen": 1119813632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005339217652958877, + "loss": 1.3867, + "theoretical_loss": 3.6389530432180033, + "tokens_seen": 1119879168 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005339057171514543, + "loss": 1.3862, + "theoretical_loss": 3.6389333549593124, + "tokens_seen": 1119944704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005338896690070211, + "loss": 1.3789, + "theoretical_loss": 3.638913668175255, + "tokens_seen": 1120010240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005338736208625877, + "loss": 1.3899, + "theoretical_loss": 3.6388939828656364, + "tokens_seen": 1120075776 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005338575727181545, + "loss": 1.245, + "theoretical_loss": 3.6388742990302587, + "tokens_seen": 1120141312 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005338415245737211, + "loss": 1.3722, + "theoretical_loss": 3.638854616668925, + "tokens_seen": 1120206848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005338254764292879, + "loss": 1.413, + "theoretical_loss": 3.638834935781439, + "tokens_seen": 1120272384 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005338094282848545, + "loss": 1.3437, + "theoretical_loss": 3.6388152563676037, + "tokens_seen": 1120337920 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005337933801404213, + "loss": 1.361, + "theoretical_loss": 3.638795578427224, + "tokens_seen": 1120403456 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005337773319959879, + "loss": 1.2868, + "theoretical_loss": 3.638775901960102, + "tokens_seen": 1120468992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005337612838515547, + "loss": 1.3439, + "theoretical_loss": 3.6387562269660414, + "tokens_seen": 1120534528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005337452357071213, + "loss": 1.3982, + "theoretical_loss": 3.638736553444846, + "tokens_seen": 1120600064 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 377426, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3275361061096191, + "objective/train/theoretical_loss": 3.63871688139632, + "objective/train/tokens_used": 1141125600, + "theoretical_loss": 3.63871688139632, + "tokens_seen": 1120665600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005337291875626881, + "loss": 1.3741, + "theoretical_loss": 3.63871688139632, + "tokens_seen": 1120665600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005337131394182547, + "loss": 1.4376, + "theoretical_loss": 3.638697210820266, + "tokens_seen": 1120731136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005336970912738215, + "loss": 1.4034, + "theoretical_loss": 3.6386775417164885, + "tokens_seen": 1120796672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005336810431293881, + "loss": 1.3859, + "theoretical_loss": 3.638657874084791, + "tokens_seen": 1120862208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005336649949849549, + "loss": 1.3789, + "theoretical_loss": 3.638638207924977, + "tokens_seen": 1120927744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005336489468405215, + "loss": 1.2965, + "theoretical_loss": 3.6386185432368503, + "tokens_seen": 1120993280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005336328986960883, + "loss": 1.3204, + "theoretical_loss": 3.638598880020215, + "tokens_seen": 1121058816 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005336168505516549, + "loss": 1.3042, + "theoretical_loss": 3.638579218274875, + "tokens_seen": 1121124352 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005336008024072217, + "loss": 1.368, + "theoretical_loss": 3.638559558000634, + "tokens_seen": 1121189888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005335847542627883, + "loss": 1.4172, + "theoretical_loss": 3.6385398991972964, + "tokens_seen": 1121255424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005335687061183551, + "loss": 1.4214, + "theoretical_loss": 3.638520241864666, + "tokens_seen": 1121320960 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005335526579739217, + "loss": 1.3088, + "theoretical_loss": 3.6385005860025457, + "tokens_seen": 1121386496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005335366098294885, + "loss": 1.2974, + "theoretical_loss": 3.6384809316107414, + "tokens_seen": 1121452032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005335205616850551, + "loss": 1.4272, + "theoretical_loss": 3.638461278689056, + "tokens_seen": 1121517568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005335045135406219, + "loss": 1.3855, + "theoretical_loss": 3.6384416272372944, + "tokens_seen": 1121583104 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005334884653961885, + "loss": 1.441, + "theoretical_loss": 3.63842197725526, + "tokens_seen": 1121648640 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005334724172517553, + "loss": 1.2518, + "theoretical_loss": 3.6384023287427576, + "tokens_seen": 1121714176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005334563691073219, + "loss": 1.2455, + "theoretical_loss": 3.638382681699591, + "tokens_seen": 1121779712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005334403209628887, + "loss": 1.3986, + "theoretical_loss": 3.6383630361255648, + "tokens_seen": 1121845248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005334242728184553, + "loss": 1.3738, + "theoretical_loss": 3.638343392020483, + "tokens_seen": 1121910784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005334082246740221, + "loss": 1.3148, + "theoretical_loss": 3.638323749384151, + "tokens_seen": 1121976320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005333921765295887, + "loss": 1.4755, + "theoretical_loss": 3.6383041082163716, + "tokens_seen": 1122041856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005333761283851555, + "loss": 1.366, + "theoretical_loss": 3.63828446851695, + "tokens_seen": 1122107392 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005333600802407221, + "loss": 1.3049, + "theoretical_loss": 3.6382648302856917, + "tokens_seen": 1122172928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005333440320962889, + "loss": 1.345, + "theoretical_loss": 3.6382451935223994, + "tokens_seen": 1122238464 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005333279839518556, + "loss": 1.4058, + "theoretical_loss": 3.638225558226879, + "tokens_seen": 1122304000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005333119358074223, + "loss": 1.3659, + "theoretical_loss": 3.6382059243989344, + "tokens_seen": 1122369536 + }, + { + "epoch": 0.34, + "learning_rate": 0.000533295887662989, + "loss": 1.3639, + "theoretical_loss": 3.6381862920383705, + "tokens_seen": 1122435072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005332798395185557, + "loss": 1.2969, + "theoretical_loss": 3.6381666611449917, + "tokens_seen": 1122500608 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005332637913741224, + "loss": 1.2742, + "theoretical_loss": 3.638147031718603, + "tokens_seen": 1122566144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005332477432296891, + "loss": 1.4018, + "theoretical_loss": 3.6381274037590092, + "tokens_seen": 1122631680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005332316950852559, + "loss": 1.4345, + "theoretical_loss": 3.638107777266015, + "tokens_seen": 1122697216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005332156469408225, + "loss": 1.4199, + "theoretical_loss": 3.6380881522394244, + "tokens_seen": 1122762752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005331995987963893, + "loss": 1.35, + "theoretical_loss": 3.6380685286790437, + "tokens_seen": 1122828288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005331835506519559, + "loss": 1.44, + "theoretical_loss": 3.638048906584677, + "tokens_seen": 1122893824 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005331675025075227, + "loss": 1.4196, + "theoretical_loss": 3.6380292859561294, + "tokens_seen": 1122959360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005331514543630893, + "loss": 1.326, + "theoretical_loss": 3.6380096667932054, + "tokens_seen": 1123024896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005331354062186561, + "loss": 1.3748, + "theoretical_loss": 3.63799004909571, + "tokens_seen": 1123090432 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005331193580742227, + "loss": 1.2627, + "theoretical_loss": 3.6379704328634492, + "tokens_seen": 1123155968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005331033099297895, + "loss": 1.3448, + "theoretical_loss": 3.6379508180962272, + "tokens_seen": 1123221504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005330872617853561, + "loss": 1.2757, + "theoretical_loss": 3.63793120479385, + "tokens_seen": 1123287040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005330712136409229, + "loss": 1.2797, + "theoretical_loss": 3.6379115929561214, + "tokens_seen": 1123352576 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005330551654964895, + "loss": 1.2881, + "theoretical_loss": 3.6378919825828477, + "tokens_seen": 1123418112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005330391173520563, + "loss": 1.3451, + "theoretical_loss": 3.637872373673834, + "tokens_seen": 1123483648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005330230692076229, + "loss": 1.4061, + "theoretical_loss": 3.637852766228885, + "tokens_seen": 1123549184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005330070210631897, + "loss": 1.3824, + "theoretical_loss": 3.637833160247806, + "tokens_seen": 1123614720 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005329909729187563, + "loss": 1.3463, + "theoretical_loss": 3.637813555730403, + "tokens_seen": 1123680256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005329749247743231, + "loss": 1.483, + "theoretical_loss": 3.637793952676481, + "tokens_seen": 1123745792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005329588766298897, + "loss": 1.401, + "theoretical_loss": 3.637774351085845, + "tokens_seen": 1123811328 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005329428284854565, + "loss": 1.3526, + "theoretical_loss": 3.637754750958302, + "tokens_seen": 1123876864 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 378684, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2597837448120117, + "objective/train/theoretical_loss": 3.637735152293655, + "objective/train/tokens_used": 1144402400, + "theoretical_loss": 3.637735152293655, + "tokens_seen": 1123942400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005329267803410231, + "loss": 1.3474, + "theoretical_loss": 3.637735152293655, + "tokens_seen": 1123942400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005329107321965899, + "loss": 1.4682, + "theoretical_loss": 3.637715555091712, + "tokens_seen": 1124007936 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005328946840521565, + "loss": 1.3396, + "theoretical_loss": 3.6376959593522766, + "tokens_seen": 1124073472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005328786359077233, + "loss": 1.2955, + "theoretical_loss": 3.637676365075156, + "tokens_seen": 1124139008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005328625877632899, + "loss": 1.3791, + "theoretical_loss": 3.6376567722601547, + "tokens_seen": 1124204544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005328465396188567, + "loss": 1.4525, + "theoretical_loss": 3.637637180907079, + "tokens_seen": 1124270080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005328304914744233, + "loss": 1.294, + "theoretical_loss": 3.637617591015734, + "tokens_seen": 1124335616 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005328144433299901, + "loss": 1.3757, + "theoretical_loss": 3.637598002585926, + "tokens_seen": 1124401152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005327983951855567, + "loss": 1.3695, + "theoretical_loss": 3.6375784156174604, + "tokens_seen": 1124466688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005327823470411235, + "loss": 1.4115, + "theoretical_loss": 3.6375588301101436, + "tokens_seen": 1124532224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005327662988966901, + "loss": 1.3728, + "theoretical_loss": 3.6375392460637808, + "tokens_seen": 1124597760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005327502507522569, + "loss": 1.3182, + "theoretical_loss": 3.6375196634781783, + "tokens_seen": 1124663296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005327342026078235, + "loss": 1.375, + "theoretical_loss": 3.637500082353142, + "tokens_seen": 1124728832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005327181544633903, + "loss": 1.3206, + "theoretical_loss": 3.6374805026884776, + "tokens_seen": 1124794368 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005327021063189569, + "loss": 1.3184, + "theoretical_loss": 3.637460924483991, + "tokens_seen": 1124859904 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005326860581745236, + "loss": 1.3875, + "theoretical_loss": 3.6374413477394887, + "tokens_seen": 1124925440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005326700100300903, + "loss": 1.2268, + "theoretical_loss": 3.637421772454777, + "tokens_seen": 1124990976 + }, + { + "epoch": 0.34, + "learning_rate": 0.000532653961885657, + "loss": 1.331, + "theoretical_loss": 3.637402198629661, + "tokens_seen": 1125056512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005326379137412237, + "loss": 1.3865, + "theoretical_loss": 3.637382626263948, + "tokens_seen": 1125122048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005326218655967904, + "loss": 1.4205, + "theoretical_loss": 3.637363055357443, + "tokens_seen": 1125187584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005326058174523571, + "loss": 1.4883, + "theoretical_loss": 3.6373434859099536, + "tokens_seen": 1125253120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005325897693079238, + "loss": 1.3364, + "theoretical_loss": 3.637323917921285, + "tokens_seen": 1125318656 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005325737211634905, + "loss": 1.3323, + "theoretical_loss": 3.637304351391244, + "tokens_seen": 1125384192 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005325576730190572, + "loss": 1.4029, + "theoretical_loss": 3.6372847863196363, + "tokens_seen": 1125449728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005325416248746238, + "loss": 1.356, + "theoretical_loss": 3.6372652227062696, + "tokens_seen": 1125515264 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005325255767301906, + "loss": 1.3703, + "theoretical_loss": 3.637245660550949, + "tokens_seen": 1125580800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005325095285857572, + "loss": 1.3003, + "theoretical_loss": 3.6372260998534816, + "tokens_seen": 1125646336 + }, + { + "epoch": 0.34, + "learning_rate": 0.000532493480441324, + "loss": 1.3414, + "theoretical_loss": 3.6372065406136733, + "tokens_seen": 1125711872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005324774322968906, + "loss": 1.3245, + "theoretical_loss": 3.6371869828313317, + "tokens_seen": 1125777408 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005324613841524574, + "loss": 1.2384, + "theoretical_loss": 3.637167426506262, + "tokens_seen": 1125842944 + }, + { + "epoch": 0.34, + "learning_rate": 0.000532445336008024, + "loss": 1.3696, + "theoretical_loss": 3.637147871638272, + "tokens_seen": 1125908480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005324292878635908, + "loss": 1.2808, + "theoretical_loss": 3.637128318227168, + "tokens_seen": 1125974016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005324132397191574, + "loss": 1.405, + "theoretical_loss": 3.637108766272756, + "tokens_seen": 1126039552 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005323971915747242, + "loss": 1.2078, + "theoretical_loss": 3.637089215774843, + "tokens_seen": 1126105088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005323811434302908, + "loss": 1.4223, + "theoretical_loss": 3.6370696667332365, + "tokens_seen": 1126170624 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005323650952858576, + "loss": 1.3003, + "theoretical_loss": 3.6370501191477427, + "tokens_seen": 1126236160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005323490471414242, + "loss": 1.3226, + "theoretical_loss": 3.6370305730181682, + "tokens_seen": 1126301696 + }, + { + "epoch": 0.34, + "learning_rate": 0.000532332998996991, + "loss": 1.4025, + "theoretical_loss": 3.6370110283443204, + "tokens_seen": 1126367232 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005323169508525576, + "loss": 1.3653, + "theoretical_loss": 3.6369914851260057, + "tokens_seen": 1126432768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005323009027081244, + "loss": 1.326, + "theoretical_loss": 3.636971943363031, + "tokens_seen": 1126498304 + }, + { + "epoch": 0.34, + "learning_rate": 0.000532284854563691, + "loss": 1.3969, + "theoretical_loss": 3.6369524030552043, + "tokens_seen": 1126563840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005322688064192578, + "loss": 1.2623, + "theoretical_loss": 3.6369328642023313, + "tokens_seen": 1126629376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005322527582748244, + "loss": 1.415, + "theoretical_loss": 3.6369133268042195, + "tokens_seen": 1126694912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005322367101303912, + "loss": 1.3816, + "theoretical_loss": 3.636893790860676, + "tokens_seen": 1126760448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005322206619859578, + "loss": 1.4577, + "theoretical_loss": 3.6368742563715077, + "tokens_seen": 1126825984 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005322046138415246, + "loss": 1.2977, + "theoretical_loss": 3.6368547233365227, + "tokens_seen": 1126891520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005321885656970912, + "loss": 1.421, + "theoretical_loss": 3.6368351917555275, + "tokens_seen": 1126957056 + }, + { + "epoch": 0.34, + "learning_rate": 0.000532172517552658, + "loss": 1.4054, + "theoretical_loss": 3.6368156616283285, + "tokens_seen": 1127022592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005321564694082246, + "loss": 1.3986, + "theoretical_loss": 3.6367961329547343, + "tokens_seen": 1127088128 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005321404212637914, + "loss": 1.3883, + "theoretical_loss": 3.6367766057345516, + "tokens_seen": 1127153664 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 379502, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4635413885116577, + "objective/train/theoretical_loss": 3.6367570799675875, + "objective/train/tokens_used": 1147679200, + "theoretical_loss": 3.6367570799675875, + "tokens_seen": 1127219200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005321243731193581, + "loss": 1.4607, + "theoretical_loss": 3.6367570799675875, + "tokens_seen": 1127219200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005321083249749248, + "loss": 1.3679, + "theoretical_loss": 3.6367375556536503, + "tokens_seen": 1127284736 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005320922768304915, + "loss": 1.4228, + "theoretical_loss": 3.636718032792546, + "tokens_seen": 1127350272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005320762286860582, + "loss": 1.3982, + "theoretical_loss": 3.6366985113840835, + "tokens_seen": 1127415808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005320601805416249, + "loss": 1.3395, + "theoretical_loss": 3.6366789914280693, + "tokens_seen": 1127481344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005320441323971916, + "loss": 1.3854, + "theoretical_loss": 3.636659472924311, + "tokens_seen": 1127546880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005320280842527583, + "loss": 1.3441, + "theoretical_loss": 3.636639955872617, + "tokens_seen": 1127612416 + }, + { + "epoch": 0.34, + "learning_rate": 0.000532012036108325, + "loss": 1.2984, + "theoretical_loss": 3.636620440272794, + "tokens_seen": 1127677952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005319959879638917, + "loss": 1.3089, + "theoretical_loss": 3.6366009261246495, + "tokens_seen": 1127743488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005319799398194584, + "loss": 1.289, + "theoretical_loss": 3.636581413427992, + "tokens_seen": 1127809024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005319638916750251, + "loss": 1.3694, + "theoretical_loss": 3.6365619021826285, + "tokens_seen": 1127874560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005319478435305918, + "loss": 1.4387, + "theoretical_loss": 3.6365423923883666, + "tokens_seen": 1127940096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005319317953861585, + "loss": 1.2967, + "theoretical_loss": 3.6365228840450152, + "tokens_seen": 1128005632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005319157472417252, + "loss": 1.2617, + "theoretical_loss": 3.636503377152381, + "tokens_seen": 1128071168 + }, + { + "epoch": 0.34, + "learning_rate": 0.000531899699097292, + "loss": 1.2746, + "theoretical_loss": 3.6364838717102725, + "tokens_seen": 1128136704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005318836509528586, + "loss": 1.2389, + "theoretical_loss": 3.636464367718497, + "tokens_seen": 1128202240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005318676028084254, + "loss": 1.2898, + "theoretical_loss": 3.636444865176863, + "tokens_seen": 1128267776 + }, + { + "epoch": 0.34, + "learning_rate": 0.000531851554663992, + "loss": 1.3015, + "theoretical_loss": 3.636425364085178, + "tokens_seen": 1128333312 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005318355065195588, + "loss": 1.3878, + "theoretical_loss": 3.63640586444325, + "tokens_seen": 1128398848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005318194583751254, + "loss": 1.3115, + "theoretical_loss": 3.636386366250888, + "tokens_seen": 1128464384 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005318034102306922, + "loss": 1.3701, + "theoretical_loss": 3.636366869507899, + "tokens_seen": 1128529920 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005317873620862588, + "loss": 1.2944, + "theoretical_loss": 3.6363473742140906, + "tokens_seen": 1128595456 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005317713139418256, + "loss": 1.3582, + "theoretical_loss": 3.6363278803692727, + "tokens_seen": 1128660992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005317552657973922, + "loss": 1.2594, + "theoretical_loss": 3.6363083879732523, + "tokens_seen": 1128726528 + }, + { + "epoch": 0.34, + "learning_rate": 0.000531739217652959, + "loss": 1.3404, + "theoretical_loss": 3.6362888970258376, + "tokens_seen": 1128792064 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005317231695085256, + "loss": 1.4112, + "theoretical_loss": 3.6362694075268367, + "tokens_seen": 1128857600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005317071213640924, + "loss": 1.4157, + "theoretical_loss": 3.636249919476059, + "tokens_seen": 1128923136 + }, + { + "epoch": 0.34, + "learning_rate": 0.000531691073219659, + "loss": 1.3547, + "theoretical_loss": 3.6362304328733117, + "tokens_seen": 1128988672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005316750250752258, + "loss": 1.2918, + "theoretical_loss": 3.6362109477184035, + "tokens_seen": 1129054208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005316589769307924, + "loss": 1.394, + "theoretical_loss": 3.6361914640111435, + "tokens_seen": 1129119744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005316429287863592, + "loss": 1.4599, + "theoretical_loss": 3.6361719817513385, + "tokens_seen": 1129185280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005316268806419258, + "loss": 1.3713, + "theoretical_loss": 3.6361525009387985, + "tokens_seen": 1129250816 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005316108324974926, + "loss": 1.2741, + "theoretical_loss": 3.6361330215733316, + "tokens_seen": 1129316352 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005315947843530592, + "loss": 1.3546, + "theoretical_loss": 3.6361135436547456, + "tokens_seen": 1129381888 + }, + { + "epoch": 0.34, + "learning_rate": 0.000531578736208626, + "loss": 1.3232, + "theoretical_loss": 3.63609406718285, + "tokens_seen": 1129447424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005315626880641926, + "loss": 1.329, + "theoretical_loss": 3.6360745921574527, + "tokens_seen": 1129512960 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005315466399197594, + "loss": 1.4433, + "theoretical_loss": 3.6360551185783634, + "tokens_seen": 1129578496 + }, + { + "epoch": 0.34, + "learning_rate": 0.000531530591775326, + "loss": 1.2452, + "theoretical_loss": 3.63603564644539, + "tokens_seen": 1129644032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005315145436308928, + "loss": 1.3164, + "theoretical_loss": 3.636016175758341, + "tokens_seen": 1129709568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005314984954864594, + "loss": 1.524, + "theoretical_loss": 3.635996706517026, + "tokens_seen": 1129775104 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005314824473420262, + "loss": 1.4299, + "theoretical_loss": 3.6359772387212526, + "tokens_seen": 1129840640 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005314663991975928, + "loss": 1.3257, + "theoretical_loss": 3.6359577723708307, + "tokens_seen": 1129906176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005314503510531596, + "loss": 1.396, + "theoretical_loss": 3.635938307465569, + "tokens_seen": 1129971712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005314343029087262, + "loss": 1.3114, + "theoretical_loss": 3.6359188440052757, + "tokens_seen": 1130037248 + }, + { + "epoch": 0.34, + "learning_rate": 0.000531418254764293, + "loss": 1.3683, + "theoretical_loss": 3.6358993819897605, + "tokens_seen": 1130102784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005314022066198596, + "loss": 1.4065, + "theoretical_loss": 3.635879921418832, + "tokens_seen": 1130168320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005313861584754264, + "loss": 1.3705, + "theoretical_loss": 3.6358604622922996, + "tokens_seen": 1130233856 + }, + { + "epoch": 0.34, + "learning_rate": 0.000531370110330993, + "loss": 1.2837, + "theoretical_loss": 3.635841004609972, + "tokens_seen": 1130299392 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005313540621865598, + "loss": 1.3338, + "theoretical_loss": 3.6358215483716587, + "tokens_seen": 1130364928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005313380140421264, + "loss": 1.3236, + "theoretical_loss": 3.635802093577168, + "tokens_seen": 1130430464 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 380591, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.375028133392334, + "objective/train/theoretical_loss": 3.63578264022631, + "objective/train/tokens_used": 1150956000, + "theoretical_loss": 3.63578264022631, + "tokens_seen": 1130496000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005313219658976932, + "loss": 1.2996, + "theoretical_loss": 3.63578264022631, + "tokens_seen": 1130496000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005313059177532598, + "loss": 1.372, + "theoretical_loss": 3.6357631883188932, + "tokens_seen": 1130561536 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005312898696088265, + "loss": 1.3764, + "theoretical_loss": 3.6357437378547273, + "tokens_seen": 1130627072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005312738214643932, + "loss": 1.2847, + "theoretical_loss": 3.635724288833621, + "tokens_seen": 1130692608 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005312577733199599, + "loss": 1.3036, + "theoretical_loss": 3.635704841255384, + "tokens_seen": 1130758144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005312417251755266, + "loss": 1.3314, + "theoretical_loss": 3.6356853951198262, + "tokens_seen": 1130823680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005312256770310933, + "loss": 1.36, + "theoretical_loss": 3.6356659504267563, + "tokens_seen": 1130889216 + }, + { + "epoch": 0.34, + "learning_rate": 0.00053120962888666, + "loss": 1.2925, + "theoretical_loss": 3.6356465071759834, + "tokens_seen": 1130954752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005311935807422267, + "loss": 1.3217, + "theoretical_loss": 3.6356270653673173, + "tokens_seen": 1131020288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005311775325977934, + "loss": 1.2822, + "theoretical_loss": 3.635607625000568, + "tokens_seen": 1131085824 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005311614844533601, + "loss": 1.2929, + "theoretical_loss": 3.6355881860755437, + "tokens_seen": 1131151360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005311454363089267, + "loss": 1.3268, + "theoretical_loss": 3.635568748592056, + "tokens_seen": 1131216896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005311293881644935, + "loss": 1.3252, + "theoretical_loss": 3.6355493125499123, + "tokens_seen": 1131282432 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005311133400200601, + "loss": 1.3213, + "theoretical_loss": 3.6355298779489242, + "tokens_seen": 1131347968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005310972918756269, + "loss": 1.3944, + "theoretical_loss": 3.6355104447889, + "tokens_seen": 1131413504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005310812437311935, + "loss": 1.2435, + "theoretical_loss": 3.6354910130696494, + "tokens_seen": 1131479040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005310651955867603, + "loss": 1.4415, + "theoretical_loss": 3.635471582790983, + "tokens_seen": 1131544576 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005310491474423269, + "loss": 1.3897, + "theoretical_loss": 3.63545215395271, + "tokens_seen": 1131610112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005310330992978937, + "loss": 1.3398, + "theoretical_loss": 3.6354327265546402, + "tokens_seen": 1131675648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005310170511534603, + "loss": 1.3494, + "theoretical_loss": 3.635413300596584, + "tokens_seen": 1131741184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005310010030090271, + "loss": 1.2869, + "theoretical_loss": 3.635393876078351, + "tokens_seen": 1131806720 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005309849548645937, + "loss": 1.3088, + "theoretical_loss": 3.6353744529997503, + "tokens_seen": 1131872256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005309689067201605, + "loss": 1.3001, + "theoretical_loss": 3.635355031360593, + "tokens_seen": 1131937792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005309528585757271, + "loss": 1.2959, + "theoretical_loss": 3.6353356111606887, + "tokens_seen": 1132003328 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005309368104312939, + "loss": 1.3803, + "theoretical_loss": 3.6353161923998476, + "tokens_seen": 1132068864 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005309207622868606, + "loss": 1.2426, + "theoretical_loss": 3.6352967750778786, + "tokens_seen": 1132134400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005309047141424273, + "loss": 1.3723, + "theoretical_loss": 3.6352773591945935, + "tokens_seen": 1132199936 + }, + { + "epoch": 0.34, + "learning_rate": 0.000530888665997994, + "loss": 1.2994, + "theoretical_loss": 3.6352579447498017, + "tokens_seen": 1132265472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005308726178535607, + "loss": 1.2576, + "theoretical_loss": 3.6352385317433136, + "tokens_seen": 1132331008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005308565697091274, + "loss": 1.3685, + "theoretical_loss": 3.6352191201749386, + "tokens_seen": 1132396544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005308405215646941, + "loss": 1.2988, + "theoretical_loss": 3.6351997100444873, + "tokens_seen": 1132462080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005308244734202608, + "loss": 1.2542, + "theoretical_loss": 3.6351803013517707, + "tokens_seen": 1132527616 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005308084252758275, + "loss": 1.3261, + "theoretical_loss": 3.6351608940965985, + "tokens_seen": 1132593152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005307923771313942, + "loss": 1.242, + "theoretical_loss": 3.6351414882787814, + "tokens_seen": 1132658688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005307763289869609, + "loss": 1.3133, + "theoretical_loss": 3.6351220838981293, + "tokens_seen": 1132724224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005307602808425276, + "loss": 1.2503, + "theoretical_loss": 3.6351026809544527, + "tokens_seen": 1132789760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005307442326980943, + "loss": 1.2665, + "theoretical_loss": 3.6350832794475627, + "tokens_seen": 1132855296 + }, + { + "epoch": 0.34, + "learning_rate": 0.000530728184553661, + "loss": 1.3992, + "theoretical_loss": 3.635063879377269, + "tokens_seen": 1132920832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005307121364092277, + "loss": 1.3293, + "theoretical_loss": 3.6350444807433826, + "tokens_seen": 1132986368 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005306960882647944, + "loss": 1.2076, + "theoretical_loss": 3.6350250835457136, + "tokens_seen": 1133051904 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005306800401203611, + "loss": 1.3282, + "theoretical_loss": 3.6350056877840733, + "tokens_seen": 1133117440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005306639919759278, + "loss": 1.3262, + "theoretical_loss": 3.6349862934582715, + "tokens_seen": 1133182976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005306479438314945, + "loss": 1.3443, + "theoretical_loss": 3.63496690056812, + "tokens_seen": 1133248512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005306318956870612, + "loss": 1.3307, + "theoretical_loss": 3.6349475091134287, + "tokens_seen": 1133314048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005306158475426279, + "loss": 1.2516, + "theoretical_loss": 3.6349281190940084, + "tokens_seen": 1133379584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005305997993981946, + "loss": 1.3576, + "theoretical_loss": 3.63490873050967, + "tokens_seen": 1133445120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005305837512537613, + "loss": 1.3443, + "theoretical_loss": 3.6348893433602245, + "tokens_seen": 1133510656 + }, + { + "epoch": 0.34, + "learning_rate": 0.000530567703109328, + "loss": 1.3465, + "theoretical_loss": 3.634869957645482, + "tokens_seen": 1133576192 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005305516549648947, + "loss": 1.385, + "theoretical_loss": 3.634850573365255, + "tokens_seen": 1133641728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005305356068204614, + "loss": 1.3111, + "theoretical_loss": 3.6348311905193524, + "tokens_seen": 1133707264 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 381475, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2271829843521118, + "objective/train/theoretical_loss": 3.6348118091075867, + "objective/train/tokens_used": 1154232800, + "theoretical_loss": 3.6348118091075867, + "tokens_seen": 1133772800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005305195586760281, + "loss": 1.191, + "theoretical_loss": 3.6348118091075867, + "tokens_seen": 1133772800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005305035105315948, + "loss": 1.2194, + "theoretical_loss": 3.634792429129768, + "tokens_seen": 1133838336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005304874623871615, + "loss": 1.2952, + "theoretical_loss": 3.634773050585708, + "tokens_seen": 1133903872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005304714142427283, + "loss": 1.2634, + "theoretical_loss": 3.6347536734752177, + "tokens_seen": 1133969408 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005304553660982949, + "loss": 1.339, + "theoretical_loss": 3.6347342977981074, + "tokens_seen": 1134034944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005304393179538617, + "loss": 1.3236, + "theoretical_loss": 3.6347149235541893, + "tokens_seen": 1134100480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005304232698094283, + "loss": 1.3053, + "theoretical_loss": 3.634695550743274, + "tokens_seen": 1134166016 + }, + { + "epoch": 0.34, + "learning_rate": 0.000530407221664995, + "loss": 1.3035, + "theoretical_loss": 3.6346761793651723, + "tokens_seen": 1134231552 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005303911735205617, + "loss": 1.3321, + "theoretical_loss": 3.6346568094196963, + "tokens_seen": 1134297088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005303751253761285, + "loss": 1.3015, + "theoretical_loss": 3.634637440906657, + "tokens_seen": 1134362624 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005303590772316951, + "loss": 1.3557, + "theoretical_loss": 3.634618073825866, + "tokens_seen": 1134428160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005303430290872619, + "loss": 1.3101, + "theoretical_loss": 3.634598708177134, + "tokens_seen": 1134493696 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005303269809428285, + "loss": 1.2787, + "theoretical_loss": 3.6345793439602723, + "tokens_seen": 1134559232 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005303109327983953, + "loss": 1.4081, + "theoretical_loss": 3.6345599811750935, + "tokens_seen": 1134624768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005302948846539619, + "loss": 1.3721, + "theoretical_loss": 3.634540619821408, + "tokens_seen": 1134690304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005302788365095287, + "loss": 1.3142, + "theoretical_loss": 3.634521259899027, + "tokens_seen": 1134755840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005302627883650953, + "loss": 1.4324, + "theoretical_loss": 3.6345019014077637, + "tokens_seen": 1134821376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005302467402206621, + "loss": 1.247, + "theoretical_loss": 3.634482544347428, + "tokens_seen": 1134886912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005302306920762287, + "loss": 1.3462, + "theoretical_loss": 3.634463188717832, + "tokens_seen": 1134952448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005302146439317955, + "loss": 1.301, + "theoretical_loss": 3.634443834518788, + "tokens_seen": 1135017984 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005301985957873621, + "loss": 1.3281, + "theoretical_loss": 3.6344244817501066, + "tokens_seen": 1135083520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005301825476429289, + "loss": 1.3277, + "theoretical_loss": 3.6344051304116007, + "tokens_seen": 1135149056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005301664994984955, + "loss": 1.4713, + "theoretical_loss": 3.634385780503081, + "tokens_seen": 1135214592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005301504513540623, + "loss": 1.3735, + "theoretical_loss": 3.6343664320243594, + "tokens_seen": 1135280128 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005301344032096289, + "loss": 1.254, + "theoretical_loss": 3.6343470849752486, + "tokens_seen": 1135345664 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005301183550651957, + "loss": 1.4025, + "theoretical_loss": 3.6343277393555597, + "tokens_seen": 1135411200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005301023069207623, + "loss": 1.2399, + "theoretical_loss": 3.634308395165105, + "tokens_seen": 1135476736 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005300862587763291, + "loss": 1.2324, + "theoretical_loss": 3.6342890524036955, + "tokens_seen": 1135542272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005300702106318957, + "loss": 1.3211, + "theoretical_loss": 3.6342697110711444, + "tokens_seen": 1135607808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005300541624874625, + "loss": 1.401, + "theoretical_loss": 3.634250371167263, + "tokens_seen": 1135673344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005300381143430291, + "loss": 1.3394, + "theoretical_loss": 3.634231032691863, + "tokens_seen": 1135738880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005300220661985959, + "loss": 1.3315, + "theoretical_loss": 3.634211695644758, + "tokens_seen": 1135804416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005300060180541625, + "loss": 1.3305, + "theoretical_loss": 3.634192360025758, + "tokens_seen": 1135869952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005299899699097293, + "loss": 1.2994, + "theoretical_loss": 3.634173025834677, + "tokens_seen": 1135935488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005299739217652959, + "loss": 1.3139, + "theoretical_loss": 3.6341536930713265, + "tokens_seen": 1136001024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005299578736208627, + "loss": 1.3998, + "theoretical_loss": 3.6341343617355184, + "tokens_seen": 1136066560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005299418254764293, + "loss": 1.3566, + "theoretical_loss": 3.634115031827065, + "tokens_seen": 1136132096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005299257773319961, + "loss": 1.3159, + "theoretical_loss": 3.6340957033457784, + "tokens_seen": 1136197632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005299097291875627, + "loss": 1.2776, + "theoretical_loss": 3.6340763762914716, + "tokens_seen": 1136263168 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005298936810431294, + "loss": 1.3585, + "theoretical_loss": 3.634057050663957, + "tokens_seen": 1136328704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005298776328986961, + "loss": 1.473, + "theoretical_loss": 3.634037726463046, + "tokens_seen": 1136394240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005298615847542628, + "loss": 1.3581, + "theoretical_loss": 3.634018403688552, + "tokens_seen": 1136459776 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005298455366098295, + "loss": 1.404, + "theoretical_loss": 3.633999082340287, + "tokens_seen": 1136525312 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005298294884653962, + "loss": 1.3263, + "theoretical_loss": 3.633979762418063, + "tokens_seen": 1136590848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005298134403209629, + "loss": 1.3379, + "theoretical_loss": 3.6339604439216937, + "tokens_seen": 1136656384 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005297973921765296, + "loss": 1.4255, + "theoretical_loss": 3.633941126850991, + "tokens_seen": 1136721920 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005297813440320963, + "loss": 1.4317, + "theoretical_loss": 3.6339218112057674, + "tokens_seen": 1136787456 + }, + { + "epoch": 0.34, + "learning_rate": 0.000529765295887663, + "loss": 1.4105, + "theoretical_loss": 3.633902496985836, + "tokens_seen": 1136852992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005297492477432296, + "loss": 1.2663, + "theoretical_loss": 3.633883184191009, + "tokens_seen": 1136918528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005297331995987964, + "loss": 1.2736, + "theoretical_loss": 3.6338638728210997, + "tokens_seen": 1136984064 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 382718, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4597383737564087, + "objective/train/theoretical_loss": 3.63384456287592, + "objective/train/tokens_used": 1157509600, + "theoretical_loss": 3.63384456287592, + "tokens_seen": 1137049600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005297171514543631, + "loss": 1.4389, + "theoretical_loss": 3.63384456287592, + "tokens_seen": 1137049600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005297011033099298, + "loss": 1.3618, + "theoretical_loss": 3.6338252543552834, + "tokens_seen": 1137115136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005296850551654965, + "loss": 1.2754, + "theoretical_loss": 3.6338059472590025, + "tokens_seen": 1137180672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005296690070210632, + "loss": 1.3212, + "theoretical_loss": 3.63378664158689, + "tokens_seen": 1137246208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005296529588766299, + "loss": 1.3643, + "theoretical_loss": 3.6337673373387585, + "tokens_seen": 1137311744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005296369107321966, + "loss": 1.39, + "theoretical_loss": 3.6337480345144217, + "tokens_seen": 1137377280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005296208625877633, + "loss": 1.2826, + "theoretical_loss": 3.633728733113692, + "tokens_seen": 1137442816 + }, + { + "epoch": 0.34, + "learning_rate": 0.00052960481444333, + "loss": 1.4381, + "theoretical_loss": 3.6337094331363824, + "tokens_seen": 1137508352 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005295887662988967, + "loss": 1.4166, + "theoretical_loss": 3.6336901345823067, + "tokens_seen": 1137573888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005295727181544634, + "loss": 1.3983, + "theoretical_loss": 3.633670837451277, + "tokens_seen": 1137639424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005295566700100301, + "loss": 1.3055, + "theoretical_loss": 3.633651541743107, + "tokens_seen": 1137704960 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005295406218655968, + "loss": 1.3178, + "theoretical_loss": 3.633632247457609, + "tokens_seen": 1137770496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005295245737211635, + "loss": 1.3164, + "theoretical_loss": 3.633612954594597, + "tokens_seen": 1137836032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005295085255767302, + "loss": 1.325, + "theoretical_loss": 3.633593663153884, + "tokens_seen": 1137901568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005294924774322969, + "loss": 1.3331, + "theoretical_loss": 3.6335743731352834, + "tokens_seen": 1137967104 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005294764292878636, + "loss": 1.3119, + "theoretical_loss": 3.633555084538608, + "tokens_seen": 1138032640 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005294603811434303, + "loss": 1.4214, + "theoretical_loss": 3.6335357973636713, + "tokens_seen": 1138098176 + }, + { + "epoch": 0.34, + "learning_rate": 0.000529444332998997, + "loss": 1.2846, + "theoretical_loss": 3.6335165116102868, + "tokens_seen": 1138163712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005294282848545637, + "loss": 1.3522, + "theoretical_loss": 3.633497227278268, + "tokens_seen": 1138229248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005294122367101304, + "loss": 1.4455, + "theoretical_loss": 3.633477944367428, + "tokens_seen": 1138294784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005293961885656971, + "loss": 1.338, + "theoretical_loss": 3.6334586628775805, + "tokens_seen": 1138360320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005293801404212638, + "loss": 1.2857, + "theoretical_loss": 3.6334393828085387, + "tokens_seen": 1138425856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005293640922768305, + "loss": 1.4045, + "theoretical_loss": 3.633420104160116, + "tokens_seen": 1138491392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005293480441323972, + "loss": 1.2807, + "theoretical_loss": 3.6334008269321263, + "tokens_seen": 1138556928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005293319959879639, + "loss": 1.3043, + "theoretical_loss": 3.6333815511243834, + "tokens_seen": 1138622464 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005293159478435306, + "loss": 1.2798, + "theoretical_loss": 3.633362276736701, + "tokens_seen": 1138688000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005292998996990973, + "loss": 1.2801, + "theoretical_loss": 3.633343003768892, + "tokens_seen": 1138753536 + }, + { + "epoch": 0.35, + "learning_rate": 0.000529283851554664, + "loss": 1.3817, + "theoretical_loss": 3.63332373222077, + "tokens_seen": 1138819072 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005292678034102307, + "loss": 1.3764, + "theoretical_loss": 3.63330446209215, + "tokens_seen": 1138884608 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005292517552657974, + "loss": 1.4049, + "theoretical_loss": 3.6332851933828447, + "tokens_seen": 1138950144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005292357071213641, + "loss": 1.2247, + "theoretical_loss": 3.6332659260926685, + "tokens_seen": 1139015680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005292196589769308, + "loss": 1.4062, + "theoretical_loss": 3.6332466602214346, + "tokens_seen": 1139081216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005292036108324975, + "loss": 1.3674, + "theoretical_loss": 3.6332273957689574, + "tokens_seen": 1139146752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005291875626880642, + "loss": 1.4022, + "theoretical_loss": 3.6332081327350503, + "tokens_seen": 1139212288 + }, + { + "epoch": 0.35, + "learning_rate": 0.000529171514543631, + "loss": 1.3372, + "theoretical_loss": 3.633188871119528, + "tokens_seen": 1139277824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005291554663991976, + "loss": 1.3026, + "theoretical_loss": 3.633169610922204, + "tokens_seen": 1139343360 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005291394182547643, + "loss": 1.2527, + "theoretical_loss": 3.6331503521428923, + "tokens_seen": 1139408896 + }, + { + "epoch": 0.35, + "learning_rate": 0.000529123370110331, + "loss": 1.4002, + "theoretical_loss": 3.6331310947814073, + "tokens_seen": 1139474432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005291073219658978, + "loss": 1.2718, + "theoretical_loss": 3.6331118388375625, + "tokens_seen": 1139539968 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005290912738214644, + "loss": 1.4031, + "theoretical_loss": 3.6330925843111723, + "tokens_seen": 1139605504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005290752256770312, + "loss": 1.4044, + "theoretical_loss": 3.633073331202051, + "tokens_seen": 1139671040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005290591775325978, + "loss": 1.4823, + "theoretical_loss": 3.6330540795100124, + "tokens_seen": 1139736576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005290431293881646, + "loss": 1.3078, + "theoretical_loss": 3.6330348292348713, + "tokens_seen": 1139802112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005290270812437312, + "loss": 1.3579, + "theoretical_loss": 3.6330155803764415, + "tokens_seen": 1139867648 + }, + { + "epoch": 0.35, + "learning_rate": 0.000529011033099298, + "loss": 1.3369, + "theoretical_loss": 3.632996332934537, + "tokens_seen": 1139933184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005289949849548646, + "loss": 1.2965, + "theoretical_loss": 3.6329770869089737, + "tokens_seen": 1139998720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005289789368104314, + "loss": 1.3081, + "theoretical_loss": 3.6329578422995636, + "tokens_seen": 1140064256 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528962888665998, + "loss": 1.3686, + "theoretical_loss": 3.632938599106123, + "tokens_seen": 1140129792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005289468405215648, + "loss": 1.3117, + "theoretical_loss": 3.6329193573284657, + "tokens_seen": 1140195328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005289307923771314, + "loss": 1.3551, + "theoretical_loss": 3.6329001169664057, + "tokens_seen": 1140260864 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 383588, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.193879246711731, + "objective/train/theoretical_loss": 3.6328808780197583, + "objective/train/tokens_used": 1160786400, + "theoretical_loss": 3.6328808780197583, + "tokens_seen": 1140326400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005289147442326982, + "loss": 1.3129, + "theoretical_loss": 3.6328808780197583, + "tokens_seen": 1140326400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005288986960882648, + "loss": 1.3148, + "theoretical_loss": 3.632861640488338, + "tokens_seen": 1140391936 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005288826479438316, + "loss": 1.4849, + "theoretical_loss": 3.6328424043719587, + "tokens_seen": 1140457472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005288665997993982, + "loss": 1.2353, + "theoretical_loss": 3.632823169670435, + "tokens_seen": 1140523008 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528850551654965, + "loss": 1.4659, + "theoretical_loss": 3.6328039363835822, + "tokens_seen": 1140588544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005288345035105316, + "loss": 1.2511, + "theoretical_loss": 3.632784704511215, + "tokens_seen": 1140654080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005288184553660984, + "loss": 1.3932, + "theoretical_loss": 3.632765474053148, + "tokens_seen": 1140719616 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528802407221665, + "loss": 1.2355, + "theoretical_loss": 3.632746245009195, + "tokens_seen": 1140785152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005287863590772318, + "loss": 1.418, + "theoretical_loss": 3.632727017379172, + "tokens_seen": 1140850688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005287703109327984, + "loss": 1.3035, + "theoretical_loss": 3.632707791162893, + "tokens_seen": 1140916224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005287542627883652, + "loss": 1.2606, + "theoretical_loss": 3.6326885663601733, + "tokens_seen": 1140981760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005287382146439318, + "loss": 1.3402, + "theoretical_loss": 3.6326693429708277, + "tokens_seen": 1141047296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005287221664994986, + "loss": 1.3486, + "theoretical_loss": 3.632650120994671, + "tokens_seen": 1141112832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005287061183550652, + "loss": 1.3213, + "theoretical_loss": 3.6326309004315185, + "tokens_seen": 1141178368 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528690070210632, + "loss": 1.2077, + "theoretical_loss": 3.632611681281185, + "tokens_seen": 1141243904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005286740220661986, + "loss": 1.3652, + "theoretical_loss": 3.632592463543485, + "tokens_seen": 1141309440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005286579739217654, + "loss": 1.3507, + "theoretical_loss": 3.632573247218235, + "tokens_seen": 1141374976 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528641925777332, + "loss": 1.3774, + "theoretical_loss": 3.6325540323052485, + "tokens_seen": 1141440512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005286258776328988, + "loss": 1.3804, + "theoretical_loss": 3.6325348188043414, + "tokens_seen": 1141506048 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005286098294884654, + "loss": 1.2376, + "theoretical_loss": 3.632515606715329, + "tokens_seen": 1141571584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005285937813440322, + "loss": 1.1911, + "theoretical_loss": 3.6324963960380257, + "tokens_seen": 1141637120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005285777331995988, + "loss": 1.3345, + "theoretical_loss": 3.632477186772248, + "tokens_seen": 1141702656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005285616850551656, + "loss": 1.3125, + "theoretical_loss": 3.6324579789178095, + "tokens_seen": 1141768192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005285456369107322, + "loss": 1.477, + "theoretical_loss": 3.6324387724745275, + "tokens_seen": 1141833728 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528529588766299, + "loss": 1.3276, + "theoretical_loss": 3.6324195674422155, + "tokens_seen": 1141899264 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005285135406218656, + "loss": 1.3045, + "theoretical_loss": 3.63240036382069, + "tokens_seen": 1141964800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005284974924774324, + "loss": 1.2982, + "theoretical_loss": 3.632381161609766, + "tokens_seen": 1142030336 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528481444332999, + "loss": 1.178, + "theoretical_loss": 3.632361960809259, + "tokens_seen": 1142095872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005284653961885657, + "loss": 1.3646, + "theoretical_loss": 3.632342761418985, + "tokens_seen": 1142161408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005284493480441324, + "loss": 1.3485, + "theoretical_loss": 3.6323235634387587, + "tokens_seen": 1142226944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005284332998996991, + "loss": 1.462, + "theoretical_loss": 3.632304366868396, + "tokens_seen": 1142292480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005284172517552658, + "loss": 1.3052, + "theoretical_loss": 3.6322851717077125, + "tokens_seen": 1142358016 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005284012036108325, + "loss": 1.2837, + "theoretical_loss": 3.632265977956524, + "tokens_seen": 1142423552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005283851554663992, + "loss": 1.3251, + "theoretical_loss": 3.6322467856146456, + "tokens_seen": 1142489088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005283691073219659, + "loss": 1.3377, + "theoretical_loss": 3.6322275946818934, + "tokens_seen": 1142554624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005283530591775326, + "loss": 1.2977, + "theoretical_loss": 3.632208405158083, + "tokens_seen": 1142620160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005283370110330993, + "loss": 1.3095, + "theoretical_loss": 3.63218921704303, + "tokens_seen": 1142685696 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528320962888666, + "loss": 1.3895, + "theoretical_loss": 3.632170030336551, + "tokens_seen": 1142751232 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005283049147442327, + "loss": 1.2499, + "theoretical_loss": 3.6321508450384608, + "tokens_seen": 1142816768 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005282888665997994, + "loss": 1.3816, + "theoretical_loss": 3.6321316611485757, + "tokens_seen": 1142882304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005282728184553661, + "loss": 1.2767, + "theoretical_loss": 3.632112478666712, + "tokens_seen": 1142947840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005282567703109328, + "loss": 1.4018, + "theoretical_loss": 3.6320932975926845, + "tokens_seen": 1143013376 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005282407221664995, + "loss": 1.2862, + "theoretical_loss": 3.6320741179263103, + "tokens_seen": 1143078912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005282246740220662, + "loss": 1.2824, + "theoretical_loss": 3.632054939667405, + "tokens_seen": 1143144448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005282086258776329, + "loss": 1.346, + "theoretical_loss": 3.6320357628157844, + "tokens_seen": 1143209984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005281925777331996, + "loss": 1.4018, + "theoretical_loss": 3.6320165873712646, + "tokens_seen": 1143275520 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005281765295887663, + "loss": 1.3485, + "theoretical_loss": 3.6319974133336625, + "tokens_seen": 1143341056 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528160481444333, + "loss": 1.3261, + "theoretical_loss": 3.6319782407027934, + "tokens_seen": 1143406592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005281444332998997, + "loss": 1.356, + "theoretical_loss": 3.631959069478474, + "tokens_seen": 1143472128 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005281283851554664, + "loss": 1.364, + "theoretical_loss": 3.6319398996605194, + "tokens_seen": 1143537664 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 384799, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5311000347137451, + "objective/train/theoretical_loss": 3.631920731248747, + "objective/train/tokens_used": 1164063200, + "theoretical_loss": 3.631920731248747, + "tokens_seen": 1143603200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005281123370110331, + "loss": 1.4664, + "theoretical_loss": 3.631920731248747, + "tokens_seen": 1143603200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005280962888665998, + "loss": 1.3345, + "theoretical_loss": 3.6319015642429733, + "tokens_seen": 1143668736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005280802407221665, + "loss": 1.3935, + "theoretical_loss": 3.631882398643013, + "tokens_seen": 1143734272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005280641925777332, + "loss": 1.2661, + "theoretical_loss": 3.6318632344486845, + "tokens_seen": 1143799808 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005280481444332999, + "loss": 1.3856, + "theoretical_loss": 3.6318440716598026, + "tokens_seen": 1143865344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005280320962888666, + "loss": 1.2881, + "theoretical_loss": 3.631824910276184, + "tokens_seen": 1143930880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005280160481444333, + "loss": 1.3577, + "theoretical_loss": 3.631805750297646, + "tokens_seen": 1143996416 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528, + "loss": 1.2978, + "theoretical_loss": 3.631786591724004, + "tokens_seen": 1144061952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005279839518555667, + "loss": 1.3174, + "theoretical_loss": 3.6317674345550754, + "tokens_seen": 1144127488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005279679037111334, + "loss": 1.3476, + "theoretical_loss": 3.6317482787906763, + "tokens_seen": 1144193024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005279518555667001, + "loss": 1.3092, + "theoretical_loss": 3.6317291244306236, + "tokens_seen": 1144258560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005279358074222668, + "loss": 1.2958, + "theoretical_loss": 3.6317099714747334, + "tokens_seen": 1144324096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005279197592778335, + "loss": 1.2297, + "theoretical_loss": 3.631690819922823, + "tokens_seen": 1144389632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005279037111334002, + "loss": 1.345, + "theoretical_loss": 3.631671669774708, + "tokens_seen": 1144455168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005278876629889669, + "loss": 1.3698, + "theoretical_loss": 3.6316525210302064, + "tokens_seen": 1144520704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005278716148445336, + "loss": 1.4033, + "theoretical_loss": 3.6316333736891337, + "tokens_seen": 1144586240 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005278555667001003, + "loss": 1.3209, + "theoretical_loss": 3.6316142277513084, + "tokens_seen": 1144651776 + }, + { + "epoch": 0.35, + "learning_rate": 0.000527839518555667, + "loss": 1.2795, + "theoretical_loss": 3.6315950832165456, + "tokens_seen": 1144717312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005278234704112337, + "loss": 1.3696, + "theoretical_loss": 3.6315759400846632, + "tokens_seen": 1144782848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005278074222668004, + "loss": 1.3169, + "theoretical_loss": 3.631556798355478, + "tokens_seen": 1144848384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005277913741223671, + "loss": 1.3511, + "theoretical_loss": 3.6315376580288063, + "tokens_seen": 1144913920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005277753259779338, + "loss": 1.3794, + "theoretical_loss": 3.631518519104466, + "tokens_seen": 1144979456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005277592778335005, + "loss": 1.3197, + "theoretical_loss": 3.631499381582273, + "tokens_seen": 1145044992 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005277432296890673, + "loss": 1.2908, + "theoretical_loss": 3.6314802454620456, + "tokens_seen": 1145110528 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005277271815446339, + "loss": 1.3856, + "theoretical_loss": 3.6314611107436, + "tokens_seen": 1145176064 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005277111334002007, + "loss": 1.3486, + "theoretical_loss": 3.6314419774267535, + "tokens_seen": 1145241600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005276950852557673, + "loss": 1.2676, + "theoretical_loss": 3.631422845511323, + "tokens_seen": 1145307136 + }, + { + "epoch": 0.35, + "learning_rate": 0.000527679037111334, + "loss": 1.3219, + "theoretical_loss": 3.6314037149971257, + "tokens_seen": 1145372672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005276629889669007, + "loss": 1.3939, + "theoretical_loss": 3.6313845858839793, + "tokens_seen": 1145438208 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005276469408224675, + "loss": 1.3421, + "theoretical_loss": 3.6313654581717008, + "tokens_seen": 1145503744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005276308926780341, + "loss": 1.3355, + "theoretical_loss": 3.631346331860107, + "tokens_seen": 1145569280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005276148445336009, + "loss": 1.3466, + "theoretical_loss": 3.631327206949016, + "tokens_seen": 1145634816 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005275987963891675, + "loss": 1.3706, + "theoretical_loss": 3.6313080834382445, + "tokens_seen": 1145700352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005275827482447343, + "loss": 1.428, + "theoretical_loss": 3.631288961327611, + "tokens_seen": 1145765888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005275667001003009, + "loss": 1.4646, + "theoretical_loss": 3.6312698406169313, + "tokens_seen": 1145831424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005275506519558677, + "loss": 1.2793, + "theoretical_loss": 3.6312507213060234, + "tokens_seen": 1145896960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005275346038114343, + "loss": 1.3848, + "theoretical_loss": 3.6312316033947054, + "tokens_seen": 1145962496 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005275185556670011, + "loss": 1.3406, + "theoretical_loss": 3.6312124868827946, + "tokens_seen": 1146028032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005275025075225677, + "loss": 1.3627, + "theoretical_loss": 3.631193371770108, + "tokens_seen": 1146093568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005274864593781345, + "loss": 1.2938, + "theoretical_loss": 3.631174258056464, + "tokens_seen": 1146159104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005274704112337011, + "loss": 1.2908, + "theoretical_loss": 3.6311551457416793, + "tokens_seen": 1146224640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005274543630892679, + "loss": 1.2411, + "theoretical_loss": 3.6311360348255723, + "tokens_seen": 1146290176 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005274383149448345, + "loss": 1.3654, + "theoretical_loss": 3.63111692530796, + "tokens_seen": 1146355712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005274222668004013, + "loss": 1.3042, + "theoretical_loss": 3.6310978171886608, + "tokens_seen": 1146421248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005274062186559679, + "loss": 1.3152, + "theoretical_loss": 3.631078710467492, + "tokens_seen": 1146486784 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005273901705115347, + "loss": 1.3923, + "theoretical_loss": 3.6310596051442716, + "tokens_seen": 1146552320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005273741223671013, + "loss": 1.3292, + "theoretical_loss": 3.6310405012188176, + "tokens_seen": 1146617856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005273580742226681, + "loss": 1.5038, + "theoretical_loss": 3.6310213986909474, + "tokens_seen": 1146683392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005273420260782347, + "loss": 1.3402, + "theoretical_loss": 3.631002297560479, + "tokens_seen": 1146748928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005273259779338015, + "loss": 1.3025, + "theoretical_loss": 3.6309831978272307, + "tokens_seen": 1146814464 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 386018, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2878888845443726, + "objective/train/theoretical_loss": 3.63096409949102, + "objective/train/tokens_used": 1167340000, + "theoretical_loss": 3.63096409949102, + "tokens_seen": 1146880000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005273099297893682, + "loss": 1.3574, + "theoretical_loss": 3.63096409949102, + "tokens_seen": 1146880000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005272938816449349, + "loss": 1.2588, + "theoretical_loss": 3.630945002551665, + "tokens_seen": 1146945536 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005272778335005016, + "loss": 1.3601, + "theoretical_loss": 3.630925907008984, + "tokens_seen": 1147011072 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005272617853560683, + "loss": 1.2096, + "theoretical_loss": 3.6309068128627953, + "tokens_seen": 1147076608 + }, + { + "epoch": 0.35, + "learning_rate": 0.000527245737211635, + "loss": 1.2368, + "theoretical_loss": 3.630887720112916, + "tokens_seen": 1147142144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005272296890672017, + "loss": 1.2394, + "theoretical_loss": 3.630868628759165, + "tokens_seen": 1147207680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005272136409227683, + "loss": 1.3448, + "theoretical_loss": 3.6308495388013604, + "tokens_seen": 1147273216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005271975927783351, + "loss": 1.2866, + "theoretical_loss": 3.6308304502393196, + "tokens_seen": 1147338752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005271815446339017, + "loss": 1.2818, + "theoretical_loss": 3.630811363072862, + "tokens_seen": 1147404288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005271654964894685, + "loss": 1.2521, + "theoretical_loss": 3.6307922773018055, + "tokens_seen": 1147469824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005271494483450351, + "loss": 1.383, + "theoretical_loss": 3.630773192925968, + "tokens_seen": 1147535360 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005271334002006019, + "loss": 1.292, + "theoretical_loss": 3.630754109945168, + "tokens_seen": 1147600896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005271173520561685, + "loss": 1.3332, + "theoretical_loss": 3.6307350283592243, + "tokens_seen": 1147666432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005271013039117353, + "loss": 1.3052, + "theoretical_loss": 3.6307159481679547, + "tokens_seen": 1147731968 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005270852557673019, + "loss": 1.2793, + "theoretical_loss": 3.630696869371178, + "tokens_seen": 1147797504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005270692076228686, + "loss": 1.2779, + "theoretical_loss": 3.6306777919687128, + "tokens_seen": 1147863040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005270531594784353, + "loss": 1.3197, + "theoretical_loss": 3.630658715960377, + "tokens_seen": 1147928576 + }, + { + "epoch": 0.35, + "learning_rate": 0.000527037111334002, + "loss": 1.2939, + "theoretical_loss": 3.63063964134599, + "tokens_seen": 1147994112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005270210631895687, + "loss": 1.3744, + "theoretical_loss": 3.6306205681253694, + "tokens_seen": 1148059648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005270050150451354, + "loss": 1.4097, + "theoretical_loss": 3.6306014962983344, + "tokens_seen": 1148125184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005269889669007021, + "loss": 1.2644, + "theoretical_loss": 3.6305824258647035, + "tokens_seen": 1148190720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005269729187562688, + "loss": 1.3718, + "theoretical_loss": 3.6305633568242954, + "tokens_seen": 1148256256 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005269568706118355, + "loss": 1.3295, + "theoretical_loss": 3.630544289176929, + "tokens_seen": 1148321792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005269408224674022, + "loss": 1.3056, + "theoretical_loss": 3.6305252229224223, + "tokens_seen": 1148387328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005269247743229689, + "loss": 1.2321, + "theoretical_loss": 3.6305061580605953, + "tokens_seen": 1148452864 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005269087261785356, + "loss": 1.3242, + "theoretical_loss": 3.630487094591266, + "tokens_seen": 1148518400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005268926780341023, + "loss": 1.3703, + "theoretical_loss": 3.630468032514253, + "tokens_seen": 1148583936 + }, + { + "epoch": 0.35, + "learning_rate": 0.000526876629889669, + "loss": 1.2667, + "theoretical_loss": 3.6304489718293755, + "tokens_seen": 1148649472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005268605817452357, + "loss": 1.3421, + "theoretical_loss": 3.630429912536453, + "tokens_seen": 1148715008 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005268445336008024, + "loss": 1.4368, + "theoretical_loss": 3.6304108546353033, + "tokens_seen": 1148780544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005268284854563691, + "loss": 1.3279, + "theoretical_loss": 3.6303917981257467, + "tokens_seen": 1148846080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005268124373119358, + "loss": 1.2554, + "theoretical_loss": 3.6303727430076007, + "tokens_seen": 1148911616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005267963891675025, + "loss": 1.2973, + "theoretical_loss": 3.630353689280686, + "tokens_seen": 1148977152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005267803410230692, + "loss": 1.3164, + "theoretical_loss": 3.63033463694482, + "tokens_seen": 1149042688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005267642928786359, + "loss": 1.241, + "theoretical_loss": 3.6303155859998233, + "tokens_seen": 1149108224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005267482447342026, + "loss": 1.2774, + "theoretical_loss": 3.630296536445514, + "tokens_seen": 1149173760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005267321965897693, + "loss": 1.4199, + "theoretical_loss": 3.630277488281712, + "tokens_seen": 1149239296 + }, + { + "epoch": 0.35, + "learning_rate": 0.000526716148445336, + "loss": 1.3624, + "theoretical_loss": 3.6302584415082357, + "tokens_seen": 1149304832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005267001003009027, + "loss": 1.2518, + "theoretical_loss": 3.630239396124905, + "tokens_seen": 1149370368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005266840521564694, + "loss": 1.3389, + "theoretical_loss": 3.630220352131539, + "tokens_seen": 1149435904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005266680040120361, + "loss": 1.3208, + "theoretical_loss": 3.6302013095279575, + "tokens_seen": 1149501440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005266519558676028, + "loss": 1.4276, + "theoretical_loss": 3.6301822683139786, + "tokens_seen": 1149566976 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005266359077231695, + "loss": 1.2476, + "theoretical_loss": 3.6301632284894225, + "tokens_seen": 1149632512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005266198595787362, + "loss": 1.2424, + "theoretical_loss": 3.630144190054109, + "tokens_seen": 1149698048 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005266038114343029, + "loss": 1.3232, + "theoretical_loss": 3.6301251530078575, + "tokens_seen": 1149763584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005265877632898696, + "loss": 1.2944, + "theoretical_loss": 3.6301061173504863, + "tokens_seen": 1149829120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005265717151454363, + "loss": 1.4103, + "theoretical_loss": 3.6300870830818166, + "tokens_seen": 1149894656 + }, + { + "epoch": 0.35, + "learning_rate": 0.000526555667001003, + "loss": 1.3234, + "theoretical_loss": 3.6300680502016665, + "tokens_seen": 1149960192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005265396188565697, + "loss": 1.3174, + "theoretical_loss": 3.630049018709856, + "tokens_seen": 1150025728 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005265235707121364, + "loss": 1.3077, + "theoretical_loss": 3.6300299886062053, + "tokens_seen": 1150091264 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 386869, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3539170026779175, + "objective/train/theoretical_loss": 3.6300109598905332, + "objective/train/tokens_used": 1170616800, + "theoretical_loss": 3.6300109598905332, + "tokens_seen": 1150156800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005265075225677031, + "loss": 1.2592, + "theoretical_loss": 3.6300109598905332, + "tokens_seen": 1150156800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005264914744232698, + "loss": 1.4126, + "theoretical_loss": 3.6299919325626604, + "tokens_seen": 1150222336 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005264754262788365, + "loss": 1.2547, + "theoretical_loss": 3.629972906622406, + "tokens_seen": 1150287872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005264593781344032, + "loss": 1.3082, + "theoretical_loss": 3.6299538820695894, + "tokens_seen": 1150353408 + }, + { + "epoch": 0.35, + "learning_rate": 0.00052644332998997, + "loss": 1.3854, + "theoretical_loss": 3.629934858904031, + "tokens_seen": 1150418944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005264272818455366, + "loss": 1.3252, + "theoretical_loss": 3.6299158371255507, + "tokens_seen": 1150484480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005264112337011033, + "loss": 1.4755, + "theoretical_loss": 3.629896816733967, + "tokens_seen": 1150550016 + }, + { + "epoch": 0.35, + "learning_rate": 0.00052639518555667, + "loss": 1.3732, + "theoretical_loss": 3.6298777977291024, + "tokens_seen": 1150615552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005263791374122368, + "loss": 1.2985, + "theoretical_loss": 3.6298587801107747, + "tokens_seen": 1150681088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005263630892678034, + "loss": 1.2555, + "theoretical_loss": 3.6298397638788042, + "tokens_seen": 1150746624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005263470411233702, + "loss": 1.3708, + "theoretical_loss": 3.629820749033011, + "tokens_seen": 1150812160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005263309929789368, + "loss": 1.2837, + "theoretical_loss": 3.6298017355732157, + "tokens_seen": 1150877696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005263149448345036, + "loss": 1.3719, + "theoretical_loss": 3.629782723499238, + "tokens_seen": 1150943232 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005262988966900702, + "loss": 1.2352, + "theoretical_loss": 3.629763712810898, + "tokens_seen": 1151008768 + }, + { + "epoch": 0.35, + "learning_rate": 0.000526282848545637, + "loss": 1.4079, + "theoretical_loss": 3.629744703508015, + "tokens_seen": 1151074304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005262668004012036, + "loss": 1.3179, + "theoretical_loss": 3.6297256955904107, + "tokens_seen": 1151139840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005262507522567704, + "loss": 1.2817, + "theoretical_loss": 3.6297066890579037, + "tokens_seen": 1151205376 + }, + { + "epoch": 0.35, + "learning_rate": 0.000526234704112337, + "loss": 1.2657, + "theoretical_loss": 3.6296876839103156, + "tokens_seen": 1151270912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005262186559679038, + "loss": 1.3571, + "theoretical_loss": 3.629668680147466, + "tokens_seen": 1151336448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005262026078234704, + "loss": 1.4269, + "theoretical_loss": 3.6296496777691756, + "tokens_seen": 1151401984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005261865596790372, + "loss": 1.3469, + "theoretical_loss": 3.629630676775264, + "tokens_seen": 1151467520 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005261705115346038, + "loss": 1.3293, + "theoretical_loss": 3.6296116771655518, + "tokens_seen": 1151533056 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005261544633901706, + "loss": 1.2247, + "theoretical_loss": 3.62959267893986, + "tokens_seen": 1151598592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005261384152457372, + "loss": 1.327, + "theoretical_loss": 3.6295736820980085, + "tokens_seen": 1151664128 + }, + { + "epoch": 0.35, + "learning_rate": 0.000526122367101304, + "loss": 1.3716, + "theoretical_loss": 3.6295546866398176, + "tokens_seen": 1151729664 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005261063189568707, + "loss": 1.3355, + "theoretical_loss": 3.629535692565108, + "tokens_seen": 1151795200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005260902708124374, + "loss": 1.3666, + "theoretical_loss": 3.6295166998737005, + "tokens_seen": 1151860736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005260742226680041, + "loss": 1.3776, + "theoretical_loss": 3.6294977085654154, + "tokens_seen": 1151926272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005260581745235708, + "loss": 1.3003, + "theoretical_loss": 3.6294787186400734, + "tokens_seen": 1151991808 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005260421263791375, + "loss": 1.3106, + "theoretical_loss": 3.629459730097495, + "tokens_seen": 1152057344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005260260782347042, + "loss": 1.2752, + "theoretical_loss": 3.6294407429375006, + "tokens_seen": 1152122880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005260100300902709, + "loss": 1.2717, + "theoretical_loss": 3.6294217571599114, + "tokens_seen": 1152188416 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005259939819458376, + "loss": 1.333, + "theoretical_loss": 3.629402772764548, + "tokens_seen": 1152253952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005259779338014043, + "loss": 1.2802, + "theoretical_loss": 3.6293837897512313, + "tokens_seen": 1152319488 + }, + { + "epoch": 0.35, + "learning_rate": 0.000525961885656971, + "loss": 1.3182, + "theoretical_loss": 3.6293648081197816, + "tokens_seen": 1152385024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005259458375125377, + "loss": 1.319, + "theoretical_loss": 3.6293458278700204, + "tokens_seen": 1152450560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005259297893681044, + "loss": 1.2196, + "theoretical_loss": 3.6293268490017674, + "tokens_seen": 1152516096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005259137412236711, + "loss": 1.4032, + "theoretical_loss": 3.629307871514845, + "tokens_seen": 1152581632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005258976930792378, + "loss": 1.347, + "theoretical_loss": 3.6292888954090734, + "tokens_seen": 1152647168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005258816449348045, + "loss": 1.3088, + "theoretical_loss": 3.629269920684273, + "tokens_seen": 1152712704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005258655967903712, + "loss": 1.3199, + "theoretical_loss": 3.6292509473402657, + "tokens_seen": 1152778240 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005258495486459379, + "loss": 1.4102, + "theoretical_loss": 3.6292319753768725, + "tokens_seen": 1152843776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005258335005015046, + "loss": 1.2946, + "theoretical_loss": 3.6292130047939137, + "tokens_seen": 1152909312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005258174523570712, + "loss": 1.4109, + "theoretical_loss": 3.6291940355912113, + "tokens_seen": 1152974848 + }, + { + "epoch": 0.35, + "learning_rate": 0.000525801404212638, + "loss": 1.3048, + "theoretical_loss": 3.6291750677685854, + "tokens_seen": 1153040384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005257853560682046, + "loss": 1.3441, + "theoretical_loss": 3.6291561013258584, + "tokens_seen": 1153105920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005257693079237714, + "loss": 1.2741, + "theoretical_loss": 3.629137136262851, + "tokens_seen": 1153171456 + }, + { + "epoch": 0.35, + "learning_rate": 0.000525753259779338, + "loss": 1.3727, + "theoretical_loss": 3.6291181725793833, + "tokens_seen": 1153236992 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005257372116349048, + "loss": 1.2986, + "theoretical_loss": 3.629099210275278, + "tokens_seen": 1153302528 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005257211634904714, + "loss": 1.4147, + "theoretical_loss": 3.6290802493503564, + "tokens_seen": 1153368064 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 388139, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4212877750396729, + "objective/train/theoretical_loss": 3.6290612898044388, + "objective/train/tokens_used": 1173893600, + "theoretical_loss": 3.6290612898044388, + "tokens_seen": 1153433600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005257051153460382, + "loss": 1.3308, + "theoretical_loss": 3.6290612898044388, + "tokens_seen": 1153433600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005256890672016048, + "loss": 1.3459, + "theoretical_loss": 3.6290423316373475, + "tokens_seen": 1153499136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005256730190571715, + "loss": 1.3637, + "theoretical_loss": 3.629023374848903, + "tokens_seen": 1153564672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005256569709127382, + "loss": 1.3223, + "theoretical_loss": 3.6290044194389277, + "tokens_seen": 1153630208 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005256409227683049, + "loss": 1.3115, + "theoretical_loss": 3.6289854654072426, + "tokens_seen": 1153695744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005256248746238716, + "loss": 1.3293, + "theoretical_loss": 3.628966512753669, + "tokens_seen": 1153761280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005256088264794383, + "loss": 1.3262, + "theoretical_loss": 3.6289475614780287, + "tokens_seen": 1153826816 + }, + { + "epoch": 0.35, + "learning_rate": 0.000525592778335005, + "loss": 1.1924, + "theoretical_loss": 3.628928611580143, + "tokens_seen": 1153892352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005255767301905717, + "loss": 1.2593, + "theoretical_loss": 3.628909663059834, + "tokens_seen": 1153957888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005255606820461384, + "loss": 1.3586, + "theoretical_loss": 3.6288907159169232, + "tokens_seen": 1154023424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005255446339017051, + "loss": 1.173, + "theoretical_loss": 3.628871770151232, + "tokens_seen": 1154088960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005255285857572718, + "loss": 1.3188, + "theoretical_loss": 3.628852825762582, + "tokens_seen": 1154154496 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005255125376128385, + "loss": 1.3508, + "theoretical_loss": 3.6288338827507953, + "tokens_seen": 1154220032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005254964894684052, + "loss": 1.2634, + "theoretical_loss": 3.628814941115693, + "tokens_seen": 1154285568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005254804413239719, + "loss": 1.3406, + "theoretical_loss": 3.628796000857098, + "tokens_seen": 1154351104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005254643931795386, + "loss": 1.2375, + "theoretical_loss": 3.628777061974831, + "tokens_seen": 1154416640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005254483450351053, + "loss": 1.3291, + "theoretical_loss": 3.628758124468715, + "tokens_seen": 1154482176 + }, + { + "epoch": 0.35, + "learning_rate": 0.000525432296890672, + "loss": 1.3058, + "theoretical_loss": 3.6287391883385705, + "tokens_seen": 1154547712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005254162487462387, + "loss": 1.3351, + "theoretical_loss": 3.6287202535842207, + "tokens_seen": 1154613248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005254002006018054, + "loss": 1.2449, + "theoretical_loss": 3.6287013202054865, + "tokens_seen": 1154678784 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005253841524573721, + "loss": 1.2602, + "theoretical_loss": 3.628682388202191, + "tokens_seen": 1154744320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005253681043129388, + "loss": 1.3141, + "theoretical_loss": 3.628663457574155, + "tokens_seen": 1154809856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005253520561685055, + "loss": 1.2782, + "theoretical_loss": 3.628644528321202, + "tokens_seen": 1154875392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005253360080240722, + "loss": 1.2779, + "theoretical_loss": 3.628625600443153, + "tokens_seen": 1154940928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005253199598796389, + "loss": 1.2693, + "theoretical_loss": 3.6286066739398306, + "tokens_seen": 1155006464 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005253039117352056, + "loss": 1.4455, + "theoretical_loss": 3.6285877488110563, + "tokens_seen": 1155072000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005252878635907723, + "loss": 1.3163, + "theoretical_loss": 3.6285688250566532, + "tokens_seen": 1155137536 + }, + { + "epoch": 0.35, + "learning_rate": 0.000525271815446339, + "loss": 1.1737, + "theoretical_loss": 3.6285499026764434, + "tokens_seen": 1155203072 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005252557673019057, + "loss": 1.362, + "theoretical_loss": 3.6285309816702482, + "tokens_seen": 1155268608 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005252397191574724, + "loss": 1.2838, + "theoretical_loss": 3.628512062037891, + "tokens_seen": 1155334144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005252236710130391, + "loss": 1.4258, + "theoretical_loss": 3.6284931437791936, + "tokens_seen": 1155399680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005252076228686058, + "loss": 1.417, + "theoretical_loss": 3.628474226893978, + "tokens_seen": 1155465216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005251915747241725, + "loss": 1.2895, + "theoretical_loss": 3.628455311382068, + "tokens_seen": 1155530752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005251755265797392, + "loss": 1.3196, + "theoretical_loss": 3.6284363972432843, + "tokens_seen": 1155596288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005251594784353059, + "loss": 1.3434, + "theoretical_loss": 3.628417484477451, + "tokens_seen": 1155661824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005251434302908726, + "loss": 1.3113, + "theoretical_loss": 3.6283985730843886, + "tokens_seen": 1155727360 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005251273821464393, + "loss": 1.2642, + "theoretical_loss": 3.628379663063921, + "tokens_seen": 1155792896 + }, + { + "epoch": 0.35, + "learning_rate": 0.000525111334002006, + "loss": 1.2885, + "theoretical_loss": 3.628360754415871, + "tokens_seen": 1155858432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005250952858575727, + "loss": 1.3405, + "theoretical_loss": 3.62834184714006, + "tokens_seen": 1155923968 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005250792377131394, + "loss": 1.4213, + "theoretical_loss": 3.628322941236312, + "tokens_seen": 1155989504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005250631895687061, + "loss": 1.3425, + "theoretical_loss": 3.6283040367044483, + "tokens_seen": 1156055040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005250471414242728, + "loss": 1.3402, + "theoretical_loss": 3.628285133544293, + "tokens_seen": 1156120576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005250310932798395, + "loss": 1.3353, + "theoretical_loss": 3.6282662317556675, + "tokens_seen": 1156186112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005250150451354063, + "loss": 1.3254, + "theoretical_loss": 3.6282473313383954, + "tokens_seen": 1156251648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005249989969909729, + "loss": 1.3593, + "theoretical_loss": 3.628228432292299, + "tokens_seen": 1156317184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005249829488465397, + "loss": 1.3537, + "theoretical_loss": 3.628209534617201, + "tokens_seen": 1156382720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005249669007021063, + "loss": 1.3434, + "theoretical_loss": 3.628190638312925, + "tokens_seen": 1156448256 + }, + { + "epoch": 0.35, + "learning_rate": 0.000524950852557673, + "loss": 1.2878, + "theoretical_loss": 3.628171743379294, + "tokens_seen": 1156513792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005249348044132397, + "loss": 1.407, + "theoretical_loss": 3.62815284981613, + "tokens_seen": 1156579328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005249187562688065, + "loss": 1.35, + "theoretical_loss": 3.6281339576232563, + "tokens_seen": 1156644864 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 389093, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3961246013641357, + "objective/train/theoretical_loss": 3.628115066800496, + "objective/train/tokens_used": 1177170400, + "theoretical_loss": 3.628115066800496, + "tokens_seen": 1156710400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005249027081243732, + "loss": 1.4008, + "theoretical_loss": 3.628115066800496, + "tokens_seen": 1156710400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005248866599799399, + "loss": 1.3283, + "theoretical_loss": 3.628096177347672, + "tokens_seen": 1156775936 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005248706118355066, + "loss": 1.4352, + "theoretical_loss": 3.6280772892646067, + "tokens_seen": 1156841472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005248545636910733, + "loss": 1.388, + "theoretical_loss": 3.6280584025511247, + "tokens_seen": 1156907008 + }, + { + "epoch": 0.35, + "learning_rate": 0.00052483851554664, + "loss": 1.3747, + "theoretical_loss": 3.628039517207048, + "tokens_seen": 1156972544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005248224674022067, + "loss": 1.3926, + "theoretical_loss": 3.6280206332322, + "tokens_seen": 1157038080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005248064192577734, + "loss": 1.3103, + "theoretical_loss": 3.628001750626404, + "tokens_seen": 1157103616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005247903711133401, + "loss": 1.2729, + "theoretical_loss": 3.627982869389483, + "tokens_seen": 1157169152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005247743229689068, + "loss": 1.2684, + "theoretical_loss": 3.6279639895212608, + "tokens_seen": 1157234688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005247582748244735, + "loss": 1.2934, + "theoretical_loss": 3.6279451110215595, + "tokens_seen": 1157300224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005247422266800402, + "loss": 1.2891, + "theoretical_loss": 3.6279262338902036, + "tokens_seen": 1157365760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005247261785356069, + "loss": 1.4099, + "theoretical_loss": 3.6279073581270165, + "tokens_seen": 1157431296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005247101303911736, + "loss": 1.3125, + "theoretical_loss": 3.6278884837318204, + "tokens_seen": 1157496832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005246940822467403, + "loss": 1.2276, + "theoretical_loss": 3.6278696107044395, + "tokens_seen": 1157562368 + }, + { + "epoch": 0.35, + "learning_rate": 0.000524678034102307, + "loss": 1.3279, + "theoretical_loss": 3.6278507390446975, + "tokens_seen": 1157627904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005246619859578737, + "loss": 1.3733, + "theoretical_loss": 3.627831868752417, + "tokens_seen": 1157693440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005246459378134404, + "loss": 1.2596, + "theoretical_loss": 3.6278129998274227, + "tokens_seen": 1157758976 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005246298896690071, + "loss": 1.2671, + "theoretical_loss": 3.6277941322695373, + "tokens_seen": 1157824512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005246138415245738, + "loss": 1.3346, + "theoretical_loss": 3.627775266078584, + "tokens_seen": 1157890048 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005245977933801405, + "loss": 1.2842, + "theoretical_loss": 3.6277564012543877, + "tokens_seen": 1157955584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005245817452357072, + "loss": 1.27, + "theoretical_loss": 3.6277375377967704, + "tokens_seen": 1158021120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005245656970912739, + "loss": 1.3695, + "theoretical_loss": 3.627718675705558, + "tokens_seen": 1158086656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005245496489468406, + "loss": 1.2826, + "theoretical_loss": 3.6276998149805717, + "tokens_seen": 1158152192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005245336008024073, + "loss": 1.3259, + "theoretical_loss": 3.627680955621637, + "tokens_seen": 1158217728 + }, + { + "epoch": 0.35, + "learning_rate": 0.000524517552657974, + "loss": 1.2803, + "theoretical_loss": 3.627662097628577, + "tokens_seen": 1158283264 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005245015045135407, + "loss": 1.3473, + "theoretical_loss": 3.6276432410012154, + "tokens_seen": 1158348800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005244854563691074, + "loss": 1.3034, + "theoretical_loss": 3.6276243857393764, + "tokens_seen": 1158414336 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005244694082246741, + "loss": 1.4376, + "theoretical_loss": 3.627605531842884, + "tokens_seen": 1158479872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005244533600802408, + "loss": 1.246, + "theoretical_loss": 3.6275866793115616, + "tokens_seen": 1158545408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005244373119358075, + "loss": 1.3351, + "theoretical_loss": 3.6275678281452333, + "tokens_seen": 1158610944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005244212637913741, + "loss": 1.3593, + "theoretical_loss": 3.627548978343723, + "tokens_seen": 1158676480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005244052156469409, + "loss": 1.3827, + "theoretical_loss": 3.627530129906855, + "tokens_seen": 1158742016 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005243891675025075, + "loss": 1.4156, + "theoretical_loss": 3.627511282834453, + "tokens_seen": 1158807552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005243731193580743, + "loss": 1.2261, + "theoretical_loss": 3.6274924371263415, + "tokens_seen": 1158873088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005243570712136409, + "loss": 1.3422, + "theoretical_loss": 3.627473592782344, + "tokens_seen": 1158938624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005243410230692077, + "loss": 1.4268, + "theoretical_loss": 3.627454749802285, + "tokens_seen": 1159004160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005243249749247743, + "loss": 1.3028, + "theoretical_loss": 3.6274359081859884, + "tokens_seen": 1159069696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005243089267803411, + "loss": 1.3061, + "theoretical_loss": 3.627417067933279, + "tokens_seen": 1159135232 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005242928786359077, + "loss": 1.326, + "theoretical_loss": 3.6273982290439806, + "tokens_seen": 1159200768 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005242768304914744, + "loss": 1.4346, + "theoretical_loss": 3.627379391517917, + "tokens_seen": 1159266304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005242607823470411, + "loss": 1.2694, + "theoretical_loss": 3.6273605553549135, + "tokens_seen": 1159331840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005242447342026078, + "loss": 1.361, + "theoretical_loss": 3.6273417205547935, + "tokens_seen": 1159397376 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005242286860581745, + "loss": 1.3586, + "theoretical_loss": 3.627322887117382, + "tokens_seen": 1159462912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005242126379137412, + "loss": 1.2966, + "theoretical_loss": 3.6273040550425035, + "tokens_seen": 1159528448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005241965897693079, + "loss": 1.3173, + "theoretical_loss": 3.627285224329982, + "tokens_seen": 1159593984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005241805416248746, + "loss": 1.3016, + "theoretical_loss": 3.6272663949796415, + "tokens_seen": 1159659520 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005241644934804413, + "loss": 1.2903, + "theoretical_loss": 3.6272475669913073, + "tokens_seen": 1159725056 + }, + { + "epoch": 0.35, + "learning_rate": 0.000524148445336008, + "loss": 1.2825, + "theoretical_loss": 3.627228740364804, + "tokens_seen": 1159790592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005241323971915747, + "loss": 1.33, + "theoretical_loss": 3.6272099150999555, + "tokens_seen": 1159856128 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005241163490471414, + "loss": 1.3692, + "theoretical_loss": 3.627191091196586, + "tokens_seen": 1159921664 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 390304, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.382239580154419, + "objective/train/theoretical_loss": 3.6271722686545216, + "objective/train/tokens_used": 1180447200, + "theoretical_loss": 3.6271722686545216, + "tokens_seen": 1159987200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005241003009027081, + "loss": 1.3223, + "theoretical_loss": 3.6271722686545216, + "tokens_seen": 1159987200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005240842527582748, + "loss": 1.323, + "theoretical_loss": 3.6271534474735865, + "tokens_seen": 1160052736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005240682046138415, + "loss": 1.3826, + "theoretical_loss": 3.6271346276536045, + "tokens_seen": 1160118272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005240521564694082, + "loss": 1.3941, + "theoretical_loss": 3.627115809194401, + "tokens_seen": 1160183808 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005240361083249749, + "loss": 1.3702, + "theoretical_loss": 3.627096992095801, + "tokens_seen": 1160249344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005240200601805416, + "loss": 1.3475, + "theoretical_loss": 3.627078176357628, + "tokens_seen": 1160314880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005240040120361083, + "loss": 1.3882, + "theoretical_loss": 3.6270593619797085, + "tokens_seen": 1160380416 + }, + { + "epoch": 0.35, + "learning_rate": 0.000523987963891675, + "loss": 1.3651, + "theoretical_loss": 3.6270405489618662, + "tokens_seen": 1160445952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005239719157472417, + "loss": 1.3723, + "theoretical_loss": 3.6270217373039264, + "tokens_seen": 1160511488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005239558676028084, + "loss": 1.3694, + "theoretical_loss": 3.627002927005714, + "tokens_seen": 1160577024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005239398194583751, + "loss": 1.3012, + "theoretical_loss": 3.626984118067054, + "tokens_seen": 1160642560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005239237713139418, + "loss": 1.322, + "theoretical_loss": 3.6269653104877713, + "tokens_seen": 1160708096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005239077231695085, + "loss": 1.3307, + "theoretical_loss": 3.626946504267691, + "tokens_seen": 1160773632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005238916750250752, + "loss": 1.3745, + "theoretical_loss": 3.6269276994066377, + "tokens_seen": 1160839168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005238756268806419, + "loss": 1.265, + "theoretical_loss": 3.6269088959044367, + "tokens_seen": 1160904704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005238595787362086, + "loss": 1.3075, + "theoretical_loss": 3.626890093760913, + "tokens_seen": 1160970240 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005238435305917753, + "loss": 1.4269, + "theoretical_loss": 3.626871292975893, + "tokens_seen": 1161035776 + }, + { + "epoch": 0.35, + "learning_rate": 0.000523827482447342, + "loss": 1.3602, + "theoretical_loss": 3.6268524935491997, + "tokens_seen": 1161101312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005238114343029087, + "loss": 1.2614, + "theoretical_loss": 3.62683369548066, + "tokens_seen": 1161166848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005237953861584754, + "loss": 1.4316, + "theoretical_loss": 3.6268148987700983, + "tokens_seen": 1161232384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005237793380140421, + "loss": 1.3308, + "theoretical_loss": 3.62679610341734, + "tokens_seen": 1161297920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005237632898696088, + "loss": 1.3643, + "theoretical_loss": 3.626777309422211, + "tokens_seen": 1161363456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005237472417251755, + "loss": 1.3086, + "theoretical_loss": 3.6267585167845358, + "tokens_seen": 1161428992 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005237311935807422, + "loss": 1.3594, + "theoretical_loss": 3.62673972550414, + "tokens_seen": 1161494528 + }, + { + "epoch": 0.35, + "learning_rate": 0.000523715145436309, + "loss": 1.4302, + "theoretical_loss": 3.6267209355808494, + "tokens_seen": 1161560064 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005236990972918757, + "loss": 1.4588, + "theoretical_loss": 3.6267021470144893, + "tokens_seen": 1161625600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005236830491474423, + "loss": 1.3555, + "theoretical_loss": 3.6266833598048844, + "tokens_seen": 1161691136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005236670010030091, + "loss": 1.4816, + "theoretical_loss": 3.626664573951861, + "tokens_seen": 1161756672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005236509528585757, + "loss": 1.3267, + "theoretical_loss": 3.626645789455245, + "tokens_seen": 1161822208 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005236349047141425, + "loss": 1.442, + "theoretical_loss": 3.6266270063148607, + "tokens_seen": 1161887744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005236188565697092, + "loss": 1.2699, + "theoretical_loss": 3.6266082245305347, + "tokens_seen": 1161953280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005236028084252759, + "loss": 1.4851, + "theoretical_loss": 3.626589444102092, + "tokens_seen": 1162018816 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005235867602808426, + "loss": 1.4912, + "theoretical_loss": 3.6265706650293588, + "tokens_seen": 1162084352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005235707121364093, + "loss": 1.506, + "theoretical_loss": 3.6265518873121607, + "tokens_seen": 1162149888 + }, + { + "epoch": 0.35, + "learning_rate": 0.000523554663991976, + "loss": 1.2892, + "theoretical_loss": 3.626533110950323, + "tokens_seen": 1162215424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005235386158475427, + "loss": 1.3803, + "theoretical_loss": 3.6265143359436722, + "tokens_seen": 1162280960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005235225677031094, + "loss": 1.3633, + "theoretical_loss": 3.6264955622920327, + "tokens_seen": 1162346496 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005235065195586761, + "loss": 1.2449, + "theoretical_loss": 3.626476789995232, + "tokens_seen": 1162412032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005234904714142428, + "loss": 1.373, + "theoretical_loss": 3.6264580190530946, + "tokens_seen": 1162477568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005234744232698095, + "loss": 1.3181, + "theoretical_loss": 3.626439249465448, + "tokens_seen": 1162543104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005234583751253762, + "loss": 1.2979, + "theoretical_loss": 3.626420481232116, + "tokens_seen": 1162608640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005234423269809429, + "loss": 1.2596, + "theoretical_loss": 3.6264017143529257, + "tokens_seen": 1162674176 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005234262788365096, + "loss": 1.2634, + "theoretical_loss": 3.626382948827703, + "tokens_seen": 1162739712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005234102306920763, + "loss": 1.3723, + "theoretical_loss": 3.626364184656274, + "tokens_seen": 1162805248 + }, + { + "epoch": 0.35, + "learning_rate": 0.000523394182547643, + "loss": 1.3086, + "theoretical_loss": 3.6263454218384648, + "tokens_seen": 1162870784 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005233781344032097, + "loss": 1.3344, + "theoretical_loss": 3.626326660374101, + "tokens_seen": 1162936320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005233620862587764, + "loss": 1.4669, + "theoretical_loss": 3.6263079002630096, + "tokens_seen": 1163001856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005233460381143431, + "loss": 1.4078, + "theoretical_loss": 3.6262891415050156, + "tokens_seen": 1163067392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005233299899699098, + "loss": 1.2619, + "theoretical_loss": 3.6262703840999455, + "tokens_seen": 1163132928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005233139418254765, + "loss": 1.2902, + "theoretical_loss": 3.6262516280476262, + "tokens_seen": 1163198464 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 391310, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3289260864257812, + "objective/train/theoretical_loss": 3.6262328733478837, + "objective/train/tokens_used": 1183724000, + "theoretical_loss": 3.6262328733478837, + "tokens_seen": 1163264000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005232978936810432, + "loss": 1.4095, + "theoretical_loss": 3.6262328733478837, + "tokens_seen": 1163264000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005232818455366099, + "loss": 1.3516, + "theoretical_loss": 3.6262141200005433, + "tokens_seen": 1163329536 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005232657973921766, + "loss": 1.3389, + "theoretical_loss": 3.6261953680054324, + "tokens_seen": 1163395072 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005232497492477433, + "loss": 1.3979, + "theoretical_loss": 3.6261766173623773, + "tokens_seen": 1163460608 + }, + { + "epoch": 0.35, + "learning_rate": 0.00052323370110331, + "loss": 1.4256, + "theoretical_loss": 3.6261578680712034, + "tokens_seen": 1163526144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005232176529588767, + "loss": 1.3421, + "theoretical_loss": 3.6261391201317377, + "tokens_seen": 1163591680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005232016048144434, + "loss": 1.3171, + "theoretical_loss": 3.6261203735438072, + "tokens_seen": 1163657216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005231855566700101, + "loss": 1.3133, + "theoretical_loss": 3.6261016283072376, + "tokens_seen": 1163722752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005231695085255768, + "loss": 1.3757, + "theoretical_loss": 3.626082884421855, + "tokens_seen": 1163788288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005231534603811435, + "loss": 1.3293, + "theoretical_loss": 3.6260641418874875, + "tokens_seen": 1163853824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005231374122367102, + "loss": 1.3661, + "theoretical_loss": 3.62604540070396, + "tokens_seen": 1163919360 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005231213640922769, + "loss": 1.3974, + "theoretical_loss": 3.6260266608711005, + "tokens_seen": 1163984896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005231053159478436, + "loss": 1.3269, + "theoretical_loss": 3.626007922388734, + "tokens_seen": 1164050432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005230892678034103, + "loss": 1.3172, + "theoretical_loss": 3.6259891852566883, + "tokens_seen": 1164115968 + }, + { + "epoch": 0.35, + "learning_rate": 0.000523073219658977, + "loss": 1.2811, + "theoretical_loss": 3.62597044947479, + "tokens_seen": 1164181504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005230571715145437, + "loss": 1.2927, + "theoretical_loss": 3.6259517150428655, + "tokens_seen": 1164247040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005230411233701104, + "loss": 1.3156, + "theoretical_loss": 3.6259329819607418, + "tokens_seen": 1164312576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005230250752256771, + "loss": 1.3063, + "theoretical_loss": 3.625914250228245, + "tokens_seen": 1164378112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005230090270812438, + "loss": 1.4155, + "theoretical_loss": 3.625895519845203, + "tokens_seen": 1164443648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005229929789368104, + "loss": 1.3127, + "theoretical_loss": 3.625876790811442, + "tokens_seen": 1164509184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005229769307923772, + "loss": 1.4032, + "theoretical_loss": 3.625858063126789, + "tokens_seen": 1164574720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005229608826479438, + "loss": 1.2937, + "theoretical_loss": 3.625839336791071, + "tokens_seen": 1164640256 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005229448345035106, + "loss": 1.3879, + "theoretical_loss": 3.625820611804115, + "tokens_seen": 1164705792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005229287863590772, + "loss": 1.334, + "theoretical_loss": 3.6258018881657472, + "tokens_seen": 1164771328 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522912738214644, + "loss": 1.3847, + "theoretical_loss": 3.6257831658757955, + "tokens_seen": 1164836864 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005228966900702106, + "loss": 1.3339, + "theoretical_loss": 3.6257644449340867, + "tokens_seen": 1164902400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005228806419257773, + "loss": 1.2804, + "theoretical_loss": 3.6257457253404475, + "tokens_seen": 1164967936 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522864593781344, + "loss": 1.212, + "theoretical_loss": 3.6257270070947056, + "tokens_seen": 1165033472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005228485456369107, + "loss": 1.3679, + "theoretical_loss": 3.6257082901966884, + "tokens_seen": 1165099008 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005228324974924774, + "loss": 1.4327, + "theoretical_loss": 3.6256895746462217, + "tokens_seen": 1165164544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005228164493480441, + "loss": 1.3036, + "theoretical_loss": 3.625670860443134, + "tokens_seen": 1165230080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005228004012036108, + "loss": 1.3006, + "theoretical_loss": 3.6256521475872514, + "tokens_seen": 1165295616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005227843530591775, + "loss": 1.2842, + "theoretical_loss": 3.6256334360784024, + "tokens_seen": 1165361152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005227683049147442, + "loss": 1.3809, + "theoretical_loss": 3.625614725916413, + "tokens_seen": 1165426688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005227522567703109, + "loss": 1.2579, + "theoretical_loss": 3.6255960171011115, + "tokens_seen": 1165492224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005227362086258776, + "loss": 1.3692, + "theoretical_loss": 3.6255773096323254, + "tokens_seen": 1165557760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005227201604814443, + "loss": 1.2346, + "theoretical_loss": 3.625558603509881, + "tokens_seen": 1165623296 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522704112337011, + "loss": 1.2338, + "theoretical_loss": 3.6255398987336065, + "tokens_seen": 1165688832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005226880641925777, + "loss": 1.3505, + "theoretical_loss": 3.6255211953033295, + "tokens_seen": 1165754368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005226720160481444, + "loss": 1.3322, + "theoretical_loss": 3.6255024932188773, + "tokens_seen": 1165819904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005226559679037111, + "loss": 1.397, + "theoretical_loss": 3.625483792480077, + "tokens_seen": 1165885440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005226399197592778, + "loss": 1.3544, + "theoretical_loss": 3.6254650930867562, + "tokens_seen": 1165950976 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005226238716148445, + "loss": 1.3373, + "theoretical_loss": 3.625446395038743, + "tokens_seen": 1166016512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005226078234704112, + "loss": 1.4169, + "theoretical_loss": 3.625427698335865, + "tokens_seen": 1166082048 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005225917753259779, + "loss": 1.4729, + "theoretical_loss": 3.625409002977949, + "tokens_seen": 1166147584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005225757271815446, + "loss": 1.3368, + "theoretical_loss": 3.6253903089648234, + "tokens_seen": 1166213120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005225596790371113, + "loss": 1.2234, + "theoretical_loss": 3.6253716162963157, + "tokens_seen": 1166278656 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522543630892678, + "loss": 1.3524, + "theoretical_loss": 3.625352924972254, + "tokens_seen": 1166344192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005225275827482447, + "loss": 1.3436, + "theoretical_loss": 3.625334234992465, + "tokens_seen": 1166409728 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005225115346038114, + "loss": 1.3063, + "theoretical_loss": 3.625315546356778, + "tokens_seen": 1166475264 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 392693, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.403052806854248, + "objective/train/theoretical_loss": 3.62529685906502, + "objective/train/tokens_used": 1187000800, + "theoretical_loss": 3.62529685906502, + "tokens_seen": 1166540800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005224954864593782, + "loss": 1.382, + "theoretical_loss": 3.62529685906502, + "tokens_seen": 1166540800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005224794383149448, + "loss": 1.3795, + "theoretical_loss": 3.625278173117019, + "tokens_seen": 1166606336 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005224633901705116, + "loss": 1.3338, + "theoretical_loss": 3.6252594885126026, + "tokens_seen": 1166671872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005224473420260782, + "loss": 1.2931, + "theoretical_loss": 3.625240805251599, + "tokens_seen": 1166737408 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522431293881645, + "loss": 1.3088, + "theoretical_loss": 3.625222123333836, + "tokens_seen": 1166802944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005224152457372116, + "loss": 1.409, + "theoretical_loss": 3.6252034427591417, + "tokens_seen": 1166868480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005223991975927784, + "loss": 1.4136, + "theoretical_loss": 3.6251847635273444, + "tokens_seen": 1166934016 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522383149448345, + "loss": 1.34, + "theoretical_loss": 3.6251660856382717, + "tokens_seen": 1166999552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005223671013039118, + "loss": 1.4028, + "theoretical_loss": 3.6251474090917517, + "tokens_seen": 1167065088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005223510531594784, + "loss": 1.2295, + "theoretical_loss": 3.625128733887613, + "tokens_seen": 1167130624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005223350050150452, + "loss": 1.3767, + "theoretical_loss": 3.6251100600256834, + "tokens_seen": 1167196160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005223189568706118, + "loss": 1.2543, + "theoretical_loss": 3.6250913875057904, + "tokens_seen": 1167261696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005223029087261786, + "loss": 1.307, + "theoretical_loss": 3.6250727163277636, + "tokens_seen": 1167327232 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005222868605817452, + "loss": 1.3599, + "theoretical_loss": 3.6250540464914307, + "tokens_seen": 1167392768 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522270812437312, + "loss": 1.3695, + "theoretical_loss": 3.6250353779966193, + "tokens_seen": 1167458304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005222547642928787, + "loss": 1.239, + "theoretical_loss": 3.625016710843159, + "tokens_seen": 1167523840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005222387161484454, + "loss": 1.3781, + "theoretical_loss": 3.6249980450308765, + "tokens_seen": 1167589376 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522222668004012, + "loss": 1.2695, + "theoretical_loss": 3.6249793805596013, + "tokens_seen": 1167654912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005222066198595788, + "loss": 1.4009, + "theoretical_loss": 3.624960717429162, + "tokens_seen": 1167720448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005221905717151455, + "loss": 1.3018, + "theoretical_loss": 3.6249420556393863, + "tokens_seen": 1167785984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005221745235707122, + "loss": 1.3338, + "theoretical_loss": 3.6249233951901028, + "tokens_seen": 1167851520 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005221584754262789, + "loss": 1.3396, + "theoretical_loss": 3.62490473608114, + "tokens_seen": 1167917056 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005221424272818456, + "loss": 1.4346, + "theoretical_loss": 3.624886078312327, + "tokens_seen": 1167982592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005221263791374123, + "loss": 1.3923, + "theoretical_loss": 3.6248674218834918, + "tokens_seen": 1168048128 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522110330992979, + "loss": 1.3886, + "theoretical_loss": 3.6248487667944627, + "tokens_seen": 1168113664 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005220942828485457, + "loss": 1.3706, + "theoretical_loss": 3.624830113045069, + "tokens_seen": 1168179200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005220782347041124, + "loss": 1.3453, + "theoretical_loss": 3.624811460635139, + "tokens_seen": 1168244736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005220621865596791, + "loss": 1.2965, + "theoretical_loss": 3.624792809564502, + "tokens_seen": 1168310272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005220461384152458, + "loss": 1.2545, + "theoretical_loss": 3.6247741598329855, + "tokens_seen": 1168375808 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005220300902708125, + "loss": 1.3799, + "theoretical_loss": 3.6247555114404193, + "tokens_seen": 1168441344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005220140421263792, + "loss": 1.2763, + "theoretical_loss": 3.624736864386632, + "tokens_seen": 1168506880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005219979939819459, + "loss": 1.3788, + "theoretical_loss": 3.6247182186714517, + "tokens_seen": 1168572416 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005219819458375126, + "loss": 1.4166, + "theoretical_loss": 3.6246995742947083, + "tokens_seen": 1168637952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005219658976930793, + "loss": 1.3735, + "theoretical_loss": 3.6246809312562296, + "tokens_seen": 1168703488 + }, + { + "epoch": 0.35, + "learning_rate": 0.000521949849548646, + "loss": 1.3781, + "theoretical_loss": 3.6246622895558454, + "tokens_seen": 1168769024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005219338014042127, + "loss": 1.4817, + "theoretical_loss": 3.624643649193384, + "tokens_seen": 1168834560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005219177532597794, + "loss": 1.321, + "theoretical_loss": 3.6246250101686748, + "tokens_seen": 1168900096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005219017051153461, + "loss": 1.2949, + "theoretical_loss": 3.624606372481547, + "tokens_seen": 1168965632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005218856569709128, + "loss": 1.3907, + "theoretical_loss": 3.624587736131829, + "tokens_seen": 1169031168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005218696088264795, + "loss": 1.3235, + "theoretical_loss": 3.62456910111935, + "tokens_seen": 1169096704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005218535606820462, + "loss": 1.3947, + "theoretical_loss": 3.6245504674439393, + "tokens_seen": 1169162240 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005218375125376129, + "loss": 1.2989, + "theoretical_loss": 3.624531835105426, + "tokens_seen": 1169227776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005218214643931796, + "loss": 1.4171, + "theoretical_loss": 3.624513204103639, + "tokens_seen": 1169293312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005218054162487463, + "loss": 1.3067, + "theoretical_loss": 3.6244945744384083, + "tokens_seen": 1169358848 + }, + { + "epoch": 0.35, + "learning_rate": 0.000521789368104313, + "loss": 1.3451, + "theoretical_loss": 3.6244759461095617, + "tokens_seen": 1169424384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005217733199598797, + "loss": 1.3654, + "theoretical_loss": 3.62445731911693, + "tokens_seen": 1169489920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005217572718154464, + "loss": 1.3649, + "theoretical_loss": 3.6244386934603416, + "tokens_seen": 1169555456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005217412236710131, + "loss": 1.3148, + "theoretical_loss": 3.6244200691396258, + "tokens_seen": 1169620992 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005217251755265798, + "loss": 1.2983, + "theoretical_loss": 3.624401446154612, + "tokens_seen": 1169686528 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005217091273821465, + "loss": 1.255, + "theoretical_loss": 3.6243828245051297, + "tokens_seen": 1169752064 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 393563, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2607676982879639, + "objective/train/theoretical_loss": 3.624364204191009, + "objective/train/tokens_used": 1190277600, + "theoretical_loss": 3.624364204191009, + "tokens_seen": 1169817600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005216930792377132, + "loss": 1.2299, + "theoretical_loss": 3.624364204191009, + "tokens_seen": 1169817600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005216770310932799, + "loss": 1.3175, + "theoretical_loss": 3.624345585212078, + "tokens_seen": 1169883136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005216609829488466, + "loss": 1.4497, + "theoretical_loss": 3.624326967568167, + "tokens_seen": 1169948672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005216449348044133, + "loss": 1.3844, + "theoretical_loss": 3.624308351259105, + "tokens_seen": 1170014208 + }, + { + "epoch": 0.35, + "learning_rate": 0.00052162888665998, + "loss": 1.3511, + "theoretical_loss": 3.624289736284722, + "tokens_seen": 1170079744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005216128385155467, + "loss": 1.2804, + "theoretical_loss": 3.6242711226448474, + "tokens_seen": 1170145280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005215967903711133, + "loss": 1.2419, + "theoretical_loss": 3.624252510339311, + "tokens_seen": 1170210816 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005215807422266801, + "loss": 1.2794, + "theoretical_loss": 3.624233899367942, + "tokens_seen": 1170276352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005215646940822467, + "loss": 1.2997, + "theoretical_loss": 3.6242152897305706, + "tokens_seen": 1170341888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005215486459378135, + "loss": 1.278, + "theoretical_loss": 3.624196681427026, + "tokens_seen": 1170407424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005215325977933801, + "loss": 1.3937, + "theoretical_loss": 3.624178074457138, + "tokens_seen": 1170472960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005215165496489469, + "loss": 1.3346, + "theoretical_loss": 3.6241594688207366, + "tokens_seen": 1170538496 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005215005015045135, + "loss": 1.3682, + "theoretical_loss": 3.6241408645176514, + "tokens_seen": 1170604032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005214844533600803, + "loss": 1.2385, + "theoretical_loss": 3.624122261547712, + "tokens_seen": 1170669568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005214684052156469, + "loss": 1.3522, + "theoretical_loss": 3.624103659910749, + "tokens_seen": 1170735104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005214523570712136, + "loss": 1.2782, + "theoretical_loss": 3.6240850596065917, + "tokens_seen": 1170800640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005214363089267803, + "loss": 1.3125, + "theoretical_loss": 3.6240664606350697, + "tokens_seen": 1170866176 + }, + { + "epoch": 0.35, + "learning_rate": 0.000521420260782347, + "loss": 1.3384, + "theoretical_loss": 3.6240478629960133, + "tokens_seen": 1170931712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005214042126379137, + "loss": 1.3271, + "theoretical_loss": 3.624029266689253, + "tokens_seen": 1170997248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005213881644934804, + "loss": 1.4168, + "theoretical_loss": 3.6240106717146174, + "tokens_seen": 1171062784 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005213721163490471, + "loss": 1.3534, + "theoretical_loss": 3.6239920780719386, + "tokens_seen": 1171128320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005213560682046138, + "loss": 1.4672, + "theoretical_loss": 3.6239734857610446, + "tokens_seen": 1171193856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005213400200601805, + "loss": 1.3727, + "theoretical_loss": 3.623954894781767, + "tokens_seen": 1171259392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005213239719157472, + "loss": 1.3473, + "theoretical_loss": 3.6239363051339346, + "tokens_seen": 1171324928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005213079237713139, + "loss": 1.3116, + "theoretical_loss": 3.623917716817379, + "tokens_seen": 1171390464 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005212918756268807, + "loss": 1.2633, + "theoretical_loss": 3.623899129831929, + "tokens_seen": 1171456000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005212758274824473, + "loss": 1.3765, + "theoretical_loss": 3.623880544177416, + "tokens_seen": 1171521536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005212597793380141, + "loss": 1.3197, + "theoretical_loss": 3.6238619598536697, + "tokens_seen": 1171587072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005212437311935807, + "loss": 1.4117, + "theoretical_loss": 3.6238433768605205, + "tokens_seen": 1171652608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005212276830491475, + "loss": 1.3921, + "theoretical_loss": 3.623824795197798, + "tokens_seen": 1171718144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005212116349047141, + "loss": 1.2963, + "theoretical_loss": 3.6238062148653336, + "tokens_seen": 1171783680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005211955867602809, + "loss": 1.3229, + "theoretical_loss": 3.623787635862957, + "tokens_seen": 1171849216 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005211795386158475, + "loss": 1.3587, + "theoretical_loss": 3.6237690581904993, + "tokens_seen": 1171914752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005211634904714143, + "loss": 1.3557, + "theoretical_loss": 3.6237504818477904, + "tokens_seen": 1171980288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005211474423269809, + "loss": 1.4415, + "theoretical_loss": 3.6237319068346605, + "tokens_seen": 1172045824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005211313941825477, + "loss": 1.31, + "theoretical_loss": 3.6237133331509406, + "tokens_seen": 1172111360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005211153460381143, + "loss": 1.3444, + "theoretical_loss": 3.623694760796461, + "tokens_seen": 1172176896 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005210992978936811, + "loss": 1.3207, + "theoretical_loss": 3.6236761897710528, + "tokens_seen": 1172242432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005210832497492477, + "loss": 1.2704, + "theoretical_loss": 3.623657620074546, + "tokens_seen": 1172307968 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005210672016048145, + "loss": 1.3772, + "theoretical_loss": 3.623639051706771, + "tokens_seen": 1172373504 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005210511534603811, + "loss": 1.3308, + "theoretical_loss": 3.6236204846675593, + "tokens_seen": 1172439040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005210351053159479, + "loss": 1.2544, + "theoretical_loss": 3.6236019189567408, + "tokens_seen": 1172504576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005210190571715145, + "loss": 1.3313, + "theoretical_loss": 3.623583354574147, + "tokens_seen": 1172570112 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005210030090270813, + "loss": 1.3694, + "theoretical_loss": 3.623564791519607, + "tokens_seen": 1172635648 + }, + { + "epoch": 0.36, + "learning_rate": 0.000520986960882648, + "loss": 1.5035, + "theoretical_loss": 3.623546229792954, + "tokens_seen": 1172701184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005209709127382147, + "loss": 1.3508, + "theoretical_loss": 3.6235276693940173, + "tokens_seen": 1172766720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005209548645937813, + "loss": 1.2774, + "theoretical_loss": 3.623509110322628, + "tokens_seen": 1172832256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005209388164493481, + "loss": 1.4738, + "theoretical_loss": 3.623490552578617, + "tokens_seen": 1172897792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005209227683049147, + "loss": 1.2105, + "theoretical_loss": 3.6234719961618147, + "tokens_seen": 1172963328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005209067201604815, + "loss": 1.2656, + "theoretical_loss": 3.623453441072053, + "tokens_seen": 1173028864 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 394748, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4302725791931152, + "objective/train/theoretical_loss": 3.623434887309162, + "objective/train/tokens_used": 1193554400, + "theoretical_loss": 3.623434887309162, + "tokens_seen": 1173094400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005208906720160482, + "loss": 1.3879, + "theoretical_loss": 3.623434887309162, + "tokens_seen": 1173094400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005208746238716149, + "loss": 1.3935, + "theoretical_loss": 3.6234163348729735, + "tokens_seen": 1173159936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005208585757271816, + "loss": 1.3199, + "theoretical_loss": 3.623397783763318, + "tokens_seen": 1173225472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005208425275827483, + "loss": 1.2786, + "theoretical_loss": 3.623379233980027, + "tokens_seen": 1173291008 + }, + { + "epoch": 0.36, + "learning_rate": 0.000520826479438315, + "loss": 1.3943, + "theoretical_loss": 3.623360685522931, + "tokens_seen": 1173356544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005208104312938817, + "loss": 1.4016, + "theoretical_loss": 3.6233421383918607, + "tokens_seen": 1173422080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005207943831494484, + "loss": 1.4327, + "theoretical_loss": 3.623323592586649, + "tokens_seen": 1173487616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005207783350050151, + "loss": 1.3676, + "theoretical_loss": 3.6233050481071256, + "tokens_seen": 1173553152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005207622868605818, + "loss": 1.297, + "theoretical_loss": 3.623286504953122, + "tokens_seen": 1173618688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005207462387161485, + "loss": 1.2294, + "theoretical_loss": 3.6232679631244697, + "tokens_seen": 1173684224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005207301905717152, + "loss": 1.2713, + "theoretical_loss": 3.623249422621, + "tokens_seen": 1173749760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005207141424272819, + "loss": 1.3539, + "theoretical_loss": 3.6232308834425435, + "tokens_seen": 1173815296 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005206980942828486, + "loss": 1.28, + "theoretical_loss": 3.623212345588933, + "tokens_seen": 1173880832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005206820461384153, + "loss": 1.4316, + "theoretical_loss": 3.623193809059998, + "tokens_seen": 1173946368 + }, + { + "epoch": 0.36, + "learning_rate": 0.000520665997993982, + "loss": 1.3246, + "theoretical_loss": 3.623175273855572, + "tokens_seen": 1174011904 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005206499498495487, + "loss": 1.3382, + "theoretical_loss": 3.6231567399754843, + "tokens_seen": 1174077440 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005206339017051154, + "loss": 1.4022, + "theoretical_loss": 3.6231382074195677, + "tokens_seen": 1174142976 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005206178535606821, + "loss": 1.3721, + "theoretical_loss": 3.6231196761876534, + "tokens_seen": 1174208512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005206018054162488, + "loss": 1.3147, + "theoretical_loss": 3.6231011462795726, + "tokens_seen": 1174274048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005205857572718155, + "loss": 1.3632, + "theoretical_loss": 3.6230826176951574, + "tokens_seen": 1174339584 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005205697091273822, + "loss": 1.3908, + "theoretical_loss": 3.623064090434239, + "tokens_seen": 1174405120 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005205536609829489, + "loss": 1.3546, + "theoretical_loss": 3.623045564496649, + "tokens_seen": 1174470656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005205376128385156, + "loss": 1.4011, + "theoretical_loss": 3.6230270398822197, + "tokens_seen": 1174536192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005205215646940823, + "loss": 1.3003, + "theoretical_loss": 3.6230085165907813, + "tokens_seen": 1174601728 + }, + { + "epoch": 0.36, + "learning_rate": 0.000520505516549649, + "loss": 1.3056, + "theoretical_loss": 3.6229899946221673, + "tokens_seen": 1174667264 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005204894684052157, + "loss": 1.354, + "theoretical_loss": 3.6229714739762082, + "tokens_seen": 1174732800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005204734202607824, + "loss": 1.3905, + "theoretical_loss": 3.622952954652736, + "tokens_seen": 1174798336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005204573721163491, + "loss": 1.3324, + "theoretical_loss": 3.622934436651583, + "tokens_seen": 1174863872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005204413239719158, + "loss": 1.2093, + "theoretical_loss": 3.6229159199725807, + "tokens_seen": 1174929408 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005204252758274825, + "loss": 1.2937, + "theoretical_loss": 3.6228974046155606, + "tokens_seen": 1174994944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005204092276830492, + "loss": 1.3784, + "theoretical_loss": 3.6228788905803553, + "tokens_seen": 1175060480 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005203931795386159, + "loss": 1.2328, + "theoretical_loss": 3.6228603778667967, + "tokens_seen": 1175126016 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005203771313941826, + "loss": 1.2746, + "theoretical_loss": 3.6228418664747157, + "tokens_seen": 1175191552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005203610832497493, + "loss": 1.3738, + "theoretical_loss": 3.622823356403945, + "tokens_seen": 1175257088 + }, + { + "epoch": 0.36, + "learning_rate": 0.000520345035105316, + "loss": 1.3804, + "theoretical_loss": 3.622804847654317, + "tokens_seen": 1175322624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005203289869608827, + "loss": 1.5001, + "theoretical_loss": 3.6227863402256633, + "tokens_seen": 1175388160 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005203129388164494, + "loss": 1.4309, + "theoretical_loss": 3.6227678341178158, + "tokens_seen": 1175453696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005202968906720161, + "loss": 1.4174, + "theoretical_loss": 3.622749329330607, + "tokens_seen": 1175519232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005202808425275828, + "loss": 1.3994, + "theoretical_loss": 3.6227308258638686, + "tokens_seen": 1175584768 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005202647943831495, + "loss": 1.3172, + "theoretical_loss": 3.6227123237174332, + "tokens_seen": 1175650304 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005202487462387162, + "loss": 1.3803, + "theoretical_loss": 3.622693822891133, + "tokens_seen": 1175715840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005202326980942829, + "loss": 1.2936, + "theoretical_loss": 3.6226753233848, + "tokens_seen": 1175781376 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005202166499498496, + "loss": 1.3158, + "theoretical_loss": 3.6226568251982667, + "tokens_seen": 1175846912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005202006018054162, + "loss": 1.3924, + "theoretical_loss": 3.622638328331365, + "tokens_seen": 1175912448 + }, + { + "epoch": 0.36, + "learning_rate": 0.000520184553660983, + "loss": 1.3513, + "theoretical_loss": 3.6226198327839274, + "tokens_seen": 1175977984 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005201685055165496, + "loss": 1.3921, + "theoretical_loss": 3.6226013385557865, + "tokens_seen": 1176043520 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005201524573721164, + "loss": 1.2881, + "theoretical_loss": 3.6225828456467744, + "tokens_seen": 1176109056 + }, + { + "epoch": 0.36, + "learning_rate": 0.000520136409227683, + "loss": 1.2704, + "theoretical_loss": 3.6225643540567236, + "tokens_seen": 1176174592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005201203610832498, + "loss": 1.3317, + "theoretical_loss": 3.622545863785467, + "tokens_seen": 1176240128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005201043129388164, + "loss": 1.3436, + "theoretical_loss": 3.6225273748328357, + "tokens_seen": 1176305664 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 395873, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5824906826019287, + "objective/train/theoretical_loss": 3.6225088871986637, + "objective/train/tokens_used": 1196831200, + "theoretical_loss": 3.6225088871986637, + "tokens_seen": 1176371200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005200882647943832, + "loss": 1.442, + "theoretical_loss": 3.6225088871986637, + "tokens_seen": 1176371200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005200722166499498, + "loss": 1.3582, + "theoretical_loss": 3.622490400882783, + "tokens_seen": 1176436736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005200561685055166, + "loss": 1.2754, + "theoretical_loss": 3.6224719158850256, + "tokens_seen": 1176502272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005200401203610832, + "loss": 1.3573, + "theoretical_loss": 3.622453432205225, + "tokens_seen": 1176567808 + }, + { + "epoch": 0.36, + "learning_rate": 0.00052002407221665, + "loss": 1.3548, + "theoretical_loss": 3.622434949843213, + "tokens_seen": 1176633344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005200080240722166, + "loss": 1.3114, + "theoretical_loss": 3.622416468798823, + "tokens_seen": 1176698880 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005199919759277834, + "loss": 1.3777, + "theoretical_loss": 3.622397989071888, + "tokens_seen": 1176764416 + }, + { + "epoch": 0.36, + "learning_rate": 0.00051997592778335, + "loss": 1.2718, + "theoretical_loss": 3.6223795106622396, + "tokens_seen": 1176829952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005199598796389168, + "loss": 1.2786, + "theoretical_loss": 3.622361033569711, + "tokens_seen": 1176895488 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005199438314944834, + "loss": 1.3207, + "theoretical_loss": 3.622342557794135, + "tokens_seen": 1176961024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005199277833500502, + "loss": 1.3457, + "theoretical_loss": 3.6223240833353447, + "tokens_seen": 1177026560 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005199117352056168, + "loss": 1.3378, + "theoretical_loss": 3.6223056101931728, + "tokens_seen": 1177092096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005198956870611836, + "loss": 1.346, + "theoretical_loss": 3.6222871383674518, + "tokens_seen": 1177157632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005198796389167502, + "loss": 1.3451, + "theoretical_loss": 3.6222686678580147, + "tokens_seen": 1177223168 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519863590772317, + "loss": 1.4074, + "theoretical_loss": 3.6222501986646947, + "tokens_seen": 1177288704 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005198475426278836, + "loss": 1.3488, + "theoretical_loss": 3.622231730787325, + "tokens_seen": 1177354240 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005198314944834504, + "loss": 1.3536, + "theoretical_loss": 3.622213264225738, + "tokens_seen": 1177419776 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519815446339017, + "loss": 1.3691, + "theoretical_loss": 3.622194798979767, + "tokens_seen": 1177485312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005197993981945838, + "loss": 1.3549, + "theoretical_loss": 3.6221763350492457, + "tokens_seen": 1177550848 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005197833500501504, + "loss": 1.3276, + "theoretical_loss": 3.6221578724340056, + "tokens_seen": 1177616384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005197673019057172, + "loss": 1.2978, + "theoretical_loss": 3.622139411133881, + "tokens_seen": 1177681920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005197512537612838, + "loss": 1.2934, + "theoretical_loss": 3.622120951148705, + "tokens_seen": 1177747456 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005197352056168506, + "loss": 1.3778, + "theoretical_loss": 3.6221024924783105, + "tokens_seen": 1177812992 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005197191574724172, + "loss": 1.3326, + "theoretical_loss": 3.6220840351225307, + "tokens_seen": 1177878528 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519703109327984, + "loss": 1.3671, + "theoretical_loss": 3.6220655790811986, + "tokens_seen": 1177944064 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005196870611835506, + "loss": 1.2988, + "theoretical_loss": 3.622047124354148, + "tokens_seen": 1178009600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005196710130391174, + "loss": 1.3643, + "theoretical_loss": 3.622028670941212, + "tokens_seen": 1178075136 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519654964894684, + "loss": 1.2444, + "theoretical_loss": 3.6220102188422243, + "tokens_seen": 1178140672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005196389167502508, + "loss": 1.4353, + "theoretical_loss": 3.6219917680570175, + "tokens_seen": 1178206208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005196228686058174, + "loss": 1.3703, + "theoretical_loss": 3.6219733185854253, + "tokens_seen": 1178271744 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005196068204613842, + "loss": 1.3585, + "theoretical_loss": 3.621954870427282, + "tokens_seen": 1178337280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005195907723169508, + "loss": 1.3883, + "theoretical_loss": 3.6219364235824187, + "tokens_seen": 1178402816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005195747241725176, + "loss": 1.4193, + "theoretical_loss": 3.6219179780506714, + "tokens_seen": 1178468352 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005195586760280842, + "loss": 1.3301, + "theoretical_loss": 3.621899533831873, + "tokens_seen": 1178533888 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519542627883651, + "loss": 1.3868, + "theoretical_loss": 3.621881090925856, + "tokens_seen": 1178599424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005195265797392177, + "loss": 1.2952, + "theoretical_loss": 3.6218626493324546, + "tokens_seen": 1178664960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005195105315947844, + "loss": 1.3821, + "theoretical_loss": 3.6218442090515026, + "tokens_seen": 1178730496 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519494483450351, + "loss": 1.4559, + "theoretical_loss": 3.6218257700828334, + "tokens_seen": 1178796032 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005194784353059178, + "loss": 1.3733, + "theoretical_loss": 3.6218073324262807, + "tokens_seen": 1178861568 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005194623871614845, + "loss": 1.2628, + "theoretical_loss": 3.6217888960816786, + "tokens_seen": 1178927104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005194463390170512, + "loss": 1.3803, + "theoretical_loss": 3.62177046104886, + "tokens_seen": 1178992640 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005194302908726179, + "loss": 1.3956, + "theoretical_loss": 3.6217520273276596, + "tokens_seen": 1179058176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005194142427281846, + "loss": 1.3624, + "theoretical_loss": 3.6217335949179104, + "tokens_seen": 1179123712 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005193981945837513, + "loss": 1.4153, + "theoretical_loss": 3.6217151638194465, + "tokens_seen": 1179189248 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519382146439318, + "loss": 1.317, + "theoretical_loss": 3.621696734032102, + "tokens_seen": 1179254784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005193660982948847, + "loss": 1.3697, + "theoretical_loss": 3.62167830555571, + "tokens_seen": 1179320320 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005193500501504514, + "loss": 1.3489, + "theoretical_loss": 3.6216598783901053, + "tokens_seen": 1179385856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005193340020060181, + "loss": 1.3594, + "theoretical_loss": 3.6216414525351217, + "tokens_seen": 1179451392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005193179538615848, + "loss": 1.3834, + "theoretical_loss": 3.6216230279905925, + "tokens_seen": 1179516928 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005193019057171515, + "loss": 1.3705, + "theoretical_loss": 3.621604604756352, + "tokens_seen": 1179582464 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 396821, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.240499496459961, + "objective/train/theoretical_loss": 3.621586182832235, + "objective/train/tokens_used": 1200108000, + "theoretical_loss": 3.621586182832235, + "tokens_seen": 1179648000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005192858575727182, + "loss": 1.2934, + "theoretical_loss": 3.621586182832235, + "tokens_seen": 1179648000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005192698094282849, + "loss": 1.2857, + "theoretical_loss": 3.621567762218074, + "tokens_seen": 1179713536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005192537612838516, + "loss": 1.3332, + "theoretical_loss": 3.621549342913705, + "tokens_seen": 1179779072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005192377131394183, + "loss": 1.2775, + "theoretical_loss": 3.621530924918961, + "tokens_seen": 1179844608 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519221664994985, + "loss": 1.3946, + "theoretical_loss": 3.6215125082336757, + "tokens_seen": 1179910144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005192056168505517, + "loss": 1.2751, + "theoretical_loss": 3.621494092857684, + "tokens_seen": 1179975680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005191895687061184, + "loss": 1.3485, + "theoretical_loss": 3.62147567879082, + "tokens_seen": 1180041216 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005191735205616851, + "loss": 1.3742, + "theoretical_loss": 3.621457266032918, + "tokens_seen": 1180106752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005191574724172518, + "loss": 1.281, + "theoretical_loss": 3.621438854583812, + "tokens_seen": 1180172288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005191414242728185, + "loss": 1.4004, + "theoretical_loss": 3.621420444443337, + "tokens_seen": 1180237824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005191253761283852, + "loss": 1.2787, + "theoretical_loss": 3.621402035611326, + "tokens_seen": 1180303360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005191093279839519, + "loss": 1.4473, + "theoretical_loss": 3.6213836280876146, + "tokens_seen": 1180368896 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005190932798395186, + "loss": 1.3844, + "theoretical_loss": 3.6213652218720367, + "tokens_seen": 1180434432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005190772316950853, + "loss": 1.38, + "theoretical_loss": 3.6213468169644267, + "tokens_seen": 1180499968 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519061183550652, + "loss": 1.3562, + "theoretical_loss": 3.621328413364619, + "tokens_seen": 1180565504 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005190451354062187, + "loss": 1.3592, + "theoretical_loss": 3.621310011072448, + "tokens_seen": 1180631040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005190290872617854, + "loss": 1.4273, + "theoretical_loss": 3.621291610087749, + "tokens_seen": 1180696576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005190130391173521, + "loss": 1.3981, + "theoretical_loss": 3.6212732104103558, + "tokens_seen": 1180762112 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005189969909729188, + "loss": 1.4812, + "theoretical_loss": 3.621254812040103, + "tokens_seen": 1180827648 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005189809428284855, + "loss": 1.3422, + "theoretical_loss": 3.621236414976825, + "tokens_seen": 1180893184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005189648946840522, + "loss": 1.5115, + "theoretical_loss": 3.621218019220357, + "tokens_seen": 1180958720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005189488465396189, + "loss": 1.3807, + "theoretical_loss": 3.621199624770534, + "tokens_seen": 1181024256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005189327983951856, + "loss": 1.2891, + "theoretical_loss": 3.621181231627189, + "tokens_seen": 1181089792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005189167502507523, + "loss": 1.3865, + "theoretical_loss": 3.6211628397901583, + "tokens_seen": 1181155328 + }, + { + "epoch": 0.36, + "learning_rate": 0.000518900702106319, + "loss": 1.363, + "theoretical_loss": 3.621144449259276, + "tokens_seen": 1181220864 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005188846539618857, + "loss": 1.3727, + "theoretical_loss": 3.6211260600343773, + "tokens_seen": 1181286400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005188686058174524, + "loss": 1.3512, + "theoretical_loss": 3.6211076721152966, + "tokens_seen": 1181351936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005188525576730191, + "loss": 1.4427, + "theoretical_loss": 3.621089285501869, + "tokens_seen": 1181417472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005188365095285858, + "loss": 1.3918, + "theoretical_loss": 3.621070900193929, + "tokens_seen": 1181483008 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005188204613841525, + "loss": 1.2408, + "theoretical_loss": 3.621052516191312, + "tokens_seen": 1181548544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005188044132397191, + "loss": 1.3617, + "theoretical_loss": 3.6210341334938527, + "tokens_seen": 1181614080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005187883650952859, + "loss": 1.3777, + "theoretical_loss": 3.621015752101386, + "tokens_seen": 1181679616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005187723169508525, + "loss": 1.3017, + "theoretical_loss": 3.620997372013747, + "tokens_seen": 1181745152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005187562688064193, + "loss": 1.4179, + "theoretical_loss": 3.6209789932307705, + "tokens_seen": 1181810688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005187402206619859, + "loss": 1.321, + "theoretical_loss": 3.620960615752292, + "tokens_seen": 1181876224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005187241725175527, + "loss": 1.2739, + "theoretical_loss": 3.620942239578146, + "tokens_seen": 1181941760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005187081243731193, + "loss": 1.3463, + "theoretical_loss": 3.620923864708168, + "tokens_seen": 1182007296 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005186920762286861, + "loss": 1.2876, + "theoretical_loss": 3.620905491142193, + "tokens_seen": 1182072832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005186760280842527, + "loss": 1.4414, + "theoretical_loss": 3.6208871188800567, + "tokens_seen": 1182138368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005186599799398195, + "loss": 1.3373, + "theoretical_loss": 3.620868747921593, + "tokens_seen": 1182203904 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005186439317953861, + "loss": 1.4227, + "theoretical_loss": 3.6208503782666384, + "tokens_seen": 1182269440 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005186278836509529, + "loss": 1.4225, + "theoretical_loss": 3.620832009915028, + "tokens_seen": 1182334976 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005186118355065195, + "loss": 1.3005, + "theoretical_loss": 3.6208136428665965, + "tokens_seen": 1182400512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005185957873620863, + "loss": 1.275, + "theoretical_loss": 3.6207952771211795, + "tokens_seen": 1182466048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005185797392176529, + "loss": 1.2888, + "theoretical_loss": 3.620776912678612, + "tokens_seen": 1182531584 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005185636910732197, + "loss": 1.3743, + "theoretical_loss": 3.6207585495387304, + "tokens_seen": 1182597120 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005185476429287863, + "loss": 1.4577, + "theoretical_loss": 3.620740187701369, + "tokens_seen": 1182662656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005185315947843531, + "loss": 1.2731, + "theoretical_loss": 3.6207218271663644, + "tokens_seen": 1182728192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005185155466399197, + "loss": 1.2278, + "theoretical_loss": 3.6207034679335504, + "tokens_seen": 1182793728 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005184994984954865, + "loss": 1.3427, + "theoretical_loss": 3.620685110002764, + "tokens_seen": 1182859264 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 398018, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4799141883850098, + "objective/train/theoretical_loss": 3.6206667533738406, + "objective/train/tokens_used": 1203384800, + "theoretical_loss": 3.6206667533738406, + "tokens_seen": 1182924800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005184834503510531, + "loss": 1.3542, + "theoretical_loss": 3.6206667533738406, + "tokens_seen": 1182924800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005184674022066199, + "loss": 1.2616, + "theoretical_loss": 3.6206483980466144, + "tokens_seen": 1182990336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005184513540621865, + "loss": 1.4289, + "theoretical_loss": 3.620630044020923, + "tokens_seen": 1183055872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005184353059177533, + "loss": 1.3788, + "theoretical_loss": 3.6206116912966007, + "tokens_seen": 1183121408 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005184192577733199, + "loss": 1.3258, + "theoretical_loss": 3.620593339873483, + "tokens_seen": 1183186944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005184032096288867, + "loss": 1.3038, + "theoretical_loss": 3.620574989751406, + "tokens_seen": 1183252480 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005183871614844533, + "loss": 1.4068, + "theoretical_loss": 3.620556640930206, + "tokens_seen": 1183318016 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005183711133400201, + "loss": 1.4106, + "theoretical_loss": 3.6205382934097177, + "tokens_seen": 1183383552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005183550651955867, + "loss": 1.1708, + "theoretical_loss": 3.6205199471897775, + "tokens_seen": 1183449088 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005183390170511535, + "loss": 1.3734, + "theoretical_loss": 3.6205016022702208, + "tokens_seen": 1183514624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005183229689067201, + "loss": 1.2589, + "theoretical_loss": 3.620483258650884, + "tokens_seen": 1183580160 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005183069207622869, + "loss": 1.3248, + "theoretical_loss": 3.6204649163316023, + "tokens_seen": 1183645696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005182908726178535, + "loss": 1.2841, + "theoretical_loss": 3.6204465753122124, + "tokens_seen": 1183711232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005182748244734203, + "loss": 1.449, + "theoretical_loss": 3.620428235592549, + "tokens_seen": 1183776768 + }, + { + "epoch": 0.36, + "learning_rate": 0.000518258776328987, + "loss": 1.3521, + "theoretical_loss": 3.6204098971724488, + "tokens_seen": 1183842304 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005182427281845537, + "loss": 1.4387, + "theoretical_loss": 3.6203915600517482, + "tokens_seen": 1183907840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005182266800401203, + "loss": 1.3705, + "theoretical_loss": 3.6203732242302826, + "tokens_seen": 1183973376 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005182106318956871, + "loss": 1.3759, + "theoretical_loss": 3.6203548897078885, + "tokens_seen": 1184038912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005181945837512537, + "loss": 1.3856, + "theoretical_loss": 3.6203365564844012, + "tokens_seen": 1184104448 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005181785356068205, + "loss": 1.3476, + "theoretical_loss": 3.6203182245596572, + "tokens_seen": 1184169984 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005181624874623871, + "loss": 1.2597, + "theoretical_loss": 3.6202998939334927, + "tokens_seen": 1184235520 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005181464393179539, + "loss": 1.2945, + "theoretical_loss": 3.6202815646057447, + "tokens_seen": 1184301056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005181303911735206, + "loss": 1.2322, + "theoretical_loss": 3.6202632365762475, + "tokens_seen": 1184366592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005181143430290873, + "loss": 1.3941, + "theoretical_loss": 3.6202449098448386, + "tokens_seen": 1184432128 + }, + { + "epoch": 0.36, + "learning_rate": 0.000518098294884654, + "loss": 1.3156, + "theoretical_loss": 3.6202265844113537, + "tokens_seen": 1184497664 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005180822467402207, + "loss": 1.2034, + "theoretical_loss": 3.6202082602756303, + "tokens_seen": 1184563200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005180661985957874, + "loss": 1.3516, + "theoretical_loss": 3.6201899374375026, + "tokens_seen": 1184628736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005180501504513541, + "loss": 1.3206, + "theoretical_loss": 3.620171615896809, + "tokens_seen": 1184694272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005180341023069208, + "loss": 1.2562, + "theoretical_loss": 3.6201532956533846, + "tokens_seen": 1184759808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005180180541624875, + "loss": 1.3374, + "theoretical_loss": 3.6201349767070665, + "tokens_seen": 1184825344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005180020060180542, + "loss": 1.4135, + "theoretical_loss": 3.62011665905769, + "tokens_seen": 1184890880 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005179859578736209, + "loss": 1.2787, + "theoretical_loss": 3.6200983427050932, + "tokens_seen": 1184956416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005179699097291876, + "loss": 1.267, + "theoretical_loss": 3.620080027649111, + "tokens_seen": 1185021952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005179538615847543, + "loss": 1.3703, + "theoretical_loss": 3.620061713889581, + "tokens_seen": 1185087488 + }, + { + "epoch": 0.36, + "learning_rate": 0.000517937813440321, + "loss": 1.2718, + "theoretical_loss": 3.620043401426339, + "tokens_seen": 1185153024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005179217652958877, + "loss": 1.1488, + "theoretical_loss": 3.6200250902592224, + "tokens_seen": 1185218560 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005179057171514544, + "loss": 1.3381, + "theoretical_loss": 3.620006780388067, + "tokens_seen": 1185284096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005178896690070211, + "loss": 1.3544, + "theoretical_loss": 3.61998847181271, + "tokens_seen": 1185349632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005178736208625878, + "loss": 1.3417, + "theoretical_loss": 3.619970164532988, + "tokens_seen": 1185415168 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005178575727181545, + "loss": 1.2851, + "theoretical_loss": 3.6199518585487374, + "tokens_seen": 1185480704 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005178415245737212, + "loss": 1.3352, + "theoretical_loss": 3.619933553859795, + "tokens_seen": 1185546240 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005178254764292879, + "loss": 1.3918, + "theoretical_loss": 3.6199152504659975, + "tokens_seen": 1185611776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005178094282848546, + "loss": 1.4179, + "theoretical_loss": 3.619896948367182, + "tokens_seen": 1185677312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005177933801404213, + "loss": 1.3772, + "theoretical_loss": 3.619878647563185, + "tokens_seen": 1185742848 + }, + { + "epoch": 0.36, + "learning_rate": 0.000517777331995988, + "loss": 1.2345, + "theoretical_loss": 3.6198603480538427, + "tokens_seen": 1185808384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005177612838515547, + "loss": 1.2903, + "theoretical_loss": 3.6198420498389936, + "tokens_seen": 1185873920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005177452357071214, + "loss": 1.2857, + "theoretical_loss": 3.6198237529184736, + "tokens_seen": 1185939456 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005177291875626881, + "loss": 1.3838, + "theoretical_loss": 3.619805457292119, + "tokens_seen": 1186004992 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005177131394182549, + "loss": 1.2333, + "theoretical_loss": 3.6197871629597684, + "tokens_seen": 1186070528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005176970912738215, + "loss": 1.3619, + "theoretical_loss": 3.6197688699212573, + "tokens_seen": 1186136064 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 398884, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.390918493270874, + "objective/train/theoretical_loss": 3.619750578176424, + "objective/train/tokens_used": 1206661600, + "theoretical_loss": 3.619750578176424, + "tokens_seen": 1186201600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005176810431293883, + "loss": 1.4301, + "theoretical_loss": 3.619750578176424, + "tokens_seen": 1186201600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005176649949849549, + "loss": 1.5078, + "theoretical_loss": 3.619732287725104, + "tokens_seen": 1186267136 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005176489468405217, + "loss": 1.3039, + "theoretical_loss": 3.619713998567135, + "tokens_seen": 1186332672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005176328986960883, + "loss": 1.3442, + "theoretical_loss": 3.619695710702355, + "tokens_seen": 1186398208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005176168505516551, + "loss": 1.2442, + "theoretical_loss": 3.6196774241306002, + "tokens_seen": 1186463744 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005176008024072217, + "loss": 1.3391, + "theoretical_loss": 3.619659138851708, + "tokens_seen": 1186529280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005175847542627885, + "loss": 1.4667, + "theoretical_loss": 3.6196408548655157, + "tokens_seen": 1186594816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005175687061183551, + "loss": 1.3806, + "theoretical_loss": 3.61962257217186, + "tokens_seen": 1186660352 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005175526579739219, + "loss": 1.3643, + "theoretical_loss": 3.619604290770579, + "tokens_seen": 1186725888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005175366098294885, + "loss": 1.3412, + "theoretical_loss": 3.619586010661509, + "tokens_seen": 1186791424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005175205616850552, + "loss": 1.3849, + "theoretical_loss": 3.6195677318444885, + "tokens_seen": 1186856960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005175045135406219, + "loss": 1.456, + "theoretical_loss": 3.619549454319354, + "tokens_seen": 1186922496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005174884653961886, + "loss": 1.2811, + "theoretical_loss": 3.6195311780859427, + "tokens_seen": 1186988032 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005174724172517553, + "loss": 1.258, + "theoretical_loss": 3.619512903144093, + "tokens_seen": 1187053568 + }, + { + "epoch": 0.36, + "learning_rate": 0.000517456369107322, + "loss": 1.4333, + "theoretical_loss": 3.619494629493641, + "tokens_seen": 1187119104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005174403209628887, + "loss": 1.3053, + "theoretical_loss": 3.6194763571344257, + "tokens_seen": 1187184640 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005174242728184554, + "loss": 1.2799, + "theoretical_loss": 3.619458086066283, + "tokens_seen": 1187250176 + }, + { + "epoch": 0.36, + "learning_rate": 0.000517408224674022, + "loss": 1.4062, + "theoretical_loss": 3.6194398162890513, + "tokens_seen": 1187315712 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005173921765295888, + "loss": 1.4374, + "theoretical_loss": 3.6194215478025678, + "tokens_seen": 1187381248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005173761283851554, + "loss": 1.3834, + "theoretical_loss": 3.6194032806066705, + "tokens_seen": 1187446784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005173600802407222, + "loss": 1.3529, + "theoretical_loss": 3.619385014701197, + "tokens_seen": 1187512320 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005173440320962888, + "loss": 1.3175, + "theoretical_loss": 3.6193667500859847, + "tokens_seen": 1187577856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005173279839518556, + "loss": 1.428, + "theoretical_loss": 3.619348486760871, + "tokens_seen": 1187643392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005173119358074222, + "loss": 1.279, + "theoretical_loss": 3.619330224725694, + "tokens_seen": 1187708928 + }, + { + "epoch": 0.36, + "learning_rate": 0.000517295887662989, + "loss": 1.3363, + "theoretical_loss": 3.6193119639802918, + "tokens_seen": 1187774464 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005172798395185556, + "loss": 1.3446, + "theoretical_loss": 3.619293704524501, + "tokens_seen": 1187840000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005172637913741224, + "loss": 1.3324, + "theoretical_loss": 3.6192754463581602, + "tokens_seen": 1187905536 + }, + { + "epoch": 0.36, + "learning_rate": 0.000517247743229689, + "loss": 1.4595, + "theoretical_loss": 3.619257189481107, + "tokens_seen": 1187971072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005172316950852558, + "loss": 1.3313, + "theoretical_loss": 3.619238933893179, + "tokens_seen": 1188036608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005172156469408224, + "loss": 1.3008, + "theoretical_loss": 3.619220679594215, + "tokens_seen": 1188102144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005171995987963892, + "loss": 1.3326, + "theoretical_loss": 3.619202426584052, + "tokens_seen": 1188167680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005171835506519558, + "loss": 1.4644, + "theoretical_loss": 3.6191841748625277, + "tokens_seen": 1188233216 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005171675025075226, + "loss": 1.2918, + "theoretical_loss": 3.619165924429481, + "tokens_seen": 1188298752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005171514543630892, + "loss": 1.3477, + "theoretical_loss": 3.61914767528475, + "tokens_seen": 1188364288 + }, + { + "epoch": 0.36, + "learning_rate": 0.000517135406218656, + "loss": 1.4141, + "theoretical_loss": 3.619129427428171, + "tokens_seen": 1188429824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005171193580742226, + "loss": 1.4162, + "theoretical_loss": 3.6191111808595835, + "tokens_seen": 1188495360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005171033099297894, + "loss": 1.2769, + "theoretical_loss": 3.6190929355788253, + "tokens_seen": 1188560896 + }, + { + "epoch": 0.36, + "learning_rate": 0.000517087261785356, + "loss": 1.2547, + "theoretical_loss": 3.619074691585735, + "tokens_seen": 1188626432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005170712136409228, + "loss": 1.3061, + "theoretical_loss": 3.6190564488801495, + "tokens_seen": 1188691968 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005170551654964894, + "loss": 1.3649, + "theoretical_loss": 3.619038207461908, + "tokens_seen": 1188757504 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005170391173520562, + "loss": 1.4416, + "theoretical_loss": 3.6190199673308485, + "tokens_seen": 1188823040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005170230692076228, + "loss": 1.4086, + "theoretical_loss": 3.6190017284868086, + "tokens_seen": 1188888576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005170070210631896, + "loss": 1.5011, + "theoretical_loss": 3.6189834909296277, + "tokens_seen": 1188954112 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005169909729187562, + "loss": 1.3175, + "theoretical_loss": 3.618965254659143, + "tokens_seen": 1189019648 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516974924774323, + "loss": 1.3763, + "theoretical_loss": 3.618947019675193, + "tokens_seen": 1189085184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005169588766298896, + "loss": 1.3976, + "theoretical_loss": 3.6189287859776167, + "tokens_seen": 1189150720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005169428284854564, + "loss": 1.3302, + "theoretical_loss": 3.6189105535662516, + "tokens_seen": 1189216256 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516926780341023, + "loss": 1.377, + "theoretical_loss": 3.618892322440937, + "tokens_seen": 1189281792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005169107321965898, + "loss": 1.3334, + "theoretical_loss": 3.61887409260151, + "tokens_seen": 1189347328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005168946840521564, + "loss": 1.2878, + "theoretical_loss": 3.6188558640478106, + "tokens_seen": 1189412864 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 400204, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2898616790771484, + "objective/train/theoretical_loss": 3.618837636779676, + "objective/train/tokens_used": 1209938400, + "theoretical_loss": 3.618837636779676, + "tokens_seen": 1189478400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005168786359077232, + "loss": 1.3022, + "theoretical_loss": 3.618837636779676, + "tokens_seen": 1189478400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005168625877632898, + "loss": 1.3268, + "theoretical_loss": 3.618819410796946, + "tokens_seen": 1189543936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005168465396188566, + "loss": 1.4245, + "theoretical_loss": 3.618801186099458, + "tokens_seen": 1189609472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005168304914744232, + "loss": 1.2795, + "theoretical_loss": 3.618782962687051, + "tokens_seen": 1189675008 + }, + { + "epoch": 0.36, + "learning_rate": 0.00051681444332999, + "loss": 1.2759, + "theoretical_loss": 3.6187647405595635, + "tokens_seen": 1189740544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005167983951855566, + "loss": 1.3017, + "theoretical_loss": 3.6187465197168347, + "tokens_seen": 1189806080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005167823470411234, + "loss": 1.2985, + "theoretical_loss": 3.6187283001587023, + "tokens_seen": 1189871616 + }, + { + "epoch": 0.36, + "learning_rate": 0.00051676629889669, + "loss": 1.264, + "theoretical_loss": 3.6187100818850055, + "tokens_seen": 1189937152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005167502507522568, + "loss": 1.327, + "theoretical_loss": 3.6186918648955837, + "tokens_seen": 1190002688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005167342026078235, + "loss": 1.392, + "theoretical_loss": 3.6186736491902742, + "tokens_seen": 1190068224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005167181544633902, + "loss": 1.267, + "theoretical_loss": 3.6186554347689164, + "tokens_seen": 1190133760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005167021063189569, + "loss": 1.3462, + "theoretical_loss": 3.6186372216313503, + "tokens_seen": 1190199296 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005166860581745236, + "loss": 1.2363, + "theoretical_loss": 3.6186190097774125, + "tokens_seen": 1190264832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005166700100300903, + "loss": 1.3146, + "theoretical_loss": 3.618600799206943, + "tokens_seen": 1190330368 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516653961885657, + "loss": 1.4205, + "theoretical_loss": 3.6185825899197814, + "tokens_seen": 1190395904 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005166379137412237, + "loss": 1.3104, + "theoretical_loss": 3.618564381915766, + "tokens_seen": 1190461440 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005166218655967904, + "loss": 1.3198, + "theoretical_loss": 3.6185461751947354, + "tokens_seen": 1190526976 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005166058174523571, + "loss": 1.3626, + "theoretical_loss": 3.6185279697565287, + "tokens_seen": 1190592512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005165897693079238, + "loss": 1.2887, + "theoretical_loss": 3.6185097656009853, + "tokens_seen": 1190658048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005165737211634905, + "loss": 1.3333, + "theoretical_loss": 3.618491562727944, + "tokens_seen": 1190723584 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005165576730190572, + "loss": 1.2951, + "theoretical_loss": 3.6184733611372435, + "tokens_seen": 1190789120 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005165416248746239, + "loss": 1.3484, + "theoretical_loss": 3.6184551608287236, + "tokens_seen": 1190854656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005165255767301906, + "loss": 1.3487, + "theoretical_loss": 3.6184369618022236, + "tokens_seen": 1190920192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005165095285857574, + "loss": 1.3191, + "theoretical_loss": 3.618418764057581, + "tokens_seen": 1190985728 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516493480441324, + "loss": 1.2842, + "theoretical_loss": 3.618400567594637, + "tokens_seen": 1191051264 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005164774322968908, + "loss": 1.3415, + "theoretical_loss": 3.6183823724132296, + "tokens_seen": 1191116800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005164613841524574, + "loss": 1.4422, + "theoretical_loss": 3.6183641785131986, + "tokens_seen": 1191182336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005164453360080242, + "loss": 1.2918, + "theoretical_loss": 3.6183459858943827, + "tokens_seen": 1191247872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005164292878635908, + "loss": 1.483, + "theoretical_loss": 3.618327794556622, + "tokens_seen": 1191313408 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005164132397191576, + "loss": 1.2428, + "theoretical_loss": 3.618309604499755, + "tokens_seen": 1191378944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005163971915747242, + "loss": 1.3704, + "theoretical_loss": 3.618291415723622, + "tokens_seen": 1191444480 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516381143430291, + "loss": 1.4868, + "theoretical_loss": 3.6182732282280607, + "tokens_seen": 1191510016 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005163650952858576, + "loss": 1.3767, + "theoretical_loss": 3.6182550420129127, + "tokens_seen": 1191575552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005163490471414244, + "loss": 1.4112, + "theoretical_loss": 3.618236857078016, + "tokens_seen": 1191641088 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516332998996991, + "loss": 1.344, + "theoretical_loss": 3.6182186734232102, + "tokens_seen": 1191706624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005163169508525578, + "loss": 1.4586, + "theoretical_loss": 3.6182004910483356, + "tokens_seen": 1191772160 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005163009027081244, + "loss": 1.3765, + "theoretical_loss": 3.618182309953231, + "tokens_seen": 1191837696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005162848545636912, + "loss": 1.2811, + "theoretical_loss": 3.618164130137736, + "tokens_seen": 1191903232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005162688064192578, + "loss": 1.3366, + "theoretical_loss": 3.61814595160169, + "tokens_seen": 1191968768 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005162527582748246, + "loss": 1.2032, + "theoretical_loss": 3.618127774344934, + "tokens_seen": 1192034304 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005162367101303912, + "loss": 1.3796, + "theoretical_loss": 3.6181095983673055, + "tokens_seen": 1192099840 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516220661985958, + "loss": 1.2893, + "theoretical_loss": 3.6180914236686457, + "tokens_seen": 1192165376 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005162046138415246, + "loss": 1.2538, + "theoretical_loss": 3.618073250248794, + "tokens_seen": 1192230912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005161885656970914, + "loss": 1.4381, + "theoretical_loss": 3.6180550781075898, + "tokens_seen": 1192296448 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516172517552658, + "loss": 1.1963, + "theoretical_loss": 3.618036907244873, + "tokens_seen": 1192361984 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005161564694082248, + "loss": 1.4107, + "theoretical_loss": 3.618018737660484, + "tokens_seen": 1192427520 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005161404212637914, + "loss": 1.294, + "theoretical_loss": 3.6180005693542614, + "tokens_seen": 1192493056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005161243731193581, + "loss": 1.3024, + "theoretical_loss": 3.617982402326046, + "tokens_seen": 1192558592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005161083249749248, + "loss": 1.341, + "theoretical_loss": 3.617964236575678, + "tokens_seen": 1192624128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005160922768304915, + "loss": 1.4014, + "theoretical_loss": 3.6179460721029955, + "tokens_seen": 1192689664 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 401134, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1924456357955933, + "objective/train/theoretical_loss": 3.6179279089078404, + "objective/train/tokens_used": 1213215200, + "theoretical_loss": 3.6179279089078404, + "tokens_seen": 1192755200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005160762286860582, + "loss": 1.2172, + "theoretical_loss": 3.6179279089078404, + "tokens_seen": 1192755200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005160601805416249, + "loss": 1.2603, + "theoretical_loss": 3.617909746990052, + "tokens_seen": 1192820736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005160441323971916, + "loss": 1.3865, + "theoretical_loss": 3.6178915863494696, + "tokens_seen": 1192886272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005160280842527583, + "loss": 1.2988, + "theoretical_loss": 3.6178734269859345, + "tokens_seen": 1192951808 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516012036108325, + "loss": 1.4073, + "theoretical_loss": 3.6178552688992855, + "tokens_seen": 1193017344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005159959879638917, + "loss": 1.3006, + "theoretical_loss": 3.6178371120893638, + "tokens_seen": 1193082880 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005159799398194583, + "loss": 1.4048, + "theoretical_loss": 3.6178189565560084, + "tokens_seen": 1193148416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005159638916750251, + "loss": 1.2509, + "theoretical_loss": 3.617800802299061, + "tokens_seen": 1193213952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005159478435305917, + "loss": 1.3963, + "theoretical_loss": 3.61778264931836, + "tokens_seen": 1193279488 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005159317953861585, + "loss": 1.3635, + "theoretical_loss": 3.617764497613746, + "tokens_seen": 1193345024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005159157472417251, + "loss": 1.4002, + "theoretical_loss": 3.6177463471850597, + "tokens_seen": 1193410560 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005158996990972919, + "loss": 1.2974, + "theoretical_loss": 3.617728198032142, + "tokens_seen": 1193476096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005158836509528585, + "loss": 1.3488, + "theoretical_loss": 3.617710050154832, + "tokens_seen": 1193541632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005158676028084253, + "loss": 1.3583, + "theoretical_loss": 3.61769190355297, + "tokens_seen": 1193607168 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005158515546639919, + "loss": 1.2704, + "theoretical_loss": 3.617673758226397, + "tokens_seen": 1193672704 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005158355065195587, + "loss": 1.3382, + "theoretical_loss": 3.6176556141749536, + "tokens_seen": 1193738240 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005158194583751253, + "loss": 1.332, + "theoretical_loss": 3.6176374713984796, + "tokens_seen": 1193803776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005158034102306921, + "loss": 1.3715, + "theoretical_loss": 3.617619329896815, + "tokens_seen": 1193869312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005157873620862587, + "loss": 1.3082, + "theoretical_loss": 3.6176011896698013, + "tokens_seen": 1193934848 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005157713139418255, + "loss": 1.3234, + "theoretical_loss": 3.617583050717278, + "tokens_seen": 1194000384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005157552657973921, + "loss": 1.415, + "theoretical_loss": 3.617564913039087, + "tokens_seen": 1194065920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005157392176529589, + "loss": 1.372, + "theoretical_loss": 3.617546776635067, + "tokens_seen": 1194131456 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005157231695085255, + "loss": 1.2965, + "theoretical_loss": 3.61752864150506, + "tokens_seen": 1194196992 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005157071213640923, + "loss": 1.278, + "theoretical_loss": 3.617510507648906, + "tokens_seen": 1194262528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005156910732196589, + "loss": 1.3113, + "theoretical_loss": 3.617492375066446, + "tokens_seen": 1194328064 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005156750250752257, + "loss": 1.2497, + "theoretical_loss": 3.6174742437575196, + "tokens_seen": 1194393600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005156589769307923, + "loss": 1.3184, + "theoretical_loss": 3.6174561137219685, + "tokens_seen": 1194459136 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005156429287863591, + "loss": 1.3982, + "theoretical_loss": 3.6174379849596336, + "tokens_seen": 1194524672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005156268806419257, + "loss": 1.2823, + "theoretical_loss": 3.617419857470355, + "tokens_seen": 1194590208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005156108324974925, + "loss": 1.2944, + "theoretical_loss": 3.6174017312539735, + "tokens_seen": 1194655744 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005155947843530591, + "loss": 1.4242, + "theoretical_loss": 3.61738360631033, + "tokens_seen": 1194721280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005155787362086259, + "loss": 1.3206, + "theoretical_loss": 3.617365482639266, + "tokens_seen": 1194786816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005155626880641925, + "loss": 1.346, + "theoretical_loss": 3.617347360240621, + "tokens_seen": 1194852352 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005155466399197593, + "loss": 1.3717, + "theoretical_loss": 3.6173292391142375, + "tokens_seen": 1194917888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005155305917753259, + "loss": 1.3413, + "theoretical_loss": 3.6173111192599547, + "tokens_seen": 1194983424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005155145436308927, + "loss": 1.3822, + "theoretical_loss": 3.6172930006776154, + "tokens_seen": 1195048960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005154984954864593, + "loss": 1.3607, + "theoretical_loss": 3.6172748833670587, + "tokens_seen": 1195114496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005154824473420261, + "loss": 1.3311, + "theoretical_loss": 3.6172567673281266, + "tokens_seen": 1195180032 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005154663991975927, + "loss": 1.3801, + "theoretical_loss": 3.61723865256066, + "tokens_seen": 1195245568 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005154503510531595, + "loss": 1.3788, + "theoretical_loss": 3.6172205390645003, + "tokens_seen": 1195311104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005154343029087261, + "loss": 1.3877, + "theoretical_loss": 3.617202426839488, + "tokens_seen": 1195376640 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005154182547642929, + "loss": 1.3405, + "theoretical_loss": 3.6171843158854644, + "tokens_seen": 1195442176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005154022066198596, + "loss": 1.2665, + "theoretical_loss": 3.6171662062022705, + "tokens_seen": 1195507712 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005153861584754263, + "loss": 1.3175, + "theoretical_loss": 3.617148097789748, + "tokens_seen": 1195573248 + }, + { + "epoch": 0.36, + "learning_rate": 0.000515370110330993, + "loss": 1.2281, + "theoretical_loss": 3.6171299906477374, + "tokens_seen": 1195638784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005153540621865597, + "loss": 1.341, + "theoretical_loss": 3.617111884776081, + "tokens_seen": 1195704320 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005153380140421264, + "loss": 1.4207, + "theoretical_loss": 3.6170937801746184, + "tokens_seen": 1195769856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005153219658976931, + "loss": 1.3215, + "theoretical_loss": 3.617075676843192, + "tokens_seen": 1195835392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005153059177532599, + "loss": 1.3226, + "theoretical_loss": 3.6170575747816436, + "tokens_seen": 1195900928 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005152898696088265, + "loss": 1.4244, + "theoretical_loss": 3.6170394739898133, + "tokens_seen": 1195966464 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 402272, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3172639608383179, + "objective/train/theoretical_loss": 3.617021374467543, + "objective/train/tokens_used": 1216492000, + "theoretical_loss": 3.617021374467543, + "tokens_seen": 1196032000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005152738214643933, + "loss": 1.3791, + "theoretical_loss": 3.617021374467543, + "tokens_seen": 1196032000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005152577733199599, + "loss": 1.3855, + "theoretical_loss": 3.617003276214674, + "tokens_seen": 1196097536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005152417251755267, + "loss": 1.4151, + "theoretical_loss": 3.6169851792310483, + "tokens_seen": 1196163072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005152256770310933, + "loss": 1.3781, + "theoretical_loss": 3.616967083516507, + "tokens_seen": 1196228608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005152096288866601, + "loss": 1.2994, + "theoretical_loss": 3.616948989070891, + "tokens_seen": 1196294144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005151935807422267, + "loss": 1.3409, + "theoretical_loss": 3.6169308958940425, + "tokens_seen": 1196359680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005151775325977935, + "loss": 1.363, + "theoretical_loss": 3.616912803985803, + "tokens_seen": 1196425216 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005151614844533601, + "loss": 1.3576, + "theoretical_loss": 3.6168947133460136, + "tokens_seen": 1196490752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005151454363089269, + "loss": 1.3667, + "theoretical_loss": 3.6168766239745165, + "tokens_seen": 1196556288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005151293881644935, + "loss": 1.3921, + "theoretical_loss": 3.616858535871153, + "tokens_seen": 1196621824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005151133400200603, + "loss": 1.4119, + "theoretical_loss": 3.616840449035765, + "tokens_seen": 1196687360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005150972918756269, + "loss": 1.3082, + "theoretical_loss": 3.6168223634681933, + "tokens_seen": 1196752896 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005150812437311937, + "loss": 1.238, + "theoretical_loss": 3.6168042791682806, + "tokens_seen": 1196818432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005150651955867603, + "loss": 1.4655, + "theoretical_loss": 3.6167861961358687, + "tokens_seen": 1196883968 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005150491474423271, + "loss": 1.2184, + "theoretical_loss": 3.6167681143707986, + "tokens_seen": 1196949504 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005150330992978937, + "loss": 1.3114, + "theoretical_loss": 3.616750033872912, + "tokens_seen": 1197015040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005150170511534605, + "loss": 1.3726, + "theoretical_loss": 3.616731954642052, + "tokens_seen": 1197080576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005150010030090271, + "loss": 1.2718, + "theoretical_loss": 3.6167138766780598, + "tokens_seen": 1197146112 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005149849548645939, + "loss": 1.4189, + "theoretical_loss": 3.6166957999807767, + "tokens_seen": 1197211648 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005149689067201605, + "loss": 1.3034, + "theoretical_loss": 3.616677724550045, + "tokens_seen": 1197277184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005149528585757273, + "loss": 1.3077, + "theoretical_loss": 3.616659650385707, + "tokens_seen": 1197342720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005149368104312939, + "loss": 1.4082, + "theoretical_loss": 3.616641577487604, + "tokens_seen": 1197408256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005149207622868607, + "loss": 1.3105, + "theoretical_loss": 3.6166235058555785, + "tokens_seen": 1197473792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005149047141424273, + "loss": 1.2711, + "theoretical_loss": 3.6166054354894728, + "tokens_seen": 1197539328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005148886659979941, + "loss": 1.3306, + "theoretical_loss": 3.616587366389128, + "tokens_seen": 1197604864 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005148726178535607, + "loss": 1.3604, + "theoretical_loss": 3.6165692985543867, + "tokens_seen": 1197670400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005148565697091275, + "loss": 1.4035, + "theoretical_loss": 3.616551231985091, + "tokens_seen": 1197735936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005148405215646941, + "loss": 1.3933, + "theoretical_loss": 3.616533166681083, + "tokens_seen": 1197801472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005148244734202609, + "loss": 1.4038, + "theoretical_loss": 3.6165151026422055, + "tokens_seen": 1197867008 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005148084252758275, + "loss": 1.4169, + "theoretical_loss": 3.6164970398682996, + "tokens_seen": 1197932544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005147923771313943, + "loss": 1.225, + "theoretical_loss": 3.616478978359208, + "tokens_seen": 1197998080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005147763289869609, + "loss": 1.2246, + "theoretical_loss": 3.6164609181147735, + "tokens_seen": 1198063616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005147602808425277, + "loss": 1.3435, + "theoretical_loss": 3.616442859134837, + "tokens_seen": 1198129152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005147442326980943, + "loss": 1.3922, + "theoretical_loss": 3.6164248014192424, + "tokens_seen": 1198194688 + }, + { + "epoch": 0.36, + "learning_rate": 0.000514728184553661, + "loss": 1.3292, + "theoretical_loss": 3.616406744967831, + "tokens_seen": 1198260224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005147121364092277, + "loss": 1.3444, + "theoretical_loss": 3.616388689780446, + "tokens_seen": 1198325760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005146960882647944, + "loss": 1.3101, + "theoretical_loss": 3.6163706358569288, + "tokens_seen": 1198391296 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005146800401203611, + "loss": 1.3164, + "theoretical_loss": 3.6163525831971217, + "tokens_seen": 1198456832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005146639919759278, + "loss": 1.3864, + "theoretical_loss": 3.6163345318008684, + "tokens_seen": 1198522368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005146479438314945, + "loss": 1.3088, + "theoretical_loss": 3.6163164816680107, + "tokens_seen": 1198587904 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005146318956870612, + "loss": 1.4454, + "theoretical_loss": 3.616298432798391, + "tokens_seen": 1198653440 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005146158475426279, + "loss": 1.3278, + "theoretical_loss": 3.6162803851918515, + "tokens_seen": 1198718976 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005145997993981946, + "loss": 1.311, + "theoretical_loss": 3.6162623388482356, + "tokens_seen": 1198784512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005145837512537612, + "loss": 1.3142, + "theoretical_loss": 3.6162442937673855, + "tokens_seen": 1198850048 + }, + { + "epoch": 0.36, + "learning_rate": 0.000514567703109328, + "loss": 1.3755, + "theoretical_loss": 3.6162262499491438, + "tokens_seen": 1198915584 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005145516549648946, + "loss": 1.3546, + "theoretical_loss": 3.616208207393353, + "tokens_seen": 1198981120 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005145356068204614, + "loss": 1.2408, + "theoretical_loss": 3.616190166099856, + "tokens_seen": 1199046656 + }, + { + "epoch": 0.36, + "learning_rate": 0.000514519558676028, + "loss": 1.3636, + "theoretical_loss": 3.616172126068496, + "tokens_seen": 1199112192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005145035105315948, + "loss": 1.325, + "theoretical_loss": 3.6161540872991145, + "tokens_seen": 1199177728 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005144874623871614, + "loss": 1.3188, + "theoretical_loss": 3.616136049791555, + "tokens_seen": 1199243264 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 403082, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2459710836410522, + "objective/train/theoretical_loss": 3.61611801354566, + "objective/train/tokens_used": 1219768800, + "theoretical_loss": 3.61611801354566, + "tokens_seen": 1199308800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005144714142427282, + "loss": 1.3009, + "theoretical_loss": 3.61611801354566, + "tokens_seen": 1199308800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005144553660982948, + "loss": 1.3199, + "theoretical_loss": 3.6160999785612726, + "tokens_seen": 1199374336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005144393179538616, + "loss": 1.2832, + "theoretical_loss": 3.6160819448382355, + "tokens_seen": 1199439872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005144232698094282, + "loss": 1.2378, + "theoretical_loss": 3.616063912376392, + "tokens_seen": 1199505408 + }, + { + "epoch": 0.36, + "learning_rate": 0.000514407221664995, + "loss": 1.3436, + "theoretical_loss": 3.6160458811755842, + "tokens_seen": 1199570944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005143911735205616, + "loss": 1.2775, + "theoretical_loss": 3.6160278512356556, + "tokens_seen": 1199636480 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005143751253761284, + "loss": 1.3245, + "theoretical_loss": 3.6160098225564488, + "tokens_seen": 1199702016 + }, + { + "epoch": 0.36, + "learning_rate": 0.000514359077231695, + "loss": 1.3329, + "theoretical_loss": 3.6159917951378073, + "tokens_seen": 1199767552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005143430290872618, + "loss": 1.3655, + "theoretical_loss": 3.6159737689795737, + "tokens_seen": 1199833088 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005143269809428284, + "loss": 1.3092, + "theoretical_loss": 3.6159557440815915, + "tokens_seen": 1199898624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005143109327983952, + "loss": 1.4172, + "theoretical_loss": 3.6159377204437027, + "tokens_seen": 1199964160 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005142948846539618, + "loss": 1.1557, + "theoretical_loss": 3.6159196980657518, + "tokens_seen": 1200029696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005142788365095286, + "loss": 1.3506, + "theoretical_loss": 3.6159016769475807, + "tokens_seen": 1200095232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005142627883650952, + "loss": 1.3797, + "theoretical_loss": 3.615883657089033, + "tokens_seen": 1200160768 + }, + { + "epoch": 0.36, + "learning_rate": 0.000514246740220662, + "loss": 1.4379, + "theoretical_loss": 3.615865638489953, + "tokens_seen": 1200226304 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005142306920762286, + "loss": 1.41, + "theoretical_loss": 3.6158476211501815, + "tokens_seen": 1200291840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005142146439317954, + "loss": 1.4144, + "theoretical_loss": 3.615829605069564, + "tokens_seen": 1200357376 + }, + { + "epoch": 0.36, + "learning_rate": 0.000514198595787362, + "loss": 1.4482, + "theoretical_loss": 3.6158115902479424, + "tokens_seen": 1200422912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005141825476429288, + "loss": 1.3713, + "theoretical_loss": 3.6157935766851605, + "tokens_seen": 1200488448 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005141664994984954, + "loss": 1.3692, + "theoretical_loss": 3.615775564381062, + "tokens_seen": 1200553984 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005141504513540622, + "loss": 1.2828, + "theoretical_loss": 3.6157575533354893, + "tokens_seen": 1200619520 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005141344032096288, + "loss": 1.2701, + "theoretical_loss": 3.6157395435482864, + "tokens_seen": 1200685056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005141183550651956, + "loss": 1.4091, + "theoretical_loss": 3.615721535019297, + "tokens_seen": 1200750592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005141023069207624, + "loss": 1.3496, + "theoretical_loss": 3.6157035277483636, + "tokens_seen": 1200816128 + }, + { + "epoch": 0.36, + "learning_rate": 0.000514086258776329, + "loss": 1.3302, + "theoretical_loss": 3.6156855217353305, + "tokens_seen": 1200881664 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005140702106318958, + "loss": 1.414, + "theoretical_loss": 3.615667516980041, + "tokens_seen": 1200947200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005140541624874624, + "loss": 1.4534, + "theoretical_loss": 3.6156495134823383, + "tokens_seen": 1201012736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005140381143430292, + "loss": 1.3739, + "theoretical_loss": 3.615631511242066, + "tokens_seen": 1201078272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005140220661985958, + "loss": 1.3974, + "theoretical_loss": 3.6156135102590685, + "tokens_seen": 1201143808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005140060180541626, + "loss": 1.3475, + "theoretical_loss": 3.615595510533188, + "tokens_seen": 1201209344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005139899699097292, + "loss": 1.3214, + "theoretical_loss": 3.6155775120642692, + "tokens_seen": 1201274880 + }, + { + "epoch": 0.36, + "learning_rate": 0.000513973921765296, + "loss": 1.2765, + "theoretical_loss": 3.6155595148521553, + "tokens_seen": 1201340416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005139578736208626, + "loss": 1.249, + "theoretical_loss": 3.6155415188966904, + "tokens_seen": 1201405952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005139418254764294, + "loss": 1.3862, + "theoretical_loss": 3.6155235241977177, + "tokens_seen": 1201471488 + }, + { + "epoch": 0.36, + "learning_rate": 0.000513925777331996, + "loss": 1.4308, + "theoretical_loss": 3.615505530755081, + "tokens_seen": 1201537024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005139097291875628, + "loss": 1.3439, + "theoretical_loss": 3.6154875385686243, + "tokens_seen": 1201602560 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005138936810431294, + "loss": 1.3738, + "theoretical_loss": 3.6154695476381913, + "tokens_seen": 1201668096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005138776328986962, + "loss": 1.3634, + "theoretical_loss": 3.6154515579636257, + "tokens_seen": 1201733632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005138615847542628, + "loss": 1.2396, + "theoretical_loss": 3.6154335695447717, + "tokens_seen": 1201799168 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005138455366098296, + "loss": 1.3243, + "theoretical_loss": 3.615415582381473, + "tokens_seen": 1201864704 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005138294884653962, + "loss": 1.3334, + "theoretical_loss": 3.6153975964735734, + "tokens_seen": 1201930240 + }, + { + "epoch": 0.36, + "learning_rate": 0.000513813440320963, + "loss": 1.3817, + "theoretical_loss": 3.615379611820917, + "tokens_seen": 1201995776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005137973921765296, + "loss": 1.4029, + "theoretical_loss": 3.6153616284233476, + "tokens_seen": 1202061312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005137813440320964, + "loss": 1.2957, + "theoretical_loss": 3.615343646280709, + "tokens_seen": 1202126848 + }, + { + "epoch": 0.36, + "learning_rate": 0.000513765295887663, + "loss": 1.3365, + "theoretical_loss": 3.615325665392846, + "tokens_seen": 1202192384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005137492477432298, + "loss": 1.3333, + "theoretical_loss": 3.6153076857596016, + "tokens_seen": 1202257920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005137331995987964, + "loss": 1.3629, + "theoretical_loss": 3.6152897073808212, + "tokens_seen": 1202323456 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005137171514543632, + "loss": 1.2974, + "theoretical_loss": 3.6152717302563477, + "tokens_seen": 1202388992 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005137011033099298, + "loss": 1.3039, + "theoretical_loss": 3.6152537543860257, + "tokens_seen": 1202454528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005136850551654966, + "loss": 1.4214, + "theoretical_loss": 3.615235779769699, + "tokens_seen": 1202520064 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 404343, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.367863416671753, + "objective/train/theoretical_loss": 3.6152178064072125, + "objective/train/tokens_used": 1223045600, + "theoretical_loss": 3.6152178064072125, + "tokens_seen": 1202585600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005136690070210632, + "loss": 1.3764, + "theoretical_loss": 3.6152178064072125, + "tokens_seen": 1202585600 + }, + { + "epoch": 0.36, + "learning_rate": 0.00051365295887663, + "loss": 1.3665, + "theoretical_loss": 3.6151998342984095, + "tokens_seen": 1202651136 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005136369107321966, + "loss": 1.3252, + "theoretical_loss": 3.6151818634431354, + "tokens_seen": 1202716672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005136208625877634, + "loss": 1.356, + "theoretical_loss": 3.615163893841233, + "tokens_seen": 1202782208 + }, + { + "epoch": 0.36, + "learning_rate": 0.00051360481444333, + "loss": 1.5441, + "theoretical_loss": 3.6151459254925484, + "tokens_seen": 1202847744 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005135887662988968, + "loss": 1.2836, + "theoretical_loss": 3.6151279583969247, + "tokens_seen": 1202913280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005135727181544634, + "loss": 1.1651, + "theoretical_loss": 3.6151099925542063, + "tokens_seen": 1202978816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005135566700100302, + "loss": 1.4052, + "theoretical_loss": 3.6150920279642373, + "tokens_seen": 1203044352 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005135406218655968, + "loss": 1.3768, + "theoretical_loss": 3.615074064626863, + "tokens_seen": 1203109888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005135245737211636, + "loss": 1.3209, + "theoretical_loss": 3.615056102541928, + "tokens_seen": 1203175424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005135085255767302, + "loss": 1.3026, + "theoretical_loss": 3.6150381417092756, + "tokens_seen": 1203240960 + }, + { + "epoch": 0.36, + "learning_rate": 0.000513492477432297, + "loss": 1.3266, + "theoretical_loss": 3.6150201821287506, + "tokens_seen": 1203306496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005134764292878636, + "loss": 1.3884, + "theoretical_loss": 3.6150022238001984, + "tokens_seen": 1203372032 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005134603811434304, + "loss": 1.2748, + "theoretical_loss": 3.6149842667234626, + "tokens_seen": 1203437568 + }, + { + "epoch": 0.36, + "learning_rate": 0.000513444332998997, + "loss": 1.2208, + "theoretical_loss": 3.6149663108983887, + "tokens_seen": 1203503104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005134282848545638, + "loss": 1.4069, + "theoretical_loss": 3.6149483563248204, + "tokens_seen": 1203568640 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005134122367101304, + "loss": 1.2371, + "theoretical_loss": 3.614930403002602, + "tokens_seen": 1203634176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005133961885656972, + "loss": 1.2641, + "theoretical_loss": 3.61491245093158, + "tokens_seen": 1203699712 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005133801404212638, + "loss": 1.2983, + "theoretical_loss": 3.614894500111597, + "tokens_seen": 1203765248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005133640922768306, + "loss": 1.4068, + "theoretical_loss": 3.6148765505424993, + "tokens_seen": 1203830784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005133480441323972, + "loss": 1.2553, + "theoretical_loss": 3.614858602224131, + "tokens_seen": 1203896320 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005133319959879639, + "loss": 1.3769, + "theoretical_loss": 3.6148406551563363, + "tokens_seen": 1203961856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005133159478435306, + "loss": 1.3133, + "theoretical_loss": 3.614822709338961, + "tokens_seen": 1204027392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005132998996990973, + "loss": 1.4305, + "theoretical_loss": 3.6148047647718493, + "tokens_seen": 1204092928 + }, + { + "epoch": 0.36, + "learning_rate": 0.000513283851554664, + "loss": 1.3142, + "theoretical_loss": 3.614786821454846, + "tokens_seen": 1204158464 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005132678034102307, + "loss": 1.2143, + "theoretical_loss": 3.6147688793877966, + "tokens_seen": 1204224000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005132517552657974, + "loss": 1.2778, + "theoretical_loss": 3.614750938570545, + "tokens_seen": 1204289536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005132357071213641, + "loss": 1.286, + "theoretical_loss": 3.6147329990029373, + "tokens_seen": 1204355072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005132196589769308, + "loss": 1.3337, + "theoretical_loss": 3.6147150606848184, + "tokens_seen": 1204420608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005132036108324975, + "loss": 1.3435, + "theoretical_loss": 3.6146971236160317, + "tokens_seen": 1204486144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005131875626880641, + "loss": 1.4116, + "theoretical_loss": 3.6146791877964244, + "tokens_seen": 1204551680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005131715145436309, + "loss": 1.2683, + "theoretical_loss": 3.6146612532258398, + "tokens_seen": 1204617216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005131554663991975, + "loss": 1.3302, + "theoretical_loss": 3.6146433199041237, + "tokens_seen": 1204682752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005131394182547643, + "loss": 1.3733, + "theoretical_loss": 3.614625387831121, + "tokens_seen": 1204748288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005131233701103309, + "loss": 1.4139, + "theoretical_loss": 3.6146074570066773, + "tokens_seen": 1204813824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005131073219658977, + "loss": 1.3556, + "theoretical_loss": 3.6145895274306374, + "tokens_seen": 1204879360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005130912738214643, + "loss": 1.3724, + "theoretical_loss": 3.6145715991028466, + "tokens_seen": 1204944896 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005130752256770311, + "loss": 1.2736, + "theoretical_loss": 3.61455367202315, + "tokens_seen": 1205010432 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005130591775325977, + "loss": 1.4024, + "theoretical_loss": 3.614535746191393, + "tokens_seen": 1205075968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005130431293881645, + "loss": 1.3528, + "theoretical_loss": 3.6145178216074205, + "tokens_seen": 1205141504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005130270812437311, + "loss": 1.3643, + "theoretical_loss": 3.6144998982710783, + "tokens_seen": 1205207040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005130110330992979, + "loss": 1.3988, + "theoretical_loss": 3.614481976182211, + "tokens_seen": 1205272576 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005129949849548645, + "loss": 1.3279, + "theoretical_loss": 3.6144640553406653, + "tokens_seen": 1205338112 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005129789368104313, + "loss": 1.3316, + "theoretical_loss": 3.614446135746285, + "tokens_seen": 1205403648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005129628886659979, + "loss": 1.4332, + "theoretical_loss": 3.6144282173989164, + "tokens_seen": 1205469184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005129468405215647, + "loss": 1.3009, + "theoretical_loss": 3.6144103002984047, + "tokens_seen": 1205534720 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005129307923771313, + "loss": 1.4368, + "theoretical_loss": 3.6143923844445958, + "tokens_seen": 1205600256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005129147442326981, + "loss": 1.3376, + "theoretical_loss": 3.614374469837334, + "tokens_seen": 1205665792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005128986960882648, + "loss": 1.4212, + "theoretical_loss": 3.6143565564764666, + "tokens_seen": 1205731328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005128826479438315, + "loss": 1.3553, + "theoretical_loss": 3.6143386443618373, + "tokens_seen": 1205796864 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 405583, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.383839726448059, + "objective/train/theoretical_loss": 3.614320733493293, + "objective/train/tokens_used": 1226322400, + "theoretical_loss": 3.614320733493293, + "tokens_seen": 1205862400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005128665997993983, + "loss": 1.3701, + "theoretical_loss": 3.614320733493293, + "tokens_seen": 1205862400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005128505516549649, + "loss": 1.3188, + "theoretical_loss": 3.6143028238706787, + "tokens_seen": 1205927936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005128345035105317, + "loss": 1.329, + "theoretical_loss": 3.61428491549384, + "tokens_seen": 1205993472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005128184553660983, + "loss": 1.3674, + "theoretical_loss": 3.6142670083626225, + "tokens_seen": 1206059008 + }, + { + "epoch": 0.37, + "learning_rate": 0.000512802407221665, + "loss": 1.3181, + "theoretical_loss": 3.6142491024768724, + "tokens_seen": 1206124544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005127863590772317, + "loss": 1.3222, + "theoretical_loss": 3.6142311978364354, + "tokens_seen": 1206190080 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005127703109327985, + "loss": 1.3119, + "theoretical_loss": 3.6142132944411562, + "tokens_seen": 1206255616 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005127542627883651, + "loss": 1.3041, + "theoretical_loss": 3.614195392290882, + "tokens_seen": 1206321152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005127382146439319, + "loss": 1.3166, + "theoretical_loss": 3.6141774913854574, + "tokens_seen": 1206386688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005127221664994985, + "loss": 1.434, + "theoretical_loss": 3.6141595917247296, + "tokens_seen": 1206452224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005127061183550653, + "loss": 1.3401, + "theoretical_loss": 3.6141416933085426, + "tokens_seen": 1206517760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005126900702106319, + "loss": 1.2208, + "theoretical_loss": 3.6141237961367434, + "tokens_seen": 1206583296 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005126740220661987, + "loss": 1.4068, + "theoretical_loss": 3.6141059002091778, + "tokens_seen": 1206648832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005126579739217653, + "loss": 1.2782, + "theoretical_loss": 3.614088005525692, + "tokens_seen": 1206714368 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005126419257773321, + "loss": 1.3517, + "theoretical_loss": 3.6140701120861314, + "tokens_seen": 1206779904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005126258776328987, + "loss": 1.3423, + "theoretical_loss": 3.6140522198903424, + "tokens_seen": 1206845440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005126098294884655, + "loss": 1.3735, + "theoretical_loss": 3.6140343289381702, + "tokens_seen": 1206910976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005125937813440321, + "loss": 1.4779, + "theoretical_loss": 3.614016439229462, + "tokens_seen": 1206976512 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005125777331995989, + "loss": 1.4023, + "theoretical_loss": 3.6139985507640633, + "tokens_seen": 1207042048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005125616850551655, + "loss": 1.3906, + "theoretical_loss": 3.61398066354182, + "tokens_seen": 1207107584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005125456369107323, + "loss": 1.4067, + "theoretical_loss": 3.6139627775625787, + "tokens_seen": 1207173120 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005125295887662989, + "loss": 1.4119, + "theoretical_loss": 3.6139448928261846, + "tokens_seen": 1207238656 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005125135406218657, + "loss": 1.4725, + "theoretical_loss": 3.6139270093324853, + "tokens_seen": 1207304192 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005124974924774323, + "loss": 1.3313, + "theoretical_loss": 3.6139091270813264, + "tokens_seen": 1207369728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005124814443329991, + "loss": 1.3469, + "theoretical_loss": 3.613891246072553, + "tokens_seen": 1207435264 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005124653961885657, + "loss": 1.3837, + "theoretical_loss": 3.613873366306013, + "tokens_seen": 1207500800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005124493480441325, + "loss": 1.4254, + "theoretical_loss": 3.613855487781552, + "tokens_seen": 1207566336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005124332998996991, + "loss": 1.3958, + "theoretical_loss": 3.6138376104990164, + "tokens_seen": 1207631872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005124172517552659, + "loss": 1.4596, + "theoretical_loss": 3.6138197344582528, + "tokens_seen": 1207697408 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005124012036108325, + "loss": 1.4737, + "theoretical_loss": 3.613801859659107, + "tokens_seen": 1207762944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005123851554663993, + "loss": 1.3003, + "theoretical_loss": 3.613783986101425, + "tokens_seen": 1207828480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005123691073219659, + "loss": 1.2779, + "theoretical_loss": 3.6137661137850543, + "tokens_seen": 1207894016 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005123530591775327, + "loss": 1.3878, + "theoretical_loss": 3.6137482427098417, + "tokens_seen": 1207959552 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005123370110330993, + "loss": 1.392, + "theoretical_loss": 3.613730372875632, + "tokens_seen": 1208025088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005123209628886661, + "loss": 1.3946, + "theoretical_loss": 3.613712504282273, + "tokens_seen": 1208090624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005123049147442327, + "loss": 1.3279, + "theoretical_loss": 3.6136946369296106, + "tokens_seen": 1208156160 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005122888665997995, + "loss": 1.2981, + "theoretical_loss": 3.6136767708174915, + "tokens_seen": 1208221696 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005122728184553661, + "loss": 1.3533, + "theoretical_loss": 3.613658905945763, + "tokens_seen": 1208287232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005122567703109329, + "loss": 1.3687, + "theoretical_loss": 3.61364104231427, + "tokens_seen": 1208352768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005122407221664995, + "loss": 1.3565, + "theoretical_loss": 3.613623179922861, + "tokens_seen": 1208418304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005122246740220663, + "loss": 1.3479, + "theoretical_loss": 3.613605318771382, + "tokens_seen": 1208483840 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005122086258776329, + "loss": 1.3981, + "theoretical_loss": 3.613587458859679, + "tokens_seen": 1208549376 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005121925777331997, + "loss": 1.4024, + "theoretical_loss": 3.6135696001876, + "tokens_seen": 1208614912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005121765295887663, + "loss": 1.2919, + "theoretical_loss": 3.613551742754991, + "tokens_seen": 1208680448 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005121604814443331, + "loss": 1.2458, + "theoretical_loss": 3.6135338865616986, + "tokens_seen": 1208745984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005121444332998997, + "loss": 1.3452, + "theoretical_loss": 3.6135160316075696, + "tokens_seen": 1208811520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005121283851554665, + "loss": 1.3305, + "theoretical_loss": 3.6134981778924518, + "tokens_seen": 1208877056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005121123370110331, + "loss": 1.3028, + "theoretical_loss": 3.613480325416191, + "tokens_seen": 1208942592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005120962888665999, + "loss": 1.421, + "theoretical_loss": 3.613462474178634, + "tokens_seen": 1209008128 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005120802407221665, + "loss": 1.3711, + "theoretical_loss": 3.6134446241796283, + "tokens_seen": 1209073664 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 406414, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3190600872039795, + "objective/train/theoretical_loss": 3.613426775419021, + "objective/train/tokens_used": 1229599200, + "theoretical_loss": 3.613426775419021, + "tokens_seen": 1209139200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005120641925777333, + "loss": 1.3376, + "theoretical_loss": 3.613426775419021, + "tokens_seen": 1209139200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005120481444332999, + "loss": 1.4591, + "theoretical_loss": 3.6134089278966584, + "tokens_seen": 1209204736 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005120320962888667, + "loss": 1.4646, + "theoretical_loss": 3.613391081612388, + "tokens_seen": 1209270272 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005120160481444333, + "loss": 1.3262, + "theoretical_loss": 3.6133732365660567, + "tokens_seen": 1209335808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005120000000000001, + "loss": 1.4101, + "theoretical_loss": 3.6133553927575113, + "tokens_seen": 1209401344 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005119839518555667, + "loss": 1.3722, + "theoretical_loss": 3.6133375501865994, + "tokens_seen": 1209466880 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005119679037111335, + "loss": 1.4411, + "theoretical_loss": 3.613319708853168, + "tokens_seen": 1209532416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005119518555667001, + "loss": 1.4105, + "theoretical_loss": 3.6133018687570635, + "tokens_seen": 1209597952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005119358074222668, + "loss": 1.3581, + "theoretical_loss": 3.613284029898134, + "tokens_seen": 1209663488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005119197592778335, + "loss": 1.2052, + "theoretical_loss": 3.613266192276226, + "tokens_seen": 1209729024 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005119037111334002, + "loss": 1.3657, + "theoretical_loss": 3.613248355891187, + "tokens_seen": 1209794560 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005118876629889669, + "loss": 1.4047, + "theoretical_loss": 3.613230520742864, + "tokens_seen": 1209860096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005118716148445336, + "loss": 1.3488, + "theoretical_loss": 3.6132126868311047, + "tokens_seen": 1209925632 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005118555667001003, + "loss": 1.3686, + "theoretical_loss": 3.6131948541557564, + "tokens_seen": 1209991168 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511839518555667, + "loss": 1.325, + "theoretical_loss": 3.613177022716666, + "tokens_seen": 1210056704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005118234704112337, + "loss": 1.4054, + "theoretical_loss": 3.613159192513681, + "tokens_seen": 1210122240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005118074222668004, + "loss": 1.369, + "theoretical_loss": 3.6131413635466485, + "tokens_seen": 1210187776 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511791374122367, + "loss": 1.3287, + "theoretical_loss": 3.613123535815417, + "tokens_seen": 1210253312 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005117753259779338, + "loss": 1.3342, + "theoretical_loss": 3.6131057093198327, + "tokens_seen": 1210318848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005117592778335004, + "loss": 1.4553, + "theoretical_loss": 3.613087884059744, + "tokens_seen": 1210384384 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005117432296890672, + "loss": 1.3799, + "theoretical_loss": 3.613070060034997, + "tokens_seen": 1210449920 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005117271815446338, + "loss": 1.4055, + "theoretical_loss": 3.613052237245441, + "tokens_seen": 1210515456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005117111334002006, + "loss": 1.4076, + "theoretical_loss": 3.6130344156909215, + "tokens_seen": 1210580992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005116950852557673, + "loss": 1.301, + "theoretical_loss": 3.6130165953712883, + "tokens_seen": 1210646528 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511679037111334, + "loss": 1.4515, + "theoretical_loss": 3.6129987762863873, + "tokens_seen": 1210712064 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005116629889669007, + "loss": 1.4408, + "theoretical_loss": 3.612980958436067, + "tokens_seen": 1210777600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005116469408224674, + "loss": 1.4515, + "theoretical_loss": 3.6129631418201744, + "tokens_seen": 1210843136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005116308926780341, + "loss": 1.3465, + "theoretical_loss": 3.612945326438558, + "tokens_seen": 1210908672 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005116148445336008, + "loss": 1.2689, + "theoretical_loss": 3.6129275122910647, + "tokens_seen": 1210974208 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005115987963891675, + "loss": 1.4271, + "theoretical_loss": 3.6129096993775427, + "tokens_seen": 1211039744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005115827482447342, + "loss": 1.3598, + "theoretical_loss": 3.6128918876978395, + "tokens_seen": 1211105280 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511566700100301, + "loss": 1.2845, + "theoretical_loss": 3.612874077251803, + "tokens_seen": 1211170816 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005115506519558676, + "loss": 1.4193, + "theoretical_loss": 3.6128562680392813, + "tokens_seen": 1211236352 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005115346038114343, + "loss": 1.31, + "theoretical_loss": 3.6128384600601215, + "tokens_seen": 1211301888 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511518555667001, + "loss": 1.3781, + "theoretical_loss": 3.6128206533141722, + "tokens_seen": 1211367424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005115025075225678, + "loss": 1.3714, + "theoretical_loss": 3.612802847801281, + "tokens_seen": 1211432960 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005114864593781344, + "loss": 1.3739, + "theoretical_loss": 3.6127850435212956, + "tokens_seen": 1211498496 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005114704112337012, + "loss": 1.4279, + "theoretical_loss": 3.612767240474064, + "tokens_seen": 1211564032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005114543630892678, + "loss": 1.3829, + "theoretical_loss": 3.612749438659435, + "tokens_seen": 1211629568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005114383149448346, + "loss": 1.45, + "theoretical_loss": 3.612731638077255, + "tokens_seen": 1211695104 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005114222668004012, + "loss": 1.3736, + "theoretical_loss": 3.612713838727373, + "tokens_seen": 1211760640 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511406218655968, + "loss": 1.3915, + "theoretical_loss": 3.612696040609638, + "tokens_seen": 1211826176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005113901705115346, + "loss": 1.2323, + "theoretical_loss": 3.612678243723896, + "tokens_seen": 1211891712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005113741223671014, + "loss": 1.2744, + "theoretical_loss": 3.612660448069997, + "tokens_seen": 1211957248 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511358074222668, + "loss": 1.3926, + "theoretical_loss": 3.6126426536477876, + "tokens_seen": 1212022784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005113420260782348, + "loss": 1.4082, + "theoretical_loss": 3.6126248604571174, + "tokens_seen": 1212088320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005113259779338014, + "loss": 1.3457, + "theoretical_loss": 3.6126070684978333, + "tokens_seen": 1212153856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005113099297893682, + "loss": 1.2181, + "theoretical_loss": 3.612589277769784, + "tokens_seen": 1212219392 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005112938816449348, + "loss": 1.3968, + "theoretical_loss": 3.612571488272818, + "tokens_seen": 1212284928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005112778335005016, + "loss": 1.4058, + "theoretical_loss": 3.6125537000067833, + "tokens_seen": 1212350464 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 407314, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.369948148727417, + "objective/train/theoretical_loss": 3.6125359129715284, + "objective/train/tokens_used": 1232876000, + "theoretical_loss": 3.6125359129715284, + "tokens_seen": 1212416000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005112617853560682, + "loss": 1.3099, + "theoretical_loss": 3.6125359129715284, + "tokens_seen": 1212416000 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511245737211635, + "loss": 1.3631, + "theoretical_loss": 3.6125181271669016, + "tokens_seen": 1212481536 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005112296890672016, + "loss": 1.3632, + "theoretical_loss": 3.6125003425927504, + "tokens_seen": 1212547072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005112136409227684, + "loss": 1.2103, + "theoretical_loss": 3.6124825592489245, + "tokens_seen": 1212612608 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511197592778335, + "loss": 1.3042, + "theoretical_loss": 3.612464777135272, + "tokens_seen": 1212678144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005111815446339018, + "loss": 1.3234, + "theoretical_loss": 3.6124469962516406, + "tokens_seen": 1212743680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005111654964894684, + "loss": 1.4118, + "theoretical_loss": 3.612429216597879, + "tokens_seen": 1212809216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005111494483450352, + "loss": 1.3437, + "theoretical_loss": 3.6124114381738357, + "tokens_seen": 1212874752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005111334002006018, + "loss": 1.3639, + "theoretical_loss": 3.61239366097936, + "tokens_seen": 1212940288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005111173520561686, + "loss": 1.3357, + "theoretical_loss": 3.6123758850142993, + "tokens_seen": 1213005824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005111013039117352, + "loss": 1.3217, + "theoretical_loss": 3.612358110278503, + "tokens_seen": 1213071360 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511085255767302, + "loss": 1.3236, + "theoretical_loss": 3.612340336771819, + "tokens_seen": 1213136896 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005110692076228686, + "loss": 1.2687, + "theoretical_loss": 3.6123225644940966, + "tokens_seen": 1213202432 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005110531594784354, + "loss": 1.3901, + "theoretical_loss": 3.612304793445184, + "tokens_seen": 1213267968 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511037111334002, + "loss": 1.4303, + "theoretical_loss": 3.61228702362493, + "tokens_seen": 1213333504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005110210631895688, + "loss": 1.2671, + "theoretical_loss": 3.6122692550331834, + "tokens_seen": 1213399040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005110050150451354, + "loss": 1.337, + "theoretical_loss": 3.612251487669792, + "tokens_seen": 1213464576 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005109889669007022, + "loss": 1.3473, + "theoretical_loss": 3.6122337215346065, + "tokens_seen": 1213530112 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005109729187562688, + "loss": 1.39, + "theoretical_loss": 3.612215956627474, + "tokens_seen": 1213595648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005109568706118356, + "loss": 1.3999, + "theoretical_loss": 3.612198192948244, + "tokens_seen": 1213661184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005109408224674022, + "loss": 1.3007, + "theoretical_loss": 3.612180430496765, + "tokens_seen": 1213726720 + }, + { + "epoch": 0.37, + "learning_rate": 0.000510924774322969, + "loss": 1.3578, + "theoretical_loss": 3.6121626692728857, + "tokens_seen": 1213792256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005109087261785356, + "loss": 1.2689, + "theoretical_loss": 3.6121449092764557, + "tokens_seen": 1213857792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005108926780341024, + "loss": 1.2337, + "theoretical_loss": 3.6121271505073236, + "tokens_seen": 1213923328 + }, + { + "epoch": 0.37, + "learning_rate": 0.000510876629889669, + "loss": 1.384, + "theoretical_loss": 3.612109392965338, + "tokens_seen": 1213988864 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005108605817452358, + "loss": 1.3511, + "theoretical_loss": 3.612091636650348, + "tokens_seen": 1214054400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005108445336008024, + "loss": 1.3522, + "theoretical_loss": 3.6120738815622033, + "tokens_seen": 1214119936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005108284854563692, + "loss": 1.2782, + "theoretical_loss": 3.6120561277007517, + "tokens_seen": 1214185472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005108124373119358, + "loss": 1.347, + "theoretical_loss": 3.612038375065843, + "tokens_seen": 1214251008 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005107963891675026, + "loss": 1.322, + "theoretical_loss": 3.612020623657326, + "tokens_seen": 1214316544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005107803410230692, + "loss": 1.3969, + "theoretical_loss": 3.61200287347505, + "tokens_seen": 1214382080 + }, + { + "epoch": 0.37, + "learning_rate": 0.000510764292878636, + "loss": 1.4987, + "theoretical_loss": 3.6119851245188643, + "tokens_seen": 1214447616 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005107482447342026, + "loss": 1.2335, + "theoretical_loss": 3.6119673767886176, + "tokens_seen": 1214513152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005107321965897694, + "loss": 1.374, + "theoretical_loss": 3.611949630284159, + "tokens_seen": 1214578688 + }, + { + "epoch": 0.37, + "learning_rate": 0.000510716148445336, + "loss": 1.4743, + "theoretical_loss": 3.6119318850053386, + "tokens_seen": 1214644224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005107001003009028, + "loss": 1.3383, + "theoretical_loss": 3.611914140952005, + "tokens_seen": 1214709760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005106840521564694, + "loss": 1.3486, + "theoretical_loss": 3.6118963981240073, + "tokens_seen": 1214775296 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005106680040120362, + "loss": 1.4368, + "theoretical_loss": 3.6118786565211947, + "tokens_seen": 1214840832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005106519558676028, + "loss": 1.3773, + "theoretical_loss": 3.6118609161434168, + "tokens_seen": 1214906368 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005106359077231696, + "loss": 1.4838, + "theoretical_loss": 3.611843176990523, + "tokens_seen": 1214971904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005106198595787362, + "loss": 1.4517, + "theoretical_loss": 3.6118254390623625, + "tokens_seen": 1215037440 + }, + { + "epoch": 0.37, + "learning_rate": 0.000510603811434303, + "loss": 1.278, + "theoretical_loss": 3.611807702358785, + "tokens_seen": 1215102976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005105877632898696, + "loss": 1.3935, + "theoretical_loss": 3.6117899668796394, + "tokens_seen": 1215168512 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005105717151454364, + "loss": 1.2404, + "theoretical_loss": 3.6117722326247756, + "tokens_seen": 1215234048 + }, + { + "epoch": 0.37, + "learning_rate": 0.000510555667001003, + "loss": 1.4078, + "theoretical_loss": 3.611754499594043, + "tokens_seen": 1215299584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005105396188565698, + "loss": 1.3754, + "theoretical_loss": 3.6117367677872907, + "tokens_seen": 1215365120 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005105235707121364, + "loss": 1.334, + "theoretical_loss": 3.6117190372043684, + "tokens_seen": 1215430656 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005105075225677031, + "loss": 1.2575, + "theoretical_loss": 3.611701307845126, + "tokens_seen": 1215496192 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005104914744232698, + "loss": 1.3585, + "theoretical_loss": 3.611683579709413, + "tokens_seen": 1215561728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005104754262788365, + "loss": 1.2716, + "theoretical_loss": 3.6116658527970786, + "tokens_seen": 1215627264 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 408591, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5982543230056763, + "objective/train/theoretical_loss": 3.611648127107973, + "objective/train/tokens_used": 1236152800, + "theoretical_loss": 3.611648127107973, + "tokens_seen": 1215692800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005104593781344032, + "loss": 1.5032, + "theoretical_loss": 3.611648127107973, + "tokens_seen": 1215692800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005104433299899699, + "loss": 1.345, + "theoretical_loss": 3.6116304026419446, + "tokens_seen": 1215758336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005104272818455366, + "loss": 1.3119, + "theoretical_loss": 3.611612679398845, + "tokens_seen": 1215823872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005104112337011033, + "loss": 1.3545, + "theoretical_loss": 3.6115949573785224, + "tokens_seen": 1215889408 + }, + { + "epoch": 0.37, + "learning_rate": 0.00051039518555667, + "loss": 1.4228, + "theoretical_loss": 3.6115772365808274, + "tokens_seen": 1215954944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005103791374122367, + "loss": 1.3251, + "theoretical_loss": 3.611559517005609, + "tokens_seen": 1216020480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005103630892678034, + "loss": 1.4169, + "theoretical_loss": 3.611541798652718, + "tokens_seen": 1216086016 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005103470411233701, + "loss": 1.4134, + "theoretical_loss": 3.6115240815220035, + "tokens_seen": 1216151552 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005103309929789368, + "loss": 1.3452, + "theoretical_loss": 3.611506365613315, + "tokens_seen": 1216217088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005103149448345035, + "loss": 1.5155, + "theoretical_loss": 3.6114886509265034, + "tokens_seen": 1216282624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005102988966900702, + "loss": 1.4263, + "theoretical_loss": 3.6114709374614176, + "tokens_seen": 1216348160 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005102828485456369, + "loss": 1.4095, + "theoretical_loss": 3.611453225217908, + "tokens_seen": 1216413696 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005102668004012036, + "loss": 1.3857, + "theoretical_loss": 3.6114355141958248, + "tokens_seen": 1216479232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005102507522567703, + "loss": 1.495, + "theoretical_loss": 3.6114178043950176, + "tokens_seen": 1216544768 + }, + { + "epoch": 0.37, + "learning_rate": 0.000510234704112337, + "loss": 1.2786, + "theoretical_loss": 3.6114000958153363, + "tokens_seen": 1216610304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005102186559679037, + "loss": 1.253, + "theoretical_loss": 3.611382388456631, + "tokens_seen": 1216675840 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005102026078234704, + "loss": 1.4706, + "theoretical_loss": 3.611364682318752, + "tokens_seen": 1216741376 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005101865596790371, + "loss": 1.4155, + "theoretical_loss": 3.611346977401549, + "tokens_seen": 1216806912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005101705115346038, + "loss": 1.4166, + "theoretical_loss": 3.6113292737048734, + "tokens_seen": 1216872448 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005101544633901705, + "loss": 1.3025, + "theoretical_loss": 3.6113115712285735, + "tokens_seen": 1216937984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005101384152457373, + "loss": 1.3737, + "theoretical_loss": 3.6112938699725, + "tokens_seen": 1217003520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005101223671013039, + "loss": 1.4429, + "theoretical_loss": 3.611276169936504, + "tokens_seen": 1217069056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005101063189568707, + "loss": 1.4673, + "theoretical_loss": 3.6112584711204345, + "tokens_seen": 1217134592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005100902708124373, + "loss": 1.3303, + "theoretical_loss": 3.611240773524143, + "tokens_seen": 1217200128 + }, + { + "epoch": 0.37, + "learning_rate": 0.000510074222668004, + "loss": 1.3982, + "theoretical_loss": 3.6112230771474785, + "tokens_seen": 1217265664 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005100581745235707, + "loss": 1.3113, + "theoretical_loss": 3.611205381990292, + "tokens_seen": 1217331200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005100421263791375, + "loss": 1.3865, + "theoretical_loss": 3.611187688052433, + "tokens_seen": 1217396736 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005100260782347041, + "loss": 1.3629, + "theoretical_loss": 3.6111699953337535, + "tokens_seen": 1217462272 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005100100300902709, + "loss": 1.3549, + "theoretical_loss": 3.6111523038341025, + "tokens_seen": 1217527808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005099939819458375, + "loss": 1.2067, + "theoretical_loss": 3.6111346135533307, + "tokens_seen": 1217593344 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005099779338014043, + "loss": 1.2571, + "theoretical_loss": 3.611116924491289, + "tokens_seen": 1217658880 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005099618856569709, + "loss": 1.2841, + "theoretical_loss": 3.611099236647827, + "tokens_seen": 1217724416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005099458375125377, + "loss": 1.321, + "theoretical_loss": 3.6110815500227957, + "tokens_seen": 1217789952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005099297893681043, + "loss": 1.5426, + "theoretical_loss": 3.611063864616046, + "tokens_seen": 1217855488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005099137412236711, + "loss": 1.2651, + "theoretical_loss": 3.6110461804274276, + "tokens_seen": 1217921024 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005098976930792377, + "loss": 1.4044, + "theoretical_loss": 3.6110284974567914, + "tokens_seen": 1217986560 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005098816449348045, + "loss": 1.2246, + "theoretical_loss": 3.611010815703988, + "tokens_seen": 1218052096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005098655967903711, + "loss": 1.2565, + "theoretical_loss": 3.610993135168868, + "tokens_seen": 1218117632 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005098495486459379, + "loss": 1.4739, + "theoretical_loss": 3.6109754558512828, + "tokens_seen": 1218183168 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005098335005015045, + "loss": 1.4314, + "theoretical_loss": 3.6109577777510813, + "tokens_seen": 1218248704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005098174523570713, + "loss": 1.4238, + "theoretical_loss": 3.610940100868116, + "tokens_seen": 1218314240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005098014042126379, + "loss": 1.4001, + "theoretical_loss": 3.6109224252022365, + "tokens_seen": 1218379776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005097853560682047, + "loss": 1.2788, + "theoretical_loss": 3.610904750753294, + "tokens_seen": 1218445312 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005097693079237713, + "loss": 1.3308, + "theoretical_loss": 3.610887077521139, + "tokens_seen": 1218510848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005097532597793381, + "loss": 1.2774, + "theoretical_loss": 3.610869405505622, + "tokens_seen": 1218576384 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005097372116349047, + "loss": 1.3115, + "theoretical_loss": 3.6108517347065954, + "tokens_seen": 1218641920 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005097211634904715, + "loss": 1.392, + "theoretical_loss": 3.610834065123908, + "tokens_seen": 1218707456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005097051153460381, + "loss": 1.3531, + "theoretical_loss": 3.610816396757412, + "tokens_seen": 1218772992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005096890672016049, + "loss": 1.3569, + "theoretical_loss": 3.610798729606958, + "tokens_seen": 1218838528 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005096730190571715, + "loss": 1.3418, + "theoretical_loss": 3.6107810636723965, + "tokens_seen": 1218904064 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 409566, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3452516794204712, + "objective/train/theoretical_loss": 3.6107633989535786, + "objective/train/tokens_used": 1239429600, + "theoretical_loss": 3.6107633989535786, + "tokens_seen": 1218969600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005096569709127383, + "loss": 1.4007, + "theoretical_loss": 3.6107633989535786, + "tokens_seen": 1218969600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005096409227683049, + "loss": 1.3499, + "theoretical_loss": 3.6107457354503554, + "tokens_seen": 1219035136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005096248746238717, + "loss": 1.4341, + "theoretical_loss": 3.610728073162578, + "tokens_seen": 1219100672 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005096088264794383, + "loss": 1.3204, + "theoretical_loss": 3.610710412090098, + "tokens_seen": 1219166208 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005095927783350051, + "loss": 1.3421, + "theoretical_loss": 3.610692752232765, + "tokens_seen": 1219231744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005095767301905717, + "loss": 1.2549, + "theoretical_loss": 3.6106750935904315, + "tokens_seen": 1219297280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005095606820461385, + "loss": 1.3546, + "theoretical_loss": 3.610657436162948, + "tokens_seen": 1219362816 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005095446339017051, + "loss": 1.2459, + "theoretical_loss": 3.6106397799501653, + "tokens_seen": 1219428352 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005095285857572719, + "loss": 1.3483, + "theoretical_loss": 3.6106221249519352, + "tokens_seen": 1219493888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005095125376128385, + "loss": 1.5332, + "theoretical_loss": 3.6106044711681085, + "tokens_seen": 1219559424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005094964894684053, + "loss": 1.4287, + "theoretical_loss": 3.6105868185985366, + "tokens_seen": 1219624960 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005094804413239719, + "loss": 1.3836, + "theoretical_loss": 3.6105691672430704, + "tokens_seen": 1219690496 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005094643931795387, + "loss": 1.3473, + "theoretical_loss": 3.6105515171015616, + "tokens_seen": 1219756032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005094483450351053, + "loss": 1.2934, + "theoretical_loss": 3.6105338681738615, + "tokens_seen": 1219821568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005094322968906721, + "loss": 1.3215, + "theoretical_loss": 3.610516220459821, + "tokens_seen": 1219887104 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005094162487462387, + "loss": 1.3114, + "theoretical_loss": 3.610498573959292, + "tokens_seen": 1219952640 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005094002006018055, + "loss": 1.3689, + "theoretical_loss": 3.610480928672125, + "tokens_seen": 1220018176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005093841524573721, + "loss": 1.396, + "theoretical_loss": 3.610463284598173, + "tokens_seen": 1220083712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005093681043129389, + "loss": 1.3632, + "theoretical_loss": 3.610445641737285, + "tokens_seen": 1220149248 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005093520561685055, + "loss": 1.3965, + "theoretical_loss": 3.610428000089315, + "tokens_seen": 1220214784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005093360080240723, + "loss": 1.3503, + "theoretical_loss": 3.6104103596541126, + "tokens_seen": 1220280320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005093199598796389, + "loss": 1.3583, + "theoretical_loss": 3.6103927204315305, + "tokens_seen": 1220345856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005093039117352057, + "loss": 1.3688, + "theoretical_loss": 3.6103750824214194, + "tokens_seen": 1220411392 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005092878635907724, + "loss": 1.3448, + "theoretical_loss": 3.6103574456236314, + "tokens_seen": 1220476928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005092718154463391, + "loss": 1.3285, + "theoretical_loss": 3.6103398100380177, + "tokens_seen": 1220542464 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005092557673019057, + "loss": 1.36, + "theoretical_loss": 3.61032217566443, + "tokens_seen": 1220608000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005092397191574725, + "loss": 1.3331, + "theoretical_loss": 3.6103045425027203, + "tokens_seen": 1220673536 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005092236710130391, + "loss": 1.4157, + "theoretical_loss": 3.61028691055274, + "tokens_seen": 1220739072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005092076228686059, + "loss": 1.3788, + "theoretical_loss": 3.6102692798143408, + "tokens_seen": 1220804608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005091915747241725, + "loss": 1.3119, + "theoretical_loss": 3.610251650287374, + "tokens_seen": 1220870144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005091755265797393, + "loss": 1.3138, + "theoretical_loss": 3.610234021971692, + "tokens_seen": 1220935680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005091594784353059, + "loss": 1.386, + "theoretical_loss": 3.6102163948671464, + "tokens_seen": 1221001216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005091434302908727, + "loss": 1.4066, + "theoretical_loss": 3.6101987689735884, + "tokens_seen": 1221066752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005091273821464393, + "loss": 1.3648, + "theoretical_loss": 3.6101811442908707, + "tokens_seen": 1221132288 + }, + { + "epoch": 0.37, + "learning_rate": 0.000509111334002006, + "loss": 1.4272, + "theoretical_loss": 3.610163520818845, + "tokens_seen": 1221197824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005090952858575727, + "loss": 1.3886, + "theoretical_loss": 3.6101458985573625, + "tokens_seen": 1221263360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005090792377131394, + "loss": 1.4172, + "theoretical_loss": 3.6101282775062753, + "tokens_seen": 1221328896 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005090631895687061, + "loss": 1.2674, + "theoretical_loss": 3.610110657665436, + "tokens_seen": 1221394432 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005090471414242728, + "loss": 1.4391, + "theoretical_loss": 3.6100930390346955, + "tokens_seen": 1221459968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005090310932798395, + "loss": 1.2714, + "theoretical_loss": 3.610075421613907, + "tokens_seen": 1221525504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005090150451354062, + "loss": 1.3585, + "theoretical_loss": 3.6100578054029215, + "tokens_seen": 1221591040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005089989969909729, + "loss": 1.3147, + "theoretical_loss": 3.610040190401591, + "tokens_seen": 1221656576 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005089829488465396, + "loss": 1.3047, + "theoretical_loss": 3.6100225766097687, + "tokens_seen": 1221722112 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005089669007021063, + "loss": 1.3707, + "theoretical_loss": 3.610004964027305, + "tokens_seen": 1221787648 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508950852557673, + "loss": 1.3458, + "theoretical_loss": 3.609987352654054, + "tokens_seen": 1221853184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005089348044132397, + "loss": 1.252, + "theoretical_loss": 3.6099697424898656, + "tokens_seen": 1221918720 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005089187562688064, + "loss": 1.3129, + "theoretical_loss": 3.609952133534594, + "tokens_seen": 1221984256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005089027081243731, + "loss": 1.3465, + "theoretical_loss": 3.6099345257880895, + "tokens_seen": 1222049792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005088866599799398, + "loss": 1.4346, + "theoretical_loss": 3.609916919250206, + "tokens_seen": 1222115328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005088706118355065, + "loss": 1.271, + "theoretical_loss": 3.6098993139207947, + "tokens_seen": 1222180864 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 410881, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3531014919281006, + "objective/train/theoretical_loss": 3.6098817097997085, + "objective/train/tokens_used": 1242706400, + "theoretical_loss": 3.6098817097997085, + "tokens_seen": 1222246400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005088545636910732, + "loss": 1.4237, + "theoretical_loss": 3.6098817097997085, + "tokens_seen": 1222246400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00050883851554664, + "loss": 1.3529, + "theoretical_loss": 3.609864106886799, + "tokens_seen": 1222311936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005088224674022066, + "loss": 1.2548, + "theoretical_loss": 3.6098465051819186, + "tokens_seen": 1222377472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005088064192577733, + "loss": 1.3921, + "theoretical_loss": 3.60982890468492, + "tokens_seen": 1222443008 + }, + { + "epoch": 0.37, + "learning_rate": 0.00050879037111334, + "loss": 1.4104, + "theoretical_loss": 3.609811305395656, + "tokens_seen": 1222508544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005087743229689068, + "loss": 1.3592, + "theoretical_loss": 3.609793707313978, + "tokens_seen": 1222574080 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005087582748244734, + "loss": 1.326, + "theoretical_loss": 3.609776110439739, + "tokens_seen": 1222639616 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005087422266800402, + "loss": 1.3587, + "theoretical_loss": 3.609758514772791, + "tokens_seen": 1222705152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005087261785356068, + "loss": 1.3131, + "theoretical_loss": 3.609740920312987, + "tokens_seen": 1222770688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005087101303911736, + "loss": 1.4157, + "theoretical_loss": 3.609723327060179, + "tokens_seen": 1222836224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005086940822467402, + "loss": 1.2677, + "theoretical_loss": 3.60970573501422, + "tokens_seen": 1222901760 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508678034102307, + "loss": 1.3885, + "theoretical_loss": 3.6096881441749624, + "tokens_seen": 1222967296 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005086619859578736, + "loss": 1.3491, + "theoretical_loss": 3.6096705545422587, + "tokens_seen": 1223032832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005086459378134404, + "loss": 1.254, + "theoretical_loss": 3.609652966115961, + "tokens_seen": 1223098368 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508629889669007, + "loss": 1.3122, + "theoretical_loss": 3.609635378895923, + "tokens_seen": 1223163904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005086138415245738, + "loss": 1.2439, + "theoretical_loss": 3.6096177928819966, + "tokens_seen": 1223229440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005085977933801404, + "loss": 1.4453, + "theoretical_loss": 3.609600208074035, + "tokens_seen": 1223294976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005085817452357072, + "loss": 1.3788, + "theoretical_loss": 3.60958262447189, + "tokens_seen": 1223360512 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005085656970912738, + "loss": 1.3732, + "theoretical_loss": 3.609565042075414, + "tokens_seen": 1223426048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005085496489468406, + "loss": 1.3837, + "theoretical_loss": 3.609547460884462, + "tokens_seen": 1223491584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005085336008024072, + "loss": 1.2681, + "theoretical_loss": 3.6095298808988847, + "tokens_seen": 1223557120 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508517552657974, + "loss": 1.4613, + "theoretical_loss": 3.609512302118536, + "tokens_seen": 1223622656 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005085015045135406, + "loss": 1.3357, + "theoretical_loss": 3.6094947245432683, + "tokens_seen": 1223688192 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005084854563691074, + "loss": 1.3543, + "theoretical_loss": 3.6094771481729344, + "tokens_seen": 1223753728 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508469408224674, + "loss": 1.3181, + "theoretical_loss": 3.609459573007387, + "tokens_seen": 1223819264 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005084533600802408, + "loss": 1.3672, + "theoretical_loss": 3.609441999046479, + "tokens_seen": 1223884800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005084373119358074, + "loss": 1.3234, + "theoretical_loss": 3.6094244262900643, + "tokens_seen": 1223950336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005084212637913742, + "loss": 1.2581, + "theoretical_loss": 3.609406854737995, + "tokens_seen": 1224015872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005084052156469408, + "loss": 1.3042, + "theoretical_loss": 3.609389284390124, + "tokens_seen": 1224081408 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005083891675025076, + "loss": 1.3076, + "theoretical_loss": 3.6093717152463043, + "tokens_seen": 1224146944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005083731193580742, + "loss": 1.3272, + "theoretical_loss": 3.60935414730639, + "tokens_seen": 1224212480 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508357071213641, + "loss": 1.3615, + "theoretical_loss": 3.6093365805702327, + "tokens_seen": 1224278016 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005083410230692076, + "loss": 1.4112, + "theoretical_loss": 3.609319015037686, + "tokens_seen": 1224343552 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005083249749247744, + "loss": 1.2945, + "theoretical_loss": 3.609301450708603, + "tokens_seen": 1224409088 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508308926780341, + "loss": 1.3688, + "theoretical_loss": 3.609283887582838, + "tokens_seen": 1224474624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005082928786359078, + "loss": 1.4659, + "theoretical_loss": 3.6092663256602426, + "tokens_seen": 1224540160 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005082768304914744, + "loss": 1.371, + "theoretical_loss": 3.60924876494067, + "tokens_seen": 1224605696 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005082607823470412, + "loss": 1.4184, + "theoretical_loss": 3.6092312054239746, + "tokens_seen": 1224671232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005082447342026078, + "loss": 1.344, + "theoretical_loss": 3.6092136471100087, + "tokens_seen": 1224736768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005082286860581746, + "loss": 1.2974, + "theoretical_loss": 3.609196089998626, + "tokens_seen": 1224802304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005082126379137412, + "loss": 1.3757, + "theoretical_loss": 3.6091785340896796, + "tokens_seen": 1224867840 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508196589769308, + "loss": 1.4648, + "theoretical_loss": 3.609160979383023, + "tokens_seen": 1224933376 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005081805416248746, + "loss": 1.286, + "theoretical_loss": 3.609143425878509, + "tokens_seen": 1224998912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005081644934804414, + "loss": 1.3262, + "theoretical_loss": 3.6091258735759917, + "tokens_seen": 1225064448 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508148445336008, + "loss": 1.3932, + "theoretical_loss": 3.609108322475324, + "tokens_seen": 1225129984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005081323971915748, + "loss": 1.3762, + "theoretical_loss": 3.60909077257636, + "tokens_seen": 1225195520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005081163490471414, + "loss": 1.3016, + "theoretical_loss": 3.6090732238789522, + "tokens_seen": 1225261056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005081003009027082, + "loss": 1.4088, + "theoretical_loss": 3.6090556763829547, + "tokens_seen": 1225326592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005080842527582749, + "loss": 1.3233, + "theoretical_loss": 3.609038130088221, + "tokens_seen": 1225392128 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005080682046138416, + "loss": 1.3382, + "theoretical_loss": 3.609020584994604, + "tokens_seen": 1225457664 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 411823, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3464809656143188, + "objective/train/theoretical_loss": 3.609003041101958, + "objective/train/tokens_used": 1245983200, + "theoretical_loss": 3.609003041101958, + "tokens_seen": 1225523200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005080521564694083, + "loss": 1.4249, + "theoretical_loss": 3.609003041101958, + "tokens_seen": 1225523200 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508036108324975, + "loss": 1.3424, + "theoretical_loss": 3.608985498410136, + "tokens_seen": 1225588736 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005080200601805417, + "loss": 1.3674, + "theoretical_loss": 3.6089679569189923, + "tokens_seen": 1225654272 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005080040120361084, + "loss": 1.291, + "theoretical_loss": 3.60895041662838, + "tokens_seen": 1225719808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005079879638916751, + "loss": 1.3059, + "theoretical_loss": 3.6089328775381526, + "tokens_seen": 1225785344 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005079719157472418, + "loss": 1.4017, + "theoretical_loss": 3.6089153396481644, + "tokens_seen": 1225850880 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005079558676028085, + "loss": 1.3051, + "theoretical_loss": 3.6088978029582686, + "tokens_seen": 1225916416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005079398194583752, + "loss": 1.3591, + "theoretical_loss": 3.608880267468319, + "tokens_seen": 1225981952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005079237713139419, + "loss": 1.3276, + "theoretical_loss": 3.6088627331781695, + "tokens_seen": 1226047488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005079077231695086, + "loss": 1.3382, + "theoretical_loss": 3.608845200087674, + "tokens_seen": 1226113024 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005078916750250753, + "loss": 1.3726, + "theoretical_loss": 3.6088276681966858, + "tokens_seen": 1226178560 + }, + { + "epoch": 0.37, + "learning_rate": 0.000507875626880642, + "loss": 1.3957, + "theoretical_loss": 3.608810137505059, + "tokens_seen": 1226244096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005078595787362086, + "loss": 1.382, + "theoretical_loss": 3.6087926080126476, + "tokens_seen": 1226309632 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005078435305917754, + "loss": 1.2793, + "theoretical_loss": 3.6087750797193054, + "tokens_seen": 1226375168 + }, + { + "epoch": 0.37, + "learning_rate": 0.000507827482447342, + "loss": 1.376, + "theoretical_loss": 3.608757552624887, + "tokens_seen": 1226440704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005078114343029088, + "loss": 1.3469, + "theoretical_loss": 3.6087400267292447, + "tokens_seen": 1226506240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005077953861584754, + "loss": 1.4904, + "theoretical_loss": 3.6087225020322338, + "tokens_seen": 1226571776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005077793380140422, + "loss": 1.3039, + "theoretical_loss": 3.6087049785337078, + "tokens_seen": 1226637312 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005077632898696088, + "loss": 1.3077, + "theoretical_loss": 3.608687456233521, + "tokens_seen": 1226702848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005077472417251756, + "loss": 1.3648, + "theoretical_loss": 3.6086699351315272, + "tokens_seen": 1226768384 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005077311935807422, + "loss": 1.3395, + "theoretical_loss": 3.6086524152275805, + "tokens_seen": 1226833920 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005077151454363089, + "loss": 1.3836, + "theoretical_loss": 3.6086348965215347, + "tokens_seen": 1226899456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005076990972918756, + "loss": 1.2947, + "theoretical_loss": 3.6086173790132445, + "tokens_seen": 1226964992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005076830491474423, + "loss": 1.4456, + "theoretical_loss": 3.608599862702564, + "tokens_seen": 1227030528 + }, + { + "epoch": 0.37, + "learning_rate": 0.000507667001003009, + "loss": 1.3566, + "theoretical_loss": 3.608582347589347, + "tokens_seen": 1227096064 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005076509528585757, + "loss": 1.3627, + "theoretical_loss": 3.608564833673447, + "tokens_seen": 1227161600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005076349047141424, + "loss": 1.3629, + "theoretical_loss": 3.60854732095472, + "tokens_seen": 1227227136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005076188565697091, + "loss": 1.2041, + "theoretical_loss": 3.608529809433019, + "tokens_seen": 1227292672 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005076028084252758, + "loss": 1.4043, + "theoretical_loss": 3.6085122991081984, + "tokens_seen": 1227358208 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005075867602808425, + "loss": 1.3448, + "theoretical_loss": 3.6084947899801127, + "tokens_seen": 1227423744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005075707121364092, + "loss": 1.3638, + "theoretical_loss": 3.608477282048616, + "tokens_seen": 1227489280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005075546639919759, + "loss": 1.3437, + "theoretical_loss": 3.6084597753135625, + "tokens_seen": 1227554816 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005075386158475426, + "loss": 1.2933, + "theoretical_loss": 3.6084422697748075, + "tokens_seen": 1227620352 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005075225677031093, + "loss": 1.3695, + "theoretical_loss": 3.608424765432205, + "tokens_seen": 1227685888 + }, + { + "epoch": 0.37, + "learning_rate": 0.000507506519558676, + "loss": 1.3435, + "theoretical_loss": 3.608407262285608, + "tokens_seen": 1227751424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005074904714142427, + "loss": 1.3852, + "theoretical_loss": 3.608389760334873, + "tokens_seen": 1227816960 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005074744232698094, + "loss": 1.3372, + "theoretical_loss": 3.6083722595798537, + "tokens_seen": 1227882496 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005074583751253761, + "loss": 1.2699, + "theoretical_loss": 3.6083547600204042, + "tokens_seen": 1227948032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005074423269809428, + "loss": 1.4889, + "theoretical_loss": 3.6083372616563794, + "tokens_seen": 1228013568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005074262788365095, + "loss": 1.3554, + "theoretical_loss": 3.6083197644876335, + "tokens_seen": 1228079104 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005074102306920762, + "loss": 1.3262, + "theoretical_loss": 3.608302268514022, + "tokens_seen": 1228144640 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005073941825476429, + "loss": 1.3699, + "theoretical_loss": 3.608284773735398, + "tokens_seen": 1228210176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005073781344032097, + "loss": 1.2882, + "theoretical_loss": 3.6082672801516176, + "tokens_seen": 1228275712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005073620862587763, + "loss": 1.2018, + "theoretical_loss": 3.608249787762534, + "tokens_seen": 1228341248 + }, + { + "epoch": 0.37, + "learning_rate": 0.000507346038114343, + "loss": 1.2808, + "theoretical_loss": 3.6082322965680036, + "tokens_seen": 1228406784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005073299899699097, + "loss": 1.4004, + "theoretical_loss": 3.6082148065678794, + "tokens_seen": 1228472320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005073139418254765, + "loss": 1.3758, + "theoretical_loss": 3.608197317762018, + "tokens_seen": 1228537856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005072978936810431, + "loss": 1.3479, + "theoretical_loss": 3.6081798301502723, + "tokens_seen": 1228603392 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005072818455366099, + "loss": 1.3519, + "theoretical_loss": 3.6081623437324977, + "tokens_seen": 1228668928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005072657973921765, + "loss": 1.4185, + "theoretical_loss": 3.60814485850855, + "tokens_seen": 1228734464 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 413048, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3460556268692017, + "objective/train/theoretical_loss": 3.608127374478282, + "objective/train/tokens_used": 1249260000, + "theoretical_loss": 3.608127374478282, + "tokens_seen": 1228800000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005072497492477433, + "loss": 1.3908, + "theoretical_loss": 3.608127374478282, + "tokens_seen": 1228800000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005072337011033099, + "loss": 1.286, + "theoretical_loss": 3.6081098916415506, + "tokens_seen": 1228865536 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005072176529588767, + "loss": 1.3386, + "theoretical_loss": 3.6080924099982092, + "tokens_seen": 1228931072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005072016048144433, + "loss": 1.3347, + "theoretical_loss": 3.608074929548114, + "tokens_seen": 1228996608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005071855566700101, + "loss": 1.4125, + "theoretical_loss": 3.6080574502911187, + "tokens_seen": 1229062144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005071695085255767, + "loss": 1.3884, + "theoretical_loss": 3.608039972227079, + "tokens_seen": 1229127680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005071534603811435, + "loss": 1.2577, + "theoretical_loss": 3.6080224953558497, + "tokens_seen": 1229193216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005071374122367101, + "loss": 1.3599, + "theoretical_loss": 3.6080050196772855, + "tokens_seen": 1229258752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005071213640922769, + "loss": 1.4663, + "theoretical_loss": 3.607987545191242, + "tokens_seen": 1229324288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005071053159478435, + "loss": 1.2423, + "theoretical_loss": 3.607970071897574, + "tokens_seen": 1229389824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005070892678034103, + "loss": 1.3145, + "theoretical_loss": 3.6079525997961364, + "tokens_seen": 1229455360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005070732196589769, + "loss": 1.3346, + "theoretical_loss": 3.6079351288867842, + "tokens_seen": 1229520896 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005070571715145437, + "loss": 1.3632, + "theoretical_loss": 3.607917659169373, + "tokens_seen": 1229586432 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005070411233701103, + "loss": 1.3559, + "theoretical_loss": 3.607900190643758, + "tokens_seen": 1229651968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005070250752256771, + "loss": 1.3823, + "theoretical_loss": 3.607882723309794, + "tokens_seen": 1229717504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005070090270812437, + "loss": 1.3266, + "theoretical_loss": 3.607865257167336, + "tokens_seen": 1229783040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005069929789368105, + "loss": 1.3763, + "theoretical_loss": 3.60784779221624, + "tokens_seen": 1229848576 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005069769307923771, + "loss": 1.4658, + "theoretical_loss": 3.6078303284563606, + "tokens_seen": 1229914112 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005069608826479439, + "loss": 1.273, + "theoretical_loss": 3.6078128658875537, + "tokens_seen": 1229979648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005069448345035105, + "loss": 1.3706, + "theoretical_loss": 3.607795404509674, + "tokens_seen": 1230045184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005069287863590773, + "loss": 1.3904, + "theoretical_loss": 3.607777944322577, + "tokens_seen": 1230110720 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005069127382146439, + "loss": 1.3611, + "theoretical_loss": 3.607760485326118, + "tokens_seen": 1230176256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005068966900702107, + "loss": 1.5583, + "theoretical_loss": 3.6077430275201525, + "tokens_seen": 1230241792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005068806419257774, + "loss": 1.3757, + "theoretical_loss": 3.607725570904536, + "tokens_seen": 1230307328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005068645937813441, + "loss": 1.4449, + "theoretical_loss": 3.6077081154791237, + "tokens_seen": 1230372864 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005068485456369108, + "loss": 1.3612, + "theoretical_loss": 3.6076906612437716, + "tokens_seen": 1230438400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005068324974924775, + "loss": 1.2978, + "theoretical_loss": 3.6076732081983347, + "tokens_seen": 1230503936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005068164493480442, + "loss": 1.3877, + "theoretical_loss": 3.6076557563426688, + "tokens_seen": 1230569472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005068004012036109, + "loss": 1.3809, + "theoretical_loss": 3.6076383056766286, + "tokens_seen": 1230635008 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005067843530591776, + "loss": 1.3526, + "theoretical_loss": 3.6076208562000707, + "tokens_seen": 1230700544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005067683049147443, + "loss": 1.3833, + "theoretical_loss": 3.6076034079128503, + "tokens_seen": 1230766080 + }, + { + "epoch": 0.37, + "learning_rate": 0.000506752256770311, + "loss": 1.3462, + "theoretical_loss": 3.6075859608148226, + "tokens_seen": 1230831616 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005067362086258777, + "loss": 1.3983, + "theoretical_loss": 3.6075685149058443, + "tokens_seen": 1230897152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005067201604814444, + "loss": 1.3677, + "theoretical_loss": 3.60755107018577, + "tokens_seen": 1230962688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005067041123370111, + "loss": 1.3699, + "theoretical_loss": 3.607533626654456, + "tokens_seen": 1231028224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005066880641925778, + "loss": 1.3833, + "theoretical_loss": 3.607516184311758, + "tokens_seen": 1231093760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005066720160481445, + "loss": 1.3335, + "theoretical_loss": 3.6074987431575307, + "tokens_seen": 1231159296 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005066559679037112, + "loss": 1.397, + "theoretical_loss": 3.6074813031916317, + "tokens_seen": 1231224832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005066399197592779, + "loss": 1.3204, + "theoretical_loss": 3.607463864413915, + "tokens_seen": 1231290368 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005066238716148446, + "loss": 1.329, + "theoretical_loss": 3.6074464268242377, + "tokens_seen": 1231355904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005066078234704113, + "loss": 1.4027, + "theoretical_loss": 3.607428990422455, + "tokens_seen": 1231421440 + }, + { + "epoch": 0.37, + "learning_rate": 0.000506591775325978, + "loss": 1.328, + "theoretical_loss": 3.607411555208423, + "tokens_seen": 1231486976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005065757271815447, + "loss": 1.3285, + "theoretical_loss": 3.6073941211819975, + "tokens_seen": 1231552512 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005065596790371114, + "loss": 1.2685, + "theoretical_loss": 3.6073766883430345, + "tokens_seen": 1231618048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005065436308926781, + "loss": 1.3626, + "theoretical_loss": 3.6073592566913897, + "tokens_seen": 1231683584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005065275827482448, + "loss": 1.3369, + "theoretical_loss": 3.6073418262269197, + "tokens_seen": 1231749120 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005065115346038115, + "loss": 1.3733, + "theoretical_loss": 3.6073243969494793, + "tokens_seen": 1231814656 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005064954864593782, + "loss": 1.3092, + "theoretical_loss": 3.6073069688589254, + "tokens_seen": 1231880192 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005064794383149449, + "loss": 1.2845, + "theoretical_loss": 3.6072895419551143, + "tokens_seen": 1231945728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005064633901705115, + "loss": 1.3873, + "theoretical_loss": 3.607272116237901, + "tokens_seen": 1232011264 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 414319, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.253023624420166, + "objective/train/theoretical_loss": 3.6072546917071424, + "objective/train/tokens_used": 1252536800, + "theoretical_loss": 3.6072546917071424, + "tokens_seen": 1232076800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005064473420260783, + "loss": 1.2735, + "theoretical_loss": 3.6072546917071424, + "tokens_seen": 1232076800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005064312938816449, + "loss": 1.3149, + "theoretical_loss": 3.607237268362695, + "tokens_seen": 1232142336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005064152457372117, + "loss": 1.4091, + "theoretical_loss": 3.607219846204414, + "tokens_seen": 1232207872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005063991975927783, + "loss": 1.4218, + "theoretical_loss": 3.607202425232156, + "tokens_seen": 1232273408 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005063831494483451, + "loss": 1.41, + "theoretical_loss": 3.607185005445777, + "tokens_seen": 1232338944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005063671013039117, + "loss": 1.3715, + "theoretical_loss": 3.607167586845134, + "tokens_seen": 1232404480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005063510531594785, + "loss": 1.3617, + "theoretical_loss": 3.6071501694300814, + "tokens_seen": 1232470016 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005063350050150451, + "loss": 1.3739, + "theoretical_loss": 3.6071327532004775, + "tokens_seen": 1232535552 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005063189568706118, + "loss": 1.2954, + "theoretical_loss": 3.607115338156178, + "tokens_seen": 1232601088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005063029087261785, + "loss": 1.3607, + "theoretical_loss": 3.6070979242970385, + "tokens_seen": 1232666624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005062868605817452, + "loss": 1.3799, + "theoretical_loss": 3.6070805116229154, + "tokens_seen": 1232732160 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005062708124373119, + "loss": 1.4167, + "theoretical_loss": 3.607063100133666, + "tokens_seen": 1232797696 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005062547642928786, + "loss": 1.3713, + "theoretical_loss": 3.607045689829146, + "tokens_seen": 1232863232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005062387161484453, + "loss": 1.3901, + "theoretical_loss": 3.607028280709213, + "tokens_seen": 1232928768 + }, + { + "epoch": 0.37, + "learning_rate": 0.000506222668004012, + "loss": 1.2927, + "theoretical_loss": 3.6070108727737207, + "tokens_seen": 1232994304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005062066198595787, + "loss": 1.285, + "theoretical_loss": 3.6069934660225287, + "tokens_seen": 1233059840 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005061905717151454, + "loss": 1.2497, + "theoretical_loss": 3.6069760604554912, + "tokens_seen": 1233125376 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005061745235707121, + "loss": 1.368, + "theoretical_loss": 3.6069586560724662, + "tokens_seen": 1233190912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005061584754262788, + "loss": 1.3899, + "theoretical_loss": 3.6069412528733094, + "tokens_seen": 1233256448 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005061424272818455, + "loss": 1.5183, + "theoretical_loss": 3.6069238508578776, + "tokens_seen": 1233321984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005061263791374122, + "loss": 1.3898, + "theoretical_loss": 3.6069064500260275, + "tokens_seen": 1233387520 + }, + { + "epoch": 0.37, + "learning_rate": 0.000506110330992979, + "loss": 1.4818, + "theoretical_loss": 3.6068890503776156, + "tokens_seen": 1233453056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005060942828485456, + "loss": 1.3643, + "theoretical_loss": 3.606871651912498, + "tokens_seen": 1233518592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005060782347041123, + "loss": 1.3459, + "theoretical_loss": 3.606854254630533, + "tokens_seen": 1233584128 + }, + { + "epoch": 0.37, + "learning_rate": 0.000506062186559679, + "loss": 1.3058, + "theoretical_loss": 3.606836858531576, + "tokens_seen": 1233649664 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005060461384152457, + "loss": 1.4604, + "theoretical_loss": 3.606819463615483, + "tokens_seen": 1233715200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005060300902708124, + "loss": 1.365, + "theoretical_loss": 3.6068020698821126, + "tokens_seen": 1233780736 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005060140421263792, + "loss": 1.316, + "theoretical_loss": 3.60678467733132, + "tokens_seen": 1233846272 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005059979939819458, + "loss": 1.3043, + "theoretical_loss": 3.606767285962963, + "tokens_seen": 1233911808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005059819458375126, + "loss": 1.3951, + "theoretical_loss": 3.6067498957768986, + "tokens_seen": 1233977344 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005059658976930792, + "loss": 1.2649, + "theoretical_loss": 3.606732506772982, + "tokens_seen": 1234042880 + }, + { + "epoch": 0.37, + "learning_rate": 0.000505949849548646, + "loss": 1.4024, + "theoretical_loss": 3.606715118951072, + "tokens_seen": 1234108416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005059338014042126, + "loss": 1.2778, + "theoretical_loss": 3.6066977323110243, + "tokens_seen": 1234173952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005059177532597794, + "loss": 1.3418, + "theoretical_loss": 3.6066803468526967, + "tokens_seen": 1234239488 + }, + { + "epoch": 0.37, + "learning_rate": 0.000505901705115346, + "loss": 1.3942, + "theoretical_loss": 3.6066629625759448, + "tokens_seen": 1234305024 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005058856569709128, + "loss": 1.2371, + "theoretical_loss": 3.6066455794806265, + "tokens_seen": 1234370560 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005058696088264794, + "loss": 1.3195, + "theoretical_loss": 3.6066281975665992, + "tokens_seen": 1234436096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005058535606820462, + "loss": 1.4218, + "theoretical_loss": 3.6066108168337188, + "tokens_seen": 1234501632 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005058375125376128, + "loss": 1.4048, + "theoretical_loss": 3.6065934372818433, + "tokens_seen": 1234567168 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005058214643931796, + "loss": 1.3959, + "theoretical_loss": 3.6065760589108296, + "tokens_seen": 1234632704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005058054162487462, + "loss": 1.4088, + "theoretical_loss": 3.606558681720534, + "tokens_seen": 1234698240 + }, + { + "epoch": 0.37, + "learning_rate": 0.000505789368104313, + "loss": 1.3888, + "theoretical_loss": 3.6065413057108144, + "tokens_seen": 1234763776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005057733199598796, + "loss": 1.3972, + "theoretical_loss": 3.606523930881528, + "tokens_seen": 1234829312 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005057572718154464, + "loss": 1.4086, + "theoretical_loss": 3.6065065572325317, + "tokens_seen": 1234894848 + }, + { + "epoch": 0.37, + "learning_rate": 0.000505741223671013, + "loss": 1.2821, + "theoretical_loss": 3.6064891847636824, + "tokens_seen": 1234960384 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005057251755265798, + "loss": 1.4076, + "theoretical_loss": 3.6064718134748377, + "tokens_seen": 1235025920 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005057091273821464, + "loss": 1.4631, + "theoretical_loss": 3.6064544433658545, + "tokens_seen": 1235091456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005056930792377132, + "loss": 1.3723, + "theoretical_loss": 3.606437074436591, + "tokens_seen": 1235156992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005056770310932799, + "loss": 1.2252, + "theoretical_loss": 3.6064197066869035, + "tokens_seen": 1235222528 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005056609829488466, + "loss": 1.3153, + "theoretical_loss": 3.606402340116649, + "tokens_seen": 1235288064 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 415196, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2908198833465576, + "objective/train/theoretical_loss": 3.606384974725686, + "objective/train/tokens_used": 1255813600, + "theoretical_loss": 3.606384974725686, + "tokens_seen": 1235353600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005056449348044133, + "loss": 1.3514, + "theoretical_loss": 3.606384974725686, + "tokens_seen": 1235353600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00050562888665998, + "loss": 1.3917, + "theoretical_loss": 3.606367610513871, + "tokens_seen": 1235419136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005056128385155467, + "loss": 1.3514, + "theoretical_loss": 3.6063502474810623, + "tokens_seen": 1235484672 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005055967903711134, + "loss": 1.3734, + "theoretical_loss": 3.6063328856271166, + "tokens_seen": 1235550208 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005055807422266801, + "loss": 1.2518, + "theoretical_loss": 3.6063155249518912, + "tokens_seen": 1235615744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005055646940822468, + "loss": 1.3106, + "theoretical_loss": 3.606298165455244, + "tokens_seen": 1235681280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005055486459378135, + "loss": 1.4104, + "theoretical_loss": 3.606280807137032, + "tokens_seen": 1235746816 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005055325977933802, + "loss": 1.284, + "theoretical_loss": 3.606263449997113, + "tokens_seen": 1235812352 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005055165496489469, + "loss": 1.3137, + "theoretical_loss": 3.606246094035345, + "tokens_seen": 1235877888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005055005015045136, + "loss": 1.364, + "theoretical_loss": 3.606228739251585, + "tokens_seen": 1235943424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005054844533600803, + "loss": 1.4349, + "theoretical_loss": 3.6062113856456905, + "tokens_seen": 1236008960 + }, + { + "epoch": 0.37, + "learning_rate": 0.000505468405215647, + "loss": 1.4239, + "theoretical_loss": 3.6061940332175193, + "tokens_seen": 1236074496 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005054523570712137, + "loss": 1.393, + "theoretical_loss": 3.606176681966929, + "tokens_seen": 1236140032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005054363089267804, + "loss": 1.4988, + "theoretical_loss": 3.6061593318937777, + "tokens_seen": 1236205568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005054202607823471, + "loss": 1.4439, + "theoretical_loss": 3.606141982997922, + "tokens_seen": 1236271104 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005054042126379138, + "loss": 1.4327, + "theoretical_loss": 3.6061246352792216, + "tokens_seen": 1236336640 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005053881644934805, + "loss": 1.3594, + "theoretical_loss": 3.606107288737532, + "tokens_seen": 1236402176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005053721163490472, + "loss": 1.357, + "theoretical_loss": 3.606089943372712, + "tokens_seen": 1236467712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005053560682046139, + "loss": 1.2525, + "theoretical_loss": 3.6060725991846194, + "tokens_seen": 1236533248 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005053400200601806, + "loss": 1.3922, + "theoretical_loss": 3.6060552561731116, + "tokens_seen": 1236598784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005053239719157473, + "loss": 1.4451, + "theoretical_loss": 3.6060379143380468, + "tokens_seen": 1236664320 + }, + { + "epoch": 0.37, + "learning_rate": 0.000505307923771314, + "loss": 1.4254, + "theoretical_loss": 3.6060205736792836, + "tokens_seen": 1236729856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005052918756268807, + "loss": 1.3562, + "theoretical_loss": 3.6060032341966783, + "tokens_seen": 1236795392 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005052758274824474, + "loss": 1.4196, + "theoretical_loss": 3.6059858958900897, + "tokens_seen": 1236860928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005052597793380141, + "loss": 1.2609, + "theoretical_loss": 3.6059685587593755, + "tokens_seen": 1236926464 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005052437311935808, + "loss": 1.4501, + "theoretical_loss": 3.605951222804394, + "tokens_seen": 1236992000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005052276830491475, + "loss": 1.2625, + "theoretical_loss": 3.605933888025003, + "tokens_seen": 1237057536 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005052116349047142, + "loss": 1.4949, + "theoretical_loss": 3.6059165544210607, + "tokens_seen": 1237123072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005051955867602809, + "loss": 1.2802, + "theoretical_loss": 3.6058992219924244, + "tokens_seen": 1237188608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005051795386158476, + "loss": 1.3527, + "theoretical_loss": 3.605881890738953, + "tokens_seen": 1237254144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005051634904714143, + "loss": 1.3396, + "theoretical_loss": 3.605864560660504, + "tokens_seen": 1237319680 + }, + { + "epoch": 0.37, + "learning_rate": 0.000505147442326981, + "loss": 1.3599, + "theoretical_loss": 3.605847231756936, + "tokens_seen": 1237385216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005051313941825477, + "loss": 1.4324, + "theoretical_loss": 3.6058299040281065, + "tokens_seen": 1237450752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005051153460381144, + "loss": 1.3222, + "theoretical_loss": 3.6058125774738743, + "tokens_seen": 1237516288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005050992978936811, + "loss": 1.3546, + "theoretical_loss": 3.6057952520940972, + "tokens_seen": 1237581824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005050832497492478, + "loss": 1.3409, + "theoretical_loss": 3.6057779278886337, + "tokens_seen": 1237647360 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005050672016048145, + "loss": 1.3855, + "theoretical_loss": 3.6057606048573416, + "tokens_seen": 1237712896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005050511534603812, + "loss": 1.2924, + "theoretical_loss": 3.6057432830000797, + "tokens_seen": 1237778432 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005050351053159478, + "loss": 1.2803, + "theoretical_loss": 3.605725962316706, + "tokens_seen": 1237843968 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005050190571715146, + "loss": 1.3581, + "theoretical_loss": 3.6057086428070786, + "tokens_seen": 1237909504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005050030090270812, + "loss": 1.3183, + "theoretical_loss": 3.6056913244710556, + "tokens_seen": 1237975040 + }, + { + "epoch": 0.38, + "learning_rate": 0.000504986960882648, + "loss": 1.3255, + "theoretical_loss": 3.6056740073084965, + "tokens_seen": 1238040576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005049709127382146, + "loss": 1.512, + "theoretical_loss": 3.6056566913192585, + "tokens_seen": 1238106112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005049548645937814, + "loss": 1.3427, + "theoretical_loss": 3.6056393765032007, + "tokens_seen": 1238171648 + }, + { + "epoch": 0.38, + "learning_rate": 0.000504938816449348, + "loss": 1.3479, + "theoretical_loss": 3.605622062860181, + "tokens_seen": 1238237184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005049227683049147, + "loss": 1.3233, + "theoretical_loss": 3.605604750390058, + "tokens_seen": 1238302720 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005049067201604814, + "loss": 1.3147, + "theoretical_loss": 3.6055874390926905, + "tokens_seen": 1238368256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005048906720160481, + "loss": 1.406, + "theoretical_loss": 3.605570128967937, + "tokens_seen": 1238433792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005048746238716148, + "loss": 1.4015, + "theoretical_loss": 3.6055528200156557, + "tokens_seen": 1238499328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005048585757271815, + "loss": 1.3123, + "theoretical_loss": 3.605535512235705, + "tokens_seen": 1238564864 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 416369, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4090187549591064, + "objective/train/theoretical_loss": 3.605518205627944, + "objective/train/tokens_used": 1259090400, + "theoretical_loss": 3.605518205627944, + "tokens_seen": 1238630400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005048425275827482, + "loss": 1.4503, + "theoretical_loss": 3.605518205627944, + "tokens_seen": 1238630400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005048264794383149, + "loss": 1.4952, + "theoretical_loss": 3.605500900192231, + "tokens_seen": 1238695936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005048104312938816, + "loss": 1.3989, + "theoretical_loss": 3.6054835959284244, + "tokens_seen": 1238761472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005047943831494483, + "loss": 1.289, + "theoretical_loss": 3.6054662928363834, + "tokens_seen": 1238827008 + }, + { + "epoch": 0.38, + "learning_rate": 0.000504778335005015, + "loss": 1.2864, + "theoretical_loss": 3.605448990915966, + "tokens_seen": 1238892544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005047622868605817, + "loss": 1.4136, + "theoretical_loss": 3.605431690167032, + "tokens_seen": 1238958080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005047462387161484, + "loss": 1.3431, + "theoretical_loss": 3.6054143905894387, + "tokens_seen": 1239023616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005047301905717151, + "loss": 1.4101, + "theoretical_loss": 3.605397092183046, + "tokens_seen": 1239089152 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005047141424272818, + "loss": 1.3925, + "theoretical_loss": 3.6053797949477118, + "tokens_seen": 1239154688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005046980942828485, + "loss": 1.2969, + "theoretical_loss": 3.605362498883296, + "tokens_seen": 1239220224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005046820461384152, + "loss": 1.3725, + "theoretical_loss": 3.605345203989656, + "tokens_seen": 1239285760 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005046659979939819, + "loss": 1.2839, + "theoretical_loss": 3.6053279102666513, + "tokens_seen": 1239351296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005046499498495487, + "loss": 1.3753, + "theoretical_loss": 3.6053106177141414, + "tokens_seen": 1239416832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005046339017051153, + "loss": 1.3842, + "theoretical_loss": 3.605293326331984, + "tokens_seen": 1239482368 + }, + { + "epoch": 0.38, + "learning_rate": 0.000504617853560682, + "loss": 1.3695, + "theoretical_loss": 3.6052760361200393, + "tokens_seen": 1239547904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005046018054162487, + "loss": 1.2988, + "theoretical_loss": 3.6052587470781647, + "tokens_seen": 1239613440 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005045857572718155, + "loss": 1.369, + "theoretical_loss": 3.605241459206221, + "tokens_seen": 1239678976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005045697091273821, + "loss": 1.4293, + "theoretical_loss": 3.605224172504066, + "tokens_seen": 1239744512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005045536609829489, + "loss": 1.3443, + "theoretical_loss": 3.6052068869715583, + "tokens_seen": 1239810048 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005045376128385155, + "loss": 1.42, + "theoretical_loss": 3.605189602608558, + "tokens_seen": 1239875584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005045215646940823, + "loss": 1.3387, + "theoretical_loss": 3.605172319414924, + "tokens_seen": 1239941120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005045055165496489, + "loss": 1.3442, + "theoretical_loss": 3.6051550373905146, + "tokens_seen": 1240006656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005044894684052157, + "loss": 1.2619, + "theoretical_loss": 3.60513775653519, + "tokens_seen": 1240072192 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005044734202607824, + "loss": 1.3328, + "theoretical_loss": 3.6051204768488088, + "tokens_seen": 1240137728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005044573721163491, + "loss": 1.3232, + "theoretical_loss": 3.6051031983312294, + "tokens_seen": 1240203264 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005044413239719158, + "loss": 1.4697, + "theoretical_loss": 3.6050859209823125, + "tokens_seen": 1240268800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005044252758274825, + "loss": 1.3618, + "theoretical_loss": 3.605068644801916, + "tokens_seen": 1240334336 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005044092276830492, + "loss": 1.3062, + "theoretical_loss": 3.6050513697899, + "tokens_seen": 1240399872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005043931795386159, + "loss": 1.3823, + "theoretical_loss": 3.6050340959461233, + "tokens_seen": 1240465408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005043771313941826, + "loss": 1.3717, + "theoretical_loss": 3.6050168232704456, + "tokens_seen": 1240530944 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005043610832497493, + "loss": 1.4089, + "theoretical_loss": 3.6049995517627256, + "tokens_seen": 1240596480 + }, + { + "epoch": 0.38, + "learning_rate": 0.000504345035105316, + "loss": 1.3627, + "theoretical_loss": 3.604982281422823, + "tokens_seen": 1240662016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005043289869608827, + "loss": 1.2779, + "theoretical_loss": 3.604965012250597, + "tokens_seen": 1240727552 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005043129388164494, + "loss": 1.483, + "theoretical_loss": 3.6049477442459077, + "tokens_seen": 1240793088 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005042968906720161, + "loss": 1.3465, + "theoretical_loss": 3.604930477408613, + "tokens_seen": 1240858624 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005042808425275828, + "loss": 1.3855, + "theoretical_loss": 3.6049132117385736, + "tokens_seen": 1240924160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005042647943831495, + "loss": 1.2742, + "theoretical_loss": 3.604895947235649, + "tokens_seen": 1240989696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005042487462387162, + "loss": 1.3288, + "theoretical_loss": 3.604878683899698, + "tokens_seen": 1241055232 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005042326980942829, + "loss": 1.3308, + "theoretical_loss": 3.60486142173058, + "tokens_seen": 1241120768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005042166499498496, + "loss": 1.4005, + "theoretical_loss": 3.604844160728155, + "tokens_seen": 1241186304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005042006018054163, + "loss": 1.3795, + "theoretical_loss": 3.6048269008922826, + "tokens_seen": 1241251840 + }, + { + "epoch": 0.38, + "learning_rate": 0.000504184553660983, + "loss": 1.3427, + "theoretical_loss": 3.604809642222822, + "tokens_seen": 1241317376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005041685055165497, + "loss": 1.2901, + "theoretical_loss": 3.6047923847196337, + "tokens_seen": 1241382912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005041524573721164, + "loss": 1.3897, + "theoretical_loss": 3.6047751283825757, + "tokens_seen": 1241448448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005041364092276831, + "loss": 1.3342, + "theoretical_loss": 3.6047578732115086, + "tokens_seen": 1241513984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005041203610832498, + "loss": 1.3681, + "theoretical_loss": 3.604740619206293, + "tokens_seen": 1241579520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005041043129388165, + "loss": 1.2255, + "theoretical_loss": 3.6047233663667866, + "tokens_seen": 1241645056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005040882647943832, + "loss": 1.4839, + "theoretical_loss": 3.604706114692851, + "tokens_seen": 1241710592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005040722166499499, + "loss": 1.2679, + "theoretical_loss": 3.604688864184345, + "tokens_seen": 1241776128 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005040561685055166, + "loss": 1.3852, + "theoretical_loss": 3.604671614841128, + "tokens_seen": 1241841664 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 417232, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1997463703155518, + "objective/train/theoretical_loss": 3.6046543666630604, + "objective/train/tokens_used": 1262367200, + "theoretical_loss": 3.6046543666630604, + "tokens_seen": 1241907200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005040401203610833, + "loss": 1.2561, + "theoretical_loss": 3.6046543666630604, + "tokens_seen": 1241907200 + }, + { + "epoch": 0.38, + "learning_rate": 0.00050402407221665, + "loss": 1.2476, + "theoretical_loss": 3.604637119650002, + "tokens_seen": 1241972736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005040080240722167, + "loss": 1.3679, + "theoretical_loss": 3.6046198738018127, + "tokens_seen": 1242038272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005039919759277834, + "loss": 1.3728, + "theoretical_loss": 3.604602629118352, + "tokens_seen": 1242103808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005039759277833501, + "loss": 1.3712, + "theoretical_loss": 3.6045853855994796, + "tokens_seen": 1242169344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005039598796389168, + "loss": 1.4702, + "theoretical_loss": 3.6045681432450567, + "tokens_seen": 1242234880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005039438314944835, + "loss": 1.3163, + "theoretical_loss": 3.6045509020549416, + "tokens_seen": 1242300416 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005039277833500502, + "loss": 1.3762, + "theoretical_loss": 3.6045336620289956, + "tokens_seen": 1242365952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005039117352056169, + "loss": 1.3988, + "theoretical_loss": 3.6045164231670777, + "tokens_seen": 1242431488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005038956870611836, + "loss": 1.3746, + "theoretical_loss": 3.6044991854690487, + "tokens_seen": 1242497024 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005038796389167503, + "loss": 1.3213, + "theoretical_loss": 3.604481948934768, + "tokens_seen": 1242562560 + }, + { + "epoch": 0.38, + "learning_rate": 0.000503863590772317, + "loss": 1.3872, + "theoretical_loss": 3.6044647135640955, + "tokens_seen": 1242628096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005038475426278837, + "loss": 1.1483, + "theoretical_loss": 3.6044474793568924, + "tokens_seen": 1242693632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005038314944834504, + "loss": 1.2868, + "theoretical_loss": 3.6044302463130173, + "tokens_seen": 1242759168 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005038154463390171, + "loss": 1.3432, + "theoretical_loss": 3.604413014432332, + "tokens_seen": 1242824704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005037993981945838, + "loss": 1.3241, + "theoretical_loss": 3.604395783714695, + "tokens_seen": 1242890240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005037833500501505, + "loss": 1.3367, + "theoretical_loss": 3.6043785541599678, + "tokens_seen": 1242955776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005037673019057172, + "loss": 1.4916, + "theoretical_loss": 3.60436132576801, + "tokens_seen": 1243021312 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005037512537612839, + "loss": 1.2502, + "theoretical_loss": 3.6043440985386814, + "tokens_seen": 1243086848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005037352056168506, + "loss": 1.338, + "theoretical_loss": 3.6043268724718436, + "tokens_seen": 1243152384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005037191574724173, + "loss": 1.3496, + "theoretical_loss": 3.604309647567356, + "tokens_seen": 1243217920 + }, + { + "epoch": 0.38, + "learning_rate": 0.000503703109327984, + "loss": 1.2946, + "theoretical_loss": 3.604292423825078, + "tokens_seen": 1243283456 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005036870611835507, + "loss": 1.3432, + "theoretical_loss": 3.6042752012448718, + "tokens_seen": 1243348992 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005036710130391174, + "loss": 1.1825, + "theoretical_loss": 3.6042579798265963, + "tokens_seen": 1243414528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005036549648946841, + "loss": 1.4187, + "theoretical_loss": 3.6042407595701125, + "tokens_seen": 1243480064 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005036389167502507, + "loss": 1.3748, + "theoretical_loss": 3.6042235404752807, + "tokens_seen": 1243545600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005036228686058175, + "loss": 1.3473, + "theoretical_loss": 3.6042063225419616, + "tokens_seen": 1243611136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005036068204613841, + "loss": 1.3553, + "theoretical_loss": 3.604189105770015, + "tokens_seen": 1243676672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005035907723169509, + "loss": 1.3541, + "theoretical_loss": 3.604171890159302, + "tokens_seen": 1243742208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005035747241725175, + "loss": 1.4371, + "theoretical_loss": 3.6041546757096823, + "tokens_seen": 1243807744 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005035586760280843, + "loss": 1.4177, + "theoretical_loss": 3.6041374624210176, + "tokens_seen": 1243873280 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005035426278836509, + "loss": 1.4302, + "theoretical_loss": 3.6041202502931675, + "tokens_seen": 1243938816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005035265797392177, + "loss": 1.4668, + "theoretical_loss": 3.6041030393259925, + "tokens_seen": 1244004352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005035105315947843, + "loss": 1.3746, + "theoretical_loss": 3.604085829519354, + "tokens_seen": 1244069888 + }, + { + "epoch": 0.38, + "learning_rate": 0.000503494483450351, + "loss": 1.3797, + "theoretical_loss": 3.604068620873112, + "tokens_seen": 1244135424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005034784353059177, + "loss": 1.3403, + "theoretical_loss": 3.6040514133871273, + "tokens_seen": 1244200960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005034623871614844, + "loss": 1.3468, + "theoretical_loss": 3.604034207061261, + "tokens_seen": 1244266496 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005034463390170511, + "loss": 1.354, + "theoretical_loss": 3.6040170018953726, + "tokens_seen": 1244332032 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005034302908726178, + "loss": 1.3894, + "theoretical_loss": 3.603999797889324, + "tokens_seen": 1244397568 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005034142427281845, + "loss": 1.4053, + "theoretical_loss": 3.6039825950429756, + "tokens_seen": 1244463104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005033981945837512, + "loss": 1.3635, + "theoretical_loss": 3.6039653933561877, + "tokens_seen": 1244528640 + }, + { + "epoch": 0.38, + "learning_rate": 0.000503382146439318, + "loss": 1.3767, + "theoretical_loss": 3.6039481928288217, + "tokens_seen": 1244594176 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005033660982948846, + "loss": 1.3488, + "theoretical_loss": 3.603930993460738, + "tokens_seen": 1244659712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005033500501504513, + "loss": 1.2356, + "theoretical_loss": 3.603913795251798, + "tokens_seen": 1244725248 + }, + { + "epoch": 0.38, + "learning_rate": 0.000503334002006018, + "loss": 1.407, + "theoretical_loss": 3.603896598201862, + "tokens_seen": 1244790784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005033179538615847, + "loss": 1.3197, + "theoretical_loss": 3.6038794023107905, + "tokens_seen": 1244856320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005033019057171515, + "loss": 1.3136, + "theoretical_loss": 3.6038622075784454, + "tokens_seen": 1244921856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005032858575727182, + "loss": 1.3627, + "theoretical_loss": 3.603845014004687, + "tokens_seen": 1244987392 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005032698094282849, + "loss": 1.3836, + "theoretical_loss": 3.6038278215893764, + "tokens_seen": 1245052928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005032537612838516, + "loss": 1.3111, + "theoretical_loss": 3.603810630332375, + "tokens_seen": 1245118464 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 418442, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4537241458892822, + "objective/train/theoretical_loss": 3.6037934402335425, + "objective/train/tokens_used": 1265644000, + "theoretical_loss": 3.6037934402335425, + "tokens_seen": 1245184000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005032377131394183, + "loss": 1.3637, + "theoretical_loss": 3.6037934402335425, + "tokens_seen": 1245184000 + }, + { + "epoch": 0.38, + "learning_rate": 0.000503221664994985, + "loss": 1.4316, + "theoretical_loss": 3.6037762512927416, + "tokens_seen": 1245249536 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005032056168505517, + "loss": 1.2916, + "theoretical_loss": 3.603759063509832, + "tokens_seen": 1245315072 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005031895687061184, + "loss": 1.3194, + "theoretical_loss": 3.603741876884676, + "tokens_seen": 1245380608 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005031735205616851, + "loss": 1.3862, + "theoretical_loss": 3.6037246914171335, + "tokens_seen": 1245446144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005031574724172518, + "loss": 1.2677, + "theoretical_loss": 3.6037075071070666, + "tokens_seen": 1245511680 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005031414242728185, + "loss": 1.333, + "theoretical_loss": 3.6036903239543356, + "tokens_seen": 1245577216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005031253761283852, + "loss": 1.2588, + "theoretical_loss": 3.6036731419588017, + "tokens_seen": 1245642752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005031093279839519, + "loss": 1.308, + "theoretical_loss": 3.603655961120327, + "tokens_seen": 1245708288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005030932798395186, + "loss": 1.3684, + "theoretical_loss": 3.6036387814387725, + "tokens_seen": 1245773824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005030772316950853, + "loss": 1.362, + "theoretical_loss": 3.6036216029139982, + "tokens_seen": 1245839360 + }, + { + "epoch": 0.38, + "learning_rate": 0.000503061183550652, + "loss": 1.2078, + "theoretical_loss": 3.603604425545867, + "tokens_seen": 1245904896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005030451354062187, + "loss": 1.3836, + "theoretical_loss": 3.603587249334239, + "tokens_seen": 1245970432 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005030290872617854, + "loss": 1.3458, + "theoretical_loss": 3.6035700742789762, + "tokens_seen": 1246035968 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005030130391173521, + "loss": 1.3773, + "theoretical_loss": 3.60355290037994, + "tokens_seen": 1246101504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005029969909729188, + "loss": 1.4511, + "theoretical_loss": 3.6035357276369906, + "tokens_seen": 1246167040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005029809428284855, + "loss": 1.3902, + "theoretical_loss": 3.603518556049991, + "tokens_seen": 1246232576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005029648946840522, + "loss": 1.3707, + "theoretical_loss": 3.603501385618802, + "tokens_seen": 1246298112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005029488465396189, + "loss": 1.3224, + "theoretical_loss": 3.6034842163432845, + "tokens_seen": 1246363648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005029327983951856, + "loss": 1.3062, + "theoretical_loss": 3.6034670482233, + "tokens_seen": 1246429184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005029167502507523, + "loss": 1.3808, + "theoretical_loss": 3.603449881258711, + "tokens_seen": 1246494720 + }, + { + "epoch": 0.38, + "learning_rate": 0.000502900702106319, + "loss": 1.4469, + "theoretical_loss": 3.6034327154493777, + "tokens_seen": 1246560256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005028846539618857, + "loss": 1.3295, + "theoretical_loss": 3.603415550795163, + "tokens_seen": 1246625792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005028686058174524, + "loss": 1.3878, + "theoretical_loss": 3.603398387295927, + "tokens_seen": 1246691328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005028525576730191, + "loss": 1.3918, + "theoretical_loss": 3.6033812249515327, + "tokens_seen": 1246756864 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005028365095285858, + "loss": 1.4321, + "theoretical_loss": 3.6033640637618403, + "tokens_seen": 1246822400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005028204613841525, + "loss": 1.3718, + "theoretical_loss": 3.6033469037267123, + "tokens_seen": 1246887936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005028044132397192, + "loss": 1.3938, + "theoretical_loss": 3.60332974484601, + "tokens_seen": 1246953472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005027883650952859, + "loss": 1.3004, + "theoretical_loss": 3.6033125871195955, + "tokens_seen": 1247019008 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005027723169508526, + "loss": 1.2841, + "theoretical_loss": 3.6032954305473304, + "tokens_seen": 1247084544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005027562688064193, + "loss": 1.376, + "theoretical_loss": 3.6032782751290755, + "tokens_seen": 1247150080 + }, + { + "epoch": 0.38, + "learning_rate": 0.000502740220661986, + "loss": 1.3305, + "theoretical_loss": 3.603261120864694, + "tokens_seen": 1247215616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005027241725175527, + "loss": 1.2501, + "theoretical_loss": 3.6032439677540467, + "tokens_seen": 1247281152 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005027081243731194, + "loss": 1.348, + "theoretical_loss": 3.6032268157969956, + "tokens_seen": 1247346688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005026920762286861, + "loss": 1.2983, + "theoretical_loss": 3.6032096649934022, + "tokens_seen": 1247412224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005026760280842528, + "loss": 1.2707, + "theoretical_loss": 3.603192515343129, + "tokens_seen": 1247477760 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005026599799398195, + "loss": 1.2921, + "theoretical_loss": 3.6031753668460373, + "tokens_seen": 1247543296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005026439317953862, + "loss": 1.2688, + "theoretical_loss": 3.6031582195019896, + "tokens_seen": 1247608832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005026278836509529, + "loss": 1.3406, + "theoretical_loss": 3.6031410733108475, + "tokens_seen": 1247674368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005026118355065196, + "loss": 1.2678, + "theoretical_loss": 3.6031239282724723, + "tokens_seen": 1247739904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005025957873620863, + "loss": 1.4064, + "theoretical_loss": 3.603106784386727, + "tokens_seen": 1247805440 + }, + { + "epoch": 0.38, + "learning_rate": 0.000502579739217653, + "loss": 1.2971, + "theoretical_loss": 3.6030896416534732, + "tokens_seen": 1247870976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005025636910732197, + "loss": 1.3555, + "theoretical_loss": 3.6030725000725723, + "tokens_seen": 1247936512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005025476429287864, + "loss": 1.3105, + "theoretical_loss": 3.6030553596438875, + "tokens_seen": 1248002048 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005025315947843531, + "loss": 1.4003, + "theoretical_loss": 3.6030382203672793, + "tokens_seen": 1248067584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005025155466399198, + "loss": 1.2548, + "theoretical_loss": 3.603021082242611, + "tokens_seen": 1248133120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005024994984954865, + "loss": 1.3411, + "theoretical_loss": 3.6030039452697444, + "tokens_seen": 1248198656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005024834503510532, + "loss": 1.3666, + "theoretical_loss": 3.6029868094485415, + "tokens_seen": 1248264192 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005024674022066199, + "loss": 1.338, + "theoretical_loss": 3.6029696747788647, + "tokens_seen": 1248329728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005024513540621866, + "loss": 1.2971, + "theoretical_loss": 3.602952541260576, + "tokens_seen": 1248395264 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 419258, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.282519817352295, + "objective/train/theoretical_loss": 3.6029354088935373, + "objective/train/tokens_used": 1268920800, + "theoretical_loss": 3.6029354088935373, + "tokens_seen": 1248460800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005024353059177533, + "loss": 1.4272, + "theoretical_loss": 3.6029354088935373, + "tokens_seen": 1248460800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00050241925777332, + "loss": 1.212, + "theoretical_loss": 3.6029182776776114, + "tokens_seen": 1248526336 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005024032096288867, + "loss": 1.3939, + "theoretical_loss": 3.60290114761266, + "tokens_seen": 1248591872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005023871614844534, + "loss": 1.3397, + "theoretical_loss": 3.6028840186985462, + "tokens_seen": 1248657408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005023711133400201, + "loss": 1.4012, + "theoretical_loss": 3.6028668909351307, + "tokens_seen": 1248722944 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005023550651955868, + "loss": 1.3113, + "theoretical_loss": 3.6028497643222774, + "tokens_seen": 1248788480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005023390170511535, + "loss": 1.2558, + "theoretical_loss": 3.602832638859848, + "tokens_seen": 1248854016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005023229689067202, + "loss": 1.1927, + "theoretical_loss": 3.6028155145477054, + "tokens_seen": 1248919552 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005023069207622869, + "loss": 1.2656, + "theoretical_loss": 3.602798391385711, + "tokens_seen": 1248985088 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005022908726178536, + "loss": 1.3359, + "theoretical_loss": 3.602781269373727, + "tokens_seen": 1249050624 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005022748244734203, + "loss": 1.3055, + "theoretical_loss": 3.6027641485116177, + "tokens_seen": 1249116160 + }, + { + "epoch": 0.38, + "learning_rate": 0.000502258776328987, + "loss": 1.2639, + "theoretical_loss": 3.602747028799244, + "tokens_seen": 1249181696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005022427281845536, + "loss": 1.5219, + "theoretical_loss": 3.6027299102364685, + "tokens_seen": 1249247232 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005022266800401204, + "loss": 1.2866, + "theoretical_loss": 3.6027127928231537, + "tokens_seen": 1249312768 + }, + { + "epoch": 0.38, + "learning_rate": 0.000502210631895687, + "loss": 1.2317, + "theoretical_loss": 3.602695676559163, + "tokens_seen": 1249378304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005021945837512538, + "loss": 1.285, + "theoretical_loss": 3.6026785614443577, + "tokens_seen": 1249443840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005021785356068204, + "loss": 1.251, + "theoretical_loss": 3.6026614474786016, + "tokens_seen": 1249509376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005021624874623872, + "loss": 1.3303, + "theoretical_loss": 3.6026443346617567, + "tokens_seen": 1249574912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005021464393179538, + "loss": 1.438, + "theoretical_loss": 3.602627222993685, + "tokens_seen": 1249640448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005021303911735206, + "loss": 1.4266, + "theoretical_loss": 3.6026101124742502, + "tokens_seen": 1249705984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005021143430290872, + "loss": 1.4137, + "theoretical_loss": 3.6025930031033147, + "tokens_seen": 1249771520 + }, + { + "epoch": 0.38, + "learning_rate": 0.000502098294884654, + "loss": 1.3768, + "theoretical_loss": 3.602575894880741, + "tokens_seen": 1249837056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005020822467402206, + "loss": 1.3509, + "theoretical_loss": 3.602558787806391, + "tokens_seen": 1249902592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005020661985957874, + "loss": 1.3474, + "theoretical_loss": 3.602541681880129, + "tokens_seen": 1249968128 + }, + { + "epoch": 0.38, + "learning_rate": 0.000502050150451354, + "loss": 1.2582, + "theoretical_loss": 3.602524577101817, + "tokens_seen": 1250033664 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005020341023069208, + "loss": 1.353, + "theoretical_loss": 3.602507473471318, + "tokens_seen": 1250099200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005020180541624874, + "loss": 1.3492, + "theoretical_loss": 3.602490370988494, + "tokens_seen": 1250164736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005020020060180542, + "loss": 1.294, + "theoretical_loss": 3.602473269653209, + "tokens_seen": 1250230272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005019859578736208, + "loss": 1.365, + "theoretical_loss": 3.602456169465325, + "tokens_seen": 1250295808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005019699097291876, + "loss": 1.4377, + "theoretical_loss": 3.6024390704247056, + "tokens_seen": 1250361344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005019538615847542, + "loss": 1.28, + "theoretical_loss": 3.6024219725312125, + "tokens_seen": 1250426880 + }, + { + "epoch": 0.38, + "learning_rate": 0.000501937813440321, + "loss": 1.324, + "theoretical_loss": 3.6024048757847105, + "tokens_seen": 1250492416 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005019217652958877, + "loss": 1.2918, + "theoretical_loss": 3.6023877801850603, + "tokens_seen": 1250557952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005019057171514544, + "loss": 1.3186, + "theoretical_loss": 3.6023706857321267, + "tokens_seen": 1250623488 + }, + { + "epoch": 0.38, + "learning_rate": 0.000501889669007021, + "loss": 1.4421, + "theoretical_loss": 3.6023535924257724, + "tokens_seen": 1250689024 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005018736208625878, + "loss": 1.3071, + "theoretical_loss": 3.602336500265859, + "tokens_seen": 1250754560 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005018575727181545, + "loss": 1.4807, + "theoretical_loss": 3.6023194092522517, + "tokens_seen": 1250820096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005018415245737212, + "loss": 1.3308, + "theoretical_loss": 3.602302319384812, + "tokens_seen": 1250885632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005018254764292879, + "loss": 1.4066, + "theoretical_loss": 3.6022852306634032, + "tokens_seen": 1250951168 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005018094282848546, + "loss": 1.3101, + "theoretical_loss": 3.602268143087889, + "tokens_seen": 1251016704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005017933801404213, + "loss": 1.2965, + "theoretical_loss": 3.6022510566581323, + "tokens_seen": 1251082240 + }, + { + "epoch": 0.38, + "learning_rate": 0.000501777331995988, + "loss": 1.4012, + "theoretical_loss": 3.6022339713739964, + "tokens_seen": 1251147776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005017612838515547, + "loss": 1.3074, + "theoretical_loss": 3.6022168872353437, + "tokens_seen": 1251213312 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005017452357071214, + "loss": 1.2211, + "theoretical_loss": 3.602199804242038, + "tokens_seen": 1251278848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005017291875626881, + "loss": 1.3246, + "theoretical_loss": 3.602182722393943, + "tokens_seen": 1251344384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005017131394182548, + "loss": 1.3118, + "theoretical_loss": 3.602165641690921, + "tokens_seen": 1251409920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005016970912738215, + "loss": 1.2714, + "theoretical_loss": 3.602148562132836, + "tokens_seen": 1251475456 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005016810431293882, + "loss": 1.2469, + "theoretical_loss": 3.6021314837195515, + "tokens_seen": 1251540992 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005016649949849549, + "loss": 1.383, + "theoretical_loss": 3.6021144064509296, + "tokens_seen": 1251606528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005016489468405216, + "loss": 1.3218, + "theoretical_loss": 3.6020973303268344, + "tokens_seen": 1251672064 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 420558, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4131748676300049, + "objective/train/theoretical_loss": 3.6020802553471296, + "objective/train/tokens_used": 1272197600, + "theoretical_loss": 3.6020802553471296, + "tokens_seen": 1251737600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005016328986960883, + "loss": 1.3026, + "theoretical_loss": 3.6020802553471296, + "tokens_seen": 1251737600 + }, + { + "epoch": 0.38, + "learning_rate": 0.000501616850551655, + "loss": 1.3667, + "theoretical_loss": 3.6020631815116784, + "tokens_seen": 1251803136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005016008024072217, + "loss": 1.4029, + "theoretical_loss": 3.602046108820344, + "tokens_seen": 1251868672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005015847542627884, + "loss": 1.2877, + "theoretical_loss": 3.60202903727299, + "tokens_seen": 1251934208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005015687061183551, + "loss": 1.3506, + "theoretical_loss": 3.6020119668694797, + "tokens_seen": 1251999744 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005015526579739218, + "loss": 1.3731, + "theoretical_loss": 3.6019948976096767, + "tokens_seen": 1252065280 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005015366098294885, + "loss": 1.3478, + "theoretical_loss": 3.6019778294934444, + "tokens_seen": 1252130816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005015205616850552, + "loss": 1.2647, + "theoretical_loss": 3.6019607625206467, + "tokens_seen": 1252196352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005015045135406219, + "loss": 1.2592, + "theoretical_loss": 3.601943696691147, + "tokens_seen": 1252261888 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005014884653961886, + "loss": 1.3061, + "theoretical_loss": 3.601926632004809, + "tokens_seen": 1252327424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005014724172517553, + "loss": 1.2887, + "theoretical_loss": 3.6019095684614957, + "tokens_seen": 1252392960 + }, + { + "epoch": 0.38, + "learning_rate": 0.000501456369107322, + "loss": 1.3371, + "theoretical_loss": 3.601892506061071, + "tokens_seen": 1252458496 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005014403209628887, + "loss": 1.395, + "theoretical_loss": 3.601875444803399, + "tokens_seen": 1252524032 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005014242728184554, + "loss": 1.3225, + "theoretical_loss": 3.6018583846883434, + "tokens_seen": 1252589568 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005014082246740221, + "loss": 1.5075, + "theoretical_loss": 3.601841325715767, + "tokens_seen": 1252655104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005013921765295888, + "loss": 1.4083, + "theoretical_loss": 3.601824267885535, + "tokens_seen": 1252720640 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005013761283851555, + "loss": 1.3404, + "theoretical_loss": 3.6018072111975092, + "tokens_seen": 1252786176 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005013600802407222, + "loss": 1.4084, + "theoretical_loss": 3.6017901556515546, + "tokens_seen": 1252851712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005013440320962889, + "loss": 1.3497, + "theoretical_loss": 3.6017731012475354, + "tokens_seen": 1252917248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005013279839518556, + "loss": 1.3632, + "theoretical_loss": 3.6017560479853143, + "tokens_seen": 1252982784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005013119358074223, + "loss": 1.3007, + "theoretical_loss": 3.601738995864756, + "tokens_seen": 1253048320 + }, + { + "epoch": 0.38, + "learning_rate": 0.000501295887662989, + "loss": 1.4, + "theoretical_loss": 3.601721944885724, + "tokens_seen": 1253113856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005012798395185557, + "loss": 1.2713, + "theoretical_loss": 3.6017048950480817, + "tokens_seen": 1253179392 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005012637913741224, + "loss": 1.4096, + "theoretical_loss": 3.601687846351694, + "tokens_seen": 1253244928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005012477432296891, + "loss": 1.1946, + "theoretical_loss": 3.6016707987964245, + "tokens_seen": 1253310464 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005012316950852558, + "loss": 1.2877, + "theoretical_loss": 3.601653752382137, + "tokens_seen": 1253376000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005012156469408225, + "loss": 1.3249, + "theoretical_loss": 3.601636707108695, + "tokens_seen": 1253441536 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005011995987963892, + "loss": 1.245, + "theoretical_loss": 3.601619662975964, + "tokens_seen": 1253507072 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005011835506519559, + "loss": 1.4088, + "theoretical_loss": 3.6016026199838063, + "tokens_seen": 1253572608 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005011675025075226, + "loss": 1.4695, + "theoretical_loss": 3.601585578132087, + "tokens_seen": 1253638144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005011514543630893, + "loss": 1.3517, + "theoretical_loss": 3.6015685374206696, + "tokens_seen": 1253703680 + }, + { + "epoch": 0.38, + "learning_rate": 0.000501135406218656, + "loss": 1.3793, + "theoretical_loss": 3.601551497849419, + "tokens_seen": 1253769216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005011193580742227, + "loss": 1.4154, + "theoretical_loss": 3.6015344594181986, + "tokens_seen": 1253834752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005011033099297894, + "loss": 1.3332, + "theoretical_loss": 3.6015174221268724, + "tokens_seen": 1253900288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005010872617853561, + "loss": 1.4641, + "theoretical_loss": 3.601500385975305, + "tokens_seen": 1253965824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005010712136409228, + "loss": 1.3193, + "theoretical_loss": 3.6014833509633606, + "tokens_seen": 1254031360 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005010551654964895, + "loss": 1.308, + "theoretical_loss": 3.6014663170909036, + "tokens_seen": 1254096896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005010391173520562, + "loss": 1.4253, + "theoretical_loss": 3.6014492843577974, + "tokens_seen": 1254162432 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005010230692076229, + "loss": 1.394, + "theoretical_loss": 3.6014322527639075, + "tokens_seen": 1254227968 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005010070210631896, + "loss": 1.3337, + "theoretical_loss": 3.6014152223090967, + "tokens_seen": 1254293504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005009909729187563, + "loss": 1.4764, + "theoretical_loss": 3.601398192993231, + "tokens_seen": 1254359040 + }, + { + "epoch": 0.38, + "learning_rate": 0.000500974924774323, + "loss": 1.2114, + "theoretical_loss": 3.601381164816173, + "tokens_seen": 1254424576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005009588766298897, + "loss": 1.2906, + "theoretical_loss": 3.6013641377777885, + "tokens_seen": 1254490112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005009428284854564, + "loss": 1.3512, + "theoretical_loss": 3.601347111877941, + "tokens_seen": 1254555648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005009267803410231, + "loss": 1.3272, + "theoretical_loss": 3.6013300871164953, + "tokens_seen": 1254621184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005009107321965898, + "loss": 1.3745, + "theoretical_loss": 3.6013130634933157, + "tokens_seen": 1254686720 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005008946840521565, + "loss": 1.3642, + "theoretical_loss": 3.6012960410082666, + "tokens_seen": 1254752256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005008786359077232, + "loss": 1.3109, + "theoretical_loss": 3.601279019661213, + "tokens_seen": 1254817792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005008625877632899, + "loss": 1.3514, + "theoretical_loss": 3.601261999452018, + "tokens_seen": 1254883328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005008465396188565, + "loss": 1.2845, + "theoretical_loss": 3.6012449803805477, + "tokens_seen": 1254948864 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 421519, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.402679443359375, + "objective/train/theoretical_loss": 3.601227962446665, + "objective/train/tokens_used": 1275474400, + "theoretical_loss": 3.601227962446665, + "tokens_seen": 1255014400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005008304914744233, + "loss": 1.3367, + "theoretical_loss": 3.601227962446665, + "tokens_seen": 1255014400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005008144433299899, + "loss": 1.3501, + "theoretical_loss": 3.6012109456502364, + "tokens_seen": 1255079936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005007983951855567, + "loss": 1.3978, + "theoretical_loss": 3.6011939299911253, + "tokens_seen": 1255145472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005007823470411233, + "loss": 1.3117, + "theoretical_loss": 3.601176915469196, + "tokens_seen": 1255211008 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005007662988966901, + "loss": 1.2574, + "theoretical_loss": 3.601159902084315, + "tokens_seen": 1255276544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005007502507522567, + "loss": 1.2656, + "theoretical_loss": 3.6011428898363445, + "tokens_seen": 1255342080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005007342026078235, + "loss": 1.3554, + "theoretical_loss": 3.60112587872515, + "tokens_seen": 1255407616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005007181544633901, + "loss": 1.3155, + "theoretical_loss": 3.6011088687505977, + "tokens_seen": 1255473152 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005007021063189569, + "loss": 1.3541, + "theoretical_loss": 3.6010918599125503, + "tokens_seen": 1255538688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005006860581745235, + "loss": 1.3029, + "theoretical_loss": 3.601074852210873, + "tokens_seen": 1255604224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005006700100300903, + "loss": 1.3996, + "theoretical_loss": 3.601057845645432, + "tokens_seen": 1255669760 + }, + { + "epoch": 0.38, + "learning_rate": 0.000500653961885657, + "loss": 1.2697, + "theoretical_loss": 3.6010408402160907, + "tokens_seen": 1255735296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005006379137412237, + "loss": 1.3648, + "theoretical_loss": 3.601023835922714, + "tokens_seen": 1255800832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005006218655967903, + "loss": 1.4372, + "theoretical_loss": 3.601006832765167, + "tokens_seen": 1255866368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005006058174523571, + "loss": 1.331, + "theoretical_loss": 3.6009898307433144, + "tokens_seen": 1255931904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005005897693079237, + "loss": 1.345, + "theoretical_loss": 3.6009728298570214, + "tokens_seen": 1255997440 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005005737211634905, + "loss": 1.2894, + "theoretical_loss": 3.6009558301061526, + "tokens_seen": 1256062976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005005576730190571, + "loss": 1.3336, + "theoretical_loss": 3.6009388314905735, + "tokens_seen": 1256128512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005005416248746239, + "loss": 1.1996, + "theoretical_loss": 3.6009218340101485, + "tokens_seen": 1256194048 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005005255767301906, + "loss": 1.3561, + "theoretical_loss": 3.600904837664743, + "tokens_seen": 1256259584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005005095285857573, + "loss": 1.3176, + "theoretical_loss": 3.6008878424542208, + "tokens_seen": 1256325120 + }, + { + "epoch": 0.38, + "learning_rate": 0.000500493480441324, + "loss": 1.3492, + "theoretical_loss": 3.600870848378449, + "tokens_seen": 1256390656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005004774322968907, + "loss": 1.2025, + "theoretical_loss": 3.6008538554372906, + "tokens_seen": 1256456192 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005004613841524574, + "loss": 1.2839, + "theoretical_loss": 3.6008368636306116, + "tokens_seen": 1256521728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005004453360080241, + "loss": 1.2362, + "theoretical_loss": 3.6008198729582777, + "tokens_seen": 1256587264 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005004292878635908, + "loss": 1.2907, + "theoretical_loss": 3.600802883420153, + "tokens_seen": 1256652800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005004132397191575, + "loss": 1.3262, + "theoretical_loss": 3.600785895016103, + "tokens_seen": 1256718336 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005003971915747242, + "loss": 1.2668, + "theoretical_loss": 3.600768907745993, + "tokens_seen": 1256783872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005003811434302909, + "loss": 1.27, + "theoretical_loss": 3.6007519216096884, + "tokens_seen": 1256849408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005003650952858576, + "loss": 1.3873, + "theoretical_loss": 3.6007349366070533, + "tokens_seen": 1256914944 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005003490471414243, + "loss": 1.266, + "theoretical_loss": 3.6007179527379547, + "tokens_seen": 1256980480 + }, + { + "epoch": 0.38, + "learning_rate": 0.000500332998996991, + "loss": 1.3877, + "theoretical_loss": 3.600700970002256, + "tokens_seen": 1257046016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005003169508525577, + "loss": 1.2647, + "theoretical_loss": 3.6006839883998243, + "tokens_seen": 1257111552 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005003009027081244, + "loss": 1.3241, + "theoretical_loss": 3.6006670079305234, + "tokens_seen": 1257177088 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005002848545636911, + "loss": 1.2356, + "theoretical_loss": 3.6006500285942193, + "tokens_seen": 1257242624 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005002688064192578, + "loss": 1.3316, + "theoretical_loss": 3.600633050390777, + "tokens_seen": 1257308160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005002527582748245, + "loss": 1.3431, + "theoretical_loss": 3.6006160733200625, + "tokens_seen": 1257373696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005002367101303912, + "loss": 1.3099, + "theoretical_loss": 3.6005990973819406, + "tokens_seen": 1257439232 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005002206619859579, + "loss": 1.3399, + "theoretical_loss": 3.6005821225762764, + "tokens_seen": 1257504768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005002046138415246, + "loss": 1.387, + "theoretical_loss": 3.6005651489029367, + "tokens_seen": 1257570304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005001885656970913, + "loss": 1.3326, + "theoretical_loss": 3.600548176361786, + "tokens_seen": 1257635840 + }, + { + "epoch": 0.38, + "learning_rate": 0.000500172517552658, + "loss": 1.2592, + "theoretical_loss": 3.6005312049526896, + "tokens_seen": 1257701376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005001564694082247, + "loss": 1.3732, + "theoretical_loss": 3.6005142346755132, + "tokens_seen": 1257766912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005001404212637914, + "loss": 1.2656, + "theoretical_loss": 3.600497265530123, + "tokens_seen": 1257832448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005001243731193581, + "loss": 1.2732, + "theoretical_loss": 3.600480297516384, + "tokens_seen": 1257897984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005001083249749248, + "loss": 1.3049, + "theoretical_loss": 3.6004633306341614, + "tokens_seen": 1257963520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005000922768304915, + "loss": 1.4004, + "theoretical_loss": 3.6004463648833216, + "tokens_seen": 1258029056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005000762286860582, + "loss": 1.3171, + "theoretical_loss": 3.6004294002637294, + "tokens_seen": 1258094592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005000601805416249, + "loss": 1.2923, + "theoretical_loss": 3.600412436775251, + "tokens_seen": 1258160128 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005000441323971916, + "loss": 1.4036, + "theoretical_loss": 3.600395474417752, + "tokens_seen": 1258225664 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 422629, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2424266338348389, + "objective/train/theoretical_loss": 3.600378513191098, + "objective/train/tokens_used": 1278751200, + "theoretical_loss": 3.600378513191098, + "tokens_seen": 1258291200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005000280842527583, + "loss": 1.3259, + "theoretical_loss": 3.600378513191098, + "tokens_seen": 1258291200 + }, + { + "epoch": 0.38, + "learning_rate": 0.000500012036108325, + "loss": 1.3379, + "theoretical_loss": 3.600361553095155, + "tokens_seen": 1258356736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004999959879638917, + "loss": 1.4047, + "theoretical_loss": 3.6003445941297882, + "tokens_seen": 1258422272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004999799398194584, + "loss": 1.2773, + "theoretical_loss": 3.600327636294864, + "tokens_seen": 1258487808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004999638916750251, + "loss": 1.3321, + "theoretical_loss": 3.6003106795902475, + "tokens_seen": 1258553344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004999478435305918, + "loss": 1.3351, + "theoretical_loss": 3.600293724015805, + "tokens_seen": 1258618880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004999317953861585, + "loss": 1.3151, + "theoretical_loss": 3.600276769571402, + "tokens_seen": 1258684416 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004999157472417252, + "loss": 1.2681, + "theoretical_loss": 3.6002598162569046, + "tokens_seen": 1258749952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004998996990972919, + "loss": 1.2733, + "theoretical_loss": 3.600242864072179, + "tokens_seen": 1258815488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004998836509528586, + "loss": 1.239, + "theoretical_loss": 3.6002259130170904, + "tokens_seen": 1258881024 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004998676028084253, + "loss": 1.3811, + "theoretical_loss": 3.6002089630915046, + "tokens_seen": 1258946560 + }, + { + "epoch": 0.38, + "learning_rate": 0.000499851554663992, + "loss": 1.2456, + "theoretical_loss": 3.6001920142952883, + "tokens_seen": 1259012096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004998355065195587, + "loss": 1.35, + "theoretical_loss": 3.6001750666283074, + "tokens_seen": 1259077632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004998194583751254, + "loss": 1.2933, + "theoretical_loss": 3.600158120090428, + "tokens_seen": 1259143168 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004998034102306921, + "loss": 1.2378, + "theoretical_loss": 3.600141174681515, + "tokens_seen": 1259208704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004997873620862588, + "loss": 1.2058, + "theoretical_loss": 3.6001242304014354, + "tokens_seen": 1259274240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004997713139418255, + "loss": 1.3714, + "theoretical_loss": 3.6001072872500552, + "tokens_seen": 1259339776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004997552657973922, + "loss": 1.3448, + "theoretical_loss": 3.6000903452272404, + "tokens_seen": 1259405312 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004997392176529589, + "loss": 1.3862, + "theoretical_loss": 3.600073404332856, + "tokens_seen": 1259470848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004997231695085256, + "loss": 1.3522, + "theoretical_loss": 3.6000564645667703, + "tokens_seen": 1259536384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004997071213640923, + "loss": 1.1841, + "theoretical_loss": 3.6000395259288482, + "tokens_seen": 1259601920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004996910732196591, + "loss": 1.3151, + "theoretical_loss": 3.6000225884189563, + "tokens_seen": 1259667456 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004996750250752257, + "loss": 1.1988, + "theoretical_loss": 3.60000565203696, + "tokens_seen": 1259732992 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004996589769307925, + "loss": 1.2379, + "theoretical_loss": 3.599988716782726, + "tokens_seen": 1259798528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004996429287863591, + "loss": 1.3703, + "theoretical_loss": 3.5999717826561204, + "tokens_seen": 1259864064 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004996268806419259, + "loss": 1.3443, + "theoretical_loss": 3.59995484965701, + "tokens_seen": 1259929600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004996108324974925, + "loss": 1.307, + "theoretical_loss": 3.5999379177852604, + "tokens_seen": 1259995136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004995947843530592, + "loss": 1.297, + "theoretical_loss": 3.5999209870407385, + "tokens_seen": 1260060672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004995787362086259, + "loss": 1.3657, + "theoretical_loss": 3.59990405742331, + "tokens_seen": 1260126208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004995626880641926, + "loss": 1.3301, + "theoretical_loss": 3.599887128932842, + "tokens_seen": 1260191744 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004995466399197593, + "loss": 1.3567, + "theoretical_loss": 3.5998702015692006, + "tokens_seen": 1260257280 + }, + { + "epoch": 0.38, + "learning_rate": 0.000499530591775326, + "loss": 1.2652, + "theoretical_loss": 3.5998532753322516, + "tokens_seen": 1260322816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004995145436308927, + "loss": 1.323, + "theoretical_loss": 3.599836350221862, + "tokens_seen": 1260388352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004994984954864594, + "loss": 1.2185, + "theoretical_loss": 3.5998194262378984, + "tokens_seen": 1260453888 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004994824473420261, + "loss": 1.312, + "theoretical_loss": 3.599802503380227, + "tokens_seen": 1260519424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004994663991975928, + "loss": 1.2526, + "theoretical_loss": 3.5997855816487143, + "tokens_seen": 1260584960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004994503510531594, + "loss": 1.1733, + "theoretical_loss": 3.599768661043227, + "tokens_seen": 1260650496 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004994343029087262, + "loss": 1.2027, + "theoretical_loss": 3.5997517415636313, + "tokens_seen": 1260716032 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004994182547642928, + "loss": 1.2417, + "theoretical_loss": 3.599734823209794, + "tokens_seen": 1260781568 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004994022066198596, + "loss": 1.2964, + "theoretical_loss": 3.5997179059815814, + "tokens_seen": 1260847104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004993861584754262, + "loss": 1.2988, + "theoretical_loss": 3.599700989878861, + "tokens_seen": 1260912640 + }, + { + "epoch": 0.38, + "learning_rate": 0.000499370110330993, + "loss": 1.3054, + "theoretical_loss": 3.599684074901498, + "tokens_seen": 1260978176 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004993540621865596, + "loss": 1.172, + "theoretical_loss": 3.5996671610493602, + "tokens_seen": 1261043712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004993380140421264, + "loss": 1.3809, + "theoretical_loss": 3.599650248322314, + "tokens_seen": 1261109248 + }, + { + "epoch": 0.38, + "learning_rate": 0.000499321965897693, + "loss": 1.2898, + "theoretical_loss": 3.599633336720226, + "tokens_seen": 1261174784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004993059177532598, + "loss": 1.2499, + "theoretical_loss": 3.599616426242963, + "tokens_seen": 1261240320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004992898696088264, + "loss": 1.2464, + "theoretical_loss": 3.5995995168903914, + "tokens_seen": 1261305856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004992738214643932, + "loss": 1.3148, + "theoretical_loss": 3.599582608662378, + "tokens_seen": 1261371392 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004992577733199598, + "loss": 1.2937, + "theoretical_loss": 3.59956570155879, + "tokens_seen": 1261436928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004992417251755266, + "loss": 1.2998, + "theoretical_loss": 3.5995487955794943, + "tokens_seen": 1261502464 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 423890, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3517919778823853, + "objective/train/theoretical_loss": 3.599531890724357, + "objective/train/tokens_used": 1282028000, + "theoretical_loss": 3.599531890724357, + "tokens_seen": 1261568000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004992256770310932, + "loss": 1.3402, + "theoretical_loss": 3.599531890724357, + "tokens_seen": 1261568000 + }, + { + "epoch": 0.38, + "learning_rate": 0.00049920962888666, + "loss": 1.2594, + "theoretical_loss": 3.599514986993246, + "tokens_seen": 1261633536 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004991935807422266, + "loss": 1.2545, + "theoretical_loss": 3.5994980843860276, + "tokens_seen": 1261699072 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004991775325977934, + "loss": 1.4063, + "theoretical_loss": 3.5994811829025686, + "tokens_seen": 1261764608 + }, + { + "epoch": 0.38, + "learning_rate": 0.00049916148445336, + "loss": 1.342, + "theoretical_loss": 3.599464282542735, + "tokens_seen": 1261830144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004991454363089268, + "loss": 1.3544, + "theoretical_loss": 3.599447383306396, + "tokens_seen": 1261895680 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004991293881644935, + "loss": 1.2332, + "theoretical_loss": 3.5994304851934174, + "tokens_seen": 1261961216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004991133400200602, + "loss": 1.3271, + "theoretical_loss": 3.5994135882036655, + "tokens_seen": 1262026752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004990972918756269, + "loss": 1.1862, + "theoretical_loss": 3.5993966923370087, + "tokens_seen": 1262092288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004990812437311936, + "loss": 1.3249, + "theoretical_loss": 3.599379797593312, + "tokens_seen": 1262157824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004990651955867603, + "loss": 1.3205, + "theoretical_loss": 3.599362903972445, + "tokens_seen": 1262223360 + }, + { + "epoch": 0.38, + "learning_rate": 0.000499049147442327, + "loss": 1.3479, + "theoretical_loss": 3.5993460114742732, + "tokens_seen": 1262288896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004990330992978937, + "loss": 1.2175, + "theoretical_loss": 3.599329120098664, + "tokens_seen": 1262354432 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004990170511534604, + "loss": 1.3575, + "theoretical_loss": 3.5993122298454843, + "tokens_seen": 1262419968 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004990010030090271, + "loss": 1.3866, + "theoretical_loss": 3.599295340714602, + "tokens_seen": 1262485504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004989849548645938, + "loss": 1.3307, + "theoretical_loss": 3.5992784527058834, + "tokens_seen": 1262551040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004989689067201605, + "loss": 1.2999, + "theoretical_loss": 3.5992615658191967, + "tokens_seen": 1262616576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004989528585757272, + "loss": 1.2711, + "theoretical_loss": 3.599244680054408, + "tokens_seen": 1262682112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004989368104312939, + "loss": 1.2886, + "theoretical_loss": 3.599227795411385, + "tokens_seen": 1262747648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004989207622868606, + "loss": 1.3036, + "theoretical_loss": 3.5992109118899958, + "tokens_seen": 1262813184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004989047141424273, + "loss": 1.3382, + "theoretical_loss": 3.599194029490106, + "tokens_seen": 1262878720 + }, + { + "epoch": 0.38, + "learning_rate": 0.000498888665997994, + "loss": 1.3411, + "theoretical_loss": 3.5991771482115844, + "tokens_seen": 1262944256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004988726178535607, + "loss": 1.3098, + "theoretical_loss": 3.5991602680542973, + "tokens_seen": 1263009792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004988565697091274, + "loss": 1.3073, + "theoretical_loss": 3.5991433890181126, + "tokens_seen": 1263075328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004988405215646941, + "loss": 1.2819, + "theoretical_loss": 3.599126511102898, + "tokens_seen": 1263140864 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004988244734202608, + "loss": 1.311, + "theoretical_loss": 3.59910963430852, + "tokens_seen": 1263206400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004988084252758275, + "loss": 1.3306, + "theoretical_loss": 3.599092758634847, + "tokens_seen": 1263271936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004987923771313942, + "loss": 1.2372, + "theoretical_loss": 3.5990758840817456, + "tokens_seen": 1263337472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004987763289869609, + "loss": 1.2025, + "theoretical_loss": 3.599059010649084, + "tokens_seen": 1263403008 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004987602808425276, + "loss": 1.2648, + "theoretical_loss": 3.5990421383367286, + "tokens_seen": 1263468544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004987442326980943, + "loss": 1.3064, + "theoretical_loss": 3.5990252671445484, + "tokens_seen": 1263534080 + }, + { + "epoch": 0.38, + "learning_rate": 0.000498728184553661, + "loss": 1.3771, + "theoretical_loss": 3.5990083970724096, + "tokens_seen": 1263599616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004987121364092277, + "loss": 1.2826, + "theoretical_loss": 3.5989915281201807, + "tokens_seen": 1263665152 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004986960882647944, + "loss": 1.3061, + "theoretical_loss": 3.5989746602877286, + "tokens_seen": 1263730688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004986800401203611, + "loss": 1.2911, + "theoretical_loss": 3.5989577935749217, + "tokens_seen": 1263796224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004986639919759278, + "loss": 1.2631, + "theoretical_loss": 3.598940927981627, + "tokens_seen": 1263861760 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004986479438314945, + "loss": 1.3076, + "theoretical_loss": 3.598924063507712, + "tokens_seen": 1263927296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004986318956870612, + "loss": 1.3675, + "theoretical_loss": 3.598907200153045, + "tokens_seen": 1263992832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004986158475426279, + "loss": 1.2881, + "theoretical_loss": 3.598890337917493, + "tokens_seen": 1264058368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004985997993981946, + "loss": 1.3467, + "theoretical_loss": 3.5988734768009243, + "tokens_seen": 1264123904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004985837512537613, + "loss": 1.3733, + "theoretical_loss": 3.5988566168032063, + "tokens_seen": 1264189440 + }, + { + "epoch": 0.38, + "learning_rate": 0.000498567703109328, + "loss": 1.3434, + "theoretical_loss": 3.598839757924207, + "tokens_seen": 1264254976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004985516549648947, + "loss": 1.2633, + "theoretical_loss": 3.5988229001637935, + "tokens_seen": 1264320512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004985356068204614, + "loss": 1.2668, + "theoretical_loss": 3.5988060435218348, + "tokens_seen": 1264386048 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004985195586760281, + "loss": 1.2314, + "theoretical_loss": 3.598789187998198, + "tokens_seen": 1264451584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004985035105315948, + "loss": 1.3266, + "theoretical_loss": 3.5987723335927506, + "tokens_seen": 1264517120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004984874623871616, + "loss": 1.2526, + "theoretical_loss": 3.5987554803053614, + "tokens_seen": 1264582656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004984714142427282, + "loss": 1.3122, + "theoretical_loss": 3.5987386281358975, + "tokens_seen": 1264648192 + }, + { + "epoch": 0.38, + "learning_rate": 0.000498455366098295, + "loss": 1.3031, + "theoretical_loss": 3.5987217770842275, + "tokens_seen": 1264713728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004984393179538616, + "loss": 1.1985, + "theoretical_loss": 3.5987049271502185, + "tokens_seen": 1264779264 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 424734, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4077988862991333, + "objective/train/theoretical_loss": 3.5986880783337387, + "objective/train/tokens_used": 1285304800, + "theoretical_loss": 3.5986880783337387, + "tokens_seen": 1264844800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004984232698094284, + "loss": 1.3125, + "theoretical_loss": 3.5986880783337387, + "tokens_seen": 1264844800 + }, + { + "epoch": 0.38, + "learning_rate": 0.000498407221664995, + "loss": 1.1478, + "theoretical_loss": 3.598671230634657, + "tokens_seen": 1264910336 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004983911735205618, + "loss": 1.2639, + "theoretical_loss": 3.59865438405284, + "tokens_seen": 1264975872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004983751253761284, + "loss": 1.2707, + "theoretical_loss": 3.5986375385881573, + "tokens_seen": 1265041408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004983590772316952, + "loss": 1.3552, + "theoretical_loss": 3.5986206942404757, + "tokens_seen": 1265106944 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004983430290872618, + "loss": 1.272, + "theoretical_loss": 3.5986038510096634, + "tokens_seen": 1265172480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004983269809428286, + "loss": 1.3361, + "theoretical_loss": 3.5985870088955894, + "tokens_seen": 1265238016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004983109327983952, + "loss": 1.278, + "theoretical_loss": 3.598570167898121, + "tokens_seen": 1265303552 + }, + { + "epoch": 0.38, + "learning_rate": 0.000498294884653962, + "loss": 1.3644, + "theoretical_loss": 3.5985533280171262, + "tokens_seen": 1265369088 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004982788365095286, + "loss": 1.5033, + "theoretical_loss": 3.5985364892524734, + "tokens_seen": 1265434624 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004982627883650954, + "loss": 1.3121, + "theoretical_loss": 3.598519651604031, + "tokens_seen": 1265500160 + }, + { + "epoch": 0.38, + "learning_rate": 0.000498246740220662, + "loss": 1.2772, + "theoretical_loss": 3.598502815071667, + "tokens_seen": 1265565696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004982306920762288, + "loss": 1.3747, + "theoretical_loss": 3.5984859796552504, + "tokens_seen": 1265631232 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004982146439317954, + "loss": 1.3262, + "theoretical_loss": 3.598469145354648, + "tokens_seen": 1265696768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004981985957873622, + "loss": 1.2549, + "theoretical_loss": 3.598452312169729, + "tokens_seen": 1265762304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004981825476429288, + "loss": 1.2031, + "theoretical_loss": 3.598435480100362, + "tokens_seen": 1265827840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004981664994984955, + "loss": 1.3247, + "theoretical_loss": 3.5984186491464145, + "tokens_seen": 1265893376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004981504513540622, + "loss": 1.2843, + "theoretical_loss": 3.598401819307756, + "tokens_seen": 1265958912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004981344032096289, + "loss": 1.3527, + "theoretical_loss": 3.598384990584253, + "tokens_seen": 1266024448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004981183550651956, + "loss": 1.3106, + "theoretical_loss": 3.5983681629757753, + "tokens_seen": 1266089984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004981023069207623, + "loss": 1.3004, + "theoretical_loss": 3.598351336482191, + "tokens_seen": 1266155520 + }, + { + "epoch": 0.38, + "learning_rate": 0.000498086258776329, + "loss": 1.3196, + "theoretical_loss": 3.598334511103369, + "tokens_seen": 1266221056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004980702106318957, + "loss": 1.3494, + "theoretical_loss": 3.5983176868391764, + "tokens_seen": 1266286592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004980541624874624, + "loss": 1.3617, + "theoretical_loss": 3.5983008636894835, + "tokens_seen": 1266352128 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004980381143430291, + "loss": 1.2671, + "theoretical_loss": 3.598284041654157, + "tokens_seen": 1266417664 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004980220661985957, + "loss": 1.2546, + "theoretical_loss": 3.598267220733067, + "tokens_seen": 1266483200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004980060180541625, + "loss": 1.2556, + "theoretical_loss": 3.598250400926081, + "tokens_seen": 1266548736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004979899699097291, + "loss": 1.2886, + "theoretical_loss": 3.5982335822330676, + "tokens_seen": 1266614272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004979739217652959, + "loss": 1.106, + "theoretical_loss": 3.5982167646538956, + "tokens_seen": 1266679808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004979578736208625, + "loss": 1.347, + "theoretical_loss": 3.5981999481884346, + "tokens_seen": 1266745344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004979418254764293, + "loss": 1.2945, + "theoretical_loss": 3.5981831328365512, + "tokens_seen": 1266810880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004979257773319959, + "loss": 1.2477, + "theoretical_loss": 3.598166318598116, + "tokens_seen": 1266876416 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004979097291875627, + "loss": 1.3321, + "theoretical_loss": 3.5981495054729966, + "tokens_seen": 1266941952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004978936810431293, + "loss": 1.1598, + "theoretical_loss": 3.5981326934610616, + "tokens_seen": 1267007488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004978776328986961, + "loss": 1.2386, + "theoretical_loss": 3.5981158825621806, + "tokens_seen": 1267073024 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004978615847542627, + "loss": 1.2771, + "theoretical_loss": 3.598099072776222, + "tokens_seen": 1267138560 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004978455366098295, + "loss": 1.2073, + "theoretical_loss": 3.598082264103054, + "tokens_seen": 1267204096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004978294884653961, + "loss": 1.1953, + "theoretical_loss": 3.5980654565425456, + "tokens_seen": 1267269632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004978134403209629, + "loss": 1.253, + "theoretical_loss": 3.598048650094566, + "tokens_seen": 1267335168 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004977973921765296, + "loss": 1.2272, + "theoretical_loss": 3.598031844758984, + "tokens_seen": 1267400704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004977813440320963, + "loss": 1.3106, + "theoretical_loss": 3.598015040535668, + "tokens_seen": 1267466240 + }, + { + "epoch": 0.38, + "learning_rate": 0.000497765295887663, + "loss": 1.4032, + "theoretical_loss": 3.597998237424487, + "tokens_seen": 1267531776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004977492477432297, + "loss": 1.3226, + "theoretical_loss": 3.597981435425311, + "tokens_seen": 1267597312 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004977331995987964, + "loss": 1.2629, + "theoretical_loss": 3.597964634538007, + "tokens_seen": 1267662848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004977171514543631, + "loss": 1.3556, + "theoretical_loss": 3.5979478347624454, + "tokens_seen": 1267728384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004977011033099298, + "loss": 1.3254, + "theoretical_loss": 3.5979310360984944, + "tokens_seen": 1267793920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004976850551654965, + "loss": 1.2801, + "theoretical_loss": 3.597914238546023, + "tokens_seen": 1267859456 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004976690070210632, + "loss": 1.2658, + "theoretical_loss": 3.5978974421049017, + "tokens_seen": 1267924992 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004976529588766299, + "loss": 1.217, + "theoretical_loss": 3.5978806467749975, + "tokens_seen": 1267990528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004976369107321966, + "loss": 1.3581, + "theoretical_loss": 3.5978638525561806, + "tokens_seen": 1268056064 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 425936, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3263369798660278, + "objective/train/theoretical_loss": 3.5978470594483194, + "objective/train/tokens_used": 1288581600, + "theoretical_loss": 3.5978470594483194, + "tokens_seen": 1268121600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004976208625877633, + "loss": 1.2524, + "theoretical_loss": 3.5978470594483194, + "tokens_seen": 1268121600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00049760481444333, + "loss": 1.2851, + "theoretical_loss": 3.5978302674512834, + "tokens_seen": 1268187136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004975887662988967, + "loss": 1.3585, + "theoretical_loss": 3.5978134765649417, + "tokens_seen": 1268252672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004975727181544634, + "loss": 1.2964, + "theoretical_loss": 3.597796686789164, + "tokens_seen": 1268318208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004975566700100301, + "loss": 1.2648, + "theoretical_loss": 3.5977798981238185, + "tokens_seen": 1268383744 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004975406218655968, + "loss": 1.2545, + "theoretical_loss": 3.597763110568775, + "tokens_seen": 1268449280 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004975245737211635, + "loss": 1.3272, + "theoretical_loss": 3.597746324123902, + "tokens_seen": 1268514816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004975085255767302, + "loss": 1.265, + "theoretical_loss": 3.5977295387890695, + "tokens_seen": 1268580352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004974924774322969, + "loss": 1.3263, + "theoretical_loss": 3.5977127545641467, + "tokens_seen": 1268645888 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004974764292878636, + "loss": 1.2467, + "theoretical_loss": 3.597695971449003, + "tokens_seen": 1268711424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004974603811434303, + "loss": 1.3192, + "theoretical_loss": 3.5976791894435065, + "tokens_seen": 1268776960 + }, + { + "epoch": 0.38, + "learning_rate": 0.000497444332998997, + "loss": 1.251, + "theoretical_loss": 3.597662408547528, + "tokens_seen": 1268842496 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004974282848545637, + "loss": 1.2984, + "theoretical_loss": 3.5976456287609357, + "tokens_seen": 1268908032 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004974122367101304, + "loss": 1.3213, + "theoretical_loss": 3.5976288500836002, + "tokens_seen": 1268973568 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004973961885656971, + "loss": 1.3017, + "theoretical_loss": 3.59761207251539, + "tokens_seen": 1269039104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004973801404212638, + "loss": 1.3569, + "theoretical_loss": 3.5975952960561743, + "tokens_seen": 1269104640 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004973640922768305, + "loss": 1.3836, + "theoretical_loss": 3.5975785207058237, + "tokens_seen": 1269170176 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004973480441323972, + "loss": 1.3471, + "theoretical_loss": 3.5975617464642067, + "tokens_seen": 1269235712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004973319959879639, + "loss": 1.3619, + "theoretical_loss": 3.5975449733311926, + "tokens_seen": 1269301248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004973159478435306, + "loss": 1.3563, + "theoretical_loss": 3.597528201306651, + "tokens_seen": 1269366784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004972998996990973, + "loss": 1.2156, + "theoretical_loss": 3.5975114303904525, + "tokens_seen": 1269432320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004972838515546641, + "loss": 1.2841, + "theoretical_loss": 3.5974946605824654, + "tokens_seen": 1269497856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004972678034102307, + "loss": 1.3345, + "theoretical_loss": 3.59747789188256, + "tokens_seen": 1269563392 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004972517552657975, + "loss": 1.3687, + "theoretical_loss": 3.5974611242906054, + "tokens_seen": 1269628928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004972357071213641, + "loss": 1.3154, + "theoretical_loss": 3.597444357806471, + "tokens_seen": 1269694464 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004972196589769309, + "loss": 1.2488, + "theoretical_loss": 3.5974275924300274, + "tokens_seen": 1269760000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004972036108324975, + "loss": 1.2299, + "theoretical_loss": 3.597410828161144, + "tokens_seen": 1269825536 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004971875626880643, + "loss": 1.3637, + "theoretical_loss": 3.5973940649996896, + "tokens_seen": 1269891072 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004971715145436309, + "loss": 1.2875, + "theoretical_loss": 3.5973773029455343, + "tokens_seen": 1269956608 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004971554663991977, + "loss": 1.4486, + "theoretical_loss": 3.5973605419985484, + "tokens_seen": 1270022144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004971394182547643, + "loss": 1.3388, + "theoretical_loss": 3.5973437821586014, + "tokens_seen": 1270087680 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004971233701103311, + "loss": 1.4281, + "theoretical_loss": 3.5973270234255628, + "tokens_seen": 1270153216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004971073219658977, + "loss": 1.3701, + "theoretical_loss": 3.597310265799302, + "tokens_seen": 1270218752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004970912738214645, + "loss": 1.2282, + "theoretical_loss": 3.5972935092796896, + "tokens_seen": 1270284288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004970752256770311, + "loss": 1.3315, + "theoretical_loss": 3.5972767538665953, + "tokens_seen": 1270349824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004970591775325979, + "loss": 1.2759, + "theoretical_loss": 3.5972599995598884, + "tokens_seen": 1270415360 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004970431293881645, + "loss": 1.3413, + "theoretical_loss": 3.5972432463594393, + "tokens_seen": 1270480896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004970270812437313, + "loss": 1.3109, + "theoretical_loss": 3.5972264942651178, + "tokens_seen": 1270546432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004970110330992979, + "loss": 1.3672, + "theoretical_loss": 3.597209743276794, + "tokens_seen": 1270611968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004969949849548647, + "loss": 1.3108, + "theoretical_loss": 3.5971929933943376, + "tokens_seen": 1270677504 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004969789368104313, + "loss": 1.332, + "theoretical_loss": 3.5971762446176183, + "tokens_seen": 1270743040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004969628886659981, + "loss": 1.3023, + "theoretical_loss": 3.5971594969465066, + "tokens_seen": 1270808576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004969468405215647, + "loss": 1.3514, + "theoretical_loss": 3.597142750380872, + "tokens_seen": 1270874112 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004969307923771315, + "loss": 1.2509, + "theoretical_loss": 3.597126004920585, + "tokens_seen": 1270939648 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004969147442326981, + "loss": 1.15, + "theoretical_loss": 3.5971092605655155, + "tokens_seen": 1271005184 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004968986960882649, + "loss": 1.2723, + "theoretical_loss": 3.597092517315533, + "tokens_seen": 1271070720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004968826479438315, + "loss": 1.1763, + "theoretical_loss": 3.5970757751705085, + "tokens_seen": 1271136256 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004968665997993983, + "loss": 1.3644, + "theoretical_loss": 3.5970590341303117, + "tokens_seen": 1271201792 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004968505516549649, + "loss": 1.2889, + "theoretical_loss": 3.5970422941948126, + "tokens_seen": 1271267328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004968345035105317, + "loss": 1.3835, + "theoretical_loss": 3.5970255553638815, + "tokens_seen": 1271332864 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 426775, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3854188919067383, + "objective/train/theoretical_loss": 3.5970088176373887, + "objective/train/tokens_used": 1291858400, + "theoretical_loss": 3.5970088176373887, + "tokens_seen": 1271398400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004968184553660983, + "loss": 1.3204, + "theoretical_loss": 3.5970088176373887, + "tokens_seen": 1271398400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004968024072216651, + "loss": 1.3744, + "theoretical_loss": 3.5969920810152045, + "tokens_seen": 1271463936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004967863590772317, + "loss": 1.3329, + "theoretical_loss": 3.596975345497199, + "tokens_seen": 1271529472 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004967703109327984, + "loss": 1.2875, + "theoretical_loss": 3.596958611083242, + "tokens_seen": 1271595008 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004967542627883651, + "loss": 1.263, + "theoretical_loss": 3.5969418777732045, + "tokens_seen": 1271660544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004967382146439318, + "loss": 1.2913, + "theoretical_loss": 3.5969251455669564, + "tokens_seen": 1271726080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004967221664994985, + "loss": 1.2597, + "theoretical_loss": 3.5969084144643677, + "tokens_seen": 1271791616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004967061183550652, + "loss": 1.211, + "theoretical_loss": 3.5968916844653096, + "tokens_seen": 1271857152 + }, + { + "epoch": 0.39, + "learning_rate": 0.000496690070210632, + "loss": 1.2919, + "theoretical_loss": 3.5968749555696515, + "tokens_seen": 1271922688 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004966740220661986, + "loss": 1.349, + "theoretical_loss": 3.5968582277772647, + "tokens_seen": 1271988224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004966579739217653, + "loss": 1.1978, + "theoretical_loss": 3.5968415010880186, + "tokens_seen": 1272053760 + }, + { + "epoch": 0.39, + "learning_rate": 0.000496641925777332, + "loss": 1.265, + "theoretical_loss": 3.5968247755017844, + "tokens_seen": 1272119296 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004966258776328986, + "loss": 1.2978, + "theoretical_loss": 3.5968080510184324, + "tokens_seen": 1272184832 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004966098294884654, + "loss": 1.3129, + "theoretical_loss": 3.596791327637833, + "tokens_seen": 1272250368 + }, + { + "epoch": 0.39, + "learning_rate": 0.000496593781344032, + "loss": 1.304, + "theoretical_loss": 3.596774605359857, + "tokens_seen": 1272315904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004965777331995988, + "loss": 1.3701, + "theoretical_loss": 3.596757884184374, + "tokens_seen": 1272381440 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004965616850551654, + "loss": 1.2884, + "theoretical_loss": 3.596741164111255, + "tokens_seen": 1272446976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004965456369107322, + "loss": 1.3437, + "theoretical_loss": 3.596724445140371, + "tokens_seen": 1272512512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004965295887662988, + "loss": 1.2489, + "theoretical_loss": 3.596707727271592, + "tokens_seen": 1272578048 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004965135406218656, + "loss": 1.2302, + "theoretical_loss": 3.596691010504789, + "tokens_seen": 1272643584 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004964974924774322, + "loss": 1.3499, + "theoretical_loss": 3.5966742948398327, + "tokens_seen": 1272709120 + }, + { + "epoch": 0.39, + "learning_rate": 0.000496481444332999, + "loss": 1.3188, + "theoretical_loss": 3.596657580276593, + "tokens_seen": 1272774656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004964653961885656, + "loss": 1.3194, + "theoretical_loss": 3.5966408668149414, + "tokens_seen": 1272840192 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004964493480441324, + "loss": 1.3017, + "theoretical_loss": 3.5966241544547484, + "tokens_seen": 1272905728 + }, + { + "epoch": 0.39, + "learning_rate": 0.000496433299899699, + "loss": 1.2717, + "theoretical_loss": 3.596607443195884, + "tokens_seen": 1272971264 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004964172517552658, + "loss": 1.32, + "theoretical_loss": 3.59659073303822, + "tokens_seen": 1273036800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004964012036108325, + "loss": 1.2705, + "theoretical_loss": 3.5965740239816264, + "tokens_seen": 1273102336 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004963851554663992, + "loss": 1.2454, + "theoretical_loss": 3.596557316025974, + "tokens_seen": 1273167872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004963691073219659, + "loss": 1.209, + "theoretical_loss": 3.596540609171134, + "tokens_seen": 1273233408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004963530591775326, + "loss": 1.2642, + "theoretical_loss": 3.596523903416977, + "tokens_seen": 1273298944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004963370110330993, + "loss": 1.3399, + "theoretical_loss": 3.5965071987633745, + "tokens_seen": 1273364480 + }, + { + "epoch": 0.39, + "learning_rate": 0.000496320962888666, + "loss": 1.2915, + "theoretical_loss": 3.5964904952101957, + "tokens_seen": 1273430016 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004963049147442327, + "loss": 1.1884, + "theoretical_loss": 3.5964737927573136, + "tokens_seen": 1273495552 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004962888665997994, + "loss": 1.2608, + "theoretical_loss": 3.596457091404597, + "tokens_seen": 1273561088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004962728184553661, + "loss": 1.2978, + "theoretical_loss": 3.5964403911519183, + "tokens_seen": 1273626624 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004962567703109328, + "loss": 1.3278, + "theoretical_loss": 3.596423691999148, + "tokens_seen": 1273692160 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004962407221664995, + "loss": 1.3374, + "theoretical_loss": 3.5964069939461565, + "tokens_seen": 1273757696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004962246740220662, + "loss": 1.2383, + "theoretical_loss": 3.596390296992816, + "tokens_seen": 1273823232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004962086258776329, + "loss": 1.4236, + "theoretical_loss": 3.5963736011389966, + "tokens_seen": 1273888768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004961925777331996, + "loss": 1.2526, + "theoretical_loss": 3.59635690638457, + "tokens_seen": 1273954304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004961765295887663, + "loss": 1.3395, + "theoretical_loss": 3.596340212729406, + "tokens_seen": 1274019840 + }, + { + "epoch": 0.39, + "learning_rate": 0.000496160481444333, + "loss": 1.3486, + "theoretical_loss": 3.596323520173377, + "tokens_seen": 1274085376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004961444332998997, + "loss": 1.3048, + "theoretical_loss": 3.5963068287163535, + "tokens_seen": 1274150912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004961283851554664, + "loss": 1.2996, + "theoretical_loss": 3.5962901383582073, + "tokens_seen": 1274216448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004961123370110331, + "loss": 1.3083, + "theoretical_loss": 3.596273449098809, + "tokens_seen": 1274281984 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004960962888665998, + "loss": 1.2555, + "theoretical_loss": 3.5962567609380285, + "tokens_seen": 1274347520 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004960802407221666, + "loss": 1.2913, + "theoretical_loss": 3.596240073875739, + "tokens_seen": 1274413056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004960641925777332, + "loss": 1.2185, + "theoretical_loss": 3.5962233879118113, + "tokens_seen": 1274478592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004960481444333, + "loss": 1.3612, + "theoretical_loss": 3.596206703046116, + "tokens_seen": 1274544128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004960320962888666, + "loss": 1.3007, + "theoretical_loss": 3.5961900192785246, + "tokens_seen": 1274609664 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 427982, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2709904909133911, + "objective/train/theoretical_loss": 3.5961733366089086, + "objective/train/tokens_used": 1295135200, + "theoretical_loss": 3.5961733366089086, + "tokens_seen": 1274675200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004960160481444334, + "loss": 1.2709, + "theoretical_loss": 3.5961733366089086, + "tokens_seen": 1274675200 + }, + { + "epoch": 0.39, + "learning_rate": 0.000496, + "loss": 1.3176, + "theoretical_loss": 3.596156655037139, + "tokens_seen": 1274740736 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004959839518555668, + "loss": 1.2992, + "theoretical_loss": 3.596139974563087, + "tokens_seen": 1274806272 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004959679037111334, + "loss": 1.2785, + "theoretical_loss": 3.5961232951866244, + "tokens_seen": 1274871808 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004959518555667002, + "loss": 1.272, + "theoretical_loss": 3.5961066169076226, + "tokens_seen": 1274937344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004959358074222668, + "loss": 1.2913, + "theoretical_loss": 3.5960899397259523, + "tokens_seen": 1275002880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004959197592778336, + "loss": 1.3332, + "theoretical_loss": 3.596073263641485, + "tokens_seen": 1275068416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004959037111334002, + "loss": 1.3603, + "theoretical_loss": 3.596056588654093, + "tokens_seen": 1275133952 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495887662988967, + "loss": 1.2949, + "theoretical_loss": 3.596039914763647, + "tokens_seen": 1275199488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004958716148445336, + "loss": 1.4417, + "theoretical_loss": 3.596023241970019, + "tokens_seen": 1275265024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004958555667001004, + "loss": 1.2076, + "theoretical_loss": 3.5960065702730795, + "tokens_seen": 1275330560 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495839518555667, + "loss": 1.4377, + "theoretical_loss": 3.5959898996727007, + "tokens_seen": 1275396096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004958234704112338, + "loss": 1.2029, + "theoretical_loss": 3.595973230168754, + "tokens_seen": 1275461632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004958074222668004, + "loss": 1.4822, + "theoretical_loss": 3.5959565617611116, + "tokens_seen": 1275527168 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004957913741223672, + "loss": 1.3551, + "theoretical_loss": 3.5959398944496446, + "tokens_seen": 1275592704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004957753259779338, + "loss": 1.3049, + "theoretical_loss": 3.5959232282342235, + "tokens_seen": 1275658240 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004957592778335006, + "loss": 1.3371, + "theoretical_loss": 3.595906563114722, + "tokens_seen": 1275723776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004957432296890672, + "loss": 1.2368, + "theoretical_loss": 3.59588989909101, + "tokens_seen": 1275789312 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495727181544634, + "loss": 1.3828, + "theoretical_loss": 3.5958732361629604, + "tokens_seen": 1275854848 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004957111334002006, + "loss": 1.342, + "theoretical_loss": 3.595856574330444, + "tokens_seen": 1275920384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004956950852557674, + "loss": 1.3444, + "theoretical_loss": 3.595839913593333, + "tokens_seen": 1275985920 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495679037111334, + "loss": 1.3372, + "theoretical_loss": 3.595823253951499, + "tokens_seen": 1276051456 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004956629889669008, + "loss": 1.2516, + "theoretical_loss": 3.595806595404813, + "tokens_seen": 1276116992 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004956469408224674, + "loss": 1.2169, + "theoretical_loss": 3.5957899379531484, + "tokens_seen": 1276182528 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004956308926780342, + "loss": 1.1793, + "theoretical_loss": 3.595773281596376, + "tokens_seen": 1276248064 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004956148445336008, + "loss": 1.3753, + "theoretical_loss": 3.595756626334367, + "tokens_seen": 1276313600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004955987963891676, + "loss": 1.3244, + "theoretical_loss": 3.595739972166995, + "tokens_seen": 1276379136 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004955827482447342, + "loss": 1.3931, + "theoretical_loss": 3.5957233190941302, + "tokens_seen": 1276444672 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495566700100301, + "loss": 1.4858, + "theoretical_loss": 3.595706667115645, + "tokens_seen": 1276510208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004955506519558676, + "loss": 1.3683, + "theoretical_loss": 3.595690016231411, + "tokens_seen": 1276575744 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004955346038114344, + "loss": 1.2999, + "theoretical_loss": 3.595673366441301, + "tokens_seen": 1276641280 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495518555667001, + "loss": 1.3359, + "theoretical_loss": 3.595656717745187, + "tokens_seen": 1276706816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004955025075225678, + "loss": 1.3228, + "theoretical_loss": 3.5956400701429394, + "tokens_seen": 1276772352 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004954864593781344, + "loss": 1.3666, + "theoretical_loss": 3.5956234236344313, + "tokens_seen": 1276837888 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004954704112337012, + "loss": 1.2305, + "theoretical_loss": 3.595606778219535, + "tokens_seen": 1276903424 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004954543630892678, + "loss": 1.233, + "theoretical_loss": 3.595590133898122, + "tokens_seen": 1276968960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004954383149448346, + "loss": 1.3041, + "theoretical_loss": 3.5955734906700645, + "tokens_seen": 1277034496 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004954222668004012, + "loss": 1.28, + "theoretical_loss": 3.5955568485352343, + "tokens_seen": 1277100032 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495406218655968, + "loss": 1.2192, + "theoretical_loss": 3.5955402074935043, + "tokens_seen": 1277165568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004953901705115346, + "loss": 1.2501, + "theoretical_loss": 3.5955235675447454, + "tokens_seen": 1277231104 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004953741223671013, + "loss": 1.342, + "theoretical_loss": 3.595506928688831, + "tokens_seen": 1277296640 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495358074222668, + "loss": 1.242, + "theoretical_loss": 3.5954902909256328, + "tokens_seen": 1277362176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004953420260782347, + "loss": 1.2875, + "theoretical_loss": 3.5954736542550227, + "tokens_seen": 1277427712 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004953259779338014, + "loss": 1.3997, + "theoretical_loss": 3.5954570186768726, + "tokens_seen": 1277493248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004953099297893681, + "loss": 1.2826, + "theoretical_loss": 3.5954403841910554, + "tokens_seen": 1277558784 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004952938816449348, + "loss": 1.3569, + "theoretical_loss": 3.595423750797443, + "tokens_seen": 1277624320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004952778335005015, + "loss": 1.3848, + "theoretical_loss": 3.595407118495908, + "tokens_seen": 1277689856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004952617853560683, + "loss": 1.2403, + "theoretical_loss": 3.5953904872863225, + "tokens_seen": 1277755392 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004952457372116349, + "loss": 1.3896, + "theoretical_loss": 3.5953738571685587, + "tokens_seen": 1277820928 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004952296890672015, + "loss": 1.3923, + "theoretical_loss": 3.5953572281424893, + "tokens_seen": 1277886464 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 429086, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.291599988937378, + "objective/train/theoretical_loss": 3.595340600207986, + "objective/train/tokens_used": 1298412000, + "theoretical_loss": 3.595340600207986, + "tokens_seen": 1277952000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004952136409227683, + "loss": 1.2622, + "theoretical_loss": 3.595340600207986, + "tokens_seen": 1277952000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004951975927783349, + "loss": 1.3463, + "theoretical_loss": 3.5953239733649216, + "tokens_seen": 1278017536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004951815446339017, + "loss": 1.3223, + "theoretical_loss": 3.595307347613169, + "tokens_seen": 1278083072 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004951654964894683, + "loss": 1.3747, + "theoretical_loss": 3.5952907229525994, + "tokens_seen": 1278148608 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004951494483450351, + "loss": 1.3882, + "theoretical_loss": 3.5952740993830865, + "tokens_seen": 1278214144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004951334002006017, + "loss": 1.4164, + "theoretical_loss": 3.595257476904502, + "tokens_seen": 1278279680 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004951173520561685, + "loss": 1.3095, + "theoretical_loss": 3.595240855516718, + "tokens_seen": 1278345216 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004951013039117351, + "loss": 1.2501, + "theoretical_loss": 3.5952242352196078, + "tokens_seen": 1278410752 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004950852557673019, + "loss": 1.3951, + "theoretical_loss": 3.5952076160130435, + "tokens_seen": 1278476288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004950692076228685, + "loss": 1.2822, + "theoretical_loss": 3.5951909978968986, + "tokens_seen": 1278541824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004950531594784353, + "loss": 1.2396, + "theoretical_loss": 3.595174380871044, + "tokens_seen": 1278607360 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495037111334002, + "loss": 1.3792, + "theoretical_loss": 3.5951577649353537, + "tokens_seen": 1278672896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004950210631895687, + "loss": 1.2545, + "theoretical_loss": 3.5951411500896997, + "tokens_seen": 1278738432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004950050150451354, + "loss": 1.2397, + "theoretical_loss": 3.595124536333955, + "tokens_seen": 1278803968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004949889669007021, + "loss": 1.3027, + "theoretical_loss": 3.5951079236679915, + "tokens_seen": 1278869504 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004949729187562688, + "loss": 1.4003, + "theoretical_loss": 3.5950913120916823, + "tokens_seen": 1278935040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004949568706118355, + "loss": 1.4203, + "theoretical_loss": 3.5950747016049007, + "tokens_seen": 1279000576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004949408224674022, + "loss": 1.3769, + "theoretical_loss": 3.595058092207519, + "tokens_seen": 1279066112 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004949247743229689, + "loss": 1.2913, + "theoretical_loss": 3.5950414838994087, + "tokens_seen": 1279131648 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004949087261785356, + "loss": 1.2267, + "theoretical_loss": 3.5950248766804447, + "tokens_seen": 1279197184 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004948926780341023, + "loss": 1.3514, + "theoretical_loss": 3.5950082705504984, + "tokens_seen": 1279262720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004948766298896691, + "loss": 1.2558, + "theoretical_loss": 3.594991665509443, + "tokens_seen": 1279328256 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004948605817452357, + "loss": 1.3068, + "theoretical_loss": 3.594975061557151, + "tokens_seen": 1279393792 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004948445336008025, + "loss": 1.2278, + "theoretical_loss": 3.5949584586934957, + "tokens_seen": 1279459328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004948284854563691, + "loss": 1.301, + "theoretical_loss": 3.59494185691835, + "tokens_seen": 1279524864 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004948124373119359, + "loss": 1.3764, + "theoretical_loss": 3.594925256231586, + "tokens_seen": 1279590400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004947963891675025, + "loss": 1.2844, + "theoretical_loss": 3.594908656633078, + "tokens_seen": 1279655936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004947803410230693, + "loss": 1.2771, + "theoretical_loss": 3.5948920581226975, + "tokens_seen": 1279721472 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004947642928786359, + "loss": 1.3355, + "theoretical_loss": 3.594875460700319, + "tokens_seen": 1279787008 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004947482447342027, + "loss": 1.3422, + "theoretical_loss": 3.594858864365813, + "tokens_seen": 1279852544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004947321965897693, + "loss": 1.3437, + "theoretical_loss": 3.594842269119055, + "tokens_seen": 1279918080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004947161484453361, + "loss": 1.3129, + "theoretical_loss": 3.5948256749599174, + "tokens_seen": 1279983616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004947001003009027, + "loss": 1.3646, + "theoretical_loss": 3.594809081888272, + "tokens_seen": 1280049152 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004946840521564695, + "loss": 1.2377, + "theoretical_loss": 3.5947924899039934, + "tokens_seen": 1280114688 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004946680040120361, + "loss": 1.312, + "theoretical_loss": 3.5947758990069536, + "tokens_seen": 1280180224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004946519558676029, + "loss": 1.4118, + "theoretical_loss": 3.594759309197027, + "tokens_seen": 1280245760 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004946359077231695, + "loss": 1.4197, + "theoretical_loss": 3.594742720474085, + "tokens_seen": 1280311296 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004946198595787363, + "loss": 1.3832, + "theoretical_loss": 3.5947261328380016, + "tokens_seen": 1280376832 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004946038114343029, + "loss": 1.3104, + "theoretical_loss": 3.5947095462886502, + "tokens_seen": 1280442368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004945877632898697, + "loss": 1.2556, + "theoretical_loss": 3.594692960825904, + "tokens_seen": 1280507904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004945717151454363, + "loss": 1.31, + "theoretical_loss": 3.5946763764496357, + "tokens_seen": 1280573440 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004945556670010031, + "loss": 1.4522, + "theoretical_loss": 3.594659793159719, + "tokens_seen": 1280638976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004945396188565697, + "loss": 1.3988, + "theoretical_loss": 3.5946432109560265, + "tokens_seen": 1280704512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004945235707121365, + "loss": 1.2574, + "theoretical_loss": 3.594626629838433, + "tokens_seen": 1280770048 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004945075225677031, + "loss": 1.3055, + "theoretical_loss": 3.5946100498068096, + "tokens_seen": 1280835584 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004944914744232699, + "loss": 1.3813, + "theoretical_loss": 3.5945934708610308, + "tokens_seen": 1280901120 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004944754262788365, + "loss": 1.3457, + "theoretical_loss": 3.59457689300097, + "tokens_seen": 1280966656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004944593781344033, + "loss": 1.3795, + "theoretical_loss": 3.5945603162265005, + "tokens_seen": 1281032192 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004944433299899699, + "loss": 1.289, + "theoretical_loss": 3.594543740537496, + "tokens_seen": 1281097728 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004944272818455367, + "loss": 1.2675, + "theoretical_loss": 3.594527165933829, + "tokens_seen": 1281163264 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 430126, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1517274379730225, + "objective/train/theoretical_loss": 3.594510592415374, + "objective/train/tokens_used": 1301688800, + "theoretical_loss": 3.594510592415374, + "tokens_seen": 1281228800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004944112337011033, + "loss": 1.3297, + "theoretical_loss": 3.594510592415374, + "tokens_seen": 1281228800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004943951855566701, + "loss": 1.332, + "theoretical_loss": 3.5944940199820037, + "tokens_seen": 1281294336 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004943791374122367, + "loss": 1.3252, + "theoretical_loss": 3.594477448633591, + "tokens_seen": 1281359872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004943630892678035, + "loss": 1.2618, + "theoretical_loss": 3.594460878370011, + "tokens_seen": 1281425408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004943470411233701, + "loss": 1.2576, + "theoretical_loss": 3.594444309191136, + "tokens_seen": 1281490944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004943309929789369, + "loss": 1.3243, + "theoretical_loss": 3.5944277410968395, + "tokens_seen": 1281556480 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004943149448345035, + "loss": 1.4041, + "theoretical_loss": 3.594411174086996, + "tokens_seen": 1281622016 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004942988966900703, + "loss": 1.2448, + "theoretical_loss": 3.594394608161478, + "tokens_seen": 1281687552 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004942828485456369, + "loss": 1.3129, + "theoretical_loss": 3.5943780433201598, + "tokens_seen": 1281753088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004942668004012037, + "loss": 1.4211, + "theoretical_loss": 3.594361479562915, + "tokens_seen": 1281818624 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004942507522567703, + "loss": 1.3106, + "theoretical_loss": 3.594344916889616, + "tokens_seen": 1281884160 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004942347041123371, + "loss": 1.2583, + "theoretical_loss": 3.5943283553001386, + "tokens_seen": 1281949696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004942186559679037, + "loss": 1.4562, + "theoretical_loss": 3.594311794794355, + "tokens_seen": 1282015232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004942026078234705, + "loss": 1.3874, + "theoretical_loss": 3.5942952353721394, + "tokens_seen": 1282080768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004941865596790371, + "loss": 1.2735, + "theoretical_loss": 3.5942786770333646, + "tokens_seen": 1282146304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004941705115346039, + "loss": 1.2348, + "theoretical_loss": 3.594262119777906, + "tokens_seen": 1282211840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004941544633901705, + "loss": 1.3056, + "theoretical_loss": 3.594245563605636, + "tokens_seen": 1282277376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004941384152457373, + "loss": 1.313, + "theoretical_loss": 3.594229008516429, + "tokens_seen": 1282342912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004941223671013039, + "loss": 1.2875, + "theoretical_loss": 3.594212454510158, + "tokens_seen": 1282408448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004941063189568707, + "loss": 1.2558, + "theoretical_loss": 3.594195901586698, + "tokens_seen": 1282473984 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004940902708124373, + "loss": 1.2904, + "theoretical_loss": 3.5941793497459225, + "tokens_seen": 1282539520 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004940742226680041, + "loss": 1.3036, + "theoretical_loss": 3.5941627989877047, + "tokens_seen": 1282605056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004940581745235707, + "loss": 1.3134, + "theoretical_loss": 3.594146249311919, + "tokens_seen": 1282670592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004940421263791375, + "loss": 1.4107, + "theoretical_loss": 3.59412970071844, + "tokens_seen": 1282736128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004940260782347041, + "loss": 1.3728, + "theoretical_loss": 3.59411315320714, + "tokens_seen": 1282801664 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004940100300902709, + "loss": 1.35, + "theoretical_loss": 3.594096606777894, + "tokens_seen": 1282867200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004939939819458375, + "loss": 1.3382, + "theoretical_loss": 3.594080061430576, + "tokens_seen": 1282932736 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004939779338014042, + "loss": 1.2969, + "theoretical_loss": 3.5940635171650603, + "tokens_seen": 1282998272 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493961885656971, + "loss": 1.4137, + "theoretical_loss": 3.5940469739812198, + "tokens_seen": 1283063808 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004939458375125376, + "loss": 1.3062, + "theoretical_loss": 3.594030431878929, + "tokens_seen": 1283129344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004939297893681043, + "loss": 1.2036, + "theoretical_loss": 3.5940138908580628, + "tokens_seen": 1283194880 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493913741223671, + "loss": 1.2865, + "theoretical_loss": 3.593997350918494, + "tokens_seen": 1283260416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004938976930792378, + "loss": 1.327, + "theoretical_loss": 3.5939808120600976, + "tokens_seen": 1283325952 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004938816449348044, + "loss": 1.2471, + "theoretical_loss": 3.5939642742827473, + "tokens_seen": 1283391488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004938655967903712, + "loss": 1.3631, + "theoretical_loss": 3.5939477375863174, + "tokens_seen": 1283457024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004938495486459378, + "loss": 1.2954, + "theoretical_loss": 3.593931201970682, + "tokens_seen": 1283522560 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004938335005015044, + "loss": 1.3171, + "theoretical_loss": 3.5939146674357154, + "tokens_seen": 1283588096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004938174523570712, + "loss": 1.4015, + "theoretical_loss": 3.593898133981291, + "tokens_seen": 1283653632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004938014042126378, + "loss": 1.4143, + "theoretical_loss": 3.5938816016072845, + "tokens_seen": 1283719168 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004937853560682046, + "loss": 1.3127, + "theoretical_loss": 3.5938650703135693, + "tokens_seen": 1283784704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004937693079237712, + "loss": 1.2435, + "theoretical_loss": 3.5938485401000193, + "tokens_seen": 1283850240 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493753259779338, + "loss": 1.2403, + "theoretical_loss": 3.59383201096651, + "tokens_seen": 1283915776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004937372116349046, + "loss": 1.2874, + "theoretical_loss": 3.5938154829129143, + "tokens_seen": 1283981312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004937211634904714, + "loss": 1.3221, + "theoretical_loss": 3.593798955939107, + "tokens_seen": 1284046848 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493705115346038, + "loss": 1.2331, + "theoretical_loss": 3.5937824300449632, + "tokens_seen": 1284112384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004936890672016048, + "loss": 1.2967, + "theoretical_loss": 3.5937659052303563, + "tokens_seen": 1284177920 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004936730190571716, + "loss": 1.2442, + "theoretical_loss": 3.5937493814951615, + "tokens_seen": 1284243456 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004936569709127382, + "loss": 1.2047, + "theoretical_loss": 3.5937328588392523, + "tokens_seen": 1284308992 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493640922768305, + "loss": 1.2752, + "theoretical_loss": 3.593716337262504, + "tokens_seen": 1284374528 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004936248746238716, + "loss": 1.3132, + "theoretical_loss": 3.59369981676479, + "tokens_seen": 1284440064 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 431379, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4823201894760132, + "objective/train/theoretical_loss": 3.5936832973459865, + "objective/train/tokens_used": 1304965600, + "theoretical_loss": 3.5936832973459865, + "tokens_seen": 1284505600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004936088264794384, + "loss": 1.3724, + "theoretical_loss": 3.5936832973459865, + "tokens_seen": 1284505600 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493592778335005, + "loss": 1.2995, + "theoretical_loss": 3.593666779005966, + "tokens_seen": 1284571136 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004935767301905718, + "loss": 1.3161, + "theoretical_loss": 3.5936502617446044, + "tokens_seen": 1284636672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004935606820461384, + "loss": 1.2323, + "theoretical_loss": 3.593633745561776, + "tokens_seen": 1284702208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004935446339017052, + "loss": 1.3966, + "theoretical_loss": 3.593617230457355, + "tokens_seen": 1284767744 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004935285857572718, + "loss": 1.3848, + "theoretical_loss": 3.593600716431216, + "tokens_seen": 1284833280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004935125376128386, + "loss": 1.3259, + "theoretical_loss": 3.593584203483234, + "tokens_seen": 1284898816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004934964894684052, + "loss": 1.4316, + "theoretical_loss": 3.5935676916132833, + "tokens_seen": 1284964352 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493480441323972, + "loss": 1.4408, + "theoretical_loss": 3.593551180821239, + "tokens_seen": 1285029888 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004934643931795386, + "loss": 1.2968, + "theoretical_loss": 3.593534671106975, + "tokens_seen": 1285095424 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004934483450351054, + "loss": 1.275, + "theoretical_loss": 3.5935181624703665, + "tokens_seen": 1285160960 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493432296890672, + "loss": 1.4033, + "theoretical_loss": 3.5935016549112877, + "tokens_seen": 1285226496 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004934162487462388, + "loss": 1.3205, + "theoretical_loss": 3.5934851484296146, + "tokens_seen": 1285292032 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004934002006018054, + "loss": 1.3356, + "theoretical_loss": 3.5934686430252203, + "tokens_seen": 1285357568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004933841524573722, + "loss": 1.3887, + "theoretical_loss": 3.5934521386979807, + "tokens_seen": 1285423104 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004933681043129388, + "loss": 1.3009, + "theoretical_loss": 3.5934356354477703, + "tokens_seen": 1285488640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004933520561685056, + "loss": 1.4457, + "theoretical_loss": 3.5934191332744634, + "tokens_seen": 1285554176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004933360080240722, + "loss": 1.3302, + "theoretical_loss": 3.593402632177936, + "tokens_seen": 1285619712 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493319959879639, + "loss": 1.4391, + "theoretical_loss": 3.593386132158062, + "tokens_seen": 1285685248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004933039117352056, + "loss": 1.2914, + "theoretical_loss": 3.5933696332147163, + "tokens_seen": 1285750784 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004932878635907724, + "loss": 1.4211, + "theoretical_loss": 3.593353135347774, + "tokens_seen": 1285816320 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493271815446339, + "loss": 1.3314, + "theoretical_loss": 3.59333663855711, + "tokens_seen": 1285881856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004932557673019058, + "loss": 1.2129, + "theoretical_loss": 3.5933201428425994, + "tokens_seen": 1285947392 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004932397191574724, + "loss": 1.4659, + "theoretical_loss": 3.5933036482041167, + "tokens_seen": 1286012928 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004932236710130392, + "loss": 1.4073, + "theoretical_loss": 3.593287154641538, + "tokens_seen": 1286078464 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004932076228686058, + "loss": 1.3336, + "theoretical_loss": 3.593270662154737, + "tokens_seen": 1286144000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004931915747241726, + "loss": 1.2649, + "theoretical_loss": 3.593254170743589, + "tokens_seen": 1286209536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004931755265797392, + "loss": 1.2477, + "theoretical_loss": 3.5932376804079698, + "tokens_seen": 1286275072 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493159478435306, + "loss": 1.3447, + "theoretical_loss": 3.5932211911477534, + "tokens_seen": 1286340608 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004931434302908726, + "loss": 1.2927, + "theoretical_loss": 3.593204702962816, + "tokens_seen": 1286406144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004931273821464394, + "loss": 1.3555, + "theoretical_loss": 3.593188215853032, + "tokens_seen": 1286471680 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493111334002006, + "loss": 1.3249, + "theoretical_loss": 3.593171729818276, + "tokens_seen": 1286537216 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004930952858575728, + "loss": 1.2865, + "theoretical_loss": 3.5931552448584245, + "tokens_seen": 1286602752 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004930792377131394, + "loss": 1.3145, + "theoretical_loss": 3.5931387609733516, + "tokens_seen": 1286668288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004930631895687062, + "loss": 1.4134, + "theoretical_loss": 3.5931222781629333, + "tokens_seen": 1286733824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004930471414242728, + "loss": 1.3083, + "theoretical_loss": 3.593105796427044, + "tokens_seen": 1286799360 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004930310932798396, + "loss": 1.3091, + "theoretical_loss": 3.5930893157655595, + "tokens_seen": 1286864896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004930150451354062, + "loss": 1.3026, + "theoretical_loss": 3.593072836178355, + "tokens_seen": 1286930432 + }, + { + "epoch": 0.39, + "learning_rate": 0.000492998996990973, + "loss": 1.3357, + "theoretical_loss": 3.593056357665305, + "tokens_seen": 1286995968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004929829488465396, + "loss": 1.502, + "theoretical_loss": 3.5930398802262857, + "tokens_seen": 1287061504 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004929669007021064, + "loss": 1.2645, + "theoretical_loss": 3.5930234038611726, + "tokens_seen": 1287127040 + }, + { + "epoch": 0.39, + "learning_rate": 0.000492950852557673, + "loss": 1.4383, + "theoretical_loss": 3.59300692856984, + "tokens_seen": 1287192576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004929348044132398, + "loss": 1.4961, + "theoretical_loss": 3.5929904543521642, + "tokens_seen": 1287258112 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004929187562688064, + "loss": 1.3086, + "theoretical_loss": 3.59297398120802, + "tokens_seen": 1287323648 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004929027081243732, + "loss": 1.3873, + "theoretical_loss": 3.5929575091372827, + "tokens_seen": 1287389184 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004928866599799398, + "loss": 1.4399, + "theoretical_loss": 3.592941038139829, + "tokens_seen": 1287454720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004928706118355066, + "loss": 1.5055, + "theoretical_loss": 3.5929245682155324, + "tokens_seen": 1287520256 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004928545636910732, + "loss": 1.3475, + "theoretical_loss": 3.5929080993642692, + "tokens_seen": 1287585792 + }, + { + "epoch": 0.39, + "learning_rate": 0.00049283851554664, + "loss": 1.3472, + "theoretical_loss": 3.5928916315859154, + "tokens_seen": 1287651328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004928224674022066, + "loss": 1.2471, + "theoretical_loss": 3.592875164880346, + "tokens_seen": 1287716864 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 432272, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3189022541046143, + "objective/train/theoretical_loss": 3.592858699247437, + "objective/train/tokens_used": 1308242400, + "theoretical_loss": 3.592858699247437, + "tokens_seen": 1287782400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004928064192577734, + "loss": 1.3203, + "theoretical_loss": 3.592858699247437, + "tokens_seen": 1287782400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00049279037111334, + "loss": 1.3671, + "theoretical_loss": 3.592842234687063, + "tokens_seen": 1287847936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004927743229689068, + "loss": 1.3745, + "theoretical_loss": 3.5928257711991005, + "tokens_seen": 1287913472 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004927582748244734, + "loss": 1.2851, + "theoretical_loss": 3.5928093087834245, + "tokens_seen": 1287979008 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004927422266800402, + "loss": 1.2529, + "theoretical_loss": 3.592792847439911, + "tokens_seen": 1288044544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004927261785356068, + "loss": 1.3913, + "theoretical_loss": 3.5927763871684353, + "tokens_seen": 1288110080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004927101303911736, + "loss": 1.3012, + "theoretical_loss": 3.5927599279688733, + "tokens_seen": 1288175616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004926940822467402, + "loss": 1.3546, + "theoretical_loss": 3.5927434698411003, + "tokens_seen": 1288241152 + }, + { + "epoch": 0.39, + "learning_rate": 0.000492678034102307, + "loss": 1.318, + "theoretical_loss": 3.5927270127849926, + "tokens_seen": 1288306688 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004926619859578736, + "loss": 1.385, + "theoretical_loss": 3.592710556800425, + "tokens_seen": 1288372224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004926459378134404, + "loss": 1.2928, + "theoretical_loss": 3.5926941018872744, + "tokens_seen": 1288437760 + }, + { + "epoch": 0.39, + "learning_rate": 0.000492629889669007, + "loss": 1.3409, + "theoretical_loss": 3.5926776480454157, + "tokens_seen": 1288503296 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004926138415245738, + "loss": 1.4072, + "theoretical_loss": 3.5926611952747245, + "tokens_seen": 1288568832 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004925977933801404, + "loss": 1.3322, + "theoretical_loss": 3.5926447435750775, + "tokens_seen": 1288634368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004925817452357071, + "loss": 1.3475, + "theoretical_loss": 3.5926282929463498, + "tokens_seen": 1288699904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004925656970912738, + "loss": 1.3439, + "theoretical_loss": 3.5926118433884175, + "tokens_seen": 1288765440 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004925496489468405, + "loss": 1.4001, + "theoretical_loss": 3.5925953949011564, + "tokens_seen": 1288830976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004925336008024073, + "loss": 1.307, + "theoretical_loss": 3.5925789474844425, + "tokens_seen": 1288896512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004925175526579739, + "loss": 1.3581, + "theoretical_loss": 3.5925625011381515, + "tokens_seen": 1288962048 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004925015045135407, + "loss": 1.408, + "theoretical_loss": 3.592546055862159, + "tokens_seen": 1289027584 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004924854563691073, + "loss": 1.3784, + "theoretical_loss": 3.5925296116563414, + "tokens_seen": 1289093120 + }, + { + "epoch": 0.39, + "learning_rate": 0.000492469408224674, + "loss": 1.4887, + "theoretical_loss": 3.5925131685205747, + "tokens_seen": 1289158656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004924533600802407, + "loss": 1.4219, + "theoretical_loss": 3.592496726454735, + "tokens_seen": 1289224192 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004924373119358075, + "loss": 1.462, + "theoretical_loss": 3.5924802854586977, + "tokens_seen": 1289289728 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004924212637913741, + "loss": 1.3384, + "theoretical_loss": 3.592463845532339, + "tokens_seen": 1289355264 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004924052156469409, + "loss": 1.2879, + "theoretical_loss": 3.592447406675536, + "tokens_seen": 1289420800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004923891675025075, + "loss": 1.3606, + "theoretical_loss": 3.592430968888163, + "tokens_seen": 1289486336 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004923731193580743, + "loss": 1.2404, + "theoretical_loss": 3.592414532170097, + "tokens_seen": 1289551872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004923570712136409, + "loss": 1.2948, + "theoretical_loss": 3.5923980965212143, + "tokens_seen": 1289617408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004923410230692077, + "loss": 1.4047, + "theoretical_loss": 3.5923816619413906, + "tokens_seen": 1289682944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004923249749247743, + "loss": 1.4074, + "theoretical_loss": 3.5923652284305025, + "tokens_seen": 1289748480 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004923089267803411, + "loss": 1.3486, + "theoretical_loss": 3.5923487959884257, + "tokens_seen": 1289814016 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004922928786359077, + "loss": 1.357, + "theoretical_loss": 3.5923323646150367, + "tokens_seen": 1289879552 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004922768304914745, + "loss": 1.2946, + "theoretical_loss": 3.5923159343102116, + "tokens_seen": 1289945088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004922607823470411, + "loss": 1.3531, + "theoretical_loss": 3.592299505073826, + "tokens_seen": 1290010624 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004922447342026079, + "loss": 1.3608, + "theoretical_loss": 3.5922830769057574, + "tokens_seen": 1290076160 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004922286860581745, + "loss": 1.3382, + "theoretical_loss": 3.592266649805881, + "tokens_seen": 1290141696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004922126379137413, + "loss": 1.3103, + "theoretical_loss": 3.592250223774074, + "tokens_seen": 1290207232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004921965897693079, + "loss": 1.3596, + "theoretical_loss": 3.592233798810212, + "tokens_seen": 1290272768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004921805416248747, + "loss": 1.4379, + "theoretical_loss": 3.592217374914171, + "tokens_seen": 1290338304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004921644934804413, + "loss": 1.4919, + "theoretical_loss": 3.5922009520858285, + "tokens_seen": 1290403840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004921484453360081, + "loss": 1.3597, + "theoretical_loss": 3.59218453032506, + "tokens_seen": 1290469376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004921323971915747, + "loss": 1.2389, + "theoretical_loss": 3.592168109631742, + "tokens_seen": 1290534912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004921163490471415, + "loss": 1.5207, + "theoretical_loss": 3.5921516900057515, + "tokens_seen": 1290600448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004921003009027081, + "loss": 1.2722, + "theoretical_loss": 3.5921352714469643, + "tokens_seen": 1290665984 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004920842527582749, + "loss": 1.3502, + "theoretical_loss": 3.592118853955257, + "tokens_seen": 1290731520 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004920682046138415, + "loss": 1.2804, + "theoretical_loss": 3.592102437530506, + "tokens_seen": 1290797056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004920521564694083, + "loss": 1.2639, + "theoretical_loss": 3.5920860221725883, + "tokens_seen": 1290862592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004920361083249749, + "loss": 1.4943, + "theoretical_loss": 3.5920696078813794, + "tokens_seen": 1290928128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004920200601805417, + "loss": 1.4752, + "theoretical_loss": 3.592053194656757, + "tokens_seen": 1290993664 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 433657, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2094658613204956, + "objective/train/theoretical_loss": 3.592036782498597, + "objective/train/tokens_used": 1311519200, + "theoretical_loss": 3.592036782498597, + "tokens_seen": 1291059200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004920040120361083, + "loss": 1.3495, + "theoretical_loss": 3.592036782498597, + "tokens_seen": 1291059200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004919879638916751, + "loss": 1.271, + "theoretical_loss": 3.5920203714067758, + "tokens_seen": 1291124736 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004919719157472417, + "loss": 1.3186, + "theoretical_loss": 3.5920039613811703, + "tokens_seen": 1291190272 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004919558676028085, + "loss": 1.3833, + "theoretical_loss": 3.5919875524216573, + "tokens_seen": 1291255808 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004919398194583751, + "loss": 1.3534, + "theoretical_loss": 3.591971144528113, + "tokens_seen": 1291321344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004919237713139419, + "loss": 1.3739, + "theoretical_loss": 3.591954737700415, + "tokens_seen": 1291386880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004919077231695085, + "loss": 1.3191, + "theoretical_loss": 3.5919383319384384, + "tokens_seen": 1291452416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004918916750250753, + "loss": 1.5903, + "theoretical_loss": 3.591921927242061, + "tokens_seen": 1291517952 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004918756268806419, + "loss": 1.342, + "theoretical_loss": 3.5919055236111594, + "tokens_seen": 1291583488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004918595787362087, + "loss": 1.338, + "theoretical_loss": 3.5918891210456105, + "tokens_seen": 1291649024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004918435305917753, + "loss": 1.467, + "theoretical_loss": 3.5918727195452904, + "tokens_seen": 1291714560 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004918274824473421, + "loss": 1.2794, + "theoretical_loss": 3.5918563191100765, + "tokens_seen": 1291780096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004918114343029087, + "loss": 1.1941, + "theoretical_loss": 3.5918399197398454, + "tokens_seen": 1291845632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004917953861584755, + "loss": 1.328, + "theoretical_loss": 3.591823521434474, + "tokens_seen": 1291911168 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004917793380140421, + "loss": 1.311, + "theoretical_loss": 3.5918071241938385, + "tokens_seen": 1291976704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004917632898696089, + "loss": 1.322, + "theoretical_loss": 3.5917907280178167, + "tokens_seen": 1292042240 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004917472417251755, + "loss": 1.2841, + "theoretical_loss": 3.5917743329062852, + "tokens_seen": 1292107776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004917311935807423, + "loss": 1.3337, + "theoretical_loss": 3.5917579388591205, + "tokens_seen": 1292173312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004917151454363089, + "loss": 1.4893, + "theoretical_loss": 3.5917415458762, + "tokens_seen": 1292238848 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004916990972918757, + "loss": 1.3748, + "theoretical_loss": 3.5917251539574004, + "tokens_seen": 1292304384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004916830491474423, + "loss": 1.3087, + "theoretical_loss": 3.5917087631025986, + "tokens_seen": 1292369920 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004916670010030091, + "loss": 1.3229, + "theoretical_loss": 3.591692373311672, + "tokens_seen": 1292435456 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004916509528585757, + "loss": 1.2367, + "theoretical_loss": 3.591675984584497, + "tokens_seen": 1292500992 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004916349047141425, + "loss": 1.3359, + "theoretical_loss": 3.591659596920951, + "tokens_seen": 1292566528 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004916188565697091, + "loss": 1.4251, + "theoretical_loss": 3.5916432103209113, + "tokens_seen": 1292632064 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004916028084252759, + "loss": 1.3154, + "theoretical_loss": 3.591626824784255, + "tokens_seen": 1292697600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004915867602808425, + "loss": 1.4047, + "theoretical_loss": 3.5916104403108577, + "tokens_seen": 1292763136 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004915707121364093, + "loss": 1.3264, + "theoretical_loss": 3.5915940569005986, + "tokens_seen": 1292828672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004915546639919759, + "loss": 1.4677, + "theoretical_loss": 3.5915776745533536, + "tokens_seen": 1292894208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004915386158475427, + "loss": 1.2761, + "theoretical_loss": 3.5915612932689998, + "tokens_seen": 1292959744 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004915225677031093, + "loss": 1.3447, + "theoretical_loss": 3.5915449130474153, + "tokens_seen": 1293025280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004915065195586761, + "loss": 1.3004, + "theoretical_loss": 3.591528533888476, + "tokens_seen": 1293090816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004914904714142427, + "loss": 1.3187, + "theoretical_loss": 3.5915121557920604, + "tokens_seen": 1293156352 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004914744232698095, + "loss": 1.3155, + "theoretical_loss": 3.591495778758045, + "tokens_seen": 1293221888 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004914583751253761, + "loss": 1.3298, + "theoretical_loss": 3.591479402786307, + "tokens_seen": 1293287424 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004914423269809429, + "loss": 1.3217, + "theoretical_loss": 3.591463027876724, + "tokens_seen": 1293352960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004914262788365095, + "loss": 1.3225, + "theoretical_loss": 3.5914466540291734, + "tokens_seen": 1293418496 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004914102306920763, + "loss": 1.3561, + "theoretical_loss": 3.5914302812435315, + "tokens_seen": 1293484032 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004913941825476429, + "loss": 1.3903, + "theoretical_loss": 3.591413909519677, + "tokens_seen": 1293549568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004913781344032097, + "loss": 1.4476, + "theoretical_loss": 3.5913975388574864, + "tokens_seen": 1293615104 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004913620862587763, + "loss": 1.3465, + "theoretical_loss": 3.591381169256838, + "tokens_seen": 1293680640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004913460381143431, + "loss": 1.3507, + "theoretical_loss": 3.5913648007176078, + "tokens_seen": 1293746176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004913299899699097, + "loss": 1.2841, + "theoretical_loss": 3.5913484332396743, + "tokens_seen": 1293811712 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004913139418254765, + "loss": 1.3726, + "theoretical_loss": 3.5913320668229143, + "tokens_seen": 1293877248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004912978936810431, + "loss": 1.2302, + "theoretical_loss": 3.5913157014672055, + "tokens_seen": 1293942784 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004912818455366099, + "loss": 1.3879, + "theoretical_loss": 3.5912993371724258, + "tokens_seen": 1294008320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004912657973921765, + "loss": 1.3029, + "theoretical_loss": 3.591282973938452, + "tokens_seen": 1294073856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004912497492477433, + "loss": 1.2827, + "theoretical_loss": 3.591266611765162, + "tokens_seen": 1294139392 + }, + { + "epoch": 0.39, + "learning_rate": 0.00049123370110331, + "loss": 1.3735, + "theoretical_loss": 3.5912502506524335, + "tokens_seen": 1294204928 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004912176529588767, + "loss": 1.4038, + "theoretical_loss": 3.5912338906001433, + "tokens_seen": 1294270464 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 434508, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4272451400756836, + "objective/train/theoretical_loss": 3.5912175316081694, + "objective/train/tokens_used": 1314796000, + "theoretical_loss": 3.5912175316081694, + "tokens_seen": 1294336000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004912016048144433, + "loss": 1.45, + "theoretical_loss": 3.5912175316081694, + "tokens_seen": 1294336000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004911855566700101, + "loss": 1.3144, + "theoretical_loss": 3.59120117367639, + "tokens_seen": 1294401536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004911695085255767, + "loss": 1.2642, + "theoretical_loss": 3.591184816804682, + "tokens_seen": 1294467072 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004911534603811434, + "loss": 1.376, + "theoretical_loss": 3.5911684609929235, + "tokens_seen": 1294532608 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004911374122367102, + "loss": 1.4699, + "theoretical_loss": 3.5911521062409912, + "tokens_seen": 1294598144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004911213640922768, + "loss": 1.2568, + "theoretical_loss": 3.591135752548764, + "tokens_seen": 1294663680 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004911053159478436, + "loss": 1.3599, + "theoretical_loss": 3.591119399916119, + "tokens_seen": 1294729216 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004910892678034102, + "loss": 1.324, + "theoretical_loss": 3.5911030483429345, + "tokens_seen": 1294794752 + }, + { + "epoch": 0.39, + "learning_rate": 0.000491073219658977, + "loss": 1.3569, + "theoretical_loss": 3.5910866978290867, + "tokens_seen": 1294860288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004910571715145436, + "loss": 1.2693, + "theoretical_loss": 3.5910703483744553, + "tokens_seen": 1294925824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004910411233701104, + "loss": 1.4145, + "theoretical_loss": 3.591053999978917, + "tokens_seen": 1294991360 + }, + { + "epoch": 0.39, + "learning_rate": 0.000491025075225677, + "loss": 1.2988, + "theoretical_loss": 3.591037652642349, + "tokens_seen": 1295056896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004910090270812438, + "loss": 1.2822, + "theoretical_loss": 3.591021306364631, + "tokens_seen": 1295122432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004909929789368104, + "loss": 1.2971, + "theoretical_loss": 3.5910049611456394, + "tokens_seen": 1295187968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004909769307923772, + "loss": 1.2812, + "theoretical_loss": 3.590988616985252, + "tokens_seen": 1295253504 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004909608826479438, + "loss": 1.3704, + "theoretical_loss": 3.5909722738833474, + "tokens_seen": 1295319040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004909448345035106, + "loss": 1.4753, + "theoretical_loss": 3.590955931839803, + "tokens_seen": 1295384576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004909287863590772, + "loss": 1.4235, + "theoretical_loss": 3.5909395908544974, + "tokens_seen": 1295450112 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490912738214644, + "loss": 1.356, + "theoretical_loss": 3.5909232509273075, + "tokens_seen": 1295515648 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004908966900702106, + "loss": 1.3655, + "theoretical_loss": 3.590906912058112, + "tokens_seen": 1295581184 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004908806419257774, + "loss": 1.335, + "theoretical_loss": 3.5908905742467887, + "tokens_seen": 1295646720 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490864593781344, + "loss": 1.31, + "theoretical_loss": 3.5908742374932157, + "tokens_seen": 1295712256 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004908485456369108, + "loss": 1.5038, + "theoretical_loss": 3.590857901797271, + "tokens_seen": 1295777792 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004908324974924774, + "loss": 1.4252, + "theoretical_loss": 3.590841567158832, + "tokens_seen": 1295843328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004908164493480442, + "loss": 1.3494, + "theoretical_loss": 3.590825233577778, + "tokens_seen": 1295908864 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004908004012036108, + "loss": 1.4036, + "theoretical_loss": 3.590808901053986, + "tokens_seen": 1295974400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004907843530591776, + "loss": 1.3434, + "theoretical_loss": 3.590792569587335, + "tokens_seen": 1296039936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004907683049147442, + "loss": 1.3556, + "theoretical_loss": 3.5907762391777025, + "tokens_seen": 1296105472 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490752256770311, + "loss": 1.3104, + "theoretical_loss": 3.5907599098249667, + "tokens_seen": 1296171008 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004907362086258776, + "loss": 1.2789, + "theoretical_loss": 3.5907435815290056, + "tokens_seen": 1296236544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004907201604814444, + "loss": 1.2265, + "theoretical_loss": 3.590727254289698, + "tokens_seen": 1296302080 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490704112337011, + "loss": 1.5031, + "theoretical_loss": 3.5907109281069216, + "tokens_seen": 1296367616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004906880641925778, + "loss": 1.27, + "theoretical_loss": 3.5906946029805544, + "tokens_seen": 1296433152 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004906720160481444, + "loss": 1.3972, + "theoretical_loss": 3.5906782789104756, + "tokens_seen": 1296498688 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004906559679037112, + "loss": 1.3149, + "theoretical_loss": 3.5906619558965627, + "tokens_seen": 1296564224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004906399197592778, + "loss": 1.2508, + "theoretical_loss": 3.5906456339386943, + "tokens_seen": 1296629760 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004906238716148446, + "loss": 1.3477, + "theoretical_loss": 3.5906293130367484, + "tokens_seen": 1296695296 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004906078234704112, + "loss": 1.4802, + "theoretical_loss": 3.5906129931906037, + "tokens_seen": 1296760832 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490591775325978, + "loss": 1.2791, + "theoretical_loss": 3.590596674400138, + "tokens_seen": 1296826368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004905757271815446, + "loss": 1.2481, + "theoretical_loss": 3.590580356665231, + "tokens_seen": 1296891904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004905596790371114, + "loss": 1.3494, + "theoretical_loss": 3.5905640399857583, + "tokens_seen": 1296957440 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490543630892678, + "loss": 1.3892, + "theoretical_loss": 3.5905477243616017, + "tokens_seen": 1297022976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004905275827482448, + "loss": 1.257, + "theoretical_loss": 3.5905314097926375, + "tokens_seen": 1297088512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004905115346038114, + "loss": 1.3527, + "theoretical_loss": 3.590515096278745, + "tokens_seen": 1297154048 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004904954864593782, + "loss": 1.4653, + "theoretical_loss": 3.5904987838198017, + "tokens_seen": 1297219584 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004904794383149448, + "loss": 1.4053, + "theoretical_loss": 3.5904824724156876, + "tokens_seen": 1297285120 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004904633901705116, + "loss": 1.2649, + "theoretical_loss": 3.5904661620662797, + "tokens_seen": 1297350656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004904473420260782, + "loss": 1.3664, + "theoretical_loss": 3.5904498527714575, + "tokens_seen": 1297416192 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490431293881645, + "loss": 1.4184, + "theoretical_loss": 3.590433544531099, + "tokens_seen": 1297481728 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004904152457372116, + "loss": 1.3453, + "theoretical_loss": 3.590417237345083, + "tokens_seen": 1297547264 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 435770, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4622042179107666, + "objective/train/theoretical_loss": 3.5904009312132885, + "objective/train/tokens_used": 1318072800, + "theoretical_loss": 3.5904009312132885, + "tokens_seen": 1297612800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004903991975927784, + "loss": 1.3757, + "theoretical_loss": 3.5904009312132885, + "tokens_seen": 1297612800 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490383149448345, + "loss": 1.449, + "theoretical_loss": 3.5903846261355934, + "tokens_seen": 1297678336 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004903671013039118, + "loss": 1.3038, + "theoretical_loss": 3.5903683221118765, + "tokens_seen": 1297743872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004903510531594784, + "loss": 1.3635, + "theoretical_loss": 3.5903520191420166, + "tokens_seen": 1297809408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004903350050150452, + "loss": 1.4401, + "theoretical_loss": 3.5903357172258925, + "tokens_seen": 1297874944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004903189568706118, + "loss": 1.2792, + "theoretical_loss": 3.590319416363383, + "tokens_seen": 1297940480 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004903029087261786, + "loss": 1.4366, + "theoretical_loss": 3.5903031165543657, + "tokens_seen": 1298006016 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004902868605817452, + "loss": 1.3782, + "theoretical_loss": 3.590286817798721, + "tokens_seen": 1298071552 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490270812437312, + "loss": 1.376, + "theoretical_loss": 3.5902705200963263, + "tokens_seen": 1298137088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004902547642928786, + "loss": 1.428, + "theoretical_loss": 3.5902542234470616, + "tokens_seen": 1298202624 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004902387161484454, + "loss": 1.3626, + "theoretical_loss": 3.5902379278508043, + "tokens_seen": 1298268160 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490222668004012, + "loss": 1.3293, + "theoretical_loss": 3.590221633307434, + "tokens_seen": 1298333696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004902066198595788, + "loss": 1.2862, + "theoretical_loss": 3.59020533981683, + "tokens_seen": 1298399232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004901905717151454, + "loss": 1.2682, + "theoretical_loss": 3.59018904737887, + "tokens_seen": 1298464768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004901745235707122, + "loss": 1.3294, + "theoretical_loss": 3.590172755993433, + "tokens_seen": 1298530304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004901584754262788, + "loss": 1.3853, + "theoretical_loss": 3.5901564656603995, + "tokens_seen": 1298595840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004901424272818456, + "loss": 1.4211, + "theoretical_loss": 3.5901401763796468, + "tokens_seen": 1298661376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004901263791374122, + "loss": 1.4769, + "theoretical_loss": 3.5901238881510538, + "tokens_seen": 1298726912 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490110330992979, + "loss": 1.4099, + "theoretical_loss": 3.5901076009745, + "tokens_seen": 1298792448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004900942828485456, + "loss": 1.3624, + "theoretical_loss": 3.5900913148498645, + "tokens_seen": 1298857984 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004900782347041124, + "loss": 1.3068, + "theoretical_loss": 3.5900750297770267, + "tokens_seen": 1298923520 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004900621865596791, + "loss": 1.4171, + "theoretical_loss": 3.590058745755864, + "tokens_seen": 1298989056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004900461384152458, + "loss": 1.2757, + "theoretical_loss": 3.590042462786257, + "tokens_seen": 1299054592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004900300902708125, + "loss": 1.4112, + "theoretical_loss": 3.590026180868084, + "tokens_seen": 1299120128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004900140421263792, + "loss": 1.2693, + "theoretical_loss": 3.5900099000012244, + "tokens_seen": 1299185664 + }, + { + "epoch": 0.39, + "learning_rate": 0.000489997993981946, + "loss": 1.3372, + "theoretical_loss": 3.589993620185557, + "tokens_seen": 1299251200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004899819458375126, + "loss": 1.3898, + "theoretical_loss": 3.589977341420961, + "tokens_seen": 1299316736 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004899658976930794, + "loss": 1.3904, + "theoretical_loss": 3.5899610637073156, + "tokens_seen": 1299382272 + }, + { + "epoch": 0.39, + "learning_rate": 0.000489949849548646, + "loss": 1.3679, + "theoretical_loss": 3.5899447870445, + "tokens_seen": 1299447808 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004899338014042128, + "loss": 1.2966, + "theoretical_loss": 3.589928511432393, + "tokens_seen": 1299513344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004899177532597794, + "loss": 1.333, + "theoretical_loss": 3.5899122368708745, + "tokens_seen": 1299578880 + }, + { + "epoch": 0.39, + "learning_rate": 0.000489901705115346, + "loss": 1.2978, + "theoretical_loss": 3.589895963359823, + "tokens_seen": 1299644416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004898856569709128, + "loss": 1.4422, + "theoretical_loss": 3.589879690899118, + "tokens_seen": 1299709952 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004898696088264794, + "loss": 1.3692, + "theoretical_loss": 3.5898634194886387, + "tokens_seen": 1299775488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004898535606820462, + "loss": 1.3866, + "theoretical_loss": 3.589847149128265, + "tokens_seen": 1299841024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004898375125376128, + "loss": 1.4209, + "theoretical_loss": 3.5898308798178755, + "tokens_seen": 1299906560 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004898214643931796, + "loss": 1.4354, + "theoretical_loss": 3.5898146115573493, + "tokens_seen": 1299972096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004898054162487462, + "loss": 1.359, + "theoretical_loss": 3.5897983443465664, + "tokens_seen": 1300037632 + }, + { + "epoch": 0.39, + "learning_rate": 0.000489789368104313, + "loss": 1.4943, + "theoretical_loss": 3.5897820781854053, + "tokens_seen": 1300103168 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004897733199598797, + "loss": 1.3495, + "theoretical_loss": 3.589765813073746, + "tokens_seen": 1300168704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004897572718154463, + "loss": 1.3841, + "theoretical_loss": 3.5897495490114686, + "tokens_seen": 1300234240 + }, + { + "epoch": 0.39, + "learning_rate": 0.000489741223671013, + "loss": 1.4819, + "theoretical_loss": 3.5897332859984505, + "tokens_seen": 1300299776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004897251755265797, + "loss": 1.2692, + "theoretical_loss": 3.5897170240345733, + "tokens_seen": 1300365312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004897091273821465, + "loss": 1.3227, + "theoretical_loss": 3.589700763119715, + "tokens_seen": 1300430848 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004896930792377131, + "loss": 1.2962, + "theoretical_loss": 3.589684503253755, + "tokens_seen": 1300496384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004896770310932799, + "loss": 1.3879, + "theoretical_loss": 3.5896682444365746, + "tokens_seen": 1300561920 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004896609829488465, + "loss": 1.3779, + "theoretical_loss": 3.589651986668051, + "tokens_seen": 1300627456 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004896449348044133, + "loss": 1.3759, + "theoretical_loss": 3.5896357299480655, + "tokens_seen": 1300692992 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004896288866599799, + "loss": 1.2914, + "theoretical_loss": 3.5896194742764966, + "tokens_seen": 1300758528 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004896128385155467, + "loss": 1.3408, + "theoretical_loss": 3.5896032196532244, + "tokens_seen": 1300824064 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 436727, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.392486810684204, + "objective/train/theoretical_loss": 3.589586966078128, + "objective/train/tokens_used": 1321349600, + "theoretical_loss": 3.589586966078128, + "tokens_seen": 1300889600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004895967903711133, + "loss": 1.2775, + "theoretical_loss": 3.589586966078128, + "tokens_seen": 1300889600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004895807422266801, + "loss": 1.3813, + "theoretical_loss": 3.5895707135510873, + "tokens_seen": 1300955136 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004895646940822467, + "loss": 1.3381, + "theoretical_loss": 3.589554462071982, + "tokens_seen": 1301020672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004895486459378135, + "loss": 1.3777, + "theoretical_loss": 3.5895382116406918, + "tokens_seen": 1301086208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004895325977933801, + "loss": 1.4776, + "theoretical_loss": 3.5895219622570957, + "tokens_seen": 1301151744 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004895165496489469, + "loss": 1.3612, + "theoretical_loss": 3.589505713921074, + "tokens_seen": 1301217280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004895005015045135, + "loss": 1.358, + "theoretical_loss": 3.5894894666325072, + "tokens_seen": 1301282816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004894844533600803, + "loss": 1.3097, + "theoretical_loss": 3.5894732203912736, + "tokens_seen": 1301348352 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004894684052156469, + "loss": 1.4728, + "theoretical_loss": 3.5894569751972534, + "tokens_seen": 1301413888 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004894523570712137, + "loss": 1.5459, + "theoretical_loss": 3.5894407310503267, + "tokens_seen": 1301479424 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004894363089267803, + "loss": 1.2955, + "theoretical_loss": 3.5894244879503727, + "tokens_seen": 1301544960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004894202607823471, + "loss": 1.3677, + "theoretical_loss": 3.589408245897272, + "tokens_seen": 1301610496 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004894042126379137, + "loss": 1.3244, + "theoretical_loss": 3.589392004890904, + "tokens_seen": 1301676032 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004893881644934805, + "loss": 1.358, + "theoretical_loss": 3.589375764931148, + "tokens_seen": 1301741568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004893721163490471, + "loss": 1.3236, + "theoretical_loss": 3.5893595260178843, + "tokens_seen": 1301807104 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004893560682046139, + "loss": 1.4503, + "theoretical_loss": 3.5893432881509937, + "tokens_seen": 1301872640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004893400200601805, + "loss": 1.2277, + "theoretical_loss": 3.589327051330355, + "tokens_seen": 1301938176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004893239719157473, + "loss": 1.2936, + "theoretical_loss": 3.589310815555848, + "tokens_seen": 1302003712 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004893079237713139, + "loss": 1.2937, + "theoretical_loss": 3.589294580827354, + "tokens_seen": 1302069248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004892918756268807, + "loss": 1.3151, + "theoretical_loss": 3.5892783471447514, + "tokens_seen": 1302134784 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004892758274824473, + "loss": 1.3537, + "theoretical_loss": 3.5892621145079207, + "tokens_seen": 1302200320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004892597793380141, + "loss": 1.2823, + "theoretical_loss": 3.5892458829167424, + "tokens_seen": 1302265856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004892437311935807, + "loss": 1.4119, + "theoretical_loss": 3.589229652371096, + "tokens_seen": 1302331392 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004892276830491475, + "loss": 1.3613, + "theoretical_loss": 3.589213422870862, + "tokens_seen": 1302396928 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004892116349047141, + "loss": 1.3079, + "theoretical_loss": 3.5891971944159193, + "tokens_seen": 1302462464 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004891955867602809, + "loss": 1.2965, + "theoretical_loss": 3.5891809670061496, + "tokens_seen": 1302528000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004891795386158475, + "loss": 1.271, + "theoretical_loss": 3.589164740641432, + "tokens_seen": 1302593536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004891634904714143, + "loss": 1.2464, + "theoretical_loss": 3.589148515321647, + "tokens_seen": 1302659072 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004891474423269809, + "loss": 1.3946, + "theoretical_loss": 3.589132291046675, + "tokens_seen": 1302724608 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004891313941825477, + "loss": 1.3117, + "theoretical_loss": 3.589116067816395, + "tokens_seen": 1302790144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004891153460381143, + "loss": 1.3672, + "theoretical_loss": 3.589099845630688, + "tokens_seen": 1302855680 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004890992978936811, + "loss": 1.3606, + "theoretical_loss": 3.589083624489435, + "tokens_seen": 1302921216 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004890832497492477, + "loss": 1.3398, + "theoretical_loss": 3.589067404392515, + "tokens_seen": 1302986752 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004890672016048145, + "loss": 1.3625, + "theoretical_loss": 3.589051185339809, + "tokens_seen": 1303052288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004890511534603811, + "loss": 1.4311, + "theoretical_loss": 3.5890349673311963, + "tokens_seen": 1303117824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004890351053159479, + "loss": 1.2492, + "theoretical_loss": 3.5890187503665576, + "tokens_seen": 1303183360 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004890190571715145, + "loss": 1.3321, + "theoretical_loss": 3.589002534445774, + "tokens_seen": 1303248896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004890030090270813, + "loss": 1.3745, + "theoretical_loss": 3.588986319568725, + "tokens_seen": 1303314432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004889869608826479, + "loss": 1.4623, + "theoretical_loss": 3.5889701057352914, + "tokens_seen": 1303379968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004889709127382147, + "loss": 1.3252, + "theoretical_loss": 3.5889538929453533, + "tokens_seen": 1303445504 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004889548645937813, + "loss": 1.3577, + "theoretical_loss": 3.5889376811987908, + "tokens_seen": 1303511040 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004889388164493481, + "loss": 1.377, + "theoretical_loss": 3.588921470495485, + "tokens_seen": 1303576576 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004889227683049147, + "loss": 1.3029, + "theoretical_loss": 3.5889052608353156, + "tokens_seen": 1303642112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004889067201604815, + "loss": 1.2623, + "theoretical_loss": 3.588889052218163, + "tokens_seen": 1303707648 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004888906720160481, + "loss": 1.2913, + "theoretical_loss": 3.588872844643909, + "tokens_seen": 1303773184 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004888746238716149, + "loss": 1.2814, + "theoretical_loss": 3.588856638112432, + "tokens_seen": 1303838720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004888585757271816, + "loss": 1.3374, + "theoretical_loss": 3.5888404326236145, + "tokens_seen": 1303904256 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004888425275827483, + "loss": 1.3847, + "theoretical_loss": 3.5888242281773355, + "tokens_seen": 1303969792 + }, + { + "epoch": 0.4, + "learning_rate": 0.000488826479438315, + "loss": 1.4997, + "theoretical_loss": 3.588808024773477, + "tokens_seen": 1304035328 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004888104312938817, + "loss": 1.2831, + "theoretical_loss": 3.5887918224119177, + "tokens_seen": 1304100864 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 437838, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.382591962814331, + "objective/train/theoretical_loss": 3.58877562109254, + "objective/train/tokens_used": 1324626400, + "theoretical_loss": 3.58877562109254, + "tokens_seen": 1304166400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004887943831494484, + "loss": 1.4017, + "theoretical_loss": 3.58877562109254, + "tokens_seen": 1304166400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004887783350050151, + "loss": 1.3708, + "theoretical_loss": 3.588759420815223, + "tokens_seen": 1304231936 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004887622868605818, + "loss": 1.3601, + "theoretical_loss": 3.5887432215798487, + "tokens_seen": 1304297472 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004887462387161485, + "loss": 1.2826, + "theoretical_loss": 3.588727023386297, + "tokens_seen": 1304363008 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004887301905717152, + "loss": 1.4542, + "theoretical_loss": 3.588710826234448, + "tokens_seen": 1304428544 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004887141424272819, + "loss": 1.2902, + "theoretical_loss": 3.5886946301241833, + "tokens_seen": 1304494080 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004886980942828486, + "loss": 1.3722, + "theoretical_loss": 3.5886784350553835, + "tokens_seen": 1304559616 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004886820461384153, + "loss": 1.281, + "theoretical_loss": 3.5886622410279294, + "tokens_seen": 1304625152 + }, + { + "epoch": 0.4, + "learning_rate": 0.000488665997993982, + "loss": 1.2987, + "theoretical_loss": 3.588646048041701, + "tokens_seen": 1304690688 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004886499498495487, + "loss": 1.2728, + "theoretical_loss": 3.5886298560965795, + "tokens_seen": 1304756224 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004886339017051155, + "loss": 1.2925, + "theoretical_loss": 3.5886136651924456, + "tokens_seen": 1304821760 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004886178535606821, + "loss": 1.3501, + "theoretical_loss": 3.58859747532918, + "tokens_seen": 1304887296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004886018054162489, + "loss": 1.3987, + "theoretical_loss": 3.588581286506664, + "tokens_seen": 1304952832 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004885857572718155, + "loss": 1.2724, + "theoretical_loss": 3.5885650987247777, + "tokens_seen": 1305018368 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004885697091273823, + "loss": 1.4087, + "theoretical_loss": 3.588548911983403, + "tokens_seen": 1305083904 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004885536609829489, + "loss": 1.3, + "theoretical_loss": 3.58853272628242, + "tokens_seen": 1305149440 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004885376128385157, + "loss": 1.4646, + "theoretical_loss": 3.5885165416217095, + "tokens_seen": 1305214976 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004885215646940823, + "loss": 1.2494, + "theoretical_loss": 3.588500358001153, + "tokens_seen": 1305280512 + }, + { + "epoch": 0.4, + "learning_rate": 0.000488505516549649, + "loss": 1.3871, + "theoretical_loss": 3.5884841754206303, + "tokens_seen": 1305346048 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004884894684052157, + "loss": 1.236, + "theoretical_loss": 3.588467993880024, + "tokens_seen": 1305411584 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004884734202607823, + "loss": 1.4106, + "theoretical_loss": 3.588451813379214, + "tokens_seen": 1305477120 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004884573721163491, + "loss": 1.2457, + "theoretical_loss": 3.588435633918081, + "tokens_seen": 1305542656 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004884413239719157, + "loss": 1.3543, + "theoretical_loss": 3.5884194554965068, + "tokens_seen": 1305608192 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004884252758274825, + "loss": 1.3426, + "theoretical_loss": 3.588403278114372, + "tokens_seen": 1305673728 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004884092276830492, + "loss": 1.2778, + "theoretical_loss": 3.5883871017715583, + "tokens_seen": 1305739264 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004883931795386159, + "loss": 1.2656, + "theoretical_loss": 3.5883709264679458, + "tokens_seen": 1305804800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004883771313941826, + "loss": 1.2422, + "theoretical_loss": 3.588354752203416, + "tokens_seen": 1305870336 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004883610832497492, + "loss": 1.2721, + "theoretical_loss": 3.5883385789778504, + "tokens_seen": 1305935872 + }, + { + "epoch": 0.4, + "learning_rate": 0.000488345035105316, + "loss": 1.3752, + "theoretical_loss": 3.5883224067911295, + "tokens_seen": 1306001408 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004883289869608826, + "loss": 1.3963, + "theoretical_loss": 3.5883062356431346, + "tokens_seen": 1306066944 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004883129388164494, + "loss": 1.3193, + "theoretical_loss": 3.588290065533747, + "tokens_seen": 1306132480 + }, + { + "epoch": 0.4, + "learning_rate": 0.000488296890672016, + "loss": 1.3588, + "theoretical_loss": 3.5882738964628484, + "tokens_seen": 1306198016 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004882808425275828, + "loss": 1.2814, + "theoretical_loss": 3.588257728430319, + "tokens_seen": 1306263552 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048826479438314946, + "loss": 1.3708, + "theoretical_loss": 3.588241561436041, + "tokens_seen": 1306329088 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004882487462387162, + "loss": 1.2912, + "theoretical_loss": 3.5882253954798946, + "tokens_seen": 1306394624 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048823269809428286, + "loss": 1.3338, + "theoretical_loss": 3.5882092305617617, + "tokens_seen": 1306460160 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004882166499498496, + "loss": 1.3748, + "theoretical_loss": 3.588193066681524, + "tokens_seen": 1306525696 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048820060180541626, + "loss": 1.3714, + "theoretical_loss": 3.588176903839062, + "tokens_seen": 1306591232 + }, + { + "epoch": 0.4, + "learning_rate": 0.000488184553660983, + "loss": 1.2621, + "theoretical_loss": 3.5881607420342574, + "tokens_seen": 1306656768 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048816850551654966, + "loss": 1.3108, + "theoretical_loss": 3.588144581266991, + "tokens_seen": 1306722304 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004881524573721164, + "loss": 1.4491, + "theoretical_loss": 3.588128421537145, + "tokens_seen": 1306787840 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048813640922768306, + "loss": 1.3293, + "theoretical_loss": 3.5881122628446005, + "tokens_seen": 1306853376 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048812036108324976, + "loss": 1.4123, + "theoretical_loss": 3.588096105189239, + "tokens_seen": 1306918912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048810431293881646, + "loss": 1.3747, + "theoretical_loss": 3.5880799485709414, + "tokens_seen": 1306984448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048808826479438316, + "loss": 1.3194, + "theoretical_loss": 3.5880637929895896, + "tokens_seen": 1307049984 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048807221664994986, + "loss": 1.4002, + "theoretical_loss": 3.588047638445065, + "tokens_seen": 1307115520 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048805616850551656, + "loss": 1.3388, + "theoretical_loss": 3.5880314849372485, + "tokens_seen": 1307181056 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048804012036108327, + "loss": 1.3694, + "theoretical_loss": 3.588015332466023, + "tokens_seen": 1307246592 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048802407221664997, + "loss": 1.3555, + "theoretical_loss": 3.5879991810312686, + "tokens_seen": 1307312128 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048800802407221667, + "loss": 1.398, + "theoretical_loss": 3.5879830306328673, + "tokens_seen": 1307377664 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 438740, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2633261680603027, + "objective/train/theoretical_loss": 3.587966881270701, + "objective/train/tokens_used": 1327903200, + "theoretical_loss": 3.587966881270701, + "tokens_seen": 1307443200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048799197592778337, + "loss": 1.2872, + "theoretical_loss": 3.587966881270701, + "tokens_seen": 1307443200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048797592778335, + "loss": 1.486, + "theoretical_loss": 3.587950732944651, + "tokens_seen": 1307508736 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048795987963891677, + "loss": 1.3327, + "theoretical_loss": 3.5879345856545988, + "tokens_seen": 1307574272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004879438314944834, + "loss": 1.3837, + "theoretical_loss": 3.5879184394004264, + "tokens_seen": 1307639808 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048792778335005017, + "loss": 1.3712, + "theoretical_loss": 3.587902294182015, + "tokens_seen": 1307705344 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004879117352056168, + "loss": 1.2574, + "theoretical_loss": 3.587886149999246, + "tokens_seen": 1307770880 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048789568706118357, + "loss": 1.4947, + "theoretical_loss": 3.587870006852002, + "tokens_seen": 1307836416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004878796389167502, + "loss": 1.3119, + "theoretical_loss": 3.587853864740164, + "tokens_seen": 1307901952 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048786359077231697, + "loss": 1.3567, + "theoretical_loss": 3.587837723663614, + "tokens_seen": 1307967488 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004878475426278836, + "loss": 1.3144, + "theoretical_loss": 3.587821583622233, + "tokens_seen": 1308033024 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004878314944834504, + "loss": 1.3094, + "theoretical_loss": 3.5878054446159044, + "tokens_seen": 1308098560 + }, + { + "epoch": 0.4, + "learning_rate": 0.000487815446339017, + "loss": 1.3086, + "theoretical_loss": 3.587789306644509, + "tokens_seen": 1308164096 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004877993981945838, + "loss": 1.4832, + "theoretical_loss": 3.5877731697079276, + "tokens_seen": 1308229632 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004877833500501504, + "loss": 1.3206, + "theoretical_loss": 3.587757033806043, + "tokens_seen": 1308295168 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004877673019057172, + "loss": 1.4024, + "theoretical_loss": 3.587740898938738, + "tokens_seen": 1308360704 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004877512537612838, + "loss": 1.248, + "theoretical_loss": 3.5877247651058926, + "tokens_seen": 1308426240 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004877352056168506, + "loss": 1.2777, + "theoretical_loss": 3.5877086323073897, + "tokens_seen": 1308491776 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004877191574724172, + "loss": 1.3802, + "theoretical_loss": 3.587692500543111, + "tokens_seen": 1308557312 + }, + { + "epoch": 0.4, + "learning_rate": 0.000487703109327984, + "loss": 1.4371, + "theoretical_loss": 3.5876763698129386, + "tokens_seen": 1308622848 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048768706118355073, + "loss": 1.4213, + "theoretical_loss": 3.5876602401167537, + "tokens_seen": 1308688384 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004876710130391174, + "loss": 1.3535, + "theoretical_loss": 3.5876441114544395, + "tokens_seen": 1308753920 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048765496489468413, + "loss": 1.4607, + "theoretical_loss": 3.587627983825876, + "tokens_seen": 1308819456 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004876389167502508, + "loss": 1.3423, + "theoretical_loss": 3.587611857230948, + "tokens_seen": 1308884992 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048762286860581754, + "loss": 1.3082, + "theoretical_loss": 3.587595731669535, + "tokens_seen": 1308950528 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004876068204613842, + "loss": 1.3249, + "theoretical_loss": 3.58757960714152, + "tokens_seen": 1309016064 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048759077231695094, + "loss": 1.3357, + "theoretical_loss": 3.587563483646785, + "tokens_seen": 1309081600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004875747241725176, + "loss": 1.3371, + "theoretical_loss": 3.5875473611852122, + "tokens_seen": 1309147136 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048755867602808434, + "loss": 1.3521, + "theoretical_loss": 3.5875312397566836, + "tokens_seen": 1309212672 + }, + { + "epoch": 0.4, + "learning_rate": 0.000487542627883651, + "loss": 1.415, + "theoretical_loss": 3.5875151193610817, + "tokens_seen": 1309278208 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048752657973921774, + "loss": 1.3108, + "theoretical_loss": 3.587498999998287, + "tokens_seen": 1309343744 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004875105315947844, + "loss": 1.2217, + "theoretical_loss": 3.5874828816681834, + "tokens_seen": 1309409280 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004874944834503511, + "loss": 1.3103, + "theoretical_loss": 3.5874667643706526, + "tokens_seen": 1309474816 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004874784353059178, + "loss": 1.2793, + "theoretical_loss": 3.587450648105577, + "tokens_seen": 1309540352 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004874623871614845, + "loss": 1.3311, + "theoretical_loss": 3.5874345328728374, + "tokens_seen": 1309605888 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004874463390170512, + "loss": 1.3557, + "theoretical_loss": 3.5874184186723177, + "tokens_seen": 1309671424 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004874302908726179, + "loss": 1.425, + "theoretical_loss": 3.5874023055038995, + "tokens_seen": 1309736960 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004874142427281846, + "loss": 1.2403, + "theoretical_loss": 3.5873861933674647, + "tokens_seen": 1309802496 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004873981945837513, + "loss": 1.384, + "theoretical_loss": 3.5873700822628964, + "tokens_seen": 1309868032 + }, + { + "epoch": 0.4, + "learning_rate": 0.000487382146439318, + "loss": 1.3239, + "theoretical_loss": 3.5873539721900762, + "tokens_seen": 1309933568 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004873660982948847, + "loss": 1.3367, + "theoretical_loss": 3.5873378631488864, + "tokens_seen": 1309999104 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048735005015045134, + "loss": 1.4561, + "theoretical_loss": 3.58732175513921, + "tokens_seen": 1310064640 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004873340020060181, + "loss": 1.3727, + "theoretical_loss": 3.5873056481609282, + "tokens_seen": 1310130176 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048731795386158474, + "loss": 1.3093, + "theoretical_loss": 3.5872895422139246, + "tokens_seen": 1310195712 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004873019057171515, + "loss": 1.4342, + "theoretical_loss": 3.587273437298081, + "tokens_seen": 1310261248 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048728585757271814, + "loss": 1.3079, + "theoretical_loss": 3.5872573334132802, + "tokens_seen": 1310326784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004872698094282849, + "loss": 1.4567, + "theoretical_loss": 3.587241230559404, + "tokens_seen": 1310392320 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048725376128385154, + "loss": 1.2264, + "theoretical_loss": 3.5872251287363355, + "tokens_seen": 1310457856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004872377131394183, + "loss": 1.3824, + "theoretical_loss": 3.587209027943956, + "tokens_seen": 1310523392 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048722166499498494, + "loss": 1.2255, + "theoretical_loss": 3.5871929281821497, + "tokens_seen": 1310588928 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004872056168505517, + "loss": 1.3629, + "theoretical_loss": 3.5871768294507973, + "tokens_seen": 1310654464 + }, + { + "debugging/Compilability": 0.0, + "debugging/distinct-1-grams": 0.6869446880300378, + "debugging/entropy-1-grams": 4.904166514984576, + "debugging/length": 583.2222222222222, + "debugging/num_segments": 9, + "debugging/score": 0.07426709959248204, + "debugging/score_std": 0.021458313531614225, + "epoch": 0.4, + "objective/train/docs_used": 440076, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2848517894744873, + "objective/train/theoretical_loss": 3.5871607317497833, + "objective/train/tokens_used": 1331180000, + "theoretical_loss": 3.5871607317497833, + "tokens_seen": 1310720000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048718956870611834, + "loss": 1.3497, + "theoretical_loss": 3.5871607317497833, + "tokens_seen": 1310720000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004871735205616851, + "loss": 1.4093, + "theoretical_loss": 3.587144635078989, + "tokens_seen": 1310785536 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048715747241725175, + "loss": 1.3719, + "theoretical_loss": 3.5871285394382966, + "tokens_seen": 1310851072 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004871414242728185, + "loss": 1.3175, + "theoretical_loss": 3.5871124448275893, + "tokens_seen": 1310916608 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048712537612838515, + "loss": 1.3838, + "theoretical_loss": 3.5870963512467497, + "tokens_seen": 1310982144 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004871093279839519, + "loss": 1.3672, + "theoretical_loss": 3.5870802586956607, + "tokens_seen": 1311047680 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048709327983951855, + "loss": 1.2974, + "theoretical_loss": 3.5870641671742045, + "tokens_seen": 1311113216 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004870772316950853, + "loss": 1.4125, + "theoretical_loss": 3.587048076682264, + "tokens_seen": 1311178752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048706118355065195, + "loss": 1.4156, + "theoretical_loss": 3.5870319872197216, + "tokens_seen": 1311244288 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004870451354062187, + "loss": 1.2327, + "theoretical_loss": 3.58701589878646, + "tokens_seen": 1311309824 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048702908726178535, + "loss": 1.2803, + "theoretical_loss": 3.5869998113823622, + "tokens_seen": 1311375360 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004870130391173521, + "loss": 1.4842, + "theoretical_loss": 3.586983725007311, + "tokens_seen": 1311440896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048699699097291875, + "loss": 1.273, + "theoretical_loss": 3.5869676396611885, + "tokens_seen": 1311506432 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004869809428284855, + "loss": 1.365, + "theoretical_loss": 3.5869515553438784, + "tokens_seen": 1311571968 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048696489468405215, + "loss": 1.3436, + "theoretical_loss": 3.586935472055263, + "tokens_seen": 1311637504 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004869488465396189, + "loss": 1.3666, + "theoretical_loss": 3.5869193897952245, + "tokens_seen": 1311703040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048693279839518555, + "loss": 1.3589, + "theoretical_loss": 3.586903308563647, + "tokens_seen": 1311768576 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004869167502507523, + "loss": 1.3215, + "theoretical_loss": 3.586887228360413, + "tokens_seen": 1311834112 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048690070210631896, + "loss": 1.2588, + "theoretical_loss": 3.586871149185405, + "tokens_seen": 1311899648 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004868846539618857, + "loss": 1.4483, + "theoretical_loss": 3.5868550710385056, + "tokens_seen": 1311965184 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048686860581745236, + "loss": 1.3977, + "theoretical_loss": 3.586838993919598, + "tokens_seen": 1312030720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004868525576730191, + "loss": 1.3464, + "theoretical_loss": 3.5868229178285658, + "tokens_seen": 1312096256 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048683650952858576, + "loss": 1.2904, + "theoretical_loss": 3.5868068427652906, + "tokens_seen": 1312161792 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004868204613841525, + "loss": 1.264, + "theoretical_loss": 3.586790768729657, + "tokens_seen": 1312227328 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048680441323971916, + "loss": 1.2785, + "theoretical_loss": 3.586774695721547, + "tokens_seen": 1312292864 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004867883650952859, + "loss": 1.3954, + "theoretical_loss": 3.5867586237408435, + "tokens_seen": 1312358400 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048677231695085256, + "loss": 1.4794, + "theoretical_loss": 3.5867425527874297, + "tokens_seen": 1312423936 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004867562688064193, + "loss": 1.3471, + "theoretical_loss": 3.586726482861189, + "tokens_seen": 1312489472 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048674022066198596, + "loss": 1.3096, + "theoretical_loss": 3.586710413962004, + "tokens_seen": 1312555008 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048672417251755266, + "loss": 1.3247, + "theoretical_loss": 3.5866943460897582, + "tokens_seen": 1312620544 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048670812437311936, + "loss": 1.4049, + "theoretical_loss": 3.5866782792443344, + "tokens_seen": 1312686080 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048669207622868606, + "loss": 1.3004, + "theoretical_loss": 3.5866622134256154, + "tokens_seen": 1312751616 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048667602808425277, + "loss": 1.4206, + "theoretical_loss": 3.586646148633485, + "tokens_seen": 1312817152 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048665997993981947, + "loss": 1.223, + "theoretical_loss": 3.586630084867826, + "tokens_seen": 1312882688 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048664393179538617, + "loss": 1.3097, + "theoretical_loss": 3.586614022128522, + "tokens_seen": 1312948224 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048662788365095287, + "loss": 1.3496, + "theoretical_loss": 3.586597960415456, + "tokens_seen": 1313013760 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048661183550651957, + "loss": 1.3372, + "theoretical_loss": 3.5865818997285106, + "tokens_seen": 1313079296 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048659578736208627, + "loss": 1.3366, + "theoretical_loss": 3.5865658400675695, + "tokens_seen": 1313144832 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004865797392176529, + "loss": 1.1644, + "theoretical_loss": 3.5865497814325162, + "tokens_seen": 1313210368 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048656369107321967, + "loss": 1.2771, + "theoretical_loss": 3.5865337238232335, + "tokens_seen": 1313275904 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004865476429287863, + "loss": 1.2773, + "theoretical_loss": 3.5865176672396055, + "tokens_seen": 1313341440 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048653159478435307, + "loss": 1.4803, + "theoretical_loss": 3.5865016116815145, + "tokens_seen": 1313406976 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004865155466399197, + "loss": 1.2962, + "theoretical_loss": 3.5864855571488445, + "tokens_seen": 1313472512 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048649949849548647, + "loss": 1.3954, + "theoretical_loss": 3.586469503641479, + "tokens_seen": 1313538048 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004864834503510532, + "loss": 1.3354, + "theoretical_loss": 3.5864534511593003, + "tokens_seen": 1313603584 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004864674022066199, + "loss": 1.3211, + "theoretical_loss": 3.5864373997021928, + "tokens_seen": 1313669120 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048645135406218663, + "loss": 1.3154, + "theoretical_loss": 3.5864213492700396, + "tokens_seen": 1313734656 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004864353059177533, + "loss": 1.3031, + "theoretical_loss": 3.586405299862724, + "tokens_seen": 1313800192 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048641925777332003, + "loss": 1.2972, + "theoretical_loss": 3.5863892514801297, + "tokens_seen": 1313865728 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004864032096288867, + "loss": 1.274, + "theoretical_loss": 3.58637320412214, + "tokens_seen": 1313931264 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 441061, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4314746856689453, + "objective/train/theoretical_loss": 3.5863571577886386, + "objective/train/tokens_used": 1334456800, + "theoretical_loss": 3.5863571577886386, + "tokens_seen": 1313996800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048638716148445343, + "loss": 1.4004, + "theoretical_loss": 3.5863571577886386, + "tokens_seen": 1313996800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004863711133400201, + "loss": 1.4439, + "theoretical_loss": 3.586341112479509, + "tokens_seen": 1314062336 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048635506519558683, + "loss": 1.3404, + "theoretical_loss": 3.5863250681946344, + "tokens_seen": 1314127872 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004863390170511535, + "loss": 1.2621, + "theoretical_loss": 3.586309024933898, + "tokens_seen": 1314193408 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048632296890672023, + "loss": 1.3316, + "theoretical_loss": 3.5862929826971843, + "tokens_seen": 1314258944 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004863069207622869, + "loss": 1.2931, + "theoretical_loss": 3.5862769414843765, + "tokens_seen": 1314324480 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048629087261785363, + "loss": 1.4122, + "theoretical_loss": 3.5862609012953577, + "tokens_seen": 1314390016 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004862748244734203, + "loss": 1.4399, + "theoretical_loss": 3.5862448621300125, + "tokens_seen": 1314455552 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048625877632898704, + "loss": 1.293, + "theoretical_loss": 3.5862288239882236, + "tokens_seen": 1314521088 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004862427281845537, + "loss": 1.2831, + "theoretical_loss": 3.5862127868698757, + "tokens_seen": 1314586624 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048622668004012044, + "loss": 1.3394, + "theoretical_loss": 3.5861967507748513, + "tokens_seen": 1314652160 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004862106318956871, + "loss": 1.3394, + "theoretical_loss": 3.5861807157030343, + "tokens_seen": 1314717696 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048619458375125384, + "loss": 1.3589, + "theoretical_loss": 3.586164681654309, + "tokens_seen": 1314783232 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004861785356068205, + "loss": 1.3801, + "theoretical_loss": 3.5861486486285594, + "tokens_seen": 1314848768 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048616248746238724, + "loss": 1.2873, + "theoretical_loss": 3.586132616625668, + "tokens_seen": 1314914304 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004861464393179539, + "loss": 1.3393, + "theoretical_loss": 3.5861165856455197, + "tokens_seen": 1314979840 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048613039117352064, + "loss": 1.3433, + "theoretical_loss": 3.586100555687998, + "tokens_seen": 1315045376 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004861143430290873, + "loss": 1.3301, + "theoretical_loss": 3.5860845267529857, + "tokens_seen": 1315110912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048609829488465404, + "loss": 1.3748, + "theoretical_loss": 3.586068498840368, + "tokens_seen": 1315176448 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004860822467402207, + "loss": 1.251, + "theoretical_loss": 3.586052471950029, + "tokens_seen": 1315241984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004860661985957874, + "loss": 1.3338, + "theoretical_loss": 3.5860364460818506, + "tokens_seen": 1315307520 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004860501504513541, + "loss": 1.3986, + "theoretical_loss": 3.5860204212357187, + "tokens_seen": 1315373056 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004860341023069208, + "loss": 1.3436, + "theoretical_loss": 3.586004397411516, + "tokens_seen": 1315438592 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004860180541624875, + "loss": 1.266, + "theoretical_loss": 3.5859883746091272, + "tokens_seen": 1315504128 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004860020060180542, + "loss": 1.2852, + "theoretical_loss": 3.5859723528284357, + "tokens_seen": 1315569664 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004859859578736209, + "loss": 1.4979, + "theoretical_loss": 3.5859563320693253, + "tokens_seen": 1315635200 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004859699097291876, + "loss": 1.246, + "theoretical_loss": 3.585940312331681, + "tokens_seen": 1315700736 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048595386158475424, + "loss": 1.3416, + "theoretical_loss": 3.5859242936153857, + "tokens_seen": 1315766272 + }, + { + "epoch": 0.4, + "learning_rate": 0.000485937813440321, + "loss": 1.407, + "theoretical_loss": 3.5859082759203234, + "tokens_seen": 1315831808 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048592176529588764, + "loss": 1.2157, + "theoretical_loss": 3.585892259246379, + "tokens_seen": 1315897344 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004859057171514544, + "loss": 1.3887, + "theoretical_loss": 3.585876243593436, + "tokens_seen": 1315962880 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048588966900702104, + "loss": 1.3234, + "theoretical_loss": 3.5858602289613786, + "tokens_seen": 1316028416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004858736208625878, + "loss": 1.3796, + "theoretical_loss": 3.585844215350091, + "tokens_seen": 1316093952 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048585757271815444, + "loss": 1.3373, + "theoretical_loss": 3.585828202759457, + "tokens_seen": 1316159488 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004858415245737212, + "loss": 1.3937, + "theoretical_loss": 3.585812191189361, + "tokens_seen": 1316225024 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048582547642928784, + "loss": 1.4095, + "theoretical_loss": 3.5857961806396874, + "tokens_seen": 1316290560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004858094282848546, + "loss": 1.373, + "theoretical_loss": 3.5857801711103194, + "tokens_seen": 1316356096 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048579338014042125, + "loss": 1.3198, + "theoretical_loss": 3.585764162601142, + "tokens_seen": 1316421632 + }, + { + "epoch": 0.4, + "learning_rate": 0.000485777331995988, + "loss": 1.3021, + "theoretical_loss": 3.5857481551120394, + "tokens_seen": 1316487168 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048576128385155465, + "loss": 1.2573, + "theoretical_loss": 3.5857321486428955, + "tokens_seen": 1316552704 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004857452357071214, + "loss": 1.3368, + "theoretical_loss": 3.585716143193595, + "tokens_seen": 1316618240 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048572918756268805, + "loss": 1.395, + "theoretical_loss": 3.5857001387640213, + "tokens_seen": 1316683776 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004857131394182548, + "loss": 1.3204, + "theoretical_loss": 3.5856841353540596, + "tokens_seen": 1316749312 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048569709127382145, + "loss": 1.2323, + "theoretical_loss": 3.585668132963594, + "tokens_seen": 1316814848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004856810431293882, + "loss": 1.442, + "theoretical_loss": 3.585652131592508, + "tokens_seen": 1316880384 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048566499498495485, + "loss": 1.244, + "theoretical_loss": 3.585636131240687, + "tokens_seen": 1316945920 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004856489468405216, + "loss": 1.3468, + "theoretical_loss": 3.5856201319080148, + "tokens_seen": 1317011456 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048563289869608825, + "loss": 1.3423, + "theoretical_loss": 3.5856041335943765, + "tokens_seen": 1317076992 + }, + { + "epoch": 0.4, + "learning_rate": 0.000485616850551655, + "loss": 1.2923, + "theoretical_loss": 3.585588136299655, + "tokens_seen": 1317142528 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048560080240722165, + "loss": 1.4071, + "theoretical_loss": 3.5855721400237366, + "tokens_seen": 1317208064 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 442341, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3773832321166992, + "objective/train/theoretical_loss": 3.585556144766504, + "objective/train/tokens_used": 1337733600, + "theoretical_loss": 3.585556144766504, + "tokens_seen": 1317273600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004855847542627884, + "loss": 1.4276, + "theoretical_loss": 3.585556144766504, + "tokens_seen": 1317273600 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048556870611835505, + "loss": 1.3344, + "theoretical_loss": 3.5855401505278426, + "tokens_seen": 1317339136 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004855526579739218, + "loss": 1.4097, + "theoretical_loss": 3.5855241573076366, + "tokens_seen": 1317404672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048553660982948846, + "loss": 1.2924, + "theoretical_loss": 3.5855081651057708, + "tokens_seen": 1317470208 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004855205616850552, + "loss": 1.3379, + "theoretical_loss": 3.5854921739221295, + "tokens_seen": 1317535744 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048550451354062186, + "loss": 1.3209, + "theoretical_loss": 3.5854761837565974, + "tokens_seen": 1317601280 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004854884653961886, + "loss": 1.3033, + "theoretical_loss": 3.5854601946090585, + "tokens_seen": 1317666816 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048547241725175526, + "loss": 1.3764, + "theoretical_loss": 3.585444206479398, + "tokens_seen": 1317732352 + }, + { + "epoch": 0.4, + "learning_rate": 0.000485456369107322, + "loss": 1.3218, + "theoretical_loss": 3.5854282193674996, + "tokens_seen": 1317797888 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048544032096288866, + "loss": 1.4137, + "theoretical_loss": 3.585412233273249, + "tokens_seen": 1317863424 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004854242728184554, + "loss": 1.3492, + "theoretical_loss": 3.5853962481965302, + "tokens_seen": 1317928960 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048540822467402206, + "loss": 1.2507, + "theoretical_loss": 3.5853802641372283, + "tokens_seen": 1317994496 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004853921765295888, + "loss": 1.243, + "theoretical_loss": 3.5853642810952273, + "tokens_seen": 1318060032 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048537612838515546, + "loss": 1.3568, + "theoretical_loss": 3.5853482990704117, + "tokens_seen": 1318125568 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004853600802407222, + "loss": 1.2306, + "theoretical_loss": 3.5853323180626675, + "tokens_seen": 1318191104 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048534403209628886, + "loss": 1.3425, + "theoretical_loss": 3.5853163380718787, + "tokens_seen": 1318256640 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004853279839518556, + "loss": 1.2497, + "theoretical_loss": 3.5853003590979293, + "tokens_seen": 1318322176 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048531193580742226, + "loss": 1.2938, + "theoretical_loss": 3.585284381140705, + "tokens_seen": 1318387712 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048529588766298897, + "loss": 1.2727, + "theoretical_loss": 3.5852684042000904, + "tokens_seen": 1318453248 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004852798395185557, + "loss": 1.3531, + "theoretical_loss": 3.58525242827597, + "tokens_seen": 1318518784 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048526379137412237, + "loss": 1.4274, + "theoretical_loss": 3.5852364533682284, + "tokens_seen": 1318584320 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004852477432296891, + "loss": 1.3313, + "theoretical_loss": 3.5852204794767513, + "tokens_seen": 1318649856 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048523169508525577, + "loss": 1.3909, + "theoretical_loss": 3.5852045066014226, + "tokens_seen": 1318715392 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004852156469408225, + "loss": 1.2567, + "theoretical_loss": 3.585188534742128, + "tokens_seen": 1318780928 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048519959879638917, + "loss": 1.2556, + "theoretical_loss": 3.5851725638987517, + "tokens_seen": 1318846464 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004851835506519559, + "loss": 1.3358, + "theoretical_loss": 3.5851565940711794, + "tokens_seen": 1318912000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048516750250752257, + "loss": 1.2443, + "theoretical_loss": 3.585140625259295, + "tokens_seen": 1318977536 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004851514543630893, + "loss": 1.4086, + "theoretical_loss": 3.585124657462984, + "tokens_seen": 1319043072 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048513540621865597, + "loss": 1.293, + "theoretical_loss": 3.585108690682131, + "tokens_seen": 1319108608 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004851193580742227, + "loss": 1.3807, + "theoretical_loss": 3.5850927249166222, + "tokens_seen": 1319174144 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004851033099297894, + "loss": 1.2991, + "theoretical_loss": 3.585076760166341, + "tokens_seen": 1319239680 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048508726178535613, + "loss": 1.3025, + "theoretical_loss": 3.5850607964311734, + "tokens_seen": 1319305216 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004850712136409228, + "loss": 1.2968, + "theoretical_loss": 3.585044833711004, + "tokens_seen": 1319370752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048505516549648953, + "loss": 1.2663, + "theoretical_loss": 3.5850288720057177, + "tokens_seen": 1319436288 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004850391173520562, + "loss": 1.2713, + "theoretical_loss": 3.5850129113152, + "tokens_seen": 1319501824 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048502306920762293, + "loss": 1.2781, + "theoretical_loss": 3.5849969516393365, + "tokens_seen": 1319567360 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004850070210631896, + "loss": 1.2317, + "theoretical_loss": 3.5849809929780108, + "tokens_seen": 1319632896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048499097291875633, + "loss": 1.306, + "theoretical_loss": 3.5849650353311096, + "tokens_seen": 1319698432 + }, + { + "epoch": 0.4, + "learning_rate": 0.000484974924774323, + "loss": 1.3761, + "theoretical_loss": 3.584949078698517, + "tokens_seen": 1319763968 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048495887662988973, + "loss": 1.2449, + "theoretical_loss": 3.584933123080118, + "tokens_seen": 1319829504 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004849428284854564, + "loss": 1.2852, + "theoretical_loss": 3.584917168475799, + "tokens_seen": 1319895040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048492678034102313, + "loss": 1.3378, + "theoretical_loss": 3.584901214885444, + "tokens_seen": 1319960576 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004849107321965898, + "loss": 1.4203, + "theoretical_loss": 3.584885262308939, + "tokens_seen": 1320026112 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048489468405215654, + "loss": 1.3073, + "theoretical_loss": 3.5848693107461687, + "tokens_seen": 1320091648 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004848786359077232, + "loss": 1.3765, + "theoretical_loss": 3.5848533601970187, + "tokens_seen": 1320157184 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048486258776328994, + "loss": 1.3058, + "theoretical_loss": 3.5848374106613745, + "tokens_seen": 1320222720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004848465396188566, + "loss": 1.2311, + "theoretical_loss": 3.5848214621391206, + "tokens_seen": 1320288256 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048483049147442334, + "loss": 1.3319, + "theoretical_loss": 3.584805514630143, + "tokens_seen": 1320353792 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048481444332999, + "loss": 1.3171, + "theoretical_loss": 3.584789568134327, + "tokens_seen": 1320419328 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048479839518555674, + "loss": 1.3266, + "theoretical_loss": 3.584773622651557, + "tokens_seen": 1320484864 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 443211, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4416449069976807, + "objective/train/theoretical_loss": 3.5847576781817203, + "objective/train/tokens_used": 1341010400, + "theoretical_loss": 3.5847576781817203, + "tokens_seen": 1320550400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004847823470411234, + "loss": 1.3684, + "theoretical_loss": 3.5847576781817203, + "tokens_seen": 1320550400 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048476629889669014, + "loss": 1.2597, + "theoretical_loss": 3.5847417347247004, + "tokens_seen": 1320615936 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004847502507522568, + "loss": 1.3509, + "theoretical_loss": 3.5847257922803832, + "tokens_seen": 1320681472 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048473420260782354, + "loss": 1.2616, + "theoretical_loss": 3.584709850848655, + "tokens_seen": 1320747008 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004847181544633902, + "loss": 1.3003, + "theoretical_loss": 3.5846939104294, + "tokens_seen": 1320812544 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048470210631895694, + "loss": 1.3852, + "theoretical_loss": 3.584677971022505, + "tokens_seen": 1320878080 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004846860581745236, + "loss": 1.3603, + "theoretical_loss": 3.5846620326278544, + "tokens_seen": 1320943616 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004846700100300903, + "loss": 1.3746, + "theoretical_loss": 3.5846460952453336, + "tokens_seen": 1321009152 + }, + { + "epoch": 0.4, + "learning_rate": 0.000484653961885657, + "loss": 1.263, + "theoretical_loss": 3.5846301588748295, + "tokens_seen": 1321074688 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004846379137412237, + "loss": 1.3042, + "theoretical_loss": 3.584614223516226, + "tokens_seen": 1321140224 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004846218655967904, + "loss": 1.3297, + "theoretical_loss": 3.58459828916941, + "tokens_seen": 1321205760 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004846058174523571, + "loss": 1.1947, + "theoretical_loss": 3.5845823558342653, + "tokens_seen": 1321271296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004845897693079238, + "loss": 1.2345, + "theoretical_loss": 3.5845664235106796, + "tokens_seen": 1321336832 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004845737211634905, + "loss": 1.5226, + "theoretical_loss": 3.584550492198537, + "tokens_seen": 1321402368 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004845576730190572, + "loss": 1.3637, + "theoretical_loss": 3.5845345618977245, + "tokens_seen": 1321467904 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004845416248746239, + "loss": 1.3617, + "theoretical_loss": 3.584518632608126, + "tokens_seen": 1321533440 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048452557673019054, + "loss": 1.2552, + "theoretical_loss": 3.584502704329629, + "tokens_seen": 1321598976 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004845095285857573, + "loss": 1.3647, + "theoretical_loss": 3.5844867770621174, + "tokens_seen": 1321664512 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048449348044132394, + "loss": 1.3693, + "theoretical_loss": 3.584470850805478, + "tokens_seen": 1321730048 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004844774322968907, + "loss": 1.2726, + "theoretical_loss": 3.584454925559597, + "tokens_seen": 1321795584 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048446138415245734, + "loss": 1.3556, + "theoretical_loss": 3.584439001324359, + "tokens_seen": 1321861120 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004844453360080241, + "loss": 1.3298, + "theoretical_loss": 3.58442307809965, + "tokens_seen": 1321926656 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048442928786359075, + "loss": 1.3846, + "theoretical_loss": 3.584407155885356, + "tokens_seen": 1321992192 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004844132397191575, + "loss": 1.2629, + "theoretical_loss": 3.5843912346813633, + "tokens_seen": 1322057728 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048439719157472415, + "loss": 1.3087, + "theoretical_loss": 3.5843753144875565, + "tokens_seen": 1322123264 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004843811434302909, + "loss": 1.2942, + "theoretical_loss": 3.5843593953038226, + "tokens_seen": 1322188800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048436509528585755, + "loss": 1.3796, + "theoretical_loss": 3.584343477130047, + "tokens_seen": 1322254336 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004843490471414243, + "loss": 1.2679, + "theoretical_loss": 3.5843275599661153, + "tokens_seen": 1322319872 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048433299899699095, + "loss": 1.4062, + "theoretical_loss": 3.5843116438119145, + "tokens_seen": 1322385408 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004843169508525577, + "loss": 1.2716, + "theoretical_loss": 3.5842957286673287, + "tokens_seen": 1322450944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048430090270812435, + "loss": 1.2989, + "theoretical_loss": 3.584279814532245, + "tokens_seen": 1322516480 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004842848545636911, + "loss": 1.3335, + "theoretical_loss": 3.5842639014065494, + "tokens_seen": 1322582016 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048426880641925775, + "loss": 1.2631, + "theoretical_loss": 3.5842479892901276, + "tokens_seen": 1322647552 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004842527582748245, + "loss": 1.2766, + "theoretical_loss": 3.5842320781828656, + "tokens_seen": 1322713088 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048423671013039115, + "loss": 1.3562, + "theoretical_loss": 3.5842161680846494, + "tokens_seen": 1322778624 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004842206619859579, + "loss": 1.3548, + "theoretical_loss": 3.5842002589953648, + "tokens_seen": 1322844160 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048420461384152455, + "loss": 1.2538, + "theoretical_loss": 3.5841843509148985, + "tokens_seen": 1322909696 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004841885656970913, + "loss": 1.2859, + "theoretical_loss": 3.5841684438431356, + "tokens_seen": 1322975232 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048417251755265796, + "loss": 1.3239, + "theoretical_loss": 3.5841525377799632, + "tokens_seen": 1323040768 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004841564694082247, + "loss": 1.3852, + "theoretical_loss": 3.584136632725267, + "tokens_seen": 1323106304 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048414042126379136, + "loss": 1.2987, + "theoretical_loss": 3.5841207286789327, + "tokens_seen": 1323171840 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004841243731193581, + "loss": 1.2242, + "theoretical_loss": 3.5841048256408463, + "tokens_seen": 1323237376 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048410832497492476, + "loss": 1.225, + "theoretical_loss": 3.584088923610895, + "tokens_seen": 1323302912 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004840922768304915, + "loss": 1.3383, + "theoretical_loss": 3.5840730225889645, + "tokens_seen": 1323368448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048407622868605827, + "loss": 1.3257, + "theoretical_loss": 3.58405712257494, + "tokens_seen": 1323433984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004840601805416249, + "loss": 1.3174, + "theoretical_loss": 3.5840412235687094, + "tokens_seen": 1323499520 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004840441323971916, + "loss": 1.398, + "theoretical_loss": 3.584025325570158, + "tokens_seen": 1323565056 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004840280842527583, + "loss": 1.2417, + "theoretical_loss": 3.5840094285791717, + "tokens_seen": 1323630592 + }, + { + "epoch": 0.4, + "learning_rate": 0.000484012036108325, + "loss": 1.4068, + "theoretical_loss": 3.5839935325956374, + "tokens_seen": 1323696128 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004839959879638917, + "loss": 1.4216, + "theoretical_loss": 3.5839776376194408, + "tokens_seen": 1323761664 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 444455, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3063066005706787, + "objective/train/theoretical_loss": 3.583961743650469, + "objective/train/tokens_used": 1344287200, + "theoretical_loss": 3.583961743650469, + "tokens_seen": 1323827200 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004839799398194584, + "loss": 1.3326, + "theoretical_loss": 3.583961743650469, + "tokens_seen": 1323827200 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004839638916750251, + "loss": 1.3388, + "theoretical_loss": 3.583945850688608, + "tokens_seen": 1323892736 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004839478435305918, + "loss": 1.3409, + "theoretical_loss": 3.5839299587337434, + "tokens_seen": 1323958272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004839317953861585, + "loss": 1.2974, + "theoretical_loss": 3.583914067785763, + "tokens_seen": 1324023808 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004839157472417252, + "loss": 1.2286, + "theoretical_loss": 3.583898177844552, + "tokens_seen": 1324089344 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048389969909729187, + "loss": 1.2968, + "theoretical_loss": 3.5838822889099973, + "tokens_seen": 1324154880 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004838836509528586, + "loss": 1.344, + "theoretical_loss": 3.5838664009819845, + "tokens_seen": 1324220416 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048386760280842527, + "loss": 1.2972, + "theoretical_loss": 3.5838505140604013, + "tokens_seen": 1324285952 + }, + { + "epoch": 0.4, + "learning_rate": 0.000483851554663992, + "loss": 1.3461, + "theoretical_loss": 3.583834628145133, + "tokens_seen": 1324351488 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048383550651955867, + "loss": 1.2023, + "theoretical_loss": 3.5838187432360673, + "tokens_seen": 1324417024 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004838194583751254, + "loss": 1.4498, + "theoretical_loss": 3.5838028593330895, + "tokens_seen": 1324482560 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048380341023069207, + "loss": 1.3108, + "theoretical_loss": 3.5837869764360866, + "tokens_seen": 1324548096 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004837873620862588, + "loss": 1.3089, + "theoretical_loss": 3.583771094544945, + "tokens_seen": 1324613632 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048377131394182547, + "loss": 1.3479, + "theoretical_loss": 3.5837552136595514, + "tokens_seen": 1324679168 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004837552657973922, + "loss": 1.3109, + "theoretical_loss": 3.583739333779792, + "tokens_seen": 1324744704 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048373921765295887, + "loss": 1.2479, + "theoretical_loss": 3.583723454905554, + "tokens_seen": 1324810240 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048372316950852563, + "loss": 1.2528, + "theoretical_loss": 3.5837075770367237, + "tokens_seen": 1324875776 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004837071213640923, + "loss": 1.2651, + "theoretical_loss": 3.5836917001731874, + "tokens_seen": 1324941312 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048369107321965903, + "loss": 1.3618, + "theoretical_loss": 3.583675824314832, + "tokens_seen": 1325006848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004836750250752257, + "loss": 1.3535, + "theoretical_loss": 3.5836599494615444, + "tokens_seen": 1325072384 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048365897693079243, + "loss": 1.3018, + "theoretical_loss": 3.583644075613211, + "tokens_seen": 1325137920 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004836429287863591, + "loss": 1.2062, + "theoretical_loss": 3.5836282027697184, + "tokens_seen": 1325203456 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048362688064192583, + "loss": 1.4019, + "theoretical_loss": 3.5836123309309533, + "tokens_seen": 1325268992 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004836108324974925, + "loss": 1.3232, + "theoretical_loss": 3.5835964600968024, + "tokens_seen": 1325334528 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048359478435305923, + "loss": 1.3022, + "theoretical_loss": 3.5835805902671525, + "tokens_seen": 1325400064 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004835787362086259, + "loss": 1.368, + "theoretical_loss": 3.583564721441891, + "tokens_seen": 1325465600 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048356268806419263, + "loss": 1.3103, + "theoretical_loss": 3.5835488536209033, + "tokens_seen": 1325531136 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004835466399197593, + "loss": 1.3369, + "theoretical_loss": 3.5835329868040775, + "tokens_seen": 1325596672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048353059177532604, + "loss": 1.3383, + "theoretical_loss": 3.5835171209913, + "tokens_seen": 1325662208 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004835145436308927, + "loss": 1.3852, + "theoretical_loss": 3.5835012561824575, + "tokens_seen": 1325727744 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048349849548645944, + "loss": 1.2991, + "theoretical_loss": 3.5834853923774364, + "tokens_seen": 1325793280 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004834824473420261, + "loss": 1.3479, + "theoretical_loss": 3.5834695295761243, + "tokens_seen": 1325858816 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048346639919759284, + "loss": 1.3692, + "theoretical_loss": 3.583453667778408, + "tokens_seen": 1325924352 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004834503510531595, + "loss": 1.3161, + "theoretical_loss": 3.583437806984174, + "tokens_seen": 1325989888 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048343430290872624, + "loss": 1.3371, + "theoretical_loss": 3.58342194719331, + "tokens_seen": 1326055424 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004834182547642929, + "loss": 1.3452, + "theoretical_loss": 3.5834060884057015, + "tokens_seen": 1326120960 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048340220661985964, + "loss": 1.4371, + "theoretical_loss": 3.5833902306212373, + "tokens_seen": 1326186496 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004833861584754263, + "loss": 1.4161, + "theoretical_loss": 3.5833743738398027, + "tokens_seen": 1326252032 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048337011033099304, + "loss": 1.2408, + "theoretical_loss": 3.583358518061286, + "tokens_seen": 1326317568 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004833540621865597, + "loss": 1.322, + "theoretical_loss": 3.5833426632855736, + "tokens_seen": 1326383104 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048333801404212644, + "loss": 1.2247, + "theoretical_loss": 3.5833268095125526, + "tokens_seen": 1326448640 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004833219658976931, + "loss": 1.3168, + "theoretical_loss": 3.5833109567421095, + "tokens_seen": 1326514176 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048330591775325984, + "loss": 1.3679, + "theoretical_loss": 3.583295104974132, + "tokens_seen": 1326579712 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004832898696088265, + "loss": 1.3412, + "theoretical_loss": 3.583279254208508, + "tokens_seen": 1326645248 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004832738214643932, + "loss": 1.3053, + "theoretical_loss": 3.5832634044451233, + "tokens_seen": 1326710784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004832577733199599, + "loss": 1.4155, + "theoretical_loss": 3.583247555683865, + "tokens_seen": 1326776320 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004832417251755266, + "loss": 1.2586, + "theoretical_loss": 3.583231707924621, + "tokens_seen": 1326841856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004832256770310933, + "loss": 1.3028, + "theoretical_loss": 3.5832158611672784, + "tokens_seen": 1326907392 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048320962888666, + "loss": 1.2988, + "theoretical_loss": 3.583200015411724, + "tokens_seen": 1326972928 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004831935807422267, + "loss": 1.3468, + "theoretical_loss": 3.5831841706578444, + "tokens_seen": 1327038464 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 445368, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4701875448226929, + "objective/train/theoretical_loss": 3.5831683269055286, + "objective/train/tokens_used": 1347564000, + "theoretical_loss": 3.5831683269055286, + "tokens_seen": 1327104000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004831775325977934, + "loss": 1.4034, + "theoretical_loss": 3.5831683269055286, + "tokens_seen": 1327104000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004831614844533601, + "loss": 1.2843, + "theoretical_loss": 3.5831524841546623, + "tokens_seen": 1327169536 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004831454363089268, + "loss": 1.2842, + "theoretical_loss": 3.583136642405133, + "tokens_seen": 1327235072 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048312938816449344, + "loss": 1.299, + "theoretical_loss": 3.5831208016568286, + "tokens_seen": 1327300608 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004831133400200602, + "loss": 1.3265, + "theoretical_loss": 3.583104961909636, + "tokens_seen": 1327366144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048309729187562684, + "loss": 1.2967, + "theoretical_loss": 3.5830891231634423, + "tokens_seen": 1327431680 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004830812437311936, + "loss": 1.3434, + "theoretical_loss": 3.583073285418135, + "tokens_seen": 1327497216 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048306519558676025, + "loss": 1.2553, + "theoretical_loss": 3.5830574486736015, + "tokens_seen": 1327562752 + }, + { + "epoch": 0.4, + "learning_rate": 0.000483049147442327, + "loss": 1.2941, + "theoretical_loss": 3.583041612929729, + "tokens_seen": 1327628288 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048303309929789365, + "loss": 1.3652, + "theoretical_loss": 3.583025778186405, + "tokens_seen": 1327693824 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004830170511534604, + "loss": 1.4218, + "theoretical_loss": 3.5830099444435173, + "tokens_seen": 1327759360 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048300100300902705, + "loss": 1.4198, + "theoretical_loss": 3.5829941117009527, + "tokens_seen": 1327824896 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004829849548645938, + "loss": 1.4086, + "theoretical_loss": 3.582978279958599, + "tokens_seen": 1327890432 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048296890672016045, + "loss": 1.2971, + "theoretical_loss": 3.5829624492163434, + "tokens_seen": 1327955968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004829528585757272, + "loss": 1.1543, + "theoretical_loss": 3.5829466194740736, + "tokens_seen": 1328021504 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048293681043129385, + "loss": 1.4362, + "theoretical_loss": 3.5829307907316768, + "tokens_seen": 1328087040 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004829207622868606, + "loss": 1.3065, + "theoretical_loss": 3.582914962989041, + "tokens_seen": 1328152576 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048290471414242725, + "loss": 1.3042, + "theoretical_loss": 3.582899136246053, + "tokens_seen": 1328218112 + }, + { + "epoch": 0.4, + "learning_rate": 0.000482888665997994, + "loss": 1.4558, + "theoretical_loss": 3.5828833105026012, + "tokens_seen": 1328283648 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048287261785356076, + "loss": 1.2149, + "theoretical_loss": 3.5828674857585723, + "tokens_seen": 1328349184 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004828565697091274, + "loss": 1.3571, + "theoretical_loss": 3.582851662013854, + "tokens_seen": 1328414720 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048284052156469416, + "loss": 1.2751, + "theoretical_loss": 3.582835839268335, + "tokens_seen": 1328480256 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004828244734202608, + "loss": 1.3507, + "theoretical_loss": 3.582820017521902, + "tokens_seen": 1328545792 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048280842527582756, + "loss": 1.3013, + "theoretical_loss": 3.5828041967744424, + "tokens_seen": 1328611328 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004827923771313942, + "loss": 1.2844, + "theoretical_loss": 3.5827883770258446, + "tokens_seen": 1328676864 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048277632898696097, + "loss": 1.3641, + "theoretical_loss": 3.5827725582759955, + "tokens_seen": 1328742400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004827602808425276, + "loss": 1.2678, + "theoretical_loss": 3.5827567405247835, + "tokens_seen": 1328807936 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048274423269809437, + "loss": 1.3406, + "theoretical_loss": 3.582740923772096, + "tokens_seen": 1328873472 + }, + { + "epoch": 0.4, + "learning_rate": 0.000482728184553661, + "loss": 1.3013, + "theoretical_loss": 3.5827251080178204, + "tokens_seen": 1328939008 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048271213640922777, + "loss": 1.2817, + "theoretical_loss": 3.582709293261845, + "tokens_seen": 1329004544 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004826960882647944, + "loss": 1.4249, + "theoretical_loss": 3.5826934795040573, + "tokens_seen": 1329070080 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048268004012036117, + "loss": 1.3481, + "theoretical_loss": 3.5826776667443454, + "tokens_seen": 1329135616 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004826639919759278, + "loss": 1.4349, + "theoretical_loss": 3.582661854982596, + "tokens_seen": 1329201152 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004826479438314945, + "loss": 1.3055, + "theoretical_loss": 3.5826460442186985, + "tokens_seen": 1329266688 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004826318956870612, + "loss": 1.3214, + "theoretical_loss": 3.582630234452539, + "tokens_seen": 1329332224 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004826158475426279, + "loss": 1.4142, + "theoretical_loss": 3.5826144256840076, + "tokens_seen": 1329397760 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004825997993981946, + "loss": 1.428, + "theoretical_loss": 3.58259861791299, + "tokens_seen": 1329463296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004825837512537613, + "loss": 1.2922, + "theoretical_loss": 3.582582811139375, + "tokens_seen": 1329528832 + }, + { + "epoch": 0.4, + "learning_rate": 0.000482567703109328, + "loss": 1.3145, + "theoretical_loss": 3.582567005363051, + "tokens_seen": 1329594368 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004825516549648947, + "loss": 1.3879, + "theoretical_loss": 3.582551200583905, + "tokens_seen": 1329659904 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004825356068204614, + "loss": 1.4077, + "theoretical_loss": 3.582535396801825, + "tokens_seen": 1329725440 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004825195586760281, + "loss": 1.2434, + "theoretical_loss": 3.5825195940167003, + "tokens_seen": 1329790976 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048250351053159477, + "loss": 1.3787, + "theoretical_loss": 3.582503792228417, + "tokens_seen": 1329856512 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004824874623871615, + "loss": 1.2914, + "theoretical_loss": 3.5824879914368646, + "tokens_seen": 1329922048 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048247141424272817, + "loss": 1.3221, + "theoretical_loss": 3.58247219164193, + "tokens_seen": 1329987584 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004824553660982949, + "loss": 1.3212, + "theoretical_loss": 3.5824563928435023, + "tokens_seen": 1330053120 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048243931795386157, + "loss": 1.3086, + "theoretical_loss": 3.5824405950414686, + "tokens_seen": 1330118656 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004824232698094283, + "loss": 1.32, + "theoretical_loss": 3.582424798235717, + "tokens_seen": 1330184192 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048240722166499497, + "loss": 1.3382, + "theoretical_loss": 3.582409002426137, + "tokens_seen": 1330249728 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004823911735205617, + "loss": 1.3155, + "theoretical_loss": 3.582393207612615, + "tokens_seen": 1330315264 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 446542, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2807539701461792, + "objective/train/theoretical_loss": 3.58237741379504, + "objective/train/tokens_used": 1350840800, + "theoretical_loss": 3.58237741379504, + "tokens_seen": 1330380800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048237512537612837, + "loss": 1.3185, + "theoretical_loss": 3.58237741379504, + "tokens_seen": 1330380800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048235907723169513, + "loss": 1.3071, + "theoretical_loss": 3.5823616209732996, + "tokens_seen": 1330446336 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004823430290872618, + "loss": 1.3834, + "theoretical_loss": 3.5823458291472825, + "tokens_seen": 1330511872 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048232698094282853, + "loss": 1.3389, + "theoretical_loss": 3.5823300383168766, + "tokens_seen": 1330577408 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004823109327983952, + "loss": 1.358, + "theoretical_loss": 3.5823142484819703, + "tokens_seen": 1330642944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048229488465396193, + "loss": 1.3405, + "theoretical_loss": 3.582298459642452, + "tokens_seen": 1330708480 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004822788365095286, + "loss": 1.2709, + "theoretical_loss": 3.5822826717982093, + "tokens_seen": 1330774016 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048226278836509533, + "loss": 1.2909, + "theoretical_loss": 3.5822668849491306, + "tokens_seen": 1330839552 + }, + { + "epoch": 0.4, + "learning_rate": 0.000482246740220662, + "loss": 1.3235, + "theoretical_loss": 3.5822510990951044, + "tokens_seen": 1330905088 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048223069207622873, + "loss": 1.3417, + "theoretical_loss": 3.5822353142360193, + "tokens_seen": 1330970624 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004822146439317954, + "loss": 1.3082, + "theoretical_loss": 3.5822195303717628, + "tokens_seen": 1331036160 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048219859578736213, + "loss": 1.2585, + "theoretical_loss": 3.5822037475022235, + "tokens_seen": 1331101696 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004821825476429288, + "loss": 1.336, + "theoretical_loss": 3.5821879656272904, + "tokens_seen": 1331167232 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048216649949849553, + "loss": 1.3647, + "theoretical_loss": 3.5821721847468515, + "tokens_seen": 1331232768 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004821504513540622, + "loss": 1.3162, + "theoretical_loss": 3.5821564048607946, + "tokens_seen": 1331298304 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048213440320962894, + "loss": 1.3587, + "theoretical_loss": 3.5821406259690085, + "tokens_seen": 1331363840 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004821183550651956, + "loss": 1.302, + "theoretical_loss": 3.5821248480713823, + "tokens_seen": 1331429376 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048210230692076234, + "loss": 1.288, + "theoretical_loss": 3.582109071167803, + "tokens_seen": 1331494912 + }, + { + "epoch": 0.4, + "learning_rate": 0.000482086258776329, + "loss": 1.3165, + "theoretical_loss": 3.58209329525816, + "tokens_seen": 1331560448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048207021063189574, + "loss": 1.3853, + "theoretical_loss": 3.5820775203423416, + "tokens_seen": 1331625984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004820541624874624, + "loss": 1.3913, + "theoretical_loss": 3.5820617464202367, + "tokens_seen": 1331691520 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048203811434302914, + "loss": 1.3406, + "theoretical_loss": 3.5820459734917334, + "tokens_seen": 1331757056 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004820220661985958, + "loss": 1.3425, + "theoretical_loss": 3.5820302015567194, + "tokens_seen": 1331822592 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048200601805416254, + "loss": 1.2499, + "theoretical_loss": 3.582014430615085, + "tokens_seen": 1331888128 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004819899699097292, + "loss": 1.4842, + "theoretical_loss": 3.5819986606667173, + "tokens_seen": 1331953664 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048197392176529594, + "loss": 1.3569, + "theoretical_loss": 3.5819828917115055, + "tokens_seen": 1332019200 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004819578736208626, + "loss": 1.3203, + "theoretical_loss": 3.581967123749338, + "tokens_seen": 1332084736 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048194182547642934, + "loss": 1.2886, + "theoretical_loss": 3.581951356780104, + "tokens_seen": 1332150272 + }, + { + "epoch": 0.4, + "learning_rate": 0.000481925777331996, + "loss": 1.2725, + "theoretical_loss": 3.581935590803691, + "tokens_seen": 1332215808 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048190972918756275, + "loss": 1.3201, + "theoretical_loss": 3.5819198258199885, + "tokens_seen": 1332281344 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004818936810431294, + "loss": 1.4101, + "theoretical_loss": 3.581904061828885, + "tokens_seen": 1332346880 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004818776328986961, + "loss": 1.2548, + "theoretical_loss": 3.581888298830269, + "tokens_seen": 1332412416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004818615847542628, + "loss": 1.2908, + "theoretical_loss": 3.5818725368240294, + "tokens_seen": 1332477952 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004818455366098295, + "loss": 1.4159, + "theoretical_loss": 3.5818567758100546, + "tokens_seen": 1332543488 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004818294884653962, + "loss": 1.422, + "theoretical_loss": 3.581841015788234, + "tokens_seen": 1332609024 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004818134403209629, + "loss": 1.2996, + "theoretical_loss": 3.5818252567584556, + "tokens_seen": 1332674560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004817973921765296, + "loss": 1.1327, + "theoretical_loss": 3.5818094987206086, + "tokens_seen": 1332740096 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004817813440320963, + "loss": 1.3226, + "theoretical_loss": 3.581793741674582, + "tokens_seen": 1332805632 + }, + { + "epoch": 0.4, + "learning_rate": 0.000481765295887663, + "loss": 1.3293, + "theoretical_loss": 3.5817779856202643, + "tokens_seen": 1332871168 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004817492477432297, + "loss": 1.3704, + "theoretical_loss": 3.581762230557544, + "tokens_seen": 1332936704 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048173319959879634, + "loss": 1.1409, + "theoretical_loss": 3.5817464764863107, + "tokens_seen": 1333002240 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004817171514543631, + "loss": 1.2467, + "theoretical_loss": 3.5817307234064524, + "tokens_seen": 1333067776 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048170110330992974, + "loss": 1.4078, + "theoretical_loss": 3.5817149713178584, + "tokens_seen": 1333133312 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004816850551654965, + "loss": 1.4079, + "theoretical_loss": 3.5816992202204183, + "tokens_seen": 1333198848 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048166900702106325, + "loss": 1.389, + "theoretical_loss": 3.5816834701140197, + "tokens_seen": 1333264384 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004816529588766299, + "loss": 1.2974, + "theoretical_loss": 3.5816677209985524, + "tokens_seen": 1333329920 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048163691073219666, + "loss": 1.3524, + "theoretical_loss": 3.581651972873905, + "tokens_seen": 1333395456 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004816208625877633, + "loss": 1.3615, + "theoretical_loss": 3.5816362257399668, + "tokens_seen": 1333460992 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048160481444333006, + "loss": 1.2211, + "theoretical_loss": 3.5816204795966264, + "tokens_seen": 1333526528 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004815887662988967, + "loss": 1.3693, + "theoretical_loss": 3.5816047344437734, + "tokens_seen": 1333592064 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 447424, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4249781370162964, + "objective/train/theoretical_loss": 3.581588990281296, + "objective/train/tokens_used": 1354117600, + "theoretical_loss": 3.581588990281296, + "tokens_seen": 1333657600 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048157271815446346, + "loss": 1.3495, + "theoretical_loss": 3.581588990281296, + "tokens_seen": 1333657600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004815566700100301, + "loss": 1.4024, + "theoretical_loss": 3.581573247109084, + "tokens_seen": 1333723136 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048154062186559686, + "loss": 1.2595, + "theoretical_loss": 3.581557504927026, + "tokens_seen": 1333788672 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004815245737211635, + "loss": 1.3404, + "theoretical_loss": 3.581541763735011, + "tokens_seen": 1333854208 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048150852557673026, + "loss": 1.4137, + "theoretical_loss": 3.5815260235329287, + "tokens_seen": 1333919744 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004814924774322969, + "loss": 1.3797, + "theoretical_loss": 3.5815102843206676, + "tokens_seen": 1333985280 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048147642928786366, + "loss": 1.1652, + "theoretical_loss": 3.581494546098117, + "tokens_seen": 1334050816 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004814603811434303, + "loss": 1.3512, + "theoretical_loss": 3.581478808865166, + "tokens_seen": 1334116352 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048144433299899706, + "loss": 1.3896, + "theoretical_loss": 3.5814630726217045, + "tokens_seen": 1334181888 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004814282848545637, + "loss": 1.4512, + "theoretical_loss": 3.5814473373676203, + "tokens_seen": 1334247424 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048141223671013046, + "loss": 1.362, + "theoretical_loss": 3.5814316031028035, + "tokens_seen": 1334312960 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004813961885656971, + "loss": 1.3385, + "theoretical_loss": 3.5814158698271434, + "tokens_seen": 1334378496 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048138014042126387, + "loss": 1.3252, + "theoretical_loss": 3.581400137540529, + "tokens_seen": 1334444032 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004813640922768305, + "loss": 1.3871, + "theoretical_loss": 3.58138440624285, + "tokens_seen": 1334509568 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048134804413239727, + "loss": 1.2668, + "theoretical_loss": 3.5813686759339944, + "tokens_seen": 1334575104 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004813319959879639, + "loss": 1.322, + "theoretical_loss": 3.581352946613853, + "tokens_seen": 1334640640 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048131594784353067, + "loss": 1.2272, + "theoretical_loss": 3.5813372182823136, + "tokens_seen": 1334706176 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004812998996990973, + "loss": 1.1931, + "theoretical_loss": 3.581321490939267, + "tokens_seen": 1334771712 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048128385155466407, + "loss": 1.3972, + "theoretical_loss": 3.5813057645846023, + "tokens_seen": 1334837248 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004812678034102307, + "loss": 1.2984, + "theoretical_loss": 3.581290039218208, + "tokens_seen": 1334902784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004812517552657974, + "loss": 1.3355, + "theoretical_loss": 3.581274314839974, + "tokens_seen": 1334968320 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004812357071213641, + "loss": 1.4606, + "theoretical_loss": 3.5812585914497896, + "tokens_seen": 1335033856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004812196589769308, + "loss": 1.4568, + "theoretical_loss": 3.5812428690475446, + "tokens_seen": 1335099392 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004812036108324975, + "loss": 1.2637, + "theoretical_loss": 3.5812271476331277, + "tokens_seen": 1335164928 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004811875626880642, + "loss": 1.2876, + "theoretical_loss": 3.581211427206429, + "tokens_seen": 1335230464 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004811715145436309, + "loss": 1.3924, + "theoretical_loss": 3.5811957077673373, + "tokens_seen": 1335296000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004811554663991976, + "loss": 1.3643, + "theoretical_loss": 3.5811799893157437, + "tokens_seen": 1335361536 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004811394182547643, + "loss": 1.3821, + "theoretical_loss": 3.5811642718515353, + "tokens_seen": 1335427072 + }, + { + "epoch": 0.4, + "learning_rate": 0.000481123370110331, + "loss": 1.2616, + "theoretical_loss": 3.5811485553746034, + "tokens_seen": 1335492608 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048110732196589767, + "loss": 1.2583, + "theoretical_loss": 3.5811328398848374, + "tokens_seen": 1335558144 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004810912738214644, + "loss": 1.3647, + "theoretical_loss": 3.581117125382126, + "tokens_seen": 1335623680 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048107522567703107, + "loss": 1.2947, + "theoretical_loss": 3.581101411866359, + "tokens_seen": 1335689216 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004810591775325978, + "loss": 1.2276, + "theoretical_loss": 3.581085699337427, + "tokens_seen": 1335754752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048104312938816447, + "loss": 1.4068, + "theoretical_loss": 3.5810699877952183, + "tokens_seen": 1335820288 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004810270812437312, + "loss": 1.2359, + "theoretical_loss": 3.5810542772396228, + "tokens_seen": 1335885824 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048101103309929787, + "loss": 1.3078, + "theoretical_loss": 3.5810385676705305, + "tokens_seen": 1335951360 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004809949849548646, + "loss": 1.3726, + "theoretical_loss": 3.581022859087832, + "tokens_seen": 1336016896 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004809789368104313, + "loss": 1.2807, + "theoretical_loss": 3.581007151491415, + "tokens_seen": 1336082432 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048096288866599803, + "loss": 1.3501, + "theoretical_loss": 3.58099144488117, + "tokens_seen": 1336147968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004809468405215647, + "loss": 1.4591, + "theoretical_loss": 3.5809757392569876, + "tokens_seen": 1336213504 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048093079237713143, + "loss": 1.2377, + "theoretical_loss": 3.580960034618757, + "tokens_seen": 1336279040 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004809147442326981, + "loss": 1.349, + "theoretical_loss": 3.580944330966367, + "tokens_seen": 1336344576 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048089869608826483, + "loss": 1.3822, + "theoretical_loss": 3.5809286282997084, + "tokens_seen": 1336410112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004808826479438315, + "loss": 1.3179, + "theoretical_loss": 3.580912926618671, + "tokens_seen": 1336475648 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048086659979939823, + "loss": 1.3979, + "theoretical_loss": 3.5808972259231444, + "tokens_seen": 1336541184 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004808505516549649, + "loss": 1.3848, + "theoretical_loss": 3.5808815262130183, + "tokens_seen": 1336606720 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048083450351053163, + "loss": 1.4087, + "theoretical_loss": 3.5808658274881826, + "tokens_seen": 1336672256 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004808184553660983, + "loss": 1.3429, + "theoretical_loss": 3.5808501297485273, + "tokens_seen": 1336737792 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048080240722166503, + "loss": 1.4033, + "theoretical_loss": 3.5808344329939423, + "tokens_seen": 1336803328 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004807863590772317, + "loss": 1.3355, + "theoretical_loss": 3.580818737224317, + "tokens_seen": 1336868864 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 448669, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3390278816223145, + "objective/train/theoretical_loss": 3.5808030424395416, + "objective/train/tokens_used": 1357394400, + "theoretical_loss": 3.5808030424395416, + "tokens_seen": 1336934400 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048077031093279844, + "loss": 1.3454, + "theoretical_loss": 3.5808030424395416, + "tokens_seen": 1336934400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004807542627883651, + "loss": 1.361, + "theoretical_loss": 3.5807873486395065, + "tokens_seen": 1336999936 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048073821464393184, + "loss": 1.3739, + "theoretical_loss": 3.580771655824101, + "tokens_seen": 1337065472 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004807221664994985, + "loss": 1.4102, + "theoretical_loss": 3.5807559639932154, + "tokens_seen": 1337131008 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048070611835506524, + "loss": 1.37, + "theoretical_loss": 3.5807402731467395, + "tokens_seen": 1337196544 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004806900702106319, + "loss": 1.338, + "theoretical_loss": 3.5807245832845638, + "tokens_seen": 1337262080 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048067402206619864, + "loss": 1.3267, + "theoretical_loss": 3.5807088944065777, + "tokens_seen": 1337327616 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004806579739217653, + "loss": 1.2232, + "theoretical_loss": 3.580693206512671, + "tokens_seen": 1337393152 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048064192577733204, + "loss": 1.3858, + "theoretical_loss": 3.580677519602735, + "tokens_seen": 1337458688 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004806258776328987, + "loss": 1.2465, + "theoretical_loss": 3.580661833676659, + "tokens_seen": 1337524224 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048060982948846544, + "loss": 1.2839, + "theoretical_loss": 3.5806461487343326, + "tokens_seen": 1337589760 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004805937813440321, + "loss": 1.2402, + "theoretical_loss": 3.5806304647756466, + "tokens_seen": 1337655296 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048057773319959884, + "loss": 1.3859, + "theoretical_loss": 3.5806147818004908, + "tokens_seen": 1337720832 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004805616850551655, + "loss": 1.4221, + "theoretical_loss": 3.5805990998087562, + "tokens_seen": 1337786368 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048054563691073224, + "loss": 1.2923, + "theoretical_loss": 3.5805834188003316, + "tokens_seen": 1337851904 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004805295887662989, + "loss": 1.3699, + "theoretical_loss": 3.580567738775108, + "tokens_seen": 1337917440 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048051354062186565, + "loss": 1.3643, + "theoretical_loss": 3.580552059732975, + "tokens_seen": 1337982976 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004804974924774323, + "loss": 1.4674, + "theoretical_loss": 3.5805363816738236, + "tokens_seen": 1338048512 + }, + { + "epoch": 0.41, + "learning_rate": 0.000480481444332999, + "loss": 1.2965, + "theoretical_loss": 3.580520704597544, + "tokens_seen": 1338114048 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048046539618856575, + "loss": 1.3388, + "theoretical_loss": 3.5805050285040254, + "tokens_seen": 1338179584 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004804493480441324, + "loss": 1.3883, + "theoretical_loss": 3.580489353393159, + "tokens_seen": 1338245120 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048043329989969915, + "loss": 1.2544, + "theoretical_loss": 3.580473679264835, + "tokens_seen": 1338310656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004804172517552658, + "loss": 1.2456, + "theoretical_loss": 3.5804580061189437, + "tokens_seen": 1338376192 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048040120361083255, + "loss": 1.268, + "theoretical_loss": 3.580442333955375, + "tokens_seen": 1338441728 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004803851554663992, + "loss": 1.4147, + "theoretical_loss": 3.5804266627740198, + "tokens_seen": 1338507264 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048036910732196595, + "loss": 1.2712, + "theoretical_loss": 3.580410992574768, + "tokens_seen": 1338572800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004803530591775326, + "loss": 1.2593, + "theoretical_loss": 3.5803953233575103, + "tokens_seen": 1338638336 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048033701103309935, + "loss": 1.2848, + "theoretical_loss": 3.5803796551221367, + "tokens_seen": 1338703872 + }, + { + "epoch": 0.41, + "learning_rate": 0.000480320962888666, + "loss": 1.3717, + "theoretical_loss": 3.5803639878685383, + "tokens_seen": 1338769408 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048030491474423275, + "loss": 1.304, + "theoretical_loss": 3.5803483215966048, + "tokens_seen": 1338834944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004802888665997994, + "loss": 1.3585, + "theoretical_loss": 3.580332656306227, + "tokens_seen": 1338900480 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048027281845536616, + "loss": 1.3024, + "theoretical_loss": 3.580316991997295, + "tokens_seen": 1338966016 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004802567703109328, + "loss": 1.3721, + "theoretical_loss": 3.5803013286697, + "tokens_seen": 1339031552 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048024072216649956, + "loss": 1.3501, + "theoretical_loss": 3.5802856663233316, + "tokens_seen": 1339097088 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004802246740220662, + "loss": 1.339, + "theoretical_loss": 3.580270004958081, + "tokens_seen": 1339162624 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048020862587763296, + "loss": 1.2659, + "theoretical_loss": 3.580254344573839, + "tokens_seen": 1339228160 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004801925777331996, + "loss": 1.3614, + "theoretical_loss": 3.580238685170495, + "tokens_seen": 1339293696 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048017652958876636, + "loss": 1.3372, + "theoretical_loss": 3.58022302674794, + "tokens_seen": 1339359232 + }, + { + "epoch": 0.41, + "learning_rate": 0.000480160481444333, + "loss": 1.3331, + "theoretical_loss": 3.5802073693060654, + "tokens_seen": 1339424768 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048014443329989976, + "loss": 1.3586, + "theoretical_loss": 3.580191712844761, + "tokens_seen": 1339490304 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004801283851554664, + "loss": 1.3113, + "theoretical_loss": 3.580176057363918, + "tokens_seen": 1339555840 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048011233701103316, + "loss": 1.307, + "theoretical_loss": 3.5801604028634264, + "tokens_seen": 1339621376 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004800962888665998, + "loss": 1.3671, + "theoretical_loss": 3.5801447493431766, + "tokens_seen": 1339686912 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048008024072216656, + "loss": 1.3435, + "theoretical_loss": 3.5801290968030606, + "tokens_seen": 1339752448 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004800641925777332, + "loss": 1.3388, + "theoretical_loss": 3.5801134452429677, + "tokens_seen": 1339817984 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048004814443329996, + "loss": 1.2955, + "theoretical_loss": 3.580097794662789, + "tokens_seen": 1339883520 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004800320962888666, + "loss": 1.3901, + "theoretical_loss": 3.5800821450624163, + "tokens_seen": 1339949056 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048001604814443337, + "loss": 1.3408, + "theoretical_loss": 3.580066496441739, + "tokens_seen": 1340014592 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048, + "loss": 1.4272, + "theoretical_loss": 3.5800508488006484, + "tokens_seen": 1340080128 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047998395185556677, + "loss": 1.3211, + "theoretical_loss": 3.580035202139035, + "tokens_seen": 1340145664 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 450060, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.155937910079956, + "objective/train/theoretical_loss": 3.58001955645679, + "objective/train/tokens_used": 1360671200, + "theoretical_loss": 3.58001955645679, + "tokens_seen": 1340211200 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004799679037111334, + "loss": 1.2487, + "theoretical_loss": 3.58001955645679, + "tokens_seen": 1340211200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047995185556670017, + "loss": 1.2357, + "theoretical_loss": 3.5800039117538036, + "tokens_seen": 1340276736 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004799358074222668, + "loss": 1.2733, + "theoretical_loss": 3.5799882680299673, + "tokens_seen": 1340342272 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047991975927783357, + "loss": 1.2845, + "theoretical_loss": 3.579972625285172, + "tokens_seen": 1340407808 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004799037111334002, + "loss": 1.3828, + "theoretical_loss": 3.579956983519308, + "tokens_seen": 1340473344 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047988766298896697, + "loss": 1.3095, + "theoretical_loss": 3.5799413427322664, + "tokens_seen": 1340538880 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004798716148445336, + "loss": 1.2779, + "theoretical_loss": 3.579925702923938, + "tokens_seen": 1340604416 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047985556670010037, + "loss": 1.3103, + "theoretical_loss": 3.5799100640942134, + "tokens_seen": 1340669952 + }, + { + "epoch": 0.41, + "learning_rate": 0.000479839518555667, + "loss": 1.2648, + "theoretical_loss": 3.5798944262429853, + "tokens_seen": 1340735488 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004798234704112337, + "loss": 1.2802, + "theoretical_loss": 3.579878789370142, + "tokens_seen": 1340801024 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004798074222668004, + "loss": 1.4822, + "theoretical_loss": 3.5798631534755767, + "tokens_seen": 1340866560 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004797913741223671, + "loss": 1.4289, + "theoretical_loss": 3.5798475185591787, + "tokens_seen": 1340932096 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004797753259779338, + "loss": 1.3829, + "theoretical_loss": 3.57983188462084, + "tokens_seen": 1340997632 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004797592778335005, + "loss": 1.4056, + "theoretical_loss": 3.5798162516604517, + "tokens_seen": 1341063168 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004797432296890672, + "loss": 1.3528, + "theoretical_loss": 3.579800619677905, + "tokens_seen": 1341128704 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004797271815446339, + "loss": 1.4509, + "theoretical_loss": 3.5797849886730897, + "tokens_seen": 1341194240 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047971113340020057, + "loss": 1.3058, + "theoretical_loss": 3.579769358645898, + "tokens_seen": 1341259776 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004796950852557673, + "loss": 1.4055, + "theoretical_loss": 3.5797537295962205, + "tokens_seen": 1341325312 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047967903711133397, + "loss": 1.2921, + "theoretical_loss": 3.579738101523949, + "tokens_seen": 1341390848 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004796629889669007, + "loss": 1.4404, + "theoretical_loss": 3.5797224744289737, + "tokens_seen": 1341456384 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047964694082246737, + "loss": 1.3229, + "theoretical_loss": 3.579706848311186, + "tokens_seen": 1341521920 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004796308926780341, + "loss": 1.308, + "theoretical_loss": 3.579691223170477, + "tokens_seen": 1341587456 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004796148445336008, + "loss": 1.2353, + "theoretical_loss": 3.579675599006739, + "tokens_seen": 1341652992 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047959879638916753, + "loss": 1.2569, + "theoretical_loss": 3.5796599758198617, + "tokens_seen": 1341718528 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004795827482447342, + "loss": 1.4666, + "theoretical_loss": 3.579644353609737, + "tokens_seen": 1341784064 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047956670010030093, + "loss": 1.2893, + "theoretical_loss": 3.579628732376256, + "tokens_seen": 1341849600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004795506519558676, + "loss": 1.3149, + "theoretical_loss": 3.57961311211931, + "tokens_seen": 1341915136 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047953460381143433, + "loss": 1.396, + "theoretical_loss": 3.5795974928387904, + "tokens_seen": 1341980672 + }, + { + "epoch": 0.41, + "learning_rate": 0.000479518555667001, + "loss": 1.3927, + "theoretical_loss": 3.579581874534588, + "tokens_seen": 1342046208 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047950250752256773, + "loss": 1.4129, + "theoretical_loss": 3.5795662572065945, + "tokens_seen": 1342111744 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004794864593781344, + "loss": 1.2728, + "theoretical_loss": 3.5795506408547015, + "tokens_seen": 1342177280 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047947041123370113, + "loss": 1.4039, + "theoretical_loss": 3.5795350254788, + "tokens_seen": 1342242816 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004794543630892678, + "loss": 1.4029, + "theoretical_loss": 3.579519411078781, + "tokens_seen": 1342308352 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047943831494483453, + "loss": 1.4798, + "theoretical_loss": 3.5795037976545365, + "tokens_seen": 1342373888 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004794222668004012, + "loss": 1.3412, + "theoretical_loss": 3.5794881852059577, + "tokens_seen": 1342439424 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047940621865596794, + "loss": 1.431, + "theoretical_loss": 3.5794725737329354, + "tokens_seen": 1342504960 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004793901705115346, + "loss": 1.2976, + "theoretical_loss": 3.579456963235362, + "tokens_seen": 1342570496 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047937412236710134, + "loss": 1.314, + "theoretical_loss": 3.579441353713128, + "tokens_seen": 1342636032 + }, + { + "epoch": 0.41, + "learning_rate": 0.000479358074222668, + "loss": 1.3727, + "theoretical_loss": 3.5794257451661258, + "tokens_seen": 1342701568 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047934202607823474, + "loss": 1.4277, + "theoretical_loss": 3.5794101375942464, + "tokens_seen": 1342767104 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004793259779338014, + "loss": 1.2167, + "theoretical_loss": 3.579394530997381, + "tokens_seen": 1342832640 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047930992978936814, + "loss": 1.2519, + "theoretical_loss": 3.579378925375422, + "tokens_seen": 1342898176 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004792938816449348, + "loss": 1.3776, + "theoretical_loss": 3.5793633207282594, + "tokens_seen": 1342963712 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047927783350050154, + "loss": 1.3619, + "theoretical_loss": 3.579347717055786, + "tokens_seen": 1343029248 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004792617853560683, + "loss": 1.3105, + "theoretical_loss": 3.5793321143578933, + "tokens_seen": 1343094784 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047924573721163494, + "loss": 1.3094, + "theoretical_loss": 3.5793165126344726, + "tokens_seen": 1343160320 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004792296890672017, + "loss": 1.2959, + "theoretical_loss": 3.579300911885415, + "tokens_seen": 1343225856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047921364092276834, + "loss": 1.3763, + "theoretical_loss": 3.5792853121106134, + "tokens_seen": 1343291392 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047919759277833504, + "loss": 1.3447, + "theoretical_loss": 3.5792697133099587, + "tokens_seen": 1343356928 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047918154463390174, + "loss": 1.3474, + "theoretical_loss": 3.5792541154833417, + "tokens_seen": 1343422464 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 450997, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3282132148742676, + "objective/train/theoretical_loss": 3.579238518630656, + "objective/train/tokens_used": 1363948000, + "theoretical_loss": 3.579238518630656, + "tokens_seen": 1343488000 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047916549648946845, + "loss": 1.2727, + "theoretical_loss": 3.579238518630656, + "tokens_seen": 1343488000 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047914944834503515, + "loss": 1.4423, + "theoretical_loss": 3.579222922751791, + "tokens_seen": 1343553536 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047913340020060185, + "loss": 1.3274, + "theoretical_loss": 3.5792073278466403, + "tokens_seen": 1343619072 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047911735205616855, + "loss": 1.2289, + "theoretical_loss": 3.5791917339150947, + "tokens_seen": 1343684608 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047910130391173525, + "loss": 1.3859, + "theoretical_loss": 3.579176140957046, + "tokens_seen": 1343750144 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047908525576730195, + "loss": 1.3617, + "theoretical_loss": 3.579160548972386, + "tokens_seen": 1343815680 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047906920762286865, + "loss": 1.292, + "theoretical_loss": 3.579144957961007, + "tokens_seen": 1343881216 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004790531594784353, + "loss": 1.3962, + "theoretical_loss": 3.5791293679228, + "tokens_seen": 1343946752 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047903711133400205, + "loss": 1.2153, + "theoretical_loss": 3.579113778857657, + "tokens_seen": 1344012288 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004790210631895687, + "loss": 1.347, + "theoretical_loss": 3.57909819076547, + "tokens_seen": 1344077824 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047900501504513545, + "loss": 1.3906, + "theoretical_loss": 3.579082603646131, + "tokens_seen": 1344143360 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004789889669007021, + "loss": 1.2948, + "theoretical_loss": 3.579067017499532, + "tokens_seen": 1344208896 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047897291875626885, + "loss": 1.475, + "theoretical_loss": 3.5790514323255636, + "tokens_seen": 1344274432 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004789568706118355, + "loss": 1.258, + "theoretical_loss": 3.579035848124119, + "tokens_seen": 1344339968 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047894082246740225, + "loss": 1.3343, + "theoretical_loss": 3.5790202648950897, + "tokens_seen": 1344405504 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004789247743229689, + "loss": 1.4002, + "theoretical_loss": 3.579004682638368, + "tokens_seen": 1344471040 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047890872617853566, + "loss": 1.295, + "theoretical_loss": 3.5789891013538453, + "tokens_seen": 1344536576 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004788926780341023, + "loss": 1.3492, + "theoretical_loss": 3.5789735210414135, + "tokens_seen": 1344602112 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047887662988966906, + "loss": 1.371, + "theoretical_loss": 3.578957941700965, + "tokens_seen": 1344667648 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004788605817452357, + "loss": 1.274, + "theoretical_loss": 3.578942363332392, + "tokens_seen": 1344733184 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047884453360080246, + "loss": 1.3448, + "theoretical_loss": 3.5789267859355856, + "tokens_seen": 1344798720 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004788284854563691, + "loss": 1.3471, + "theoretical_loss": 3.5789112095104385, + "tokens_seen": 1344864256 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047881243731193586, + "loss": 1.3998, + "theoretical_loss": 3.5788956340568427, + "tokens_seen": 1344929792 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004787963891675025, + "loss": 1.3067, + "theoretical_loss": 3.57888005957469, + "tokens_seen": 1344995328 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047878034102306926, + "loss": 1.3002, + "theoretical_loss": 3.578864486063873, + "tokens_seen": 1345060864 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004787642928786359, + "loss": 1.3365, + "theoretical_loss": 3.578848913524283, + "tokens_seen": 1345126400 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047874824473420266, + "loss": 1.3591, + "theoretical_loss": 3.578833341955813, + "tokens_seen": 1345191936 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004787321965897693, + "loss": 1.3407, + "theoretical_loss": 3.5788177713583544, + "tokens_seen": 1345257472 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047871614844533606, + "loss": 1.3225, + "theoretical_loss": 3.5788022017317997, + "tokens_seen": 1345323008 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004787001003009027, + "loss": 1.4138, + "theoretical_loss": 3.5787866330760414, + "tokens_seen": 1345388544 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047868405215646946, + "loss": 1.3204, + "theoretical_loss": 3.5787710653909706, + "tokens_seen": 1345454080 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004786680040120361, + "loss": 1.3798, + "theoretical_loss": 3.5787554986764807, + "tokens_seen": 1345519616 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047865195586760287, + "loss": 1.3444, + "theoretical_loss": 3.5787399329324634, + "tokens_seen": 1345585152 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004786359077231695, + "loss": 1.3323, + "theoretical_loss": 3.5787243681588112, + "tokens_seen": 1345650688 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047861985957873627, + "loss": 1.3313, + "theoretical_loss": 3.5787088043554154, + "tokens_seen": 1345716224 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004786038114343029, + "loss": 1.325, + "theoretical_loss": 3.5786932415221697, + "tokens_seen": 1345781760 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047858776328986967, + "loss": 1.3466, + "theoretical_loss": 3.578677679658965, + "tokens_seen": 1345847296 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004785717151454363, + "loss": 1.3353, + "theoretical_loss": 3.5786621187656946, + "tokens_seen": 1345912832 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047855566700100307, + "loss": 1.3241, + "theoretical_loss": 3.5786465588422507, + "tokens_seen": 1345978368 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004785396188565697, + "loss": 1.299, + "theoretical_loss": 3.5786309998885253, + "tokens_seen": 1346043904 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047852357071213647, + "loss": 1.3289, + "theoretical_loss": 3.578615441904411, + "tokens_seen": 1346109440 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004785075225677031, + "loss": 1.2738, + "theoretical_loss": 3.5785998848897997, + "tokens_seen": 1346174976 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047849147442326987, + "loss": 1.3434, + "theoretical_loss": 3.5785843288445847, + "tokens_seen": 1346240512 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004784754262788365, + "loss": 1.3577, + "theoretical_loss": 3.5785687737686573, + "tokens_seen": 1346306048 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004784593781344033, + "loss": 1.389, + "theoretical_loss": 3.5785532196619108, + "tokens_seen": 1346371584 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004784593781344033, + "loss": 1.3129, + "theoretical_loss": 3.5785376665242374, + "tokens_seen": 1346437120 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004784433299899699, + "loss": 1.3693, + "theoretical_loss": 3.578522114355529, + "tokens_seen": 1346502656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004784272818455366, + "loss": 1.2745, + "theoretical_loss": 3.5785065631556794, + "tokens_seen": 1346568192 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004784112337011033, + "loss": 1.2882, + "theoretical_loss": 3.5784910129245793, + "tokens_seen": 1346633728 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047839518555667, + "loss": 1.3367, + "theoretical_loss": 3.578475463662123, + "tokens_seen": 1346699264 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 452300, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2804325819015503, + "objective/train/theoretical_loss": 3.5784599153682017, + "objective/train/tokens_used": 1367224800, + "theoretical_loss": 3.5784599153682017, + "tokens_seen": 1346764800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004783791374122367, + "loss": 1.2656, + "theoretical_loss": 3.5784599153682017, + "tokens_seen": 1346764800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004783630892678034, + "loss": 1.2759, + "theoretical_loss": 3.578444368042709, + "tokens_seen": 1346830336 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004783470411233701, + "loss": 1.4242, + "theoretical_loss": 3.578428821685536, + "tokens_seen": 1346895872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004783309929789368, + "loss": 1.2695, + "theoretical_loss": 3.578413276296577, + "tokens_seen": 1346961408 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004783149448345035, + "loss": 1.3269, + "theoretical_loss": 3.5783977318757234, + "tokens_seen": 1347026944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004782988966900702, + "loss": 1.1833, + "theoretical_loss": 3.578382188422869, + "tokens_seen": 1347092480 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047828284854563687, + "loss": 1.3813, + "theoretical_loss": 3.578366645937905, + "tokens_seen": 1347158016 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004782668004012036, + "loss": 1.4579, + "theoretical_loss": 3.578351104420724, + "tokens_seen": 1347223552 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004782507522567703, + "loss": 1.2303, + "theoretical_loss": 3.57833556387122, + "tokens_seen": 1347289088 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047823470411233703, + "loss": 1.3805, + "theoretical_loss": 3.5783200242892854, + "tokens_seen": 1347354624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004782186559679037, + "loss": 1.3677, + "theoretical_loss": 3.578304485674812, + "tokens_seen": 1347420160 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047820260782347043, + "loss": 1.313, + "theoretical_loss": 3.5782889480276934, + "tokens_seen": 1347485696 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004781865596790371, + "loss": 1.3977, + "theoretical_loss": 3.578273411347822, + "tokens_seen": 1347551232 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047817051153460383, + "loss": 1.3989, + "theoretical_loss": 3.57825787563509, + "tokens_seen": 1347616768 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004781544633901705, + "loss": 1.383, + "theoretical_loss": 3.578242340889391, + "tokens_seen": 1347682304 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047813841524573723, + "loss": 1.3729, + "theoretical_loss": 3.5782268071106174, + "tokens_seen": 1347747840 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004781223671013039, + "loss": 1.2073, + "theoretical_loss": 3.5782112742986625, + "tokens_seen": 1347813376 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047810631895687063, + "loss": 1.328, + "theoretical_loss": 3.5781957424534183, + "tokens_seen": 1347878912 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004780902708124373, + "loss": 1.3598, + "theoretical_loss": 3.578180211574778, + "tokens_seen": 1347944448 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047807422266800403, + "loss": 1.3714, + "theoretical_loss": 3.578164681662635, + "tokens_seen": 1348009984 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004780581745235708, + "loss": 1.3467, + "theoretical_loss": 3.578149152716881, + "tokens_seen": 1348075520 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047804212637913744, + "loss": 1.4247, + "theoretical_loss": 3.57813362473741, + "tokens_seen": 1348141056 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004780260782347042, + "loss": 1.3423, + "theoretical_loss": 3.578118097724114, + "tokens_seen": 1348206592 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047801003009027084, + "loss": 1.3935, + "theoretical_loss": 3.578102571676887, + "tokens_seen": 1348272128 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004779939819458376, + "loss": 1.3446, + "theoretical_loss": 3.578087046595621, + "tokens_seen": 1348337664 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047797793380140424, + "loss": 1.3589, + "theoretical_loss": 3.5780715224802093, + "tokens_seen": 1348403200 + }, + { + "epoch": 0.41, + "learning_rate": 0.000477961885656971, + "loss": 1.2795, + "theoretical_loss": 3.5780559993305445, + "tokens_seen": 1348468736 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047794583751253764, + "loss": 1.2751, + "theoretical_loss": 3.5780404771465206, + "tokens_seen": 1348534272 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004779297893681044, + "loss": 1.3591, + "theoretical_loss": 3.5780249559280297, + "tokens_seen": 1348599808 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047791374122367104, + "loss": 1.2809, + "theoretical_loss": 3.578009435674965, + "tokens_seen": 1348665344 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004778976930792378, + "loss": 1.3244, + "theoretical_loss": 3.5779939163872196, + "tokens_seen": 1348730880 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047788164493480444, + "loss": 1.3271, + "theoretical_loss": 3.577978398064687, + "tokens_seen": 1348796416 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004778655967903712, + "loss": 1.3408, + "theoretical_loss": 3.577962880707259, + "tokens_seen": 1348861952 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047784954864593784, + "loss": 1.3476, + "theoretical_loss": 3.57794736431483, + "tokens_seen": 1348927488 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004778335005015046, + "loss": 1.3463, + "theoretical_loss": 3.5779318488872924, + "tokens_seen": 1348993024 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047781745235707124, + "loss": 1.2752, + "theoretical_loss": 3.57791633442454, + "tokens_seen": 1349058560 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047780140421263794, + "loss": 1.2638, + "theoretical_loss": 3.577900820926465, + "tokens_seen": 1349124096 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047778535606820465, + "loss": 1.3608, + "theoretical_loss": 3.577885308392962, + "tokens_seen": 1349189632 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047776930792377135, + "loss": 1.243, + "theoretical_loss": 3.577869796823922, + "tokens_seen": 1349255168 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047775325977933805, + "loss": 1.3345, + "theoretical_loss": 3.5778542862192406, + "tokens_seen": 1349320704 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047773721163490475, + "loss": 1.3975, + "theoretical_loss": 3.577838776578809, + "tokens_seen": 1349386240 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047772116349047145, + "loss": 1.373, + "theoretical_loss": 3.577823267902522, + "tokens_seen": 1349451776 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047770511534603815, + "loss": 1.2175, + "theoretical_loss": 3.5778077601902716, + "tokens_seen": 1349517312 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047768906720160485, + "loss": 1.2608, + "theoretical_loss": 3.577792253441952, + "tokens_seen": 1349582848 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047767301905717155, + "loss": 1.2634, + "theoretical_loss": 3.577776747657456, + "tokens_seen": 1349648384 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004776569709127382, + "loss": 1.3247, + "theoretical_loss": 3.5777612428366767, + "tokens_seen": 1349713920 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047764092276830495, + "loss": 1.2539, + "theoretical_loss": 3.577745738979508, + "tokens_seen": 1349779456 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004776248746238716, + "loss": 1.2019, + "theoretical_loss": 3.5777302360858423, + "tokens_seen": 1349844992 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047760882647943835, + "loss": 1.2937, + "theoretical_loss": 3.577714734155574, + "tokens_seen": 1349910528 + }, + { + "epoch": 0.41, + "learning_rate": 0.000477592778335005, + "loss": 1.3451, + "theoretical_loss": 3.5776992331885964, + "tokens_seen": 1349976064 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 453237, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.378117322921753, + "objective/train/theoretical_loss": 3.577683733184802, + "objective/train/tokens_used": 1370501600, + "theoretical_loss": 3.577683733184802, + "tokens_seen": 1350041600 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047757673019057175, + "loss": 1.343, + "theoretical_loss": 3.577683733184802, + "tokens_seen": 1350041600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004775606820461384, + "loss": 1.2933, + "theoretical_loss": 3.577668234144085, + "tokens_seen": 1350107136 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047754463390170516, + "loss": 1.3646, + "theoretical_loss": 3.577652736066338, + "tokens_seen": 1350172672 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004775285857572718, + "loss": 1.2913, + "theoretical_loss": 3.5776372389514557, + "tokens_seen": 1350238208 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047751253761283856, + "loss": 1.2864, + "theoretical_loss": 3.57762174279933, + "tokens_seen": 1350303744 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004774964894684052, + "loss": 1.2866, + "theoretical_loss": 3.5776062476098565, + "tokens_seen": 1350369280 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047748044132397196, + "loss": 1.3274, + "theoretical_loss": 3.577590753382926, + "tokens_seen": 1350434816 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004774643931795386, + "loss": 1.3718, + "theoretical_loss": 3.577575260118434, + "tokens_seen": 1350500352 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047744834503510536, + "loss": 1.1962, + "theoretical_loss": 3.577559767816273, + "tokens_seen": 1350565888 + }, + { + "epoch": 0.41, + "learning_rate": 0.000477432296890672, + "loss": 1.3703, + "theoretical_loss": 3.5775442764763374, + "tokens_seen": 1350631424 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047741624874623876, + "loss": 1.377, + "theoretical_loss": 3.5775287860985197, + "tokens_seen": 1350696960 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004774002006018054, + "loss": 1.2782, + "theoretical_loss": 3.5775132966827146, + "tokens_seen": 1350762496 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047738415245737216, + "loss": 1.3029, + "theoretical_loss": 3.5774978082288147, + "tokens_seen": 1350828032 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004773681043129388, + "loss": 1.4288, + "theoretical_loss": 3.5774823207367143, + "tokens_seen": 1350893568 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047735205616850556, + "loss": 1.247, + "theoretical_loss": 3.5774668342063065, + "tokens_seen": 1350959104 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004773360080240722, + "loss": 1.2179, + "theoretical_loss": 3.5774513486374855, + "tokens_seen": 1351024640 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047731995987963896, + "loss": 1.3451, + "theoretical_loss": 3.5774358640301447, + "tokens_seen": 1351090176 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004773039117352056, + "loss": 1.3644, + "theoretical_loss": 3.577420380384177, + "tokens_seen": 1351155712 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047728786359077237, + "loss": 1.3545, + "theoretical_loss": 3.577404897699477, + "tokens_seen": 1351221248 + }, + { + "epoch": 0.41, + "learning_rate": 0.000477271815446339, + "loss": 1.3961, + "theoretical_loss": 3.5773894159759383, + "tokens_seen": 1351286784 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047725576730190577, + "loss": 1.3106, + "theoretical_loss": 3.577373935213455, + "tokens_seen": 1351352320 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004772397191574724, + "loss": 1.3005, + "theoretical_loss": 3.57735845541192, + "tokens_seen": 1351417856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047722367101303917, + "loss": 1.3862, + "theoretical_loss": 3.5773429765712272, + "tokens_seen": 1351483392 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004772076228686058, + "loss": 1.3219, + "theoretical_loss": 3.5773274986912704, + "tokens_seen": 1351548928 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047719157472417257, + "loss": 1.3756, + "theoretical_loss": 3.577312021771944, + "tokens_seen": 1351614464 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004771755265797392, + "loss": 1.3361, + "theoretical_loss": 3.577296545813141, + "tokens_seen": 1351680000 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047715947843530597, + "loss": 1.2426, + "theoretical_loss": 3.577281070814756, + "tokens_seen": 1351745536 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004771434302908726, + "loss": 1.349, + "theoretical_loss": 3.577265596776682, + "tokens_seen": 1351811072 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047712738214643937, + "loss": 1.3574, + "theoretical_loss": 3.5772501236988132, + "tokens_seen": 1351876608 + }, + { + "epoch": 0.41, + "learning_rate": 0.000477111334002006, + "loss": 1.3271, + "theoretical_loss": 3.577234651581044, + "tokens_seen": 1351942144 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047709528585757277, + "loss": 1.3236, + "theoretical_loss": 3.577219180423268, + "tokens_seen": 1352007680 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004770792377131394, + "loss": 1.2752, + "theoretical_loss": 3.577203710225378, + "tokens_seen": 1352073216 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004770631895687062, + "loss": 1.2766, + "theoretical_loss": 3.5771882409872697, + "tokens_seen": 1352138752 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004770471414242728, + "loss": 1.3151, + "theoretical_loss": 3.5771727727088356, + "tokens_seen": 1352204288 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004770310932798395, + "loss": 1.3158, + "theoretical_loss": 3.577157305389971, + "tokens_seen": 1352269824 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004770150451354062, + "loss": 1.3837, + "theoretical_loss": 3.5771418390305687, + "tokens_seen": 1352335360 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004769989969909729, + "loss": 1.3076, + "theoretical_loss": 3.577126373630523, + "tokens_seen": 1352400896 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004769829488465396, + "loss": 1.3721, + "theoretical_loss": 3.577110909189728, + "tokens_seen": 1352466432 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004769669007021063, + "loss": 1.3973, + "theoretical_loss": 3.5770954457080784, + "tokens_seen": 1352531968 + }, + { + "epoch": 0.41, + "learning_rate": 0.000476950852557673, + "loss": 1.3649, + "theoretical_loss": 3.577079983185467, + "tokens_seen": 1352597504 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004769348044132397, + "loss": 1.3097, + "theoretical_loss": 3.577064521621789, + "tokens_seen": 1352663040 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004769187562688064, + "loss": 1.3028, + "theoretical_loss": 3.577049061016937, + "tokens_seen": 1352728576 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004769027081243731, + "loss": 1.3148, + "theoretical_loss": 3.5770336013708066, + "tokens_seen": 1352794112 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047688665997993977, + "loss": 1.337, + "theoretical_loss": 3.5770181426832917, + "tokens_seen": 1352859648 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047687061183550653, + "loss": 1.2915, + "theoretical_loss": 3.5770026849542855, + "tokens_seen": 1352925184 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004768545636910733, + "loss": 1.2745, + "theoretical_loss": 3.576987228183683, + "tokens_seen": 1352990720 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047683851554663993, + "loss": 1.263, + "theoretical_loss": 3.576971772371378, + "tokens_seen": 1353056256 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004768224674022067, + "loss": 1.3746, + "theoretical_loss": 3.576956317517264, + "tokens_seen": 1353121792 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047680641925777333, + "loss": 1.3158, + "theoretical_loss": 3.576940863621237, + "tokens_seen": 1353187328 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004767903711133401, + "loss": 1.2643, + "theoretical_loss": 3.57692541068319, + "tokens_seen": 1353252864 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 454374, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4121934175491333, + "objective/train/theoretical_loss": 3.5769099587030166, + "objective/train/tokens_used": 1373778400, + "theoretical_loss": 3.5769099587030166, + "tokens_seen": 1353318400 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047677432296890673, + "loss": 1.3257, + "theoretical_loss": 3.5769099587030166, + "tokens_seen": 1353318400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004767582748244735, + "loss": 1.2999, + "theoretical_loss": 3.5768945076806125, + "tokens_seen": 1353383936 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047674222668004013, + "loss": 1.317, + "theoretical_loss": 3.576879057615871, + "tokens_seen": 1353449472 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004767261785356069, + "loss": 1.3362, + "theoretical_loss": 3.5768636085086865, + "tokens_seen": 1353515008 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047671013039117353, + "loss": 1.3674, + "theoretical_loss": 3.5768481603589537, + "tokens_seen": 1353580544 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004766940822467403, + "loss": 1.3929, + "theoretical_loss": 3.5768327131665667, + "tokens_seen": 1353646080 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047667803410230694, + "loss": 1.3458, + "theoretical_loss": 3.5768172669314193, + "tokens_seen": 1353711616 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004766619859578737, + "loss": 1.3494, + "theoretical_loss": 3.5768018216534068, + "tokens_seen": 1353777152 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047664593781344034, + "loss": 1.3724, + "theoretical_loss": 3.5767863773324233, + "tokens_seen": 1353842688 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004766298896690071, + "loss": 1.3222, + "theoretical_loss": 3.576770933968362, + "tokens_seen": 1353908224 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047661384152457374, + "loss": 1.4499, + "theoretical_loss": 3.5767554915611193, + "tokens_seen": 1353973760 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004765977933801405, + "loss": 1.3342, + "theoretical_loss": 3.5767400501105877, + "tokens_seen": 1354039296 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047658174523570714, + "loss": 1.3485, + "theoretical_loss": 3.5767246096166634, + "tokens_seen": 1354104832 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004765656970912739, + "loss": 1.2786, + "theoretical_loss": 3.576709170079239, + "tokens_seen": 1354170368 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047654964894684054, + "loss": 1.3495, + "theoretical_loss": 3.57669373149821, + "tokens_seen": 1354235904 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004765336008024073, + "loss": 1.3129, + "theoretical_loss": 3.576678293873471, + "tokens_seen": 1354301440 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047651755265797394, + "loss": 1.3542, + "theoretical_loss": 3.5766628572049166, + "tokens_seen": 1354366976 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004765015045135407, + "loss": 1.3111, + "theoretical_loss": 3.5766474214924404, + "tokens_seen": 1354432512 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047648545636910734, + "loss": 1.3236, + "theoretical_loss": 3.5766319867359377, + "tokens_seen": 1354498048 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004764694082246741, + "loss": 1.3899, + "theoretical_loss": 3.5766165529353033, + "tokens_seen": 1354563584 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047645336008024074, + "loss": 1.308, + "theoretical_loss": 3.5766011200904306, + "tokens_seen": 1354629120 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004764373119358075, + "loss": 1.41, + "theoretical_loss": 3.5765856882012153, + "tokens_seen": 1354694656 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047642126379137415, + "loss": 1.3213, + "theoretical_loss": 3.5765702572675515, + "tokens_seen": 1354760192 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047640521564694085, + "loss": 1.3043, + "theoretical_loss": 3.5765548272893337, + "tokens_seen": 1354825728 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047638916750250755, + "loss": 1.3058, + "theoretical_loss": 3.5765393982664566, + "tokens_seen": 1354891264 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047637311935807425, + "loss": 1.2932, + "theoretical_loss": 3.5765239701988154, + "tokens_seen": 1354956800 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047635707121364095, + "loss": 1.3814, + "theoretical_loss": 3.5765085430863044, + "tokens_seen": 1355022336 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047634102306920765, + "loss": 1.3512, + "theoretical_loss": 3.576493116928818, + "tokens_seen": 1355087872 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047632497492477435, + "loss": 1.3128, + "theoretical_loss": 3.576477691726251, + "tokens_seen": 1355153408 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047630892678034105, + "loss": 1.3136, + "theoretical_loss": 3.5764622674784983, + "tokens_seen": 1355218944 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047629287863590775, + "loss": 1.4381, + "theoretical_loss": 3.576446844185454, + "tokens_seen": 1355284480 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047627683049147445, + "loss": 1.2857, + "theoretical_loss": 3.576431421847014, + "tokens_seen": 1355350016 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004762607823470411, + "loss": 1.3775, + "theoretical_loss": 3.5764160004630723, + "tokens_seen": 1355415552 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047624473420260785, + "loss": 1.2717, + "theoretical_loss": 3.576400580033524, + "tokens_seen": 1355481088 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004762286860581745, + "loss": 1.2164, + "theoretical_loss": 3.5763851605582637, + "tokens_seen": 1355546624 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047621263791374125, + "loss": 1.3729, + "theoretical_loss": 3.576369742037186, + "tokens_seen": 1355612160 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004761965897693079, + "loss": 1.3306, + "theoretical_loss": 3.576354324470186, + "tokens_seen": 1355677696 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047618054162487465, + "loss": 1.2721, + "theoretical_loss": 3.5763389078571586, + "tokens_seen": 1355743232 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004761644934804413, + "loss": 1.3621, + "theoretical_loss": 3.5763234921979983, + "tokens_seen": 1355808768 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047614844533600806, + "loss": 1.3879, + "theoretical_loss": 3.576308077492601, + "tokens_seen": 1355874304 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004761323971915747, + "loss": 1.3844, + "theoretical_loss": 3.57629266374086, + "tokens_seen": 1355939840 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047611634904714146, + "loss": 1.2722, + "theoretical_loss": 3.5762772509426712, + "tokens_seen": 1356005376 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004761003009027081, + "loss": 1.3234, + "theoretical_loss": 3.5762618390979295, + "tokens_seen": 1356070912 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047608425275827486, + "loss": 1.3334, + "theoretical_loss": 3.57624642820653, + "tokens_seen": 1356136448 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004760682046138415, + "loss": 1.3057, + "theoretical_loss": 3.576231018268367, + "tokens_seen": 1356201984 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047605215646940826, + "loss": 1.3257, + "theoretical_loss": 3.576215609283336, + "tokens_seen": 1356267520 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004760361083249749, + "loss": 1.366, + "theoretical_loss": 3.576200201251332, + "tokens_seen": 1356333056 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047602006018054166, + "loss": 1.3436, + "theoretical_loss": 3.5761847941722493, + "tokens_seen": 1356398592 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004760040120361083, + "loss": 1.3706, + "theoretical_loss": 3.576169388045984, + "tokens_seen": 1356464128 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047598796389167506, + "loss": 1.3702, + "theoretical_loss": 3.576153982872431, + "tokens_seen": 1356529664 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 455288, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3363687992095947, + "objective/train/theoretical_loss": 3.5761385786514843, + "objective/train/tokens_used": 1377055200, + "theoretical_loss": 3.5761385786514843, + "tokens_seen": 1356595200 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004759719157472417, + "loss": 1.3978, + "theoretical_loss": 3.5761385786514843, + "tokens_seen": 1356595200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047595586760280846, + "loss": 1.323, + "theoretical_loss": 3.5761231753830405, + "tokens_seen": 1356660736 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004759398194583751, + "loss": 1.32, + "theoretical_loss": 3.576107773066993, + "tokens_seen": 1356726272 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047592377131394186, + "loss": 1.3252, + "theoretical_loss": 3.576092371703238, + "tokens_seen": 1356791808 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004759077231695085, + "loss": 1.2315, + "theoretical_loss": 3.5760769712916707, + "tokens_seen": 1356857344 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047589167502507527, + "loss": 1.2437, + "theoretical_loss": 3.576061571832186, + "tokens_seen": 1356922880 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004758756268806419, + "loss": 1.3991, + "theoretical_loss": 3.576046173324679, + "tokens_seen": 1356988416 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047585957873620867, + "loss": 1.3663, + "theoretical_loss": 3.576030775769045, + "tokens_seen": 1357053952 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004758435305917753, + "loss": 1.4817, + "theoretical_loss": 3.5760153791651783, + "tokens_seen": 1357119488 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047582748244734207, + "loss": 1.2868, + "theoretical_loss": 3.5759999835129754, + "tokens_seen": 1357185024 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004758114343029087, + "loss": 1.2257, + "theoretical_loss": 3.5759845888123314, + "tokens_seen": 1357250560 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047579538615847547, + "loss": 1.2162, + "theoretical_loss": 3.575969195063141, + "tokens_seen": 1357316096 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004757793380140421, + "loss": 1.3297, + "theoretical_loss": 3.5759538022652997, + "tokens_seen": 1357381632 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047576328986960887, + "loss": 1.4411, + "theoretical_loss": 3.5759384104187024, + "tokens_seen": 1357447168 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004757472417251755, + "loss": 1.2751, + "theoretical_loss": 3.575923019523245, + "tokens_seen": 1357512704 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047573119358074227, + "loss": 1.2694, + "theoretical_loss": 3.5759076295788224, + "tokens_seen": 1357578240 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004757151454363089, + "loss": 1.3627, + "theoretical_loss": 3.5758922405853304, + "tokens_seen": 1357643776 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004756990972918757, + "loss": 1.3891, + "theoretical_loss": 3.5758768525426636, + "tokens_seen": 1357709312 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004756830491474423, + "loss": 1.3531, + "theoretical_loss": 3.5758614654507177, + "tokens_seen": 1357774848 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004756670010030091, + "loss": 1.3185, + "theoretical_loss": 3.5758460793093887, + "tokens_seen": 1357840384 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004756509528585758, + "loss": 1.2617, + "theoretical_loss": 3.575830694118571, + "tokens_seen": 1357905920 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004756349047141424, + "loss": 1.3435, + "theoretical_loss": 3.5758153098781604, + "tokens_seen": 1357971456 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004756188565697092, + "loss": 1.3266, + "theoretical_loss": 3.5757999265880525, + "tokens_seen": 1358036992 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004756028084252758, + "loss": 1.4084, + "theoretical_loss": 3.5757845442481426, + "tokens_seen": 1358102528 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004755867602808426, + "loss": 1.2733, + "theoretical_loss": 3.575769162858326, + "tokens_seen": 1358168064 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004755707121364092, + "loss": 1.2879, + "theoretical_loss": 3.5757537824184986, + "tokens_seen": 1358233600 + }, + { + "epoch": 0.41, + "learning_rate": 0.000475554663991976, + "loss": 1.2708, + "theoretical_loss": 3.5757384029285557, + "tokens_seen": 1358299136 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004755386158475426, + "loss": 1.3172, + "theoretical_loss": 3.5757230243883926, + "tokens_seen": 1358364672 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004755225677031094, + "loss": 1.3364, + "theoretical_loss": 3.5757076467979045, + "tokens_seen": 1358430208 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047550651955867603, + "loss": 1.358, + "theoretical_loss": 3.575692270156988, + "tokens_seen": 1358495744 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004754904714142428, + "loss": 1.3155, + "theoretical_loss": 3.5756768944655377, + "tokens_seen": 1358561280 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047547442326980943, + "loss": 1.2822, + "theoretical_loss": 3.5756615197234503, + "tokens_seen": 1358626816 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004754583751253762, + "loss": 1.3729, + "theoretical_loss": 3.5756461459306204, + "tokens_seen": 1358692352 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047544232698094283, + "loss": 1.4255, + "theoretical_loss": 3.575630773086943, + "tokens_seen": 1358757888 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004754262788365096, + "loss": 1.3004, + "theoretical_loss": 3.575615401192316, + "tokens_seen": 1358823424 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047541023069207623, + "loss": 1.3632, + "theoretical_loss": 3.5756000302466324, + "tokens_seen": 1358888960 + }, + { + "epoch": 0.41, + "learning_rate": 0.000475394182547643, + "loss": 1.4778, + "theoretical_loss": 3.575584660249789, + "tokens_seen": 1358954496 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047537813440320963, + "loss": 1.4276, + "theoretical_loss": 3.5755692912016825, + "tokens_seen": 1359020032 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004753620862587764, + "loss": 1.3964, + "theoretical_loss": 3.5755539231022073, + "tokens_seen": 1359085568 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047534603811434303, + "loss": 1.4651, + "theoretical_loss": 3.575538555951259, + "tokens_seen": 1359151104 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004753299899699098, + "loss": 1.4241, + "theoretical_loss": 3.5755231897487345, + "tokens_seen": 1359216640 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047531394182547643, + "loss": 1.2928, + "theoretical_loss": 3.5755078244945286, + "tokens_seen": 1359282176 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004752978936810432, + "loss": 1.3191, + "theoretical_loss": 3.5754924601885367, + "tokens_seen": 1359347712 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047528184553660984, + "loss": 1.2639, + "theoretical_loss": 3.5754770968306557, + "tokens_seen": 1359413248 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004752657973921766, + "loss": 1.3114, + "theoretical_loss": 3.575461734420781, + "tokens_seen": 1359478784 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047524974924774324, + "loss": 1.3596, + "theoretical_loss": 3.575446372958808, + "tokens_seen": 1359544320 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047523370110331, + "loss": 1.3917, + "theoretical_loss": 3.5754310124446325, + "tokens_seen": 1359609856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047521765295887664, + "loss": 1.3005, + "theoretical_loss": 3.575415652878151, + "tokens_seen": 1359675392 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004752016048144434, + "loss": 1.3469, + "theoretical_loss": 3.575400294259259, + "tokens_seen": 1359740928 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047518555667001004, + "loss": 1.3826, + "theoretical_loss": 3.5753849365878523, + "tokens_seen": 1359806464 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 456448, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3537814617156982, + "objective/train/theoretical_loss": 3.5753695798638265, + "objective/train/tokens_used": 1380332000, + "theoretical_loss": 3.5753695798638265, + "tokens_seen": 1359872000 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004751695085255768, + "loss": 1.3345, + "theoretical_loss": 3.5753695798638265, + "tokens_seen": 1359872000 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047515346038114344, + "loss": 1.3273, + "theoretical_loss": 3.5753542240870777, + "tokens_seen": 1359937536 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004751374122367102, + "loss": 1.2333, + "theoretical_loss": 3.5753388692575023, + "tokens_seen": 1360003072 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047512136409227684, + "loss": 1.351, + "theoretical_loss": 3.575323515374996, + "tokens_seen": 1360068608 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004751053159478436, + "loss": 1.3505, + "theoretical_loss": 3.5753081624394545, + "tokens_seen": 1360134144 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047508926780341024, + "loss": 1.2973, + "theoretical_loss": 3.5752928104507737, + "tokens_seen": 1360199680 + }, + { + "epoch": 0.41, + "learning_rate": 0.000475073219658977, + "loss": 1.2254, + "theoretical_loss": 3.5752774594088503, + "tokens_seen": 1360265216 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047505717151454364, + "loss": 1.2612, + "theoretical_loss": 3.57526210931358, + "tokens_seen": 1360330752 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004750411233701104, + "loss": 1.311, + "theoretical_loss": 3.5752467601648577, + "tokens_seen": 1360396288 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047502507522567705, + "loss": 1.3675, + "theoretical_loss": 3.575231411962581, + "tokens_seen": 1360461824 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047500902708124375, + "loss": 1.3748, + "theoretical_loss": 3.575216064706645, + "tokens_seen": 1360527360 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047499297893681045, + "loss": 1.2639, + "theoretical_loss": 3.575200718396946, + "tokens_seen": 1360592896 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047497693079237715, + "loss": 1.4052, + "theoretical_loss": 3.5751853730333805, + "tokens_seen": 1360658432 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047496088264794385, + "loss": 1.3142, + "theoretical_loss": 3.5751700286158443, + "tokens_seen": 1360723968 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047494483450351055, + "loss": 1.3593, + "theoretical_loss": 3.5751546851442333, + "tokens_seen": 1360789504 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047492878635907725, + "loss": 1.2908, + "theoretical_loss": 3.575139342618444, + "tokens_seen": 1360855040 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047491273821464395, + "loss": 1.3743, + "theoretical_loss": 3.5751240010383727, + "tokens_seen": 1360920576 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047489669007021065, + "loss": 1.3119, + "theoretical_loss": 3.575108660403915, + "tokens_seen": 1360986112 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047488064192577735, + "loss": 1.2732, + "theoretical_loss": 3.575093320714967, + "tokens_seen": 1361051648 + }, + { + "epoch": 0.41, + "learning_rate": 0.000474864593781344, + "loss": 1.328, + "theoretical_loss": 3.5750779819714253, + "tokens_seen": 1361117184 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047484854563691075, + "loss": 1.3256, + "theoretical_loss": 3.5750626441731868, + "tokens_seen": 1361182720 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004748324974924774, + "loss": 1.3487, + "theoretical_loss": 3.575047307320146, + "tokens_seen": 1361248256 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047481644934804415, + "loss": 1.3667, + "theoretical_loss": 3.5750319714122005, + "tokens_seen": 1361313792 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004748004012036108, + "loss": 1.3611, + "theoretical_loss": 3.5750166364492464, + "tokens_seen": 1361379328 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047478435305917756, + "loss": 1.4097, + "theoretical_loss": 3.5750013024311795, + "tokens_seen": 1361444864 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004747683049147442, + "loss": 1.3568, + "theoretical_loss": 3.574985969357897, + "tokens_seen": 1361510400 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047475225677031096, + "loss": 1.2948, + "theoretical_loss": 3.5749706372292938, + "tokens_seen": 1361575936 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004747362086258776, + "loss": 1.3441, + "theoretical_loss": 3.5749553060452675, + "tokens_seen": 1361641472 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047472016048144436, + "loss": 1.3073, + "theoretical_loss": 3.574939975805714, + "tokens_seen": 1361707008 + }, + { + "epoch": 0.41, + "learning_rate": 0.000474704112337011, + "loss": 1.2755, + "theoretical_loss": 3.5749246465105298, + "tokens_seen": 1361772544 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047468806419257776, + "loss": 1.4098, + "theoretical_loss": 3.5749093181596105, + "tokens_seen": 1361838080 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004746720160481444, + "loss": 1.3682, + "theoretical_loss": 3.5748939907528534, + "tokens_seen": 1361903616 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047465596790371116, + "loss": 1.4077, + "theoretical_loss": 3.574878664290155, + "tokens_seen": 1361969152 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004746399197592778, + "loss": 1.29, + "theoretical_loss": 3.574863338771411, + "tokens_seen": 1362034688 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047462387161484456, + "loss": 1.3673, + "theoretical_loss": 3.5748480141965184, + "tokens_seen": 1362100224 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004746078234704112, + "loss": 1.3854, + "theoretical_loss": 3.5748326905653736, + "tokens_seen": 1362165760 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047459177532597796, + "loss": 1.3443, + "theoretical_loss": 3.5748173678778725, + "tokens_seen": 1362231296 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004745757271815446, + "loss": 1.3209, + "theoretical_loss": 3.5748020461339123, + "tokens_seen": 1362296832 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047455967903711136, + "loss": 1.2935, + "theoretical_loss": 3.5747867253333894, + "tokens_seen": 1362362368 + }, + { + "epoch": 0.41, + "learning_rate": 0.000474543630892678, + "loss": 1.4221, + "theoretical_loss": 3.5747714054762003, + "tokens_seen": 1362427904 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047452758274824477, + "loss": 1.2886, + "theoretical_loss": 3.5747560865622408, + "tokens_seen": 1362493440 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004745115346038114, + "loss": 1.3608, + "theoretical_loss": 3.574740768591409, + "tokens_seen": 1362558976 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047449548645937817, + "loss": 1.3151, + "theoretical_loss": 3.5747254515635998, + "tokens_seen": 1362624512 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004744794383149448, + "loss": 1.3692, + "theoretical_loss": 3.5747101354787114, + "tokens_seen": 1362690048 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047446339017051157, + "loss": 1.315, + "theoretical_loss": 3.5746948203366387, + "tokens_seen": 1362755584 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004744473420260783, + "loss": 1.432, + "theoretical_loss": 3.57467950613728, + "tokens_seen": 1362821120 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047443129388164497, + "loss": 1.4033, + "theoretical_loss": 3.5746641928805305, + "tokens_seen": 1362886656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004744152457372117, + "loss": 1.4039, + "theoretical_loss": 3.574648880566288, + "tokens_seen": 1362952192 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047439919759277837, + "loss": 1.2897, + "theoretical_loss": 3.5746335691944484, + "tokens_seen": 1363017728 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004743831494483451, + "loss": 1.3326, + "theoretical_loss": 3.5746182587649087, + "tokens_seen": 1363083264 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 457783, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.313234806060791, + "objective/train/theoretical_loss": 3.5746029492775655, + "objective/train/tokens_used": 1383608800, + "theoretical_loss": 3.5746029492775655, + "tokens_seen": 1363148800 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047436710130391177, + "loss": 1.3109, + "theoretical_loss": 3.5746029492775655, + "tokens_seen": 1363148800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004743510531594785, + "loss": 1.3396, + "theoretical_loss": 3.5745876407323163, + "tokens_seen": 1363214336 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004743350050150452, + "loss": 1.3384, + "theoretical_loss": 3.574572333129056, + "tokens_seen": 1363279872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004743189568706119, + "loss": 1.3015, + "theoretical_loss": 3.5745570264676836, + "tokens_seen": 1363345408 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004743029087261786, + "loss": 1.3519, + "theoretical_loss": 3.574541720748094, + "tokens_seen": 1363410944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004742868605817453, + "loss": 1.3502, + "theoretical_loss": 3.574526415970185, + "tokens_seen": 1363476480 + }, + { + "epoch": 0.41, + "learning_rate": 0.000474270812437312, + "loss": 1.3159, + "theoretical_loss": 3.5745111121338535, + "tokens_seen": 1363542016 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004742547642928787, + "loss": 1.2457, + "theoretical_loss": 3.5744958092389956, + "tokens_seen": 1363607552 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004742387161484453, + "loss": 1.3811, + "theoretical_loss": 3.5744805072855086, + "tokens_seen": 1363673088 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004742226680040121, + "loss": 1.3374, + "theoretical_loss": 3.5744652062732896, + "tokens_seen": 1363738624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004742066198595787, + "loss": 1.3209, + "theoretical_loss": 3.5744499062022346, + "tokens_seen": 1363804160 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004741905717151455, + "loss": 1.3802, + "theoretical_loss": 3.5744346070722415, + "tokens_seen": 1363869696 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004741745235707121, + "loss": 1.2311, + "theoretical_loss": 3.574419308883207, + "tokens_seen": 1363935232 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004741584754262789, + "loss": 1.3884, + "theoretical_loss": 3.5744040116350266, + "tokens_seen": 1364000768 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004741424272818455, + "loss": 1.2998, + "theoretical_loss": 3.5743887153275997, + "tokens_seen": 1364066304 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004741263791374123, + "loss": 1.2812, + "theoretical_loss": 3.5743734199608213, + "tokens_seen": 1364131840 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047411033099297893, + "loss": 1.3264, + "theoretical_loss": 3.574358125534589, + "tokens_seen": 1364197376 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004740942828485457, + "loss": 1.3705, + "theoretical_loss": 3.5743428320488, + "tokens_seen": 1364262912 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047407823470411233, + "loss": 1.2873, + "theoretical_loss": 3.574327539503351, + "tokens_seen": 1364328448 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004740621865596791, + "loss": 1.254, + "theoretical_loss": 3.5743122478981393, + "tokens_seen": 1364393984 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047404613841524573, + "loss": 1.302, + "theoretical_loss": 3.5742969572330616, + "tokens_seen": 1364459520 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004740300902708125, + "loss": 1.2448, + "theoretical_loss": 3.574281667508015, + "tokens_seen": 1364525056 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047401404212637913, + "loss": 1.3864, + "theoretical_loss": 3.574266378722897, + "tokens_seen": 1364590592 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004739979939819459, + "loss": 1.3481, + "theoretical_loss": 3.5742510908776044, + "tokens_seen": 1364656128 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047398194583751253, + "loss": 1.3396, + "theoretical_loss": 3.574235803972034, + "tokens_seen": 1364721664 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004739658976930793, + "loss": 1.3188, + "theoretical_loss": 3.574220518006083, + "tokens_seen": 1364787200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047394984954864593, + "loss": 1.3528, + "theoretical_loss": 3.574205232979649, + "tokens_seen": 1364852736 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004739338014042127, + "loss": 1.322, + "theoretical_loss": 3.574189948892629, + "tokens_seen": 1364918272 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047391775325977934, + "loss": 1.3503, + "theoretical_loss": 3.57417466574492, + "tokens_seen": 1364983808 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004739017051153461, + "loss": 1.3435, + "theoretical_loss": 3.5741593835364185, + "tokens_seen": 1365049344 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047388565697091274, + "loss": 1.3456, + "theoretical_loss": 3.574144102267023, + "tokens_seen": 1365114880 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004738696088264795, + "loss": 1.2507, + "theoretical_loss": 3.5741288219366294, + "tokens_seen": 1365180416 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047385356068204614, + "loss": 1.3631, + "theoretical_loss": 3.5741135425451365, + "tokens_seen": 1365245952 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004738375125376129, + "loss": 1.3467, + "theoretical_loss": 3.5740982640924397, + "tokens_seen": 1365311488 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047382146439317954, + "loss": 1.3542, + "theoretical_loss": 3.5740829865784383, + "tokens_seen": 1365377024 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004738054162487463, + "loss": 1.4454, + "theoretical_loss": 3.5740677100030274, + "tokens_seen": 1365442560 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047378936810431294, + "loss": 1.3942, + "theoretical_loss": 3.574052434366106, + "tokens_seen": 1365508096 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004737733199598797, + "loss": 1.3959, + "theoretical_loss": 3.5740371596675713, + "tokens_seen": 1365573632 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047375727181544634, + "loss": 1.3032, + "theoretical_loss": 3.574021885907319, + "tokens_seen": 1365639168 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004737412236710131, + "loss": 1.3207, + "theoretical_loss": 3.574006613085248, + "tokens_seen": 1365704704 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047372517552657974, + "loss": 1.2565, + "theoretical_loss": 3.5739913412012556, + "tokens_seen": 1365770240 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004737091273821465, + "loss": 1.2285, + "theoretical_loss": 3.573976070255238, + "tokens_seen": 1365835776 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047369307923771314, + "loss": 1.3508, + "theoretical_loss": 3.573960800247094, + "tokens_seen": 1365901312 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004736770310932799, + "loss": 1.3147, + "theoretical_loss": 3.5739455311767205, + "tokens_seen": 1365966848 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047366098294884655, + "loss": 1.3043, + "theoretical_loss": 3.573930263044015, + "tokens_seen": 1366032384 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004736449348044133, + "loss": 1.4096, + "theoretical_loss": 3.573914995848874, + "tokens_seen": 1366097920 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047362888665997995, + "loss": 1.2915, + "theoretical_loss": 3.573899729591196, + "tokens_seen": 1366163456 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004736128385155467, + "loss": 1.2662, + "theoretical_loss": 3.5738844642708782, + "tokens_seen": 1366228992 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047359679037111335, + "loss": 1.4209, + "theoretical_loss": 3.5738691998878176, + "tokens_seen": 1366294528 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047358074222668005, + "loss": 1.2719, + "theoretical_loss": 3.5738539364419126, + "tokens_seen": 1366360064 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 458648, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3905396461486816, + "objective/train/theoretical_loss": 3.5738386739330603, + "objective/train/tokens_used": 1386885600, + "theoretical_loss": 3.5738386739330603, + "tokens_seen": 1366425600 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047356469408224675, + "loss": 1.3947, + "theoretical_loss": 3.5738386739330603, + "tokens_seen": 1366425600 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047354864593781345, + "loss": 1.3053, + "theoretical_loss": 3.573823412361158, + "tokens_seen": 1366491136 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047353259779338015, + "loss": 1.318, + "theoretical_loss": 3.5738081517261033, + "tokens_seen": 1366556672 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047351654964894685, + "loss": 1.3456, + "theoretical_loss": 3.573792892027795, + "tokens_seen": 1366622208 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047350050150451355, + "loss": 1.3967, + "theoretical_loss": 3.573777633266128, + "tokens_seen": 1366687744 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047348445336008025, + "loss": 1.3089, + "theoretical_loss": 3.5737623754410026, + "tokens_seen": 1366753280 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004734684052156469, + "loss": 1.3499, + "theoretical_loss": 3.573747118552315, + "tokens_seen": 1366818816 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047345235707121365, + "loss": 1.341, + "theoretical_loss": 3.573731862599963, + "tokens_seen": 1366884352 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004734363089267803, + "loss": 1.4185, + "theoretical_loss": 3.5737166075838447, + "tokens_seen": 1366949888 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047342026078234706, + "loss": 1.3863, + "theoretical_loss": 3.573701353503857, + "tokens_seen": 1367015424 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004734042126379137, + "loss": 1.4295, + "theoretical_loss": 3.5736861003598985, + "tokens_seen": 1367080960 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047338816449348046, + "loss": 1.3948, + "theoretical_loss": 3.573670848151866, + "tokens_seen": 1367146496 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004733721163490471, + "loss": 1.25, + "theoretical_loss": 3.573655596879658, + "tokens_seen": 1367212032 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047335606820461386, + "loss": 1.3141, + "theoretical_loss": 3.573640346543172, + "tokens_seen": 1367277568 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004733400200601805, + "loss": 1.3778, + "theoretical_loss": 3.5736250971423056, + "tokens_seen": 1367343104 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047332397191574726, + "loss": 1.2543, + "theoretical_loss": 3.5736098486769556, + "tokens_seen": 1367408640 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004733079237713139, + "loss": 1.3102, + "theoretical_loss": 3.573594601147022, + "tokens_seen": 1367474176 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047329187562688066, + "loss": 1.3389, + "theoretical_loss": 3.5735793545524004, + "tokens_seen": 1367539712 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004732758274824473, + "loss": 1.2981, + "theoretical_loss": 3.57356410889299, + "tokens_seen": 1367605248 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047325977933801406, + "loss": 1.282, + "theoretical_loss": 3.5735488641686883, + "tokens_seen": 1367670784 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004732437311935808, + "loss": 1.4454, + "theoretical_loss": 3.5735336203793926, + "tokens_seen": 1367736320 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047322768304914746, + "loss": 1.4393, + "theoretical_loss": 3.5735183775250015, + "tokens_seen": 1367801856 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004732116349047142, + "loss": 1.3012, + "theoretical_loss": 3.5735031356054123, + "tokens_seen": 1367867392 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047319558676028086, + "loss": 1.2726, + "theoretical_loss": 3.5734878946205235, + "tokens_seen": 1367932928 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004731795386158476, + "loss": 1.348, + "theoretical_loss": 3.5734726545702324, + "tokens_seen": 1367998464 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047316349047141427, + "loss": 1.3849, + "theoretical_loss": 3.573457415454437, + "tokens_seen": 1368064000 + }, + { + "epoch": 0.41, + "learning_rate": 0.000473147442326981, + "loss": 1.2892, + "theoretical_loss": 3.5734421772730354, + "tokens_seen": 1368129536 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047313139418254767, + "loss": 1.3821, + "theoretical_loss": 3.5734269400259255, + "tokens_seen": 1368195072 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004731153460381144, + "loss": 1.3389, + "theoretical_loss": 3.5734117037130053, + "tokens_seen": 1368260608 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047309929789368107, + "loss": 1.4114, + "theoretical_loss": 3.573396468334173, + "tokens_seen": 1368326144 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004730832497492478, + "loss": 1.2412, + "theoretical_loss": 3.5733812338893265, + "tokens_seen": 1368391680 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047306720160481447, + "loss": 1.3724, + "theoretical_loss": 3.5733660003783636, + "tokens_seen": 1368457216 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004730511534603812, + "loss": 1.3053, + "theoretical_loss": 3.573350767801182, + "tokens_seen": 1368522752 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047303510531594787, + "loss": 1.3152, + "theoretical_loss": 3.5733355361576806, + "tokens_seen": 1368588288 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004730190571715146, + "loss": 1.3617, + "theoretical_loss": 3.573320305447757, + "tokens_seen": 1368653824 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047300300902708127, + "loss": 1.3034, + "theoretical_loss": 3.5733050756713096, + "tokens_seen": 1368719360 + }, + { + "epoch": 0.41, + "learning_rate": 0.000472986960882648, + "loss": 1.3214, + "theoretical_loss": 3.573289846828236, + "tokens_seen": 1368784896 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004729709127382147, + "loss": 1.224, + "theoretical_loss": 3.5732746189184343, + "tokens_seen": 1368850432 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004729548645937814, + "loss": 1.24, + "theoretical_loss": 3.5732593919418028, + "tokens_seen": 1368915968 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004729388164493481, + "loss": 1.2527, + "theoretical_loss": 3.57324416589824, + "tokens_seen": 1368981504 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004729227683049148, + "loss": 1.3667, + "theoretical_loss": 3.5732289407876436, + "tokens_seen": 1369047040 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004729067201604815, + "loss": 1.4329, + "theoretical_loss": 3.5732137166099123, + "tokens_seen": 1369112576 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004728906720160482, + "loss": 1.3794, + "theoretical_loss": 3.5731984933649437, + "tokens_seen": 1369178112 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004728746238716149, + "loss": 1.3097, + "theoretical_loss": 3.573183271052636, + "tokens_seen": 1369243648 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004728585757271816, + "loss": 1.3117, + "theoretical_loss": 3.573168049672888, + "tokens_seen": 1369309184 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004728425275827483, + "loss": 1.2598, + "theoretical_loss": 3.573152829225598, + "tokens_seen": 1369374720 + }, + { + "epoch": 0.41, + "learning_rate": 0.000472826479438315, + "loss": 1.2374, + "theoretical_loss": 3.573137609710663, + "tokens_seen": 1369440256 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004728104312938816, + "loss": 1.3218, + "theoretical_loss": 3.573122391127982, + "tokens_seen": 1369505792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004727943831494484, + "loss": 1.3227, + "theoretical_loss": 3.5731071734774544, + "tokens_seen": 1369571328 + }, + { + "epoch": 0.42, + "learning_rate": 0.000472778335005015, + "loss": 1.2823, + "theoretical_loss": 3.5730919567589767, + "tokens_seen": 1369636864 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 459889, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1413302421569824, + "objective/train/theoretical_loss": 3.573076740972449, + "objective/train/tokens_used": 1390162400, + "theoretical_loss": 3.573076740972449, + "tokens_seen": 1369702400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004727622868605818, + "loss": 1.2678, + "theoretical_loss": 3.573076740972449, + "tokens_seen": 1369702400 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047274623871614843, + "loss": 1.2219, + "theoretical_loss": 3.573061526117768, + "tokens_seen": 1369767936 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004727301905717152, + "loss": 1.4363, + "theoretical_loss": 3.573046312194833, + "tokens_seen": 1369833472 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047271414242728183, + "loss": 1.2873, + "theoretical_loss": 3.5730310992035417, + "tokens_seen": 1369899008 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004726980942828486, + "loss": 1.2869, + "theoretical_loss": 3.573015887143793, + "tokens_seen": 1369964544 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047268204613841523, + "loss": 1.2866, + "theoretical_loss": 3.5730006760154853, + "tokens_seen": 1370030080 + }, + { + "epoch": 0.42, + "learning_rate": 0.000472665997993982, + "loss": 1.4166, + "theoretical_loss": 3.572985465818517, + "tokens_seen": 1370095616 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047264994984954863, + "loss": 1.3116, + "theoretical_loss": 3.5729702565527868, + "tokens_seen": 1370161152 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004726339017051154, + "loss": 1.3564, + "theoretical_loss": 3.572955048218192, + "tokens_seen": 1370226688 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047261785356068203, + "loss": 1.3333, + "theoretical_loss": 3.5729398408146325, + "tokens_seen": 1370292224 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004726018054162488, + "loss": 1.2981, + "theoretical_loss": 3.5729246343420056, + "tokens_seen": 1370357760 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047258575727181543, + "loss": 1.4071, + "theoretical_loss": 3.5729094288002106, + "tokens_seen": 1370423296 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004725697091273822, + "loss": 1.2527, + "theoretical_loss": 3.5728942241891457, + "tokens_seen": 1370488832 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047255366098294884, + "loss": 1.3149, + "theoretical_loss": 3.5728790205087093, + "tokens_seen": 1370554368 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004725376128385156, + "loss": 1.3044, + "theoretical_loss": 3.5728638177588, + "tokens_seen": 1370619904 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047252156469408224, + "loss": 1.3668, + "theoretical_loss": 3.572848615939317, + "tokens_seen": 1370685440 + }, + { + "epoch": 0.42, + "learning_rate": 0.000472505516549649, + "loss": 1.2671, + "theoretical_loss": 3.572833415050158, + "tokens_seen": 1370750976 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047248946840521564, + "loss": 1.4222, + "theoretical_loss": 3.572818215091222, + "tokens_seen": 1370816512 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004724734202607824, + "loss": 1.3043, + "theoretical_loss": 3.572803016062408, + "tokens_seen": 1370882048 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047245737211634904, + "loss": 1.2599, + "theoretical_loss": 3.572787817963613, + "tokens_seen": 1370947584 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004724413239719158, + "loss": 1.4172, + "theoretical_loss": 3.572772620794738, + "tokens_seen": 1371013120 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047242527582748244, + "loss": 1.4319, + "theoretical_loss": 3.5727574245556797, + "tokens_seen": 1371078656 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004724092276830492, + "loss": 1.2669, + "theoretical_loss": 3.572742229246338, + "tokens_seen": 1371144192 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047239317953861584, + "loss": 1.4297, + "theoretical_loss": 3.572727034866611, + "tokens_seen": 1371209728 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004723771313941826, + "loss": 1.4387, + "theoretical_loss": 3.572711841416397, + "tokens_seen": 1371275264 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047236108324974924, + "loss": 1.255, + "theoretical_loss": 3.5726966488955956, + "tokens_seen": 1371340800 + }, + { + "epoch": 0.42, + "learning_rate": 0.000472345035105316, + "loss": 1.3279, + "theoretical_loss": 3.572681457304105, + "tokens_seen": 1371406336 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047232898696088264, + "loss": 1.4369, + "theoretical_loss": 3.5726662666418245, + "tokens_seen": 1371471872 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004723129388164494, + "loss": 1.3403, + "theoretical_loss": 3.5726510769086524, + "tokens_seen": 1371537408 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047229689067201605, + "loss": 1.3083, + "theoretical_loss": 3.5726358881044873, + "tokens_seen": 1371602944 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004722808425275828, + "loss": 1.3177, + "theoretical_loss": 3.572620700229228, + "tokens_seen": 1371668480 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047226479438314945, + "loss": 1.3407, + "theoretical_loss": 3.5726055132827748, + "tokens_seen": 1371734016 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004722487462387162, + "loss": 1.4713, + "theoretical_loss": 3.572590327265024, + "tokens_seen": 1371799552 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047223269809428285, + "loss": 1.4398, + "theoretical_loss": 3.572575142175876, + "tokens_seen": 1371865088 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004722166499498496, + "loss": 1.3684, + "theoretical_loss": 3.57255995801523, + "tokens_seen": 1371930624 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047220060180541625, + "loss": 1.3469, + "theoretical_loss": 3.5725447747829833, + "tokens_seen": 1371996160 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047218455366098295, + "loss": 1.3924, + "theoretical_loss": 3.572529592479037, + "tokens_seen": 1372061696 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047216850551654965, + "loss": 1.338, + "theoretical_loss": 3.572514411103288, + "tokens_seen": 1372127232 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047215245737211635, + "loss": 1.2579, + "theoretical_loss": 3.572499230655636, + "tokens_seen": 1372192768 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047213640922768305, + "loss": 1.3848, + "theoretical_loss": 3.57248405113598, + "tokens_seen": 1372258304 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047212036108324975, + "loss": 1.4693, + "theoretical_loss": 3.572468872544219, + "tokens_seen": 1372323840 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047210431293881645, + "loss": 1.367, + "theoretical_loss": 3.572453694880252, + "tokens_seen": 1372389376 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047208826479438315, + "loss": 1.3872, + "theoretical_loss": 3.5724385181439775, + "tokens_seen": 1372454912 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047207221664994985, + "loss": 1.366, + "theoretical_loss": 3.572423342335295, + "tokens_seen": 1372520448 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047205616850551656, + "loss": 1.4044, + "theoretical_loss": 3.5724081674541033, + "tokens_seen": 1372585984 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004720401203610833, + "loss": 1.3479, + "theoretical_loss": 3.572392993500302, + "tokens_seen": 1372651520 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047202407221664996, + "loss": 1.3709, + "theoretical_loss": 3.572377820473789, + "tokens_seen": 1372717056 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004720080240722167, + "loss": 1.3712, + "theoretical_loss": 3.5723626483744644, + "tokens_seen": 1372782592 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047199197592778336, + "loss": 1.3318, + "theoretical_loss": 3.572347477202227, + "tokens_seen": 1372848128 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004719759277833501, + "loss": 1.3188, + "theoretical_loss": 3.5723323069569757, + "tokens_seen": 1372913664 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 460768, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2740132808685303, + "objective/train/theoretical_loss": 3.57231713763861, + "objective/train/tokens_used": 1393439200, + "theoretical_loss": 3.57231713763861, + "tokens_seen": 1372979200 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047195987963891676, + "loss": 1.2705, + "theoretical_loss": 3.57231713763861, + "tokens_seen": 1372979200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004719438314944835, + "loss": 1.3173, + "theoretical_loss": 3.572301969247028, + "tokens_seen": 1373044736 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047192778335005016, + "loss": 1.3587, + "theoretical_loss": 3.57228680178213, + "tokens_seen": 1373110272 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004719117352056169, + "loss": 1.297, + "theoretical_loss": 3.572271635243815, + "tokens_seen": 1373175808 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047189568706118356, + "loss": 1.31, + "theoretical_loss": 3.5722564696319816, + "tokens_seen": 1373241344 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004718796389167503, + "loss": 1.3391, + "theoretical_loss": 3.57224130494653, + "tokens_seen": 1373306880 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047186359077231696, + "loss": 1.288, + "theoretical_loss": 3.572226141187358, + "tokens_seen": 1373372416 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004718475426278837, + "loss": 1.294, + "theoretical_loss": 3.5722109783543656, + "tokens_seen": 1373437952 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047183149448345036, + "loss": 1.3175, + "theoretical_loss": 3.5721958164474525, + "tokens_seen": 1373503488 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004718154463390171, + "loss": 1.3887, + "theoretical_loss": 3.572180655466517, + "tokens_seen": 1373569024 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047179939819458377, + "loss": 1.4171, + "theoretical_loss": 3.572165495411459, + "tokens_seen": 1373634560 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004717833500501505, + "loss": 1.3422, + "theoretical_loss": 3.572150336282178, + "tokens_seen": 1373700096 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047176730190571717, + "loss": 1.3605, + "theoretical_loss": 3.572135178078572, + "tokens_seen": 1373765632 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004717512537612839, + "loss": 1.2494, + "theoretical_loss": 3.572120020800542, + "tokens_seen": 1373831168 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047173520561685057, + "loss": 1.3234, + "theoretical_loss": 3.5721048644479865, + "tokens_seen": 1373896704 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004717191574724173, + "loss": 1.3181, + "theoretical_loss": 3.572089709020805, + "tokens_seen": 1373962240 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047170310932798397, + "loss": 1.3014, + "theoretical_loss": 3.5720745545188963, + "tokens_seen": 1374027776 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004716870611835507, + "loss": 1.4695, + "theoretical_loss": 3.5720594009421607, + "tokens_seen": 1374093312 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047167101303911737, + "loss": 1.2828, + "theoretical_loss": 3.572044248290497, + "tokens_seen": 1374158848 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004716549648946841, + "loss": 1.4203, + "theoretical_loss": 3.5720290965638046, + "tokens_seen": 1374224384 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047163891675025077, + "loss": 1.2572, + "theoretical_loss": 3.5720139457619835, + "tokens_seen": 1374289920 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004716228686058175, + "loss": 1.3911, + "theoretical_loss": 3.571998795884933, + "tokens_seen": 1374355456 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004716068204613842, + "loss": 1.1725, + "theoretical_loss": 3.5719836469325514, + "tokens_seen": 1374420992 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047159077231695093, + "loss": 1.3756, + "theoretical_loss": 3.5719684989047398, + "tokens_seen": 1374486528 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004715747241725176, + "loss": 1.3474, + "theoretical_loss": 3.5719533518013966, + "tokens_seen": 1374552064 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004715586760280843, + "loss": 1.3653, + "theoretical_loss": 3.5719382056224225, + "tokens_seen": 1374617600 + }, + { + "epoch": 0.42, + "learning_rate": 0.000471542627883651, + "loss": 1.2804, + "theoretical_loss": 3.5719230603677152, + "tokens_seen": 1374683136 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004715265797392177, + "loss": 1.4509, + "theoretical_loss": 3.5719079160371763, + "tokens_seen": 1374748672 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004715105315947844, + "loss": 1.3045, + "theoretical_loss": 3.5718927726307035, + "tokens_seen": 1374814208 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004714944834503511, + "loss": 1.2463, + "theoretical_loss": 3.5718776301481974, + "tokens_seen": 1374879744 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004714784353059178, + "loss": 1.2716, + "theoretical_loss": 3.5718624885895576, + "tokens_seen": 1374945280 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004714623871614845, + "loss": 1.315, + "theoretical_loss": 3.571847347954683, + "tokens_seen": 1375010816 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004714463390170512, + "loss": 1.3062, + "theoretical_loss": 3.5718322082434737, + "tokens_seen": 1375076352 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004714302908726179, + "loss": 1.2935, + "theoretical_loss": 3.5718170694558298, + "tokens_seen": 1375141888 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004714142427281845, + "loss": 1.383, + "theoretical_loss": 3.5718019315916503, + "tokens_seen": 1375207424 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004713981945837513, + "loss": 1.3552, + "theoretical_loss": 3.5717867946508353, + "tokens_seen": 1375272960 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047138214643931793, + "loss": 1.2173, + "theoretical_loss": 3.5717716586332835, + "tokens_seen": 1375338496 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004713660982948847, + "loss": 1.3346, + "theoretical_loss": 3.571756523538896, + "tokens_seen": 1375404032 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047135005015045133, + "loss": 1.3706, + "theoretical_loss": 3.5717413893675714, + "tokens_seen": 1375469568 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004713340020060181, + "loss": 1.3294, + "theoretical_loss": 3.5717262561192102, + "tokens_seen": 1375535104 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047131795386158473, + "loss": 1.3575, + "theoretical_loss": 3.5717111237937114, + "tokens_seen": 1375600640 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004713019057171515, + "loss": 1.3182, + "theoretical_loss": 3.5716959923909757, + "tokens_seen": 1375666176 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047128585757271813, + "loss": 1.3512, + "theoretical_loss": 3.5716808619109015, + "tokens_seen": 1375731712 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004712698094282849, + "loss": 1.3108, + "theoretical_loss": 3.5716657323533902, + "tokens_seen": 1375797248 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047125376128385153, + "loss": 1.3223, + "theoretical_loss": 3.57165060371834, + "tokens_seen": 1375862784 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004712377131394183, + "loss": 1.2804, + "theoretical_loss": 3.5716354760056523, + "tokens_seen": 1375928320 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047122166499498493, + "loss": 1.3762, + "theoretical_loss": 3.5716203492152263, + "tokens_seen": 1375993856 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004712056168505517, + "loss": 1.342, + "theoretical_loss": 3.5716052233469613, + "tokens_seen": 1376059392 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047118956870611834, + "loss": 1.3211, + "theoretical_loss": 3.571590098400758, + "tokens_seen": 1376124928 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004711735205616851, + "loss": 1.3122, + "theoretical_loss": 3.5715749743765155, + "tokens_seen": 1376190464 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 462043, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3740935325622559, + "objective/train/theoretical_loss": 3.5715598512741344, + "objective/train/tokens_used": 1396716000, + "theoretical_loss": 3.5715598512741344, + "tokens_seen": 1376256000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047115747241725174, + "loss": 1.3715, + "theoretical_loss": 3.5715598512741344, + "tokens_seen": 1376256000 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004711414242728185, + "loss": 1.4589, + "theoretical_loss": 3.571544729093514, + "tokens_seen": 1376321536 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047112537612838514, + "loss": 1.3145, + "theoretical_loss": 3.5715296078345546, + "tokens_seen": 1376387072 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004711093279839519, + "loss": 1.3687, + "theoretical_loss": 3.571514487497156, + "tokens_seen": 1376452608 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047109327983951854, + "loss": 1.2702, + "theoretical_loss": 3.5714993680812186, + "tokens_seen": 1376518144 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004710772316950853, + "loss": 1.3214, + "theoretical_loss": 3.571484249586642, + "tokens_seen": 1376583680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047106118355065194, + "loss": 1.3622, + "theoretical_loss": 3.571469132013326, + "tokens_seen": 1376649216 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004710451354062187, + "loss": 1.2574, + "theoretical_loss": 3.5714540153611702, + "tokens_seen": 1376714752 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047102908726178534, + "loss": 1.2868, + "theoretical_loss": 3.5714388996300763, + "tokens_seen": 1376780288 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004710130391173521, + "loss": 1.3985, + "theoretical_loss": 3.5714237848199426, + "tokens_seen": 1376845824 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047099699097291874, + "loss": 1.2811, + "theoretical_loss": 3.5714086709306705, + "tokens_seen": 1376911360 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004709809428284855, + "loss": 1.3024, + "theoretical_loss": 3.571393557962159, + "tokens_seen": 1376976896 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047096489468405214, + "loss": 1.3559, + "theoretical_loss": 3.5713784459143088, + "tokens_seen": 1377042432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004709488465396189, + "loss": 1.2328, + "theoretical_loss": 3.5713633347870193, + "tokens_seen": 1377107968 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047093279839518555, + "loss": 1.3789, + "theoretical_loss": 3.5713482245801917, + "tokens_seen": 1377173504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004709167502507523, + "loss": 1.3142, + "theoretical_loss": 3.5713331152937253, + "tokens_seen": 1377239040 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047090070210631895, + "loss": 1.2707, + "theoretical_loss": 3.5713180069275205, + "tokens_seen": 1377304576 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004708846539618857, + "loss": 1.3661, + "theoretical_loss": 3.571302899481478, + "tokens_seen": 1377370112 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004708686058174524, + "loss": 1.2541, + "theoretical_loss": 3.5712877929554967, + "tokens_seen": 1377435648 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004708525576730191, + "loss": 1.2824, + "theoretical_loss": 3.571272687349478, + "tokens_seen": 1377501184 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004708365095285858, + "loss": 1.43, + "theoretical_loss": 3.5712575826633213, + "tokens_seen": 1377566720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004708204613841525, + "loss": 1.2732, + "theoretical_loss": 3.5712424788969273, + "tokens_seen": 1377632256 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004708044132397192, + "loss": 1.303, + "theoretical_loss": 3.5712273760501967, + "tokens_seen": 1377697792 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047078836509528585, + "loss": 1.3798, + "theoretical_loss": 3.5712122741230283, + "tokens_seen": 1377763328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004707723169508526, + "loss": 1.344, + "theoretical_loss": 3.5711971731153236, + "tokens_seen": 1377828864 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047075626880641925, + "loss": 1.2814, + "theoretical_loss": 3.571182073026983, + "tokens_seen": 1377894400 + }, + { + "epoch": 0.42, + "learning_rate": 0.000470740220661986, + "loss": 1.321, + "theoretical_loss": 3.571166973857906, + "tokens_seen": 1377959936 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047072417251755265, + "loss": 1.2907, + "theoretical_loss": 3.571151875607993, + "tokens_seen": 1378025472 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004707081243731194, + "loss": 1.3214, + "theoretical_loss": 3.571136778277145, + "tokens_seen": 1378091008 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047069207622868605, + "loss": 1.2896, + "theoretical_loss": 3.571121681865262, + "tokens_seen": 1378156544 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004706760280842528, + "loss": 1.3386, + "theoretical_loss": 3.5711065863722444, + "tokens_seen": 1378222080 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047065997993981946, + "loss": 1.3329, + "theoretical_loss": 3.5710914917979917, + "tokens_seen": 1378287616 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004706439317953862, + "loss": 1.3, + "theoretical_loss": 3.5710763981424063, + "tokens_seen": 1378353152 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047062788365095286, + "loss": 1.3914, + "theoretical_loss": 3.5710613054053866, + "tokens_seen": 1378418688 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004706118355065196, + "loss": 1.226, + "theoretical_loss": 3.5710462135868344, + "tokens_seen": 1378484224 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047059578736208626, + "loss": 1.3071, + "theoretical_loss": 3.571031122686649, + "tokens_seen": 1378549760 + }, + { + "epoch": 0.42, + "learning_rate": 0.000470579739217653, + "loss": 1.3541, + "theoretical_loss": 3.571016032704732, + "tokens_seen": 1378615296 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047056369107321966, + "loss": 1.2992, + "theoretical_loss": 3.571000943640983, + "tokens_seen": 1378680832 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004705476429287864, + "loss": 1.4267, + "theoretical_loss": 3.5709858554953033, + "tokens_seen": 1378746368 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047053159478435306, + "loss": 1.2772, + "theoretical_loss": 3.5709707682675926, + "tokens_seen": 1378811904 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004705155466399198, + "loss": 1.4052, + "theoretical_loss": 3.5709556819577513, + "tokens_seen": 1378877440 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047049949849548646, + "loss": 1.354, + "theoretical_loss": 3.570940596565681, + "tokens_seen": 1378942976 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004704834503510532, + "loss": 1.3964, + "theoretical_loss": 3.5709255120912817, + "tokens_seen": 1379008512 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047046740220661986, + "loss": 1.1837, + "theoretical_loss": 3.5709104285344537, + "tokens_seen": 1379074048 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004704513540621866, + "loss": 1.3102, + "theoretical_loss": 3.5708953458950976, + "tokens_seen": 1379139584 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047043530591775327, + "loss": 1.3306, + "theoretical_loss": 3.5708802641731148, + "tokens_seen": 1379205120 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047041925777332, + "loss": 1.3896, + "theoretical_loss": 3.5708651833684053, + "tokens_seen": 1379270656 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047040320962888667, + "loss": 1.3955, + "theoretical_loss": 3.5708501034808693, + "tokens_seen": 1379336192 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004703871614844534, + "loss": 1.339, + "theoretical_loss": 3.5708350245104077, + "tokens_seen": 1379401728 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047037111334002007, + "loss": 1.3313, + "theoretical_loss": 3.570819946456922, + "tokens_seen": 1379467264 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 463003, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3521519899368286, + "objective/train/theoretical_loss": 3.5708048693203116, + "objective/train/tokens_used": 1399992800, + "theoretical_loss": 3.5708048693203116, + "tokens_seen": 1379532800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004703550651955868, + "loss": 1.3013, + "theoretical_loss": 3.5708048693203116, + "tokens_seen": 1379532800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047033901705115347, + "loss": 1.3763, + "theoretical_loss": 3.570789793100478, + "tokens_seen": 1379598336 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004703229689067202, + "loss": 1.2739, + "theoretical_loss": 3.570774717797322, + "tokens_seen": 1379663872 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047030692076228687, + "loss": 1.3336, + "theoretical_loss": 3.5707596434107445, + "tokens_seen": 1379729408 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004702908726178536, + "loss": 1.3077, + "theoretical_loss": 3.5707445699406453, + "tokens_seen": 1379794944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047027482447342027, + "loss": 1.2372, + "theoretical_loss": 3.5707294973869255, + "tokens_seen": 1379860480 + }, + { + "epoch": 0.42, + "learning_rate": 0.000470258776328987, + "loss": 1.3269, + "theoretical_loss": 3.570714425749486, + "tokens_seen": 1379926016 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047024272818455367, + "loss": 1.3211, + "theoretical_loss": 3.570699355028228, + "tokens_seen": 1379991552 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047022668004012043, + "loss": 1.3326, + "theoretical_loss": 3.570684285223052, + "tokens_seen": 1380057088 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004702106318956871, + "loss": 1.3618, + "theoretical_loss": 3.570669216333858, + "tokens_seen": 1380122624 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047019458375125383, + "loss": 1.2397, + "theoretical_loss": 3.570654148360548, + "tokens_seen": 1380188160 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004701785356068205, + "loss": 1.2833, + "theoretical_loss": 3.570639081303023, + "tokens_seen": 1380253696 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004701624874623872, + "loss": 1.299, + "theoretical_loss": 3.570624015161182, + "tokens_seen": 1380319232 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004701464393179539, + "loss": 1.2476, + "theoretical_loss": 3.570608949934928, + "tokens_seen": 1380384768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004701303911735206, + "loss": 1.3759, + "theoretical_loss": 3.5705938856241612, + "tokens_seen": 1380450304 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004701143430290873, + "loss": 1.2601, + "theoretical_loss": 3.570578822228782, + "tokens_seen": 1380515840 + }, + { + "epoch": 0.42, + "learning_rate": 0.000470098294884654, + "loss": 1.3817, + "theoretical_loss": 3.570563759748692, + "tokens_seen": 1380581376 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004700822467402207, + "loss": 1.3232, + "theoretical_loss": 3.5705486981837913, + "tokens_seen": 1380646912 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004700661985957874, + "loss": 1.2852, + "theoretical_loss": 3.5705336375339813, + "tokens_seen": 1380712448 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004700501504513541, + "loss": 1.4252, + "theoretical_loss": 3.570518577799164, + "tokens_seen": 1380777984 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004700341023069208, + "loss": 1.2892, + "theoretical_loss": 3.5705035189792387, + "tokens_seen": 1380843520 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047001805416248743, + "loss": 1.3714, + "theoretical_loss": 3.570488461074107, + "tokens_seen": 1380909056 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004700020060180542, + "loss": 1.38, + "theoretical_loss": 3.5704734040836703, + "tokens_seen": 1380974592 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046998595787362083, + "loss": 1.2231, + "theoretical_loss": 3.5704583480078296, + "tokens_seen": 1381040128 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004699699097291876, + "loss": 1.3391, + "theoretical_loss": 3.5704432928464858, + "tokens_seen": 1381105664 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046995386158475423, + "loss": 1.4116, + "theoretical_loss": 3.57042823859954, + "tokens_seen": 1381171200 + }, + { + "epoch": 0.42, + "learning_rate": 0.000469937813440321, + "loss": 1.2973, + "theoretical_loss": 3.5704131852668923, + "tokens_seen": 1381236736 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046992176529588763, + "loss": 1.2999, + "theoretical_loss": 3.570398132848445, + "tokens_seen": 1381302272 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004699057171514544, + "loss": 1.3806, + "theoretical_loss": 3.570383081344099, + "tokens_seen": 1381367808 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046988966900702103, + "loss": 1.32, + "theoretical_loss": 3.5703680307537553, + "tokens_seen": 1381433344 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004698736208625878, + "loss": 1.3567, + "theoretical_loss": 3.570352981077315, + "tokens_seen": 1381498880 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046985757271815443, + "loss": 1.3425, + "theoretical_loss": 3.57033793231468, + "tokens_seen": 1381564416 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004698415245737212, + "loss": 1.3485, + "theoretical_loss": 3.57032288446575, + "tokens_seen": 1381629952 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046982547642928784, + "loss": 1.2665, + "theoretical_loss": 3.5703078375304274, + "tokens_seen": 1381695488 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004698094282848546, + "loss": 1.3152, + "theoretical_loss": 3.5702927915086127, + "tokens_seen": 1381761024 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046979338014042124, + "loss": 1.427, + "theoretical_loss": 3.5702777464002073, + "tokens_seen": 1381826560 + }, + { + "epoch": 0.42, + "learning_rate": 0.000469777331995988, + "loss": 1.407, + "theoretical_loss": 3.5702627022051128, + "tokens_seen": 1381892096 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046976128385155464, + "loss": 1.3846, + "theoretical_loss": 3.57024765892323, + "tokens_seen": 1381957632 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004697452357071214, + "loss": 1.3046, + "theoretical_loss": 3.57023261655446, + "tokens_seen": 1382023168 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046972918756268804, + "loss": 1.3372, + "theoretical_loss": 3.570217575098705, + "tokens_seen": 1382088704 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004697131394182548, + "loss": 1.3188, + "theoretical_loss": 3.5702025345558654, + "tokens_seen": 1382154240 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046969709127382144, + "loss": 1.3798, + "theoretical_loss": 3.5701874949258423, + "tokens_seen": 1382219776 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004696810431293882, + "loss": 1.2812, + "theoretical_loss": 3.5701724562085384, + "tokens_seen": 1382285312 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046966499498495495, + "loss": 1.3829, + "theoretical_loss": 3.570157418403854, + "tokens_seen": 1382350848 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004696489468405216, + "loss": 1.4301, + "theoretical_loss": 3.5701423815116904, + "tokens_seen": 1382416384 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046963289869608835, + "loss": 1.3868, + "theoretical_loss": 3.570127345531949, + "tokens_seen": 1382481920 + }, + { + "epoch": 0.42, + "learning_rate": 0.000469616850551655, + "loss": 1.4893, + "theoretical_loss": 3.5701123104645314, + "tokens_seen": 1382547456 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046960080240722175, + "loss": 1.4028, + "theoretical_loss": 3.5700972763093395, + "tokens_seen": 1382612992 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004695847542627884, + "loss": 1.3122, + "theoretical_loss": 3.570082243066274, + "tokens_seen": 1382678528 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046956870611835515, + "loss": 1.432, + "theoretical_loss": 3.5700672107352363, + "tokens_seen": 1382744064 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 464319, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3647305965423584, + "objective/train/theoretical_loss": 3.570052179316128, + "objective/train/tokens_used": 1403269600, + "theoretical_loss": 3.570052179316128, + "tokens_seen": 1382809600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004695526579739218, + "loss": 1.3253, + "theoretical_loss": 3.570052179316128, + "tokens_seen": 1382809600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004695366098294885, + "loss": 1.3306, + "theoretical_loss": 3.570037148808851, + "tokens_seen": 1382875136 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004695205616850552, + "loss": 1.3643, + "theoretical_loss": 3.570022119213306, + "tokens_seen": 1382940672 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004695045135406219, + "loss": 1.304, + "theoretical_loss": 3.5700070905293955, + "tokens_seen": 1383006208 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004694884653961886, + "loss": 1.3723, + "theoretical_loss": 3.56999206275702, + "tokens_seen": 1383071744 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004694724172517553, + "loss": 1.3308, + "theoretical_loss": 3.5699770358960814, + "tokens_seen": 1383137280 + }, + { + "epoch": 0.42, + "learning_rate": 0.000469456369107322, + "loss": 1.3572, + "theoretical_loss": 3.5699620099464813, + "tokens_seen": 1383202816 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004694403209628887, + "loss": 1.2558, + "theoretical_loss": 3.569946984908122, + "tokens_seen": 1383268352 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004694242728184554, + "loss": 1.3347, + "theoretical_loss": 3.5699319607809032, + "tokens_seen": 1383333888 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004694082246740221, + "loss": 1.3705, + "theoretical_loss": 3.569916937564728, + "tokens_seen": 1383399424 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046939217652958875, + "loss": 1.3819, + "theoretical_loss": 3.569901915259498, + "tokens_seen": 1383464960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004693761283851555, + "loss": 1.3042, + "theoretical_loss": 3.569886893865114, + "tokens_seen": 1383530496 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046936008024072215, + "loss": 1.2829, + "theoretical_loss": 3.5698718733814783, + "tokens_seen": 1383596032 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004693440320962889, + "loss": 1.3505, + "theoretical_loss": 3.569856853808492, + "tokens_seen": 1383661568 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046932798395185555, + "loss": 1.375, + "theoretical_loss": 3.5698418351460575, + "tokens_seen": 1383727104 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004693119358074223, + "loss": 1.2966, + "theoretical_loss": 3.5698268173940755, + "tokens_seen": 1383792640 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046929588766298896, + "loss": 1.2349, + "theoretical_loss": 3.5698118005524484, + "tokens_seen": 1383858176 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004692798395185557, + "loss": 1.3181, + "theoretical_loss": 3.5697967846210776, + "tokens_seen": 1383923712 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046926379137412236, + "loss": 1.2792, + "theoretical_loss": 3.5697817695998655, + "tokens_seen": 1383989248 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004692477432296891, + "loss": 1.2963, + "theoretical_loss": 3.569766755488713, + "tokens_seen": 1384054784 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046923169508525576, + "loss": 1.3114, + "theoretical_loss": 3.569751742287522, + "tokens_seen": 1384120320 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004692156469408225, + "loss": 1.2276, + "theoretical_loss": 3.569736729996194, + "tokens_seen": 1384185856 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046919959879638916, + "loss": 1.3369, + "theoretical_loss": 3.5697217186146317, + "tokens_seen": 1384251392 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004691835506519559, + "loss": 1.4601, + "theoretical_loss": 3.5697067081427365, + "tokens_seen": 1384316928 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046916750250752256, + "loss": 1.3295, + "theoretical_loss": 3.56969169858041, + "tokens_seen": 1384382464 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004691514543630893, + "loss": 1.3331, + "theoretical_loss": 3.569676689927554, + "tokens_seen": 1384448000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046913540621865596, + "loss": 1.4236, + "theoretical_loss": 3.5696616821840705, + "tokens_seen": 1384513536 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004691193580742227, + "loss": 1.4035, + "theoretical_loss": 3.5696466753498615, + "tokens_seen": 1384579072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046910330992978936, + "loss": 1.3353, + "theoretical_loss": 3.5696316694248287, + "tokens_seen": 1384644608 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004690872617853561, + "loss": 1.3365, + "theoretical_loss": 3.5696166644088736, + "tokens_seen": 1384710144 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046907121364092276, + "loss": 1.3888, + "theoretical_loss": 3.569601660301899, + "tokens_seen": 1384775680 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004690551654964895, + "loss": 1.233, + "theoretical_loss": 3.569586657103806, + "tokens_seen": 1384841216 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046903911735205617, + "loss": 1.2804, + "theoretical_loss": 3.569571654814497, + "tokens_seen": 1384906752 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004690230692076229, + "loss": 1.349, + "theoretical_loss": 3.5695566534338736, + "tokens_seen": 1384972288 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046900702106318957, + "loss": 1.2633, + "theoretical_loss": 3.5695416529618385, + "tokens_seen": 1385037824 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004689909729187563, + "loss": 1.2326, + "theoretical_loss": 3.569526653398293, + "tokens_seen": 1385103360 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046897492477432297, + "loss": 1.3941, + "theoretical_loss": 3.569511654743139, + "tokens_seen": 1385168896 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004689588766298897, + "loss": 1.4688, + "theoretical_loss": 3.569496656996279, + "tokens_seen": 1385234432 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046894282848545637, + "loss": 1.3033, + "theoretical_loss": 3.569481660157615, + "tokens_seen": 1385299968 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004689267803410231, + "loss": 1.359, + "theoretical_loss": 3.5694666642270487, + "tokens_seen": 1385365504 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046891073219658977, + "loss": 1.3083, + "theoretical_loss": 3.569451669204482, + "tokens_seen": 1385431040 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004688946840521565, + "loss": 1.3679, + "theoretical_loss": 3.5694366750898174, + "tokens_seen": 1385496576 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046887863590772317, + "loss": 1.3626, + "theoretical_loss": 3.569421681882957, + "tokens_seen": 1385562112 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046886258776328993, + "loss": 1.3638, + "theoretical_loss": 3.569406689583803, + "tokens_seen": 1385627648 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004688465396188566, + "loss": 1.3713, + "theoretical_loss": 3.5693916981922564, + "tokens_seen": 1385693184 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046883049147442333, + "loss": 1.3344, + "theoretical_loss": 3.569376707708221, + "tokens_seen": 1385758720 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046881444332999, + "loss": 1.2535, + "theoretical_loss": 3.5693617181315984, + "tokens_seen": 1385824256 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046879839518555673, + "loss": 1.3254, + "theoretical_loss": 3.56934672946229, + "tokens_seen": 1385889792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004687823470411234, + "loss": 1.3007, + "theoretical_loss": 3.5693317417001986, + "tokens_seen": 1385955328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004687662988966901, + "loss": 1.3234, + "theoretical_loss": 3.5693167548452265, + "tokens_seen": 1386020864 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 465326, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.428898572921753, + "objective/train/theoretical_loss": 3.5693017688972755, + "objective/train/tokens_used": 1406546400, + "theoretical_loss": 3.5693017688972755, + "tokens_seen": 1386086400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004687502507522568, + "loss": 1.375, + "theoretical_loss": 3.5693017688972755, + "tokens_seen": 1386086400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004687342026078235, + "loss": 1.2942, + "theoretical_loss": 3.5692867838562483, + "tokens_seen": 1386151936 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004687181544633902, + "loss": 1.2628, + "theoretical_loss": 3.5692717997220464, + "tokens_seen": 1386217472 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004687021063189569, + "loss": 1.3732, + "theoretical_loss": 3.569256816494573, + "tokens_seen": 1386283008 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004686860581745236, + "loss": 1.2398, + "theoretical_loss": 3.56924183417373, + "tokens_seen": 1386348544 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004686700100300903, + "loss": 1.3177, + "theoretical_loss": 3.569226852759419, + "tokens_seen": 1386414080 + }, + { + "epoch": 0.42, + "learning_rate": 0.000468653961885657, + "loss": 1.2729, + "theoretical_loss": 3.569211872251543, + "tokens_seen": 1386479616 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004686379137412237, + "loss": 1.3639, + "theoretical_loss": 3.5691968926500053, + "tokens_seen": 1386545152 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046862186559679033, + "loss": 1.3352, + "theoretical_loss": 3.569181913954706, + "tokens_seen": 1386610688 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004686058174523571, + "loss": 1.3415, + "theoretical_loss": 3.5691669361655487, + "tokens_seen": 1386676224 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046858976930792373, + "loss": 1.3406, + "theoretical_loss": 3.5691519592824363, + "tokens_seen": 1386741760 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004685737211634905, + "loss": 1.2054, + "theoretical_loss": 3.5691369833052695, + "tokens_seen": 1386807296 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046855767301905713, + "loss": 1.348, + "theoretical_loss": 3.5691220082339523, + "tokens_seen": 1386872832 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004685416248746239, + "loss": 1.3887, + "theoretical_loss": 3.5691070340683866, + "tokens_seen": 1386938368 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046852557673019053, + "loss": 1.2267, + "theoretical_loss": 3.569092060808475, + "tokens_seen": 1387003904 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004685095285857573, + "loss": 1.3383, + "theoretical_loss": 3.569077088454119, + "tokens_seen": 1387069440 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046849348044132393, + "loss": 1.3754, + "theoretical_loss": 3.569062117005222, + "tokens_seen": 1387134976 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004684774322968907, + "loss": 1.4019, + "theoretical_loss": 3.5690471464616866, + "tokens_seen": 1387200512 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046846138415245744, + "loss": 1.3249, + "theoretical_loss": 3.5690321768234146, + "tokens_seen": 1387266048 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004684453360080241, + "loss": 1.3314, + "theoretical_loss": 3.5690172080903086, + "tokens_seen": 1387331584 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046842928786359084, + "loss": 1.3259, + "theoretical_loss": 3.5690022402622716, + "tokens_seen": 1387397120 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004684132397191575, + "loss": 1.3604, + "theoretical_loss": 3.5689872733392054, + "tokens_seen": 1387462656 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046839719157472425, + "loss": 1.2822, + "theoretical_loss": 3.568972307321013, + "tokens_seen": 1387528192 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004683811434302909, + "loss": 1.3639, + "theoretical_loss": 3.5689573422075975, + "tokens_seen": 1387593728 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046836509528585765, + "loss": 1.2465, + "theoretical_loss": 3.5689423779988605, + "tokens_seen": 1387659264 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004683490471414243, + "loss": 1.2601, + "theoretical_loss": 3.5689274146947048, + "tokens_seen": 1387724800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046833299899699105, + "loss": 1.3717, + "theoretical_loss": 3.5689124522950335, + "tokens_seen": 1387790336 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004683169508525577, + "loss": 1.2633, + "theoretical_loss": 3.568897490799749, + "tokens_seen": 1387855872 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046830090270812445, + "loss": 1.3051, + "theoretical_loss": 3.5688825302087537, + "tokens_seen": 1387921408 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004682848545636911, + "loss": 1.3714, + "theoretical_loss": 3.5688675705219497, + "tokens_seen": 1387986944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046826880641925785, + "loss": 1.2908, + "theoretical_loss": 3.568852611739241, + "tokens_seen": 1388052480 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004682527582748245, + "loss": 1.3092, + "theoretical_loss": 3.5688376538605295, + "tokens_seen": 1388118016 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046823671013039125, + "loss": 1.444, + "theoretical_loss": 3.5688226968857184, + "tokens_seen": 1388183552 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004682206619859579, + "loss": 1.2964, + "theoretical_loss": 3.5688077408147096, + "tokens_seen": 1388249088 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046820461384152465, + "loss": 1.2782, + "theoretical_loss": 3.5687927856474064, + "tokens_seen": 1388314624 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004681885656970913, + "loss": 1.4044, + "theoretical_loss": 3.568777831383711, + "tokens_seen": 1388380160 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046817251755265805, + "loss": 1.3039, + "theoretical_loss": 3.568762878023527, + "tokens_seen": 1388445696 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004681564694082247, + "loss": 1.3588, + "theoretical_loss": 3.5687479255667567, + "tokens_seen": 1388511232 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046814042126379146, + "loss": 1.2748, + "theoretical_loss": 3.5687329740133027, + "tokens_seen": 1388576768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004681243731193581, + "loss": 1.4446, + "theoretical_loss": 3.5687180233630684, + "tokens_seen": 1388642304 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004681083249749248, + "loss": 1.4675, + "theoretical_loss": 3.5687030736159557, + "tokens_seen": 1388707840 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004680922768304915, + "loss": 1.3458, + "theoretical_loss": 3.5686881247718683, + "tokens_seen": 1388773376 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004680762286860582, + "loss": 1.4207, + "theoretical_loss": 3.5686731768307087, + "tokens_seen": 1388838912 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004680601805416249, + "loss": 1.3286, + "theoretical_loss": 3.568658229792379, + "tokens_seen": 1388904448 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004680441323971916, + "loss": 1.3601, + "theoretical_loss": 3.5686432836567836, + "tokens_seen": 1388969984 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004680280842527583, + "loss": 1.316, + "theoretical_loss": 3.5686283384238244, + "tokens_seen": 1389035520 + }, + { + "epoch": 0.42, + "learning_rate": 0.000468012036108325, + "loss": 1.2939, + "theoretical_loss": 3.5686133940934046, + "tokens_seen": 1389101056 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046799598796389165, + "loss": 1.3304, + "theoretical_loss": 3.5685984506654265, + "tokens_seen": 1389166592 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004679799398194584, + "loss": 1.414, + "theoretical_loss": 3.568583508139794, + "tokens_seen": 1389232128 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046796389167502505, + "loss": 1.1521, + "theoretical_loss": 3.5685685665164097, + "tokens_seen": 1389297664 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 466536, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.399806261062622, + "objective/train/theoretical_loss": 3.568553625795176, + "objective/train/tokens_used": 1409823200, + "theoretical_loss": 3.568553625795176, + "tokens_seen": 1389363200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004679478435305918, + "loss": 1.4144, + "theoretical_loss": 3.568553625795176, + "tokens_seen": 1389363200 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046793179538615846, + "loss": 1.3563, + "theoretical_loss": 3.568538685975997, + "tokens_seen": 1389428736 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004679157472417252, + "loss": 1.2399, + "theoretical_loss": 3.5685237470587747, + "tokens_seen": 1389494272 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046789969909729186, + "loss": 1.2987, + "theoretical_loss": 3.5685088090434123, + "tokens_seen": 1389559808 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004678836509528586, + "loss": 1.2261, + "theoretical_loss": 3.5684938719298134, + "tokens_seen": 1389625344 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046786760280842526, + "loss": 1.301, + "theoretical_loss": 3.5684789357178803, + "tokens_seen": 1389690880 + }, + { + "epoch": 0.42, + "learning_rate": 0.000467851554663992, + "loss": 1.3076, + "theoretical_loss": 3.5684640004075163, + "tokens_seen": 1389756416 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046783550651955866, + "loss": 1.2074, + "theoretical_loss": 3.5684490659986245, + "tokens_seen": 1389821952 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004678194583751254, + "loss": 1.3984, + "theoretical_loss": 3.5684341324911086, + "tokens_seen": 1389887488 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046780341023069206, + "loss": 1.2291, + "theoretical_loss": 3.5684191998848704, + "tokens_seen": 1389953024 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004677873620862588, + "loss": 1.4563, + "theoretical_loss": 3.5684042681798145, + "tokens_seen": 1390018560 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046777131394182546, + "loss": 1.26, + "theoretical_loss": 3.5683893373758426, + "tokens_seen": 1390084096 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004677552657973922, + "loss": 1.289, + "theoretical_loss": 3.5683744074728585, + "tokens_seen": 1390149632 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046773921765295886, + "loss": 1.2825, + "theoretical_loss": 3.5683594784707657, + "tokens_seen": 1390215168 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004677231695085256, + "loss": 1.3856, + "theoretical_loss": 3.568344550369467, + "tokens_seen": 1390280704 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046770712136409226, + "loss": 1.448, + "theoretical_loss": 3.5683296231688653, + "tokens_seen": 1390346240 + }, + { + "epoch": 0.42, + "learning_rate": 0.000467691073219659, + "loss": 1.3966, + "theoretical_loss": 3.568314696868865, + "tokens_seen": 1390411776 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046767502507522567, + "loss": 1.4233, + "theoretical_loss": 3.5682997714693676, + "tokens_seen": 1390477312 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004676589769307924, + "loss": 1.3166, + "theoretical_loss": 3.5682848469702773, + "tokens_seen": 1390542848 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046764292878635907, + "loss": 1.2426, + "theoretical_loss": 3.5682699233714974, + "tokens_seen": 1390608384 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004676268806419258, + "loss": 1.2408, + "theoretical_loss": 3.568255000672931, + "tokens_seen": 1390673920 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046761083249749247, + "loss": 1.3111, + "theoretical_loss": 3.5682400788744815, + "tokens_seen": 1390739456 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004675947843530592, + "loss": 1.3709, + "theoretical_loss": 3.5682251579760518, + "tokens_seen": 1390804992 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046757873620862587, + "loss": 1.397, + "theoretical_loss": 3.5682102379775458, + "tokens_seen": 1390870528 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004675626880641926, + "loss": 1.3926, + "theoretical_loss": 3.5681953188788667, + "tokens_seen": 1390936064 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046754663991975927, + "loss": 1.2739, + "theoretical_loss": 3.568180400679917, + "tokens_seen": 1391001600 + }, + { + "epoch": 0.42, + "learning_rate": 0.000467530591775326, + "loss": 1.2746, + "theoretical_loss": 3.568165483380601, + "tokens_seen": 1391067136 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046751454363089267, + "loss": 1.3454, + "theoretical_loss": 3.568150566980822, + "tokens_seen": 1391132672 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004674984954864594, + "loss": 1.3187, + "theoretical_loss": 3.5681356514804827, + "tokens_seen": 1391198208 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004674824473420261, + "loss": 1.3364, + "theoretical_loss": 3.568120736879487, + "tokens_seen": 1391263744 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046746639919759283, + "loss": 1.3145, + "theoretical_loss": 3.5681058231777385, + "tokens_seen": 1391329280 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004674503510531595, + "loss": 1.504, + "theoretical_loss": 3.5680909103751404, + "tokens_seen": 1391394816 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046743430290872623, + "loss": 1.4819, + "theoretical_loss": 3.5680759984715964, + "tokens_seen": 1391460352 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004674182547642929, + "loss": 1.3309, + "theoretical_loss": 3.568061087467009, + "tokens_seen": 1391525888 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046740220661985963, + "loss": 1.3604, + "theoretical_loss": 3.568046177361283, + "tokens_seen": 1391591424 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004673861584754263, + "loss": 1.3768, + "theoretical_loss": 3.5680312681543205, + "tokens_seen": 1391656960 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046737011033099303, + "loss": 1.2238, + "theoretical_loss": 3.5680163598460264, + "tokens_seen": 1391722496 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004673540621865597, + "loss": 1.3065, + "theoretical_loss": 3.5680014524363033, + "tokens_seen": 1391788032 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004673380140421264, + "loss": 1.2669, + "theoretical_loss": 3.5679865459250553, + "tokens_seen": 1391853568 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004673219658976931, + "loss": 1.3582, + "theoretical_loss": 3.5679716403121855, + "tokens_seen": 1391919104 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004673059177532598, + "loss": 1.326, + "theoretical_loss": 3.5679567355975976, + "tokens_seen": 1391984640 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004672898696088265, + "loss": 1.3256, + "theoretical_loss": 3.567941831781195, + "tokens_seen": 1392050176 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004672738214643932, + "loss": 1.3533, + "theoretical_loss": 3.5679269288628817, + "tokens_seen": 1392115712 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046725777331995994, + "loss": 1.3419, + "theoretical_loss": 3.567912026842561, + "tokens_seen": 1392181248 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004672417251755266, + "loss": 1.3674, + "theoretical_loss": 3.567897125720137, + "tokens_seen": 1392246784 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046722567703109334, + "loss": 1.3898, + "theoretical_loss": 3.567882225495512, + "tokens_seen": 1392312320 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046720962888666, + "loss": 1.3017, + "theoretical_loss": 3.5678673261685914, + "tokens_seen": 1392377856 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046719358074222674, + "loss": 1.3046, + "theoretical_loss": 3.567852427739278, + "tokens_seen": 1392443392 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004671775325977934, + "loss": 1.3052, + "theoretical_loss": 3.567837530207475, + "tokens_seen": 1392508928 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046716148445336014, + "loss": 1.3724, + "theoretical_loss": 3.567822633573087, + "tokens_seen": 1392574464 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 467439, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3975262641906738, + "objective/train/theoretical_loss": 3.5678077378360173, + "objective/train/tokens_used": 1413100000, + "theoretical_loss": 3.5678077378360173, + "tokens_seen": 1392640000 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004671454363089268, + "loss": 1.3818, + "theoretical_loss": 3.5678077378360173, + "tokens_seen": 1392640000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046712938816449354, + "loss": 1.3425, + "theoretical_loss": 3.5677928429961705, + "tokens_seen": 1392705536 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004671133400200602, + "loss": 1.3975, + "theoretical_loss": 3.5677779490534487, + "tokens_seen": 1392771072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046709729187562694, + "loss": 1.3079, + "theoretical_loss": 3.567763056007756, + "tokens_seen": 1392836608 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004670812437311936, + "loss": 1.264, + "theoretical_loss": 3.5677481638589974, + "tokens_seen": 1392902144 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046706519558676034, + "loss": 1.2806, + "theoretical_loss": 3.567733272607076, + "tokens_seen": 1392967680 + }, + { + "epoch": 0.42, + "learning_rate": 0.000467049147442327, + "loss": 1.4315, + "theoretical_loss": 3.567718382251895, + "tokens_seen": 1393033216 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046703309929789375, + "loss": 1.3923, + "theoretical_loss": 3.5677034927933593, + "tokens_seen": 1393098752 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004670170511534604, + "loss": 1.3241, + "theoretical_loss": 3.567688604231372, + "tokens_seen": 1393164288 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046700100300902715, + "loss": 1.3409, + "theoretical_loss": 3.567673716565837, + "tokens_seen": 1393229824 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004669849548645938, + "loss": 1.2756, + "theoretical_loss": 3.5676588297966583, + "tokens_seen": 1393295360 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046696890672016055, + "loss": 1.3889, + "theoretical_loss": 3.5676439439237395, + "tokens_seen": 1393360896 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004669528585757272, + "loss": 1.4035, + "theoretical_loss": 3.5676290589469852, + "tokens_seen": 1393426432 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046693681043129395, + "loss": 1.2813, + "theoretical_loss": 3.567614174866299, + "tokens_seen": 1393491968 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004669207622868606, + "loss": 1.2742, + "theoretical_loss": 3.567599291681584, + "tokens_seen": 1393557504 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046690471414242735, + "loss": 1.3633, + "theoretical_loss": 3.5675844093927456, + "tokens_seen": 1393623040 + }, + { + "epoch": 0.42, + "learning_rate": 0.000466888665997994, + "loss": 1.2263, + "theoretical_loss": 3.5675695279996864, + "tokens_seen": 1393688576 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046687261785356075, + "loss": 1.4491, + "theoretical_loss": 3.5675546475023108, + "tokens_seen": 1393754112 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004668565697091274, + "loss": 1.344, + "theoretical_loss": 3.567539767900523, + "tokens_seen": 1393819648 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046684052156469415, + "loss": 1.4258, + "theoretical_loss": 3.5675248891942273, + "tokens_seen": 1393885184 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004668244734202608, + "loss": 1.357, + "theoretical_loss": 3.5675100113833267, + "tokens_seen": 1393950720 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046680842527582755, + "loss": 1.3401, + "theoretical_loss": 3.567495134467726, + "tokens_seen": 1394016256 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004667923771313942, + "loss": 1.3624, + "theoretical_loss": 3.5674802584473295, + "tokens_seen": 1394081792 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046677632898696096, + "loss": 1.4048, + "theoretical_loss": 3.56746538332204, + "tokens_seen": 1394147328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004667602808425276, + "loss": 1.36, + "theoretical_loss": 3.5674505090917634, + "tokens_seen": 1394212864 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046674423269809436, + "loss": 1.2604, + "theoretical_loss": 3.567435635756402, + "tokens_seen": 1394278400 + }, + { + "epoch": 0.42, + "learning_rate": 0.000466728184553661, + "loss": 1.3066, + "theoretical_loss": 3.567420763315861, + "tokens_seen": 1394343936 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004667121364092277, + "loss": 1.2258, + "theoretical_loss": 3.567405891770044, + "tokens_seen": 1394409472 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004666960882647944, + "loss": 1.2764, + "theoretical_loss": 3.5673910211188553, + "tokens_seen": 1394475008 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004666800401203611, + "loss": 1.2881, + "theoretical_loss": 3.567376151362199, + "tokens_seen": 1394540544 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004666639919759278, + "loss": 1.3034, + "theoretical_loss": 3.5673612824999794, + "tokens_seen": 1394606080 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004666479438314945, + "loss": 1.3499, + "theoretical_loss": 3.5673464145321003, + "tokens_seen": 1394671616 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004666318956870612, + "loss": 1.3134, + "theoretical_loss": 3.5673315474584664, + "tokens_seen": 1394737152 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004666158475426279, + "loss": 1.2493, + "theoretical_loss": 3.5673166812789816, + "tokens_seen": 1394802688 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004665997993981946, + "loss": 1.3878, + "theoretical_loss": 3.56730181599355, + "tokens_seen": 1394868224 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004665837512537613, + "loss": 1.346, + "theoretical_loss": 3.5672869516020764, + "tokens_seen": 1394933760 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046656770310932796, + "loss": 1.3826, + "theoretical_loss": 3.567272088104464, + "tokens_seen": 1394999296 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004665516549648947, + "loss": 1.3058, + "theoretical_loss": 3.567257225500618, + "tokens_seen": 1395064832 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046653560682046136, + "loss": 1.2699, + "theoretical_loss": 3.5672423637904425, + "tokens_seen": 1395130368 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004665195586760281, + "loss": 1.3409, + "theoretical_loss": 3.567227502973841, + "tokens_seen": 1395195904 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046650351053159476, + "loss": 1.4202, + "theoretical_loss": 3.5672126430507194, + "tokens_seen": 1395261440 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004664874623871615, + "loss": 1.3728, + "theoretical_loss": 3.5671977840209808, + "tokens_seen": 1395326976 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046647141424272816, + "loss": 1.321, + "theoretical_loss": 3.56718292588453, + "tokens_seen": 1395392512 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004664553660982949, + "loss": 1.402, + "theoretical_loss": 3.56716806864127, + "tokens_seen": 1395458048 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046643931795386156, + "loss": 1.2982, + "theoretical_loss": 3.5671532122911076, + "tokens_seen": 1395523584 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004664232698094283, + "loss": 1.3687, + "theoretical_loss": 3.5671383568339454, + "tokens_seen": 1395589120 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046640722166499496, + "loss": 1.3898, + "theoretical_loss": 3.567123502269688, + "tokens_seen": 1395654656 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004663911735205617, + "loss": 1.3348, + "theoretical_loss": 3.5671086485982406, + "tokens_seen": 1395720192 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046637512537612836, + "loss": 1.2986, + "theoretical_loss": 3.567093795819507, + "tokens_seen": 1395785728 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004663590772316951, + "loss": 1.2645, + "theoretical_loss": 3.5670789439333914, + "tokens_seen": 1395851264 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 468614, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.128334403038025, + "objective/train/theoretical_loss": 3.5670640929397988, + "objective/train/tokens_used": 1416376800, + "theoretical_loss": 3.5670640929397988, + "tokens_seen": 1395916800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046634302908726176, + "loss": 1.151, + "theoretical_loss": 3.5670640929397988, + "tokens_seen": 1395916800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004663269809428285, + "loss": 1.3527, + "theoretical_loss": 3.5670492428386336, + "tokens_seen": 1395982336 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046631093279839517, + "loss": 1.2882, + "theoretical_loss": 3.5670343936298, + "tokens_seen": 1396047872 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004662948846539619, + "loss": 1.2289, + "theoretical_loss": 3.5670195453132028, + "tokens_seen": 1396113408 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046627883650952857, + "loss": 1.2635, + "theoretical_loss": 3.5670046978887457, + "tokens_seen": 1396178944 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004662627883650953, + "loss": 1.3362, + "theoretical_loss": 3.5669898513563347, + "tokens_seen": 1396244480 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046624674022066197, + "loss": 1.3272, + "theoretical_loss": 3.5669750057158733, + "tokens_seen": 1396310016 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004662306920762287, + "loss": 1.3136, + "theoretical_loss": 3.566960160967266, + "tokens_seen": 1396375552 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046621464393179537, + "loss": 1.2719, + "theoretical_loss": 3.5669453171104175, + "tokens_seen": 1396441088 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004661985957873621, + "loss": 1.3208, + "theoretical_loss": 3.566930474145233, + "tokens_seen": 1396506624 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046618254764292877, + "loss": 1.3715, + "theoretical_loss": 3.566915632071616, + "tokens_seen": 1396572160 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004661664994984955, + "loss": 1.3914, + "theoretical_loss": 3.5669007908894725, + "tokens_seen": 1396637696 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046615045135406217, + "loss": 1.2028, + "theoretical_loss": 3.5668859505987056, + "tokens_seen": 1396703232 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004661344032096289, + "loss": 1.313, + "theoretical_loss": 3.5668711111992213, + "tokens_seen": 1396768768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004661183550651956, + "loss": 1.3542, + "theoretical_loss": 3.5668562726909236, + "tokens_seen": 1396834304 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046610230692076233, + "loss": 1.3529, + "theoretical_loss": 3.5668414350737168, + "tokens_seen": 1396899840 + }, + { + "epoch": 0.42, + "learning_rate": 0.000466086258776329, + "loss": 1.2057, + "theoretical_loss": 3.5668265983475065, + "tokens_seen": 1396965376 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046607021063189573, + "loss": 1.3048, + "theoretical_loss": 3.5668117625121964, + "tokens_seen": 1397030912 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046605416248746243, + "loss": 1.3327, + "theoretical_loss": 3.566796927567692, + "tokens_seen": 1397096448 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046603811434302913, + "loss": 1.2781, + "theoretical_loss": 3.566782093513898, + "tokens_seen": 1397161984 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046602206619859583, + "loss": 1.2973, + "theoretical_loss": 3.5667672603507183, + "tokens_seen": 1397227520 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046600601805416253, + "loss": 1.3553, + "theoretical_loss": 3.5667524280780585, + "tokens_seen": 1397293056 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046598996990972923, + "loss": 1.4168, + "theoretical_loss": 3.5667375966958232, + "tokens_seen": 1397358592 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046597392176529593, + "loss": 1.1694, + "theoretical_loss": 3.566722766203917, + "tokens_seen": 1397424128 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046595787362086263, + "loss": 1.2808, + "theoretical_loss": 3.5667079366022447, + "tokens_seen": 1397489664 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004659418254764293, + "loss": 1.4308, + "theoretical_loss": 3.5666931078907114, + "tokens_seen": 1397555200 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046592577733199603, + "loss": 1.42, + "theoretical_loss": 3.5666782800692216, + "tokens_seen": 1397620736 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004659097291875627, + "loss": 1.2872, + "theoretical_loss": 3.566663453137681, + "tokens_seen": 1397686272 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046589368104312944, + "loss": 1.4318, + "theoretical_loss": 3.566648627095993, + "tokens_seen": 1397751808 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004658776328986961, + "loss": 1.3049, + "theoretical_loss": 3.5666338019440635, + "tokens_seen": 1397817344 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046586158475426284, + "loss": 1.3829, + "theoretical_loss": 3.5666189776817974, + "tokens_seen": 1397882880 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004658455366098295, + "loss": 1.2541, + "theoretical_loss": 3.5666041543090987, + "tokens_seen": 1397948416 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046582948846539624, + "loss": 1.3511, + "theoretical_loss": 3.566589331825874, + "tokens_seen": 1398013952 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004658134403209629, + "loss": 1.3563, + "theoretical_loss": 3.566574510232026, + "tokens_seen": 1398079488 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046579739217652964, + "loss": 1.368, + "theoretical_loss": 3.5665596895274616, + "tokens_seen": 1398145024 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004657813440320963, + "loss": 1.3602, + "theoretical_loss": 3.566544869712085, + "tokens_seen": 1398210560 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046576529588766304, + "loss": 1.3254, + "theoretical_loss": 3.5665300507858007, + "tokens_seen": 1398276096 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004657492477432297, + "loss": 1.3215, + "theoretical_loss": 3.5665152327485146, + "tokens_seen": 1398341632 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046573319959879644, + "loss": 1.3215, + "theoretical_loss": 3.5665004156001308, + "tokens_seen": 1398407168 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004657171514543631, + "loss": 1.3927, + "theoretical_loss": 3.566485599340555, + "tokens_seen": 1398472704 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046570110330992984, + "loss": 1.3609, + "theoretical_loss": 3.5664707839696925, + "tokens_seen": 1398538240 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004656850551654965, + "loss": 1.3725, + "theoretical_loss": 3.566455969487448, + "tokens_seen": 1398603776 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046566900702106325, + "loss": 1.252, + "theoretical_loss": 3.5664411558937257, + "tokens_seen": 1398669312 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004656529588766299, + "loss": 1.3539, + "theoretical_loss": 3.566426343188432, + "tokens_seen": 1398734848 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046563691073219665, + "loss": 1.3519, + "theoretical_loss": 3.5664115313714713, + "tokens_seen": 1398800384 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004656208625877633, + "loss": 1.2448, + "theoretical_loss": 3.566396720442749, + "tokens_seen": 1398865920 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046560481444333005, + "loss": 1.3535, + "theoretical_loss": 3.5663819104021695, + "tokens_seen": 1398931456 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004655887662988967, + "loss": 1.3861, + "theoretical_loss": 3.5663671012496394, + "tokens_seen": 1398996992 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046557271815446345, + "loss": 1.433, + "theoretical_loss": 3.566352292985062, + "tokens_seen": 1399062528 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004655566700100301, + "loss": 1.3061, + "theoretical_loss": 3.5663374856083445, + "tokens_seen": 1399128064 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 469650, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3298465013504028, + "objective/train/theoretical_loss": 3.5663226791193905, + "objective/train/tokens_used": 1419653600, + "theoretical_loss": 3.5663226791193905, + "tokens_seen": 1399193600 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046554062186559685, + "loss": 1.2899, + "theoretical_loss": 3.5663226791193905, + "tokens_seen": 1399193600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004655245737211635, + "loss": 1.3579, + "theoretical_loss": 3.566307873518105, + "tokens_seen": 1399259136 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046550852557673025, + "loss": 1.4427, + "theoretical_loss": 3.566293068804395, + "tokens_seen": 1399324672 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004654924774322969, + "loss": 1.3123, + "theoretical_loss": 3.5662782649781644, + "tokens_seen": 1399390208 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046547642928786365, + "loss": 1.2523, + "theoretical_loss": 3.566263462039318, + "tokens_seen": 1399455744 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004654603811434303, + "loss": 1.3049, + "theoretical_loss": 3.5662486599877625, + "tokens_seen": 1399521280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046544433299899705, + "loss": 1.3442, + "theoretical_loss": 3.566233858823402, + "tokens_seen": 1399586816 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004654282848545637, + "loss": 1.2045, + "theoretical_loss": 3.566219058546143, + "tokens_seen": 1399652352 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046541223671013046, + "loss": 1.301, + "theoretical_loss": 3.566204259155889, + "tokens_seen": 1399717888 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004653961885656971, + "loss": 1.4409, + "theoretical_loss": 3.566189460652547, + "tokens_seen": 1399783424 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046538014042126386, + "loss": 1.3064, + "theoretical_loss": 3.5661746630360214, + "tokens_seen": 1399848960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004653640922768305, + "loss": 1.2533, + "theoretical_loss": 3.5661598663062177, + "tokens_seen": 1399914496 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046534804413239726, + "loss": 1.3432, + "theoretical_loss": 3.5661450704630413, + "tokens_seen": 1399980032 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004653319959879639, + "loss": 1.3104, + "theoretical_loss": 3.5661302755063975, + "tokens_seen": 1400045568 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004653159478435306, + "loss": 1.3057, + "theoretical_loss": 3.566115481436192, + "tokens_seen": 1400111104 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004652998996990973, + "loss": 1.3049, + "theoretical_loss": 3.5661006882523303, + "tokens_seen": 1400176640 + }, + { + "epoch": 0.42, + "learning_rate": 0.000465283851554664, + "loss": 1.333, + "theoretical_loss": 3.5660858959547173, + "tokens_seen": 1400242176 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004652678034102307, + "loss": 1.3225, + "theoretical_loss": 3.5660711045432585, + "tokens_seen": 1400307712 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004652517552657974, + "loss": 1.3437, + "theoretical_loss": 3.56605631401786, + "tokens_seen": 1400373248 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004652357071213641, + "loss": 1.2676, + "theoretical_loss": 3.5660415243784263, + "tokens_seen": 1400438784 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004652196589769308, + "loss": 1.3211, + "theoretical_loss": 3.566026735624863, + "tokens_seen": 1400504320 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004652036108324975, + "loss": 1.2517, + "theoretical_loss": 3.5660119477570764, + "tokens_seen": 1400569856 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004651875626880642, + "loss": 1.3758, + "theoretical_loss": 3.5659971607749714, + "tokens_seen": 1400635392 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046517151454363086, + "loss": 1.3239, + "theoretical_loss": 3.5659823746784536, + "tokens_seen": 1400700928 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004651554663991976, + "loss": 1.2681, + "theoretical_loss": 3.5659675894674288, + "tokens_seen": 1400766464 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046513941825476426, + "loss": 1.3821, + "theoretical_loss": 3.565952805141802, + "tokens_seen": 1400832000 + }, + { + "epoch": 0.42, + "learning_rate": 0.000465123370110331, + "loss": 1.4107, + "theoretical_loss": 3.565938021701479, + "tokens_seen": 1400897536 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046510732196589766, + "loss": 1.3933, + "theoretical_loss": 3.5659232391463656, + "tokens_seen": 1400963072 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004650912738214644, + "loss": 1.2784, + "theoretical_loss": 3.5659084574763673, + "tokens_seen": 1401028608 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046507522567703106, + "loss": 1.2199, + "theoretical_loss": 3.56589367669139, + "tokens_seen": 1401094144 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004650591775325978, + "loss": 1.3609, + "theoretical_loss": 3.565878896791338, + "tokens_seen": 1401159680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046504312938816446, + "loss": 1.4024, + "theoretical_loss": 3.5658641177761186, + "tokens_seen": 1401225216 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004650270812437312, + "loss": 1.2853, + "theoretical_loss": 3.5658493396456366, + "tokens_seen": 1401290752 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046501103309929786, + "loss": 1.3588, + "theoretical_loss": 3.5658345623997976, + "tokens_seen": 1401356288 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004649949849548646, + "loss": 1.3526, + "theoretical_loss": 3.565819786038508, + "tokens_seen": 1401421824 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046497893681043126, + "loss": 1.3236, + "theoretical_loss": 3.5658050105616725, + "tokens_seen": 1401487360 + }, + { + "epoch": 0.42, + "learning_rate": 0.000464962888665998, + "loss": 1.3269, + "theoretical_loss": 3.5657902359691978, + "tokens_seen": 1401552896 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046494684052156467, + "loss": 1.4531, + "theoretical_loss": 3.565775462260988, + "tokens_seen": 1401618432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004649307923771314, + "loss": 1.3345, + "theoretical_loss": 3.565760689436951, + "tokens_seen": 1401683968 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046491474423269807, + "loss": 1.3002, + "theoretical_loss": 3.565745917496991, + "tokens_seen": 1401749504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004648986960882648, + "loss": 1.2906, + "theoretical_loss": 3.5657311464410144, + "tokens_seen": 1401815040 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046488264794383147, + "loss": 1.3056, + "theoretical_loss": 3.5657163762689263, + "tokens_seen": 1401880576 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004648665997993982, + "loss": 1.243, + "theoretical_loss": 3.5657016069806335, + "tokens_seen": 1401946112 + }, + { + "epoch": 0.42, + "learning_rate": 0.000464850551654965, + "loss": 1.2916, + "theoretical_loss": 3.565686838576041, + "tokens_seen": 1402011648 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004648345035105316, + "loss": 1.4426, + "theoretical_loss": 3.5656720710550553, + "tokens_seen": 1402077184 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004648184553660984, + "loss": 1.3129, + "theoretical_loss": 3.5656573044175817, + "tokens_seen": 1402142720 + }, + { + "epoch": 0.42, + "learning_rate": 0.000464802407221665, + "loss": 1.2652, + "theoretical_loss": 3.5656425386635267, + "tokens_seen": 1402208256 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004647863590772318, + "loss": 1.5096, + "theoretical_loss": 3.565627773792795, + "tokens_seen": 1402273792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004647703109327984, + "loss": 1.2868, + "theoretical_loss": 3.5656130098052934, + "tokens_seen": 1402339328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004647542627883652, + "loss": 1.2618, + "theoretical_loss": 3.5655982467009277, + "tokens_seen": 1402404864 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 470842, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3026866912841797, + "objective/train/theoretical_loss": 3.5655834844796033, + "objective/train/tokens_used": 1422930400, + "theoretical_loss": 3.5655834844796033, + "tokens_seen": 1402470400 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046473821464393183, + "loss": 1.3822, + "theoretical_loss": 3.5655834844796033, + "tokens_seen": 1402470400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004647221664994986, + "loss": 1.3368, + "theoretical_loss": 3.5655687231412267, + "tokens_seen": 1402535936 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046470611835506523, + "loss": 1.346, + "theoretical_loss": 3.5655539626857036, + "tokens_seen": 1402601472 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046469007021063193, + "loss": 1.2459, + "theoretical_loss": 3.56553920311294, + "tokens_seen": 1402667008 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046467402206619863, + "loss": 1.2949, + "theoretical_loss": 3.5655244444228416, + "tokens_seen": 1402732544 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046465797392176533, + "loss": 1.3743, + "theoretical_loss": 3.5655096866153144, + "tokens_seen": 1402798080 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046464192577733203, + "loss": 1.3485, + "theoretical_loss": 3.5654949296902654, + "tokens_seen": 1402863616 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046462587763289873, + "loss": 1.3674, + "theoretical_loss": 3.5654801736475994, + "tokens_seen": 1402929152 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046460982948846543, + "loss": 1.2972, + "theoretical_loss": 3.5654654184872228, + "tokens_seen": 1402994688 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046459378134403213, + "loss": 1.3483, + "theoretical_loss": 3.5654506642090418, + "tokens_seen": 1403060224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046457773319959883, + "loss": 1.2416, + "theoretical_loss": 3.565435910812962, + "tokens_seen": 1403125760 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046456168505516553, + "loss": 1.3794, + "theoretical_loss": 3.56542115829889, + "tokens_seen": 1403191296 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004645456369107322, + "loss": 1.3798, + "theoretical_loss": 3.5654064066667317, + "tokens_seen": 1403256832 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046452958876629894, + "loss": 1.3422, + "theoretical_loss": 3.565391655916393, + "tokens_seen": 1403322368 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004645135406218656, + "loss": 1.2489, + "theoretical_loss": 3.5653769060477805, + "tokens_seen": 1403387904 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046449749247743234, + "loss": 1.2822, + "theoretical_loss": 3.5653621570608003, + "tokens_seen": 1403453440 + }, + { + "epoch": 0.43, + "learning_rate": 0.000464481444332999, + "loss": 1.3095, + "theoretical_loss": 3.5653474089553576, + "tokens_seen": 1403518976 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046446539618856574, + "loss": 1.2985, + "theoretical_loss": 3.565332661731359, + "tokens_seen": 1403584512 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004644493480441324, + "loss": 1.2564, + "theoretical_loss": 3.565317915388712, + "tokens_seen": 1403650048 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046443329989969914, + "loss": 1.2824, + "theoretical_loss": 3.5653031699273203, + "tokens_seen": 1403715584 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004644172517552658, + "loss": 1.2888, + "theoretical_loss": 3.565288425347092, + "tokens_seen": 1403781120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046440120361083254, + "loss": 1.2191, + "theoretical_loss": 3.5652736816479327, + "tokens_seen": 1403846656 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004643851554663992, + "loss": 1.3087, + "theoretical_loss": 3.5652589388297486, + "tokens_seen": 1403912192 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046436910732196594, + "loss": 1.2847, + "theoretical_loss": 3.565244196892446, + "tokens_seen": 1403977728 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004643530591775326, + "loss": 1.2152, + "theoretical_loss": 3.565229455835931, + "tokens_seen": 1404043264 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046433701103309934, + "loss": 1.2608, + "theoretical_loss": 3.565214715660111, + "tokens_seen": 1404108800 + }, + { + "epoch": 0.43, + "learning_rate": 0.000464320962888666, + "loss": 1.3817, + "theoretical_loss": 3.56519997636489, + "tokens_seen": 1404174336 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046430491474423274, + "loss": 1.2184, + "theoretical_loss": 3.5651852379501765, + "tokens_seen": 1404239872 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004642888665997994, + "loss": 1.3529, + "theoretical_loss": 3.5651705004158756, + "tokens_seen": 1404305408 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046427281845536615, + "loss": 1.2785, + "theoretical_loss": 3.5651557637618936, + "tokens_seen": 1404370944 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004642567703109328, + "loss": 1.2324, + "theoretical_loss": 3.5651410279881377, + "tokens_seen": 1404436480 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046424072216649955, + "loss": 1.329, + "theoretical_loss": 3.5651262930945133, + "tokens_seen": 1404502016 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004642246740220662, + "loss": 1.3064, + "theoretical_loss": 3.5651115590809273, + "tokens_seen": 1404567552 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046420862587763295, + "loss": 1.2996, + "theoretical_loss": 3.565096825947286, + "tokens_seen": 1404633088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004641925777331996, + "loss": 1.297, + "theoretical_loss": 3.565082093693496, + "tokens_seen": 1404698624 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046417652958876635, + "loss": 1.3461, + "theoretical_loss": 3.5650673623194633, + "tokens_seen": 1404764160 + }, + { + "epoch": 0.43, + "learning_rate": 0.000464160481444333, + "loss": 1.366, + "theoretical_loss": 3.5650526318250946, + "tokens_seen": 1404829696 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046414443329989975, + "loss": 1.4017, + "theoretical_loss": 3.565037902210296, + "tokens_seen": 1404895232 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004641283851554664, + "loss": 1.1948, + "theoretical_loss": 3.5650231734749744, + "tokens_seen": 1404960768 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046411233701103315, + "loss": 1.3518, + "theoretical_loss": 3.565008445619036, + "tokens_seen": 1405026304 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004640962888665998, + "loss": 1.2568, + "theoretical_loss": 3.5649937186423877, + "tokens_seen": 1405091840 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046408024072216655, + "loss": 1.3657, + "theoretical_loss": 3.564978992544935, + "tokens_seen": 1405157376 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004640641925777332, + "loss": 1.4002, + "theoretical_loss": 3.5649642673265856, + "tokens_seen": 1405222912 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046404814443329996, + "loss": 1.2962, + "theoretical_loss": 3.564949542987245, + "tokens_seen": 1405288448 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004640320962888666, + "loss": 1.3722, + "theoretical_loss": 3.5649348195268207, + "tokens_seen": 1405353984 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046401604814443336, + "loss": 1.2351, + "theoretical_loss": 3.564920096945218, + "tokens_seen": 1405419520 + }, + { + "epoch": 0.43, + "learning_rate": 0.000464, + "loss": 1.328, + "theoretical_loss": 3.564905375242345, + "tokens_seen": 1405485056 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046398395185556676, + "loss": 1.3564, + "theoretical_loss": 3.5648906544181074, + "tokens_seen": 1405550592 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004639679037111334, + "loss": 1.2927, + "theoretical_loss": 3.5648759344724117, + "tokens_seen": 1405616128 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046395185556670016, + "loss": 1.2815, + "theoretical_loss": 3.5648612154051644, + "tokens_seen": 1405681664 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 471752, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.491056203842163, + "objective/train/theoretical_loss": 3.564846497216273, + "objective/train/tokens_used": 1426207200, + "theoretical_loss": 3.564846497216273, + "tokens_seen": 1405747200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004639358074222668, + "loss": 1.3665, + "theoretical_loss": 3.564846497216273, + "tokens_seen": 1405747200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004639197592778335, + "loss": 1.3182, + "theoretical_loss": 3.564831779905644, + "tokens_seen": 1405812736 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004639037111334002, + "loss": 1.3252, + "theoretical_loss": 3.564817063473183, + "tokens_seen": 1405878272 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004638876629889669, + "loss": 1.2465, + "theoretical_loss": 3.564802347918797, + "tokens_seen": 1405943808 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004638716148445336, + "loss": 1.3465, + "theoretical_loss": 3.5647876332423936, + "tokens_seen": 1406009344 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004638555667001003, + "loss": 1.2294, + "theoretical_loss": 3.564772919443879, + "tokens_seen": 1406074880 + }, + { + "epoch": 0.43, + "learning_rate": 0.000463839518555667, + "loss": 1.2853, + "theoretical_loss": 3.564758206523159, + "tokens_seen": 1406140416 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004638234704112337, + "loss": 1.3808, + "theoretical_loss": 3.5647434944801417, + "tokens_seen": 1406205952 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004638074222668004, + "loss": 1.2815, + "theoretical_loss": 3.564728783314733, + "tokens_seen": 1406271488 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004637913741223671, + "loss": 1.2508, + "theoretical_loss": 3.56471407302684, + "tokens_seen": 1406337024 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046377532597793376, + "loss": 1.3086, + "theoretical_loss": 3.56469936361637, + "tokens_seen": 1406402560 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004637592778335005, + "loss": 1.309, + "theoretical_loss": 3.5646846550832283, + "tokens_seen": 1406468096 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046374322968906716, + "loss": 1.3229, + "theoretical_loss": 3.5646699474273227, + "tokens_seen": 1406533632 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004637271815446339, + "loss": 1.2341, + "theoretical_loss": 3.5646552406485608, + "tokens_seen": 1406599168 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046371113340020056, + "loss": 1.3117, + "theoretical_loss": 3.5646405347468475, + "tokens_seen": 1406664704 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004636950852557673, + "loss": 1.3195, + "theoretical_loss": 3.5646258297220905, + "tokens_seen": 1406730240 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046367903711133396, + "loss": 1.3438, + "theoretical_loss": 3.5646111255741975, + "tokens_seen": 1406795776 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004636629889669007, + "loss": 1.2993, + "theoretical_loss": 3.564596422303074, + "tokens_seen": 1406861312 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046364694082246747, + "loss": 1.3483, + "theoretical_loss": 3.5645817199086283, + "tokens_seen": 1406926848 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004636308926780341, + "loss": 1.3506, + "theoretical_loss": 3.5645670183907665, + "tokens_seen": 1406992384 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046361484453360087, + "loss": 1.2594, + "theoretical_loss": 3.5645523177493947, + "tokens_seen": 1407057920 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004635987963891675, + "loss": 1.2768, + "theoretical_loss": 3.5645376179844215, + "tokens_seen": 1407123456 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004635827482447343, + "loss": 1.3181, + "theoretical_loss": 3.5645229190957526, + "tokens_seen": 1407188992 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004635667001003009, + "loss": 1.3721, + "theoretical_loss": 3.5645082210832952, + "tokens_seen": 1407254528 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004635506519558677, + "loss": 1.3782, + "theoretical_loss": 3.5644935239469566, + "tokens_seen": 1407320064 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004635346038114343, + "loss": 1.2865, + "theoretical_loss": 3.564478827686644, + "tokens_seen": 1407385600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004635185556670011, + "loss": 1.4146, + "theoretical_loss": 3.5644641323022634, + "tokens_seen": 1407451136 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004635025075225677, + "loss": 1.3308, + "theoretical_loss": 3.564449437793723, + "tokens_seen": 1407516672 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004634864593781345, + "loss": 1.2313, + "theoretical_loss": 3.5644347441609288, + "tokens_seen": 1407582208 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004634704112337011, + "loss": 1.2939, + "theoretical_loss": 3.564420051403788, + "tokens_seen": 1407647744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004634543630892679, + "loss": 1.3033, + "theoretical_loss": 3.5644053595222083, + "tokens_seen": 1407713280 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004634383149448345, + "loss": 1.3209, + "theoretical_loss": 3.564390668516096, + "tokens_seen": 1407778816 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004634222668004013, + "loss": 1.4386, + "theoretical_loss": 3.5643759783853595, + "tokens_seen": 1407844352 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004634062186559679, + "loss": 1.3385, + "theoretical_loss": 3.564361289129904, + "tokens_seen": 1407909888 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004633901705115347, + "loss": 1.3646, + "theoretical_loss": 3.564346600749638, + "tokens_seen": 1407975424 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046337412236710133, + "loss": 1.3764, + "theoretical_loss": 3.5643319132444686, + "tokens_seen": 1408040960 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004633580742226681, + "loss": 1.2875, + "theoretical_loss": 3.564317226614302, + "tokens_seen": 1408106496 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046334202607823473, + "loss": 1.1642, + "theoretical_loss": 3.564302540859046, + "tokens_seen": 1408172032 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004633259779338015, + "loss": 1.4003, + "theoretical_loss": 3.5642878559786073, + "tokens_seen": 1408237568 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046330992978936813, + "loss": 1.3716, + "theoretical_loss": 3.5642731719728937, + "tokens_seen": 1408303104 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046329388164493483, + "loss": 1.2402, + "theoretical_loss": 3.5642584888418125, + "tokens_seen": 1408368640 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046327783350050153, + "loss": 1.3014, + "theoretical_loss": 3.5642438065852704, + "tokens_seen": 1408434176 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046326178535606823, + "loss": 1.2789, + "theoretical_loss": 3.5642291252031746, + "tokens_seen": 1408499712 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046324573721163493, + "loss": 1.3755, + "theoretical_loss": 3.5642144446954322, + "tokens_seen": 1408565248 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046322968906720163, + "loss": 1.2905, + "theoretical_loss": 3.564199765061951, + "tokens_seen": 1408630784 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046321364092276833, + "loss": 1.4412, + "theoretical_loss": 3.564185086302638, + "tokens_seen": 1408696320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046319759277833503, + "loss": 1.2412, + "theoretical_loss": 3.564170408417401, + "tokens_seen": 1408761856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046318154463390174, + "loss": 1.2649, + "theoretical_loss": 3.564155731406146, + "tokens_seen": 1408827392 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046316549648946844, + "loss": 1.3384, + "theoretical_loss": 3.5641410552687813, + "tokens_seen": 1408892928 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004631494483450351, + "loss": 1.2401, + "theoretical_loss": 3.564126380005214, + "tokens_seen": 1408958464 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 473021, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3816249370574951, + "objective/train/theoretical_loss": 3.5641117056153515, + "objective/train/tokens_used": 1429484000, + "theoretical_loss": 3.5641117056153515, + "tokens_seen": 1409024000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046313340020060184, + "loss": 1.4074, + "theoretical_loss": 3.5641117056153515, + "tokens_seen": 1409024000 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004631173520561685, + "loss": 1.3027, + "theoretical_loss": 3.5640970320991014, + "tokens_seen": 1409089536 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046310130391173524, + "loss": 1.3182, + "theoretical_loss": 3.5640823594563704, + "tokens_seen": 1409155072 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004630852557673019, + "loss": 1.225, + "theoretical_loss": 3.564067687687066, + "tokens_seen": 1409220608 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046306920762286864, + "loss": 1.2488, + "theoretical_loss": 3.564053016791096, + "tokens_seen": 1409286144 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004630531594784353, + "loss": 1.3409, + "theoretical_loss": 3.564038346768368, + "tokens_seen": 1409351680 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046303711133400204, + "loss": 1.2559, + "theoretical_loss": 3.564023677618789, + "tokens_seen": 1409417216 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004630210631895687, + "loss": 1.3289, + "theoretical_loss": 3.564009009342266, + "tokens_seen": 1409482752 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046300501504513544, + "loss": 1.2364, + "theoretical_loss": 3.5639943419387072, + "tokens_seen": 1409548288 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004629889669007021, + "loss": 1.3942, + "theoretical_loss": 3.56397967540802, + "tokens_seen": 1409613824 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046297291875626884, + "loss": 1.4127, + "theoretical_loss": 3.5639650097501114, + "tokens_seen": 1409679360 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004629568706118355, + "loss": 1.3688, + "theoretical_loss": 3.5639503449648893, + "tokens_seen": 1409744896 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046294082246740224, + "loss": 1.3236, + "theoretical_loss": 3.5639356810522607, + "tokens_seen": 1409810432 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004629247743229689, + "loss": 1.1882, + "theoretical_loss": 3.5639210180121337, + "tokens_seen": 1409875968 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046290872617853565, + "loss": 1.2427, + "theoretical_loss": 3.5639063558444155, + "tokens_seen": 1409941504 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004628926780341023, + "loss": 1.364, + "theoretical_loss": 3.563891694549014, + "tokens_seen": 1410007040 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046287662988966905, + "loss": 1.3528, + "theoretical_loss": 3.563877034125837, + "tokens_seen": 1410072576 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004628605817452357, + "loss": 1.3439, + "theoretical_loss": 3.563862374574791, + "tokens_seen": 1410138112 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046284453360080245, + "loss": 1.3109, + "theoretical_loss": 3.5638477158957844, + "tokens_seen": 1410203648 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004628284854563691, + "loss": 1.3087, + "theoretical_loss": 3.5638330580887243, + "tokens_seen": 1410269184 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046281243731193585, + "loss": 1.4207, + "theoretical_loss": 3.5638184011535188, + "tokens_seen": 1410334720 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004627963891675025, + "loss": 1.3335, + "theoretical_loss": 3.563803745090075, + "tokens_seen": 1410400256 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046278034102306925, + "loss": 1.328, + "theoretical_loss": 3.563789089898301, + "tokens_seen": 1410465792 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004627642928786359, + "loss": 1.2656, + "theoretical_loss": 3.5637744355781047, + "tokens_seen": 1410531328 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046274824473420265, + "loss": 1.3268, + "theoretical_loss": 3.5637597821293934, + "tokens_seen": 1410596864 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004627321965897693, + "loss": 1.1607, + "theoretical_loss": 3.563745129552075, + "tokens_seen": 1410662400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046271614844533605, + "loss": 1.3328, + "theoretical_loss": 3.5637304778460566, + "tokens_seen": 1410727936 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004627001003009027, + "loss": 1.3885, + "theoretical_loss": 3.5637158270112463, + "tokens_seen": 1410793472 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046268405215646945, + "loss": 1.2589, + "theoretical_loss": 3.5637011770475517, + "tokens_seen": 1410859008 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004626680040120361, + "loss": 1.3258, + "theoretical_loss": 3.5636865279548813, + "tokens_seen": 1410924544 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046265195586760286, + "loss": 1.3359, + "theoretical_loss": 3.5636718797331417, + "tokens_seen": 1410990080 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004626359077231695, + "loss": 1.2994, + "theoretical_loss": 3.5636572323822415, + "tokens_seen": 1411055616 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046261985957873626, + "loss": 1.4495, + "theoretical_loss": 3.5636425859020884, + "tokens_seen": 1411121152 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004626038114343029, + "loss": 1.3747, + "theoretical_loss": 3.56362794029259, + "tokens_seen": 1411186688 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046258776328986966, + "loss": 1.2321, + "theoretical_loss": 3.5636132955536537, + "tokens_seen": 1411252224 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004625717151454363, + "loss": 1.3645, + "theoretical_loss": 3.5635986516851874, + "tokens_seen": 1411317760 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046255566700100306, + "loss": 1.301, + "theoretical_loss": 3.5635840086871005, + "tokens_seen": 1411383296 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004625396188565697, + "loss": 1.3656, + "theoretical_loss": 3.5635693665592987, + "tokens_seen": 1411448832 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004625235707121364, + "loss": 1.3231, + "theoretical_loss": 3.5635547253016915, + "tokens_seen": 1411514368 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004625075225677031, + "loss": 1.3584, + "theoretical_loss": 3.5635400849141856, + "tokens_seen": 1411579904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004624914744232698, + "loss": 1.2762, + "theoretical_loss": 3.5635254453966896, + "tokens_seen": 1411645440 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004624754262788365, + "loss": 1.4195, + "theoretical_loss": 3.563510806749111, + "tokens_seen": 1411710976 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004624593781344032, + "loss": 1.3402, + "theoretical_loss": 3.5634961689713585, + "tokens_seen": 1411776512 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046244332998996996, + "loss": 1.1941, + "theoretical_loss": 3.563481532063339, + "tokens_seen": 1411842048 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004624272818455366, + "loss": 1.2568, + "theoretical_loss": 3.563466896024961, + "tokens_seen": 1411907584 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046241123370110337, + "loss": 1.2525, + "theoretical_loss": 3.563452260856133, + "tokens_seen": 1411973120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046239518555667, + "loss": 1.3847, + "theoretical_loss": 3.5634376265567616, + "tokens_seen": 1412038656 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046237913741223677, + "loss": 1.3101, + "theoretical_loss": 3.563422993126756, + "tokens_seen": 1412104192 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004623630892678034, + "loss": 1.3066, + "theoretical_loss": 3.5634083605660236, + "tokens_seen": 1412169728 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046234704112337017, + "loss": 1.3575, + "theoretical_loss": 3.563393728874473, + "tokens_seen": 1412235264 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 474225, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4415783882141113, + "objective/train/theoretical_loss": 3.5633790980520113, + "objective/train/tokens_used": 1432760800, + "theoretical_loss": 3.5633790980520113, + "tokens_seen": 1412300800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004623309929789368, + "loss": 1.4305, + "theoretical_loss": 3.5633790980520113, + "tokens_seen": 1412300800 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046231494483450357, + "loss": 1.3028, + "theoretical_loss": 3.5633644680985475, + "tokens_seen": 1412366336 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004622988966900702, + "loss": 1.2806, + "theoretical_loss": 3.5633498390139895, + "tokens_seen": 1412431872 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046228284854563697, + "loss": 1.2758, + "theoretical_loss": 3.563335210798245, + "tokens_seen": 1412497408 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004622668004012036, + "loss": 1.2755, + "theoretical_loss": 3.563320583451222, + "tokens_seen": 1412562944 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046225075225677037, + "loss": 1.3708, + "theoretical_loss": 3.5633059569728287, + "tokens_seen": 1412628480 + }, + { + "epoch": 0.43, + "learning_rate": 0.000462234704112337, + "loss": 1.2688, + "theoretical_loss": 3.563291331362974, + "tokens_seen": 1412694016 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004622186559679038, + "loss": 1.2986, + "theoretical_loss": 3.563276706621565, + "tokens_seen": 1412759552 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004622026078234704, + "loss": 1.1771, + "theoretical_loss": 3.563262082748511, + "tokens_seen": 1412825088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004621865596790372, + "loss": 1.2169, + "theoretical_loss": 3.5632474597437183, + "tokens_seen": 1412890624 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004621705115346038, + "loss": 1.2954, + "theoretical_loss": 3.5632328376070967, + "tokens_seen": 1412956160 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004621544633901706, + "loss": 1.4088, + "theoretical_loss": 3.563218216338554, + "tokens_seen": 1413021696 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004621384152457372, + "loss": 1.3772, + "theoretical_loss": 3.563203595937999, + "tokens_seen": 1413087232 + }, + { + "epoch": 0.43, + "learning_rate": 0.000462122367101304, + "loss": 1.3154, + "theoretical_loss": 3.5631889764053386, + "tokens_seen": 1413152768 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004621063189568706, + "loss": 1.2969, + "theoretical_loss": 3.5631743577404817, + "tokens_seen": 1413218304 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004620902708124374, + "loss": 1.2884, + "theoretical_loss": 3.5631597399433366, + "tokens_seen": 1413283840 + }, + { + "epoch": 0.43, + "learning_rate": 0.000462074222668004, + "loss": 1.2522, + "theoretical_loss": 3.563145123013811, + "tokens_seen": 1413349376 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004620581745235708, + "loss": 1.2835, + "theoretical_loss": 3.563130506951814, + "tokens_seen": 1413414912 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004620421263791374, + "loss": 1.2298, + "theoretical_loss": 3.5631158917572536, + "tokens_seen": 1413480448 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004620260782347042, + "loss": 1.2495, + "theoretical_loss": 3.5631012774300386, + "tokens_seen": 1413545984 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046201003009027083, + "loss": 1.2452, + "theoretical_loss": 3.563086663970076, + "tokens_seen": 1413611520 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004619939819458376, + "loss": 1.351, + "theoretical_loss": 3.5630720513772753, + "tokens_seen": 1413677056 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046197793380140423, + "loss": 1.2684, + "theoretical_loss": 3.563057439651544, + "tokens_seen": 1413742592 + }, + { + "epoch": 0.43, + "learning_rate": 0.000461961885656971, + "loss": 1.2259, + "theoretical_loss": 3.5630428287927915, + "tokens_seen": 1413808128 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046194583751253763, + "loss": 1.2958, + "theoretical_loss": 3.5630282188009255, + "tokens_seen": 1413873664 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004619297893681044, + "loss": 1.2495, + "theoretical_loss": 3.5630136096758545, + "tokens_seen": 1413939200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046191374122367103, + "loss": 1.2337, + "theoretical_loss": 3.562999001417486, + "tokens_seen": 1414004736 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004618976930792378, + "loss": 1.4248, + "theoretical_loss": 3.5629843940257304, + "tokens_seen": 1414070272 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046188164493480443, + "loss": 1.3784, + "theoretical_loss": 3.562969787500495, + "tokens_seen": 1414135808 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046186559679037113, + "loss": 1.2026, + "theoretical_loss": 3.5629551818416876, + "tokens_seen": 1414201344 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046184954864593783, + "loss": 1.2322, + "theoretical_loss": 3.5629405770492175, + "tokens_seen": 1414266880 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046183350050150453, + "loss": 1.2609, + "theoretical_loss": 3.562925973122993, + "tokens_seen": 1414332416 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046181745235707123, + "loss": 1.3234, + "theoretical_loss": 3.562911370062923, + "tokens_seen": 1414397952 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046180140421263794, + "loss": 1.2756, + "theoretical_loss": 3.5628967678689154, + "tokens_seen": 1414463488 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046178535606820464, + "loss": 1.2495, + "theoretical_loss": 3.5628821665408785, + "tokens_seen": 1414529024 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046176930792377134, + "loss": 1.21, + "theoretical_loss": 3.5628675660787215, + "tokens_seen": 1414594560 + }, + { + "epoch": 0.43, + "learning_rate": 0.000461753259779338, + "loss": 1.2475, + "theoretical_loss": 3.5628529664823527, + "tokens_seen": 1414660096 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046173721163490474, + "loss": 1.2722, + "theoretical_loss": 3.562838367751681, + "tokens_seen": 1414725632 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004617211634904714, + "loss": 1.293, + "theoretical_loss": 3.562823769886614, + "tokens_seen": 1414791168 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046170511534603814, + "loss": 1.3017, + "theoretical_loss": 3.562809172887061, + "tokens_seen": 1414856704 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004616890672016048, + "loss": 1.3139, + "theoretical_loss": 3.56279457675293, + "tokens_seen": 1414922240 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046167301905717154, + "loss": 1.4239, + "theoretical_loss": 3.5627799814841308, + "tokens_seen": 1414987776 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004616569709127382, + "loss": 1.2464, + "theoretical_loss": 3.562765387080571, + "tokens_seen": 1415053312 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046164092276830494, + "loss": 1.3636, + "theoretical_loss": 3.5627507935421594, + "tokens_seen": 1415118848 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004616248746238716, + "loss": 1.3659, + "theoretical_loss": 3.5627362008688053, + "tokens_seen": 1415184384 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046160882647943834, + "loss": 1.3007, + "theoretical_loss": 3.562721609060416, + "tokens_seen": 1415249920 + }, + { + "epoch": 0.43, + "learning_rate": 0.000461592778335005, + "loss": 1.3657, + "theoretical_loss": 3.562707018116902, + "tokens_seen": 1415315456 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046157673019057174, + "loss": 1.3027, + "theoretical_loss": 3.5626924280381704, + "tokens_seen": 1415380992 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004615606820461384, + "loss": 1.2507, + "theoretical_loss": 3.562677838824131, + "tokens_seen": 1415446528 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046154463390170515, + "loss": 1.2592, + "theoretical_loss": 3.5626632504746913, + "tokens_seen": 1415512064 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 475088, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1855875253677368, + "objective/train/theoretical_loss": 3.5626486629897616, + "objective/train/tokens_used": 1436037600, + "theoretical_loss": 3.5626486629897616, + "tokens_seen": 1415577600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004615285857572718, + "loss": 1.2318, + "theoretical_loss": 3.5626486629897616, + "tokens_seen": 1415577600 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046151253761283855, + "loss": 1.283, + "theoretical_loss": 3.5626340763692497, + "tokens_seen": 1415643136 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004614964894684052, + "loss": 1.1989, + "theoretical_loss": 3.562619490613064, + "tokens_seen": 1415708672 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046148044132397195, + "loss": 1.2893, + "theoretical_loss": 3.5626049057211144, + "tokens_seen": 1415774208 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004614643931795386, + "loss": 1.2727, + "theoretical_loss": 3.5625903216933086, + "tokens_seen": 1415839744 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046144834503510535, + "loss": 1.154, + "theoretical_loss": 3.562575738529556, + "tokens_seen": 1415905280 + }, + { + "epoch": 0.43, + "learning_rate": 0.000461432296890672, + "loss": 1.3396, + "theoretical_loss": 3.5625611562297657, + "tokens_seen": 1415970816 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046141624874623875, + "loss": 1.3941, + "theoretical_loss": 3.562546574793845, + "tokens_seen": 1416036352 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004614002006018054, + "loss": 1.3347, + "theoretical_loss": 3.562531994221705, + "tokens_seen": 1416101888 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046138415245737215, + "loss": 1.2259, + "theoretical_loss": 3.5625174145132528, + "tokens_seen": 1416167424 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004613681043129388, + "loss": 1.3095, + "theoretical_loss": 3.5625028356683983, + "tokens_seen": 1416232960 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046135205616850555, + "loss": 1.3344, + "theoretical_loss": 3.5624882576870496, + "tokens_seen": 1416298496 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004613360080240722, + "loss": 1.5102, + "theoretical_loss": 3.5624736805691164, + "tokens_seen": 1416364032 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046131995987963895, + "loss": 1.2662, + "theoretical_loss": 3.562459104314507, + "tokens_seen": 1416429568 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004613039117352056, + "loss": 1.3415, + "theoretical_loss": 3.562444528923131, + "tokens_seen": 1416495104 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046128786359077236, + "loss": 1.3411, + "theoretical_loss": 3.5624299543948963, + "tokens_seen": 1416560640 + }, + { + "epoch": 0.43, + "learning_rate": 0.000461271815446339, + "loss": 1.3028, + "theoretical_loss": 3.5624153807297123, + "tokens_seen": 1416626176 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046125576730190576, + "loss": 1.3979, + "theoretical_loss": 3.5624008079274887, + "tokens_seen": 1416691712 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046123971915747246, + "loss": 1.2643, + "theoretical_loss": 3.5623862359881335, + "tokens_seen": 1416757248 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046122367101303916, + "loss": 1.3072, + "theoretical_loss": 3.5623716649115558, + "tokens_seen": 1416822784 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046120762286860586, + "loss": 1.3432, + "theoretical_loss": 3.5623570946976657, + "tokens_seen": 1416888320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046119157472417256, + "loss": 1.3149, + "theoretical_loss": 3.562342525346371, + "tokens_seen": 1416953856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046117552657973926, + "loss": 1.3099, + "theoretical_loss": 3.562327956857581, + "tokens_seen": 1417019392 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046115947843530596, + "loss": 1.2656, + "theoretical_loss": 3.562313389231205, + "tokens_seen": 1417084928 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046114343029087266, + "loss": 1.1917, + "theoretical_loss": 3.5622988224671523, + "tokens_seen": 1417150464 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046112738214643936, + "loss": 1.3449, + "theoretical_loss": 3.5622842565653317, + "tokens_seen": 1417216000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046111133400200606, + "loss": 1.3432, + "theoretical_loss": 3.5622696915256515, + "tokens_seen": 1417281536 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004610952858575727, + "loss": 1.4002, + "theoretical_loss": 3.5622551273480223, + "tokens_seen": 1417347072 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046107923771313946, + "loss": 1.3465, + "theoretical_loss": 3.562240564032352, + "tokens_seen": 1417412608 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004610631895687061, + "loss": 1.4118, + "theoretical_loss": 3.5622260015785505, + "tokens_seen": 1417478144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046104714142427287, + "loss": 1.3078, + "theoretical_loss": 3.562211439986527, + "tokens_seen": 1417543680 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004610310932798395, + "loss": 1.34, + "theoretical_loss": 3.5621968792561898, + "tokens_seen": 1417609216 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046101504513540627, + "loss": 1.2498, + "theoretical_loss": 3.562182319387449, + "tokens_seen": 1417674752 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004609989969909729, + "loss": 1.3904, + "theoretical_loss": 3.5621677603802127, + "tokens_seen": 1417740288 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046098294884653967, + "loss": 1.3391, + "theoretical_loss": 3.5621532022343914, + "tokens_seen": 1417805824 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004609669007021063, + "loss": 1.3944, + "theoretical_loss": 3.5621386449498935, + "tokens_seen": 1417871360 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046095085255767307, + "loss": 1.25, + "theoretical_loss": 3.5621240885266285, + "tokens_seen": 1417936896 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004609348044132397, + "loss": 1.1472, + "theoretical_loss": 3.562109532964506, + "tokens_seen": 1418002432 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046091875626880647, + "loss": 1.2777, + "theoretical_loss": 3.562094978263435, + "tokens_seen": 1418067968 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004609027081243731, + "loss": 1.2624, + "theoretical_loss": 3.562080424423324, + "tokens_seen": 1418133504 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046088665997993987, + "loss": 1.3042, + "theoretical_loss": 3.5620658714440827, + "tokens_seen": 1418199040 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004608706118355065, + "loss": 1.214, + "theoretical_loss": 3.5620513193256214, + "tokens_seen": 1418264576 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046085456369107327, + "loss": 1.1787, + "theoretical_loss": 3.5620367680678484, + "tokens_seen": 1418330112 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004608385155466399, + "loss": 1.329, + "theoretical_loss": 3.562022217670673, + "tokens_seen": 1418395648 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004608224674022067, + "loss": 1.2944, + "theoretical_loss": 3.5620076681340054, + "tokens_seen": 1418461184 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004608064192577733, + "loss": 1.2893, + "theoretical_loss": 3.5619931194577537, + "tokens_seen": 1418526720 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004607903711133401, + "loss": 1.3621, + "theoretical_loss": 3.561978571641828, + "tokens_seen": 1418592256 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004607743229689067, + "loss": 1.2604, + "theoretical_loss": 3.561964024686138, + "tokens_seen": 1418657792 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004607582748244735, + "loss": 1.2343, + "theoretical_loss": 3.5619494785905923, + "tokens_seen": 1418723328 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004607422266800401, + "loss": 1.4144, + "theoretical_loss": 3.5619349333551016, + "tokens_seen": 1418788864 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 476156, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3921765089035034, + "objective/train/theoretical_loss": 3.5619203889795736, + "objective/train/tokens_used": 1439314400, + "theoretical_loss": 3.5619203889795736, + "tokens_seen": 1418854400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004607261785356069, + "loss": 1.3558, + "theoretical_loss": 3.5619203889795736, + "tokens_seen": 1418854400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004607101303911735, + "loss": 1.3041, + "theoretical_loss": 3.5619058454639188, + "tokens_seen": 1418919936 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004606940822467403, + "loss": 1.3763, + "theoretical_loss": 3.5618913028080463, + "tokens_seen": 1418985472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004606780341023069, + "loss": 1.3323, + "theoretical_loss": 3.561876761011866, + "tokens_seen": 1419051008 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004606619859578737, + "loss": 1.3684, + "theoretical_loss": 3.561862220075287, + "tokens_seen": 1419116544 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004606459378134403, + "loss": 1.1713, + "theoretical_loss": 3.561847679998219, + "tokens_seen": 1419182080 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004606298896690071, + "loss": 1.1765, + "theoretical_loss": 3.5618331407805712, + "tokens_seen": 1419247616 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046061384152457373, + "loss": 1.3827, + "theoretical_loss": 3.5618186024222536, + "tokens_seen": 1419313152 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004605977933801405, + "loss": 1.2884, + "theoretical_loss": 3.561804064923175, + "tokens_seen": 1419378688 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046058174523570713, + "loss": 1.3631, + "theoretical_loss": 3.5617895282832457, + "tokens_seen": 1419444224 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004605656970912739, + "loss": 1.3333, + "theoretical_loss": 3.561774992502375, + "tokens_seen": 1419509760 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046054964894684053, + "loss": 1.286, + "theoretical_loss": 3.5617604575804727, + "tokens_seen": 1419575296 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004605336008024073, + "loss": 1.2231, + "theoretical_loss": 3.5617459235174476, + "tokens_seen": 1419640832 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046051755265797393, + "loss": 1.2878, + "theoretical_loss": 3.56173139031321, + "tokens_seen": 1419706368 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004605015045135407, + "loss": 1.2333, + "theoretical_loss": 3.561716857967669, + "tokens_seen": 1419771904 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046048545636910733, + "loss": 1.1826, + "theoretical_loss": 3.5617023264807353, + "tokens_seen": 1419837440 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046046940822467403, + "loss": 1.3458, + "theoretical_loss": 3.5616877958523174, + "tokens_seen": 1419902976 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046045336008024073, + "loss": 1.261, + "theoretical_loss": 3.561673266082325, + "tokens_seen": 1419968512 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046043731193580744, + "loss": 1.3043, + "theoretical_loss": 3.561658737170669, + "tokens_seen": 1420034048 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046042126379137414, + "loss": 1.2932, + "theoretical_loss": 3.5616442091172575, + "tokens_seen": 1420099584 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046040521564694084, + "loss": 1.2602, + "theoretical_loss": 3.5616296819220015, + "tokens_seen": 1420165120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046038916750250754, + "loss": 1.2364, + "theoretical_loss": 3.5616151555848097, + "tokens_seen": 1420230656 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046037311935807424, + "loss": 1.3497, + "theoretical_loss": 3.561600630105592, + "tokens_seen": 1420296192 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046035707121364094, + "loss": 1.3755, + "theoretical_loss": 3.561586105484259, + "tokens_seen": 1420361728 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046034102306920764, + "loss": 1.2862, + "theoretical_loss": 3.56157158172072, + "tokens_seen": 1420427264 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004603249749247743, + "loss": 1.331, + "theoretical_loss": 3.561557058814884, + "tokens_seen": 1420492800 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046030892678034104, + "loss": 1.2845, + "theoretical_loss": 3.5615425367666615, + "tokens_seen": 1420558336 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004602928786359077, + "loss": 1.2585, + "theoretical_loss": 3.5615280155759628, + "tokens_seen": 1420623872 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046027683049147444, + "loss": 1.3023, + "theoretical_loss": 3.5615134952426963, + "tokens_seen": 1420689408 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004602607823470411, + "loss": 1.294, + "theoretical_loss": 3.5614989757667734, + "tokens_seen": 1420754944 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046024473420260784, + "loss": 1.2831, + "theoretical_loss": 3.5614844571481026, + "tokens_seen": 1420820480 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004602286860581745, + "loss": 1.2303, + "theoretical_loss": 3.561469939386595, + "tokens_seen": 1420886016 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046021263791374124, + "loss": 1.3566, + "theoretical_loss": 3.561455422482159, + "tokens_seen": 1420951552 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004601965897693079, + "loss": 1.2763, + "theoretical_loss": 3.5614409064347057, + "tokens_seen": 1421017088 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046018054162487465, + "loss": 1.2519, + "theoretical_loss": 3.561426391244144, + "tokens_seen": 1421082624 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004601644934804413, + "loss": 1.2943, + "theoretical_loss": 3.561411876910385, + "tokens_seen": 1421148160 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046014844533600805, + "loss": 1.1506, + "theoretical_loss": 3.561397363433338, + "tokens_seen": 1421213696 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004601323971915747, + "loss": 1.4133, + "theoretical_loss": 3.5613828508129126, + "tokens_seen": 1421279232 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046011634904714145, + "loss": 1.337, + "theoretical_loss": 3.561368339049019, + "tokens_seen": 1421344768 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004601003009027081, + "loss": 1.3713, + "theoretical_loss": 3.5613538281415673, + "tokens_seen": 1421410304 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046008425275827485, + "loss": 1.3428, + "theoretical_loss": 3.561339318090467, + "tokens_seen": 1421475840 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004600682046138415, + "loss": 1.2585, + "theoretical_loss": 3.5613248088956286, + "tokens_seen": 1421541376 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046005215646940825, + "loss": 1.2722, + "theoretical_loss": 3.561310300556962, + "tokens_seen": 1421606912 + }, + { + "epoch": 0.43, + "learning_rate": 0.000460036108324975, + "loss": 1.1406, + "theoretical_loss": 3.5612957930743776, + "tokens_seen": 1421672448 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046002006018054165, + "loss": 1.4369, + "theoretical_loss": 3.5612812864477847, + "tokens_seen": 1421737984 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004600040120361084, + "loss": 1.3137, + "theoretical_loss": 3.5612667806770935, + "tokens_seen": 1421803520 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045998796389167505, + "loss": 1.3503, + "theoretical_loss": 3.5612522757622145, + "tokens_seen": 1421869056 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004599719157472418, + "loss": 1.2199, + "theoretical_loss": 3.561237771703057, + "tokens_seen": 1421934592 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045995586760280845, + "loss": 1.3518, + "theoretical_loss": 3.561223268499532, + "tokens_seen": 1422000128 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004599398194583752, + "loss": 1.3655, + "theoretical_loss": 3.561208766151549, + "tokens_seen": 1422065664 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 477084, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3397164344787598, + "objective/train/theoretical_loss": 3.561194264659018, + "objective/train/tokens_used": 1442591200, + "theoretical_loss": 3.561194264659018, + "tokens_seen": 1422131200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045992377131394186, + "loss": 1.368, + "theoretical_loss": 3.561194264659018, + "tokens_seen": 1422131200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004599077231695086, + "loss": 1.342, + "theoretical_loss": 3.5611797640218494, + "tokens_seen": 1422196736 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045989167502507526, + "loss": 1.373, + "theoretical_loss": 3.5611652642399534, + "tokens_seen": 1422262272 + }, + { + "epoch": 0.43, + "learning_rate": 0.000459875626880642, + "loss": 1.3054, + "theoretical_loss": 3.56115076531324, + "tokens_seen": 1422327808 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045985957873620866, + "loss": 1.2597, + "theoretical_loss": 3.5611362672416194, + "tokens_seen": 1422393344 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045984353059177536, + "loss": 1.275, + "theoretical_loss": 3.5611217700250015, + "tokens_seen": 1422458880 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045982748244734206, + "loss": 1.3644, + "theoretical_loss": 3.561107273663297, + "tokens_seen": 1422524416 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045981143430290876, + "loss": 1.2652, + "theoretical_loss": 3.561092778156416, + "tokens_seen": 1422589952 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045979538615847546, + "loss": 1.3335, + "theoretical_loss": 3.5610782835042682, + "tokens_seen": 1422655488 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045977933801404216, + "loss": 1.2909, + "theoretical_loss": 3.5610637897067647, + "tokens_seen": 1422721024 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045976328986960886, + "loss": 1.2856, + "theoretical_loss": 3.561049296763815, + "tokens_seen": 1422786560 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045974724172517556, + "loss": 1.3185, + "theoretical_loss": 3.5610348046753293, + "tokens_seen": 1422852096 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045973119358074226, + "loss": 1.1983, + "theoretical_loss": 3.561020313441219, + "tokens_seen": 1422917632 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045971514543630896, + "loss": 1.2979, + "theoretical_loss": 3.561005823061393, + "tokens_seen": 1422983168 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004596990972918756, + "loss": 1.3143, + "theoretical_loss": 3.560991333535762, + "tokens_seen": 1423048704 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045968304914744236, + "loss": 1.335, + "theoretical_loss": 3.5609768448642365, + "tokens_seen": 1423114240 + }, + { + "epoch": 0.43, + "learning_rate": 0.000459667001003009, + "loss": 1.3229, + "theoretical_loss": 3.5609623570467273, + "tokens_seen": 1423179776 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045965095285857577, + "loss": 1.2293, + "theoretical_loss": 3.560947870083144, + "tokens_seen": 1423245312 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004596349047141424, + "loss": 1.342, + "theoretical_loss": 3.5609333839733965, + "tokens_seen": 1423310848 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045961885656970917, + "loss": 1.3353, + "theoretical_loss": 3.5609188987173965, + "tokens_seen": 1423376384 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004596028084252758, + "loss": 1.3138, + "theoretical_loss": 3.560904414315054, + "tokens_seen": 1423441920 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045958676028084257, + "loss": 1.3179, + "theoretical_loss": 3.5608899307662787, + "tokens_seen": 1423507456 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004595707121364092, + "loss": 1.2747, + "theoretical_loss": 3.5608754480709814, + "tokens_seen": 1423572992 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045955466399197597, + "loss": 1.398, + "theoretical_loss": 3.5608609662290727, + "tokens_seen": 1423638528 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004595386158475426, + "loss": 1.3593, + "theoretical_loss": 3.5608464852404627, + "tokens_seen": 1423704064 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045952256770310937, + "loss": 1.3231, + "theoretical_loss": 3.5608320051050626, + "tokens_seen": 1423769600 + }, + { + "epoch": 0.43, + "learning_rate": 0.000459506519558676, + "loss": 1.3361, + "theoretical_loss": 3.560817525822782, + "tokens_seen": 1423835136 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045949047141424277, + "loss": 1.2657, + "theoretical_loss": 3.5608030473935313, + "tokens_seen": 1423900672 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004594744232698094, + "loss": 1.36, + "theoretical_loss": 3.5607885698172215, + "tokens_seen": 1423966208 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004594583751253762, + "loss": 1.2496, + "theoretical_loss": 3.560774093093763, + "tokens_seen": 1424031744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004594423269809428, + "loss": 1.3415, + "theoretical_loss": 3.5607596172230664, + "tokens_seen": 1424097280 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004594262788365096, + "loss": 1.3427, + "theoretical_loss": 3.560745142205042, + "tokens_seen": 1424162816 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004594102306920762, + "loss": 1.2926, + "theoretical_loss": 3.5607306680396, + "tokens_seen": 1424228352 + }, + { + "epoch": 0.43, + "learning_rate": 0.000459394182547643, + "loss": 1.334, + "theoretical_loss": 3.560716194726652, + "tokens_seen": 1424293888 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004593781344032096, + "loss": 1.3027, + "theoretical_loss": 3.5607017222661073, + "tokens_seen": 1424359424 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004593620862587764, + "loss": 1.2977, + "theoretical_loss": 3.560687250657878, + "tokens_seen": 1424424960 + }, + { + "epoch": 0.43, + "learning_rate": 0.000459346038114343, + "loss": 1.2586, + "theoretical_loss": 3.560672779901873, + "tokens_seen": 1424490496 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004593299899699098, + "loss": 1.335, + "theoretical_loss": 3.5606583099980043, + "tokens_seen": 1424556032 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004593139418254764, + "loss": 1.4184, + "theoretical_loss": 3.560643840946182, + "tokens_seen": 1424621568 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004592978936810432, + "loss": 1.3255, + "theoretical_loss": 3.560629372746316, + "tokens_seen": 1424687104 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004592818455366098, + "loss": 1.3755, + "theoretical_loss": 3.5606149053983183, + "tokens_seen": 1424752640 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004592657973921766, + "loss": 1.3917, + "theoretical_loss": 3.5606004389020987, + "tokens_seen": 1424818176 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045924974924774323, + "loss": 1.3441, + "theoretical_loss": 3.560585973257568, + "tokens_seen": 1424883712 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045923370110331, + "loss": 1.1777, + "theoretical_loss": 3.560571508464637, + "tokens_seen": 1424949248 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045921765295887663, + "loss": 1.3447, + "theoretical_loss": 3.5605570445232164, + "tokens_seen": 1425014784 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004592016048144434, + "loss": 1.3868, + "theoretical_loss": 3.560542581433217, + "tokens_seen": 1425080320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045918555667001003, + "loss": 1.2555, + "theoretical_loss": 3.560528119194549, + "tokens_seen": 1425145856 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004591695085255768, + "loss": 1.3703, + "theoretical_loss": 3.5605136578071237, + "tokens_seen": 1425211392 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045915346038114343, + "loss": 1.256, + "theoretical_loss": 3.560499197270852, + "tokens_seen": 1425276928 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004591374122367102, + "loss": 1.2993, + "theoretical_loss": 3.5604847375856448, + "tokens_seen": 1425342464 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 478424, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3589322566986084, + "objective/train/theoretical_loss": 3.5604702787514118, + "objective/train/tokens_used": 1445868000, + "theoretical_loss": 3.5604702787514118, + "tokens_seen": 1425408000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045912136409227683, + "loss": 1.2625, + "theoretical_loss": 3.5604702787514118, + "tokens_seen": 1425408000 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004591053159478436, + "loss": 1.322, + "theoretical_loss": 3.5604558207680643, + "tokens_seen": 1425473536 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045908926780341023, + "loss": 1.3262, + "theoretical_loss": 3.5604413636355137, + "tokens_seen": 1425539072 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045907321965897693, + "loss": 1.2828, + "theoretical_loss": 3.56042690735367, + "tokens_seen": 1425604608 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045905717151454364, + "loss": 1.229, + "theoretical_loss": 3.560412451922445, + "tokens_seen": 1425670144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045904112337011034, + "loss": 1.3612, + "theoretical_loss": 3.5603979973417488, + "tokens_seen": 1425735680 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045902507522567704, + "loss": 1.1432, + "theoretical_loss": 3.5603835436114926, + "tokens_seen": 1425801216 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045900902708124374, + "loss": 1.3389, + "theoretical_loss": 3.5603690907315864, + "tokens_seen": 1425866752 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045899297893681044, + "loss": 1.2868, + "theoretical_loss": 3.5603546387019422, + "tokens_seen": 1425932288 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045897693079237714, + "loss": 1.2895, + "theoretical_loss": 3.560340187522471, + "tokens_seen": 1425997824 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045896088264794384, + "loss": 1.2537, + "theoretical_loss": 3.560325737193083, + "tokens_seen": 1426063360 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045894483450351054, + "loss": 1.2988, + "theoretical_loss": 3.5603112877136893, + "tokens_seen": 1426128896 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004589287863590772, + "loss": 1.3919, + "theoretical_loss": 3.560296839084201, + "tokens_seen": 1426194432 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045891273821464394, + "loss": 1.2831, + "theoretical_loss": 3.5602823913045283, + "tokens_seen": 1426259968 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004588966900702106, + "loss": 1.2351, + "theoretical_loss": 3.5602679443745835, + "tokens_seen": 1426325504 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045888064192577734, + "loss": 1.3111, + "theoretical_loss": 3.5602534982942773, + "tokens_seen": 1426391040 + }, + { + "epoch": 0.43, + "learning_rate": 0.000458864593781344, + "loss": 1.3329, + "theoretical_loss": 3.56023905306352, + "tokens_seen": 1426456576 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045884854563691074, + "loss": 1.3195, + "theoretical_loss": 3.5602246086822227, + "tokens_seen": 1426522112 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004588324974924775, + "loss": 1.3218, + "theoretical_loss": 3.5602101651502966, + "tokens_seen": 1426587648 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045881644934804415, + "loss": 1.3029, + "theoretical_loss": 3.560195722467653, + "tokens_seen": 1426653184 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004588004012036109, + "loss": 1.348, + "theoretical_loss": 3.560181280634203, + "tokens_seen": 1426718720 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045878435305917755, + "loss": 1.3028, + "theoretical_loss": 3.560166839649857, + "tokens_seen": 1426784256 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004587683049147443, + "loss": 1.3608, + "theoretical_loss": 3.5601523995145268, + "tokens_seen": 1426849792 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045875225677031095, + "loss": 1.1981, + "theoretical_loss": 3.560137960228123, + "tokens_seen": 1426915328 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004587362086258777, + "loss": 1.2624, + "theoretical_loss": 3.5601235217905574, + "tokens_seen": 1426980864 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045872016048144435, + "loss": 1.2641, + "theoretical_loss": 3.56010908420174, + "tokens_seen": 1427046400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004587041123370111, + "loss": 1.3413, + "theoretical_loss": 3.5600946474615824, + "tokens_seen": 1427111936 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045868806419257775, + "loss": 1.362, + "theoretical_loss": 3.5600802115699968, + "tokens_seen": 1427177472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004586720160481445, + "loss": 1.2466, + "theoretical_loss": 3.560065776526893, + "tokens_seen": 1427243008 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045865596790371115, + "loss": 1.3808, + "theoretical_loss": 3.5600513423321827, + "tokens_seen": 1427308544 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004586399197592779, + "loss": 1.2685, + "theoretical_loss": 3.5600369089857766, + "tokens_seen": 1427374080 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045862387161484455, + "loss": 1.2599, + "theoretical_loss": 3.5600224764875867, + "tokens_seen": 1427439616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004586078234704113, + "loss": 1.3494, + "theoretical_loss": 3.5600080448375238, + "tokens_seen": 1427505152 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045859177532597795, + "loss": 1.3027, + "theoretical_loss": 3.5599936140354993, + "tokens_seen": 1427570688 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004585757271815447, + "loss": 1.33, + "theoretical_loss": 3.559979184081424, + "tokens_seen": 1427636224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045855967903711136, + "loss": 1.3565, + "theoretical_loss": 3.5599647549752094, + "tokens_seen": 1427701760 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004585436308926781, + "loss": 1.2961, + "theoretical_loss": 3.559950326716767, + "tokens_seen": 1427767296 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045852758274824476, + "loss": 1.222, + "theoretical_loss": 3.5599358993060077, + "tokens_seen": 1427832832 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004585115346038115, + "loss": 1.2649, + "theoretical_loss": 3.559921472742843, + "tokens_seen": 1427898368 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045849548645937816, + "loss": 1.2992, + "theoretical_loss": 3.5599070470271847, + "tokens_seen": 1427963904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004584794383149449, + "loss": 1.239, + "theoretical_loss": 3.5598926221589435, + "tokens_seen": 1428029440 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045846339017051156, + "loss": 1.2707, + "theoretical_loss": 3.5598781981380307, + "tokens_seen": 1428094976 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045844734202607826, + "loss": 1.2908, + "theoretical_loss": 3.5598637749643576, + "tokens_seen": 1428160512 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045843129388164496, + "loss": 1.3567, + "theoretical_loss": 3.559849352637836, + "tokens_seen": 1428226048 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045841524573721166, + "loss": 1.3475, + "theoretical_loss": 3.5598349311583766, + "tokens_seen": 1428291584 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045839919759277836, + "loss": 1.2968, + "theoretical_loss": 3.559820510525892, + "tokens_seen": 1428357120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045838314944834506, + "loss": 1.2214, + "theoretical_loss": 3.5598060907402918, + "tokens_seen": 1428422656 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045836710130391176, + "loss": 1.2678, + "theoretical_loss": 3.559791671801489, + "tokens_seen": 1428488192 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045835105315947846, + "loss": 1.2237, + "theoretical_loss": 3.5597772537093944, + "tokens_seen": 1428553728 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045833500501504516, + "loss": 1.305, + "theoretical_loss": 3.5597628364639196, + "tokens_seen": 1428619264 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 479356, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.245415449142456, + "objective/train/theoretical_loss": 3.559748420064976, + "objective/train/tokens_used": 1449144800, + "theoretical_loss": 3.559748420064976, + "tokens_seen": 1428684800 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045831895687061186, + "loss": 1.2871, + "theoretical_loss": 3.559748420064976, + "tokens_seen": 1428684800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004583029087261785, + "loss": 1.2833, + "theoretical_loss": 3.559734004512475, + "tokens_seen": 1428750336 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045828686058174527, + "loss": 1.2819, + "theoretical_loss": 3.5597195898063276, + "tokens_seen": 1428815872 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004582708124373119, + "loss": 1.2679, + "theoretical_loss": 3.5597051759464464, + "tokens_seen": 1428881408 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045825476429287867, + "loss": 1.3871, + "theoretical_loss": 3.5596907629327417, + "tokens_seen": 1428946944 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004582387161484453, + "loss": 1.3208, + "theoretical_loss": 3.559676350765126, + "tokens_seen": 1429012480 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045822266800401207, + "loss": 1.22, + "theoretical_loss": 3.5596619394435107, + "tokens_seen": 1429078016 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004582066198595787, + "loss": 1.2834, + "theoretical_loss": 3.5596475289678065, + "tokens_seen": 1429143552 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045819057171514547, + "loss": 1.2182, + "theoretical_loss": 3.559633119337926, + "tokens_seen": 1429209088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004581745235707121, + "loss": 1.2952, + "theoretical_loss": 3.55961871055378, + "tokens_seen": 1429274624 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045815847542627887, + "loss": 1.2874, + "theoretical_loss": 3.5596043026152806, + "tokens_seen": 1429340160 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004581424272818455, + "loss": 1.2538, + "theoretical_loss": 3.5595898955223393, + "tokens_seen": 1429405696 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045812637913741227, + "loss": 1.2469, + "theoretical_loss": 3.5595754892748674, + "tokens_seen": 1429471232 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004581103309929789, + "loss": 1.2872, + "theoretical_loss": 3.5595610838727767, + "tokens_seen": 1429536768 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004580942828485457, + "loss": 1.3236, + "theoretical_loss": 3.559546679315979, + "tokens_seen": 1429602304 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004580782347041123, + "loss": 1.3737, + "theoretical_loss": 3.559532275604386, + "tokens_seen": 1429667840 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004580621865596791, + "loss": 1.379, + "theoretical_loss": 3.559517872737909, + "tokens_seen": 1429733376 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004580461384152457, + "loss": 1.2007, + "theoretical_loss": 3.55950347071646, + "tokens_seen": 1429798912 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004580300902708125, + "loss": 1.3352, + "theoretical_loss": 3.5594890695399504, + "tokens_seen": 1429864448 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004580140421263791, + "loss": 1.2399, + "theoretical_loss": 3.559474669208292, + "tokens_seen": 1429929984 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004579979939819459, + "loss": 1.2412, + "theoretical_loss": 3.559460269721397, + "tokens_seen": 1429995520 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004579819458375125, + "loss": 1.3351, + "theoretical_loss": 3.5594458710791765, + "tokens_seen": 1430061056 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004579658976930793, + "loss": 1.2859, + "theoretical_loss": 3.5594314732815424, + "tokens_seen": 1430126592 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004579498495486459, + "loss": 1.2754, + "theoretical_loss": 3.5594170763284065, + "tokens_seen": 1430192128 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004579338014042127, + "loss": 1.2804, + "theoretical_loss": 3.559402680219681, + "tokens_seen": 1430257664 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004579177532597793, + "loss": 1.3938, + "theoretical_loss": 3.559388284955277, + "tokens_seen": 1430323200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004579017051153461, + "loss": 1.4642, + "theoretical_loss": 3.559373890535107, + "tokens_seen": 1430388736 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045788565697091273, + "loss": 1.3394, + "theoretical_loss": 3.5593594969590816, + "tokens_seen": 1430454272 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004578696088264795, + "loss": 1.2821, + "theoretical_loss": 3.559345104227114, + "tokens_seen": 1430519808 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045785356068204613, + "loss": 1.2923, + "theoretical_loss": 3.5593307123391154, + "tokens_seen": 1430585344 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004578375125376129, + "loss": 1.2905, + "theoretical_loss": 3.5593163212949976, + "tokens_seen": 1430650880 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045782146439317953, + "loss": 1.3761, + "theoretical_loss": 3.5593019310946725, + "tokens_seen": 1430716416 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004578054162487463, + "loss": 1.3591, + "theoretical_loss": 3.559287541738052, + "tokens_seen": 1430781952 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045778936810431293, + "loss": 1.2329, + "theoretical_loss": 3.5592731532250483, + "tokens_seen": 1430847488 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004577733199598797, + "loss": 1.2312, + "theoretical_loss": 3.559258765555573, + "tokens_seen": 1430913024 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045775727181544633, + "loss": 1.5046, + "theoretical_loss": 3.559244378729538, + "tokens_seen": 1430978560 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004577412236710131, + "loss": 1.3033, + "theoretical_loss": 3.5592299927468556, + "tokens_seen": 1431044096 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045772517552657973, + "loss": 1.3342, + "theoretical_loss": 3.5592156076074373, + "tokens_seen": 1431109632 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004577091273821465, + "loss": 1.2985, + "theoretical_loss": 3.559201223311195, + "tokens_seen": 1431175168 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045769307923771314, + "loss": 1.278, + "theoretical_loss": 3.5591868398580413, + "tokens_seen": 1431240704 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045767703109327984, + "loss": 1.2854, + "theoretical_loss": 3.559172457247887, + "tokens_seen": 1431306240 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045766098294884654, + "loss": 1.5248, + "theoretical_loss": 3.5591580754806458, + "tokens_seen": 1431371776 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045764493480441324, + "loss": 1.3153, + "theoretical_loss": 3.5591436945562283, + "tokens_seen": 1431437312 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045762888665998, + "loss": 1.3561, + "theoretical_loss": 3.5591293144745464, + "tokens_seen": 1431502848 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045761283851554664, + "loss": 1.3035, + "theoretical_loss": 3.559114935235513, + "tokens_seen": 1431568384 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004575967903711134, + "loss": 1.2852, + "theoretical_loss": 3.5591005568390406, + "tokens_seen": 1431633920 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045758074222668004, + "loss": 1.2484, + "theoretical_loss": 3.5590861792850403, + "tokens_seen": 1431699456 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004575646940822468, + "loss": 1.326, + "theoretical_loss": 3.559071802573424, + "tokens_seen": 1431764992 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045754864593781344, + "loss": 1.399, + "theoretical_loss": 3.5590574267041046, + "tokens_seen": 1431830528 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004575325977933802, + "loss": 1.4098, + "theoretical_loss": 3.5590430516769938, + "tokens_seen": 1431896064 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 480635, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3885066509246826, + "objective/train/theoretical_loss": 3.5590286774920035, + "objective/train/tokens_used": 1452421600, + "theoretical_loss": 3.5590286774920035, + "tokens_seen": 1431961600 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045751654964894684, + "loss": 1.3568, + "theoretical_loss": 3.5590286774920035, + "tokens_seen": 1431961600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004575005015045136, + "loss": 1.3758, + "theoretical_loss": 3.5590143041490463, + "tokens_seen": 1432027136 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045748445336008024, + "loss": 1.3635, + "theoretical_loss": 3.558999931648034, + "tokens_seen": 1432092672 + }, + { + "epoch": 0.43, + "learning_rate": 0.000457468405215647, + "loss": 1.2606, + "theoretical_loss": 3.5589855599888782, + "tokens_seen": 1432158208 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045745235707121364, + "loss": 1.2629, + "theoretical_loss": 3.5589711891714924, + "tokens_seen": 1432223744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004574363089267804, + "loss": 1.2715, + "theoretical_loss": 3.5589568191957883, + "tokens_seen": 1432289280 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045742026078234705, + "loss": 1.4599, + "theoretical_loss": 3.5589424500616778, + "tokens_seen": 1432354816 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004574042126379138, + "loss": 1.4052, + "theoretical_loss": 3.558928081769073, + "tokens_seen": 1432420352 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045738816449348045, + "loss": 1.2824, + "theoretical_loss": 3.558913714317886, + "tokens_seen": 1432485888 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004573721163490472, + "loss": 1.3131, + "theoretical_loss": 3.5588993477080297, + "tokens_seen": 1432551424 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045735606820461385, + "loss": 1.3909, + "theoretical_loss": 3.558884981939416, + "tokens_seen": 1432616960 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004573400200601806, + "loss": 1.1918, + "theoretical_loss": 3.5588706170119577, + "tokens_seen": 1432682496 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045732397191574725, + "loss": 1.2211, + "theoretical_loss": 3.5588562529255663, + "tokens_seen": 1432748032 + }, + { + "epoch": 0.43, + "learning_rate": 0.000457307923771314, + "loss": 1.3774, + "theoretical_loss": 3.5588418896801537, + "tokens_seen": 1432813568 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045729187562688065, + "loss": 1.2349, + "theoretical_loss": 3.558827527275633, + "tokens_seen": 1432879104 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004572758274824474, + "loss": 1.2705, + "theoretical_loss": 3.558813165711917, + "tokens_seen": 1432944640 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045725977933801405, + "loss": 1.2108, + "theoretical_loss": 3.5587988049889168, + "tokens_seen": 1433010176 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004572437311935808, + "loss": 1.3224, + "theoretical_loss": 3.5587844451065456, + "tokens_seen": 1433075712 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045722768304914745, + "loss": 1.2045, + "theoretical_loss": 3.5587700860647153, + "tokens_seen": 1433141248 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004572116349047142, + "loss": 1.3315, + "theoretical_loss": 3.558755727863338, + "tokens_seen": 1433206784 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045719558676028085, + "loss": 1.3738, + "theoretical_loss": 3.558741370502327, + "tokens_seen": 1433272320 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004571795386158476, + "loss": 1.2957, + "theoretical_loss": 3.5587270139815943, + "tokens_seen": 1433337856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045716349047141426, + "loss": 1.41, + "theoretical_loss": 3.5587126583010518, + "tokens_seen": 1433403392 + }, + { + "epoch": 0.43, + "learning_rate": 0.000457147442326981, + "loss": 1.3075, + "theoretical_loss": 3.5586983034606128, + "tokens_seen": 1433468928 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045713139418254766, + "loss": 1.2395, + "theoretical_loss": 3.558683949460189, + "tokens_seen": 1433534464 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004571153460381144, + "loss": 1.3525, + "theoretical_loss": 3.5586695962996933, + "tokens_seen": 1433600000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045709929789368106, + "loss": 1.2121, + "theoretical_loss": 3.5586552439790378, + "tokens_seen": 1433665536 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004570832497492478, + "loss": 1.3105, + "theoretical_loss": 3.5586408924981354, + "tokens_seen": 1433731072 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045706720160481446, + "loss": 1.3357, + "theoretical_loss": 3.558626541856898, + "tokens_seen": 1433796608 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045705115346038116, + "loss": 1.3028, + "theoretical_loss": 3.5586121920552385, + "tokens_seen": 1433862144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045703510531594786, + "loss": 1.3872, + "theoretical_loss": 3.5585978430930694, + "tokens_seen": 1433927680 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045701905717151456, + "loss": 1.3249, + "theoretical_loss": 3.5585834949703026, + "tokens_seen": 1433993216 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045700300902708126, + "loss": 1.3845, + "theoretical_loss": 3.558569147686852, + "tokens_seen": 1434058752 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045698696088264796, + "loss": 1.3745, + "theoretical_loss": 3.5585548012426287, + "tokens_seen": 1434124288 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045697091273821466, + "loss": 1.3304, + "theoretical_loss": 3.5585404556375466, + "tokens_seen": 1434189824 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045695486459378136, + "loss": 1.2257, + "theoretical_loss": 3.558526110871517, + "tokens_seen": 1434255360 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045693881644934807, + "loss": 1.3389, + "theoretical_loss": 3.558511766944453, + "tokens_seen": 1434320896 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045692276830491477, + "loss": 1.2873, + "theoretical_loss": 3.558497423856268, + "tokens_seen": 1434386432 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004569067201604814, + "loss": 1.4315, + "theoretical_loss": 3.558483081606873, + "tokens_seen": 1434451968 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045689067201604817, + "loss": 1.2989, + "theoretical_loss": 3.558468740196182, + "tokens_seen": 1434517504 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004568746238716148, + "loss": 1.3377, + "theoretical_loss": 3.5584543996241074, + "tokens_seen": 1434583040 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045685857572718157, + "loss": 1.2988, + "theoretical_loss": 3.558440059890561, + "tokens_seen": 1434648576 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004568425275827482, + "loss": 1.3134, + "theoretical_loss": 3.558425720995456, + "tokens_seen": 1434714112 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045682647943831497, + "loss": 1.4479, + "theoretical_loss": 3.558411382938706, + "tokens_seen": 1434779648 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004568104312938816, + "loss": 1.2924, + "theoretical_loss": 3.558397045720222, + "tokens_seen": 1434845184 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045679438314944837, + "loss": 1.3414, + "theoretical_loss": 3.5583827093399183, + "tokens_seen": 1434910720 + }, + { + "epoch": 0.43, + "learning_rate": 0.000456778335005015, + "loss": 1.2888, + "theoretical_loss": 3.5583683737977063, + "tokens_seen": 1434976256 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045676228686058177, + "loss": 1.3414, + "theoretical_loss": 3.5583540390934996, + "tokens_seen": 1435041792 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004567462387161484, + "loss": 1.2786, + "theoretical_loss": 3.55833970522721, + "tokens_seen": 1435107328 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004567301905717152, + "loss": 1.2949, + "theoretical_loss": 3.558325372198752, + "tokens_seen": 1435172864 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 481622, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3161892890930176, + "objective/train/theoretical_loss": 3.5583110400080367, + "objective/train/tokens_used": 1455698400, + "theoretical_loss": 3.5583110400080367, + "tokens_seen": 1435238400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004567141424272818, + "loss": 1.379, + "theoretical_loss": 3.5583110400080367, + "tokens_seen": 1435238400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004566980942828486, + "loss": 1.3631, + "theoretical_loss": 3.5582967086549777, + "tokens_seen": 1435303936 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004566820461384152, + "loss": 1.2946, + "theoretical_loss": 3.558282378139487, + "tokens_seen": 1435369472 + }, + { + "epoch": 0.43, + "learning_rate": 0.000456665997993982, + "loss": 1.3131, + "theoretical_loss": 3.5582680484614784, + "tokens_seen": 1435435008 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004566499498495486, + "loss": 1.2119, + "theoretical_loss": 3.5582537196208643, + "tokens_seen": 1435500544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004566339017051154, + "loss": 1.3031, + "theoretical_loss": 3.5582393916175574, + "tokens_seen": 1435566080 + }, + { + "epoch": 0.44, + "learning_rate": 0.000456617853560682, + "loss": 1.3375, + "theoretical_loss": 3.558225064451471, + "tokens_seen": 1435631616 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004566018054162488, + "loss": 1.3074, + "theoretical_loss": 3.5582107381225176, + "tokens_seen": 1435697152 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004565857572718154, + "loss": 1.2224, + "theoretical_loss": 3.5581964126306103, + "tokens_seen": 1435762688 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004565697091273822, + "loss": 1.2397, + "theoretical_loss": 3.5581820879756614, + "tokens_seen": 1435828224 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004565536609829488, + "loss": 1.3009, + "theoretical_loss": 3.5581677641575844, + "tokens_seen": 1435893760 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004565376128385156, + "loss": 1.2711, + "theoretical_loss": 3.558153441176292, + "tokens_seen": 1435959296 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045652156469408223, + "loss": 1.307, + "theoretical_loss": 3.558139119031697, + "tokens_seen": 1436024832 + }, + { + "epoch": 0.44, + "learning_rate": 0.000456505516549649, + "loss": 1.437, + "theoretical_loss": 3.558124797723713, + "tokens_seen": 1436090368 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045648946840521563, + "loss": 1.3788, + "theoretical_loss": 3.558110477252252, + "tokens_seen": 1436155904 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004564734202607824, + "loss": 1.3812, + "theoretical_loss": 3.5580961576172276, + "tokens_seen": 1436221440 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045645737211634903, + "loss": 1.2538, + "theoretical_loss": 3.558081838818553, + "tokens_seen": 1436286976 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004564413239719158, + "loss": 1.3411, + "theoretical_loss": 3.5580675208561403, + "tokens_seen": 1436352512 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004564252758274825, + "loss": 1.2705, + "theoretical_loss": 3.5580532037299033, + "tokens_seen": 1436418048 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004564092276830492, + "loss": 1.3729, + "theoretical_loss": 3.5580388874397544, + "tokens_seen": 1436483584 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004563931795386159, + "loss": 1.3112, + "theoretical_loss": 3.5580245719856074, + "tokens_seen": 1436549120 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004563771313941826, + "loss": 1.2366, + "theoretical_loss": 3.558010257367375, + "tokens_seen": 1436614656 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004563610832497493, + "loss": 1.2951, + "theoretical_loss": 3.5579959435849693, + "tokens_seen": 1436680192 + }, + { + "epoch": 0.44, + "learning_rate": 0.000456345035105316, + "loss": 1.3306, + "theoretical_loss": 3.557981630638305, + "tokens_seen": 1436745728 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004563289869608827, + "loss": 1.421, + "theoretical_loss": 3.557967318527295, + "tokens_seen": 1436811264 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004563129388164494, + "loss": 1.2585, + "theoretical_loss": 3.557953007251851, + "tokens_seen": 1436876800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004562968906720161, + "loss": 1.3057, + "theoretical_loss": 3.5579386968118865, + "tokens_seen": 1436942336 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045628084252758274, + "loss": 1.2843, + "theoretical_loss": 3.557924387207316, + "tokens_seen": 1437007872 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004562647943831495, + "loss": 1.3638, + "theoretical_loss": 3.5579100784380513, + "tokens_seen": 1437073408 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045624874623871614, + "loss": 1.3064, + "theoretical_loss": 3.5578957705040057, + "tokens_seen": 1437138944 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004562326980942829, + "loss": 1.2938, + "theoretical_loss": 3.557881463405093, + "tokens_seen": 1437204480 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045621664994984954, + "loss": 1.2259, + "theoretical_loss": 3.557867157141226, + "tokens_seen": 1437270016 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004562006018054163, + "loss": 1.3679, + "theoretical_loss": 3.557852851712318, + "tokens_seen": 1437335552 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045618455366098294, + "loss": 1.3252, + "theoretical_loss": 3.557838547118282, + "tokens_seen": 1437401088 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004561685055165497, + "loss": 1.3509, + "theoretical_loss": 3.5578242433590317, + "tokens_seen": 1437466624 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045615245737211634, + "loss": 1.283, + "theoretical_loss": 3.5578099404344794, + "tokens_seen": 1437532160 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004561364092276831, + "loss": 1.3375, + "theoretical_loss": 3.5577956383445386, + "tokens_seen": 1437597696 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045612036108324974, + "loss": 1.2761, + "theoretical_loss": 3.557781337089123, + "tokens_seen": 1437663232 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004561043129388165, + "loss": 1.2984, + "theoretical_loss": 3.557767036668146, + "tokens_seen": 1437728768 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045608826479438314, + "loss": 1.3462, + "theoretical_loss": 3.557752737081521, + "tokens_seen": 1437794304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004560722166499499, + "loss": 1.2343, + "theoretical_loss": 3.55773843832916, + "tokens_seen": 1437859840 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045605616850551655, + "loss": 1.36, + "theoretical_loss": 3.5577241404109774, + "tokens_seen": 1437925376 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004560401203610833, + "loss": 1.3435, + "theoretical_loss": 3.5577098433268866, + "tokens_seen": 1437990912 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045602407221664995, + "loss": 1.4761, + "theoretical_loss": 3.5576955470768, + "tokens_seen": 1438056448 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004560080240722167, + "loss": 1.3683, + "theoretical_loss": 3.5576812516606324, + "tokens_seen": 1438121984 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045599197592778335, + "loss": 1.329, + "theoretical_loss": 3.557666957078296, + "tokens_seen": 1438187520 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004559759277833501, + "loss": 1.3379, + "theoretical_loss": 3.557652663329704, + "tokens_seen": 1438253056 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045595987963891675, + "loss": 1.4506, + "theoretical_loss": 3.5576383704147707, + "tokens_seen": 1438318592 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004559438314944835, + "loss": 1.2574, + "theoretical_loss": 3.5576240783334097, + "tokens_seen": 1438384128 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045592778335005015, + "loss": 1.2259, + "theoretical_loss": 3.557609787085533, + "tokens_seen": 1438449664 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 482849, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2713879346847534, + "objective/train/theoretical_loss": 3.5575954966710546, + "objective/train/tokens_used": 1458975200, + "theoretical_loss": 3.5575954966710546, + "tokens_seen": 1438515200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004559117352056169, + "loss": 1.2522, + "theoretical_loss": 3.5575954966710546, + "tokens_seen": 1438515200 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045589568706118355, + "loss": 1.2682, + "theoretical_loss": 3.5575812070898887, + "tokens_seen": 1438580736 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004558796389167503, + "loss": 1.3228, + "theoretical_loss": 3.557566918341948, + "tokens_seen": 1438646272 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045586359077231695, + "loss": 1.361, + "theoretical_loss": 3.5575526304271463, + "tokens_seen": 1438711808 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004558475426278837, + "loss": 1.2867, + "theoretical_loss": 3.5575383433453966, + "tokens_seen": 1438777344 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045583149448345035, + "loss": 1.2955, + "theoretical_loss": 3.557524057096613, + "tokens_seen": 1438842880 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004558154463390171, + "loss": 1.3134, + "theoretical_loss": 3.5575097716807087, + "tokens_seen": 1438908416 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045579939819458376, + "loss": 1.2428, + "theoretical_loss": 3.557495487097597, + "tokens_seen": 1438973952 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004557833500501505, + "loss": 1.3008, + "theoretical_loss": 3.557481203347192, + "tokens_seen": 1439039488 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045576730190571716, + "loss": 1.3362, + "theoretical_loss": 3.5574669204294067, + "tokens_seen": 1439105024 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004557512537612839, + "loss": 1.298, + "theoretical_loss": 3.557452638344155, + "tokens_seen": 1439170560 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045573520561685056, + "loss": 1.2211, + "theoretical_loss": 3.5574383570913497, + "tokens_seen": 1439236096 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004557191574724173, + "loss": 1.3366, + "theoretical_loss": 3.5574240766709058, + "tokens_seen": 1439301632 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045570310932798396, + "loss": 1.3071, + "theoretical_loss": 3.5574097970827356, + "tokens_seen": 1439367168 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004556870611835507, + "loss": 1.2954, + "theoretical_loss": 3.5573955183267536, + "tokens_seen": 1439432704 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045567101303911736, + "loss": 1.4258, + "theoretical_loss": 3.5573812404028726, + "tokens_seen": 1439498240 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004556549648946841, + "loss": 1.3103, + "theoretical_loss": 3.557366963311007, + "tokens_seen": 1439563776 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045563891675025076, + "loss": 1.3403, + "theoretical_loss": 3.5573526870510697, + "tokens_seen": 1439629312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045562286860581746, + "loss": 1.3593, + "theoretical_loss": 3.5573384116229745, + "tokens_seen": 1439694848 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045560682046138416, + "loss": 1.3412, + "theoretical_loss": 3.5573241370266357, + "tokens_seen": 1439760384 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045559077231695086, + "loss": 1.3049, + "theoretical_loss": 3.557309863261967, + "tokens_seen": 1439825920 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045557472417251756, + "loss": 1.2632, + "theoretical_loss": 3.5572955903288808, + "tokens_seen": 1439891456 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045555867602808427, + "loss": 1.2165, + "theoretical_loss": 3.557281318227292, + "tokens_seen": 1439956992 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045554262788365097, + "loss": 1.2038, + "theoretical_loss": 3.557267046957114, + "tokens_seen": 1440022528 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045552657973921767, + "loss": 1.4544, + "theoretical_loss": 3.5572527765182604, + "tokens_seen": 1440088064 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004555105315947843, + "loss": 1.3677, + "theoretical_loss": 3.557238506910646, + "tokens_seen": 1440153600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045549448345035107, + "loss": 1.3547, + "theoretical_loss": 3.5572242381341823, + "tokens_seen": 1440219136 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004554784353059177, + "loss": 1.3557, + "theoretical_loss": 3.557209970188785, + "tokens_seen": 1440284672 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045546238716148447, + "loss": 1.2771, + "theoretical_loss": 3.5571957030743677, + "tokens_seen": 1440350208 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004554463390170511, + "loss": 1.2832, + "theoretical_loss": 3.557181436790843, + "tokens_seen": 1440415744 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045543029087261787, + "loss": 1.3751, + "theoretical_loss": 3.5571671713381257, + "tokens_seen": 1440481280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004554142427281845, + "loss": 1.2117, + "theoretical_loss": 3.5571529067161296, + "tokens_seen": 1440546816 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045539819458375127, + "loss": 1.3891, + "theoretical_loss": 3.5571386429247687, + "tokens_seen": 1440612352 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004553821464393179, + "loss": 1.3857, + "theoretical_loss": 3.557124379963956, + "tokens_seen": 1440677888 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004553660982948847, + "loss": 1.4717, + "theoretical_loss": 3.5571101178336058, + "tokens_seen": 1440743424 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004553500501504513, + "loss": 1.3517, + "theoretical_loss": 3.557095856533632, + "tokens_seen": 1440808960 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004553340020060181, + "loss": 1.1584, + "theoretical_loss": 3.5570815960639486, + "tokens_seen": 1440874496 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004553179538615847, + "loss": 1.3169, + "theoretical_loss": 3.5570673364244696, + "tokens_seen": 1440940032 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004553019057171515, + "loss": 1.4056, + "theoretical_loss": 3.557053077615109, + "tokens_seen": 1441005568 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004552858575727181, + "loss": 1.2782, + "theoretical_loss": 3.5570388196357796, + "tokens_seen": 1441071104 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004552698094282849, + "loss": 1.338, + "theoretical_loss": 3.5570245624863968, + "tokens_seen": 1441136640 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045525376128385163, + "loss": 1.2896, + "theoretical_loss": 3.557010306166874, + "tokens_seen": 1441202176 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004552377131394183, + "loss": 1.3373, + "theoretical_loss": 3.5569960506771245, + "tokens_seen": 1441267712 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045522166499498503, + "loss": 1.4077, + "theoretical_loss": 3.5569817960170633, + "tokens_seen": 1441333248 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004552056168505517, + "loss": 1.2653, + "theoretical_loss": 3.5569675421866043, + "tokens_seen": 1441398784 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045518956870611843, + "loss": 1.2131, + "theoretical_loss": 3.5569532891856603, + "tokens_seen": 1441464320 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004551735205616851, + "loss": 1.2864, + "theoretical_loss": 3.556939037014147, + "tokens_seen": 1441529856 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045515747241725184, + "loss": 1.2779, + "theoretical_loss": 3.5569247856719772, + "tokens_seen": 1441595392 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004551414242728185, + "loss": 1.339, + "theoretical_loss": 3.5569105351590657, + "tokens_seen": 1441660928 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045512537612838524, + "loss": 1.3434, + "theoretical_loss": 3.556896285475326, + "tokens_seen": 1441726464 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 483810, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2746837139129639, + "objective/train/theoretical_loss": 3.5568820366206726, + "objective/train/tokens_used": 1462252000, + "theoretical_loss": 3.5568820366206726, + "tokens_seen": 1441792000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004551093279839519, + "loss": 1.3041, + "theoretical_loss": 3.5568820366206726, + "tokens_seen": 1441792000 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045509327983951864, + "loss": 1.2942, + "theoretical_loss": 3.556867788595019, + "tokens_seen": 1441857536 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004550772316950853, + "loss": 1.2462, + "theoretical_loss": 3.5568535413982794, + "tokens_seen": 1441923072 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045506118355065204, + "loss": 1.4316, + "theoretical_loss": 3.556839295030369, + "tokens_seen": 1441988608 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004550451354062187, + "loss": 1.3575, + "theoretical_loss": 3.5568250494912004, + "tokens_seen": 1442054144 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045502908726178544, + "loss": 1.3017, + "theoretical_loss": 3.556810804780689, + "tokens_seen": 1442119680 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004550130391173521, + "loss": 1.3111, + "theoretical_loss": 3.5567965608987473, + "tokens_seen": 1442185216 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004549969909729188, + "loss": 1.2188, + "theoretical_loss": 3.5567823178452915, + "tokens_seen": 1442250752 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004549809428284855, + "loss": 1.2008, + "theoretical_loss": 3.5567680756202344, + "tokens_seen": 1442316288 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004549648946840522, + "loss": 1.2739, + "theoretical_loss": 3.5567538342234903, + "tokens_seen": 1442381824 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004549488465396189, + "loss": 1.18, + "theoretical_loss": 3.556739593654974, + "tokens_seen": 1442447360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004549327983951856, + "loss": 1.2894, + "theoretical_loss": 3.556725353914599, + "tokens_seen": 1442512896 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004549167502507523, + "loss": 1.2844, + "theoretical_loss": 3.5567111150022805, + "tokens_seen": 1442578432 + }, + { + "epoch": 0.44, + "learning_rate": 0.000454900702106319, + "loss": 1.4224, + "theoretical_loss": 3.5566968769179317, + "tokens_seen": 1442643968 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004548846539618857, + "loss": 1.3313, + "theoretical_loss": 3.5566826396614672, + "tokens_seen": 1442709504 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004548686058174524, + "loss": 1.3673, + "theoretical_loss": 3.556668403232801, + "tokens_seen": 1442775040 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045485255767301904, + "loss": 1.2996, + "theoretical_loss": 3.556654167631848, + "tokens_seen": 1442840576 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004548365095285858, + "loss": 1.3492, + "theoretical_loss": 3.5566399328585225, + "tokens_seen": 1442906112 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045482046138415244, + "loss": 1.3996, + "theoretical_loss": 3.556625698912738, + "tokens_seen": 1442971648 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004548044132397192, + "loss": 1.3516, + "theoretical_loss": 3.5566114657944095, + "tokens_seen": 1443037184 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045478836509528584, + "loss": 1.3208, + "theoretical_loss": 3.5565972335034512, + "tokens_seen": 1443102720 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004547723169508526, + "loss": 1.2532, + "theoretical_loss": 3.556583002039777, + "tokens_seen": 1443168256 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045475626880641924, + "loss": 1.2962, + "theoretical_loss": 3.5565687714033016, + "tokens_seen": 1443233792 + }, + { + "epoch": 0.44, + "learning_rate": 0.000454740220661986, + "loss": 1.2418, + "theoretical_loss": 3.5565545415939397, + "tokens_seen": 1443299328 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045472417251755264, + "loss": 1.279, + "theoretical_loss": 3.556540312611605, + "tokens_seen": 1443364864 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004547081243731194, + "loss": 1.3042, + "theoretical_loss": 3.5565260844562117, + "tokens_seen": 1443430400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045469207622868605, + "loss": 1.3724, + "theoretical_loss": 3.556511857127675, + "tokens_seen": 1443495936 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004546760280842528, + "loss": 1.2836, + "theoretical_loss": 3.5564976306259095, + "tokens_seen": 1443561472 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045465997993981945, + "loss": 1.3978, + "theoretical_loss": 3.556483404950829, + "tokens_seen": 1443627008 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004546439317953862, + "loss": 1.3526, + "theoretical_loss": 3.556469180102348, + "tokens_seen": 1443692544 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045462788365095285, + "loss": 1.3248, + "theoretical_loss": 3.5564549560803806, + "tokens_seen": 1443758080 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004546118355065196, + "loss": 1.2878, + "theoretical_loss": 3.556440732884842, + "tokens_seen": 1443823616 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045459578736208625, + "loss": 1.3183, + "theoretical_loss": 3.5564265105156463, + "tokens_seen": 1443889152 + }, + { + "epoch": 0.44, + "learning_rate": 0.000454579739217653, + "loss": 1.2843, + "theoretical_loss": 3.556412288972708, + "tokens_seen": 1443954688 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045456369107321965, + "loss": 1.3958, + "theoretical_loss": 3.556398068255942, + "tokens_seen": 1444020224 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004545476429287864, + "loss": 1.3342, + "theoretical_loss": 3.556383848365262, + "tokens_seen": 1444085760 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045453159478435305, + "loss": 1.4241, + "theoretical_loss": 3.556369629300583, + "tokens_seen": 1444151296 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004545155466399198, + "loss": 1.2689, + "theoretical_loss": 3.5563554110618196, + "tokens_seen": 1444216832 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045449949849548645, + "loss": 1.3332, + "theoretical_loss": 3.556341193648886, + "tokens_seen": 1444282368 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004544834503510532, + "loss": 1.2805, + "theoretical_loss": 3.5563269770616976, + "tokens_seen": 1444347904 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045446740220661985, + "loss": 1.3438, + "theoretical_loss": 3.556312761300168, + "tokens_seen": 1444413440 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004544513540621866, + "loss": 1.3181, + "theoretical_loss": 3.5562985463642125, + "tokens_seen": 1444478976 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045443530591775326, + "loss": 1.3549, + "theoretical_loss": 3.556284332253745, + "tokens_seen": 1444544512 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045441925777332, + "loss": 1.3919, + "theoretical_loss": 3.5562701189686807, + "tokens_seen": 1444610048 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045440320962888666, + "loss": 1.3744, + "theoretical_loss": 3.556255906508934, + "tokens_seen": 1444675584 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004543871614844534, + "loss": 1.2959, + "theoretical_loss": 3.5562416948744193, + "tokens_seen": 1444741120 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045437111334002006, + "loss": 1.1631, + "theoretical_loss": 3.5562274840650523, + "tokens_seen": 1444806656 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004543550651955868, + "loss": 1.4316, + "theoretical_loss": 3.556213274080746, + "tokens_seen": 1444872192 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045433901705115346, + "loss": 1.3729, + "theoretical_loss": 3.5561990649214166, + "tokens_seen": 1444937728 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004543229689067202, + "loss": 1.3261, + "theoretical_loss": 3.5561848565869782, + "tokens_seen": 1445003264 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 485005, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2077436447143555, + "objective/train/theoretical_loss": 3.556170649077345, + "objective/train/tokens_used": 1465528800, + "theoretical_loss": 3.556170649077345, + "tokens_seen": 1445068800 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045430692076228686, + "loss": 1.299, + "theoretical_loss": 3.556170649077345, + "tokens_seen": 1445068800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004542908726178536, + "loss": 1.3906, + "theoretical_loss": 3.5561564423924326, + "tokens_seen": 1445134336 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045427482447342026, + "loss": 1.3759, + "theoretical_loss": 3.556142236532155, + "tokens_seen": 1445199872 + }, + { + "epoch": 0.44, + "learning_rate": 0.000454258776328987, + "loss": 1.1993, + "theoretical_loss": 3.556128031496428, + "tokens_seen": 1445265408 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045424272818455366, + "loss": 1.3354, + "theoretical_loss": 3.5561138272851647, + "tokens_seen": 1445330944 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045422668004012036, + "loss": 1.3714, + "theoretical_loss": 3.556099623898281, + "tokens_seen": 1445396480 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045421063189568706, + "loss": 1.3137, + "theoretical_loss": 3.5560854213356916, + "tokens_seen": 1445462016 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045419458375125377, + "loss": 1.3455, + "theoretical_loss": 3.556071219597311, + "tokens_seen": 1445527552 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045417853560682047, + "loss": 1.3484, + "theoretical_loss": 3.5560570186830542, + "tokens_seen": 1445593088 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045416248746238717, + "loss": 1.3403, + "theoretical_loss": 3.556042818592836, + "tokens_seen": 1445658624 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045414643931795387, + "loss": 1.2567, + "theoretical_loss": 3.5560286193265713, + "tokens_seen": 1445724160 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045413039117352057, + "loss": 1.3307, + "theoretical_loss": 3.556014420884175, + "tokens_seen": 1445789696 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045411434302908727, + "loss": 1.3201, + "theoretical_loss": 3.5560002232655616, + "tokens_seen": 1445855232 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045409829488465397, + "loss": 1.2798, + "theoretical_loss": 3.555986026470646, + "tokens_seen": 1445920768 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004540822467402206, + "loss": 1.2078, + "theoretical_loss": 3.5559718304993435, + "tokens_seen": 1445986304 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045406619859578737, + "loss": 1.3716, + "theoretical_loss": 3.5559576353515685, + "tokens_seen": 1446051840 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004540501504513541, + "loss": 1.3648, + "theoretical_loss": 3.555943441027236, + "tokens_seen": 1446117376 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045403410230692077, + "loss": 1.2856, + "theoretical_loss": 3.5559292475262616, + "tokens_seen": 1446182912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004540180541624875, + "loss": 1.4444, + "theoretical_loss": 3.5559150548485596, + "tokens_seen": 1446248448 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045400200601805417, + "loss": 1.3344, + "theoretical_loss": 3.555900862994045, + "tokens_seen": 1446313984 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045398595787362093, + "loss": 1.2735, + "theoretical_loss": 3.5558866719626328, + "tokens_seen": 1446379520 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004539699097291876, + "loss": 1.3613, + "theoretical_loss": 3.555872481754238, + "tokens_seen": 1446445056 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045395386158475433, + "loss": 1.3414, + "theoretical_loss": 3.555858292368775, + "tokens_seen": 1446510592 + }, + { + "epoch": 0.44, + "learning_rate": 0.000453937813440321, + "loss": 1.3846, + "theoretical_loss": 3.5558441038061606, + "tokens_seen": 1446576128 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045392176529588773, + "loss": 1.2806, + "theoretical_loss": 3.5558299160663074, + "tokens_seen": 1446641664 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004539057171514544, + "loss": 1.3078, + "theoretical_loss": 3.555815729149132, + "tokens_seen": 1446707200 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045388966900702113, + "loss": 1.435, + "theoretical_loss": 3.555801543054549, + "tokens_seen": 1446772736 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004538736208625878, + "loss": 1.3391, + "theoretical_loss": 3.5557873577824735, + "tokens_seen": 1446838272 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045385757271815453, + "loss": 1.2468, + "theoretical_loss": 3.5557731733328204, + "tokens_seen": 1446903808 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004538415245737212, + "loss": 1.3014, + "theoretical_loss": 3.555758989705505, + "tokens_seen": 1446969344 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045382547642928793, + "loss": 1.2969, + "theoretical_loss": 3.5557448069004423, + "tokens_seen": 1447034880 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004538094282848546, + "loss": 1.2944, + "theoretical_loss": 3.555730624917547, + "tokens_seen": 1447100416 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045379338014042134, + "loss": 1.2632, + "theoretical_loss": 3.555716443756735, + "tokens_seen": 1447165952 + }, + { + "epoch": 0.44, + "learning_rate": 0.000453777331995988, + "loss": 1.3009, + "theoretical_loss": 3.555702263417921, + "tokens_seen": 1447231488 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045376128385155474, + "loss": 1.3152, + "theoretical_loss": 3.55568808390102, + "tokens_seen": 1447297024 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004537452357071214, + "loss": 1.3284, + "theoretical_loss": 3.555673905205947, + "tokens_seen": 1447362560 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045372918756268814, + "loss": 1.4173, + "theoretical_loss": 3.555659727332618, + "tokens_seen": 1447428096 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004537131394182548, + "loss": 1.4036, + "theoretical_loss": 3.555645550280947, + "tokens_seen": 1447493632 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045369709127382154, + "loss": 1.3324, + "theoretical_loss": 3.5556313740508503, + "tokens_seen": 1447559168 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004536810431293882, + "loss": 1.347, + "theoretical_loss": 3.5556171986422425, + "tokens_seen": 1447624704 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045366499498495494, + "loss": 1.3555, + "theoretical_loss": 3.5556030240550385, + "tokens_seen": 1447690240 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004536489468405216, + "loss": 1.26, + "theoretical_loss": 3.5555888502891544, + "tokens_seen": 1447755776 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045363289869608834, + "loss": 1.3767, + "theoretical_loss": 3.555574677344505, + "tokens_seen": 1447821312 + }, + { + "epoch": 0.44, + "learning_rate": 0.000453616850551655, + "loss": 1.3768, + "theoretical_loss": 3.5555605052210053, + "tokens_seen": 1447886848 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004536008024072217, + "loss": 1.3414, + "theoretical_loss": 3.5555463339185707, + "tokens_seen": 1447952384 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004535847542627884, + "loss": 1.3862, + "theoretical_loss": 3.555532163437116, + "tokens_seen": 1448017920 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004535687061183551, + "loss": 1.4174, + "theoretical_loss": 3.555517993776558, + "tokens_seen": 1448083456 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004535526579739218, + "loss": 1.3539, + "theoretical_loss": 3.5555038249368103, + "tokens_seen": 1448148992 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004535366098294885, + "loss": 1.3942, + "theoretical_loss": 3.555489656917789, + "tokens_seen": 1448214528 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004535205616850552, + "loss": 1.3464, + "theoretical_loss": 3.5554754897194103, + "tokens_seen": 1448280064 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 485883, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.440194845199585, + "objective/train/theoretical_loss": 3.5554613233415875, + "objective/train/tokens_used": 1468805600, + "theoretical_loss": 3.5554613233415875, + "tokens_seen": 1448345600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004535045135406219, + "loss": 1.3957, + "theoretical_loss": 3.5554613233415875, + "tokens_seen": 1448345600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004534884653961886, + "loss": 1.1841, + "theoretical_loss": 3.5554471577842373, + "tokens_seen": 1448411136 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004534724172517553, + "loss": 1.2246, + "theoretical_loss": 3.5554329930472752, + "tokens_seen": 1448476672 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045345636910732194, + "loss": 1.3249, + "theoretical_loss": 3.5554188291306157, + "tokens_seen": 1448542208 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004534403209628887, + "loss": 1.2904, + "theoretical_loss": 3.5554046660341747, + "tokens_seen": 1448607744 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045342427281845534, + "loss": 1.3259, + "theoretical_loss": 3.5553905037578675, + "tokens_seen": 1448673280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004534082246740221, + "loss": 1.3397, + "theoretical_loss": 3.55537634230161, + "tokens_seen": 1448738816 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045339217652958874, + "loss": 1.2773, + "theoretical_loss": 3.5553621816653167, + "tokens_seen": 1448804352 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004533761283851555, + "loss": 1.2989, + "theoretical_loss": 3.5553480218489035, + "tokens_seen": 1448869888 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045336008024072214, + "loss": 1.3951, + "theoretical_loss": 3.5553338628522866, + "tokens_seen": 1448935424 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004533440320962889, + "loss": 1.2479, + "theoretical_loss": 3.55531970467538, + "tokens_seen": 1449000960 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045332798395185555, + "loss": 1.2973, + "theoretical_loss": 3.5553055473181, + "tokens_seen": 1449066496 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004533119358074223, + "loss": 1.3126, + "theoretical_loss": 3.5552913907803623, + "tokens_seen": 1449132032 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045329588766298895, + "loss": 1.4397, + "theoretical_loss": 3.5552772350620816, + "tokens_seen": 1449197568 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004532798395185557, + "loss": 1.284, + "theoretical_loss": 3.555263080163174, + "tokens_seen": 1449263104 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045326379137412235, + "loss": 1.3189, + "theoretical_loss": 3.5552489260835554, + "tokens_seen": 1449328640 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004532477432296891, + "loss": 1.2923, + "theoretical_loss": 3.55523477282314, + "tokens_seen": 1449394176 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045323169508525575, + "loss": 1.384, + "theoretical_loss": 3.555220620381845, + "tokens_seen": 1449459712 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004532156469408225, + "loss": 1.3066, + "theoretical_loss": 3.555206468759585, + "tokens_seen": 1449525248 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045319959879638915, + "loss": 1.329, + "theoretical_loss": 3.5551923179562754, + "tokens_seen": 1449590784 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004531835506519559, + "loss": 1.2898, + "theoretical_loss": 3.5551781679718326, + "tokens_seen": 1449656320 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045316750250752255, + "loss": 1.4265, + "theoretical_loss": 3.5551640188061713, + "tokens_seen": 1449721856 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004531514543630893, + "loss": 1.2649, + "theoretical_loss": 3.555149870459208, + "tokens_seen": 1449787392 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045313540621865595, + "loss": 1.3998, + "theoretical_loss": 3.5551357229308573, + "tokens_seen": 1449852928 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004531193580742227, + "loss": 1.2399, + "theoretical_loss": 3.5551215762210355, + "tokens_seen": 1449918464 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045310330992978935, + "loss": 1.2495, + "theoretical_loss": 3.5551074303296586, + "tokens_seen": 1449984000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004530872617853561, + "loss": 1.2773, + "theoretical_loss": 3.5550932852566413, + "tokens_seen": 1450049536 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045307121364092276, + "loss": 1.3231, + "theoretical_loss": 3.5550791410019, + "tokens_seen": 1450115072 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004530551654964895, + "loss": 1.2919, + "theoretical_loss": 3.55506499756535, + "tokens_seen": 1450180608 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045303911735205616, + "loss": 1.4347, + "theoretical_loss": 3.555050854946907, + "tokens_seen": 1450246144 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004530230692076229, + "loss": 1.3137, + "theoretical_loss": 3.5550367131464875, + "tokens_seen": 1450311680 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045300702106318956, + "loss": 1.2355, + "theoretical_loss": 3.555022572164006, + "tokens_seen": 1450377216 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004529909729187563, + "loss": 1.2252, + "theoretical_loss": 3.5550084319993793, + "tokens_seen": 1450442752 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045297492477432296, + "loss": 1.2676, + "theoretical_loss": 3.5549942926525224, + "tokens_seen": 1450508288 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004529588766298897, + "loss": 1.3313, + "theoretical_loss": 3.554980154123351, + "tokens_seen": 1450573824 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045294282848545636, + "loss": 1.2972, + "theoretical_loss": 3.554966016411782, + "tokens_seen": 1450639360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004529267803410231, + "loss": 1.2923, + "theoretical_loss": 3.5549518795177297, + "tokens_seen": 1450704896 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045291073219658976, + "loss": 1.3751, + "theoretical_loss": 3.554937743441111, + "tokens_seen": 1450770432 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004528946840521565, + "loss": 1.347, + "theoretical_loss": 3.5549236081818414, + "tokens_seen": 1450835968 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045287863590772316, + "loss": 1.3123, + "theoretical_loss": 3.554909473739836, + "tokens_seen": 1450901504 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004528625877632899, + "loss": 1.3425, + "theoretical_loss": 3.554895340115012, + "tokens_seen": 1450967040 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004528465396188566, + "loss": 1.3134, + "theoretical_loss": 3.554881207307284, + "tokens_seen": 1451032576 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045283049147442326, + "loss": 1.2973, + "theoretical_loss": 3.5548670753165688, + "tokens_seen": 1451098112 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045281444332999, + "loss": 1.1967, + "theoretical_loss": 3.554852944142781, + "tokens_seen": 1451163648 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045279839518555667, + "loss": 1.1971, + "theoretical_loss": 3.5548388137858384, + "tokens_seen": 1451229184 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004527823470411234, + "loss": 1.2516, + "theoretical_loss": 3.5548246842456557, + "tokens_seen": 1451294720 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045276629889669007, + "loss": 1.2867, + "theoretical_loss": 3.554810555522148, + "tokens_seen": 1451360256 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004527502507522568, + "loss": 1.3382, + "theoretical_loss": 3.554796427615233, + "tokens_seen": 1451425792 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045273420260782347, + "loss": 1.2944, + "theoretical_loss": 3.554782300524826, + "tokens_seen": 1451491328 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004527181544633902, + "loss": 1.3326, + "theoretical_loss": 3.5547681742508423, + "tokens_seen": 1451556864 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 487012, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2259671688079834, + "objective/train/theoretical_loss": 3.554754048793198, + "objective/train/tokens_used": 1472082400, + "theoretical_loss": 3.554754048793198, + "tokens_seen": 1451622400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045270210631895687, + "loss": 1.2535, + "theoretical_loss": 3.554754048793198, + "tokens_seen": 1451622400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004526860581745236, + "loss": 1.3089, + "theoretical_loss": 3.55473992415181, + "tokens_seen": 1451687936 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045267001003009027, + "loss": 1.313, + "theoretical_loss": 3.5547258003265934, + "tokens_seen": 1451753472 + }, + { + "epoch": 0.44, + "learning_rate": 0.000452653961885657, + "loss": 1.2195, + "theoretical_loss": 3.5547116773174645, + "tokens_seen": 1451819008 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045263791374122367, + "loss": 1.2883, + "theoretical_loss": 3.554697555124339, + "tokens_seen": 1451884544 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045262186559679043, + "loss": 1.335, + "theoretical_loss": 3.5546834337471336, + "tokens_seen": 1451950080 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004526058174523571, + "loss": 1.3661, + "theoretical_loss": 3.554669313185764, + "tokens_seen": 1452015616 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045258976930792383, + "loss": 1.3341, + "theoretical_loss": 3.5546551934401456, + "tokens_seen": 1452081152 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004525737211634905, + "loss": 1.3922, + "theoretical_loss": 3.5546410745101955, + "tokens_seen": 1452146688 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045255767301905723, + "loss": 1.3309, + "theoretical_loss": 3.554626956395829, + "tokens_seen": 1452212224 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004525416248746239, + "loss": 1.3254, + "theoretical_loss": 3.554612839096963, + "tokens_seen": 1452277760 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045252557673019063, + "loss": 1.2518, + "theoretical_loss": 3.5545987226135125, + "tokens_seen": 1452343296 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004525095285857573, + "loss": 1.2557, + "theoretical_loss": 3.5545846069453946, + "tokens_seen": 1452408832 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045249348044132403, + "loss": 1.3505, + "theoretical_loss": 3.554570492092525, + "tokens_seen": 1452474368 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004524774322968907, + "loss": 1.2946, + "theoretical_loss": 3.55455637805482, + "tokens_seen": 1452539904 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045246138415245743, + "loss": 1.2938, + "theoretical_loss": 3.5545422648321954, + "tokens_seen": 1452605440 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004524453360080241, + "loss": 1.4115, + "theoretical_loss": 3.5545281524245675, + "tokens_seen": 1452670976 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045242928786359083, + "loss": 1.256, + "theoretical_loss": 3.5545140408318527, + "tokens_seen": 1452736512 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004524132397191575, + "loss": 1.3221, + "theoretical_loss": 3.554499930053967, + "tokens_seen": 1452802048 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045239719157472424, + "loss": 1.3062, + "theoretical_loss": 3.5544858200908265, + "tokens_seen": 1452867584 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004523811434302909, + "loss": 1.3071, + "theoretical_loss": 3.5544717109423476, + "tokens_seen": 1452933120 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045236509528585764, + "loss": 1.275, + "theoretical_loss": 3.5544576026084465, + "tokens_seen": 1452998656 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004523490471414243, + "loss": 1.2948, + "theoretical_loss": 3.554443495089039, + "tokens_seen": 1453064192 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045233299899699104, + "loss": 1.3114, + "theoretical_loss": 3.554429388384042, + "tokens_seen": 1453129728 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004523169508525577, + "loss": 1.2334, + "theoretical_loss": 3.554415282493372, + "tokens_seen": 1453195264 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045230090270812444, + "loss": 1.3717, + "theoretical_loss": 3.5544011774169446, + "tokens_seen": 1453260800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004522848545636911, + "loss": 1.3352, + "theoretical_loss": 3.554387073154676, + "tokens_seen": 1453326336 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045226880641925784, + "loss": 1.3201, + "theoretical_loss": 3.554372969706482, + "tokens_seen": 1453391872 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004522527582748245, + "loss": 1.1697, + "theoretical_loss": 3.5543588670722808, + "tokens_seen": 1453457408 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045223671013039124, + "loss": 1.3069, + "theoretical_loss": 3.554344765251987, + "tokens_seen": 1453522944 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004522206619859579, + "loss": 1.2158, + "theoretical_loss": 3.5543306642455175, + "tokens_seen": 1453588480 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004522046138415246, + "loss": 1.2928, + "theoretical_loss": 3.5543165640527885, + "tokens_seen": 1453654016 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004521885656970913, + "loss": 1.2231, + "theoretical_loss": 3.554302464673717, + "tokens_seen": 1453719552 + }, + { + "epoch": 0.44, + "learning_rate": 0.000452172517552658, + "loss": 1.3529, + "theoretical_loss": 3.554288366108219, + "tokens_seen": 1453785088 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004521564694082247, + "loss": 1.3699, + "theoretical_loss": 3.55427426835621, + "tokens_seen": 1453850624 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004521404212637914, + "loss": 1.3374, + "theoretical_loss": 3.5542601714176074, + "tokens_seen": 1453916160 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004521243731193581, + "loss": 1.2796, + "theoretical_loss": 3.554246075292328, + "tokens_seen": 1453981696 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004521083249749248, + "loss": 1.2764, + "theoretical_loss": 3.554231979980287, + "tokens_seen": 1454047232 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004520922768304915, + "loss": 1.2804, + "theoretical_loss": 3.5542178854814015, + "tokens_seen": 1454112768 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004520762286860582, + "loss": 1.2819, + "theoretical_loss": 3.5542037917955875, + "tokens_seen": 1454178304 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045206018054162484, + "loss": 1.3271, + "theoretical_loss": 3.5541896989227624, + "tokens_seen": 1454243840 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004520441323971916, + "loss": 1.3196, + "theoretical_loss": 3.5541756068628416, + "tokens_seen": 1454309376 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045202808425275824, + "loss": 1.2465, + "theoretical_loss": 3.5541615156157422, + "tokens_seen": 1454374912 + }, + { + "epoch": 0.44, + "learning_rate": 0.000452012036108325, + "loss": 1.3355, + "theoretical_loss": 3.5541474251813807, + "tokens_seen": 1454440448 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045199598796389164, + "loss": 1.3128, + "theoretical_loss": 3.5541333355596736, + "tokens_seen": 1454505984 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004519799398194584, + "loss": 1.266, + "theoretical_loss": 3.5541192467505365, + "tokens_seen": 1454571520 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045196389167502504, + "loss": 1.2017, + "theoretical_loss": 3.554105158753887, + "tokens_seen": 1454637056 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004519478435305918, + "loss": 1.2959, + "theoretical_loss": 3.5540910715696423, + "tokens_seen": 1454702592 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045193179538615845, + "loss": 1.2276, + "theoretical_loss": 3.5540769851977174, + "tokens_seen": 1454768128 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004519157472417252, + "loss": 1.2987, + "theoretical_loss": 3.554062899638029, + "tokens_seen": 1454833664 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 487903, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4405250549316406, + "objective/train/theoretical_loss": 3.5540488148904945, + "objective/train/tokens_used": 1475359200, + "theoretical_loss": 3.5540488148904945, + "tokens_seen": 1454899200 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045189969909729185, + "loss": 1.313, + "theoretical_loss": 3.5540488148904945, + "tokens_seen": 1454899200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004518836509528586, + "loss": 1.3153, + "theoretical_loss": 3.5540347309550304, + "tokens_seen": 1454964736 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045186760280842525, + "loss": 1.3729, + "theoretical_loss": 3.554020647831553, + "tokens_seen": 1455030272 + }, + { + "epoch": 0.44, + "learning_rate": 0.000451851554663992, + "loss": 1.2942, + "theoretical_loss": 3.5540065655199786, + "tokens_seen": 1455095808 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045183550651955865, + "loss": 1.2284, + "theoretical_loss": 3.553992484020224, + "tokens_seen": 1455161344 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004518194583751254, + "loss": 1.3839, + "theoretical_loss": 3.5539784033322066, + "tokens_seen": 1455226880 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045180341023069205, + "loss": 1.2961, + "theoretical_loss": 3.5539643234558422, + "tokens_seen": 1455292416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004517873620862588, + "loss": 1.28, + "theoretical_loss": 3.553950244391048, + "tokens_seen": 1455357952 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045177131394182545, + "loss": 1.2359, + "theoretical_loss": 3.55393616613774, + "tokens_seen": 1455423488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004517552657973922, + "loss": 1.2961, + "theoretical_loss": 3.5539220886958356, + "tokens_seen": 1455489024 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045173921765295885, + "loss": 1.2413, + "theoretical_loss": 3.5539080120652513, + "tokens_seen": 1455554560 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004517231695085256, + "loss": 1.3649, + "theoretical_loss": 3.5538939362459034, + "tokens_seen": 1455620096 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045170712136409226, + "loss": 1.2859, + "theoretical_loss": 3.553879861237709, + "tokens_seen": 1455685632 + }, + { + "epoch": 0.44, + "learning_rate": 0.000451691073219659, + "loss": 1.287, + "theoretical_loss": 3.553865787040585, + "tokens_seen": 1455751168 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045167502507522566, + "loss": 1.3115, + "theoretical_loss": 3.5538517136544483, + "tokens_seen": 1455816704 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004516589769307924, + "loss": 1.2647, + "theoretical_loss": 3.553837641079215, + "tokens_seen": 1455882240 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045164292878635917, + "loss": 1.2529, + "theoretical_loss": 3.553823569314802, + "tokens_seen": 1455947776 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004516268806419258, + "loss": 1.3434, + "theoretical_loss": 3.5538094983611264, + "tokens_seen": 1456013312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045161083249749257, + "loss": 1.1867, + "theoretical_loss": 3.553795428218105, + "tokens_seen": 1456078848 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004515947843530592, + "loss": 1.387, + "theoretical_loss": 3.5537813588856544, + "tokens_seen": 1456144384 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004515787362086259, + "loss": 1.3508, + "theoretical_loss": 3.553767290363692, + "tokens_seen": 1456209920 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004515626880641926, + "loss": 1.3123, + "theoretical_loss": 3.5537532226521336, + "tokens_seen": 1456275456 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004515466399197593, + "loss": 1.3477, + "theoretical_loss": 3.553739155750897, + "tokens_seen": 1456340992 + }, + { + "epoch": 0.44, + "learning_rate": 0.000451530591775326, + "loss": 1.3485, + "theoretical_loss": 3.553725089659898, + "tokens_seen": 1456406528 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004515145436308927, + "loss": 1.3496, + "theoretical_loss": 3.5537110243790546, + "tokens_seen": 1456472064 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004514984954864594, + "loss": 1.4036, + "theoretical_loss": 3.553696959908283, + "tokens_seen": 1456537600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004514824473420261, + "loss": 1.3022, + "theoretical_loss": 3.553682896247501, + "tokens_seen": 1456603136 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004514663991975928, + "loss": 1.2692, + "theoretical_loss": 3.5536688333966246, + "tokens_seen": 1456668672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004514503510531595, + "loss": 1.1335, + "theoretical_loss": 3.5536547713555704, + "tokens_seen": 1456734208 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045143430290872617, + "loss": 1.3056, + "theoretical_loss": 3.5536407101242564, + "tokens_seen": 1456799744 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004514182547642929, + "loss": 1.2785, + "theoretical_loss": 3.553626649702599, + "tokens_seen": 1456865280 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045140220661985957, + "loss": 1.2605, + "theoretical_loss": 3.5536125900905153, + "tokens_seen": 1456930816 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004513861584754263, + "loss": 1.2189, + "theoretical_loss": 3.553598531287922, + "tokens_seen": 1456996352 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045137011033099297, + "loss": 1.2632, + "theoretical_loss": 3.5535844732947366, + "tokens_seen": 1457061888 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004513540621865597, + "loss": 1.3776, + "theoretical_loss": 3.553570416110875, + "tokens_seen": 1457127424 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045133801404212637, + "loss": 1.3405, + "theoretical_loss": 3.553556359736256, + "tokens_seen": 1457192960 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004513219658976931, + "loss": 1.3776, + "theoretical_loss": 3.553542304170795, + "tokens_seen": 1457258496 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045130591775325977, + "loss": 1.3449, + "theoretical_loss": 3.55352824941441, + "tokens_seen": 1457324032 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004512898696088265, + "loss": 1.2835, + "theoretical_loss": 3.5535141954670175, + "tokens_seen": 1457389568 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045127382146439317, + "loss": 1.3431, + "theoretical_loss": 3.5535001423285344, + "tokens_seen": 1457455104 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004512577733199599, + "loss": 1.3769, + "theoretical_loss": 3.5534860899988785, + "tokens_seen": 1457520640 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004512417251755266, + "loss": 1.3933, + "theoretical_loss": 3.5534720384779668, + "tokens_seen": 1457586176 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045122567703109333, + "loss": 1.3153, + "theoretical_loss": 3.5534579877657153, + "tokens_seen": 1457651712 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045120962888666, + "loss": 1.3405, + "theoretical_loss": 3.5534439378620424, + "tokens_seen": 1457717248 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045119358074222673, + "loss": 1.2997, + "theoretical_loss": 3.553429888766865, + "tokens_seen": 1457782784 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004511775325977934, + "loss": 1.3291, + "theoretical_loss": 3.553415840480099, + "tokens_seen": 1457848320 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045116148445336013, + "loss": 1.3659, + "theoretical_loss": 3.5534017930016635, + "tokens_seen": 1457913856 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004511454363089268, + "loss": 1.2452, + "theoretical_loss": 3.5533877463314743, + "tokens_seen": 1457979392 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045112938816449353, + "loss": 1.4578, + "theoretical_loss": 3.553373700469449, + "tokens_seen": 1458044928 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004511133400200602, + "loss": 1.2215, + "theoretical_loss": 3.5533596554155045, + "tokens_seen": 1458110464 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 489171, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.413554310798645, + "objective/train/theoretical_loss": 3.5533456111695583, + "objective/train/tokens_used": 1478636000, + "theoretical_loss": 3.5533456111695583, + "tokens_seen": 1458176000 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045109729187562693, + "loss": 1.3237, + "theoretical_loss": 3.5533456111695583, + "tokens_seen": 1458176000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004510812437311936, + "loss": 1.3097, + "theoretical_loss": 3.5533315677315276, + "tokens_seen": 1458241536 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045106519558676033, + "loss": 1.3404, + "theoretical_loss": 3.5533175251013294, + "tokens_seen": 1458307072 + }, + { + "epoch": 0.44, + "learning_rate": 0.000451049147442327, + "loss": 1.2486, + "theoretical_loss": 3.553303483278881, + "tokens_seen": 1458372608 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045103309929789374, + "loss": 1.3064, + "theoretical_loss": 3.5532894422640995, + "tokens_seen": 1458438144 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004510170511534604, + "loss": 1.3998, + "theoretical_loss": 3.5532754020569026, + "tokens_seen": 1458503680 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045100100300902714, + "loss": 1.3538, + "theoretical_loss": 3.5532613626572074, + "tokens_seen": 1458569216 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004509849548645938, + "loss": 1.3451, + "theoretical_loss": 3.553247324064931, + "tokens_seen": 1458634752 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045096890672016054, + "loss": 1.3197, + "theoretical_loss": 3.5532332862799914, + "tokens_seen": 1458700288 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004509528585757272, + "loss": 1.3852, + "theoretical_loss": 3.5532192493023045, + "tokens_seen": 1458765824 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045093681043129394, + "loss": 1.2242, + "theoretical_loss": 3.5532052131317884, + "tokens_seen": 1458831360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004509207622868606, + "loss": 1.2423, + "theoretical_loss": 3.553191177768361, + "tokens_seen": 1458896896 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045090471414242734, + "loss": 1.3741, + "theoretical_loss": 3.553177143211938, + "tokens_seen": 1458962432 + }, + { + "epoch": 0.44, + "learning_rate": 0.000450888665997994, + "loss": 1.3684, + "theoretical_loss": 3.5531631094624387, + "tokens_seen": 1459027968 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045087261785356074, + "loss": 1.3882, + "theoretical_loss": 3.5531490765197797, + "tokens_seen": 1459093504 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004508565697091274, + "loss": 1.3671, + "theoretical_loss": 3.553135044383878, + "tokens_seen": 1459159040 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045084052156469414, + "loss": 1.3389, + "theoretical_loss": 3.553121013054651, + "tokens_seen": 1459224576 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004508244734202608, + "loss": 1.3312, + "theoretical_loss": 3.5531069825320167, + "tokens_seen": 1459290112 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004508084252758275, + "loss": 1.3107, + "theoretical_loss": 3.553092952815892, + "tokens_seen": 1459355648 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004507923771313942, + "loss": 1.335, + "theoretical_loss": 3.5530789239061944, + "tokens_seen": 1459421184 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004507763289869609, + "loss": 1.3087, + "theoretical_loss": 3.5530648958028417, + "tokens_seen": 1459486720 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004507602808425276, + "loss": 1.2736, + "theoretical_loss": 3.5530508685057507, + "tokens_seen": 1459552256 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004507442326980943, + "loss": 1.3638, + "theoretical_loss": 3.5530368420148397, + "tokens_seen": 1459617792 + }, + { + "epoch": 0.44, + "learning_rate": 0.000450728184553661, + "loss": 1.3524, + "theoretical_loss": 3.5530228163300253, + "tokens_seen": 1459683328 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004507121364092277, + "loss": 1.3522, + "theoretical_loss": 3.5530087914512256, + "tokens_seen": 1459748864 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004506960882647944, + "loss": 1.2369, + "theoretical_loss": 3.5529947673783573, + "tokens_seen": 1459814400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004506800401203611, + "loss": 1.2734, + "theoretical_loss": 3.5529807441113395, + "tokens_seen": 1459879936 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045066399197592774, + "loss": 1.3226, + "theoretical_loss": 3.5529667216500878, + "tokens_seen": 1459945472 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004506479438314945, + "loss": 1.3201, + "theoretical_loss": 3.5529526999945213, + "tokens_seen": 1460011008 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045063189568706114, + "loss": 1.3557, + "theoretical_loss": 3.5529386791445567, + "tokens_seen": 1460076544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004506158475426279, + "loss": 1.3174, + "theoretical_loss": 3.552924659100112, + "tokens_seen": 1460142080 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045059979939819454, + "loss": 1.3624, + "theoretical_loss": 3.552910639861104, + "tokens_seen": 1460207616 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004505837512537613, + "loss": 1.308, + "theoretical_loss": 3.5528966214274513, + "tokens_seen": 1460273152 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045056770310932795, + "loss": 1.2808, + "theoretical_loss": 3.5528826037990706, + "tokens_seen": 1460338688 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004505516549648947, + "loss": 1.2842, + "theoretical_loss": 3.55286858697588, + "tokens_seen": 1460404224 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045053560682046135, + "loss": 1.2757, + "theoretical_loss": 3.552854570957797, + "tokens_seen": 1460469760 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004505195586760281, + "loss": 1.2751, + "theoretical_loss": 3.5528405557447393, + "tokens_seen": 1460535296 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045050351053159475, + "loss": 1.3406, + "theoretical_loss": 3.552826541336625, + "tokens_seen": 1460600832 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004504874623871615, + "loss": 1.2714, + "theoretical_loss": 3.5528125277333706, + "tokens_seen": 1460666368 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045047141424272815, + "loss": 1.3913, + "theoretical_loss": 3.552798514934895, + "tokens_seen": 1460731904 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004504553660982949, + "loss": 1.386, + "theoretical_loss": 3.5527845029411145, + "tokens_seen": 1460797440 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045043931795386166, + "loss": 1.2745, + "theoretical_loss": 3.5527704917519483, + "tokens_seen": 1460862976 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004504232698094283, + "loss": 1.2067, + "theoretical_loss": 3.5527564813673127, + "tokens_seen": 1460928512 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045040722166499506, + "loss": 1.2071, + "theoretical_loss": 3.552742471787127, + "tokens_seen": 1460994048 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004503911735205617, + "loss": 1.4215, + "theoretical_loss": 3.5527284630113076, + "tokens_seen": 1461059584 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045037512537612846, + "loss": 1.4245, + "theoretical_loss": 3.5527144550397725, + "tokens_seen": 1461125120 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004503590772316951, + "loss": 1.24, + "theoretical_loss": 3.5527004478724398, + "tokens_seen": 1461190656 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045034302908726186, + "loss": 1.1814, + "theoretical_loss": 3.552686441509227, + "tokens_seen": 1461256192 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004503269809428285, + "loss": 1.3061, + "theoretical_loss": 3.5526724359500523, + "tokens_seen": 1461321728 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045031093279839526, + "loss": 1.3079, + "theoretical_loss": 3.552658431194833, + "tokens_seen": 1461387264 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 490450, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2836272716522217, + "objective/train/theoretical_loss": 3.552644427243487, + "objective/train/tokens_used": 1481912800, + "theoretical_loss": 3.552644427243487, + "tokens_seen": 1461452800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004502948846539619, + "loss": 1.2955, + "theoretical_loss": 3.552644427243487, + "tokens_seen": 1461452800 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045027883650952867, + "loss": 1.4117, + "theoretical_loss": 3.5526304240959323, + "tokens_seen": 1461518336 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004502627883650953, + "loss": 1.2808, + "theoretical_loss": 3.5526164217520866, + "tokens_seen": 1461583872 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045024674022066207, + "loss": 1.2112, + "theoretical_loss": 3.5526024202118673, + "tokens_seen": 1461649408 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004502306920762287, + "loss": 1.3372, + "theoretical_loss": 3.5525884194751933, + "tokens_seen": 1461714944 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045021464393179547, + "loss": 1.3712, + "theoretical_loss": 3.5525744195419815, + "tokens_seen": 1461780480 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004501985957873621, + "loss": 1.2657, + "theoretical_loss": 3.55256042041215, + "tokens_seen": 1461846016 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004501825476429288, + "loss": 1.236, + "theoretical_loss": 3.5525464220856176, + "tokens_seen": 1461911552 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004501664994984955, + "loss": 1.3177, + "theoretical_loss": 3.5525324245623007, + "tokens_seen": 1461977088 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004501504513540622, + "loss": 1.3865, + "theoretical_loss": 3.5525184278421182, + "tokens_seen": 1462042624 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004501344032096289, + "loss": 1.2844, + "theoretical_loss": 3.552504431924987, + "tokens_seen": 1462108160 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004501183550651956, + "loss": 1.2484, + "theoretical_loss": 3.552490436810827, + "tokens_seen": 1462173696 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004501023069207623, + "loss": 1.3447, + "theoretical_loss": 3.5524764424995543, + "tokens_seen": 1462239232 + }, + { + "epoch": 0.44, + "learning_rate": 0.000450086258776329, + "loss": 1.3489, + "theoretical_loss": 3.552462448991087, + "tokens_seen": 1462304768 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004500702106318957, + "loss": 1.3081, + "theoretical_loss": 3.5524484562853447, + "tokens_seen": 1462370304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004500541624874624, + "loss": 1.3625, + "theoretical_loss": 3.5524344643822436, + "tokens_seen": 1462435840 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045003811434302907, + "loss": 1.2521, + "theoretical_loss": 3.552420473281702, + "tokens_seen": 1462501376 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004500220661985958, + "loss": 1.3067, + "theoretical_loss": 3.5524064829836384, + "tokens_seen": 1462566912 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045000601805416247, + "loss": 1.283, + "theoretical_loss": 3.5523924934879707, + "tokens_seen": 1462632448 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004499899699097292, + "loss": 1.3991, + "theoretical_loss": 3.552378504794617, + "tokens_seen": 1462697984 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044997392176529587, + "loss": 1.3523, + "theoretical_loss": 3.552364516903495, + "tokens_seen": 1462763520 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004499578736208626, + "loss": 1.2914, + "theoretical_loss": 3.552350529814523, + "tokens_seen": 1462829056 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044994182547642927, + "loss": 1.2439, + "theoretical_loss": 3.552336543527619, + "tokens_seen": 1462894592 + }, + { + "epoch": 0.44, + "learning_rate": 0.000449925777331996, + "loss": 1.1822, + "theoretical_loss": 3.552322558042701, + "tokens_seen": 1462960128 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044990972918756267, + "loss": 1.2918, + "theoretical_loss": 3.5523085733596877, + "tokens_seen": 1463025664 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004498936810431294, + "loss": 1.3432, + "theoretical_loss": 3.5522945894784965, + "tokens_seen": 1463091200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004498776328986961, + "loss": 1.186, + "theoretical_loss": 3.5522806063990457, + "tokens_seen": 1463156736 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044986158475426283, + "loss": 1.3742, + "theoretical_loss": 3.5522666241212533, + "tokens_seen": 1463222272 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004498455366098295, + "loss": 1.2982, + "theoretical_loss": 3.552252642645038, + "tokens_seen": 1463287808 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044982948846539623, + "loss": 1.394, + "theoretical_loss": 3.552238661970317, + "tokens_seen": 1463353344 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004498134403209629, + "loss": 1.1875, + "theoretical_loss": 3.552224682097009, + "tokens_seen": 1463418880 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044979739217652963, + "loss": 1.3574, + "theoretical_loss": 3.5522107030250325, + "tokens_seen": 1463484416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004497813440320963, + "loss": 1.2868, + "theoretical_loss": 3.552196724754305, + "tokens_seen": 1463549952 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044976529588766303, + "loss": 1.335, + "theoretical_loss": 3.5521827472847454, + "tokens_seen": 1463615488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004497492477432297, + "loss": 1.2571, + "theoretical_loss": 3.5521687706162712, + "tokens_seen": 1463681024 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044973319959879643, + "loss": 1.2657, + "theoretical_loss": 3.5521547947488017, + "tokens_seen": 1463746560 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004497171514543631, + "loss": 1.2836, + "theoretical_loss": 3.5521408196822537, + "tokens_seen": 1463812096 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044970110330992983, + "loss": 1.2512, + "theoretical_loss": 3.552126845416546, + "tokens_seen": 1463877632 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004496850551654965, + "loss": 1.3302, + "theoretical_loss": 3.5521128719515978, + "tokens_seen": 1463943168 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044966900702106324, + "loss": 1.2591, + "theoretical_loss": 3.5520988992873264, + "tokens_seen": 1464008704 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004496529588766299, + "loss": 1.3432, + "theoretical_loss": 3.5520849274236497, + "tokens_seen": 1464074240 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044963691073219664, + "loss": 1.3326, + "theoretical_loss": 3.5520709563604873, + "tokens_seen": 1464139776 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004496208625877633, + "loss": 1.3833, + "theoretical_loss": 3.552056986097756, + "tokens_seen": 1464205312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044960481444333004, + "loss": 1.2468, + "theoretical_loss": 3.5520430166353756, + "tokens_seen": 1464270848 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004495887662988967, + "loss": 1.3173, + "theoretical_loss": 3.5520290479732637, + "tokens_seen": 1464336384 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044957271815446344, + "loss": 1.2926, + "theoretical_loss": 3.552015080111338, + "tokens_seen": 1464401920 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004495566700100301, + "loss": 1.2533, + "theoretical_loss": 3.552001113049518, + "tokens_seen": 1464467456 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044954062186559684, + "loss": 1.1938, + "theoretical_loss": 3.551987146787722, + "tokens_seen": 1464532992 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004495245737211635, + "loss": 1.2698, + "theoretical_loss": 3.5519731813258675, + "tokens_seen": 1464598528 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044950852557673024, + "loss": 1.1891, + "theoretical_loss": 3.5519592166638727, + "tokens_seen": 1464664064 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 491422, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3803954124450684, + "objective/train/theoretical_loss": 3.5519452528016577, + "objective/train/tokens_used": 1485189600, + "theoretical_loss": 3.5519452528016577, + "tokens_seen": 1464729600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004494924774322969, + "loss": 1.3201, + "theoretical_loss": 3.5519452528016577, + "tokens_seen": 1464729600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044947642928786364, + "loss": 1.2366, + "theoretical_loss": 3.5519312897391395, + "tokens_seen": 1464795136 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004494603811434303, + "loss": 1.3456, + "theoretical_loss": 3.551917327476237, + "tokens_seen": 1464860672 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044944433299899704, + "loss": 1.2998, + "theoretical_loss": 3.5519033660128683, + "tokens_seen": 1464926208 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004494282848545637, + "loss": 1.3234, + "theoretical_loss": 3.5518894053489523, + "tokens_seen": 1464991744 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004494122367101304, + "loss": 1.3636, + "theoretical_loss": 3.551875445484407, + "tokens_seen": 1465057280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004493961885656971, + "loss": 1.3595, + "theoretical_loss": 3.551861486419152, + "tokens_seen": 1465122816 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004493801404212638, + "loss": 1.3471, + "theoretical_loss": 3.551847528153104, + "tokens_seen": 1465188352 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004493640922768305, + "loss": 1.2771, + "theoretical_loss": 3.551833570686182, + "tokens_seen": 1465253888 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004493480441323972, + "loss": 1.3855, + "theoretical_loss": 3.5518196140183056, + "tokens_seen": 1465319424 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004493319959879639, + "loss": 1.3506, + "theoretical_loss": 3.5518056581493926, + "tokens_seen": 1465384960 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004493159478435306, + "loss": 1.2656, + "theoretical_loss": 3.5517917030793615, + "tokens_seen": 1465450496 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004492998996990973, + "loss": 1.3191, + "theoretical_loss": 3.5517777488081315, + "tokens_seen": 1465516032 + }, + { + "epoch": 0.44, + "learning_rate": 0.000449283851554664, + "loss": 1.3627, + "theoretical_loss": 3.55176379533562, + "tokens_seen": 1465581568 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044926780341023064, + "loss": 1.2993, + "theoretical_loss": 3.5517498426617458, + "tokens_seen": 1465647104 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004492517552657974, + "loss": 1.2695, + "theoretical_loss": 3.5517358907864285, + "tokens_seen": 1465712640 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044923570712136415, + "loss": 1.3515, + "theoretical_loss": 3.5517219397095854, + "tokens_seen": 1465778176 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004492196589769308, + "loss": 1.2987, + "theoretical_loss": 3.5517079894311365, + "tokens_seen": 1465843712 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044920361083249755, + "loss": 1.3813, + "theoretical_loss": 3.551694039950999, + "tokens_seen": 1465909248 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004491875626880642, + "loss": 1.2322, + "theoretical_loss": 3.551680091269092, + "tokens_seen": 1465974784 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044917151454363096, + "loss": 1.2963, + "theoretical_loss": 3.551666143385335, + "tokens_seen": 1466040320 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004491554663991976, + "loss": 1.2863, + "theoretical_loss": 3.5516521962996457, + "tokens_seen": 1466105856 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044913941825476436, + "loss": 1.1739, + "theoretical_loss": 3.551638250011943, + "tokens_seen": 1466171392 + }, + { + "epoch": 0.44, + "learning_rate": 0.000449123370110331, + "loss": 1.3681, + "theoretical_loss": 3.5516243045221456, + "tokens_seen": 1466236928 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044910732196589776, + "loss": 1.357, + "theoretical_loss": 3.551610359830172, + "tokens_seen": 1466302464 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004490912738214644, + "loss": 1.2856, + "theoretical_loss": 3.5515964159359417, + "tokens_seen": 1466368000 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044907522567703116, + "loss": 1.3846, + "theoretical_loss": 3.5515824728393723, + "tokens_seen": 1466433536 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004490591775325978, + "loss": 1.2296, + "theoretical_loss": 3.551568530540383, + "tokens_seen": 1466499072 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044904312938816456, + "loss": 1.3763, + "theoretical_loss": 3.551554589038893, + "tokens_seen": 1466564608 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004490270812437312, + "loss": 1.3708, + "theoretical_loss": 3.55154064833482, + "tokens_seen": 1466630144 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044901103309929796, + "loss": 1.3562, + "theoretical_loss": 3.551526708428084, + "tokens_seen": 1466695680 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004489949849548646, + "loss": 1.3149, + "theoretical_loss": 3.5515127693186033, + "tokens_seen": 1466761216 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044897893681043136, + "loss": 1.2556, + "theoretical_loss": 3.551498831006296, + "tokens_seen": 1466826752 + }, + { + "epoch": 0.44, + "learning_rate": 0.000448962888665998, + "loss": 1.2372, + "theoretical_loss": 3.5514848934910814, + "tokens_seen": 1466892288 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044894684052156476, + "loss": 1.2254, + "theoretical_loss": 3.5514709567728784, + "tokens_seen": 1466957824 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004489307923771314, + "loss": 1.321, + "theoretical_loss": 3.551457020851606, + "tokens_seen": 1467023360 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044891474423269817, + "loss": 1.224, + "theoretical_loss": 3.5514430857271826, + "tokens_seen": 1467088896 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004488986960882648, + "loss": 1.3188, + "theoretical_loss": 3.5514291513995273, + "tokens_seen": 1467154432 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044888264794383157, + "loss": 1.2998, + "theoretical_loss": 3.551415217868559, + "tokens_seen": 1467219968 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004488665997993982, + "loss": 1.2508, + "theoretical_loss": 3.5514012851341965, + "tokens_seen": 1467285504 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044885055165496497, + "loss": 1.3836, + "theoretical_loss": 3.5513873531963585, + "tokens_seen": 1467351040 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004488345035105316, + "loss": 1.2725, + "theoretical_loss": 3.551373422054964, + "tokens_seen": 1467416576 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044881845536609837, + "loss": 1.269, + "theoretical_loss": 3.551359491709932, + "tokens_seen": 1467482112 + }, + { + "epoch": 0.44, + "learning_rate": 0.000448802407221665, + "loss": 1.3287, + "theoretical_loss": 3.551345562161181, + "tokens_seen": 1467547648 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044878635907723177, + "loss": 1.2812, + "theoretical_loss": 3.5513316334086307, + "tokens_seen": 1467613184 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004487703109327984, + "loss": 1.2979, + "theoretical_loss": 3.5513177054522, + "tokens_seen": 1467678720 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004487542627883651, + "loss": 1.3208, + "theoretical_loss": 3.551303778291807, + "tokens_seen": 1467744256 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004487382146439318, + "loss": 1.3323, + "theoretical_loss": 3.551289851927371, + "tokens_seen": 1467809792 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004487221664994985, + "loss": 1.2646, + "theoretical_loss": 3.5512759263588114, + "tokens_seen": 1467875328 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004487061183550652, + "loss": 1.3031, + "theoretical_loss": 3.5512620015860463, + "tokens_seen": 1467940864 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 492285, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1124306917190552, + "objective/train/theoretical_loss": 3.551248077608996, + "objective/train/tokens_used": 1488466400, + "theoretical_loss": 3.551248077608996, + "tokens_seen": 1468006400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004486900702106319, + "loss": 1.2451, + "theoretical_loss": 3.551248077608996, + "tokens_seen": 1468006400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004486740220661986, + "loss": 1.2949, + "theoretical_loss": 3.5512341544275783, + "tokens_seen": 1468071936 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004486579739217653, + "loss": 1.2772, + "theoretical_loss": 3.5512202320417128, + "tokens_seen": 1468137472 + }, + { + "epoch": 0.44, + "learning_rate": 0.000448641925777332, + "loss": 1.3196, + "theoretical_loss": 3.551206310451319, + "tokens_seen": 1468203008 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004486258776328987, + "loss": 1.3246, + "theoretical_loss": 3.5511923896563147, + "tokens_seen": 1468268544 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044860982948846537, + "loss": 1.3011, + "theoretical_loss": 3.5511784696566204, + "tokens_seen": 1468334080 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004485937813440321, + "loss": 1.2091, + "theoretical_loss": 3.5511645504521536, + "tokens_seen": 1468399616 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044857773319959877, + "loss": 1.2992, + "theoretical_loss": 3.5511506320428348, + "tokens_seen": 1468465152 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004485616850551655, + "loss": 1.2837, + "theoretical_loss": 3.551136714428582, + "tokens_seen": 1468530688 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044854563691073217, + "loss": 1.2785, + "theoretical_loss": 3.551122797609315, + "tokens_seen": 1468596224 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004485295887662989, + "loss": 1.2684, + "theoretical_loss": 3.551108881584953, + "tokens_seen": 1468661760 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004485135406218656, + "loss": 1.2748, + "theoretical_loss": 3.5510949663554148, + "tokens_seen": 1468727296 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044849749247743233, + "loss": 1.3099, + "theoretical_loss": 3.5510810519206197, + "tokens_seen": 1468792832 + }, + { + "epoch": 0.45, + "learning_rate": 0.000448481444332999, + "loss": 1.3105, + "theoretical_loss": 3.5510671382804864, + "tokens_seen": 1468858368 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044846539618856573, + "loss": 1.1873, + "theoretical_loss": 3.5510532254349343, + "tokens_seen": 1468923904 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004484493480441324, + "loss": 1.3178, + "theoretical_loss": 3.551039313383883, + "tokens_seen": 1468989440 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044843329989969913, + "loss": 1.3457, + "theoretical_loss": 3.5510254021272516, + "tokens_seen": 1469054976 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004484172517552658, + "loss": 1.3076, + "theoretical_loss": 3.5510114916649584, + "tokens_seen": 1469120512 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044840120361083253, + "loss": 1.332, + "theoretical_loss": 3.5509975819969233, + "tokens_seen": 1469186048 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004483851554663992, + "loss": 1.372, + "theoretical_loss": 3.550983673123066, + "tokens_seen": 1469251584 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044836910732196593, + "loss": 1.2497, + "theoretical_loss": 3.5509697650433054, + "tokens_seen": 1469317120 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004483530591775326, + "loss": 1.338, + "theoretical_loss": 3.55095585775756, + "tokens_seen": 1469382656 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044833701103309933, + "loss": 1.1832, + "theoretical_loss": 3.5509419512657496, + "tokens_seen": 1469448192 + }, + { + "epoch": 0.45, + "learning_rate": 0.000448320962888666, + "loss": 1.2157, + "theoretical_loss": 3.550928045567794, + "tokens_seen": 1469513728 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044830491474423274, + "loss": 1.222, + "theoretical_loss": 3.550914140663612, + "tokens_seen": 1469579264 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004482888665997994, + "loss": 1.2205, + "theoretical_loss": 3.550900236553123, + "tokens_seen": 1469644800 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044827281845536614, + "loss": 1.2296, + "theoretical_loss": 3.5508863332362455, + "tokens_seen": 1469710336 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004482567703109328, + "loss": 1.3189, + "theoretical_loss": 3.5508724307129, + "tokens_seen": 1469775872 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044824072216649954, + "loss": 1.2367, + "theoretical_loss": 3.5508585289830052, + "tokens_seen": 1469841408 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004482246740220662, + "loss": 1.288, + "theoretical_loss": 3.5508446280464807, + "tokens_seen": 1469906944 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044820862587763294, + "loss": 1.2607, + "theoretical_loss": 3.5508307279032456, + "tokens_seen": 1469972480 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004481925777331996, + "loss": 1.1821, + "theoretical_loss": 3.5508168285532196, + "tokens_seen": 1470038016 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044817652958876634, + "loss": 1.3742, + "theoretical_loss": 3.5508029299963217, + "tokens_seen": 1470103552 + }, + { + "epoch": 0.45, + "learning_rate": 0.000448160481444333, + "loss": 1.3117, + "theoretical_loss": 3.5507890322324718, + "tokens_seen": 1470169088 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044814443329989974, + "loss": 1.1871, + "theoretical_loss": 3.5507751352615884, + "tokens_seen": 1470234624 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004481283851554664, + "loss": 1.2141, + "theoretical_loss": 3.5507612390835916, + "tokens_seen": 1470300160 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044811233701103314, + "loss": 1.2809, + "theoretical_loss": 3.550747343698401, + "tokens_seen": 1470365696 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004480962888665998, + "loss": 1.3651, + "theoretical_loss": 3.5507334491059357, + "tokens_seen": 1470431232 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044808024072216654, + "loss": 1.2923, + "theoretical_loss": 3.550719555306115, + "tokens_seen": 1470496768 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004480641925777332, + "loss": 1.3626, + "theoretical_loss": 3.550705662298859, + "tokens_seen": 1470562304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044804814443329995, + "loss": 1.2943, + "theoretical_loss": 3.550691770084086, + "tokens_seen": 1470627840 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044803209628886665, + "loss": 1.2904, + "theoretical_loss": 3.5506778786617166, + "tokens_seen": 1470693376 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044801604814443335, + "loss": 1.1644, + "theoretical_loss": 3.55066398803167, + "tokens_seen": 1470758912 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044800000000000005, + "loss": 1.3377, + "theoretical_loss": 3.5506500981938656, + "tokens_seen": 1470824448 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004479839518555667, + "loss": 1.4048, + "theoretical_loss": 3.5506362091482226, + "tokens_seen": 1470889984 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044796790371113345, + "loss": 1.2446, + "theoretical_loss": 3.550622320894661, + "tokens_seen": 1470955520 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004479518555667001, + "loss": 1.2955, + "theoretical_loss": 3.5506084334331005, + "tokens_seen": 1471021056 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044793580742226685, + "loss": 1.2552, + "theoretical_loss": 3.5505945467634596, + "tokens_seen": 1471086592 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004479197592778335, + "loss": 1.2722, + "theoretical_loss": 3.550580660885659, + "tokens_seen": 1471152128 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044790371113340025, + "loss": 1.3282, + "theoretical_loss": 3.550566775799618, + "tokens_seen": 1471217664 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 493595, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.409515619277954, + "objective/train/theoretical_loss": 3.550552891505256, + "objective/train/tokens_used": 1491743200, + "theoretical_loss": 3.550552891505256, + "tokens_seen": 1471283200 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004478876629889669, + "loss": 1.272, + "theoretical_loss": 3.550552891505256, + "tokens_seen": 1471283200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044787161484453365, + "loss": 1.2476, + "theoretical_loss": 3.5505390080024926, + "tokens_seen": 1471348736 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004478555667001003, + "loss": 1.3048, + "theoretical_loss": 3.550525125291248, + "tokens_seen": 1471414272 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044783951855566705, + "loss": 1.2262, + "theoretical_loss": 3.5505112433714405, + "tokens_seen": 1471479808 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004478234704112337, + "loss": 1.2819, + "theoretical_loss": 3.550497362242991, + "tokens_seen": 1471545344 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044780742226680046, + "loss": 1.2758, + "theoretical_loss": 3.5504834819058186, + "tokens_seen": 1471610880 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004477913741223671, + "loss": 1.2836, + "theoretical_loss": 3.550469602359843, + "tokens_seen": 1471676416 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044777532597793386, + "loss": 1.3172, + "theoretical_loss": 3.5504557236049843, + "tokens_seen": 1471741952 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004477592778335005, + "loss": 1.3781, + "theoretical_loss": 3.5504418456411613, + "tokens_seen": 1471807488 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044774322968906726, + "loss": 1.4068, + "theoretical_loss": 3.550427968468294, + "tokens_seen": 1471873024 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004477271815446339, + "loss": 1.2669, + "theoretical_loss": 3.550414092086303, + "tokens_seen": 1471938560 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044771113340020066, + "loss": 1.241, + "theoretical_loss": 3.550400216495107, + "tokens_seen": 1472004096 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004476950852557673, + "loss": 1.318, + "theoretical_loss": 3.550386341694626, + "tokens_seen": 1472069632 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044767903711133406, + "loss": 1.1884, + "theoretical_loss": 3.55037246768478, + "tokens_seen": 1472135168 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004476629889669007, + "loss": 1.3407, + "theoretical_loss": 3.5503585944654885, + "tokens_seen": 1472200704 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044764694082246746, + "loss": 1.4194, + "theoretical_loss": 3.550344722036671, + "tokens_seen": 1472266240 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004476308926780341, + "loss": 1.2354, + "theoretical_loss": 3.5503308503982476, + "tokens_seen": 1472331776 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044761484453360086, + "loss": 1.3369, + "theoretical_loss": 3.5503169795501384, + "tokens_seen": 1472397312 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004475987963891675, + "loss": 1.3405, + "theoretical_loss": 3.550303109492263, + "tokens_seen": 1472462848 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044758274824473426, + "loss": 1.3397, + "theoretical_loss": 3.55028924022454, + "tokens_seen": 1472528384 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004475667001003009, + "loss": 1.2805, + "theoretical_loss": 3.550275371746891, + "tokens_seen": 1472593920 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044755065195586767, + "loss": 1.2799, + "theoretical_loss": 3.5502615040592356, + "tokens_seen": 1472659456 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004475346038114343, + "loss": 1.3203, + "theoretical_loss": 3.5502476371614926, + "tokens_seen": 1472724992 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044751855566700107, + "loss": 1.2079, + "theoretical_loss": 3.5502337710535827, + "tokens_seen": 1472790528 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004475025075225677, + "loss": 1.3942, + "theoretical_loss": 3.5502199057354256, + "tokens_seen": 1472856064 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044748645937813447, + "loss": 1.3379, + "theoretical_loss": 3.550206041206941, + "tokens_seen": 1472921600 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004474704112337011, + "loss": 1.2404, + "theoretical_loss": 3.5501921774680483, + "tokens_seen": 1472987136 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044745436308926787, + "loss": 1.2891, + "theoretical_loss": 3.5501783145186687, + "tokens_seen": 1473052672 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004474383149448345, + "loss": 1.2657, + "theoretical_loss": 3.5501644523587212, + "tokens_seen": 1473118208 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044742226680040127, + "loss": 1.3169, + "theoretical_loss": 3.5501505909881264, + "tokens_seen": 1473183744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004474062186559679, + "loss": 1.2235, + "theoretical_loss": 3.5501367304068028, + "tokens_seen": 1473249280 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044739017051153467, + "loss": 1.2876, + "theoretical_loss": 3.550122870614672, + "tokens_seen": 1473314816 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004473741223671013, + "loss": 1.2713, + "theoretical_loss": 3.5501090116116534, + "tokens_seen": 1473380352 + }, + { + "epoch": 0.45, + "learning_rate": 0.000447358074222668, + "loss": 1.2368, + "theoretical_loss": 3.550095153397667, + "tokens_seen": 1473445888 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004473420260782347, + "loss": 1.2598, + "theoretical_loss": 3.550081295972632, + "tokens_seen": 1473511424 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004473259779338014, + "loss": 1.2551, + "theoretical_loss": 3.550067439336469, + "tokens_seen": 1473576960 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004473099297893681, + "loss": 1.3723, + "theoretical_loss": 3.550053583489098, + "tokens_seen": 1473642496 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004472938816449348, + "loss": 1.2455, + "theoretical_loss": 3.5500397284304395, + "tokens_seen": 1473708032 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004472778335005015, + "loss": 1.3906, + "theoretical_loss": 3.550025874160413, + "tokens_seen": 1473773568 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004472617853560682, + "loss": 1.3666, + "theoretical_loss": 3.550012020678939, + "tokens_seen": 1473839104 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004472457372116349, + "loss": 1.3299, + "theoretical_loss": 3.549998167985937, + "tokens_seen": 1473904640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004472296890672016, + "loss": 1.3547, + "theoretical_loss": 3.5499843160813276, + "tokens_seen": 1473970176 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044721364092276827, + "loss": 1.2667, + "theoretical_loss": 3.54997046496503, + "tokens_seen": 1474035712 + }, + { + "epoch": 0.45, + "learning_rate": 0.000447197592778335, + "loss": 1.2703, + "theoretical_loss": 3.549956614636965, + "tokens_seen": 1474101248 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044718154463390167, + "loss": 1.2809, + "theoretical_loss": 3.549942765097053, + "tokens_seen": 1474166784 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004471654964894684, + "loss": 1.314, + "theoretical_loss": 3.5499289163452126, + "tokens_seen": 1474232320 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044714944834503507, + "loss": 1.3864, + "theoretical_loss": 3.549915068381366, + "tokens_seen": 1474297856 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044713340020060183, + "loss": 1.2958, + "theoretical_loss": 3.549901221205432, + "tokens_seen": 1474363392 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004471173520561685, + "loss": 1.2209, + "theoretical_loss": 3.549887374817331, + "tokens_seen": 1474428928 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044710130391173523, + "loss": 1.1318, + "theoretical_loss": 3.5498735292169834, + "tokens_seen": 1474494464 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 494397, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3106197118759155, + "objective/train/theoretical_loss": 3.5498596844043093, + "objective/train/tokens_used": 1495020000, + "theoretical_loss": 3.5498596844043093, + "tokens_seen": 1474560000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004470852557673019, + "loss": 1.1911, + "theoretical_loss": 3.5498596844043093, + "tokens_seen": 1474560000 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044706920762286863, + "loss": 1.2254, + "theoretical_loss": 3.549845840379229, + "tokens_seen": 1474625536 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004470531594784353, + "loss": 1.3583, + "theoretical_loss": 3.549831997141662, + "tokens_seen": 1474691072 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044703711133400203, + "loss": 1.267, + "theoretical_loss": 3.549818154691529, + "tokens_seen": 1474756608 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004470210631895687, + "loss": 1.2699, + "theoretical_loss": 3.5498043130287504, + "tokens_seen": 1474822144 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044700501504513543, + "loss": 1.2423, + "theoretical_loss": 3.549790472153246, + "tokens_seen": 1474887680 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004469889669007021, + "loss": 1.3835, + "theoretical_loss": 3.549776632064937, + "tokens_seen": 1474953216 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044697291875626883, + "loss": 1.3148, + "theoretical_loss": 3.5497627927637425, + "tokens_seen": 1475018752 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004469568706118355, + "loss": 1.3392, + "theoretical_loss": 3.5497489542495835, + "tokens_seen": 1475084288 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044694082246740224, + "loss": 1.3905, + "theoretical_loss": 3.5497351165223794, + "tokens_seen": 1475149824 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004469247743229689, + "loss": 1.2646, + "theoretical_loss": 3.5497212795820516, + "tokens_seen": 1475215360 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044690872617853564, + "loss": 1.3821, + "theoretical_loss": 3.54970744342852, + "tokens_seen": 1475280896 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004468926780341023, + "loss": 1.2743, + "theoretical_loss": 3.5496936080617045, + "tokens_seen": 1475346432 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044687662988966904, + "loss": 1.2287, + "theoretical_loss": 3.5496797734815257, + "tokens_seen": 1475411968 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004468605817452357, + "loss": 1.2634, + "theoretical_loss": 3.5496659396879044, + "tokens_seen": 1475477504 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044684453360080244, + "loss": 1.3642, + "theoretical_loss": 3.5496521066807603, + "tokens_seen": 1475543040 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004468284854563692, + "loss": 1.3364, + "theoretical_loss": 3.5496382744600137, + "tokens_seen": 1475608576 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044681243731193584, + "loss": 1.2063, + "theoretical_loss": 3.549624443025585, + "tokens_seen": 1475674112 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004467963891675026, + "loss": 1.2851, + "theoretical_loss": 3.5496106123773954, + "tokens_seen": 1475739648 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044678034102306924, + "loss": 1.2888, + "theoretical_loss": 3.5495967825153647, + "tokens_seen": 1475805184 + }, + { + "epoch": 0.45, + "learning_rate": 0.000446764292878636, + "loss": 1.3448, + "theoretical_loss": 3.5495829534394128, + "tokens_seen": 1475870720 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044674824473420264, + "loss": 1.3652, + "theoretical_loss": 3.5495691251494614, + "tokens_seen": 1475936256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044673219658976934, + "loss": 1.2462, + "theoretical_loss": 3.54955529764543, + "tokens_seen": 1476001792 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044671614844533604, + "loss": 1.4395, + "theoretical_loss": 3.549541470927239, + "tokens_seen": 1476067328 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044670010030090274, + "loss": 1.3211, + "theoretical_loss": 3.549527644994809, + "tokens_seen": 1476132864 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044668405215646945, + "loss": 1.2914, + "theoretical_loss": 3.5495138198480607, + "tokens_seen": 1476198400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044666800401203615, + "loss": 1.2559, + "theoretical_loss": 3.549499995486914, + "tokens_seen": 1476263936 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044665195586760285, + "loss": 1.2727, + "theoretical_loss": 3.5494861719112905, + "tokens_seen": 1476329472 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044663590772316955, + "loss": 1.3265, + "theoretical_loss": 3.549472349121109, + "tokens_seen": 1476395008 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044661985957873625, + "loss": 1.3374, + "theoretical_loss": 3.5494585271162915, + "tokens_seen": 1476460544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044660381143430295, + "loss": 1.2819, + "theoretical_loss": 3.549444705896758, + "tokens_seen": 1476526080 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004465877632898696, + "loss": 1.3293, + "theoretical_loss": 3.5494308854624292, + "tokens_seen": 1476591616 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044657171514543635, + "loss": 1.3372, + "theoretical_loss": 3.5494170658132256, + "tokens_seen": 1476657152 + }, + { + "epoch": 0.45, + "learning_rate": 0.000446555667001003, + "loss": 1.295, + "theoretical_loss": 3.5494032469490673, + "tokens_seen": 1476722688 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044653961885656975, + "loss": 1.2969, + "theoretical_loss": 3.5493894288698753, + "tokens_seen": 1476788224 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004465235707121364, + "loss": 1.265, + "theoretical_loss": 3.54937561157557, + "tokens_seen": 1476853760 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044650752256770315, + "loss": 1.3506, + "theoretical_loss": 3.5493617950660727, + "tokens_seen": 1476919296 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004464914744232698, + "loss": 1.2906, + "theoretical_loss": 3.549347979341303, + "tokens_seen": 1476984832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044647542627883655, + "loss": 1.3151, + "theoretical_loss": 3.5493341644011815, + "tokens_seen": 1477050368 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004464593781344032, + "loss": 1.2969, + "theoretical_loss": 3.5493203502456296, + "tokens_seen": 1477115904 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044644332998996995, + "loss": 1.3257, + "theoretical_loss": 3.5493065368745675, + "tokens_seen": 1477181440 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004464272818455366, + "loss": 1.2564, + "theoretical_loss": 3.549292724287916, + "tokens_seen": 1477246976 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044641123370110336, + "loss": 1.199, + "theoretical_loss": 3.5492789124855957, + "tokens_seen": 1477312512 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044639518555667, + "loss": 1.3125, + "theoretical_loss": 3.5492651014675274, + "tokens_seen": 1477378048 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044637913741223676, + "loss": 1.4284, + "theoretical_loss": 3.5492512912336314, + "tokens_seen": 1477443584 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004463630892678034, + "loss": 1.3104, + "theoretical_loss": 3.5492374817838286, + "tokens_seen": 1477509120 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044634704112337016, + "loss": 1.2588, + "theoretical_loss": 3.54922367311804, + "tokens_seen": 1477574656 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004463309929789368, + "loss": 1.2301, + "theoretical_loss": 3.549209865236185, + "tokens_seen": 1477640192 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044631494483450356, + "loss": 1.2794, + "theoretical_loss": 3.549196058138187, + "tokens_seen": 1477705728 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004462988966900702, + "loss": 1.279, + "theoretical_loss": 3.549182251823964, + "tokens_seen": 1477771264 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 495688, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2247810363769531, + "objective/train/theoretical_loss": 3.549168446293438, + "objective/train/tokens_used": 1498296800, + "theoretical_loss": 3.549168446293438, + "tokens_seen": 1477836800 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044628284854563696, + "loss": 1.2837, + "theoretical_loss": 3.549168446293438, + "tokens_seen": 1477836800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004462668004012036, + "loss": 1.2858, + "theoretical_loss": 3.54915464154653, + "tokens_seen": 1477902336 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044625075225677036, + "loss": 1.3078, + "theoretical_loss": 3.54914083758316, + "tokens_seen": 1477967872 + }, + { + "epoch": 0.45, + "learning_rate": 0.000446234704112337, + "loss": 1.229, + "theoretical_loss": 3.5491270344032495, + "tokens_seen": 1478033408 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044621865596790376, + "loss": 1.3989, + "theoretical_loss": 3.549113232006719, + "tokens_seen": 1478098944 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004462026078234704, + "loss": 1.3031, + "theoretical_loss": 3.549099430393489, + "tokens_seen": 1478164480 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044618655967903716, + "loss": 1.2627, + "theoretical_loss": 3.5490856295634807, + "tokens_seen": 1478230016 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004461705115346038, + "loss": 1.3746, + "theoretical_loss": 3.549071829516615, + "tokens_seen": 1478295552 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044615446339017057, + "loss": 1.305, + "theoretical_loss": 3.5490580302528123, + "tokens_seen": 1478361088 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004461384152457372, + "loss": 1.2692, + "theoretical_loss": 3.549044231771994, + "tokens_seen": 1478426624 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044612236710130397, + "loss": 1.2892, + "theoretical_loss": 3.549030434074081, + "tokens_seen": 1478492160 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004461063189568706, + "loss": 1.2861, + "theoretical_loss": 3.5490166371589935, + "tokens_seen": 1478557696 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044609027081243737, + "loss": 1.3145, + "theoretical_loss": 3.549002841026653, + "tokens_seen": 1478623232 + }, + { + "epoch": 0.45, + "learning_rate": 0.000446074222668004, + "loss": 1.2981, + "theoretical_loss": 3.54898904567698, + "tokens_seen": 1478688768 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044605817452357077, + "loss": 1.3612, + "theoretical_loss": 3.548975251109896, + "tokens_seen": 1478754304 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004460421263791374, + "loss": 1.3104, + "theoretical_loss": 3.548961457325321, + "tokens_seen": 1478819840 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044602607823470417, + "loss": 1.2714, + "theoretical_loss": 3.548947664323177, + "tokens_seen": 1478885376 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004460100300902708, + "loss": 1.2712, + "theoretical_loss": 3.548933872103384, + "tokens_seen": 1478950912 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044599398194583757, + "loss": 1.3135, + "theoretical_loss": 3.548920080665863, + "tokens_seen": 1479016448 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004459779338014042, + "loss": 1.318, + "theoretical_loss": 3.548906290010536, + "tokens_seen": 1479081984 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004459618856569709, + "loss": 1.3726, + "theoretical_loss": 3.548892500137323, + "tokens_seen": 1479147520 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004459458375125376, + "loss": 1.3166, + "theoretical_loss": 3.5488787110461457, + "tokens_seen": 1479213056 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004459297893681043, + "loss": 1.3793, + "theoretical_loss": 3.5488649227369247, + "tokens_seen": 1479278592 + }, + { + "epoch": 0.45, + "learning_rate": 0.000445913741223671, + "loss": 1.4259, + "theoretical_loss": 3.548851135209581, + "tokens_seen": 1479344128 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004458976930792377, + "loss": 1.3343, + "theoretical_loss": 3.548837348464035, + "tokens_seen": 1479409664 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004458816449348044, + "loss": 1.2835, + "theoretical_loss": 3.548823562500209, + "tokens_seen": 1479475200 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004458655967903711, + "loss": 1.3106, + "theoretical_loss": 3.5488097773180236, + "tokens_seen": 1479540736 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004458495486459378, + "loss": 1.5154, + "theoretical_loss": 3.5487959929173996, + "tokens_seen": 1479606272 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004458335005015045, + "loss": 1.3065, + "theoretical_loss": 3.548782209298258, + "tokens_seen": 1479671808 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044581745235707117, + "loss": 1.3041, + "theoretical_loss": 3.54876842646052, + "tokens_seen": 1479737344 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004458014042126379, + "loss": 1.2106, + "theoretical_loss": 3.5487546444041076, + "tokens_seen": 1479802880 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044578535606820457, + "loss": 1.3655, + "theoretical_loss": 3.5487408631289403, + "tokens_seen": 1479868416 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044576930792377133, + "loss": 1.3113, + "theoretical_loss": 3.54872708263494, + "tokens_seen": 1479933952 + }, + { + "epoch": 0.45, + "learning_rate": 0.000445753259779338, + "loss": 1.3416, + "theoretical_loss": 3.548713302922028, + "tokens_seen": 1479999488 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044573721163490473, + "loss": 1.3106, + "theoretical_loss": 3.5486995239901256, + "tokens_seen": 1480065024 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004457211634904714, + "loss": 1.3696, + "theoretical_loss": 3.5486857458391534, + "tokens_seen": 1480130560 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044570511534603813, + "loss": 1.3216, + "theoretical_loss": 3.5486719684690327, + "tokens_seen": 1480196096 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004456890672016048, + "loss": 1.2756, + "theoretical_loss": 3.548658191879685, + "tokens_seen": 1480261632 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044567301905717153, + "loss": 1.3497, + "theoretical_loss": 3.5486444160710313, + "tokens_seen": 1480327168 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004456569709127382, + "loss": 1.325, + "theoretical_loss": 3.548630641042992, + "tokens_seen": 1480392704 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044564092276830493, + "loss": 1.2649, + "theoretical_loss": 3.54861686679549, + "tokens_seen": 1480458240 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004456248746238717, + "loss": 1.333, + "theoretical_loss": 3.5486030933284454, + "tokens_seen": 1480523776 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044560882647943833, + "loss": 1.3912, + "theoretical_loss": 3.5485893206417796, + "tokens_seen": 1480589312 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004455927783350051, + "loss": 1.3423, + "theoretical_loss": 3.5485755487354136, + "tokens_seen": 1480654848 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044557673019057173, + "loss": 1.3831, + "theoretical_loss": 3.5485617776092693, + "tokens_seen": 1480720384 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004455606820461385, + "loss": 1.2653, + "theoretical_loss": 3.5485480072632676, + "tokens_seen": 1480785920 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044554463390170514, + "loss": 1.2235, + "theoretical_loss": 3.5485342376973295, + "tokens_seen": 1480851456 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004455285857572719, + "loss": 1.3191, + "theoretical_loss": 3.5485204689113767, + "tokens_seen": 1480916992 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044551253761283854, + "loss": 1.3674, + "theoretical_loss": 3.548506700905331, + "tokens_seen": 1480982528 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004454964894684053, + "loss": 1.4616, + "theoretical_loss": 3.5484929336791122, + "tokens_seen": 1481048064 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 496572, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3179576396942139, + "objective/train/theoretical_loss": 3.5484791672326423, + "objective/train/tokens_used": 1501573600, + "theoretical_loss": 3.5484791672326423, + "tokens_seen": 1481113600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044548044132397194, + "loss": 1.2927, + "theoretical_loss": 3.5484791672326423, + "tokens_seen": 1481113600 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004454643931795387, + "loss": 1.3033, + "theoretical_loss": 3.548465401565844, + "tokens_seen": 1481179136 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044544834503510534, + "loss": 1.3021, + "theoretical_loss": 3.548451636678637, + "tokens_seen": 1481244672 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004454322968906721, + "loss": 1.2335, + "theoretical_loss": 3.5484378725709425, + "tokens_seen": 1481310208 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044541624874623874, + "loss": 1.3487, + "theoretical_loss": 3.548424109242683, + "tokens_seen": 1481375744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004454002006018055, + "loss": 1.2137, + "theoretical_loss": 3.54841034669378, + "tokens_seen": 1481441280 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044538415245737214, + "loss": 1.4135, + "theoretical_loss": 3.548396584924154, + "tokens_seen": 1481506816 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004453681043129389, + "loss": 1.2251, + "theoretical_loss": 3.5483828239337267, + "tokens_seen": 1481572352 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044535205616850554, + "loss": 1.4295, + "theoretical_loss": 3.5483690637224194, + "tokens_seen": 1481637888 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044533600802407224, + "loss": 1.3799, + "theoretical_loss": 3.548355304290154, + "tokens_seen": 1481703424 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044531995987963895, + "loss": 1.3247, + "theoretical_loss": 3.548341545636851, + "tokens_seen": 1481768960 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044530391173520565, + "loss": 1.2748, + "theoretical_loss": 3.5483277877624326, + "tokens_seen": 1481834496 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044528786359077235, + "loss": 1.2615, + "theoretical_loss": 3.5483140306668206, + "tokens_seen": 1481900032 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044527181544633905, + "loss": 1.3339, + "theoretical_loss": 3.548300274349936, + "tokens_seen": 1481965568 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044525576730190575, + "loss": 1.2702, + "theoretical_loss": 3.5482865188117, + "tokens_seen": 1482031104 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044523971915747245, + "loss": 1.4656, + "theoretical_loss": 3.5482727640520344, + "tokens_seen": 1482096640 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044522367101303915, + "loss": 1.3425, + "theoretical_loss": 3.5482590100708604, + "tokens_seen": 1482162176 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044520762286860585, + "loss": 1.2492, + "theoretical_loss": 3.5482452568681007, + "tokens_seen": 1482227712 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004451915747241725, + "loss": 1.236, + "theoretical_loss": 3.5482315044436747, + "tokens_seen": 1482293248 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044517552657973925, + "loss": 1.2805, + "theoretical_loss": 3.5482177527975063, + "tokens_seen": 1482358784 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004451594784353059, + "loss": 1.3446, + "theoretical_loss": 3.5482040019295153, + "tokens_seen": 1482424320 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044514343029087265, + "loss": 1.3252, + "theoretical_loss": 3.548190251839624, + "tokens_seen": 1482489856 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004451273821464393, + "loss": 1.3548, + "theoretical_loss": 3.5481765025277543, + "tokens_seen": 1482555392 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044511133400200605, + "loss": 1.397, + "theoretical_loss": 3.548162753993827, + "tokens_seen": 1482620928 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004450952858575727, + "loss": 1.2837, + "theoretical_loss": 3.548149006237764, + "tokens_seen": 1482686464 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044507923771313945, + "loss": 1.3748, + "theoretical_loss": 3.548135259259487, + "tokens_seen": 1482752000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004450631895687061, + "loss": 1.2917, + "theoretical_loss": 3.548121513058918, + "tokens_seen": 1482817536 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044504714142427286, + "loss": 1.3039, + "theoretical_loss": 3.548107767635978, + "tokens_seen": 1482883072 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004450310932798395, + "loss": 1.2809, + "theoretical_loss": 3.548094022990589, + "tokens_seen": 1482948608 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044501504513540626, + "loss": 1.3502, + "theoretical_loss": 3.548080279122672, + "tokens_seen": 1483014144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004449989969909729, + "loss": 1.1451, + "theoretical_loss": 3.5480665360321497, + "tokens_seen": 1483079680 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044498294884653966, + "loss": 1.2604, + "theoretical_loss": 3.548052793718943, + "tokens_seen": 1483145216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004449669007021063, + "loss": 1.2212, + "theoretical_loss": 3.548039052182974, + "tokens_seen": 1483210752 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044495085255767306, + "loss": 1.3196, + "theoretical_loss": 3.5480253114241647, + "tokens_seen": 1483276288 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004449348044132397, + "loss": 1.2527, + "theoretical_loss": 3.5480115714424354, + "tokens_seen": 1483341824 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044491875626880646, + "loss": 1.1341, + "theoretical_loss": 3.5479978322377095, + "tokens_seen": 1483407360 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004449027081243731, + "loss": 1.3636, + "theoretical_loss": 3.547984093809908, + "tokens_seen": 1483472896 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044488665997993986, + "loss": 1.2996, + "theoretical_loss": 3.5479703561589524, + "tokens_seen": 1483538432 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004448706118355065, + "loss": 1.3028, + "theoretical_loss": 3.547956619284765, + "tokens_seen": 1483603968 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044485456369107326, + "loss": 1.3174, + "theoretical_loss": 3.5479428831872672, + "tokens_seen": 1483669504 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004448385155466399, + "loss": 1.4079, + "theoretical_loss": 3.5479291478663812, + "tokens_seen": 1483735040 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044482246740220666, + "loss": 1.2716, + "theoretical_loss": 3.5479154133220283, + "tokens_seen": 1483800576 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004448064192577733, + "loss": 1.2796, + "theoretical_loss": 3.5479016795541307, + "tokens_seen": 1483866112 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044479037111334007, + "loss": 1.3138, + "theoretical_loss": 3.5478879465626094, + "tokens_seen": 1483931648 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004447743229689067, + "loss": 1.3321, + "theoretical_loss": 3.547874214347387, + "tokens_seen": 1483997184 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044475827482447347, + "loss": 1.3997, + "theoretical_loss": 3.547860482908386, + "tokens_seen": 1484062720 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004447422266800401, + "loss": 1.3465, + "theoretical_loss": 3.5478467522455266, + "tokens_seen": 1484128256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044472617853560687, + "loss": 1.2211, + "theoretical_loss": 3.5478330223587315, + "tokens_seen": 1484193792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004447101303911735, + "loss": 1.2692, + "theoretical_loss": 3.5478192932479233, + "tokens_seen": 1484259328 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044469408224674027, + "loss": 1.3578, + "theoretical_loss": 3.5478055649130225, + "tokens_seen": 1484324864 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 497900, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2213529348373413, + "objective/train/theoretical_loss": 3.5477918373539516, + "objective/train/tokens_used": 1504850400, + "theoretical_loss": 3.5477918373539516, + "tokens_seen": 1484390400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004446780341023069, + "loss": 1.3075, + "theoretical_loss": 3.5477918373539516, + "tokens_seen": 1484390400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044466198595787367, + "loss": 1.3878, + "theoretical_loss": 3.5477781105706327, + "tokens_seen": 1484455936 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004446459378134403, + "loss": 1.3598, + "theoretical_loss": 3.5477643845629876, + "tokens_seen": 1484521472 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044462988966900707, + "loss": 1.3405, + "theoretical_loss": 3.547750659330938, + "tokens_seen": 1484587008 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004446138415245737, + "loss": 1.3402, + "theoretical_loss": 3.547736934874406, + "tokens_seen": 1484652544 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004445977933801405, + "loss": 1.1972, + "theoretical_loss": 3.547723211193314, + "tokens_seen": 1484718080 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004445817452357071, + "loss": 1.2758, + "theoretical_loss": 3.547709488287583, + "tokens_seen": 1484783616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004445656970912738, + "loss": 1.2796, + "theoretical_loss": 3.547695766157136, + "tokens_seen": 1484849152 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004445496489468405, + "loss": 1.2446, + "theoretical_loss": 3.547682044801894, + "tokens_seen": 1484914688 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004445336008024072, + "loss": 1.3446, + "theoretical_loss": 3.54766832422178, + "tokens_seen": 1484980224 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004445175526579739, + "loss": 1.299, + "theoretical_loss": 3.547654604416715, + "tokens_seen": 1485045760 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004445015045135406, + "loss": 1.1966, + "theoretical_loss": 3.547640885386622, + "tokens_seen": 1485111296 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004444854563691073, + "loss": 1.2917, + "theoretical_loss": 3.547627167131422, + "tokens_seen": 1485176832 + }, + { + "epoch": 0.45, + "learning_rate": 0.000444469408224674, + "loss": 1.23, + "theoretical_loss": 3.5476134496510383, + "tokens_seen": 1485242368 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004444533600802407, + "loss": 1.2974, + "theoretical_loss": 3.5475997329453914, + "tokens_seen": 1485307904 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004444373119358074, + "loss": 1.3243, + "theoretical_loss": 3.547586017014405, + "tokens_seen": 1485373440 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004444212637913742, + "loss": 1.2422, + "theoretical_loss": 3.5475723018579997, + "tokens_seen": 1485438976 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004444052156469408, + "loss": 1.1543, + "theoretical_loss": 3.5475585874760984, + "tokens_seen": 1485504512 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004443891675025076, + "loss": 1.3314, + "theoretical_loss": 3.547544873868623, + "tokens_seen": 1485570048 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044437311935807423, + "loss": 1.2806, + "theoretical_loss": 3.5475311610354963, + "tokens_seen": 1485635584 + }, + { + "epoch": 0.45, + "learning_rate": 0.000444357071213641, + "loss": 1.3674, + "theoretical_loss": 3.547517448976639, + "tokens_seen": 1485701120 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044434102306920763, + "loss": 1.3682, + "theoretical_loss": 3.547503737691975, + "tokens_seen": 1485766656 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004443249749247744, + "loss": 1.342, + "theoretical_loss": 3.547490027181424, + "tokens_seen": 1485832192 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044430892678034103, + "loss": 1.2189, + "theoretical_loss": 3.547476317444911, + "tokens_seen": 1485897728 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004442928786359078, + "loss": 1.279, + "theoretical_loss": 3.547462608482356, + "tokens_seen": 1485963264 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044427683049147443, + "loss": 1.3384, + "theoretical_loss": 3.5474489002936824, + "tokens_seen": 1486028800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004442607823470412, + "loss": 1.411, + "theoretical_loss": 3.5474351928788117, + "tokens_seen": 1486094336 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044424473420260783, + "loss": 1.4026, + "theoretical_loss": 3.547421486237666, + "tokens_seen": 1486159872 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004442286860581746, + "loss": 1.3369, + "theoretical_loss": 3.5474077803701682, + "tokens_seen": 1486225408 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044421263791374123, + "loss": 1.3357, + "theoretical_loss": 3.5473940752762405, + "tokens_seen": 1486290944 + }, + { + "epoch": 0.45, + "learning_rate": 0.000444196589769308, + "loss": 1.336, + "theoretical_loss": 3.5473803709558043, + "tokens_seen": 1486356480 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044418054162487464, + "loss": 1.2241, + "theoretical_loss": 3.5473666674087827, + "tokens_seen": 1486422016 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004441644934804414, + "loss": 1.276, + "theoretical_loss": 3.547352964635097, + "tokens_seen": 1486487552 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044414844533600804, + "loss": 1.3731, + "theoretical_loss": 3.547339262634671, + "tokens_seen": 1486553088 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004441323971915748, + "loss": 1.3268, + "theoretical_loss": 3.5473255614074257, + "tokens_seen": 1486618624 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044411634904714144, + "loss": 1.3405, + "theoretical_loss": 3.5473118609532834, + "tokens_seen": 1486684160 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004441003009027082, + "loss": 1.3448, + "theoretical_loss": 3.547298161272167, + "tokens_seen": 1486749696 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044408425275827484, + "loss": 1.3078, + "theoretical_loss": 3.5472844623639985, + "tokens_seen": 1486815232 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004440682046138416, + "loss": 1.3369, + "theoretical_loss": 3.5472707642287, + "tokens_seen": 1486880768 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044405215646940824, + "loss": 1.1991, + "theoretical_loss": 3.5472570668661945, + "tokens_seen": 1486946304 + }, + { + "epoch": 0.45, + "learning_rate": 0.000444036108324975, + "loss": 1.2801, + "theoretical_loss": 3.547243370276404, + "tokens_seen": 1487011840 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044402006018054164, + "loss": 1.4029, + "theoretical_loss": 3.547229674459251, + "tokens_seen": 1487077376 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004440040120361084, + "loss": 1.3469, + "theoretical_loss": 3.547215979414657, + "tokens_seen": 1487142912 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044398796389167504, + "loss": 1.3631, + "theoretical_loss": 3.5472022851425455, + "tokens_seen": 1487208448 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004439719157472418, + "loss": 1.2682, + "theoretical_loss": 3.547188591642838, + "tokens_seen": 1487273984 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044395586760280844, + "loss": 1.3008, + "theoretical_loss": 3.5471748989154577, + "tokens_seen": 1487339520 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044393981945837515, + "loss": 1.3619, + "theoretical_loss": 3.5471612069603267, + "tokens_seen": 1487405056 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044392377131394185, + "loss": 1.2172, + "theoretical_loss": 3.5471475157773673, + "tokens_seen": 1487470592 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044390772316950855, + "loss": 1.3858, + "theoretical_loss": 3.547133825366502, + "tokens_seen": 1487536128 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044389167502507525, + "loss": 1.321, + "theoretical_loss": 3.547120135727653, + "tokens_seen": 1487601664 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 498821, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4604065418243408, + "objective/train/theoretical_loss": 3.5471064468607434, + "objective/train/tokens_used": 1508127200, + "theoretical_loss": 3.5471064468607434, + "tokens_seen": 1487667200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044387562688064195, + "loss": 1.4423, + "theoretical_loss": 3.5471064468607434, + "tokens_seen": 1487667200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044385957873620865, + "loss": 1.1887, + "theoretical_loss": 3.547092758765695, + "tokens_seen": 1487732736 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044384353059177535, + "loss": 1.1948, + "theoretical_loss": 3.5470790714424307, + "tokens_seen": 1487798272 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044382748244734205, + "loss": 1.2935, + "theoretical_loss": 3.547065384890873, + "tokens_seen": 1487863808 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044381143430290875, + "loss": 1.2731, + "theoretical_loss": 3.547051699110944, + "tokens_seen": 1487929344 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004437953861584754, + "loss": 1.2875, + "theoretical_loss": 3.547038014102567, + "tokens_seen": 1487994880 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044377933801404215, + "loss": 1.2711, + "theoretical_loss": 3.547024329865663, + "tokens_seen": 1488060416 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004437632898696088, + "loss": 1.2718, + "theoretical_loss": 3.547010646400156, + "tokens_seen": 1488125952 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044374724172517555, + "loss": 1.4034, + "theoretical_loss": 3.5469969637059684, + "tokens_seen": 1488191488 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004437311935807422, + "loss": 1.341, + "theoretical_loss": 3.546983281783022, + "tokens_seen": 1488257024 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044371514543630895, + "loss": 1.2758, + "theoretical_loss": 3.54696960063124, + "tokens_seen": 1488322560 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004436990972918756, + "loss": 1.2209, + "theoretical_loss": 3.5469559202505447, + "tokens_seen": 1488388096 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044368304914744236, + "loss": 1.4072, + "theoretical_loss": 3.546942240640859, + "tokens_seen": 1488453632 + }, + { + "epoch": 0.45, + "learning_rate": 0.000443667001003009, + "loss": 1.3209, + "theoretical_loss": 3.546928561802105, + "tokens_seen": 1488519168 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044365095285857576, + "loss": 1.2755, + "theoretical_loss": 3.546914883734206, + "tokens_seen": 1488584704 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004436349047141424, + "loss": 1.1996, + "theoretical_loss": 3.5469012064370835, + "tokens_seen": 1488650240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044361885656970916, + "loss": 1.3091, + "theoretical_loss": 3.5468875299106615, + "tokens_seen": 1488715776 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004436028084252758, + "loss": 1.3795, + "theoretical_loss": 3.5468738541548617, + "tokens_seen": 1488781312 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044358676028084256, + "loss": 1.2914, + "theoretical_loss": 3.546860179169607, + "tokens_seen": 1488846848 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004435707121364092, + "loss": 1.2904, + "theoretical_loss": 3.5468465049548206, + "tokens_seen": 1488912384 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044355466399197596, + "loss": 1.3453, + "theoretical_loss": 3.5468328315104243, + "tokens_seen": 1488977920 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004435386158475426, + "loss": 1.3484, + "theoretical_loss": 3.546819158836341, + "tokens_seen": 1489043456 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044352256770310936, + "loss": 1.3253, + "theoretical_loss": 3.546805486932494, + "tokens_seen": 1489108992 + }, + { + "epoch": 0.45, + "learning_rate": 0.000443506519558676, + "loss": 1.2508, + "theoretical_loss": 3.546791815798805, + "tokens_seen": 1489174528 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044349047141424276, + "loss": 1.3773, + "theoretical_loss": 3.546778145435198, + "tokens_seen": 1489240064 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004434744232698094, + "loss": 1.2105, + "theoretical_loss": 3.546764475841595, + "tokens_seen": 1489305600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044345837512537616, + "loss": 1.3785, + "theoretical_loss": 3.5467508070179186, + "tokens_seen": 1489371136 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004434423269809428, + "loss": 1.3551, + "theoretical_loss": 3.546737138964092, + "tokens_seen": 1489436672 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044342627883650957, + "loss": 1.2329, + "theoretical_loss": 3.5467234716800373, + "tokens_seen": 1489502208 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004434102306920762, + "loss": 1.3633, + "theoretical_loss": 3.5467098051656785, + "tokens_seen": 1489567744 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044339418254764297, + "loss": 1.3787, + "theoretical_loss": 3.546696139420937, + "tokens_seen": 1489633280 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004433781344032096, + "loss": 1.2233, + "theoretical_loss": 3.5466824744457366, + "tokens_seen": 1489698816 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044336208625877637, + "loss": 1.2936, + "theoretical_loss": 3.5466688102399995, + "tokens_seen": 1489764352 + }, + { + "epoch": 0.45, + "learning_rate": 0.000443346038114343, + "loss": 1.3799, + "theoretical_loss": 3.5466551468036487, + "tokens_seen": 1489829888 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044332998996990977, + "loss": 1.3461, + "theoretical_loss": 3.5466414841366074, + "tokens_seen": 1489895424 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004433139418254764, + "loss": 1.3389, + "theoretical_loss": 3.546627822238798, + "tokens_seen": 1489960960 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044329789368104317, + "loss": 1.3126, + "theoretical_loss": 3.5466141611101434, + "tokens_seen": 1490026496 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004432818455366098, + "loss": 1.2598, + "theoretical_loss": 3.546600500750567, + "tokens_seen": 1490092032 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044326579739217657, + "loss": 1.3039, + "theoretical_loss": 3.546586841159991, + "tokens_seen": 1490157568 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004432497492477432, + "loss": 1.2855, + "theoretical_loss": 3.546573182338338, + "tokens_seen": 1490223104 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044323370110331, + "loss": 1.3369, + "theoretical_loss": 3.546559524285532, + "tokens_seen": 1490288640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004432176529588767, + "loss": 1.305, + "theoretical_loss": 3.5465458670014955, + "tokens_seen": 1490354176 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004432016048144434, + "loss": 1.3263, + "theoretical_loss": 3.546532210486151, + "tokens_seen": 1490419712 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004431855566700101, + "loss": 1.3127, + "theoretical_loss": 3.5465185547394222, + "tokens_seen": 1490485248 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004431695085255767, + "loss": 1.3767, + "theoretical_loss": 3.546504899761231, + "tokens_seen": 1490550784 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004431534603811435, + "loss": 1.3478, + "theoretical_loss": 3.5464912455515014, + "tokens_seen": 1490616320 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004431374122367101, + "loss": 1.3366, + "theoretical_loss": 3.546477592110156, + "tokens_seen": 1490681856 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004431213640922769, + "loss": 1.254, + "theoretical_loss": 3.5464639394371167, + "tokens_seen": 1490747392 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004431053159478435, + "loss": 1.2013, + "theoretical_loss": 3.5464502875323083, + "tokens_seen": 1490812928 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004430892678034103, + "loss": 1.3132, + "theoretical_loss": 3.5464366363956534, + "tokens_seen": 1490878464 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 499903, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3060457706451416, + "objective/train/theoretical_loss": 3.546422986027074, + "objective/train/tokens_used": 1511404000, + "theoretical_loss": 3.546422986027074, + "tokens_seen": 1490944000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004430732196589769, + "loss": 1.272, + "theoretical_loss": 3.546422986027074, + "tokens_seen": 1490944000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004430571715145437, + "loss": 1.3209, + "theoretical_loss": 3.5464093364264935, + "tokens_seen": 1491009536 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004430411233701103, + "loss": 1.2822, + "theoretical_loss": 3.5463956875938356, + "tokens_seen": 1491075072 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004430250752256771, + "loss": 1.3027, + "theoretical_loss": 3.546382039529023, + "tokens_seen": 1491140608 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044300902708124373, + "loss": 1.2638, + "theoretical_loss": 3.5463683922319786, + "tokens_seen": 1491206144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004429929789368105, + "loss": 1.2887, + "theoretical_loss": 3.546354745702625, + "tokens_seen": 1491271680 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044297693079237713, + "loss": 1.3575, + "theoretical_loss": 3.5463410999408866, + "tokens_seen": 1491337216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004429608826479439, + "loss": 1.2515, + "theoretical_loss": 3.5463274549466854, + "tokens_seen": 1491402752 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044294483450351053, + "loss": 1.3469, + "theoretical_loss": 3.546313810719945, + "tokens_seen": 1491468288 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004429287863590773, + "loss": 1.3389, + "theoretical_loss": 3.546300167260588, + "tokens_seen": 1491533824 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044291273821464393, + "loss": 1.3137, + "theoretical_loss": 3.546286524568538, + "tokens_seen": 1491599360 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004428966900702107, + "loss": 1.2875, + "theoretical_loss": 3.546272882643718, + "tokens_seen": 1491664896 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044288064192577733, + "loss": 1.258, + "theoretical_loss": 3.546259241486051, + "tokens_seen": 1491730432 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004428645937813441, + "loss": 1.2512, + "theoretical_loss": 3.5462456010954604, + "tokens_seen": 1491795968 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044284854563691073, + "loss": 1.2033, + "theoretical_loss": 3.54623196147187, + "tokens_seen": 1491861504 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004428324974924775, + "loss": 1.2309, + "theoretical_loss": 3.5462183226152013, + "tokens_seen": 1491927040 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044281644934804414, + "loss": 1.2305, + "theoretical_loss": 3.546204684525379, + "tokens_seen": 1491992576 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004428004012036109, + "loss": 1.2936, + "theoretical_loss": 3.546191047202325, + "tokens_seen": 1492058112 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044278435305917754, + "loss": 1.2525, + "theoretical_loss": 3.546177410645964, + "tokens_seen": 1492123648 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004427683049147443, + "loss": 1.2254, + "theoretical_loss": 3.5461637748562183, + "tokens_seen": 1492189184 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044275225677031094, + "loss": 1.1758, + "theoretical_loss": 3.546150139833011, + "tokens_seen": 1492254720 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004427362086258777, + "loss": 1.2772, + "theoretical_loss": 3.5461365055762664, + "tokens_seen": 1492320256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044272016048144434, + "loss": 1.3466, + "theoretical_loss": 3.546122872085906, + "tokens_seen": 1492385792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004427041123370111, + "loss": 1.221, + "theoretical_loss": 3.546109239361855, + "tokens_seen": 1492451328 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044268806419257774, + "loss": 1.3131, + "theoretical_loss": 3.5460956074040357, + "tokens_seen": 1492516864 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004426720160481445, + "loss": 1.1914, + "theoretical_loss": 3.5460819762123714, + "tokens_seen": 1492582400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044265596790371114, + "loss": 1.3059, + "theoretical_loss": 3.546068345786785, + "tokens_seen": 1492647936 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004426399197592779, + "loss": 1.2706, + "theoretical_loss": 3.546054716127201, + "tokens_seen": 1492713472 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044262387161484454, + "loss": 1.3403, + "theoretical_loss": 3.5460410872335415, + "tokens_seen": 1492779008 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004426078234704113, + "loss": 1.3431, + "theoretical_loss": 3.5460274591057304, + "tokens_seen": 1492844544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044259177532597794, + "loss": 1.3342, + "theoretical_loss": 3.546013831743691, + "tokens_seen": 1492910080 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004425757271815447, + "loss": 1.3687, + "theoretical_loss": 3.546000205147347, + "tokens_seen": 1492975616 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044255967903711135, + "loss": 1.2215, + "theoretical_loss": 3.545986579316621, + "tokens_seen": 1493041152 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004425436308926781, + "loss": 1.3595, + "theoretical_loss": 3.545972954251437, + "tokens_seen": 1493106688 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044252758274824475, + "loss": 1.2245, + "theoretical_loss": 3.5459593299517183, + "tokens_seen": 1493172224 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044251153460381145, + "loss": 1.2358, + "theoretical_loss": 3.545945706417388, + "tokens_seen": 1493237760 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044249548645937815, + "loss": 1.2601, + "theoretical_loss": 3.5459320836483696, + "tokens_seen": 1493303296 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044247943831494485, + "loss": 1.2364, + "theoretical_loss": 3.545918461644587, + "tokens_seen": 1493368832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044246339017051155, + "loss": 1.3508, + "theoretical_loss": 3.5459048404059628, + "tokens_seen": 1493434368 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044244734202607825, + "loss": 1.2605, + "theoretical_loss": 3.5458912199324217, + "tokens_seen": 1493499904 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044243129388164495, + "loss": 1.183, + "theoretical_loss": 3.5458776002238857, + "tokens_seen": 1493565440 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044241524573721165, + "loss": 1.295, + "theoretical_loss": 3.545863981280279, + "tokens_seen": 1493630976 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004423991975927783, + "loss": 1.3173, + "theoretical_loss": 3.5458503631015255, + "tokens_seen": 1493696512 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044238314944834505, + "loss": 1.2528, + "theoretical_loss": 3.5458367456875477, + "tokens_seen": 1493762048 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004423671013039117, + "loss": 1.3235, + "theoretical_loss": 3.5458231290382694, + "tokens_seen": 1493827584 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044235105315947845, + "loss": 1.3364, + "theoretical_loss": 3.545809513153615, + "tokens_seen": 1493893120 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004423350050150451, + "loss": 1.2103, + "theoretical_loss": 3.545795898033507, + "tokens_seen": 1493958656 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044231895687061186, + "loss": 1.3311, + "theoretical_loss": 3.545782283677869, + "tokens_seen": 1494024192 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004423029087261785, + "loss": 1.2977, + "theoretical_loss": 3.545768670086625, + "tokens_seen": 1494089728 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044228686058174526, + "loss": 1.3293, + "theoretical_loss": 3.545755057259699, + "tokens_seen": 1494155264 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 500805, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4166927337646484, + "objective/train/theoretical_loss": 3.545741445197013, + "objective/train/tokens_used": 1514680800, + "theoretical_loss": 3.545741445197013, + "tokens_seen": 1494220800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004422708124373119, + "loss": 1.3803, + "theoretical_loss": 3.545741445197013, + "tokens_seen": 1494220800 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044225476429287866, + "loss": 1.2705, + "theoretical_loss": 3.5457278338984923, + "tokens_seen": 1494286336 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004422387161484453, + "loss": 1.2304, + "theoretical_loss": 3.545714223364059, + "tokens_seen": 1494351872 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044222266800401206, + "loss": 1.1331, + "theoretical_loss": 3.545700613593638, + "tokens_seen": 1494417408 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004422066198595787, + "loss": 1.2417, + "theoretical_loss": 3.545687004587152, + "tokens_seen": 1494482944 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044219057171514546, + "loss": 1.2671, + "theoretical_loss": 3.5456733963445255, + "tokens_seen": 1494548480 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004421745235707121, + "loss": 1.2321, + "theoretical_loss": 3.545659788865681, + "tokens_seen": 1494614016 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044215847542627886, + "loss": 1.3777, + "theoretical_loss": 3.545646182150543, + "tokens_seen": 1494679552 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004421424272818455, + "loss": 1.228, + "theoretical_loss": 3.5456325761990346, + "tokens_seen": 1494745088 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044212637913741226, + "loss": 1.2912, + "theoretical_loss": 3.5456189710110797, + "tokens_seen": 1494810624 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004421103309929789, + "loss": 1.3349, + "theoretical_loss": 3.5456053665866025, + "tokens_seen": 1494876160 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044209428284854566, + "loss": 1.2647, + "theoretical_loss": 3.545591762925526, + "tokens_seen": 1494941696 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004420782347041123, + "loss": 1.3457, + "theoretical_loss": 3.545578160027774, + "tokens_seen": 1495007232 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044206218655967907, + "loss": 1.2867, + "theoretical_loss": 3.5455645578932704, + "tokens_seen": 1495072768 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004420461384152457, + "loss": 1.169, + "theoretical_loss": 3.545550956521939, + "tokens_seen": 1495138304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044203009027081247, + "loss": 1.2631, + "theoretical_loss": 3.5455373559137033, + "tokens_seen": 1495203840 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004420140421263792, + "loss": 1.2381, + "theoretical_loss": 3.5455237560684867, + "tokens_seen": 1495269376 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044199799398194587, + "loss": 1.234, + "theoretical_loss": 3.5455101569862135, + "tokens_seen": 1495334912 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004419819458375126, + "loss": 1.2497, + "theoretical_loss": 3.545496558666808, + "tokens_seen": 1495400448 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044196589769307927, + "loss": 1.2863, + "theoretical_loss": 3.5454829611101926, + "tokens_seen": 1495465984 + }, + { + "epoch": 0.45, + "learning_rate": 0.000441949849548646, + "loss": 1.1738, + "theoretical_loss": 3.5454693643162916, + "tokens_seen": 1495531520 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044193380140421267, + "loss": 1.1725, + "theoretical_loss": 3.54545576828503, + "tokens_seen": 1495597056 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004419177532597794, + "loss": 1.3059, + "theoretical_loss": 3.5454421730163297, + "tokens_seen": 1495662592 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044190170511534607, + "loss": 1.2743, + "theoretical_loss": 3.5454285785101156, + "tokens_seen": 1495728128 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044188565697091277, + "loss": 1.2422, + "theoretical_loss": 3.5454149847663112, + "tokens_seen": 1495793664 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004418696088264795, + "loss": 1.2817, + "theoretical_loss": 3.545401391784841, + "tokens_seen": 1495859200 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004418535606820462, + "loss": 1.3426, + "theoretical_loss": 3.545387799565628, + "tokens_seen": 1495924736 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004418375125376129, + "loss": 1.2575, + "theoretical_loss": 3.545374208108597, + "tokens_seen": 1495990272 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004418214643931796, + "loss": 1.3284, + "theoretical_loss": 3.5453606174136705, + "tokens_seen": 1496055808 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004418054162487463, + "loss": 1.3364, + "theoretical_loss": 3.5453470274807737, + "tokens_seen": 1496121344 + }, + { + "epoch": 0.45, + "learning_rate": 0.000441789368104313, + "loss": 1.282, + "theoretical_loss": 3.5453334383098296, + "tokens_seen": 1496186880 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004417733199598797, + "loss": 1.2636, + "theoretical_loss": 3.5453198499007623, + "tokens_seen": 1496252416 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004417572718154464, + "loss": 1.3119, + "theoretical_loss": 3.5453062622534963, + "tokens_seen": 1496317952 + }, + { + "epoch": 0.45, + "learning_rate": 0.000441741223671013, + "loss": 1.2526, + "theoretical_loss": 3.5452926753679552, + "tokens_seen": 1496383488 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004417251755265798, + "loss": 1.3335, + "theoretical_loss": 3.5452790892440627, + "tokens_seen": 1496449024 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004417091273821464, + "loss": 1.3197, + "theoretical_loss": 3.545265503881742, + "tokens_seen": 1496514560 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004416930792377132, + "loss": 1.3034, + "theoretical_loss": 3.5452519192809193, + "tokens_seen": 1496580096 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004416770310932798, + "loss": 1.3558, + "theoretical_loss": 3.5452383354415167, + "tokens_seen": 1496645632 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004416609829488466, + "loss": 1.2489, + "theoretical_loss": 3.545224752363459, + "tokens_seen": 1496711168 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044164493480441323, + "loss": 1.3406, + "theoretical_loss": 3.5452111700466697, + "tokens_seen": 1496776704 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044162888665998, + "loss": 1.3659, + "theoretical_loss": 3.5451975884910727, + "tokens_seen": 1496842240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044161283851554663, + "loss": 1.2632, + "theoretical_loss": 3.5451840076965926, + "tokens_seen": 1496907776 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004415967903711134, + "loss": 1.333, + "theoretical_loss": 3.5451704276631535, + "tokens_seen": 1496973312 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044158074222668003, + "loss": 1.2692, + "theoretical_loss": 3.545156848390679, + "tokens_seen": 1497038848 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004415646940822468, + "loss": 1.246, + "theoretical_loss": 3.545143269879093, + "tokens_seen": 1497104384 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044154864593781343, + "loss": 1.3659, + "theoretical_loss": 3.5451296921283197, + "tokens_seen": 1497169920 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004415325977933802, + "loss": 1.2116, + "theoretical_loss": 3.5451161151382835, + "tokens_seen": 1497235456 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044151654964894683, + "loss": 1.3613, + "theoretical_loss": 3.5451025389089086, + "tokens_seen": 1497300992 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004415005015045136, + "loss": 1.3407, + "theoretical_loss": 3.545088963440118, + "tokens_seen": 1497366528 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044148445336008023, + "loss": 1.416, + "theoretical_loss": 3.545075388731837, + "tokens_seen": 1497432064 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 501929, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1920337677001953, + "objective/train/theoretical_loss": 3.545061814783989, + "objective/train/tokens_used": 1517957600, + "theoretical_loss": 3.545061814783989, + "tokens_seen": 1497497600 + }, + { + "epoch": 0.45, + "learning_rate": 0.000441468405215647, + "loss": 1.2595, + "theoretical_loss": 3.545061814783989, + "tokens_seen": 1497497600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044145235707121364, + "loss": 1.32, + "theoretical_loss": 3.5450482415964992, + "tokens_seen": 1497563136 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004414363089267804, + "loss": 1.2764, + "theoretical_loss": 3.54503466916929, + "tokens_seen": 1497628672 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044142026078234704, + "loss": 1.2148, + "theoretical_loss": 3.5450210975022873, + "tokens_seen": 1497694208 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004414042126379138, + "loss": 1.3527, + "theoretical_loss": 3.545007526595414, + "tokens_seen": 1497759744 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044138816449348044, + "loss": 1.3511, + "theoretical_loss": 3.544993956448595, + "tokens_seen": 1497825280 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004413721163490472, + "loss": 1.2421, + "theoretical_loss": 3.544980387061753, + "tokens_seen": 1497890816 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044135606820461384, + "loss": 1.2499, + "theoretical_loss": 3.5449668184348146, + "tokens_seen": 1497956352 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004413400200601806, + "loss": 1.2888, + "theoretical_loss": 3.5449532505677026, + "tokens_seen": 1498021888 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044132397191574724, + "loss": 1.2662, + "theoretical_loss": 3.544939683460341, + "tokens_seen": 1498087424 + }, + { + "epoch": 0.45, + "learning_rate": 0.000441307923771314, + "loss": 1.2896, + "theoretical_loss": 3.544926117112655, + "tokens_seen": 1498152960 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044129187562688064, + "loss": 1.2332, + "theoretical_loss": 3.544912551524568, + "tokens_seen": 1498218496 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004412758274824474, + "loss": 1.2381, + "theoretical_loss": 3.5448989866960043, + "tokens_seen": 1498284032 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044125977933801404, + "loss": 1.2528, + "theoretical_loss": 3.544885422626889, + "tokens_seen": 1498349568 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004412437311935808, + "loss": 1.2944, + "theoretical_loss": 3.5448718593171447, + "tokens_seen": 1498415104 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044122768304914744, + "loss": 1.2595, + "theoretical_loss": 3.544858296766697, + "tokens_seen": 1498480640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004412116349047142, + "loss": 1.3792, + "theoretical_loss": 3.54484473497547, + "tokens_seen": 1498546176 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044119558676028085, + "loss": 1.2765, + "theoretical_loss": 3.5448311739433884, + "tokens_seen": 1498611712 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004411795386158476, + "loss": 1.2502, + "theoretical_loss": 3.5448176136703755, + "tokens_seen": 1498677248 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044116349047141425, + "loss": 1.2886, + "theoretical_loss": 3.544804054156356, + "tokens_seen": 1498742784 + }, + { + "epoch": 0.45, + "learning_rate": 0.000441147442326981, + "loss": 1.3088, + "theoretical_loss": 3.544790495401255, + "tokens_seen": 1498808320 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044113139418254765, + "loss": 1.3935, + "theoretical_loss": 3.5447769374049956, + "tokens_seen": 1498873856 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044111534603811435, + "loss": 1.2399, + "theoretical_loss": 3.544763380167503, + "tokens_seen": 1498939392 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044109929789368105, + "loss": 1.4056, + "theoretical_loss": 3.5447498236887007, + "tokens_seen": 1499004928 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044108324974924775, + "loss": 1.2911, + "theoretical_loss": 3.5447362679685144, + "tokens_seen": 1499070464 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044106720160481445, + "loss": 1.3031, + "theoretical_loss": 3.544722713006868, + "tokens_seen": 1499136000 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044105115346038115, + "loss": 1.2811, + "theoretical_loss": 3.544709158803685, + "tokens_seen": 1499201536 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044103510531594785, + "loss": 1.2504, + "theoretical_loss": 3.5446956053588905, + "tokens_seen": 1499267072 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044101905717151455, + "loss": 1.3346, + "theoretical_loss": 3.544682052672409, + "tokens_seen": 1499332608 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044100300902708125, + "loss": 1.1265, + "theoretical_loss": 3.544668500744165, + "tokens_seen": 1499398144 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044098696088264795, + "loss": 1.3361, + "theoretical_loss": 3.5446549495740824, + "tokens_seen": 1499463680 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004409709127382146, + "loss": 1.2644, + "theoretical_loss": 3.5446413991620864, + "tokens_seen": 1499529216 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044095486459378136, + "loss": 1.3488, + "theoretical_loss": 3.544627849508101, + "tokens_seen": 1499594752 + }, + { + "epoch": 0.45, + "learning_rate": 0.000440938816449348, + "loss": 1.3129, + "theoretical_loss": 3.5446143006120505, + "tokens_seen": 1499660288 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044092276830491476, + "loss": 1.3208, + "theoretical_loss": 3.54460075247386, + "tokens_seen": 1499725824 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004409067201604814, + "loss": 1.3258, + "theoretical_loss": 3.5445872050934533, + "tokens_seen": 1499791360 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044089067201604816, + "loss": 1.1752, + "theoretical_loss": 3.5445736584707555, + "tokens_seen": 1499856896 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004408746238716148, + "loss": 1.2849, + "theoretical_loss": 3.5445601126056907, + "tokens_seen": 1499922432 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044085857572718156, + "loss": 1.3171, + "theoretical_loss": 3.544546567498183, + "tokens_seen": 1499987968 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004408425275827482, + "loss": 1.2543, + "theoretical_loss": 3.5445330231481584, + "tokens_seen": 1500053504 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044082647943831496, + "loss": 1.2205, + "theoretical_loss": 3.5445194795555404, + "tokens_seen": 1500119040 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004408104312938817, + "loss": 1.3444, + "theoretical_loss": 3.5445059367202534, + "tokens_seen": 1500184576 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044079438314944836, + "loss": 1.3537, + "theoretical_loss": 3.5444923946422224, + "tokens_seen": 1500250112 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004407783350050151, + "loss": 1.3226, + "theoretical_loss": 3.5444788533213725, + "tokens_seen": 1500315648 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044076228686058176, + "loss": 1.3159, + "theoretical_loss": 3.5444653127576267, + "tokens_seen": 1500381184 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004407462387161485, + "loss": 1.265, + "theoretical_loss": 3.544451772950911, + "tokens_seen": 1500446720 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044073019057171516, + "loss": 1.3354, + "theoretical_loss": 3.5444382339011495, + "tokens_seen": 1500512256 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004407141424272819, + "loss": 1.3545, + "theoretical_loss": 3.544424695608267, + "tokens_seen": 1500577792 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044069809428284857, + "loss": 1.2056, + "theoretical_loss": 3.5444111580721875, + "tokens_seen": 1500643328 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004406820461384153, + "loss": 1.2853, + "theoretical_loss": 3.5443976212928368, + "tokens_seen": 1500708864 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 502894, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.272984266281128, + "objective/train/theoretical_loss": 3.544384085270139, + "objective/train/tokens_used": 1521234400, + "theoretical_loss": 3.544384085270139, + "tokens_seen": 1500774400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044066599799398197, + "loss": 1.3016, + "theoretical_loss": 3.544384085270139, + "tokens_seen": 1500774400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004406499498495487, + "loss": 1.2469, + "theoretical_loss": 3.544370550004018, + "tokens_seen": 1500839936 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044063390170511537, + "loss": 1.3869, + "theoretical_loss": 3.5443570154944, + "tokens_seen": 1500905472 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004406178535606821, + "loss": 1.323, + "theoretical_loss": 3.5443434817412083, + "tokens_seen": 1500971008 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044060180541624877, + "loss": 1.2838, + "theoretical_loss": 3.5443299487443687, + "tokens_seen": 1501036544 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004405857572718155, + "loss": 1.256, + "theoretical_loss": 3.544316416503805, + "tokens_seen": 1501102080 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044056970912738217, + "loss": 1.3683, + "theoretical_loss": 3.5443028850194427, + "tokens_seen": 1501167616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004405536609829489, + "loss": 1.334, + "theoretical_loss": 3.5442893542912053, + "tokens_seen": 1501233152 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044053761283851557, + "loss": 1.3719, + "theoretical_loss": 3.544275824319019, + "tokens_seen": 1501298688 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004405215646940823, + "loss": 1.2759, + "theoretical_loss": 3.5442622951028078, + "tokens_seen": 1501364224 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044050551654964897, + "loss": 1.2972, + "theoretical_loss": 3.5442487666424967, + "tokens_seen": 1501429760 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004404894684052157, + "loss": 1.3218, + "theoretical_loss": 3.5442352389380103, + "tokens_seen": 1501495296 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004404734202607824, + "loss": 1.2625, + "theoretical_loss": 3.544221711989273, + "tokens_seen": 1501560832 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004404573721163491, + "loss": 1.3805, + "theoretical_loss": 3.544208185796211, + "tokens_seen": 1501626368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004404413239719158, + "loss": 1.3547, + "theoretical_loss": 3.5441946603587478, + "tokens_seen": 1501691904 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004404252758274825, + "loss": 1.349, + "theoretical_loss": 3.5441811356768085, + "tokens_seen": 1501757440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004404092276830492, + "loss": 1.3398, + "theoretical_loss": 3.544167611750318, + "tokens_seen": 1501822976 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004403931795386159, + "loss": 1.2158, + "theoretical_loss": 3.5441540885792007, + "tokens_seen": 1501888512 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004403771313941826, + "loss": 1.2974, + "theoretical_loss": 3.5441405661633825, + "tokens_seen": 1501954048 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004403610832497493, + "loss": 1.2863, + "theoretical_loss": 3.5441270445027873, + "tokens_seen": 1502019584 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004403450351053159, + "loss": 1.2833, + "theoretical_loss": 3.5441135235973404, + "tokens_seen": 1502085120 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004403289869608827, + "loss": 1.3349, + "theoretical_loss": 3.5441000034469665, + "tokens_seen": 1502150656 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004403129388164493, + "loss": 1.3148, + "theoretical_loss": 3.544086484051591, + "tokens_seen": 1502216192 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004402968906720161, + "loss": 1.3139, + "theoretical_loss": 3.5440729654111385, + "tokens_seen": 1502281728 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044028084252758273, + "loss": 1.3666, + "theoretical_loss": 3.544059447525533, + "tokens_seen": 1502347264 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004402647943831495, + "loss": 1.342, + "theoretical_loss": 3.544045930394701, + "tokens_seen": 1502412800 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044024874623871613, + "loss": 1.2179, + "theoretical_loss": 3.5440324140185666, + "tokens_seen": 1502478336 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004402326980942829, + "loss": 1.3526, + "theoretical_loss": 3.5440188983970544, + "tokens_seen": 1502543872 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044021664994984953, + "loss": 1.3061, + "theoretical_loss": 3.54400538353009, + "tokens_seen": 1502609408 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004402006018054163, + "loss": 1.2093, + "theoretical_loss": 3.5439918694175985, + "tokens_seen": 1502674944 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044018455366098293, + "loss": 1.2408, + "theoretical_loss": 3.543978356059504, + "tokens_seen": 1502740480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004401685055165497, + "loss": 1.3752, + "theoretical_loss": 3.5439648434557327, + "tokens_seen": 1502806016 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044015245737211633, + "loss": 1.1801, + "theoretical_loss": 3.543951331606208, + "tokens_seen": 1502871552 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004401364092276831, + "loss": 1.2551, + "theoretical_loss": 3.543937820510857, + "tokens_seen": 1502937088 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044012036108324973, + "loss": 1.1826, + "theoretical_loss": 3.5439243101696025, + "tokens_seen": 1503002624 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004401043129388165, + "loss": 1.3067, + "theoretical_loss": 3.5439108005823714, + "tokens_seen": 1503068160 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044008826479438314, + "loss": 1.3107, + "theoretical_loss": 3.5438972917490874, + "tokens_seen": 1503133696 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004400722166499499, + "loss": 1.2832, + "theoretical_loss": 3.5438837836696764, + "tokens_seen": 1503199232 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044005616850551654, + "loss": 1.3399, + "theoretical_loss": 3.5438702763440633, + "tokens_seen": 1503264768 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004400401203610833, + "loss": 1.268, + "theoretical_loss": 3.5438567697721726, + "tokens_seen": 1503330304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044002407221664994, + "loss": 1.3108, + "theoretical_loss": 3.5438432639539297, + "tokens_seen": 1503395840 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004400080240722167, + "loss": 1.3275, + "theoretical_loss": 3.5438297588892604, + "tokens_seen": 1503461376 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043999197592778334, + "loss": 1.3376, + "theoretical_loss": 3.543816254578089, + "tokens_seen": 1503526912 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004399759277833501, + "loss": 1.3758, + "theoretical_loss": 3.5438027510203405, + "tokens_seen": 1503592448 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043995987963891674, + "loss": 1.3116, + "theoretical_loss": 3.5437892482159405, + "tokens_seen": 1503657984 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004399438314944835, + "loss": 1.2753, + "theoretical_loss": 3.5437757461648145, + "tokens_seen": 1503723520 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043992778335005014, + "loss": 1.3406, + "theoretical_loss": 3.543762244866887, + "tokens_seen": 1503789056 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004399117352056169, + "loss": 1.3919, + "theoretical_loss": 3.543748744322083, + "tokens_seen": 1503854592 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043989568706118354, + "loss": 1.1844, + "theoretical_loss": 3.5437352445303283, + "tokens_seen": 1503920128 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004398796389167503, + "loss": 1.2561, + "theoretical_loss": 3.5437217454915473, + "tokens_seen": 1503985664 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 504139, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3649600744247437, + "objective/train/theoretical_loss": 3.5437082472056662, + "objective/train/tokens_used": 1524511200, + "theoretical_loss": 3.5437082472056662, + "tokens_seen": 1504051200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043986359077231694, + "loss": 1.2784, + "theoretical_loss": 3.5437082472056662, + "tokens_seen": 1504051200 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004398475426278837, + "loss": 1.2817, + "theoretical_loss": 3.5436947496726097, + "tokens_seen": 1504116736 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043983149448345035, + "loss": 1.3325, + "theoretical_loss": 3.5436812528923025, + "tokens_seen": 1504182272 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004398154463390171, + "loss": 1.3115, + "theoretical_loss": 3.5436677568646706, + "tokens_seen": 1504247808 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043979939819458375, + "loss": 1.3038, + "theoretical_loss": 3.5436542615896394, + "tokens_seen": 1504313344 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004397833500501505, + "loss": 1.1666, + "theoretical_loss": 3.5436407670671333, + "tokens_seen": 1504378880 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043976730190571715, + "loss": 1.237, + "theoretical_loss": 3.5436272732970777, + "tokens_seen": 1504444416 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004397512537612839, + "loss": 1.2513, + "theoretical_loss": 3.543613780279398, + "tokens_seen": 1504509952 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043973520561685055, + "loss": 1.1959, + "theoretical_loss": 3.5436002880140203, + "tokens_seen": 1504575488 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043971915747241725, + "loss": 1.3554, + "theoretical_loss": 3.543586796500869, + "tokens_seen": 1504641024 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043970310932798395, + "loss": 1.2286, + "theoretical_loss": 3.5435733057398693, + "tokens_seen": 1504706560 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043968706118355065, + "loss": 1.3703, + "theoretical_loss": 3.5435598157309474, + "tokens_seen": 1504772096 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043967101303911735, + "loss": 1.2902, + "theoretical_loss": 3.5435463264740275, + "tokens_seen": 1504837632 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043965496489468405, + "loss": 1.3072, + "theoretical_loss": 3.5435328379690354, + "tokens_seen": 1504903168 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004396389167502508, + "loss": 1.2065, + "theoretical_loss": 3.543519350215897, + "tokens_seen": 1504968704 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043962286860581745, + "loss": 1.3759, + "theoretical_loss": 3.543505863214537, + "tokens_seen": 1505034240 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004396068204613842, + "loss": 1.2626, + "theoretical_loss": 3.5434923769648803, + "tokens_seen": 1505099776 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043959077231695085, + "loss": 1.2391, + "theoretical_loss": 3.5434788914668536, + "tokens_seen": 1505165312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004395747241725176, + "loss": 1.2324, + "theoretical_loss": 3.5434654067203812, + "tokens_seen": 1505230848 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043955867602808426, + "loss": 1.2531, + "theoretical_loss": 3.543451922725389, + "tokens_seen": 1505296384 + }, + { + "epoch": 0.46, + "learning_rate": 0.000439542627883651, + "loss": 1.3445, + "theoretical_loss": 3.5434384394818026, + "tokens_seen": 1505361920 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043952657973921766, + "loss": 1.3479, + "theoretical_loss": 3.543424956989547, + "tokens_seen": 1505427456 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004395105315947844, + "loss": 1.2726, + "theoretical_loss": 3.5434114752485475, + "tokens_seen": 1505492992 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043949448345035106, + "loss": 1.3928, + "theoretical_loss": 3.54339799425873, + "tokens_seen": 1505558528 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004394784353059178, + "loss": 1.235, + "theoretical_loss": 3.5433845140200195, + "tokens_seen": 1505624064 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043946238716148446, + "loss": 1.3208, + "theoretical_loss": 3.543371034532342, + "tokens_seen": 1505689600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004394463390170512, + "loss": 1.2994, + "theoretical_loss": 3.5433575557956223, + "tokens_seen": 1505755136 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043943029087261786, + "loss": 1.3554, + "theoretical_loss": 3.5433440778097864, + "tokens_seen": 1505820672 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004394142427281846, + "loss": 1.3126, + "theoretical_loss": 3.5433306005747593, + "tokens_seen": 1505886208 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043939819458375126, + "loss": 1.3214, + "theoretical_loss": 3.5433171240904673, + "tokens_seen": 1505951744 + }, + { + "epoch": 0.46, + "learning_rate": 0.000439382146439318, + "loss": 1.208, + "theoretical_loss": 3.543303648356835, + "tokens_seen": 1506017280 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043936609829488466, + "loss": 1.2476, + "theoretical_loss": 3.543290173373789, + "tokens_seen": 1506082816 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004393500501504514, + "loss": 1.3929, + "theoretical_loss": 3.543276699141254, + "tokens_seen": 1506148352 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043933400200601806, + "loss": 1.2842, + "theoretical_loss": 3.5432632256591554, + "tokens_seen": 1506213888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004393179538615848, + "loss": 1.3316, + "theoretical_loss": 3.5432497529274194, + "tokens_seen": 1506279424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043930190571715147, + "loss": 1.2433, + "theoretical_loss": 3.543236280945971, + "tokens_seen": 1506344960 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004392858575727182, + "loss": 1.2448, + "theoretical_loss": 3.5432228097147362, + "tokens_seen": 1506410496 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043926980942828487, + "loss": 1.3293, + "theoretical_loss": 3.5432093392336403, + "tokens_seen": 1506476032 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004392537612838516, + "loss": 1.3229, + "theoretical_loss": 3.5431958695026093, + "tokens_seen": 1506541568 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043923771313941827, + "loss": 1.3073, + "theoretical_loss": 3.5431824005215686, + "tokens_seen": 1506607104 + }, + { + "epoch": 0.46, + "learning_rate": 0.000439221664994985, + "loss": 1.2099, + "theoretical_loss": 3.5431689322904436, + "tokens_seen": 1506672640 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043920561685055167, + "loss": 1.2974, + "theoretical_loss": 3.54315546480916, + "tokens_seen": 1506738176 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004391895687061184, + "loss": 1.3451, + "theoretical_loss": 3.543141998077644, + "tokens_seen": 1506803712 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043917352056168507, + "loss": 1.171, + "theoretical_loss": 3.54312853209582, + "tokens_seen": 1506869248 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004391574724172518, + "loss": 1.2839, + "theoretical_loss": 3.5431150668636153, + "tokens_seen": 1506934784 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043914142427281847, + "loss": 1.2949, + "theoretical_loss": 3.543101602380954, + "tokens_seen": 1507000320 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043912537612838523, + "loss": 1.2533, + "theoretical_loss": 3.5430881386477626, + "tokens_seen": 1507065856 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004391093279839519, + "loss": 1.2842, + "theoretical_loss": 3.5430746756639673, + "tokens_seen": 1507131392 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004390932798395186, + "loss": 1.4479, + "theoretical_loss": 3.5430612134294925, + "tokens_seen": 1507196928 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004390772316950853, + "loss": 1.2583, + "theoretical_loss": 3.5430477519442647, + "tokens_seen": 1507262464 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 505087, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2310185432434082, + "objective/train/theoretical_loss": 3.54303429120821, + "objective/train/tokens_used": 1527788000, + "theoretical_loss": 3.54303429120821, + "tokens_seen": 1507328000 + }, + { + "epoch": 0.46, + "learning_rate": 0.000439061183550652, + "loss": 1.3043, + "theoretical_loss": 3.54303429120821, + "tokens_seen": 1507328000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004390451354062187, + "loss": 1.4286, + "theoretical_loss": 3.5430208312212534, + "tokens_seen": 1507393536 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004390290872617854, + "loss": 1.2822, + "theoretical_loss": 3.5430073719833213, + "tokens_seen": 1507459072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004390130391173521, + "loss": 1.2726, + "theoretical_loss": 3.5429939134943385, + "tokens_seen": 1507524608 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004389969909729188, + "loss": 1.2838, + "theoretical_loss": 3.5429804557542317, + "tokens_seen": 1507590144 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004389809428284855, + "loss": 1.2117, + "theoretical_loss": 3.5429669987629264, + "tokens_seen": 1507655680 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004389648946840522, + "loss": 1.3165, + "theoretical_loss": 3.542953542520348, + "tokens_seen": 1507721216 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004389488465396188, + "loss": 1.2695, + "theoretical_loss": 3.5429400870264227, + "tokens_seen": 1507786752 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004389327983951856, + "loss": 1.3088, + "theoretical_loss": 3.542926632281076, + "tokens_seen": 1507852288 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043891675025075223, + "loss": 1.2226, + "theoretical_loss": 3.542913178284235, + "tokens_seen": 1507917824 + }, + { + "epoch": 0.46, + "learning_rate": 0.000438900702106319, + "loss": 1.2687, + "theoretical_loss": 3.5428997250358236, + "tokens_seen": 1507983360 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043888465396188563, + "loss": 1.3363, + "theoretical_loss": 3.5428862725357684, + "tokens_seen": 1508048896 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004388686058174524, + "loss": 1.308, + "theoretical_loss": 3.5428728207839955, + "tokens_seen": 1508114432 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043885255767301903, + "loss": 1.2982, + "theoretical_loss": 3.542859369780431, + "tokens_seen": 1508179968 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004388365095285858, + "loss": 1.2222, + "theoretical_loss": 3.542845919525, + "tokens_seen": 1508245504 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043882046138415243, + "loss": 1.2818, + "theoretical_loss": 3.542832470017629, + "tokens_seen": 1508311040 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004388044132397192, + "loss": 1.2244, + "theoretical_loss": 3.5428190212582438, + "tokens_seen": 1508376576 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043878836509528583, + "loss": 1.2179, + "theoretical_loss": 3.54280557324677, + "tokens_seen": 1508442112 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004387723169508526, + "loss": 1.2499, + "theoretical_loss": 3.5427921259831336, + "tokens_seen": 1508507648 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043875626880641923, + "loss": 1.2349, + "theoretical_loss": 3.5427786794672613, + "tokens_seen": 1508573184 + }, + { + "epoch": 0.46, + "learning_rate": 0.000438740220661986, + "loss": 1.3937, + "theoretical_loss": 3.542765233699078, + "tokens_seen": 1508638720 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043872417251755263, + "loss": 1.2186, + "theoretical_loss": 3.5427517886785096, + "tokens_seen": 1508704256 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004387081243731194, + "loss": 1.1481, + "theoretical_loss": 3.5427383444054827, + "tokens_seen": 1508769792 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043869207622868604, + "loss": 1.3161, + "theoretical_loss": 3.5427249008799233, + "tokens_seen": 1508835328 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004386760280842528, + "loss": 1.1781, + "theoretical_loss": 3.542711458101757, + "tokens_seen": 1508900864 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043865997993981944, + "loss": 1.2086, + "theoretical_loss": 3.54269801607091, + "tokens_seen": 1508966400 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004386439317953862, + "loss": 1.241, + "theoretical_loss": 3.542684574787308, + "tokens_seen": 1509031936 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043862788365095284, + "loss": 1.206, + "theoretical_loss": 3.5426711342508774, + "tokens_seen": 1509097472 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004386118355065196, + "loss": 1.2558, + "theoretical_loss": 3.542657694461544, + "tokens_seen": 1509163008 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043859578736208624, + "loss": 1.2302, + "theoretical_loss": 3.5426442554192343, + "tokens_seen": 1509228544 + }, + { + "epoch": 0.46, + "learning_rate": 0.000438579739217653, + "loss": 1.2997, + "theoretical_loss": 3.5426308171238734, + "tokens_seen": 1509294080 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043856369107321964, + "loss": 1.2498, + "theoretical_loss": 3.542617379575388, + "tokens_seen": 1509359616 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004385476429287864, + "loss": 1.3063, + "theoretical_loss": 3.542603942773704, + "tokens_seen": 1509425152 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043853159478435304, + "loss": 1.2168, + "theoretical_loss": 3.5425905067187475, + "tokens_seen": 1509490688 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004385155466399198, + "loss": 1.21, + "theoretical_loss": 3.542577071410445, + "tokens_seen": 1509556224 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043849949849548644, + "loss": 1.2149, + "theoretical_loss": 3.5425636368487217, + "tokens_seen": 1509621760 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004384834503510532, + "loss": 1.3009, + "theoretical_loss": 3.542550203033504, + "tokens_seen": 1509687296 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043846740220661984, + "loss": 1.2862, + "theoretical_loss": 3.542536769964719, + "tokens_seen": 1509752832 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004384513540621866, + "loss": 1.3723, + "theoretical_loss": 3.542523337642292, + "tokens_seen": 1509818368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004384353059177533, + "loss": 1.264, + "theoretical_loss": 3.5425099060661487, + "tokens_seen": 1509883904 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043841925777332, + "loss": 1.234, + "theoretical_loss": 3.5424964752362156, + "tokens_seen": 1509949440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004384032096288867, + "loss": 1.2317, + "theoretical_loss": 3.5424830451524194, + "tokens_seen": 1510014976 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004383871614844534, + "loss": 1.3667, + "theoretical_loss": 3.542469615814686, + "tokens_seen": 1510080512 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004383711133400201, + "loss": 1.2616, + "theoretical_loss": 3.542456187222941, + "tokens_seen": 1510146048 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004383550651955868, + "loss": 1.3187, + "theoretical_loss": 3.5424427593771117, + "tokens_seen": 1510211584 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004383390170511535, + "loss": 1.3061, + "theoretical_loss": 3.542429332277123, + "tokens_seen": 1510277120 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043832296890672015, + "loss": 1.2844, + "theoretical_loss": 3.5424159059229017, + "tokens_seen": 1510342656 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004383069207622869, + "loss": 1.2588, + "theoretical_loss": 3.542402480314374, + "tokens_seen": 1510408192 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043829087261785355, + "loss": 1.3067, + "theoretical_loss": 3.5423890554514665, + "tokens_seen": 1510473728 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004382748244734203, + "loss": 1.2576, + "theoretical_loss": 3.5423756313341057, + "tokens_seen": 1510539264 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 506269, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1935040950775146, + "objective/train/theoretical_loss": 3.542362207962216, + "objective/train/tokens_used": 1531064800, + "theoretical_loss": 3.542362207962216, + "tokens_seen": 1510604800 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043825877632898695, + "loss": 1.1419, + "theoretical_loss": 3.542362207962216, + "tokens_seen": 1510604800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004382427281845537, + "loss": 1.3459, + "theoretical_loss": 3.542348785335726, + "tokens_seen": 1510670336 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043822668004012035, + "loss": 1.27, + "theoretical_loss": 3.5423353634545607, + "tokens_seen": 1510735872 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004382106318956871, + "loss": 1.2022, + "theoretical_loss": 3.5423219423186465, + "tokens_seen": 1510801408 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043819458375125376, + "loss": 1.2414, + "theoretical_loss": 3.54230852192791, + "tokens_seen": 1510866944 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004381785356068205, + "loss": 1.3336, + "theoretical_loss": 3.542295102282277, + "tokens_seen": 1510932480 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043816248746238716, + "loss": 1.1944, + "theoretical_loss": 3.542281683381674, + "tokens_seen": 1510998016 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004381464393179539, + "loss": 1.0949, + "theoretical_loss": 3.542268265226028, + "tokens_seen": 1511063552 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043813039117352056, + "loss": 1.3464, + "theoretical_loss": 3.5422548478152645, + "tokens_seen": 1511129088 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004381143430290873, + "loss": 1.2643, + "theoretical_loss": 3.54224143114931, + "tokens_seen": 1511194624 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043809829488465396, + "loss": 1.2565, + "theoretical_loss": 3.542228015228091, + "tokens_seen": 1511260160 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004380822467402207, + "loss": 1.2449, + "theoretical_loss": 3.542214600051534, + "tokens_seen": 1511325696 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043806619859578736, + "loss": 1.1884, + "theoretical_loss": 3.5422011856195654, + "tokens_seen": 1511391232 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004380501504513541, + "loss": 1.1885, + "theoretical_loss": 3.5421877719321113, + "tokens_seen": 1511456768 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043803410230692076, + "loss": 1.3002, + "theoretical_loss": 3.542174358989098, + "tokens_seen": 1511522304 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004380180541624875, + "loss": 1.2442, + "theoretical_loss": 3.5421609467904522, + "tokens_seen": 1511587840 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043800200601805416, + "loss": 1.2248, + "theoretical_loss": 3.5421475353361007, + "tokens_seen": 1511653376 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004379859578736209, + "loss": 1.2079, + "theoretical_loss": 3.542134124625969, + "tokens_seen": 1511718912 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043796990972918756, + "loss": 1.179, + "theoretical_loss": 3.5421207146599842, + "tokens_seen": 1511784448 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004379538615847543, + "loss": 1.2244, + "theoretical_loss": 3.5421073054380727, + "tokens_seen": 1511849984 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043793781344032097, + "loss": 1.2915, + "theoretical_loss": 3.5420938969601607, + "tokens_seen": 1511915520 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004379217652958877, + "loss": 1.2193, + "theoretical_loss": 3.5420804892261746, + "tokens_seen": 1511981056 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043790571715145437, + "loss": 1.2582, + "theoretical_loss": 3.5420670822360414, + "tokens_seen": 1512046592 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004378896690070211, + "loss": 1.4046, + "theoretical_loss": 3.5420536759896875, + "tokens_seen": 1512112128 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043787362086258777, + "loss": 1.3198, + "theoretical_loss": 3.542040270487039, + "tokens_seen": 1512177664 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004378575727181545, + "loss": 1.2916, + "theoretical_loss": 3.5420268657280225, + "tokens_seen": 1512243200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043784152457372117, + "loss": 1.2808, + "theoretical_loss": 3.542013461712565, + "tokens_seen": 1512308736 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004378254764292879, + "loss": 1.2988, + "theoretical_loss": 3.542000058440592, + "tokens_seen": 1512374272 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043780942828485457, + "loss": 1.3036, + "theoretical_loss": 3.5419866559120314, + "tokens_seen": 1512439808 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004377933801404213, + "loss": 1.2818, + "theoretical_loss": 3.541973254126809, + "tokens_seen": 1512505344 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043777733199598797, + "loss": 1.3089, + "theoretical_loss": 3.541959853084851, + "tokens_seen": 1512570880 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004377612838515547, + "loss": 1.1615, + "theoretical_loss": 3.5419464527860844, + "tokens_seen": 1512636416 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004377452357071214, + "loss": 1.3224, + "theoretical_loss": 3.5419330532304363, + "tokens_seen": 1512701952 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043772918756268813, + "loss": 1.2271, + "theoretical_loss": 3.541919654417833, + "tokens_seen": 1512767488 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004377131394182548, + "loss": 1.2903, + "theoretical_loss": 3.5419062563482004, + "tokens_seen": 1512833024 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004376970912738215, + "loss": 1.3538, + "theoretical_loss": 3.5418928590214653, + "tokens_seen": 1512898560 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004376810431293882, + "loss": 1.304, + "theoretical_loss": 3.5418794624375556, + "tokens_seen": 1512964096 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004376649949849549, + "loss": 1.2842, + "theoretical_loss": 3.5418660665963966, + "tokens_seen": 1513029632 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004376489468405216, + "loss": 1.1832, + "theoretical_loss": 3.541852671497915, + "tokens_seen": 1513095168 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004376328986960883, + "loss": 1.4902, + "theoretical_loss": 3.541839277142038, + "tokens_seen": 1513160704 + }, + { + "epoch": 0.46, + "learning_rate": 0.000437616850551655, + "loss": 1.2467, + "theoretical_loss": 3.541825883528692, + "tokens_seen": 1513226240 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004376008024072217, + "loss": 1.3917, + "theoretical_loss": 3.5418124906578043, + "tokens_seen": 1513291776 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004375847542627884, + "loss": 1.2329, + "theoretical_loss": 3.541799098529301, + "tokens_seen": 1513357312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004375687061183551, + "loss": 1.2874, + "theoretical_loss": 3.5417857071431085, + "tokens_seen": 1513422848 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004375526579739217, + "loss": 1.3836, + "theoretical_loss": 3.541772316499154, + "tokens_seen": 1513488384 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004375366098294885, + "loss": 1.1754, + "theoretical_loss": 3.541758926597364, + "tokens_seen": 1513553920 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043752056168505513, + "loss": 1.3074, + "theoretical_loss": 3.541745537437665, + "tokens_seen": 1513619456 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004375045135406219, + "loss": 1.266, + "theoretical_loss": 3.541732149019985, + "tokens_seen": 1513684992 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043748846539618853, + "loss": 1.3281, + "theoretical_loss": 3.5417187613442493, + "tokens_seen": 1513750528 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004374724172517553, + "loss": 1.2361, + "theoretical_loss": 3.541705374410385, + "tokens_seen": 1513816064 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 507531, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.190418004989624, + "objective/train/theoretical_loss": 3.5416919882183193, + "objective/train/tokens_used": 1534341600, + "theoretical_loss": 3.5416919882183193, + "tokens_seen": 1513881600 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043745636910732193, + "loss": 1.2266, + "theoretical_loss": 3.5416919882183193, + "tokens_seen": 1513881600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004374403209628887, + "loss": 1.3258, + "theoretical_loss": 3.5416786027679787, + "tokens_seen": 1513947136 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043742427281845533, + "loss": 1.2911, + "theoretical_loss": 3.5416652180592902, + "tokens_seen": 1514012672 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004374082246740221, + "loss": 1.4003, + "theoretical_loss": 3.54165183409218, + "tokens_seen": 1514078208 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043739217652958873, + "loss": 1.3224, + "theoretical_loss": 3.541638450866576, + "tokens_seen": 1514143744 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004373761283851555, + "loss": 1.2397, + "theoretical_loss": 3.5416250683824044, + "tokens_seen": 1514209280 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043736008024072213, + "loss": 1.3816, + "theoretical_loss": 3.5416116866395915, + "tokens_seen": 1514274816 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004373440320962889, + "loss": 1.3999, + "theoretical_loss": 3.541598305638065, + "tokens_seen": 1514340352 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043732798395185554, + "loss": 1.3185, + "theoretical_loss": 3.5415849253777516, + "tokens_seen": 1514405888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004373119358074223, + "loss": 1.1787, + "theoretical_loss": 3.541571545858578, + "tokens_seen": 1514471424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043729588766298894, + "loss": 1.2119, + "theoretical_loss": 3.5415581670804706, + "tokens_seen": 1514536960 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004372798395185557, + "loss": 1.3185, + "theoretical_loss": 3.5415447890433573, + "tokens_seen": 1514602496 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043726379137412234, + "loss": 1.3128, + "theoretical_loss": 3.5415314117471643, + "tokens_seen": 1514668032 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004372477432296891, + "loss": 1.3142, + "theoretical_loss": 3.5415180351918187, + "tokens_seen": 1514733568 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043723169508525585, + "loss": 1.3487, + "theoretical_loss": 3.5415046593772477, + "tokens_seen": 1514799104 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004372156469408225, + "loss": 1.2566, + "theoretical_loss": 3.5414912843033775, + "tokens_seen": 1514864640 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043719959879638925, + "loss": 1.2866, + "theoretical_loss": 3.5414779099701352, + "tokens_seen": 1514930176 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004371835506519559, + "loss": 1.3375, + "theoretical_loss": 3.5414645363774486, + "tokens_seen": 1514995712 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043716750250752265, + "loss": 1.3559, + "theoretical_loss": 3.5414511635252444, + "tokens_seen": 1515061248 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004371514543630893, + "loss": 1.3581, + "theoretical_loss": 3.5414377914134487, + "tokens_seen": 1515126784 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043713540621865605, + "loss": 1.2881, + "theoretical_loss": 3.541424420041989, + "tokens_seen": 1515192320 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004371193580742227, + "loss": 1.2667, + "theoretical_loss": 3.541411049410793, + "tokens_seen": 1515257856 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043710330992978945, + "loss": 1.2893, + "theoretical_loss": 3.541397679519786, + "tokens_seen": 1515323392 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004370872617853561, + "loss": 1.302, + "theoretical_loss": 3.5413843103688967, + "tokens_seen": 1515388928 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043707121364092285, + "loss": 1.42, + "theoretical_loss": 3.5413709419580517, + "tokens_seen": 1515454464 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004370551654964895, + "loss": 1.2374, + "theoretical_loss": 3.5413575742871775, + "tokens_seen": 1515520000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004370391173520562, + "loss": 1.2864, + "theoretical_loss": 3.5413442073562016, + "tokens_seen": 1515585536 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004370230692076229, + "loss": 1.1992, + "theoretical_loss": 3.541330841165051, + "tokens_seen": 1515651072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004370070210631896, + "loss": 1.3588, + "theoretical_loss": 3.5413174757136527, + "tokens_seen": 1515716608 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004369909729187563, + "loss": 1.3657, + "theoretical_loss": 3.5413041110019337, + "tokens_seen": 1515782144 + }, + { + "epoch": 0.46, + "learning_rate": 0.000436974924774323, + "loss": 1.3434, + "theoretical_loss": 3.541290747029821, + "tokens_seen": 1515847680 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004369588766298897, + "loss": 1.347, + "theoretical_loss": 3.541277383797242, + "tokens_seen": 1515913216 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004369428284854564, + "loss": 1.4375, + "theoretical_loss": 3.5412640213041238, + "tokens_seen": 1515978752 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043692678034102305, + "loss": 1.1744, + "theoretical_loss": 3.5412506595503928, + "tokens_seen": 1516044288 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004369107321965898, + "loss": 1.2759, + "theoretical_loss": 3.541237298535977, + "tokens_seen": 1516109824 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043689468405215645, + "loss": 1.3161, + "theoretical_loss": 3.5412239382608037, + "tokens_seen": 1516175360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004368786359077232, + "loss": 1.2691, + "theoretical_loss": 3.541210578724799, + "tokens_seen": 1516240896 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043686258776328985, + "loss": 1.3528, + "theoretical_loss": 3.541197219927891, + "tokens_seen": 1516306432 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004368465396188566, + "loss": 1.2354, + "theoretical_loss": 3.541183861870006, + "tokens_seen": 1516371968 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043683049147442326, + "loss": 1.1958, + "theoretical_loss": 3.541170504551072, + "tokens_seen": 1516437504 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043681444332999, + "loss": 1.3622, + "theoretical_loss": 3.541157147971016, + "tokens_seen": 1516503040 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043679839518555666, + "loss": 1.2206, + "theoretical_loss": 3.5411437921297653, + "tokens_seen": 1516568576 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004367823470411234, + "loss": 1.2189, + "theoretical_loss": 3.5411304370272467, + "tokens_seen": 1516634112 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043676629889669006, + "loss": 1.294, + "theoretical_loss": 3.5411170826633875, + "tokens_seen": 1516699648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004367502507522568, + "loss": 1.3575, + "theoretical_loss": 3.541103729038115, + "tokens_seen": 1516765184 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043673420260782346, + "loss": 1.2574, + "theoretical_loss": 3.541090376151357, + "tokens_seen": 1516830720 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004367181544633902, + "loss": 1.2853, + "theoretical_loss": 3.5410770240030396, + "tokens_seen": 1516896256 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043670210631895686, + "loss": 1.2021, + "theoretical_loss": 3.541063672593091, + "tokens_seen": 1516961792 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004366860581745236, + "loss": 1.2567, + "theoretical_loss": 3.541050321921438, + "tokens_seen": 1517027328 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043667001003009026, + "loss": 1.3325, + "theoretical_loss": 3.5410369719880084, + "tokens_seen": 1517092864 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 508543, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.311399221420288, + "objective/train/theoretical_loss": 3.5410236227927285, + "objective/train/tokens_used": 1537618400, + "theoretical_loss": 3.5410236227927285, + "tokens_seen": 1517158400 + }, + { + "epoch": 0.46, + "learning_rate": 0.000436653961885657, + "loss": 1.2733, + "theoretical_loss": 3.5410236227927285, + "tokens_seen": 1517158400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043663791374122366, + "loss": 1.3425, + "theoretical_loss": 3.5410102743355267, + "tokens_seen": 1517223936 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004366218655967904, + "loss": 1.2955, + "theoretical_loss": 3.54099692661633, + "tokens_seen": 1517289472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043660581745235706, + "loss": 1.3617, + "theoretical_loss": 3.540983579635065, + "tokens_seen": 1517355008 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004365897693079238, + "loss": 1.3975, + "theoretical_loss": 3.5409702333916604, + "tokens_seen": 1517420544 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043657372116349047, + "loss": 1.1151, + "theoretical_loss": 3.5409568878860425, + "tokens_seen": 1517486080 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004365576730190572, + "loss": 1.2129, + "theoretical_loss": 3.5409435431181384, + "tokens_seen": 1517551616 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043654162487462387, + "loss": 1.2832, + "theoretical_loss": 3.5409301990878763, + "tokens_seen": 1517617152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004365255767301906, + "loss": 1.2076, + "theoretical_loss": 3.5409168557951833, + "tokens_seen": 1517682688 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043650952858575727, + "loss": 1.3613, + "theoretical_loss": 3.540903513239987, + "tokens_seen": 1517748224 + }, + { + "epoch": 0.46, + "learning_rate": 0.000436493480441324, + "loss": 1.2954, + "theoretical_loss": 3.5408901714222143, + "tokens_seen": 1517813760 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043647743229689067, + "loss": 1.3134, + "theoretical_loss": 3.540876830341793, + "tokens_seen": 1517879296 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004364613841524574, + "loss": 1.3425, + "theoretical_loss": 3.5408634899986504, + "tokens_seen": 1517944832 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043644533600802407, + "loss": 1.323, + "theoretical_loss": 3.5408501503927132, + "tokens_seen": 1518010368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004364292878635908, + "loss": 1.2203, + "theoretical_loss": 3.5408368115239104, + "tokens_seen": 1518075904 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043641323971915747, + "loss": 1.2025, + "theoretical_loss": 3.540823473392168, + "tokens_seen": 1518141440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004363971915747242, + "loss": 1.31, + "theoretical_loss": 3.5408101359974147, + "tokens_seen": 1518206976 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004363811434302909, + "loss": 1.3091, + "theoretical_loss": 3.5407967993395766, + "tokens_seen": 1518272512 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043636509528585763, + "loss": 1.1995, + "theoretical_loss": 3.540783463418582, + "tokens_seen": 1518338048 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004363490471414243, + "loss": 1.3549, + "theoretical_loss": 3.5407701282343584, + "tokens_seen": 1518403584 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043633299899699103, + "loss": 1.2267, + "theoretical_loss": 3.540756793786833, + "tokens_seen": 1518469120 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004363169508525577, + "loss": 1.2063, + "theoretical_loss": 3.5407434600759338, + "tokens_seen": 1518534656 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043630090270812443, + "loss": 1.3824, + "theoretical_loss": 3.540730127101588, + "tokens_seen": 1518600192 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004362848545636911, + "loss": 1.2785, + "theoretical_loss": 3.5407167948637226, + "tokens_seen": 1518665728 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004362688064192578, + "loss": 1.4351, + "theoretical_loss": 3.540703463362266, + "tokens_seen": 1518731264 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004362527582748245, + "loss": 1.3, + "theoretical_loss": 3.5406901325971454, + "tokens_seen": 1518796800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004362367101303912, + "loss": 1.202, + "theoretical_loss": 3.5406768025682887, + "tokens_seen": 1518862336 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004362206619859579, + "loss": 1.3048, + "theoretical_loss": 3.5406634732756226, + "tokens_seen": 1518927872 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004362046138415246, + "loss": 1.2868, + "theoretical_loss": 3.5406501447190752, + "tokens_seen": 1518993408 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004361885656970913, + "loss": 1.294, + "theoretical_loss": 3.5406368168985742, + "tokens_seen": 1519058944 + }, + { + "epoch": 0.46, + "learning_rate": 0.000436172517552658, + "loss": 1.2987, + "theoretical_loss": 3.540623489814047, + "tokens_seen": 1519124480 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043615646940822463, + "loss": 1.3564, + "theoretical_loss": 3.5406101634654217, + "tokens_seen": 1519190016 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004361404212637914, + "loss": 1.2843, + "theoretical_loss": 3.540596837852626, + "tokens_seen": 1519255552 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043612437311935803, + "loss": 1.2792, + "theoretical_loss": 3.5405835129755863, + "tokens_seen": 1519321088 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004361083249749248, + "loss": 1.3048, + "theoretical_loss": 3.5405701888342307, + "tokens_seen": 1519386624 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043609227683049143, + "loss": 1.2529, + "theoretical_loss": 3.540556865428488, + "tokens_seen": 1519452160 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004360762286860582, + "loss": 1.261, + "theoretical_loss": 3.5405435427582845, + "tokens_seen": 1519517696 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043606018054162483, + "loss": 1.4229, + "theoretical_loss": 3.5405302208235483, + "tokens_seen": 1519583232 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004360441323971916, + "loss": 1.3048, + "theoretical_loss": 3.540516899624208, + "tokens_seen": 1519648768 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043602808425275834, + "loss": 1.2527, + "theoretical_loss": 3.5405035791601898, + "tokens_seen": 1519714304 + }, + { + "epoch": 0.46, + "learning_rate": 0.000436012036108325, + "loss": 1.3167, + "theoretical_loss": 3.5404902594314223, + "tokens_seen": 1519779840 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043599598796389174, + "loss": 1.2432, + "theoretical_loss": 3.5404769404378325, + "tokens_seen": 1519845376 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004359799398194584, + "loss": 1.2814, + "theoretical_loss": 3.540463622179349, + "tokens_seen": 1519910912 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043596389167502514, + "loss": 1.2208, + "theoretical_loss": 3.5404503046558995, + "tokens_seen": 1519976448 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004359478435305918, + "loss": 1.2732, + "theoretical_loss": 3.5404369878674107, + "tokens_seen": 1520041984 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043593179538615855, + "loss": 1.2721, + "theoretical_loss": 3.5404236718138113, + "tokens_seen": 1520107520 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004359157472417252, + "loss": 1.3007, + "theoretical_loss": 3.540410356495029, + "tokens_seen": 1520173056 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043589969909729195, + "loss": 1.3334, + "theoretical_loss": 3.5403970419109916, + "tokens_seen": 1520238592 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004358836509528586, + "loss": 1.2909, + "theoretical_loss": 3.5403837280616264, + "tokens_seen": 1520304128 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043586760280842535, + "loss": 1.266, + "theoretical_loss": 3.540370414946861, + "tokens_seen": 1520369664 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 509751, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2503533363342285, + "objective/train/theoretical_loss": 3.5403571025666247, + "objective/train/tokens_used": 1540895200, + "theoretical_loss": 3.5403571025666247, + "tokens_seen": 1520435200 + }, + { + "epoch": 0.46, + "learning_rate": 0.000435851554663992, + "loss": 1.2463, + "theoretical_loss": 3.5403571025666247, + "tokens_seen": 1520435200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043583550651955875, + "loss": 1.3067, + "theoretical_loss": 3.5403437909208435, + "tokens_seen": 1520500736 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004358194583751254, + "loss": 1.2966, + "theoretical_loss": 3.5403304800094464, + "tokens_seen": 1520566272 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043580341023069215, + "loss": 1.2952, + "theoretical_loss": 3.5403171698323606, + "tokens_seen": 1520631808 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004357873620862588, + "loss": 1.3056, + "theoretical_loss": 3.540303860389514, + "tokens_seen": 1520697344 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043577131394182555, + "loss": 1.3061, + "theoretical_loss": 3.540290551680835, + "tokens_seen": 1520762880 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004357552657973922, + "loss": 1.3502, + "theoretical_loss": 3.5402772437062513, + "tokens_seen": 1520828416 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043573921765295895, + "loss": 1.2762, + "theoretical_loss": 3.5402639364656903, + "tokens_seen": 1520893952 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004357231695085256, + "loss": 1.2874, + "theoretical_loss": 3.5402506299590804, + "tokens_seen": 1520959488 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043570712136409235, + "loss": 1.2866, + "theoretical_loss": 3.540237324186349, + "tokens_seen": 1521025024 + }, + { + "epoch": 0.46, + "learning_rate": 0.000435691073219659, + "loss": 1.3873, + "theoretical_loss": 3.5402240191474244, + "tokens_seen": 1521090560 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043567502507522576, + "loss": 1.2747, + "theoretical_loss": 3.5402107148422344, + "tokens_seen": 1521156096 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004356589769307924, + "loss": 1.2825, + "theoretical_loss": 3.5401974112707073, + "tokens_seen": 1521221632 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004356429287863591, + "loss": 1.4176, + "theoretical_loss": 3.5401841084327703, + "tokens_seen": 1521287168 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004356268806419258, + "loss": 1.3239, + "theoretical_loss": 3.5401708063283515, + "tokens_seen": 1521352704 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004356108324974925, + "loss": 1.3202, + "theoretical_loss": 3.5401575049573792, + "tokens_seen": 1521418240 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004355947843530592, + "loss": 1.2148, + "theoretical_loss": 3.5401442043197813, + "tokens_seen": 1521483776 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004355787362086259, + "loss": 1.326, + "theoretical_loss": 3.540130904415486, + "tokens_seen": 1521549312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004355626880641926, + "loss": 1.3531, + "theoretical_loss": 3.5401176052444208, + "tokens_seen": 1521614848 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004355466399197593, + "loss": 1.3342, + "theoretical_loss": 3.540104306806514, + "tokens_seen": 1521680384 + }, + { + "epoch": 0.46, + "learning_rate": 0.000435530591775326, + "loss": 1.3025, + "theoretical_loss": 3.5400910091016935, + "tokens_seen": 1521745920 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004355145436308927, + "loss": 1.3313, + "theoretical_loss": 3.540077712129887, + "tokens_seen": 1521811456 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043549849548645935, + "loss": 1.2824, + "theoretical_loss": 3.540064415891023, + "tokens_seen": 1521876992 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004354824473420261, + "loss": 1.2822, + "theoretical_loss": 3.54005112038503, + "tokens_seen": 1521942528 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043546639919759276, + "loss": 1.3042, + "theoretical_loss": 3.5400378256118348, + "tokens_seen": 1522008064 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004354503510531595, + "loss": 1.3068, + "theoretical_loss": 3.5400245315713668, + "tokens_seen": 1522073600 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043543430290872616, + "loss": 1.2594, + "theoretical_loss": 3.5400112382635527, + "tokens_seen": 1522139136 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004354182547642929, + "loss": 1.2435, + "theoretical_loss": 3.5399979456883215, + "tokens_seen": 1522204672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043540220661985956, + "loss": 1.2718, + "theoretical_loss": 3.5399846538456012, + "tokens_seen": 1522270208 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004353861584754263, + "loss": 1.3395, + "theoretical_loss": 3.5399713627353195, + "tokens_seen": 1522335744 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043537011033099296, + "loss": 1.274, + "theoretical_loss": 3.539958072357405, + "tokens_seen": 1522401280 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004353540621865597, + "loss": 1.3374, + "theoretical_loss": 3.5399447827117854, + "tokens_seen": 1522466816 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043533801404212636, + "loss": 1.4313, + "theoretical_loss": 3.5399314937983895, + "tokens_seen": 1522532352 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004353219658976931, + "loss": 1.3218, + "theoretical_loss": 3.5399182056171448, + "tokens_seen": 1522597888 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043530591775325976, + "loss": 1.3027, + "theoretical_loss": 3.539904918167979, + "tokens_seen": 1522663424 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004352898696088265, + "loss": 1.2387, + "theoretical_loss": 3.5398916314508213, + "tokens_seen": 1522728960 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043527382146439316, + "loss": 1.24, + "theoretical_loss": 3.5398783454656, + "tokens_seen": 1522794496 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004352577733199599, + "loss": 1.3221, + "theoretical_loss": 3.539865060212242, + "tokens_seen": 1522860032 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043524172517552656, + "loss": 1.3932, + "theoretical_loss": 3.5398517756906767, + "tokens_seen": 1522925568 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004352256770310933, + "loss": 1.223, + "theoretical_loss": 3.539838491900831, + "tokens_seen": 1522991104 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043520962888665997, + "loss": 1.2602, + "theoretical_loss": 3.5398252088426347, + "tokens_seen": 1523056640 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004351935807422267, + "loss": 1.3085, + "theoretical_loss": 3.5398119265160153, + "tokens_seen": 1523122176 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043517753259779337, + "loss": 1.3413, + "theoretical_loss": 3.539798644920901, + "tokens_seen": 1523187712 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004351614844533601, + "loss": 1.3216, + "theoretical_loss": 3.539785364057219, + "tokens_seen": 1523253248 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043514543630892677, + "loss": 1.4126, + "theoretical_loss": 3.5397720839249, + "tokens_seen": 1523318784 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004351293881644935, + "loss": 1.3832, + "theoretical_loss": 3.5397588045238697, + "tokens_seen": 1523384320 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043511334002006017, + "loss": 1.3739, + "theoretical_loss": 3.5397455258540584, + "tokens_seen": 1523449856 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004350972918756269, + "loss": 1.3289, + "theoretical_loss": 3.539732247915393, + "tokens_seen": 1523515392 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043508124373119357, + "loss": 1.3113, + "theoretical_loss": 3.5397189707078027, + "tokens_seen": 1523580928 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004350651955867603, + "loss": 1.2262, + "theoretical_loss": 3.5397056942312153, + "tokens_seen": 1523646464 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 510713, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2261080741882324, + "objective/train/theoretical_loss": 3.5396924184855587, + "objective/train/tokens_used": 1544172000, + "theoretical_loss": 3.5396924184855587, + "tokens_seen": 1523712000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043504914744232697, + "loss": 1.2512, + "theoretical_loss": 3.5396924184855587, + "tokens_seen": 1523712000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004350330992978937, + "loss": 1.2323, + "theoretical_loss": 3.5396791434707624, + "tokens_seen": 1523777536 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004350170511534604, + "loss": 1.4104, + "theoretical_loss": 3.539665869186754, + "tokens_seen": 1523843072 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043500100300902713, + "loss": 1.3584, + "theoretical_loss": 3.5396525956334615, + "tokens_seen": 1523908608 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004349849548645938, + "loss": 1.3525, + "theoretical_loss": 3.539639322810814, + "tokens_seen": 1523974144 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043496890672016053, + "loss": 1.3044, + "theoretical_loss": 3.5396260507187396, + "tokens_seen": 1524039680 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004349528585757272, + "loss": 1.3124, + "theoretical_loss": 3.5396127793571663, + "tokens_seen": 1524105216 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043493681043129393, + "loss": 1.2217, + "theoretical_loss": 3.539599508726023, + "tokens_seen": 1524170752 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004349207622868606, + "loss": 1.2942, + "theoretical_loss": 3.5395862388252377, + "tokens_seen": 1524236288 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043490471414242733, + "loss": 1.2468, + "theoretical_loss": 3.5395729696547393, + "tokens_seen": 1524301824 + }, + { + "epoch": 0.46, + "learning_rate": 0.000434888665997994, + "loss": 1.3595, + "theoretical_loss": 3.539559701214456, + "tokens_seen": 1524367360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004348726178535607, + "loss": 1.2068, + "theoretical_loss": 3.5395464335043156, + "tokens_seen": 1524432896 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004348565697091274, + "loss": 1.2839, + "theoretical_loss": 3.5395331665242473, + "tokens_seen": 1524498432 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004348405215646941, + "loss": 1.3361, + "theoretical_loss": 3.53951990027418, + "tokens_seen": 1524563968 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043482447342026083, + "loss": 1.3117, + "theoretical_loss": 3.5395066347540407, + "tokens_seen": 1524629504 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004348084252758275, + "loss": 1.3429, + "theoretical_loss": 3.5394933699637585, + "tokens_seen": 1524695040 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043479237713139424, + "loss": 1.4326, + "theoretical_loss": 3.5394801059032623, + "tokens_seen": 1524760576 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004347763289869609, + "loss": 1.2817, + "theoretical_loss": 3.5394668425724802, + "tokens_seen": 1524826112 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043476028084252764, + "loss": 1.3371, + "theoretical_loss": 3.539453579971341, + "tokens_seen": 1524891648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004347442326980943, + "loss": 1.3175, + "theoretical_loss": 3.539440318099773, + "tokens_seen": 1524957184 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043472818455366104, + "loss": 1.2191, + "theoretical_loss": 3.539427056957704, + "tokens_seen": 1525022720 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004347121364092277, + "loss": 1.275, + "theoretical_loss": 3.5394137965450643, + "tokens_seen": 1525088256 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043469608826479444, + "loss": 1.3256, + "theoretical_loss": 3.539400536861781, + "tokens_seen": 1525153792 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004346800401203611, + "loss": 1.2468, + "theoretical_loss": 3.5393872779077826, + "tokens_seen": 1525219328 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043466399197592784, + "loss": 1.2451, + "theoretical_loss": 3.539374019682999, + "tokens_seen": 1525284864 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004346479438314945, + "loss": 1.3357, + "theoretical_loss": 3.5393607621873566, + "tokens_seen": 1525350400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043463189568706124, + "loss": 1.3227, + "theoretical_loss": 3.539347505420786, + "tokens_seen": 1525415936 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004346158475426279, + "loss": 1.2107, + "theoretical_loss": 3.5393342493832147, + "tokens_seen": 1525481472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043459979939819464, + "loss": 1.3845, + "theoretical_loss": 3.539320994074572, + "tokens_seen": 1525547008 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004345837512537613, + "loss": 1.3037, + "theoretical_loss": 3.5393077394947854, + "tokens_seen": 1525612544 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043456770310932804, + "loss": 1.3719, + "theoretical_loss": 3.5392944856437847, + "tokens_seen": 1525678080 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004345516549648947, + "loss": 1.2568, + "theoretical_loss": 3.539281232521498, + "tokens_seen": 1525743616 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043453560682046145, + "loss": 1.3319, + "theoretical_loss": 3.539267980127854, + "tokens_seen": 1525809152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004345195586760281, + "loss": 1.268, + "theoretical_loss": 3.5392547284627813, + "tokens_seen": 1525874688 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043450351053159485, + "loss": 1.3637, + "theoretical_loss": 3.5392414775262084, + "tokens_seen": 1525940224 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004344874623871615, + "loss": 1.3438, + "theoretical_loss": 3.5392282273180644, + "tokens_seen": 1526005760 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043447141424272825, + "loss": 1.3017, + "theoretical_loss": 3.5392149778382773, + "tokens_seen": 1526071296 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004344553660982949, + "loss": 1.3636, + "theoretical_loss": 3.539201729086777, + "tokens_seen": 1526136832 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043443931795386165, + "loss": 1.2316, + "theoretical_loss": 3.5391884810634906, + "tokens_seen": 1526202368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004344232698094283, + "loss": 1.2293, + "theoretical_loss": 3.539175233768348, + "tokens_seen": 1526267904 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043440722166499505, + "loss": 1.228, + "theoretical_loss": 3.5391619872012776, + "tokens_seen": 1526333440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004343911735205617, + "loss": 1.2361, + "theoretical_loss": 3.539148741362208, + "tokens_seen": 1526398976 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043437512537612845, + "loss": 1.3815, + "theoretical_loss": 3.5391354962510677, + "tokens_seen": 1526464512 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004343590772316951, + "loss": 1.2754, + "theoretical_loss": 3.539122251867786, + "tokens_seen": 1526530048 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043434302908726185, + "loss": 1.3377, + "theoretical_loss": 3.5391090082122916, + "tokens_seen": 1526595584 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004343269809428285, + "loss": 1.3253, + "theoretical_loss": 3.5390957652845128, + "tokens_seen": 1526661120 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043431093279839526, + "loss": 1.3318, + "theoretical_loss": 3.539082523084379, + "tokens_seen": 1526726656 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004342948846539619, + "loss": 1.297, + "theoretical_loss": 3.539069281611818, + "tokens_seen": 1526792192 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043427883650952866, + "loss": 1.3165, + "theoretical_loss": 3.5390560408667593, + "tokens_seen": 1526857728 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004342627883650953, + "loss": 1.2439, + "theoretical_loss": 3.539042800849132, + "tokens_seen": 1526923264 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 511958, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.314308524131775, + "objective/train/theoretical_loss": 3.5390295615588645, + "objective/train/tokens_used": 1547448800, + "theoretical_loss": 3.5390295615588645, + "tokens_seen": 1526988800 + }, + { + "epoch": 0.46, + "learning_rate": 0.000434246740220662, + "loss": 1.2534, + "theoretical_loss": 3.5390295615588645, + "tokens_seen": 1526988800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004342306920762287, + "loss": 1.2488, + "theoretical_loss": 3.5390163229958853, + "tokens_seen": 1527054336 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004342146439317954, + "loss": 1.3445, + "theoretical_loss": 3.5390030851601244, + "tokens_seen": 1527119872 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004341985957873621, + "loss": 1.3672, + "theoretical_loss": 3.538989848051509, + "tokens_seen": 1527185408 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004341825476429288, + "loss": 1.4039, + "theoretical_loss": 3.5389766116699692, + "tokens_seen": 1527250944 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004341664994984955, + "loss": 1.3695, + "theoretical_loss": 3.5389633760154338, + "tokens_seen": 1527316480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004341504513540622, + "loss": 1.3595, + "theoretical_loss": 3.5389501410878306, + "tokens_seen": 1527382016 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004341344032096289, + "loss": 1.3764, + "theoretical_loss": 3.5389369068870895, + "tokens_seen": 1527447552 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004341183550651956, + "loss": 1.3329, + "theoretical_loss": 3.538923673413139, + "tokens_seen": 1527513088 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043410230692076225, + "loss": 1.2879, + "theoretical_loss": 3.5389104406659087, + "tokens_seen": 1527578624 + }, + { + "epoch": 0.46, + "learning_rate": 0.000434086258776329, + "loss": 1.3723, + "theoretical_loss": 3.538897208645327, + "tokens_seen": 1527644160 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043407021063189566, + "loss": 1.258, + "theoretical_loss": 3.5388839773513223, + "tokens_seen": 1527709696 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004340541624874624, + "loss": 1.2726, + "theoretical_loss": 3.5388707467838243, + "tokens_seen": 1527775232 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043403811434302906, + "loss": 1.3699, + "theoretical_loss": 3.5388575169427616, + "tokens_seen": 1527840768 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004340220661985958, + "loss": 1.3514, + "theoretical_loss": 3.538844287828063, + "tokens_seen": 1527906304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043400601805416246, + "loss": 1.3933, + "theoretical_loss": 3.5388310594396577, + "tokens_seen": 1527971840 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004339899699097292, + "loss": 1.2657, + "theoretical_loss": 3.538817831777475, + "tokens_seen": 1528037376 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043397392176529586, + "loss": 1.3165, + "theoretical_loss": 3.5388046048414434, + "tokens_seen": 1528102912 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004339578736208626, + "loss": 1.3014, + "theoretical_loss": 3.538791378631492, + "tokens_seen": 1528168448 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043394182547642926, + "loss": 1.2957, + "theoretical_loss": 3.5387781531475504, + "tokens_seen": 1528233984 + }, + { + "epoch": 0.46, + "learning_rate": 0.000433925777331996, + "loss": 1.2912, + "theoretical_loss": 3.5387649283895466, + "tokens_seen": 1528299520 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043390972918756266, + "loss": 1.2831, + "theoretical_loss": 3.53875170435741, + "tokens_seen": 1528365056 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004338936810431294, + "loss": 1.2042, + "theoretical_loss": 3.53873848105107, + "tokens_seen": 1528430592 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043387763289869606, + "loss": 1.3423, + "theoretical_loss": 3.538725258470455, + "tokens_seen": 1528496128 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004338615847542628, + "loss": 1.3012, + "theoretical_loss": 3.538712036615495, + "tokens_seen": 1528561664 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043384553660982947, + "loss": 1.2643, + "theoretical_loss": 3.5386988154861188, + "tokens_seen": 1528627200 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004338294884653962, + "loss": 1.3501, + "theoretical_loss": 3.538685595082254, + "tokens_seen": 1528692736 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043381344032096287, + "loss": 1.2993, + "theoretical_loss": 3.5386723754038316, + "tokens_seen": 1528758272 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004337973921765296, + "loss": 1.3957, + "theoretical_loss": 3.5386591564507803, + "tokens_seen": 1528823808 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043378134403209627, + "loss": 1.3315, + "theoretical_loss": 3.5386459382230284, + "tokens_seen": 1528889344 + }, + { + "epoch": 0.46, + "learning_rate": 0.000433765295887663, + "loss": 1.3887, + "theoretical_loss": 3.5386327207205057, + "tokens_seen": 1528954880 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043374924774322967, + "loss": 1.2869, + "theoretical_loss": 3.538619503943141, + "tokens_seen": 1529020416 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004337331995987964, + "loss": 1.3352, + "theoretical_loss": 3.5386062878908637, + "tokens_seen": 1529085952 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043371715145436307, + "loss": 1.3301, + "theoretical_loss": 3.538593072563603, + "tokens_seen": 1529151488 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004337011033099298, + "loss": 1.2331, + "theoretical_loss": 3.5385798579612873, + "tokens_seen": 1529217024 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043368505516549647, + "loss": 1.3779, + "theoretical_loss": 3.5385666440838466, + "tokens_seen": 1529282560 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004336690070210632, + "loss": 1.3058, + "theoretical_loss": 3.53855343093121, + "tokens_seen": 1529348096 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043365295887662987, + "loss": 1.2652, + "theoretical_loss": 3.5385402185033064, + "tokens_seen": 1529413632 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043363691073219663, + "loss": 1.3195, + "theoretical_loss": 3.5385270068000647, + "tokens_seen": 1529479168 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043362086258776333, + "loss": 1.2086, + "theoretical_loss": 3.538513795821415, + "tokens_seen": 1529544704 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043360481444333003, + "loss": 1.1886, + "theoretical_loss": 3.538500585567286, + "tokens_seen": 1529610240 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043358876629889673, + "loss": 1.2729, + "theoretical_loss": 3.5384873760376068, + "tokens_seen": 1529675776 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043357271815446343, + "loss": 1.2027, + "theoretical_loss": 3.538474167232307, + "tokens_seen": 1529741312 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043355667001003013, + "loss": 1.2775, + "theoretical_loss": 3.5384609591513154, + "tokens_seen": 1529806848 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043354062186559683, + "loss": 1.3396, + "theoretical_loss": 3.5384477517945614, + "tokens_seen": 1529872384 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043352457372116353, + "loss": 1.2684, + "theoretical_loss": 3.538434545161974, + "tokens_seen": 1529937920 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043350852557673023, + "loss": 1.2376, + "theoretical_loss": 3.5384213392534836, + "tokens_seen": 1530003456 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043349247743229693, + "loss": 1.3755, + "theoretical_loss": 3.538408134069018, + "tokens_seen": 1530068992 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004334764292878636, + "loss": 1.3668, + "theoretical_loss": 3.5383949296085078, + "tokens_seen": 1530134528 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043346038114343033, + "loss": 1.2863, + "theoretical_loss": 3.538381725871881, + "tokens_seen": 1530200064 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 512859, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3156123161315918, + "objective/train/theoretical_loss": 3.5383685228590687, + "objective/train/tokens_used": 1550725600, + "theoretical_loss": 3.5383685228590687, + "tokens_seen": 1530265600 + }, + { + "epoch": 0.46, + "learning_rate": 0.000433444332998997, + "loss": 1.3095, + "theoretical_loss": 3.5383685228590687, + "tokens_seen": 1530265600 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043342828485456374, + "loss": 1.4053, + "theoretical_loss": 3.5383553205699982, + "tokens_seen": 1530331136 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004334122367101304, + "loss": 1.255, + "theoretical_loss": 3.5383421190046, + "tokens_seen": 1530396672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043339618856569714, + "loss": 1.354, + "theoretical_loss": 3.5383289181628035, + "tokens_seen": 1530462208 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004333801404212638, + "loss": 1.2306, + "theoretical_loss": 3.5383157180445375, + "tokens_seen": 1530527744 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043336409227683054, + "loss": 1.2193, + "theoretical_loss": 3.538302518649732, + "tokens_seen": 1530593280 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004333480441323972, + "loss": 1.2127, + "theoretical_loss": 3.5382893199783156, + "tokens_seen": 1530658816 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043333199598796394, + "loss": 1.3155, + "theoretical_loss": 3.5382761220302186, + "tokens_seen": 1530724352 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004333159478435306, + "loss": 1.3606, + "theoretical_loss": 3.5382629248053696, + "tokens_seen": 1530789888 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043329989969909734, + "loss": 1.3639, + "theoretical_loss": 3.538249728303698, + "tokens_seen": 1530855424 + }, + { + "epoch": 0.46, + "learning_rate": 0.000433283851554664, + "loss": 1.3253, + "theoretical_loss": 3.538236532525134, + "tokens_seen": 1530920960 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043326780341023074, + "loss": 1.3735, + "theoretical_loss": 3.538223337469607, + "tokens_seen": 1530986496 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004332517552657974, + "loss": 1.3221, + "theoretical_loss": 3.538210143137045, + "tokens_seen": 1531052032 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043323570712136414, + "loss": 1.3907, + "theoretical_loss": 3.5381969495273786, + "tokens_seen": 1531117568 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004332196589769308, + "loss": 1.2666, + "theoretical_loss": 3.538183756640538, + "tokens_seen": 1531183104 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043320361083249754, + "loss": 1.3686, + "theoretical_loss": 3.5381705644764505, + "tokens_seen": 1531248640 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004331875626880642, + "loss": 1.3164, + "theoretical_loss": 3.5381573730350473, + "tokens_seen": 1531314176 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043317151454363095, + "loss": 1.3652, + "theoretical_loss": 3.538144182316258, + "tokens_seen": 1531379712 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004331554663991976, + "loss": 1.2924, + "theoretical_loss": 3.538130992320011, + "tokens_seen": 1531445248 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043313941825476435, + "loss": 1.2772, + "theoretical_loss": 3.5381178030462364, + "tokens_seen": 1531510784 + }, + { + "epoch": 0.46, + "learning_rate": 0.000433123370110331, + "loss": 1.2965, + "theoretical_loss": 3.538104614494863, + "tokens_seen": 1531576320 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043310732196589775, + "loss": 1.2767, + "theoretical_loss": 3.5380914266658214, + "tokens_seen": 1531641856 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004330912738214644, + "loss": 1.3514, + "theoretical_loss": 3.538078239559041, + "tokens_seen": 1531707392 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043307522567703115, + "loss": 1.254, + "theoretical_loss": 3.5380650531744506, + "tokens_seen": 1531772928 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004330591775325978, + "loss": 1.3651, + "theoretical_loss": 3.5380518675119808, + "tokens_seen": 1531838464 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043304312938816455, + "loss": 1.2714, + "theoretical_loss": 3.5380386825715595, + "tokens_seen": 1531904000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004330270812437312, + "loss": 1.2929, + "theoretical_loss": 3.538025498353118, + "tokens_seen": 1531969536 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043301103309929795, + "loss": 1.2908, + "theoretical_loss": 3.5380123148565854, + "tokens_seen": 1532035072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004329949849548646, + "loss": 1.2462, + "theoretical_loss": 3.5379991320818904, + "tokens_seen": 1532100608 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043297893681043135, + "loss": 1.2493, + "theoretical_loss": 3.537985950028964, + "tokens_seen": 1532166144 + }, + { + "epoch": 0.46, + "learning_rate": 0.000432962888665998, + "loss": 1.3372, + "theoretical_loss": 3.5379727686977347, + "tokens_seen": 1532231680 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043294684052156475, + "loss": 1.3136, + "theoretical_loss": 3.5379595880881327, + "tokens_seen": 1532297216 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004329307923771314, + "loss": 1.2955, + "theoretical_loss": 3.537946408200087, + "tokens_seen": 1532362752 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043291474423269816, + "loss": 1.4018, + "theoretical_loss": 3.537933229033528, + "tokens_seen": 1532428288 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004328986960882648, + "loss": 1.3813, + "theoretical_loss": 3.537920050588385, + "tokens_seen": 1532493824 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043288264794383156, + "loss": 1.2688, + "theoretical_loss": 3.537906872864588, + "tokens_seen": 1532559360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004328665997993982, + "loss": 1.2696, + "theoretical_loss": 3.537893695862066, + "tokens_seen": 1532624896 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004328505516549649, + "loss": 1.3571, + "theoretical_loss": 3.5378805195807494, + "tokens_seen": 1532690432 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004328345035105316, + "loss": 1.2047, + "theoretical_loss": 3.537867344020567, + "tokens_seen": 1532755968 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004328184553660983, + "loss": 1.3288, + "theoretical_loss": 3.5378541691814496, + "tokens_seen": 1532821504 + }, + { + "epoch": 0.46, + "learning_rate": 0.000432802407221665, + "loss": 1.2466, + "theoretical_loss": 3.537840995063326, + "tokens_seen": 1532887040 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004327863590772317, + "loss": 1.3258, + "theoretical_loss": 3.5378278216661267, + "tokens_seen": 1532952576 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004327703109327984, + "loss": 1.3445, + "theoretical_loss": 3.537814648989781, + "tokens_seen": 1533018112 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004327542627883651, + "loss": 1.3186, + "theoretical_loss": 3.5378014770342183, + "tokens_seen": 1533083648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004327382146439318, + "loss": 1.3301, + "theoretical_loss": 3.537788305799369, + "tokens_seen": 1533149184 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004327221664994985, + "loss": 1.3567, + "theoretical_loss": 3.5377751352851625, + "tokens_seen": 1533214720 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043270611835506516, + "loss": 1.245, + "theoretical_loss": 3.537761965491529, + "tokens_seen": 1533280256 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004326900702106319, + "loss": 1.3002, + "theoretical_loss": 3.5377487964183976, + "tokens_seen": 1533345792 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043267402206619856, + "loss": 1.3425, + "theoretical_loss": 3.5377356280656977, + "tokens_seen": 1533411328 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004326579739217653, + "loss": 1.3162, + "theoretical_loss": 3.537722460433361, + "tokens_seen": 1533476864 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 514171, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.121462345123291, + "objective/train/theoretical_loss": 3.5377092935213152, + "objective/train/tokens_used": 1554002400, + "theoretical_loss": 3.5377092935213152, + "tokens_seen": 1533542400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043264192577733196, + "loss": 1.1911, + "theoretical_loss": 3.5377092935213152, + "tokens_seen": 1533542400 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004326258776328987, + "loss": 1.3494, + "theoretical_loss": 3.537696127329492, + "tokens_seen": 1533607936 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043260982948846536, + "loss": 1.2347, + "theoretical_loss": 3.5376829618578194, + "tokens_seen": 1533673472 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004325937813440321, + "loss": 1.2744, + "theoretical_loss": 3.5376697971062283, + "tokens_seen": 1533739008 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043257773319959876, + "loss": 1.3188, + "theoretical_loss": 3.537656633074649, + "tokens_seen": 1533804544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004325616850551655, + "loss": 1.2974, + "theoretical_loss": 3.53764346976301, + "tokens_seen": 1533870080 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043254563691073216, + "loss": 1.3428, + "theoretical_loss": 3.5376303071712423, + "tokens_seen": 1533935616 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004325295887662989, + "loss": 1.3152, + "theoretical_loss": 3.537617145299275, + "tokens_seen": 1534001152 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043251354062186556, + "loss": 1.3238, + "theoretical_loss": 3.5376039841470384, + "tokens_seen": 1534066688 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004324974924774323, + "loss": 1.3654, + "theoretical_loss": 3.5375908237144627, + "tokens_seen": 1534132224 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043248144433299896, + "loss": 1.3386, + "theoretical_loss": 3.5375776640014767, + "tokens_seen": 1534197760 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004324653961885657, + "loss": 1.346, + "theoretical_loss": 3.537564505008012, + "tokens_seen": 1534263296 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043244934804413237, + "loss": 1.2496, + "theoretical_loss": 3.537551346733997, + "tokens_seen": 1534328832 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004324332998996991, + "loss": 1.2911, + "theoretical_loss": 3.5375381891793625, + "tokens_seen": 1534394368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004324172517552659, + "loss": 1.2738, + "theoretical_loss": 3.537525032344038, + "tokens_seen": 1534459904 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004324012036108325, + "loss": 1.1993, + "theoretical_loss": 3.5375118762279545, + "tokens_seen": 1534525440 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004323851554663993, + "loss": 1.3667, + "theoretical_loss": 3.5374987208310404, + "tokens_seen": 1534590976 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004323691073219659, + "loss": 1.3241, + "theoretical_loss": 3.5374855661532263, + "tokens_seen": 1534656512 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004323530591775327, + "loss": 1.3171, + "theoretical_loss": 3.5374724121944423, + "tokens_seen": 1534722048 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004323370110330993, + "loss": 1.3691, + "theoretical_loss": 3.5374592589546188, + "tokens_seen": 1534787584 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004323209628886661, + "loss": 1.3629, + "theoretical_loss": 3.537446106433685, + "tokens_seen": 1534853120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004323049147442327, + "loss": 1.3614, + "theoretical_loss": 3.537432954631571, + "tokens_seen": 1534918656 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004322888665997995, + "loss": 1.2834, + "theoretical_loss": 3.537419803548208, + "tokens_seen": 1534984192 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043227281845536613, + "loss": 1.3042, + "theoretical_loss": 3.537406653183525, + "tokens_seen": 1535049728 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004322567703109329, + "loss": 1.3077, + "theoretical_loss": 3.5373935035374515, + "tokens_seen": 1535115264 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043224072216649953, + "loss": 1.2718, + "theoretical_loss": 3.5373803546099185, + "tokens_seen": 1535180800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043222467402206623, + "loss": 1.3983, + "theoretical_loss": 3.537367206400856, + "tokens_seen": 1535246336 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043220862587763293, + "loss": 1.2716, + "theoretical_loss": 3.5373540589101937, + "tokens_seen": 1535311872 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043219257773319963, + "loss": 1.325, + "theoretical_loss": 3.5373409121378625, + "tokens_seen": 1535377408 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043217652958876633, + "loss": 1.3545, + "theoretical_loss": 3.5373277660837914, + "tokens_seen": 1535442944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043216048144433303, + "loss": 1.2573, + "theoretical_loss": 3.537314620747911, + "tokens_seen": 1535508480 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043214443329989973, + "loss": 1.3863, + "theoretical_loss": 3.5373014761301516, + "tokens_seen": 1535574016 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043212838515546643, + "loss": 1.2602, + "theoretical_loss": 3.5372883322304425, + "tokens_seen": 1535639552 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043211233701103313, + "loss": 1.3346, + "theoretical_loss": 3.537275189048715, + "tokens_seen": 1535705088 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043209628886659983, + "loss": 1.3068, + "theoretical_loss": 3.5372620465848987, + "tokens_seen": 1535770624 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004320802407221665, + "loss": 1.36, + "theoretical_loss": 3.5372489048389237, + "tokens_seen": 1535836160 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043206419257773324, + "loss": 1.3981, + "theoretical_loss": 3.5372357638107204, + "tokens_seen": 1535901696 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004320481444332999, + "loss": 1.2923, + "theoretical_loss": 3.5372226235002184, + "tokens_seen": 1535967232 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043203209628886664, + "loss": 1.3062, + "theoretical_loss": 3.5372094839073487, + "tokens_seen": 1536032768 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004320160481444333, + "loss": 1.3202, + "theoretical_loss": 3.537196345032041, + "tokens_seen": 1536098304 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043200000000000004, + "loss": 1.4005, + "theoretical_loss": 3.537183206874225, + "tokens_seen": 1536163840 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004319839518555667, + "loss": 1.2083, + "theoretical_loss": 3.5371700694338317, + "tokens_seen": 1536229376 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043196790371113344, + "loss": 1.3219, + "theoretical_loss": 3.5371569327107917, + "tokens_seen": 1536294912 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004319518555667001, + "loss": 1.288, + "theoretical_loss": 3.537143796705034, + "tokens_seen": 1536360448 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043193580742226684, + "loss": 1.3064, + "theoretical_loss": 3.5371306614164895, + "tokens_seen": 1536425984 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004319197592778335, + "loss": 1.2217, + "theoretical_loss": 3.5371175268450887, + "tokens_seen": 1536491520 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043190371113340024, + "loss": 1.2825, + "theoretical_loss": 3.537104392990761, + "tokens_seen": 1536557056 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004318876629889669, + "loss": 1.2399, + "theoretical_loss": 3.537091259853438, + "tokens_seen": 1536622592 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043187161484453364, + "loss": 1.3843, + "theoretical_loss": 3.537078127433049, + "tokens_seen": 1536688128 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004318555667001003, + "loss": 1.3808, + "theoretical_loss": 3.5370649957295246, + "tokens_seen": 1536753664 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 515162, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2471009492874146, + "objective/train/theoretical_loss": 3.5370518647427946, + "objective/train/tokens_used": 1557279200, + "theoretical_loss": 3.5370518647427946, + "tokens_seen": 1536819200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043183951855566704, + "loss": 1.2905, + "theoretical_loss": 3.5370518647427946, + "tokens_seen": 1536819200 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004318234704112337, + "loss": 1.2587, + "theoretical_loss": 3.53703873447279, + "tokens_seen": 1536884736 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043180742226680045, + "loss": 1.231, + "theoretical_loss": 3.5370256049194406, + "tokens_seen": 1536950272 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004317913741223671, + "loss": 1.2213, + "theoretical_loss": 3.537012476082678, + "tokens_seen": 1537015808 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043177532597793385, + "loss": 1.3391, + "theoretical_loss": 3.53699934796243, + "tokens_seen": 1537081344 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004317592778335005, + "loss": 1.4384, + "theoretical_loss": 3.5369862205586298, + "tokens_seen": 1537146880 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043174322968906725, + "loss": 1.2812, + "theoretical_loss": 3.5369730938712056, + "tokens_seen": 1537212416 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004317271815446339, + "loss": 1.2214, + "theoretical_loss": 3.536959967900089, + "tokens_seen": 1537277952 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043171113340020065, + "loss": 1.1356, + "theoretical_loss": 3.53694684264521, + "tokens_seen": 1537343488 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004316950852557673, + "loss": 1.315, + "theoretical_loss": 3.5369337181064986, + "tokens_seen": 1537409024 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043167903711133405, + "loss": 1.2999, + "theoretical_loss": 3.5369205942838855, + "tokens_seen": 1537474560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004316629889669007, + "loss": 1.3237, + "theoretical_loss": 3.536907471177302, + "tokens_seen": 1537540096 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043164694082246745, + "loss": 1.2985, + "theoretical_loss": 3.536894348786676, + "tokens_seen": 1537605632 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004316308926780341, + "loss": 1.2835, + "theoretical_loss": 3.536881227111941, + "tokens_seen": 1537671168 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043161484453360085, + "loss": 1.3468, + "theoretical_loss": 3.536868106153026, + "tokens_seen": 1537736704 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004315987963891675, + "loss": 1.2031, + "theoretical_loss": 3.5368549859098612, + "tokens_seen": 1537802240 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043158274824473425, + "loss": 1.3878, + "theoretical_loss": 3.5368418663823773, + "tokens_seen": 1537867776 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004315667001003009, + "loss": 1.2901, + "theoretical_loss": 3.536828747570505, + "tokens_seen": 1537933312 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043155065195586766, + "loss": 1.4773, + "theoretical_loss": 3.536815629474174, + "tokens_seen": 1537998848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004315346038114343, + "loss": 1.2473, + "theoretical_loss": 3.536802512093316, + "tokens_seen": 1538064384 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043151855566700106, + "loss": 1.3166, + "theoretical_loss": 3.5367893954278604, + "tokens_seen": 1538129920 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004315025075225677, + "loss": 1.2257, + "theoretical_loss": 3.5367762794777384, + "tokens_seen": 1538195456 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043148645937813446, + "loss": 1.2275, + "theoretical_loss": 3.5367631642428803, + "tokens_seen": 1538260992 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004314704112337011, + "loss": 1.2817, + "theoretical_loss": 3.5367500497232163, + "tokens_seen": 1538326528 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004314543630892678, + "loss": 1.2398, + "theoretical_loss": 3.5367369359186775, + "tokens_seen": 1538392064 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004314383149448345, + "loss": 1.2908, + "theoretical_loss": 3.5367238228291944, + "tokens_seen": 1538457600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004314222668004012, + "loss": 1.3107, + "theoretical_loss": 3.536710710454697, + "tokens_seen": 1538523136 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004314062186559679, + "loss": 1.3775, + "theoretical_loss": 3.536697598795116, + "tokens_seen": 1538588672 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004313901705115346, + "loss": 1.1865, + "theoretical_loss": 3.5366844878503825, + "tokens_seen": 1538654208 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004313741223671013, + "loss": 1.3973, + "theoretical_loss": 3.5366713776204266, + "tokens_seen": 1538719744 + }, + { + "epoch": 0.47, + "learning_rate": 0.000431358074222668, + "loss": 1.4222, + "theoretical_loss": 3.5366582681051795, + "tokens_seen": 1538785280 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004313420260782347, + "loss": 1.3048, + "theoretical_loss": 3.5366451593045705, + "tokens_seen": 1538850816 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004313259779338014, + "loss": 1.3019, + "theoretical_loss": 3.5366320512185316, + "tokens_seen": 1538916352 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043130992978936806, + "loss": 1.4575, + "theoretical_loss": 3.536618943846993, + "tokens_seen": 1538981888 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004312938816449348, + "loss": 1.2935, + "theoretical_loss": 3.5366058371898847, + "tokens_seen": 1539047424 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043127783350050146, + "loss": 1.2728, + "theoretical_loss": 3.536592731247138, + "tokens_seen": 1539112960 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004312617853560682, + "loss": 1.3154, + "theoretical_loss": 3.5365796260186837, + "tokens_seen": 1539178496 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043124573721163486, + "loss": 1.3419, + "theoretical_loss": 3.5365665215044517, + "tokens_seen": 1539244032 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004312296890672016, + "loss": 1.2948, + "theoretical_loss": 3.536553417704374, + "tokens_seen": 1539309568 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043121364092276837, + "loss": 1.3514, + "theoretical_loss": 3.536540314618379, + "tokens_seen": 1539375104 + }, + { + "epoch": 0.47, + "learning_rate": 0.000431197592778335, + "loss": 1.2423, + "theoretical_loss": 3.5365272122463995, + "tokens_seen": 1539440640 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043118154463390177, + "loss": 1.3184, + "theoretical_loss": 3.5365141105883655, + "tokens_seen": 1539506176 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004311654964894684, + "loss": 1.2177, + "theoretical_loss": 3.5365010096442075, + "tokens_seen": 1539571712 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043114944834503517, + "loss": 1.3086, + "theoretical_loss": 3.5364879094138564, + "tokens_seen": 1539637248 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004311334002006018, + "loss": 1.3674, + "theoretical_loss": 3.5364748098972436, + "tokens_seen": 1539702784 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004311173520561686, + "loss": 1.3462, + "theoretical_loss": 3.536461711094298, + "tokens_seen": 1539768320 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004311013039117352, + "loss": 1.2895, + "theoretical_loss": 3.5364486130049526, + "tokens_seen": 1539833856 + }, + { + "epoch": 0.47, + "learning_rate": 0.000431085255767302, + "loss": 1.3519, + "theoretical_loss": 3.5364355156291367, + "tokens_seen": 1539899392 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004310692076228686, + "loss": 1.3387, + "theoretical_loss": 3.5364224189667812, + "tokens_seen": 1539964928 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004310531594784354, + "loss": 1.2045, + "theoretical_loss": 3.536409323017817, + "tokens_seen": 1540030464 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 516450, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1523512601852417, + "objective/train/theoretical_loss": 3.536396227782175, + "objective/train/tokens_used": 1560556000, + "theoretical_loss": 3.536396227782175, + "tokens_seen": 1540096000 + }, + { + "epoch": 0.47, + "learning_rate": 0.000431037111334002, + "loss": 1.2728, + "theoretical_loss": 3.536396227782175, + "tokens_seen": 1540096000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004310210631895688, + "loss": 1.2614, + "theoretical_loss": 3.5363831332597866, + "tokens_seen": 1540161536 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004310050150451354, + "loss": 1.3329, + "theoretical_loss": 3.5363700394505813, + "tokens_seen": 1540227072 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004309889669007022, + "loss": 1.3524, + "theoretical_loss": 3.5363569463544913, + "tokens_seen": 1540292608 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004309729187562688, + "loss": 1.398, + "theoretical_loss": 3.536343853971446, + "tokens_seen": 1540358144 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004309568706118356, + "loss": 1.345, + "theoretical_loss": 3.536330762301377, + "tokens_seen": 1540423680 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004309408224674022, + "loss": 1.3059, + "theoretical_loss": 3.536317671344216, + "tokens_seen": 1540489216 + }, + { + "epoch": 0.47, + "learning_rate": 0.000430924774322969, + "loss": 1.3272, + "theoretical_loss": 3.536304581099892, + "tokens_seen": 1540554752 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004309087261785356, + "loss": 1.2714, + "theoretical_loss": 3.5362914915683374, + "tokens_seen": 1540620288 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004308926780341024, + "loss": 1.2463, + "theoretical_loss": 3.536278402749482, + "tokens_seen": 1540685824 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043087662988966903, + "loss": 1.2279, + "theoretical_loss": 3.5362653146432574, + "tokens_seen": 1540751360 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004308605817452358, + "loss": 1.3438, + "theoretical_loss": 3.5362522272495944, + "tokens_seen": 1540816896 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043084453360080243, + "loss": 1.4957, + "theoretical_loss": 3.5362391405684237, + "tokens_seen": 1540882432 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004308284854563692, + "loss": 1.2327, + "theoretical_loss": 3.536226054599676, + "tokens_seen": 1540947968 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043081243731193583, + "loss": 1.5373, + "theoretical_loss": 3.536212969343283, + "tokens_seen": 1541013504 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043079638916750253, + "loss": 1.3178, + "theoretical_loss": 3.5361998847991747, + "tokens_seen": 1541079040 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043078034102306923, + "loss": 1.2739, + "theoretical_loss": 3.5361868009672826, + "tokens_seen": 1541144576 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043076429287863593, + "loss": 1.3036, + "theoretical_loss": 3.5361737178475376, + "tokens_seen": 1541210112 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043074824473420263, + "loss": 1.3397, + "theoretical_loss": 3.5361606354398702, + "tokens_seen": 1541275648 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043073219658976933, + "loss": 1.2485, + "theoretical_loss": 3.536147553744212, + "tokens_seen": 1541341184 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043071614844533603, + "loss": 1.3287, + "theoretical_loss": 3.5361344727604935, + "tokens_seen": 1541406720 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043070010030090274, + "loss": 1.4, + "theoretical_loss": 3.5361213924886465, + "tokens_seen": 1541472256 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004306840521564694, + "loss": 1.296, + "theoretical_loss": 3.536108312928601, + "tokens_seen": 1541537792 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043066800401203614, + "loss": 1.2847, + "theoretical_loss": 3.536095234080288, + "tokens_seen": 1541603328 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004306519558676028, + "loss": 1.2352, + "theoretical_loss": 3.5360821559436397, + "tokens_seen": 1541668864 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043063590772316954, + "loss": 1.3323, + "theoretical_loss": 3.5360690785185858, + "tokens_seen": 1541734400 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004306198595787362, + "loss": 1.2805, + "theoretical_loss": 3.5360560018050577, + "tokens_seen": 1541799936 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043060381143430294, + "loss": 1.3551, + "theoretical_loss": 3.5360429258029873, + "tokens_seen": 1541865472 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004305877632898696, + "loss": 1.3619, + "theoretical_loss": 3.5360298505123047, + "tokens_seen": 1541931008 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043057171514543634, + "loss": 1.3813, + "theoretical_loss": 3.536016775932941, + "tokens_seen": 1541996544 + }, + { + "epoch": 0.47, + "learning_rate": 0.000430555667001003, + "loss": 1.2507, + "theoretical_loss": 3.5360037020648276, + "tokens_seen": 1542062080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043053961885656974, + "loss": 1.3182, + "theoretical_loss": 3.5359906289078955, + "tokens_seen": 1542127616 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004305235707121364, + "loss": 1.2877, + "theoretical_loss": 3.5359775564620763, + "tokens_seen": 1542193152 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043050752256770314, + "loss": 1.3067, + "theoretical_loss": 3.5359644847273, + "tokens_seen": 1542258688 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004304914744232698, + "loss": 1.2747, + "theoretical_loss": 3.535951413703499, + "tokens_seen": 1542324224 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043047542627883654, + "loss": 1.3439, + "theoretical_loss": 3.535938343390603, + "tokens_seen": 1542389760 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004304593781344032, + "loss": 1.2986, + "theoretical_loss": 3.535925273788544, + "tokens_seen": 1542455296 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043044332998996995, + "loss": 1.3168, + "theoretical_loss": 3.5359122048972527, + "tokens_seen": 1542520832 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004304272818455366, + "loss": 1.3188, + "theoretical_loss": 3.5358991367166617, + "tokens_seen": 1542586368 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043041123370110335, + "loss": 1.292, + "theoretical_loss": 3.5358860692466996, + "tokens_seen": 1542651904 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043039518555667, + "loss": 1.3397, + "theoretical_loss": 3.5358730024872997, + "tokens_seen": 1542717440 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043037913741223675, + "loss": 1.2441, + "theoretical_loss": 3.535859936438393, + "tokens_seen": 1542782976 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004303630892678034, + "loss": 1.294, + "theoretical_loss": 3.5358468710999094, + "tokens_seen": 1542848512 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043034704112337015, + "loss": 1.3804, + "theoretical_loss": 3.535833806471781, + "tokens_seen": 1542914048 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004303309929789368, + "loss": 1.3572, + "theoretical_loss": 3.5358207425539385, + "tokens_seen": 1542979584 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043031494483450355, + "loss": 1.2731, + "theoretical_loss": 3.5358076793463145, + "tokens_seen": 1543045120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004302988966900702, + "loss": 1.3027, + "theoretical_loss": 3.5357946168488383, + "tokens_seen": 1543110656 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043028284854563695, + "loss": 1.3671, + "theoretical_loss": 3.535781555061442, + "tokens_seen": 1543176192 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004302668004012036, + "loss": 1.4963, + "theoretical_loss": 3.5357684939840572, + "tokens_seen": 1543241728 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043025075225677035, + "loss": 1.3042, + "theoretical_loss": 3.535755433616615, + "tokens_seen": 1543307264 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 517633, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2595796585083008, + "objective/train/theoretical_loss": 3.535742373959047, + "objective/train/tokens_used": 1563832800, + "theoretical_loss": 3.535742373959047, + "tokens_seen": 1543372800 + }, + { + "epoch": 0.47, + "learning_rate": 0.000430234704112337, + "loss": 1.2567, + "theoretical_loss": 3.535742373959047, + "tokens_seen": 1543372800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043021865596790375, + "loss": 1.3183, + "theoretical_loss": 3.5357293150112827, + "tokens_seen": 1543438336 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004302026078234704, + "loss": 1.3431, + "theoretical_loss": 3.535716256773256, + "tokens_seen": 1543503872 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043018655967903716, + "loss": 1.2666, + "theoretical_loss": 3.535703199244896, + "tokens_seen": 1543569408 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004301705115346038, + "loss": 1.2314, + "theoretical_loss": 3.5356901424261347, + "tokens_seen": 1543634944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043015446339017056, + "loss": 1.2401, + "theoretical_loss": 3.5356770863169036, + "tokens_seen": 1543700480 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004301384152457372, + "loss": 1.299, + "theoretical_loss": 3.5356640309171348, + "tokens_seen": 1543766016 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043012236710130396, + "loss": 1.2254, + "theoretical_loss": 3.5356509762267585, + "tokens_seen": 1543831552 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004301063189568706, + "loss": 1.2598, + "theoretical_loss": 3.535637922245706, + "tokens_seen": 1543897088 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043009027081243736, + "loss": 1.3127, + "theoretical_loss": 3.535624868973909, + "tokens_seen": 1543962624 + }, + { + "epoch": 0.47, + "learning_rate": 0.000430074222668004, + "loss": 1.411, + "theoretical_loss": 3.5356118164112997, + "tokens_seen": 1544028160 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043005817452357076, + "loss": 1.282, + "theoretical_loss": 3.5355987645578084, + "tokens_seen": 1544093696 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004300421263791374, + "loss": 1.3043, + "theoretical_loss": 3.535585713413366, + "tokens_seen": 1544159232 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004300260782347041, + "loss": 1.3206, + "theoretical_loss": 3.5355726629779056, + "tokens_seen": 1544224768 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043001003009027086, + "loss": 1.2718, + "theoretical_loss": 3.5355596132513565, + "tokens_seen": 1544290304 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004299939819458375, + "loss": 1.3313, + "theoretical_loss": 3.5355465642336523, + "tokens_seen": 1544355840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042997793380140426, + "loss": 1.245, + "theoretical_loss": 3.535533515924723, + "tokens_seen": 1544421376 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004299618856569709, + "loss": 1.3345, + "theoretical_loss": 3.5355204683245, + "tokens_seen": 1544486912 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042994583751253767, + "loss": 1.3975, + "theoretical_loss": 3.535507421432916, + "tokens_seen": 1544552448 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004299297893681043, + "loss": 1.3733, + "theoretical_loss": 3.535494375249901, + "tokens_seen": 1544617984 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042991374122367107, + "loss": 1.2574, + "theoretical_loss": 3.535481329775387, + "tokens_seen": 1544683520 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004298976930792377, + "loss": 1.3012, + "theoretical_loss": 3.5354682850093058, + "tokens_seen": 1544749056 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042988164493480447, + "loss": 1.2129, + "theoretical_loss": 3.5354552409515883, + "tokens_seen": 1544814592 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004298655967903711, + "loss": 1.2319, + "theoretical_loss": 3.5354421976021664, + "tokens_seen": 1544880128 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042984954864593787, + "loss": 1.2392, + "theoretical_loss": 3.535429154960971, + "tokens_seen": 1544945664 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004298335005015045, + "loss": 1.273, + "theoretical_loss": 3.5354161130279347, + "tokens_seen": 1545011200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042981745235707127, + "loss": 1.3392, + "theoretical_loss": 3.535403071802988, + "tokens_seen": 1545076736 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004298014042126379, + "loss": 1.2347, + "theoretical_loss": 3.535390031286063, + "tokens_seen": 1545142272 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042978535606820467, + "loss": 1.446, + "theoretical_loss": 3.535376991477091, + "tokens_seen": 1545207808 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004297693079237713, + "loss": 1.2679, + "theoretical_loss": 3.535363952376003, + "tokens_seen": 1545273344 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042975325977933807, + "loss": 1.3099, + "theoretical_loss": 3.5353509139827324, + "tokens_seen": 1545338880 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004297372116349047, + "loss": 1.3434, + "theoretical_loss": 3.5353378762972083, + "tokens_seen": 1545404416 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004297211634904715, + "loss": 1.2683, + "theoretical_loss": 3.535324839319364, + "tokens_seen": 1545469952 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004297051153460381, + "loss": 1.3098, + "theoretical_loss": 3.5353118030491304, + "tokens_seen": 1545535488 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004296890672016049, + "loss": 1.3316, + "theoretical_loss": 3.535298767486439, + "tokens_seen": 1545601024 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004296730190571715, + "loss": 1.2115, + "theoretical_loss": 3.5352857326312224, + "tokens_seen": 1545666560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004296569709127383, + "loss": 1.2262, + "theoretical_loss": 3.5352726984834106, + "tokens_seen": 1545732096 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004296409227683049, + "loss": 1.2519, + "theoretical_loss": 3.535259665042936, + "tokens_seen": 1545797632 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004296248746238717, + "loss": 1.4258, + "theoretical_loss": 3.5352466323097307, + "tokens_seen": 1545863168 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004296088264794383, + "loss": 1.3336, + "theoretical_loss": 3.535233600283726, + "tokens_seen": 1545928704 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004295927783350051, + "loss": 1.2113, + "theoretical_loss": 3.5352205689648537, + "tokens_seen": 1545994240 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004295767301905717, + "loss": 1.2761, + "theoretical_loss": 3.5352075383530446, + "tokens_seen": 1546059776 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004295606820461385, + "loss": 1.3123, + "theoretical_loss": 3.5351945084482312, + "tokens_seen": 1546125312 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004295446339017051, + "loss": 1.3042, + "theoretical_loss": 3.535181479250345, + "tokens_seen": 1546190848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004295285857572719, + "loss": 1.3349, + "theoretical_loss": 3.5351684507593175, + "tokens_seen": 1546256384 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042951253761283853, + "loss": 1.2305, + "theoretical_loss": 3.5351554229750812, + "tokens_seen": 1546321920 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004294964894684053, + "loss": 1.2464, + "theoretical_loss": 3.5351423958975667, + "tokens_seen": 1546387456 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042948044132397193, + "loss": 1.3007, + "theoretical_loss": 3.5351293695267056, + "tokens_seen": 1546452992 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004294643931795387, + "loss": 1.2223, + "theoretical_loss": 3.5351163438624313, + "tokens_seen": 1546518528 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042944834503510533, + "loss": 1.2771, + "theoretical_loss": 3.5351033189046737, + "tokens_seen": 1546584064 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 518472, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.236567497253418, + "objective/train/theoretical_loss": 3.535090294653365, + "objective/train/tokens_used": 1567109600, + "theoretical_loss": 3.535090294653365, + "tokens_seen": 1546649600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004294322968906721, + "loss": 1.2046, + "theoretical_loss": 3.535090294653365, + "tokens_seen": 1546649600 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042941624874623873, + "loss": 1.3078, + "theoretical_loss": 3.535077271108438, + "tokens_seen": 1546715136 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042940020060180543, + "loss": 1.2607, + "theoretical_loss": 3.5350642482698236, + "tokens_seen": 1546780672 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042938415245737213, + "loss": 1.2726, + "theoretical_loss": 3.535051226137453, + "tokens_seen": 1546846208 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042936810431293883, + "loss": 1.3819, + "theoretical_loss": 3.5350382047112587, + "tokens_seen": 1546911744 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042935205616850553, + "loss": 1.316, + "theoretical_loss": 3.535025183991173, + "tokens_seen": 1546977280 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042933600802407223, + "loss": 1.2514, + "theoretical_loss": 3.535012163977127, + "tokens_seen": 1547042816 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042931995987963894, + "loss": 1.212, + "theoretical_loss": 3.5349991446690523, + "tokens_seen": 1547108352 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042930391173520564, + "loss": 1.2569, + "theoretical_loss": 3.5349861260668813, + "tokens_seen": 1547173888 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042928786359077234, + "loss": 1.2488, + "theoretical_loss": 3.5349731081705453, + "tokens_seen": 1547239424 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042927181544633904, + "loss": 1.3093, + "theoretical_loss": 3.5349600909799763, + "tokens_seen": 1547304960 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004292557673019057, + "loss": 1.3115, + "theoretical_loss": 3.534947074495107, + "tokens_seen": 1547370496 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042923971915747244, + "loss": 1.2775, + "theoretical_loss": 3.5349340587158675, + "tokens_seen": 1547436032 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004292236710130391, + "loss": 1.3298, + "theoretical_loss": 3.5349210436421914, + "tokens_seen": 1547501568 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042920762286860584, + "loss": 1.2702, + "theoretical_loss": 3.5349080292740096, + "tokens_seen": 1547567104 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004291915747241725, + "loss": 1.3547, + "theoretical_loss": 3.534895015611254, + "tokens_seen": 1547632640 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042917552657973924, + "loss": 1.3815, + "theoretical_loss": 3.534882002653857, + "tokens_seen": 1547698176 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004291594784353059, + "loss": 1.2962, + "theoretical_loss": 3.5348689904017507, + "tokens_seen": 1547763712 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042914343029087264, + "loss": 1.2315, + "theoretical_loss": 3.534855978854866, + "tokens_seen": 1547829248 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004291273821464393, + "loss": 1.236, + "theoretical_loss": 3.5348429680131357, + "tokens_seen": 1547894784 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042911133400200604, + "loss": 1.3738, + "theoretical_loss": 3.5348299578764912, + "tokens_seen": 1547960320 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004290952858575727, + "loss": 1.3437, + "theoretical_loss": 3.534816948444864, + "tokens_seen": 1548025856 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042907923771313945, + "loss": 1.2632, + "theoretical_loss": 3.534803939718188, + "tokens_seen": 1548091392 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004290631895687061, + "loss": 1.3386, + "theoretical_loss": 3.534790931696393, + "tokens_seen": 1548156928 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042904714142427285, + "loss": 1.2929, + "theoretical_loss": 3.5347779243794113, + "tokens_seen": 1548222464 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004290310932798395, + "loss": 1.2576, + "theoretical_loss": 3.5347649177671765, + "tokens_seen": 1548288000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042901504513540625, + "loss": 1.2951, + "theoretical_loss": 3.534751911859619, + "tokens_seen": 1548353536 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004289989969909729, + "loss": 1.3275, + "theoretical_loss": 3.534738906656671, + "tokens_seen": 1548419072 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042898294884653965, + "loss": 1.4164, + "theoretical_loss": 3.5347259021582653, + "tokens_seen": 1548484608 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004289669007021063, + "loss": 1.2645, + "theoretical_loss": 3.534712898364333, + "tokens_seen": 1548550144 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042895085255767305, + "loss": 1.3163, + "theoretical_loss": 3.534699895274807, + "tokens_seen": 1548615680 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004289348044132397, + "loss": 1.1835, + "theoretical_loss": 3.5346868928896185, + "tokens_seen": 1548681216 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042891875626880645, + "loss": 1.3344, + "theoretical_loss": 3.5346738912087003, + "tokens_seen": 1548746752 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004289027081243731, + "loss": 1.2556, + "theoretical_loss": 3.5346608902319834, + "tokens_seen": 1548812288 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042888665997993985, + "loss": 1.2936, + "theoretical_loss": 3.534647889959401, + "tokens_seen": 1548877824 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004288706118355065, + "loss": 1.2922, + "theoretical_loss": 3.5346348903908846, + "tokens_seen": 1548943360 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042885456369107325, + "loss": 1.2536, + "theoretical_loss": 3.534621891526366, + "tokens_seen": 1549008896 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004288385155466399, + "loss": 1.3227, + "theoretical_loss": 3.534608893365778, + "tokens_seen": 1549074432 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042882246740220666, + "loss": 1.3009, + "theoretical_loss": 3.534595895909053, + "tokens_seen": 1549139968 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004288064192577734, + "loss": 1.2954, + "theoretical_loss": 3.534582899156122, + "tokens_seen": 1549205504 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042879037111334006, + "loss": 1.2277, + "theoretical_loss": 3.5345699031069175, + "tokens_seen": 1549271040 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042877432296890676, + "loss": 1.1816, + "theoretical_loss": 3.534556907761372, + "tokens_seen": 1549336576 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042875827482447346, + "loss": 1.3137, + "theoretical_loss": 3.5345439131194167, + "tokens_seen": 1549402112 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042874222668004016, + "loss": 1.2871, + "theoretical_loss": 3.534530919180985, + "tokens_seen": 1549467648 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042872617853560686, + "loss": 1.3037, + "theoretical_loss": 3.5345179259460084, + "tokens_seen": 1549533184 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042871013039117356, + "loss": 1.2883, + "theoretical_loss": 3.534504933414419, + "tokens_seen": 1549598720 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042869408224674026, + "loss": 1.3328, + "theoretical_loss": 3.5344919415861495, + "tokens_seen": 1549664256 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042867803410230696, + "loss": 1.2862, + "theoretical_loss": 3.5344789504611316, + "tokens_seen": 1549729792 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042866198595787366, + "loss": 1.2269, + "theoretical_loss": 3.5344659600392974, + "tokens_seen": 1549795328 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042864593781344036, + "loss": 1.3232, + "theoretical_loss": 3.5344529703205794, + "tokens_seen": 1549860864 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 519652, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1471920013427734, + "objective/train/theoretical_loss": 3.53443998130491, + "objective/train/tokens_used": 1570386400, + "theoretical_loss": 3.53443998130491, + "tokens_seen": 1549926400 + }, + { + "epoch": 0.47, + "learning_rate": 0.000428629889669007, + "loss": 1.2064, + "theoretical_loss": 3.53443998130491, + "tokens_seen": 1549926400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042861384152457376, + "loss": 1.1769, + "theoretical_loss": 3.5344269929922207, + "tokens_seen": 1549991936 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004285977933801404, + "loss": 1.4177, + "theoretical_loss": 3.5344140053824447, + "tokens_seen": 1550057472 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042858174523570716, + "loss": 1.261, + "theoretical_loss": 3.534401018475513, + "tokens_seen": 1550123008 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004285656970912738, + "loss": 1.2789, + "theoretical_loss": 3.5343880322713592, + "tokens_seen": 1550188544 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042854964894684057, + "loss": 1.3154, + "theoretical_loss": 3.534375046769915, + "tokens_seen": 1550254080 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004285336008024072, + "loss": 1.3079, + "theoretical_loss": 3.5343620619711125, + "tokens_seen": 1550319616 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042851755265797397, + "loss": 1.2498, + "theoretical_loss": 3.5343490778748836, + "tokens_seen": 1550385152 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004285015045135406, + "loss": 1.2783, + "theoretical_loss": 3.5343360944811613, + "tokens_seen": 1550450688 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042848545636910737, + "loss": 1.3021, + "theoretical_loss": 3.5343231117898783, + "tokens_seen": 1550516224 + }, + { + "epoch": 0.47, + "learning_rate": 0.000428469408224674, + "loss": 1.2649, + "theoretical_loss": 3.5343101298009656, + "tokens_seen": 1550581760 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042845336008024077, + "loss": 1.2425, + "theoretical_loss": 3.534297148514357, + "tokens_seen": 1550647296 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004284373119358074, + "loss": 1.2251, + "theoretical_loss": 3.5342841679299832, + "tokens_seen": 1550712832 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042842126379137417, + "loss": 1.3239, + "theoretical_loss": 3.5342711880477777, + "tokens_seen": 1550778368 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004284052156469408, + "loss": 1.3285, + "theoretical_loss": 3.534258208867672, + "tokens_seen": 1550843904 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042838916750250757, + "loss": 1.3055, + "theoretical_loss": 3.5342452303895997, + "tokens_seen": 1550909440 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004283731193580742, + "loss": 1.2488, + "theoretical_loss": 3.534232252613492, + "tokens_seen": 1550974976 + }, + { + "epoch": 0.47, + "learning_rate": 0.000428357071213641, + "loss": 1.3286, + "theoretical_loss": 3.5342192755392823, + "tokens_seen": 1551040512 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004283410230692076, + "loss": 1.3346, + "theoretical_loss": 3.534206299166902, + "tokens_seen": 1551106048 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004283249749247744, + "loss": 1.2439, + "theoretical_loss": 3.5341933234962837, + "tokens_seen": 1551171584 + }, + { + "epoch": 0.47, + "learning_rate": 0.000428308926780341, + "loss": 1.3698, + "theoretical_loss": 3.53418034852736, + "tokens_seen": 1551237120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004282928786359078, + "loss": 1.2807, + "theoretical_loss": 3.534167374260064, + "tokens_seen": 1551302656 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004282768304914744, + "loss": 1.2501, + "theoretical_loss": 3.5341544006943266, + "tokens_seen": 1551368192 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004282607823470412, + "loss": 1.4101, + "theoretical_loss": 3.5341414278300816, + "tokens_seen": 1551433728 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004282447342026078, + "loss": 1.3603, + "theoretical_loss": 3.5341284556672603, + "tokens_seen": 1551499264 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004282286860581746, + "loss": 1.2832, + "theoretical_loss": 3.5341154842057962, + "tokens_seen": 1551564800 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004282126379137412, + "loss": 1.3274, + "theoretical_loss": 3.5341025134456214, + "tokens_seen": 1551630336 + }, + { + "epoch": 0.47, + "learning_rate": 0.000428196589769308, + "loss": 1.2631, + "theoretical_loss": 3.534089543386668, + "tokens_seen": 1551695872 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004281805416248746, + "loss": 1.3186, + "theoretical_loss": 3.5340765740288687, + "tokens_seen": 1551761408 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004281644934804414, + "loss": 1.3629, + "theoretical_loss": 3.5340636053721566, + "tokens_seen": 1551826944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042814844533600803, + "loss": 1.1716, + "theoretical_loss": 3.534050637416463, + "tokens_seen": 1551892480 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004281323971915748, + "loss": 1.2977, + "theoretical_loss": 3.5340376701617213, + "tokens_seen": 1551958016 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042811634904714143, + "loss": 1.3734, + "theoretical_loss": 3.534024703607864, + "tokens_seen": 1552023552 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004281003009027082, + "loss": 1.2986, + "theoretical_loss": 3.5340117377548226, + "tokens_seen": 1552089088 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042808425275827483, + "loss": 1.2099, + "theoretical_loss": 3.533998772602531, + "tokens_seen": 1552154624 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004280682046138416, + "loss": 1.3073, + "theoretical_loss": 3.533985808150921, + "tokens_seen": 1552220160 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042805215646940823, + "loss": 1.2613, + "theoretical_loss": 3.5339728443999254, + "tokens_seen": 1552285696 + }, + { + "epoch": 0.47, + "learning_rate": 0.000428036108324975, + "loss": 1.3253, + "theoretical_loss": 3.5339598813494764, + "tokens_seen": 1552351232 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042802006018054163, + "loss": 1.3133, + "theoretical_loss": 3.533946918999507, + "tokens_seen": 1552416768 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042800401203610833, + "loss": 1.2276, + "theoretical_loss": 3.5339339573499498, + "tokens_seen": 1552482304 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042798796389167503, + "loss": 1.2819, + "theoretical_loss": 3.5339209964007368, + "tokens_seen": 1552547840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042797191574724173, + "loss": 1.3577, + "theoretical_loss": 3.533908036151801, + "tokens_seen": 1552613376 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042795586760280844, + "loss": 1.2769, + "theoretical_loss": 3.5338950766030752, + "tokens_seen": 1552678912 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042793981945837514, + "loss": 1.3473, + "theoretical_loss": 3.533882117754492, + "tokens_seen": 1552744448 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042792377131394184, + "loss": 1.2914, + "theoretical_loss": 3.5338691596059837, + "tokens_seen": 1552809984 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042790772316950854, + "loss": 1.3106, + "theoretical_loss": 3.533856202157483, + "tokens_seen": 1552875520 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042789167502507524, + "loss": 1.254, + "theoretical_loss": 3.5338432454089226, + "tokens_seen": 1552941056 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042787562688064194, + "loss": 1.2376, + "theoretical_loss": 3.5338302893602354, + "tokens_seen": 1553006592 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004278595787362086, + "loss": 1.2185, + "theoretical_loss": 3.5338173340113537, + "tokens_seen": 1553072128 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042784353059177534, + "loss": 1.3752, + "theoretical_loss": 3.533804379362211, + "tokens_seen": 1553137664 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 520524, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.32770836353302, + "objective/train/theoretical_loss": 3.5337914254127383, + "objective/train/tokens_used": 1573663200, + "theoretical_loss": 3.5337914254127383, + "tokens_seen": 1553203200 + }, + { + "epoch": 0.47, + "learning_rate": 0.000427827482447342, + "loss": 1.3689, + "theoretical_loss": 3.5337914254127383, + "tokens_seen": 1553203200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042781143430290874, + "loss": 1.2887, + "theoretical_loss": 3.5337784721628696, + "tokens_seen": 1553268736 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004277953861584754, + "loss": 1.3047, + "theoretical_loss": 3.5337655196125373, + "tokens_seen": 1553334272 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042777933801404214, + "loss": 1.3148, + "theoretical_loss": 3.533752567761674, + "tokens_seen": 1553399808 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004277632898696088, + "loss": 1.3365, + "theoretical_loss": 3.533739616610213, + "tokens_seen": 1553465344 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042774724172517554, + "loss": 1.3883, + "theoretical_loss": 3.5337266661580857, + "tokens_seen": 1553530880 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004277311935807422, + "loss": 1.3837, + "theoretical_loss": 3.5337137164052264, + "tokens_seen": 1553596416 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042771514543630894, + "loss": 1.2435, + "theoretical_loss": 3.5337007673515672, + "tokens_seen": 1553661952 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004276990972918756, + "loss": 1.2839, + "theoretical_loss": 3.5336878189970404, + "tokens_seen": 1553727488 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042768304914744235, + "loss": 1.4372, + "theoretical_loss": 3.5336748713415793, + "tokens_seen": 1553793024 + }, + { + "epoch": 0.47, + "learning_rate": 0.000427667001003009, + "loss": 1.3402, + "theoretical_loss": 3.533661924385117, + "tokens_seen": 1553858560 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042765095285857575, + "loss": 1.3643, + "theoretical_loss": 3.5336489781275855, + "tokens_seen": 1553924096 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004276349047141424, + "loss": 1.2836, + "theoretical_loss": 3.5336360325689173, + "tokens_seen": 1553989632 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042761885656970915, + "loss": 1.2715, + "theoretical_loss": 3.5336230877090467, + "tokens_seen": 1554055168 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004276028084252759, + "loss": 1.2843, + "theoretical_loss": 3.533610143547905, + "tokens_seen": 1554120704 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042758676028084255, + "loss": 1.2929, + "theoretical_loss": 3.533597200085426, + "tokens_seen": 1554186240 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004275707121364093, + "loss": 1.2736, + "theoretical_loss": 3.533584257321542, + "tokens_seen": 1554251776 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042755466399197595, + "loss": 1.3559, + "theoretical_loss": 3.5335713152561863, + "tokens_seen": 1554317312 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004275386158475427, + "loss": 1.2667, + "theoretical_loss": 3.5335583738892913, + "tokens_seen": 1554382848 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042752256770310935, + "loss": 1.2962, + "theoretical_loss": 3.53354543322079, + "tokens_seen": 1554448384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004275065195586761, + "loss": 1.2324, + "theoretical_loss": 3.5335324932506156, + "tokens_seen": 1554513920 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042749047141424275, + "loss": 1.1982, + "theoretical_loss": 3.5335195539787003, + "tokens_seen": 1554579456 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004274744232698095, + "loss": 1.3705, + "theoretical_loss": 3.5335066154049777, + "tokens_seen": 1554644992 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042745837512537616, + "loss": 1.3359, + "theoretical_loss": 3.53349367752938, + "tokens_seen": 1554710528 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004274423269809429, + "loss": 1.3287, + "theoretical_loss": 3.5334807403518407, + "tokens_seen": 1554776064 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042742627883650956, + "loss": 1.3146, + "theoretical_loss": 3.533467803872292, + "tokens_seen": 1554841600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004274102306920763, + "loss": 1.2455, + "theoretical_loss": 3.533454868090668, + "tokens_seen": 1554907136 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042739418254764296, + "loss": 1.2268, + "theoretical_loss": 3.533441933006901, + "tokens_seen": 1554972672 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042737813440320966, + "loss": 1.3357, + "theoretical_loss": 3.5334289986209235, + "tokens_seen": 1555038208 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042736208625877636, + "loss": 1.2692, + "theoretical_loss": 3.533416064932669, + "tokens_seen": 1555103744 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042734603811434306, + "loss": 1.3651, + "theoretical_loss": 3.53340313194207, + "tokens_seen": 1555169280 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042732998996990976, + "loss": 1.3358, + "theoretical_loss": 3.5333901996490598, + "tokens_seen": 1555234816 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042731394182547646, + "loss": 1.3439, + "theoretical_loss": 3.533377268053572, + "tokens_seen": 1555300352 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042729789368104316, + "loss": 1.3057, + "theoretical_loss": 3.533364337155538, + "tokens_seen": 1555365888 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042728184553660986, + "loss": 1.2499, + "theoretical_loss": 3.5333514069548926, + "tokens_seen": 1555431424 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042726579739217656, + "loss": 1.2716, + "theoretical_loss": 3.5333384774515677, + "tokens_seen": 1555496960 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042724974924774326, + "loss": 1.3315, + "theoretical_loss": 3.533325548645496, + "tokens_seen": 1555562496 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004272337011033099, + "loss": 1.3259, + "theoretical_loss": 3.533312620536612, + "tokens_seen": 1555628032 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042721765295887666, + "loss": 1.2772, + "theoretical_loss": 3.533299693124847, + "tokens_seen": 1555693568 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004272016048144433, + "loss": 1.3266, + "theoretical_loss": 3.5332867664101353, + "tokens_seen": 1555759104 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042718555667001007, + "loss": 1.3005, + "theoretical_loss": 3.5332738403924093, + "tokens_seen": 1555824640 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004271695085255767, + "loss": 1.3301, + "theoretical_loss": 3.533260915071603, + "tokens_seen": 1555890176 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042715346038114347, + "loss": 1.2411, + "theoretical_loss": 3.5332479904476477, + "tokens_seen": 1555955712 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004271374122367101, + "loss": 1.3077, + "theoretical_loss": 3.533235066520478, + "tokens_seen": 1556021248 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042712136409227687, + "loss": 1.2867, + "theoretical_loss": 3.5332221432900264, + "tokens_seen": 1556086784 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004271053159478435, + "loss": 1.4103, + "theoretical_loss": 3.5332092207562265, + "tokens_seen": 1556152320 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042708926780341027, + "loss": 1.3604, + "theoretical_loss": 3.5331962989190107, + "tokens_seen": 1556217856 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004270732196589769, + "loss": 1.3854, + "theoretical_loss": 3.5331833777783124, + "tokens_seen": 1556283392 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042705717151454367, + "loss": 1.2996, + "theoretical_loss": 3.533170457334065, + "tokens_seen": 1556348928 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004270411233701103, + "loss": 1.2552, + "theoretical_loss": 3.5331575375862014, + "tokens_seen": 1556414464 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 521491, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3240457773208618, + "objective/train/theoretical_loss": 3.5331446185346547, + "objective/train/tokens_used": 1576940000, + "theoretical_loss": 3.5331446185346547, + "tokens_seen": 1556480000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042702507522567707, + "loss": 1.2918, + "theoretical_loss": 3.5331446185346547, + "tokens_seen": 1556480000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004270090270812437, + "loss": 1.2906, + "theoretical_loss": 3.533131700179358, + "tokens_seen": 1556545536 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004269929789368105, + "loss": 1.2264, + "theoretical_loss": 3.533118782520245, + "tokens_seen": 1556611072 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004269769307923771, + "loss": 1.3078, + "theoretical_loss": 3.533105865557248, + "tokens_seen": 1556676608 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004269608826479439, + "loss": 1.2517, + "theoretical_loss": 3.5330929492903005, + "tokens_seen": 1556742144 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004269448345035105, + "loss": 1.4029, + "theoretical_loss": 3.5330800337193367, + "tokens_seen": 1556807680 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004269287863590773, + "loss": 1.3347, + "theoretical_loss": 3.533067118844288, + "tokens_seen": 1556873216 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004269127382146439, + "loss": 1.4732, + "theoretical_loss": 3.5330542046650892, + "tokens_seen": 1556938752 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004268966900702107, + "loss": 1.3391, + "theoretical_loss": 3.5330412911816724, + "tokens_seen": 1557004288 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004268806419257773, + "loss": 1.3001, + "theoretical_loss": 3.5330283783939715, + "tokens_seen": 1557069824 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004268645937813441, + "loss": 1.3208, + "theoretical_loss": 3.5330154663019195, + "tokens_seen": 1557135360 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004268485456369107, + "loss": 1.3041, + "theoretical_loss": 3.53300255490545, + "tokens_seen": 1557200896 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004268324974924775, + "loss": 1.2906, + "theoretical_loss": 3.5329896442044957, + "tokens_seen": 1557266432 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004268164493480441, + "loss": 1.3582, + "theoretical_loss": 3.5329767341989897, + "tokens_seen": 1557331968 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004268004012036109, + "loss": 1.3053, + "theoretical_loss": 3.532963824888866, + "tokens_seen": 1557397504 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042678435305917753, + "loss": 1.2449, + "theoretical_loss": 3.532950916274058, + "tokens_seen": 1557463040 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004267683049147443, + "loss": 1.3824, + "theoretical_loss": 3.532938008354498, + "tokens_seen": 1557528576 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042675225677031093, + "loss": 1.3398, + "theoretical_loss": 3.53292510113012, + "tokens_seen": 1557594112 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004267362086258777, + "loss": 1.3104, + "theoretical_loss": 3.5329121946008573, + "tokens_seen": 1557659648 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042672016048144433, + "loss": 1.2383, + "theoretical_loss": 3.532899288766643, + "tokens_seen": 1557725184 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004267041123370111, + "loss": 1.2848, + "theoretical_loss": 3.5328863836274103, + "tokens_seen": 1557790720 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042668806419257773, + "loss": 1.3185, + "theoretical_loss": 3.532873479183093, + "tokens_seen": 1557856256 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004266720160481445, + "loss": 1.203, + "theoretical_loss": 3.5328605754336246, + "tokens_seen": 1557921792 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042665596790371113, + "loss": 1.3224, + "theoretical_loss": 3.5328476723789377, + "tokens_seen": 1557987328 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004266399197592779, + "loss": 1.2763, + "theoretical_loss": 3.532834770018966, + "tokens_seen": 1558052864 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042662387161484453, + "loss": 1.4266, + "theoretical_loss": 3.5328218683536425, + "tokens_seen": 1558118400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042660782347041123, + "loss": 1.3181, + "theoretical_loss": 3.532808967382902, + "tokens_seen": 1558183936 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042659177532597794, + "loss": 1.2314, + "theoretical_loss": 3.532796067106676, + "tokens_seen": 1558249472 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042657572718154464, + "loss": 1.3438, + "theoretical_loss": 3.532783167524899, + "tokens_seen": 1558315008 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042655967903711134, + "loss": 1.3015, + "theoretical_loss": 3.5327702686375044, + "tokens_seen": 1558380544 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042654363089267804, + "loss": 1.3237, + "theoretical_loss": 3.5327573704444255, + "tokens_seen": 1558446080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042652758274824474, + "loss": 1.3257, + "theoretical_loss": 3.5327444729455957, + "tokens_seen": 1558511616 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042651153460381144, + "loss": 1.2575, + "theoretical_loss": 3.5327315761409483, + "tokens_seen": 1558577152 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042649548645937814, + "loss": 1.3749, + "theoretical_loss": 3.532718680030417, + "tokens_seen": 1558642688 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042647943831494484, + "loss": 1.3282, + "theoretical_loss": 3.5327057846139347, + "tokens_seen": 1558708224 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004264633901705115, + "loss": 1.2417, + "theoretical_loss": 3.532692889891435, + "tokens_seen": 1558773760 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042644734202607824, + "loss": 1.3793, + "theoretical_loss": 3.5326799958628525, + "tokens_seen": 1558839296 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004264312938816449, + "loss": 1.2934, + "theoretical_loss": 3.5326671025281193, + "tokens_seen": 1558904832 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042641524573721164, + "loss": 1.2813, + "theoretical_loss": 3.53265420988717, + "tokens_seen": 1558970368 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004263991975927784, + "loss": 1.333, + "theoretical_loss": 3.5326413179399374, + "tokens_seen": 1559035904 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042638314944834504, + "loss": 1.1715, + "theoretical_loss": 3.5326284266863546, + "tokens_seen": 1559101440 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004263671013039118, + "loss": 1.2605, + "theoretical_loss": 3.5326155361263565, + "tokens_seen": 1559166976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042635105315947844, + "loss": 1.3299, + "theoretical_loss": 3.532602646259875, + "tokens_seen": 1559232512 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004263350050150452, + "loss": 1.3748, + "theoretical_loss": 3.532589757086845, + "tokens_seen": 1559298048 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042631895687061185, + "loss": 1.373, + "theoretical_loss": 3.532576868607199, + "tokens_seen": 1559363584 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004263029087261786, + "loss": 1.1886, + "theoretical_loss": 3.5325639808208718, + "tokens_seen": 1559429120 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042628686058174525, + "loss": 1.2684, + "theoretical_loss": 3.532551093727796, + "tokens_seen": 1559494656 + }, + { + "epoch": 0.47, + "learning_rate": 0.000426270812437312, + "loss": 1.2345, + "theoretical_loss": 3.5325382073279052, + "tokens_seen": 1559560192 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042625476429287865, + "loss": 1.2726, + "theoretical_loss": 3.5325253216211334, + "tokens_seen": 1559625728 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004262387161484454, + "loss": 1.3175, + "theoretical_loss": 3.532512436607414, + "tokens_seen": 1559691264 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 522672, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3013179302215576, + "objective/train/theoretical_loss": 3.5324995522866804, + "objective/train/tokens_used": 1580216800, + "theoretical_loss": 3.5324995522866804, + "tokens_seen": 1559756800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042622266800401205, + "loss": 1.2442, + "theoretical_loss": 3.5324995522866804, + "tokens_seen": 1559756800 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004262066198595788, + "loss": 1.3117, + "theoretical_loss": 3.5324866686588665, + "tokens_seen": 1559822336 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042619057171514545, + "loss": 1.3347, + "theoretical_loss": 3.532473785723906, + "tokens_seen": 1559887872 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004261745235707122, + "loss": 1.2703, + "theoretical_loss": 3.5324609034817325, + "tokens_seen": 1559953408 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042615847542627885, + "loss": 1.2673, + "theoretical_loss": 3.532448021932279, + "tokens_seen": 1560018944 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004261424272818456, + "loss": 1.1718, + "theoretical_loss": 3.53243514107548, + "tokens_seen": 1560084480 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042612637913741225, + "loss": 1.2249, + "theoretical_loss": 3.532422260911269, + "tokens_seen": 1560150016 + }, + { + "epoch": 0.47, + "learning_rate": 0.000426110330992979, + "loss": 1.3452, + "theoretical_loss": 3.532409381439579, + "tokens_seen": 1560215552 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042609428284854565, + "loss": 1.4643, + "theoretical_loss": 3.5323965026603448, + "tokens_seen": 1560281088 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004260782347041124, + "loss": 1.3409, + "theoretical_loss": 3.532383624573499, + "tokens_seen": 1560346624 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042606218655967906, + "loss": 1.2447, + "theoretical_loss": 3.532370747178976, + "tokens_seen": 1560412160 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004260461384152458, + "loss": 1.2747, + "theoretical_loss": 3.5323578704767096, + "tokens_seen": 1560477696 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042603009027081246, + "loss": 1.2598, + "theoretical_loss": 3.5323449944666327, + "tokens_seen": 1560543232 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004260140421263792, + "loss": 1.2654, + "theoretical_loss": 3.5323321191486796, + "tokens_seen": 1560608768 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042599799398194586, + "loss": 1.2982, + "theoretical_loss": 3.532319244522784, + "tokens_seen": 1560674304 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042598194583751256, + "loss": 1.3188, + "theoretical_loss": 3.5323063705888798, + "tokens_seen": 1560739840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042596589769307926, + "loss": 1.4243, + "theoretical_loss": 3.5322934973469002, + "tokens_seen": 1560805376 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042594984954864596, + "loss": 1.4556, + "theoretical_loss": 3.5322806247967793, + "tokens_seen": 1560870912 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042593380140421266, + "loss": 1.3688, + "theoretical_loss": 3.532267752938451, + "tokens_seen": 1560936448 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042591775325977936, + "loss": 1.2601, + "theoretical_loss": 3.5322548817718493, + "tokens_seen": 1561001984 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042590170511534606, + "loss": 1.3325, + "theoretical_loss": 3.532242011296907, + "tokens_seen": 1561067520 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042588565697091276, + "loss": 1.2179, + "theoretical_loss": 3.532229141513559, + "tokens_seen": 1561133056 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042586960882647946, + "loss": 1.3719, + "theoretical_loss": 3.5322162724217385, + "tokens_seen": 1561198592 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042585356068204616, + "loss": 1.3862, + "theoretical_loss": 3.5322034040213794, + "tokens_seen": 1561264128 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004258375125376128, + "loss": 1.216, + "theoretical_loss": 3.5321905363124158, + "tokens_seen": 1561329664 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042582146439317957, + "loss": 1.3263, + "theoretical_loss": 3.5321776692947813, + "tokens_seen": 1561395200 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004258054162487462, + "loss": 1.3129, + "theoretical_loss": 3.5321648029684094, + "tokens_seen": 1561460736 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042578936810431297, + "loss": 1.3676, + "theoretical_loss": 3.532151937333235, + "tokens_seen": 1561526272 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004257733199598796, + "loss": 1.4322, + "theoretical_loss": 3.5321390723891906, + "tokens_seen": 1561591808 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042575727181544637, + "loss": 1.4022, + "theoretical_loss": 3.532126208136211, + "tokens_seen": 1561657344 + }, + { + "epoch": 0.47, + "learning_rate": 0.000425741223671013, + "loss": 1.3105, + "theoretical_loss": 3.5321133445742294, + "tokens_seen": 1561722880 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042572517552657977, + "loss": 1.2388, + "theoretical_loss": 3.532100481703181, + "tokens_seen": 1561788416 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004257091273821464, + "loss": 1.364, + "theoretical_loss": 3.532087619522998, + "tokens_seen": 1561853952 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042569307923771317, + "loss": 1.3245, + "theoretical_loss": 3.5320747580336156, + "tokens_seen": 1561919488 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004256770310932798, + "loss": 1.2795, + "theoretical_loss": 3.532061897234967, + "tokens_seen": 1561985024 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042566098294884657, + "loss": 1.3224, + "theoretical_loss": 3.5320490371269866, + "tokens_seen": 1562050560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004256449348044132, + "loss": 1.3358, + "theoretical_loss": 3.532036177709608, + "tokens_seen": 1562116096 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042562888665998, + "loss": 1.2707, + "theoretical_loss": 3.5320233189827652, + "tokens_seen": 1562181632 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004256128385155466, + "loss": 1.4499, + "theoretical_loss": 3.5320104609463923, + "tokens_seen": 1562247168 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004255967903711134, + "loss": 1.3217, + "theoretical_loss": 3.5319976036004235, + "tokens_seen": 1562312704 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042558074222668, + "loss": 1.2959, + "theoretical_loss": 3.531984746944792, + "tokens_seen": 1562378240 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004255646940822468, + "loss": 1.3239, + "theoretical_loss": 3.531971890979432, + "tokens_seen": 1562443776 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004255486459378134, + "loss": 1.2033, + "theoretical_loss": 3.531959035704278, + "tokens_seen": 1562509312 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004255325977933802, + "loss": 1.2849, + "theoretical_loss": 3.531946181119263, + "tokens_seen": 1562574848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004255165496489468, + "loss": 1.3024, + "theoretical_loss": 3.5319333272243227, + "tokens_seen": 1562640384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004255005015045136, + "loss": 1.3079, + "theoretical_loss": 3.5319204740193895, + "tokens_seen": 1562705920 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004254844533600802, + "loss": 1.3268, + "theoretical_loss": 3.5319076215043985, + "tokens_seen": 1562771456 + }, + { + "epoch": 0.47, + "learning_rate": 0.000425468405215647, + "loss": 1.2136, + "theoretical_loss": 3.531894769679283, + "tokens_seen": 1562836992 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004254523570712136, + "loss": 1.1638, + "theoretical_loss": 3.531881918543977, + "tokens_seen": 1562902528 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004254363089267804, + "loss": 1.3084, + "theoretical_loss": 3.5318690680984153, + "tokens_seen": 1562968064 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 523864, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2828794717788696, + "objective/train/theoretical_loss": 3.5318562183425315, + "objective/train/tokens_used": 1583493600, + "theoretical_loss": 3.5318562183425315, + "tokens_seen": 1563033600 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042542026078234703, + "loss": 1.3545, + "theoretical_loss": 3.5318562183425315, + "tokens_seen": 1563033600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004254042126379138, + "loss": 1.2865, + "theoretical_loss": 3.5318433692762596, + "tokens_seen": 1563099136 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042538816449348043, + "loss": 1.2645, + "theoretical_loss": 3.5318305208995335, + "tokens_seen": 1563164672 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004253721163490472, + "loss": 1.2767, + "theoretical_loss": 3.5318176732122883, + "tokens_seen": 1563230208 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042535606820461383, + "loss": 1.3011, + "theoretical_loss": 3.531804826214457, + "tokens_seen": 1563295744 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004253400200601806, + "loss": 1.2318, + "theoretical_loss": 3.531791979905974, + "tokens_seen": 1563361280 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042532397191574723, + "loss": 1.2611, + "theoretical_loss": 3.531779134286773, + "tokens_seen": 1563426816 + }, + { + "epoch": 0.47, + "learning_rate": 0.000425307923771314, + "loss": 1.2714, + "theoretical_loss": 3.5317662893567894, + "tokens_seen": 1563492352 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042529187562688063, + "loss": 1.3322, + "theoretical_loss": 3.5317534451159567, + "tokens_seen": 1563557888 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004252758274824474, + "loss": 1.3916, + "theoretical_loss": 3.5317406015642083, + "tokens_seen": 1563623424 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042525977933801403, + "loss": 1.2943, + "theoretical_loss": 3.5317277587014786, + "tokens_seen": 1563688960 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004252437311935808, + "loss": 1.4274, + "theoretical_loss": 3.5317149165277026, + "tokens_seen": 1563754496 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042522768304914743, + "loss": 1.3747, + "theoretical_loss": 3.5317020750428143, + "tokens_seen": 1563820032 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042521163490471414, + "loss": 1.2321, + "theoretical_loss": 3.531689234246747, + "tokens_seen": 1563885568 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004251955867602809, + "loss": 1.2014, + "theoretical_loss": 3.531676394139436, + "tokens_seen": 1563951104 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042517953861584754, + "loss": 1.265, + "theoretical_loss": 3.531663554720814, + "tokens_seen": 1564016640 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004251634904714143, + "loss": 1.2123, + "theoretical_loss": 3.531650715990817, + "tokens_seen": 1564082176 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042514744232698094, + "loss": 1.35, + "theoretical_loss": 3.531637877949378, + "tokens_seen": 1564147712 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004251313941825477, + "loss": 1.2883, + "theoretical_loss": 3.5316250405964316, + "tokens_seen": 1564213248 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042511534603811434, + "loss": 1.2547, + "theoretical_loss": 3.531612203931912, + "tokens_seen": 1564278784 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004250992978936811, + "loss": 1.2528, + "theoretical_loss": 3.531599367955754, + "tokens_seen": 1564344320 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042508324974924774, + "loss": 1.4118, + "theoretical_loss": 3.5315865326678906, + "tokens_seen": 1564409856 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004250672016048145, + "loss": 1.4012, + "theoretical_loss": 3.531573698068257, + "tokens_seen": 1564475392 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042505115346038114, + "loss": 1.2908, + "theoretical_loss": 3.531560864156787, + "tokens_seen": 1564540928 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004250351053159479, + "loss": 1.3058, + "theoretical_loss": 3.5315480309334157, + "tokens_seen": 1564606464 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042501905717151454, + "loss": 1.313, + "theoretical_loss": 3.5315351983980765, + "tokens_seen": 1564672000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004250030090270813, + "loss": 1.3286, + "theoretical_loss": 3.5315223665507034, + "tokens_seen": 1564737536 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042498696088264794, + "loss": 1.2794, + "theoretical_loss": 3.531509535391232, + "tokens_seen": 1564803072 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004249709127382147, + "loss": 1.342, + "theoretical_loss": 3.531496704919596, + "tokens_seen": 1564868608 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042495486459378135, + "loss": 1.301, + "theoretical_loss": 3.5314838751357294, + "tokens_seen": 1564934144 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004249388164493481, + "loss": 1.2521, + "theoretical_loss": 3.5314710460395666, + "tokens_seen": 1564999680 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042492276830491475, + "loss": 1.2948, + "theoretical_loss": 3.5314582176310427, + "tokens_seen": 1565065216 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004249067201604815, + "loss": 1.3802, + "theoretical_loss": 3.5314453899100906, + "tokens_seen": 1565130752 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042489067201604815, + "loss": 1.2886, + "theoretical_loss": 3.5314325628766463, + "tokens_seen": 1565196288 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004248746238716149, + "loss": 1.3553, + "theoretical_loss": 3.5314197365306432, + "tokens_seen": 1565261824 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042485857572718155, + "loss": 1.3395, + "theoretical_loss": 3.5314069108720156, + "tokens_seen": 1565327360 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004248425275827483, + "loss": 1.2585, + "theoretical_loss": 3.5313940859006983, + "tokens_seen": 1565392896 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042482647943831495, + "loss": 1.309, + "theoretical_loss": 3.531381261616626, + "tokens_seen": 1565458432 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004248104312938817, + "loss": 1.282, + "theoretical_loss": 3.5313684380197317, + "tokens_seen": 1565523968 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042479438314944835, + "loss": 1.3635, + "theoretical_loss": 3.5313556151099514, + "tokens_seen": 1565589504 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004247783350050151, + "loss": 1.3329, + "theoretical_loss": 3.5313427928872185, + "tokens_seen": 1565655040 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042476228686058175, + "loss": 1.3536, + "theoretical_loss": 3.5313299713514685, + "tokens_seen": 1565720576 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004247462387161485, + "loss": 1.2668, + "theoretical_loss": 3.531317150502635, + "tokens_seen": 1565786112 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042473019057171515, + "loss": 1.2595, + "theoretical_loss": 3.531304330340652, + "tokens_seen": 1565851648 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004247141424272819, + "loss": 1.3059, + "theoretical_loss": 3.5312915108654552, + "tokens_seen": 1565917184 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042469809428284856, + "loss": 1.3389, + "theoretical_loss": 3.5312786920769783, + "tokens_seen": 1565982720 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004246820461384153, + "loss": 1.3542, + "theoretical_loss": 3.5312658739751557, + "tokens_seen": 1566048256 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042466599799398196, + "loss": 1.3757, + "theoretical_loss": 3.5312530565599225, + "tokens_seen": 1566113792 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004246499498495487, + "loss": 1.2794, + "theoretical_loss": 3.531240239831212, + "tokens_seen": 1566179328 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042463390170511536, + "loss": 1.2843, + "theoretical_loss": 3.5312274237889607, + "tokens_seen": 1566244864 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 524888, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4155240058898926, + "objective/train/theoretical_loss": 3.531214608433101, + "objective/train/tokens_used": 1586770400, + "theoretical_loss": 3.531214608433101, + "tokens_seen": 1566310400 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004246178535606821, + "loss": 1.3172, + "theoretical_loss": 3.531214608433101, + "tokens_seen": 1566310400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042460180541624876, + "loss": 1.2831, + "theoretical_loss": 3.531201793763569, + "tokens_seen": 1566375936 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004245857572718155, + "loss": 1.3451, + "theoretical_loss": 3.531188979780298, + "tokens_seen": 1566441472 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042456970912738216, + "loss": 1.2661, + "theoretical_loss": 3.531176166483223, + "tokens_seen": 1566507008 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042455366098294886, + "loss": 1.3355, + "theoretical_loss": 3.5311633538722793, + "tokens_seen": 1566572544 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042453761283851556, + "loss": 1.3159, + "theoretical_loss": 3.5311505419474005, + "tokens_seen": 1566638080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042452156469408226, + "loss": 1.2903, + "theoretical_loss": 3.5311377307085214, + "tokens_seen": 1566703616 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042450551654964896, + "loss": 1.1181, + "theoretical_loss": 3.5311249201555768, + "tokens_seen": 1566769152 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042448946840521566, + "loss": 1.2878, + "theoretical_loss": 3.5311121102885012, + "tokens_seen": 1566834688 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042447342026078236, + "loss": 1.37, + "theoretical_loss": 3.531099301107229, + "tokens_seen": 1566900224 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042445737211634907, + "loss": 1.3192, + "theoretical_loss": 3.5310864926116947, + "tokens_seen": 1566965760 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004244413239719157, + "loss": 1.2813, + "theoretical_loss": 3.5310736848018336, + "tokens_seen": 1567031296 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042442527582748247, + "loss": 1.2733, + "theoretical_loss": 3.5310608776775796, + "tokens_seen": 1567096832 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004244092276830491, + "loss": 1.2068, + "theoretical_loss": 3.531048071238868, + "tokens_seen": 1567162368 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042439317953861587, + "loss": 1.3221, + "theoretical_loss": 3.5310352654856327, + "tokens_seen": 1567227904 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004243771313941825, + "loss": 1.2494, + "theoretical_loss": 3.5310224604178084, + "tokens_seen": 1567293440 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042436108324974927, + "loss": 1.4405, + "theoretical_loss": 3.5310096560353306, + "tokens_seen": 1567358976 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004243450351053159, + "loss": 1.2756, + "theoretical_loss": 3.530996852338133, + "tokens_seen": 1567424512 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042432898696088267, + "loss": 1.299, + "theoretical_loss": 3.530984049326151, + "tokens_seen": 1567490048 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004243129388164493, + "loss": 1.2588, + "theoretical_loss": 3.530971246999319, + "tokens_seen": 1567555584 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042429689067201607, + "loss": 1.2105, + "theoretical_loss": 3.5309584453575713, + "tokens_seen": 1567621120 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004242808425275827, + "loss": 1.2394, + "theoretical_loss": 3.530945644400843, + "tokens_seen": 1567686656 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042426479438314947, + "loss": 1.3263, + "theoretical_loss": 3.530932844129069, + "tokens_seen": 1567752192 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004242487462387161, + "loss": 1.3468, + "theoretical_loss": 3.5309200445421833, + "tokens_seen": 1567817728 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004242326980942829, + "loss": 1.3322, + "theoretical_loss": 3.5309072456401216, + "tokens_seen": 1567883264 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004242166499498495, + "loss": 1.1928, + "theoretical_loss": 3.530894447422818, + "tokens_seen": 1567948800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004242006018054163, + "loss": 1.3663, + "theoretical_loss": 3.5308816498902074, + "tokens_seen": 1568014336 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004241845536609829, + "loss": 1.2905, + "theoretical_loss": 3.5308688530422243, + "tokens_seen": 1568079872 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004241685055165497, + "loss": 1.3351, + "theoretical_loss": 3.5308560568788048, + "tokens_seen": 1568145408 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004241524573721163, + "loss": 1.2627, + "theoretical_loss": 3.5308432613998812, + "tokens_seen": 1568210944 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004241364092276831, + "loss": 1.4045, + "theoretical_loss": 3.5308304666053907, + "tokens_seen": 1568276480 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004241203610832497, + "loss": 1.3299, + "theoretical_loss": 3.5308176724952665, + "tokens_seen": 1568342016 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004241043129388165, + "loss": 1.3265, + "theoretical_loss": 3.5308048790694437, + "tokens_seen": 1568407552 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004240882647943831, + "loss": 1.1841, + "theoretical_loss": 3.5307920863278577, + "tokens_seen": 1568473088 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004240722166499499, + "loss": 1.2533, + "theoretical_loss": 3.5307792942704435, + "tokens_seen": 1568538624 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004240561685055165, + "loss": 1.4414, + "theoretical_loss": 3.5307665028971345, + "tokens_seen": 1568604160 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004240401203610833, + "loss": 1.2273, + "theoretical_loss": 3.530753712207867, + "tokens_seen": 1568669696 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042402407221664993, + "loss": 1.2805, + "theoretical_loss": 3.5307409222025754, + "tokens_seen": 1568735232 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004240080240722167, + "loss": 1.2144, + "theoretical_loss": 3.530728132881194, + "tokens_seen": 1568800768 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042399197592778344, + "loss": 1.3788, + "theoretical_loss": 3.5307153442436583, + "tokens_seen": 1568866304 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004239759277833501, + "loss": 1.254, + "theoretical_loss": 3.530702556289903, + "tokens_seen": 1568931840 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042395987963891684, + "loss": 1.3153, + "theoretical_loss": 3.530689769019863, + "tokens_seen": 1568997376 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004239438314944835, + "loss": 1.4061, + "theoretical_loss": 3.530676982433473, + "tokens_seen": 1569062912 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004239277833500502, + "loss": 1.3393, + "theoretical_loss": 3.530664196530668, + "tokens_seen": 1569128448 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004239117352056169, + "loss": 1.3325, + "theoretical_loss": 3.5306514113113834, + "tokens_seen": 1569193984 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004238956870611836, + "loss": 1.3402, + "theoretical_loss": 3.5306386267755534, + "tokens_seen": 1569259520 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004238796389167503, + "loss": 1.2606, + "theoretical_loss": 3.530625842923113, + "tokens_seen": 1569325056 + }, + { + "epoch": 0.48, + "learning_rate": 0.000423863590772317, + "loss": 1.1971, + "theoretical_loss": 3.5306130597539975, + "tokens_seen": 1569390592 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004238475426278837, + "loss": 1.435, + "theoretical_loss": 3.5306002772681415, + "tokens_seen": 1569456128 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004238314944834504, + "loss": 1.3341, + "theoretical_loss": 3.530587495465481, + "tokens_seen": 1569521664 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 526097, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2334527969360352, + "objective/train/theoretical_loss": 3.5305747143459487, + "objective/train/tokens_used": 1590047200, + "theoretical_loss": 3.5305747143459487, + "tokens_seen": 1569587200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004238154463390171, + "loss": 1.2915, + "theoretical_loss": 3.5305747143459487, + "tokens_seen": 1569587200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004237993981945838, + "loss": 1.262, + "theoretical_loss": 3.530561933909482, + "tokens_seen": 1569652736 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042378335005015044, + "loss": 1.3578, + "theoretical_loss": 3.530549154156014, + "tokens_seen": 1569718272 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004237673019057172, + "loss": 1.2499, + "theoretical_loss": 3.5305363750854806, + "tokens_seen": 1569783808 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042375125376128384, + "loss": 1.3204, + "theoretical_loss": 3.5305235966978175, + "tokens_seen": 1569849344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004237352056168506, + "loss": 1.2857, + "theoretical_loss": 3.5305108189929584, + "tokens_seen": 1569914880 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042371915747241724, + "loss": 1.2165, + "theoretical_loss": 3.530498041970839, + "tokens_seen": 1569980416 + }, + { + "epoch": 0.48, + "learning_rate": 0.000423703109327984, + "loss": 1.2779, + "theoretical_loss": 3.530485265631394, + "tokens_seen": 1570045952 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042368706118355064, + "loss": 1.3183, + "theoretical_loss": 3.5304724899745583, + "tokens_seen": 1570111488 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004236710130391174, + "loss": 1.272, + "theoretical_loss": 3.5304597150002675, + "tokens_seen": 1570177024 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042365496489468404, + "loss": 1.2711, + "theoretical_loss": 3.5304469407084564, + "tokens_seen": 1570242560 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004236389167502508, + "loss": 1.3257, + "theoretical_loss": 3.5304341670990596, + "tokens_seen": 1570308096 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042362286860581744, + "loss": 1.274, + "theoretical_loss": 3.5304213941720133, + "tokens_seen": 1570373632 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004236068204613842, + "loss": 1.361, + "theoretical_loss": 3.5304086219272515, + "tokens_seen": 1570439168 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042359077231695085, + "loss": 1.2634, + "theoretical_loss": 3.5303958503647097, + "tokens_seen": 1570504704 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004235747241725176, + "loss": 1.3646, + "theoretical_loss": 3.5303830794843227, + "tokens_seen": 1570570240 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042355867602808425, + "loss": 1.2974, + "theoretical_loss": 3.530370309286026, + "tokens_seen": 1570635776 + }, + { + "epoch": 0.48, + "learning_rate": 0.000423542627883651, + "loss": 1.2879, + "theoretical_loss": 3.5303575397697546, + "tokens_seen": 1570701312 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042352657973921765, + "loss": 1.3977, + "theoretical_loss": 3.530344770935444, + "tokens_seen": 1570766848 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004235105315947844, + "loss": 1.2272, + "theoretical_loss": 3.5303320027830285, + "tokens_seen": 1570832384 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042349448345035105, + "loss": 1.2839, + "theoretical_loss": 3.5303192353124437, + "tokens_seen": 1570897920 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004234784353059178, + "loss": 1.3116, + "theoretical_loss": 3.5303064685236243, + "tokens_seen": 1570963456 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042346238716148445, + "loss": 1.3329, + "theoretical_loss": 3.5302937024165058, + "tokens_seen": 1571028992 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004234463390170512, + "loss": 1.426, + "theoretical_loss": 3.530280936991024, + "tokens_seen": 1571094528 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042343029087261785, + "loss": 1.4062, + "theoretical_loss": 3.5302681722471134, + "tokens_seen": 1571160064 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004234142427281846, + "loss": 1.2104, + "theoretical_loss": 3.530255408184709, + "tokens_seen": 1571225600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042339819458375125, + "loss": 1.3025, + "theoretical_loss": 3.5302426448037467, + "tokens_seen": 1571291136 + }, + { + "epoch": 0.48, + "learning_rate": 0.000423382146439318, + "loss": 1.3241, + "theoretical_loss": 3.5302298821041607, + "tokens_seen": 1571356672 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042336609829488465, + "loss": 1.3077, + "theoretical_loss": 3.5302171200858874, + "tokens_seen": 1571422208 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004233500501504514, + "loss": 1.3636, + "theoretical_loss": 3.5302043587488607, + "tokens_seen": 1571487744 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042333400200601806, + "loss": 1.2708, + "theoretical_loss": 3.530191598093017, + "tokens_seen": 1571553280 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004233179538615848, + "loss": 1.4087, + "theoretical_loss": 3.5301788381182906, + "tokens_seen": 1571618816 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042330190571715146, + "loss": 1.2877, + "theoretical_loss": 3.5301660788246174, + "tokens_seen": 1571684352 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004232858575727182, + "loss": 1.419, + "theoretical_loss": 3.530153320211933, + "tokens_seen": 1571749888 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042326980942828486, + "loss": 1.2931, + "theoretical_loss": 3.530140562280171, + "tokens_seen": 1571815424 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004232537612838516, + "loss": 1.2951, + "theoretical_loss": 3.530127805029269, + "tokens_seen": 1571880960 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042323771313941826, + "loss": 1.2114, + "theoretical_loss": 3.53011504845916, + "tokens_seen": 1571946496 + }, + { + "epoch": 0.48, + "learning_rate": 0.000423221664994985, + "loss": 1.2458, + "theoretical_loss": 3.530102292569781, + "tokens_seen": 1572012032 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042320561685055166, + "loss": 1.2399, + "theoretical_loss": 3.5300895373610666, + "tokens_seen": 1572077568 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004231895687061184, + "loss": 1.3002, + "theoretical_loss": 3.5300767828329516, + "tokens_seen": 1572143104 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042317352056168506, + "loss": 1.268, + "theoretical_loss": 3.5300640289853726, + "tokens_seen": 1572208640 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042315747241725176, + "loss": 1.4284, + "theoretical_loss": 3.5300512758182636, + "tokens_seen": 1572274176 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042314142427281846, + "loss": 1.3799, + "theoretical_loss": 3.530038523331561, + "tokens_seen": 1572339712 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042312537612838516, + "loss": 1.2414, + "theoretical_loss": 3.5300257715251995, + "tokens_seen": 1572405248 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042310932798395186, + "loss": 1.3376, + "theoretical_loss": 3.5300130203991147, + "tokens_seen": 1572470784 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042309327983951856, + "loss": 1.4151, + "theoretical_loss": 3.5300002699532413, + "tokens_seen": 1572536320 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042307723169508527, + "loss": 1.3123, + "theoretical_loss": 3.5299875201875155, + "tokens_seen": 1572601856 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042306118355065197, + "loss": 1.3207, + "theoretical_loss": 3.529974771101873, + "tokens_seen": 1572667392 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042304513540621867, + "loss": 1.3893, + "theoretical_loss": 3.529962022696248, + "tokens_seen": 1572732928 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042302908726178537, + "loss": 1.2399, + "theoretical_loss": 3.5299492749705768, + "tokens_seen": 1572798464 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 527076, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3733115196228027, + "objective/train/theoretical_loss": 3.5299365279247943, + "objective/train/tokens_used": 1593324000, + "theoretical_loss": 3.5299365279247943, + "tokens_seen": 1572864000 + }, + { + "epoch": 0.48, + "learning_rate": 0.000423013039117352, + "loss": 1.3588, + "theoretical_loss": 3.5299365279247943, + "tokens_seen": 1572864000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042299699097291877, + "loss": 1.2936, + "theoretical_loss": 3.529923781558836, + "tokens_seen": 1572929536 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004229809428284854, + "loss": 1.3017, + "theoretical_loss": 3.5299110358726375, + "tokens_seen": 1572995072 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042296489468405217, + "loss": 1.4578, + "theoretical_loss": 3.5298982908661345, + "tokens_seen": 1573060608 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004229488465396188, + "loss": 1.2904, + "theoretical_loss": 3.529885546539262, + "tokens_seen": 1573126144 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042293279839518557, + "loss": 1.2763, + "theoretical_loss": 3.5298728028919557, + "tokens_seen": 1573191680 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004229167502507522, + "loss": 1.3212, + "theoretical_loss": 3.5298600599241503, + "tokens_seen": 1573257216 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042290070210631897, + "loss": 1.2949, + "theoretical_loss": 3.5298473176357823, + "tokens_seen": 1573322752 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004228846539618856, + "loss": 1.3182, + "theoretical_loss": 3.5298345760267864, + "tokens_seen": 1573388288 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004228686058174524, + "loss": 1.2748, + "theoretical_loss": 3.529821835097099, + "tokens_seen": 1573453824 + }, + { + "epoch": 0.48, + "learning_rate": 0.000422852557673019, + "loss": 1.2204, + "theoretical_loss": 3.5298090948466543, + "tokens_seen": 1573519360 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004228365095285858, + "loss": 1.347, + "theoretical_loss": 3.5297963552753893, + "tokens_seen": 1573584896 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042282046138415253, + "loss": 1.2267, + "theoretical_loss": 3.5297836163832383, + "tokens_seen": 1573650432 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004228044132397192, + "loss": 1.275, + "theoretical_loss": 3.529770878170137, + "tokens_seen": 1573715968 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042278836509528593, + "loss": 1.2845, + "theoretical_loss": 3.529758140636022, + "tokens_seen": 1573781504 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004227723169508526, + "loss": 1.2745, + "theoretical_loss": 3.5297454037808276, + "tokens_seen": 1573847040 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042275626880641933, + "loss": 1.2914, + "theoretical_loss": 3.529732667604489, + "tokens_seen": 1573912576 + }, + { + "epoch": 0.48, + "learning_rate": 0.000422740220661986, + "loss": 1.2849, + "theoretical_loss": 3.5297199321069437, + "tokens_seen": 1573978112 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042272417251755273, + "loss": 1.2802, + "theoretical_loss": 3.5297071972881255, + "tokens_seen": 1574043648 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004227081243731194, + "loss": 1.2531, + "theoretical_loss": 3.5296944631479703, + "tokens_seen": 1574109184 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042269207622868614, + "loss": 1.2373, + "theoretical_loss": 3.5296817296864145, + "tokens_seen": 1574174720 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004226760280842528, + "loss": 1.3647, + "theoretical_loss": 3.5296689969033928, + "tokens_seen": 1574240256 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042265997993981954, + "loss": 1.2165, + "theoretical_loss": 3.5296562647988408, + "tokens_seen": 1574305792 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004226439317953862, + "loss": 1.3965, + "theoretical_loss": 3.5296435333726945, + "tokens_seen": 1574371328 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042262788365095294, + "loss": 1.2745, + "theoretical_loss": 3.52963080262489, + "tokens_seen": 1574436864 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004226118355065196, + "loss": 1.3204, + "theoretical_loss": 3.529618072555362, + "tokens_seen": 1574502400 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042259578736208634, + "loss": 1.3651, + "theoretical_loss": 3.5296053431640466, + "tokens_seen": 1574567936 + }, + { + "epoch": 0.48, + "learning_rate": 0.000422579739217653, + "loss": 1.2688, + "theoretical_loss": 3.529592614450879, + "tokens_seen": 1574633472 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042256369107321974, + "loss": 1.302, + "theoretical_loss": 3.5295798864157955, + "tokens_seen": 1574699008 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004225476429287864, + "loss": 1.2396, + "theoretical_loss": 3.529567159058731, + "tokens_seen": 1574764544 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004225315947843531, + "loss": 1.2741, + "theoretical_loss": 3.529554432379622, + "tokens_seen": 1574830080 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004225155466399198, + "loss": 1.3089, + "theoretical_loss": 3.529541706378404, + "tokens_seen": 1574895616 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004224994984954865, + "loss": 1.2228, + "theoretical_loss": 3.5295289810550123, + "tokens_seen": 1574961152 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004224834503510532, + "loss": 1.2698, + "theoretical_loss": 3.5295162564093827, + "tokens_seen": 1575026688 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004224674022066199, + "loss": 1.2509, + "theoretical_loss": 3.5295035324414505, + "tokens_seen": 1575092224 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004224513540621866, + "loss": 1.2494, + "theoretical_loss": 3.5294908091511523, + "tokens_seen": 1575157760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004224353059177533, + "loss": 1.3338, + "theoretical_loss": 3.5294780865384237, + "tokens_seen": 1575223296 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042241925777332, + "loss": 1.3868, + "theoretical_loss": 3.5294653646031993, + "tokens_seen": 1575288832 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004224032096288867, + "loss": 1.305, + "theoretical_loss": 3.529452643345416, + "tokens_seen": 1575354368 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042238716148445334, + "loss": 1.2398, + "theoretical_loss": 3.5294399227650097, + "tokens_seen": 1575419904 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004223711133400201, + "loss": 1.3808, + "theoretical_loss": 3.5294272028619154, + "tokens_seen": 1575485440 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042235506519558674, + "loss": 1.3114, + "theoretical_loss": 3.529414483636069, + "tokens_seen": 1575550976 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004223390170511535, + "loss": 1.2883, + "theoretical_loss": 3.5294017650874063, + "tokens_seen": 1575616512 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042232296890672014, + "loss": 1.3171, + "theoretical_loss": 3.5293890472158633, + "tokens_seen": 1575682048 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004223069207622869, + "loss": 1.3048, + "theoretical_loss": 3.5293763300213756, + "tokens_seen": 1575747584 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042229087261785354, + "loss": 1.2641, + "theoretical_loss": 3.5293636135038793, + "tokens_seen": 1575813120 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004222748244734203, + "loss": 1.2291, + "theoretical_loss": 3.52935089766331, + "tokens_seen": 1575878656 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042225877632898694, + "loss": 1.3643, + "theoretical_loss": 3.529338182499603, + "tokens_seen": 1575944192 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004222427281845537, + "loss": 1.2591, + "theoretical_loss": 3.5293254680126953, + "tokens_seen": 1576009728 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042222668004012035, + "loss": 1.3474, + "theoretical_loss": 3.5293127542025213, + "tokens_seen": 1576075264 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 528401, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.289647102355957, + "objective/train/theoretical_loss": 3.529300041069018, + "objective/train/tokens_used": 1596600800, + "theoretical_loss": 3.529300041069018, + "tokens_seen": 1576140800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004222106318956871, + "loss": 1.3653, + "theoretical_loss": 3.529300041069018, + "tokens_seen": 1576140800 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042219458375125375, + "loss": 1.2696, + "theoretical_loss": 3.5292873286121207, + "tokens_seen": 1576206336 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004221785356068205, + "loss": 1.3086, + "theoretical_loss": 3.5292746168317657, + "tokens_seen": 1576271872 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042216248746238715, + "loss": 1.2939, + "theoretical_loss": 3.5292619057278887, + "tokens_seen": 1576337408 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004221464393179539, + "loss": 1.3272, + "theoretical_loss": 3.529249195300425, + "tokens_seen": 1576402944 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042213039117352055, + "loss": 1.3479, + "theoretical_loss": 3.529236485549311, + "tokens_seen": 1576468480 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004221143430290873, + "loss": 1.3035, + "theoretical_loss": 3.529223776474483, + "tokens_seen": 1576534016 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042209829488465395, + "loss": 1.2684, + "theoretical_loss": 3.5292110680758757, + "tokens_seen": 1576599552 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004220822467402207, + "loss": 1.2601, + "theoretical_loss": 3.5291983603534263, + "tokens_seen": 1576665088 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042206619859578735, + "loss": 1.274, + "theoretical_loss": 3.52918565330707, + "tokens_seen": 1576730624 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004220501504513541, + "loss": 1.2333, + "theoretical_loss": 3.529172946936743, + "tokens_seen": 1576796160 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042203410230692075, + "loss": 1.3183, + "theoretical_loss": 3.5291602412423813, + "tokens_seen": 1576861696 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004220180541624875, + "loss": 1.3126, + "theoretical_loss": 3.5291475362239204, + "tokens_seen": 1576927232 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042200200601805415, + "loss": 1.3129, + "theoretical_loss": 3.5291348318812963, + "tokens_seen": 1576992768 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004219859578736209, + "loss": 1.1857, + "theoretical_loss": 3.5291221282144454, + "tokens_seen": 1577058304 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042196990972918756, + "loss": 1.3501, + "theoretical_loss": 3.5291094252233037, + "tokens_seen": 1577123840 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004219538615847543, + "loss": 1.3067, + "theoretical_loss": 3.5290967229078074, + "tokens_seen": 1577189376 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042193781344032096, + "loss": 1.274, + "theoretical_loss": 3.5290840212678916, + "tokens_seen": 1577254912 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004219217652958877, + "loss": 1.2622, + "theoretical_loss": 3.5290713203034922, + "tokens_seen": 1577320448 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042190571715145436, + "loss": 1.1857, + "theoretical_loss": 3.5290586200145464, + "tokens_seen": 1577385984 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004218896690070211, + "loss": 1.2918, + "theoretical_loss": 3.5290459204009896, + "tokens_seen": 1577451520 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042187362086258776, + "loss": 1.2541, + "theoretical_loss": 3.5290332214627576, + "tokens_seen": 1577517056 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004218575727181545, + "loss": 1.3235, + "theoretical_loss": 3.5290205231997867, + "tokens_seen": 1577582592 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042184152457372116, + "loss": 1.3156, + "theoretical_loss": 3.5290078256120134, + "tokens_seen": 1577648128 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004218254764292879, + "loss": 1.3458, + "theoretical_loss": 3.5289951286993726, + "tokens_seen": 1577713664 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042180942828485456, + "loss": 1.3338, + "theoretical_loss": 3.528982432461801, + "tokens_seen": 1577779200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004217933801404213, + "loss": 1.3097, + "theoretical_loss": 3.528969736899235, + "tokens_seen": 1577844736 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042177733199598796, + "loss": 1.2871, + "theoretical_loss": 3.5289570420116103, + "tokens_seen": 1577910272 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042176128385155466, + "loss": 1.3468, + "theoretical_loss": 3.5289443477988627, + "tokens_seen": 1577975808 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042174523570712136, + "loss": 1.3533, + "theoretical_loss": 3.528931654260929, + "tokens_seen": 1578041344 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042172918756268806, + "loss": 1.2591, + "theoretical_loss": 3.5289189613977454, + "tokens_seen": 1578106880 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042171313941825477, + "loss": 1.2751, + "theoretical_loss": 3.528906269209247, + "tokens_seen": 1578172416 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042169709127382147, + "loss": 1.2951, + "theoretical_loss": 3.5288935776953703, + "tokens_seen": 1578237952 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042168104312938817, + "loss": 1.3469, + "theoretical_loss": 3.5288808868560517, + "tokens_seen": 1578303488 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042166499498495487, + "loss": 1.3073, + "theoretical_loss": 3.5288681966912274, + "tokens_seen": 1578369024 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042164894684052157, + "loss": 1.2532, + "theoretical_loss": 3.5288555072008334, + "tokens_seen": 1578434560 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042163289869608827, + "loss": 1.2275, + "theoretical_loss": 3.528842818384806, + "tokens_seen": 1578500096 + }, + { + "epoch": 0.48, + "learning_rate": 0.000421616850551655, + "loss": 1.3028, + "theoretical_loss": 3.5288301302430813, + "tokens_seen": 1578565632 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042160080240722167, + "loss": 1.2366, + "theoretical_loss": 3.528817442775595, + "tokens_seen": 1578631168 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004215847542627884, + "loss": 1.297, + "theoretical_loss": 3.528804755982284, + "tokens_seen": 1578696704 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042156870611835507, + "loss": 1.1885, + "theoretical_loss": 3.5287920698630835, + "tokens_seen": 1578762240 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004215526579739218, + "loss": 1.3305, + "theoretical_loss": 3.5287793844179314, + "tokens_seen": 1578827776 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042153660982948847, + "loss": 1.3154, + "theoretical_loss": 3.528766699646762, + "tokens_seen": 1578893312 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042152056168505523, + "loss": 1.2767, + "theoretical_loss": 3.528754015549513, + "tokens_seen": 1578958848 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004215045135406219, + "loss": 1.2104, + "theoretical_loss": 3.5287413321261196, + "tokens_seen": 1579024384 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042148846539618863, + "loss": 1.3082, + "theoretical_loss": 3.528728649376519, + "tokens_seen": 1579089920 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004214724172517553, + "loss": 1.2154, + "theoretical_loss": 3.528715967300646, + "tokens_seen": 1579155456 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042145636910732203, + "loss": 1.3719, + "theoretical_loss": 3.5287032858984384, + "tokens_seen": 1579220992 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004214403209628887, + "loss": 1.3689, + "theoretical_loss": 3.5286906051698317, + "tokens_seen": 1579286528 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042142427281845543, + "loss": 1.3586, + "theoretical_loss": 3.5286779251147626, + "tokens_seen": 1579352064 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 529316, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2476038932800293, + "objective/train/theoretical_loss": 3.5286652457331664, + "objective/train/tokens_used": 1599877600, + "theoretical_loss": 3.5286652457331664, + "tokens_seen": 1579417600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004214082246740221, + "loss": 1.3399, + "theoretical_loss": 3.5286652457331664, + "tokens_seen": 1579417600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042139217652958883, + "loss": 1.2368, + "theoretical_loss": 3.528652567024981, + "tokens_seen": 1579483136 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004213761283851555, + "loss": 1.2149, + "theoretical_loss": 3.5286398889901407, + "tokens_seen": 1579548672 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042136008024072223, + "loss": 1.2429, + "theoretical_loss": 3.5286272116285833, + "tokens_seen": 1579614208 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004213440320962889, + "loss": 1.2743, + "theoretical_loss": 3.528614534940245, + "tokens_seen": 1579679744 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042132798395185563, + "loss": 1.2867, + "theoretical_loss": 3.5286018589250614, + "tokens_seen": 1579745280 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004213119358074223, + "loss": 1.1426, + "theoretical_loss": 3.5285891835829695, + "tokens_seen": 1579810816 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042129588766298904, + "loss": 1.2519, + "theoretical_loss": 3.528576508913905, + "tokens_seen": 1579876352 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004212798395185557, + "loss": 1.2552, + "theoretical_loss": 3.528563834917805, + "tokens_seen": 1579941888 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042126379137412244, + "loss": 1.3386, + "theoretical_loss": 3.528551161594605, + "tokens_seen": 1580007424 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004212477432296891, + "loss": 1.3024, + "theoretical_loss": 3.528538488944242, + "tokens_seen": 1580072960 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042123169508525584, + "loss": 1.3447, + "theoretical_loss": 3.5285258169666522, + "tokens_seen": 1580138496 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004212156469408225, + "loss": 1.2856, + "theoretical_loss": 3.5285131456617718, + "tokens_seen": 1580204032 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042119959879638924, + "loss": 1.2508, + "theoretical_loss": 3.5285004750295377, + "tokens_seen": 1580269568 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004211835506519559, + "loss": 1.2427, + "theoretical_loss": 3.5284878050698865, + "tokens_seen": 1580335104 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042116750250752264, + "loss": 1.3434, + "theoretical_loss": 3.5284751357827533, + "tokens_seen": 1580400640 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004211514543630893, + "loss": 1.2483, + "theoretical_loss": 3.528462467168075, + "tokens_seen": 1580466176 + }, + { + "epoch": 0.48, + "learning_rate": 0.000421135406218656, + "loss": 1.2159, + "theoretical_loss": 3.5284497992257893, + "tokens_seen": 1580531712 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004211193580742227, + "loss": 1.2647, + "theoretical_loss": 3.528437131955831, + "tokens_seen": 1580597248 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004211033099297894, + "loss": 1.3201, + "theoretical_loss": 3.528424465358137, + "tokens_seen": 1580662784 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004210872617853561, + "loss": 1.377, + "theoretical_loss": 3.5284117994326447, + "tokens_seen": 1580728320 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004210712136409228, + "loss": 1.2718, + "theoretical_loss": 3.5283991341792893, + "tokens_seen": 1580793856 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004210551654964895, + "loss": 1.2739, + "theoretical_loss": 3.5283864695980083, + "tokens_seen": 1580859392 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004210391173520562, + "loss": 1.2492, + "theoretical_loss": 3.5283738056887373, + "tokens_seen": 1580924928 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004210230692076229, + "loss": 1.249, + "theoretical_loss": 3.5283611424514127, + "tokens_seen": 1580990464 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004210070210631896, + "loss": 1.2172, + "theoretical_loss": 3.528348479885972, + "tokens_seen": 1581056000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042099097291875624, + "loss": 1.3031, + "theoretical_loss": 3.5283358179923514, + "tokens_seen": 1581121536 + }, + { + "epoch": 0.48, + "learning_rate": 0.000420974924774323, + "loss": 1.2905, + "theoretical_loss": 3.5283231567704867, + "tokens_seen": 1581187072 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042095887662988964, + "loss": 1.2849, + "theoretical_loss": 3.528310496220315, + "tokens_seen": 1581252608 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004209428284854564, + "loss": 1.2904, + "theoretical_loss": 3.5282978363417725, + "tokens_seen": 1581318144 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042092678034102304, + "loss": 1.3633, + "theoretical_loss": 3.528285177134796, + "tokens_seen": 1581383680 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004209107321965898, + "loss": 1.1911, + "theoretical_loss": 3.528272518599322, + "tokens_seen": 1581449216 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042089468405215644, + "loss": 1.3286, + "theoretical_loss": 3.5282598607352877, + "tokens_seen": 1581514752 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004208786359077232, + "loss": 1.2025, + "theoretical_loss": 3.5282472035426284, + "tokens_seen": 1581580288 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042086258776328984, + "loss": 1.3582, + "theoretical_loss": 3.528234547021281, + "tokens_seen": 1581645824 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004208465396188566, + "loss": 1.2729, + "theoretical_loss": 3.528221891171183, + "tokens_seen": 1581711360 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042083049147442325, + "loss": 1.4335, + "theoretical_loss": 3.52820923599227, + "tokens_seen": 1581776896 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042081444332999, + "loss": 1.252, + "theoretical_loss": 3.528196581484479, + "tokens_seen": 1581842432 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042079839518555665, + "loss": 1.3587, + "theoretical_loss": 3.5281839276477465, + "tokens_seen": 1581907968 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004207823470411234, + "loss": 1.2302, + "theoretical_loss": 3.5281712744820095, + "tokens_seen": 1581973504 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042076629889669005, + "loss": 1.2506, + "theoretical_loss": 3.5281586219872043, + "tokens_seen": 1582039040 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004207502507522568, + "loss": 1.3141, + "theoretical_loss": 3.5281459701632674, + "tokens_seen": 1582104576 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042073420260782345, + "loss": 1.3975, + "theoretical_loss": 3.528133319010135, + "tokens_seen": 1582170112 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004207181544633902, + "loss": 1.307, + "theoretical_loss": 3.528120668527745, + "tokens_seen": 1582235648 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042070210631895685, + "loss": 1.2699, + "theoretical_loss": 3.528108018716033, + "tokens_seen": 1582301184 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004206860581745236, + "loss": 1.249, + "theoretical_loss": 3.5280953695749364, + "tokens_seen": 1582366720 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042067001003009025, + "loss": 1.2198, + "theoretical_loss": 3.5280827211043917, + "tokens_seen": 1582432256 + }, + { + "epoch": 0.48, + "learning_rate": 0.000420653961885657, + "loss": 1.33, + "theoretical_loss": 3.5280700733043346, + "tokens_seen": 1582497792 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042063791374122365, + "loss": 1.3147, + "theoretical_loss": 3.528057426174703, + "tokens_seen": 1582563328 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004206218655967904, + "loss": 1.2517, + "theoretical_loss": 3.528044779715434, + "tokens_seen": 1582628864 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 530619, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1984028816223145, + "objective/train/theoretical_loss": 3.5280321339264624, + "objective/train/tokens_used": 1603154400, + "theoretical_loss": 3.5280321339264624, + "tokens_seen": 1582694400 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042060581745235705, + "loss": 1.3051, + "theoretical_loss": 3.5280321339264624, + "tokens_seen": 1582694400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004205897693079238, + "loss": 1.3102, + "theoretical_loss": 3.5280194888077263, + "tokens_seen": 1582759936 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042057372116349046, + "loss": 1.2483, + "theoretical_loss": 3.5280068443591626, + "tokens_seen": 1582825472 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004205576730190572, + "loss": 1.4325, + "theoretical_loss": 3.5279942005807072, + "tokens_seen": 1582891008 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042054162487462386, + "loss": 1.2919, + "theoretical_loss": 3.5279815574722972, + "tokens_seen": 1582956544 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004205255767301906, + "loss": 1.3096, + "theoretical_loss": 3.5279689150338696, + "tokens_seen": 1583022080 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042050952858575726, + "loss": 1.3325, + "theoretical_loss": 3.527956273265361, + "tokens_seen": 1583087616 + }, + { + "epoch": 0.48, + "learning_rate": 0.000420493480441324, + "loss": 1.2714, + "theoretical_loss": 3.527943632166708, + "tokens_seen": 1583153152 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042047743229689066, + "loss": 1.2232, + "theoretical_loss": 3.527930991737848, + "tokens_seen": 1583218688 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004204613841524574, + "loss": 1.305, + "theoretical_loss": 3.527918351978717, + "tokens_seen": 1583284224 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042044533600802406, + "loss": 1.2146, + "theoretical_loss": 3.5279057128892513, + "tokens_seen": 1583349760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004204292878635908, + "loss": 1.3379, + "theoretical_loss": 3.5278930744693895, + "tokens_seen": 1583415296 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004204132397191575, + "loss": 1.3599, + "theoretical_loss": 3.527880436719067, + "tokens_seen": 1583480832 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004203971915747242, + "loss": 1.2844, + "theoretical_loss": 3.527867799638221, + "tokens_seen": 1583546368 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004203811434302909, + "loss": 1.2863, + "theoretical_loss": 3.5278551632267883, + "tokens_seen": 1583611904 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042036509528585756, + "loss": 1.2745, + "theoretical_loss": 3.5278425274847063, + "tokens_seen": 1583677440 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004203490471414243, + "loss": 1.386, + "theoretical_loss": 3.527829892411911, + "tokens_seen": 1583742976 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042033299899699097, + "loss": 1.3353, + "theoretical_loss": 3.5278172580083393, + "tokens_seen": 1583808512 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004203169508525577, + "loss": 1.2358, + "theoretical_loss": 3.5278046242739287, + "tokens_seen": 1583874048 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042030090270812437, + "loss": 1.2919, + "theoretical_loss": 3.5277919912086158, + "tokens_seen": 1583939584 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004202848545636911, + "loss": 1.4091, + "theoretical_loss": 3.5277793588123374, + "tokens_seen": 1584005120 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042026880641925777, + "loss": 1.2692, + "theoretical_loss": 3.52776672708503, + "tokens_seen": 1584070656 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004202527582748245, + "loss": 1.2129, + "theoretical_loss": 3.527754096026632, + "tokens_seen": 1584136192 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042023671013039117, + "loss": 1.3177, + "theoretical_loss": 3.527741465637078, + "tokens_seen": 1584201728 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004202206619859579, + "loss": 1.2905, + "theoretical_loss": 3.5277288359163066, + "tokens_seen": 1584267264 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042020461384152457, + "loss": 1.2648, + "theoretical_loss": 3.5277162068642545, + "tokens_seen": 1584332800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004201885656970913, + "loss": 1.2313, + "theoretical_loss": 3.5277035784808577, + "tokens_seen": 1584398336 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042017251755265797, + "loss": 1.313, + "theoretical_loss": 3.5276909507660545, + "tokens_seen": 1584463872 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004201564694082247, + "loss": 1.3639, + "theoretical_loss": 3.527678323719781, + "tokens_seen": 1584529408 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004201404212637914, + "loss": 1.2945, + "theoretical_loss": 3.527665697341974, + "tokens_seen": 1584594944 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042012437311935813, + "loss": 1.269, + "theoretical_loss": 3.527653071632572, + "tokens_seen": 1584660480 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004201083249749248, + "loss": 1.2947, + "theoretical_loss": 3.5276404465915094, + "tokens_seen": 1584726016 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042009227683049153, + "loss": 1.248, + "theoretical_loss": 3.5276278222187254, + "tokens_seen": 1584791552 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004200762286860582, + "loss": 1.252, + "theoretical_loss": 3.5276151985141557, + "tokens_seen": 1584857088 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042006018054162493, + "loss": 1.2908, + "theoretical_loss": 3.5276025754777383, + "tokens_seen": 1584922624 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004200441323971916, + "loss": 1.2689, + "theoretical_loss": 3.527589953109409, + "tokens_seen": 1584988160 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042002808425275833, + "loss": 1.2224, + "theoretical_loss": 3.527577331409106, + "tokens_seen": 1585053696 + }, + { + "epoch": 0.48, + "learning_rate": 0.000420012036108325, + "loss": 1.2175, + "theoretical_loss": 3.5275647103767653, + "tokens_seen": 1585119232 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041999598796389173, + "loss": 1.2662, + "theoretical_loss": 3.527552090012325, + "tokens_seen": 1585184768 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004199799398194584, + "loss": 1.1799, + "theoretical_loss": 3.5275394703157215, + "tokens_seen": 1585250304 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041996389167502513, + "loss": 1.2508, + "theoretical_loss": 3.5275268512868916, + "tokens_seen": 1585315840 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004199478435305918, + "loss": 1.2666, + "theoretical_loss": 3.5275142329257734, + "tokens_seen": 1585381376 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041993179538615854, + "loss": 1.2242, + "theoretical_loss": 3.527501615232303, + "tokens_seen": 1585446912 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004199157472417252, + "loss": 1.3351, + "theoretical_loss": 3.5274889982064175, + "tokens_seen": 1585512448 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041989969909729194, + "loss": 1.302, + "theoretical_loss": 3.5274763818480546, + "tokens_seen": 1585577984 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004198836509528586, + "loss": 1.3432, + "theoretical_loss": 3.527463766157151, + "tokens_seen": 1585643520 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041986760280842534, + "loss": 1.2863, + "theoretical_loss": 3.5274511511336435, + "tokens_seen": 1585709056 + }, + { + "epoch": 0.48, + "learning_rate": 0.000419851554663992, + "loss": 1.2974, + "theoretical_loss": 3.5274385367774697, + "tokens_seen": 1585774592 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041983550651955874, + "loss": 1.3053, + "theoretical_loss": 3.5274259230885665, + "tokens_seen": 1585840128 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004198194583751254, + "loss": 1.3219, + "theoretical_loss": 3.527413310066872, + "tokens_seen": 1585905664 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 531508, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2401823997497559, + "objective/train/theoretical_loss": 3.5274006977123213, + "objective/train/tokens_used": 1606431200, + "theoretical_loss": 3.5274006977123213, + "tokens_seen": 1585971200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041980341023069214, + "loss": 1.2822, + "theoretical_loss": 3.5274006977123213, + "tokens_seen": 1585971200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004197873620862588, + "loss": 1.2111, + "theoretical_loss": 3.527388086024853, + "tokens_seen": 1586036736 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041977131394182554, + "loss": 1.3603, + "theoretical_loss": 3.527375475004404, + "tokens_seen": 1586102272 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004197552657973922, + "loss": 1.3557, + "theoretical_loss": 3.527362864650912, + "tokens_seen": 1586167808 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004197392176529589, + "loss": 1.3688, + "theoretical_loss": 3.527350254964313, + "tokens_seen": 1586233344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004197231695085256, + "loss": 1.2819, + "theoretical_loss": 3.527337645944545, + "tokens_seen": 1586298880 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004197071213640923, + "loss": 1.3064, + "theoretical_loss": 3.527325037591545, + "tokens_seen": 1586364416 + }, + { + "epoch": 0.48, + "learning_rate": 0.000419691073219659, + "loss": 1.3509, + "theoretical_loss": 3.52731242990525, + "tokens_seen": 1586429952 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004196750250752257, + "loss": 1.2732, + "theoretical_loss": 3.527299822885597, + "tokens_seen": 1586495488 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004196589769307924, + "loss": 1.2875, + "theoretical_loss": 3.527287216532524, + "tokens_seen": 1586561024 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004196429287863591, + "loss": 1.3171, + "theoretical_loss": 3.527274610845968, + "tokens_seen": 1586626560 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004196268806419258, + "loss": 1.2842, + "theoretical_loss": 3.527262005825866, + "tokens_seen": 1586692096 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004196108324974925, + "loss": 1.2986, + "theoretical_loss": 3.5272494014721554, + "tokens_seen": 1586757632 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041959478435305914, + "loss": 1.2124, + "theoretical_loss": 3.5272367977847727, + "tokens_seen": 1586823168 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004195787362086259, + "loss": 1.4275, + "theoretical_loss": 3.5272241947636562, + "tokens_seen": 1586888704 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041956268806419254, + "loss": 1.3391, + "theoretical_loss": 3.527211592408743, + "tokens_seen": 1586954240 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004195466399197593, + "loss": 1.3117, + "theoretical_loss": 3.5271989907199695, + "tokens_seen": 1587019776 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041953059177532594, + "loss": 1.3273, + "theoretical_loss": 3.527186389697274, + "tokens_seen": 1587085312 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004195145436308927, + "loss": 1.3016, + "theoretical_loss": 3.5271737893405932, + "tokens_seen": 1587150848 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041949849548645934, + "loss": 1.1969, + "theoretical_loss": 3.5271611896498647, + "tokens_seen": 1587216384 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004194824473420261, + "loss": 1.2943, + "theoretical_loss": 3.5271485906250257, + "tokens_seen": 1587281920 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041946639919759275, + "loss": 1.2469, + "theoretical_loss": 3.5271359922660137, + "tokens_seen": 1587347456 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004194503510531595, + "loss": 1.2952, + "theoretical_loss": 3.527123394572765, + "tokens_seen": 1587412992 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041943430290872615, + "loss": 1.2892, + "theoretical_loss": 3.5271107975452187, + "tokens_seen": 1587478528 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004194182547642929, + "loss": 1.3364, + "theoretical_loss": 3.5270982011833114, + "tokens_seen": 1587544064 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041940220661985955, + "loss": 1.237, + "theoretical_loss": 3.5270856054869797, + "tokens_seen": 1587609600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004193861584754263, + "loss": 1.2828, + "theoretical_loss": 3.527073010456162, + "tokens_seen": 1587675136 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041937011033099295, + "loss": 1.3282, + "theoretical_loss": 3.5270604160907943, + "tokens_seen": 1587740672 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004193540621865597, + "loss": 1.4049, + "theoretical_loss": 3.5270478223908155, + "tokens_seen": 1587806208 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041933801404212635, + "loss": 1.3352, + "theoretical_loss": 3.5270352293561626, + "tokens_seen": 1587871744 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004193219658976931, + "loss": 1.2145, + "theoretical_loss": 3.5270226369867723, + "tokens_seen": 1587937280 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041930591775325975, + "loss": 1.3095, + "theoretical_loss": 3.5270100452825828, + "tokens_seen": 1588002816 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004192898696088265, + "loss": 1.3382, + "theoretical_loss": 3.5269974542435314, + "tokens_seen": 1588068352 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041927382146439315, + "loss": 1.2923, + "theoretical_loss": 3.5269848638695547, + "tokens_seen": 1588133888 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004192577733199599, + "loss": 1.2993, + "theoretical_loss": 3.526972274160591, + "tokens_seen": 1588199424 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041924172517552655, + "loss": 1.2885, + "theoretical_loss": 3.5269596851165774, + "tokens_seen": 1588264960 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004192256770310933, + "loss": 1.2316, + "theoretical_loss": 3.526947096737451, + "tokens_seen": 1588330496 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041920962888666006, + "loss": 1.2396, + "theoretical_loss": 3.5269345090231505, + "tokens_seen": 1588396032 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004191935807422267, + "loss": 1.2591, + "theoretical_loss": 3.526921921973612, + "tokens_seen": 1588461568 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041917753259779347, + "loss": 1.2781, + "theoretical_loss": 3.5269093355887735, + "tokens_seen": 1588527104 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004191614844533601, + "loss": 1.2644, + "theoretical_loss": 3.5268967498685724, + "tokens_seen": 1588592640 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041914543630892687, + "loss": 1.356, + "theoretical_loss": 3.5268841648129463, + "tokens_seen": 1588658176 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004191293881644935, + "loss": 1.3601, + "theoretical_loss": 3.5268715804218322, + "tokens_seen": 1588723712 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041911334002006027, + "loss": 1.3441, + "theoretical_loss": 3.526858996695169, + "tokens_seen": 1588789248 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004190972918756269, + "loss": 1.2989, + "theoretical_loss": 3.5268464136328923, + "tokens_seen": 1588854784 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004190812437311936, + "loss": 1.311, + "theoretical_loss": 3.526833831234941, + "tokens_seen": 1588920320 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004190651955867603, + "loss": 1.2844, + "theoretical_loss": 3.5268212495012525, + "tokens_seen": 1588985856 + }, + { + "epoch": 0.48, + "learning_rate": 0.000419049147442327, + "loss": 1.2975, + "theoretical_loss": 3.5268086684317637, + "tokens_seen": 1589051392 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004190330992978937, + "loss": 1.3467, + "theoretical_loss": 3.526796088026412, + "tokens_seen": 1589116928 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004190170511534604, + "loss": 1.4015, + "theoretical_loss": 3.526783508285136, + "tokens_seen": 1589182464 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 532777, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3931702375411987, + "objective/train/theoretical_loss": 3.526770929207873, + "objective/train/tokens_used": 1609708000, + "theoretical_loss": 3.526770929207873, + "tokens_seen": 1589248000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004190010030090271, + "loss": 1.3721, + "theoretical_loss": 3.526770929207873, + "tokens_seen": 1589248000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004189849548645938, + "loss": 1.3258, + "theoretical_loss": 3.5267583507945597, + "tokens_seen": 1589313536 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041896890672016047, + "loss": 1.3597, + "theoretical_loss": 3.5267457730451346, + "tokens_seen": 1589379072 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004189528585757272, + "loss": 1.3431, + "theoretical_loss": 3.526733195959535, + "tokens_seen": 1589444608 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041893681043129387, + "loss": 1.3008, + "theoretical_loss": 3.5267206195376977, + "tokens_seen": 1589510144 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004189207622868606, + "loss": 1.2865, + "theoretical_loss": 3.5267080437795615, + "tokens_seen": 1589575680 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041890471414242727, + "loss": 1.3016, + "theoretical_loss": 3.526695468685064, + "tokens_seen": 1589641216 + }, + { + "epoch": 0.48, + "learning_rate": 0.000418888665997994, + "loss": 1.1346, + "theoretical_loss": 3.526682894254142, + "tokens_seen": 1589706752 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041887261785356067, + "loss": 1.3811, + "theoretical_loss": 3.5266703204867333, + "tokens_seen": 1589772288 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004188565697091274, + "loss": 1.2418, + "theoretical_loss": 3.526657747382776, + "tokens_seen": 1589837824 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041884052156469407, + "loss": 1.375, + "theoretical_loss": 3.526645174942207, + "tokens_seen": 1589903360 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004188244734202608, + "loss": 1.2092, + "theoretical_loss": 3.5266326031649653, + "tokens_seen": 1589968896 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041880842527582747, + "loss": 1.3599, + "theoretical_loss": 3.526620032050987, + "tokens_seen": 1590034432 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004187923771313942, + "loss": 1.265, + "theoretical_loss": 3.5266074616002108, + "tokens_seen": 1590099968 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004187763289869609, + "loss": 1.3309, + "theoretical_loss": 3.526594891812574, + "tokens_seen": 1590165504 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041876028084252763, + "loss": 1.2433, + "theoretical_loss": 3.526582322688014, + "tokens_seen": 1590231040 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004187442326980943, + "loss": 1.2087, + "theoretical_loss": 3.526569754226469, + "tokens_seen": 1590296576 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041872818455366103, + "loss": 1.3058, + "theoretical_loss": 3.526557186427877, + "tokens_seen": 1590362112 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004187121364092277, + "loss": 1.3389, + "theoretical_loss": 3.5265446192921748, + "tokens_seen": 1590427648 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041869608826479443, + "loss": 1.2142, + "theoretical_loss": 3.5265320528193005, + "tokens_seen": 1590493184 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004186800401203611, + "loss": 1.3372, + "theoretical_loss": 3.5265194870091925, + "tokens_seen": 1590558720 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041866399197592783, + "loss": 1.3173, + "theoretical_loss": 3.5265069218617873, + "tokens_seen": 1590624256 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004186479438314945, + "loss": 1.4553, + "theoretical_loss": 3.5264943573770235, + "tokens_seen": 1590689792 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041863189568706123, + "loss": 1.3482, + "theoretical_loss": 3.526481793554839, + "tokens_seen": 1590755328 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004186158475426279, + "loss": 1.3073, + "theoretical_loss": 3.526469230395171, + "tokens_seen": 1590820864 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041859979939819463, + "loss": 1.3264, + "theoretical_loss": 3.5264566678979574, + "tokens_seen": 1590886400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004185837512537613, + "loss": 1.2387, + "theoretical_loss": 3.526444106063136, + "tokens_seen": 1590951936 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041856770310932804, + "loss": 1.2303, + "theoretical_loss": 3.526431544890645, + "tokens_seen": 1591017472 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004185516549648947, + "loss": 1.2593, + "theoretical_loss": 3.5264189843804212, + "tokens_seen": 1591083008 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041853560682046144, + "loss": 1.3082, + "theoretical_loss": 3.5264064245324036, + "tokens_seen": 1591148544 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004185195586760281, + "loss": 1.2804, + "theoretical_loss": 3.5263938653465297, + "tokens_seen": 1591214080 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041850351053159484, + "loss": 1.2827, + "theoretical_loss": 3.5263813068227368, + "tokens_seen": 1591279616 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004184874623871615, + "loss": 1.2933, + "theoretical_loss": 3.526368748960963, + "tokens_seen": 1591345152 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041847141424272824, + "loss": 1.363, + "theoretical_loss": 3.5263561917611463, + "tokens_seen": 1591410688 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004184553660982949, + "loss": 1.2557, + "theoretical_loss": 3.5263436352232245, + "tokens_seen": 1591476224 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041843931795386164, + "loss": 1.3466, + "theoretical_loss": 3.526331079347135, + "tokens_seen": 1591541760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004184232698094283, + "loss": 1.3541, + "theoretical_loss": 3.5263185241328165, + "tokens_seen": 1591607296 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041840722166499504, + "loss": 1.3778, + "theoretical_loss": 3.526305969580206, + "tokens_seen": 1591672832 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004183911735205617, + "loss": 1.3468, + "theoretical_loss": 3.5262934156892425, + "tokens_seen": 1591738368 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041837512537612844, + "loss": 1.3134, + "theoretical_loss": 3.5262808624598625, + "tokens_seen": 1591803904 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004183590772316951, + "loss": 1.2955, + "theoretical_loss": 3.526268309892005, + "tokens_seen": 1591869440 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041834302908726184, + "loss": 1.2964, + "theoretical_loss": 3.5262557579856075, + "tokens_seen": 1591934976 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004183269809428285, + "loss": 1.3082, + "theoretical_loss": 3.5262432067406078, + "tokens_seen": 1592000512 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004183109327983952, + "loss": 1.2465, + "theoretical_loss": 3.526230656156944, + "tokens_seen": 1592066048 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004182948846539619, + "loss": 1.2867, + "theoretical_loss": 3.5262181062345537, + "tokens_seen": 1592131584 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004182788365095286, + "loss": 1.3035, + "theoretical_loss": 3.526205556973375, + "tokens_seen": 1592197120 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004182627883650953, + "loss": 1.3289, + "theoretical_loss": 3.5261930083733466, + "tokens_seen": 1592262656 + }, + { + "epoch": 0.48, + "learning_rate": 0.000418246740220662, + "loss": 1.3158, + "theoretical_loss": 3.5261804604344054, + "tokens_seen": 1592328192 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004182306920762287, + "loss": 1.3363, + "theoretical_loss": 3.52616791315649, + "tokens_seen": 1592393728 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004182146439317954, + "loss": 1.3114, + "theoretical_loss": 3.526155366539538, + "tokens_seen": 1592459264 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 534133, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2955944538116455, + "objective/train/theoretical_loss": 3.5261428205834875, + "objective/train/tokens_used": 1612984800, + "theoretical_loss": 3.5261428205834875, + "tokens_seen": 1592524800 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041819859578736204, + "loss": 1.3085, + "theoretical_loss": 3.5261428205834875, + "tokens_seen": 1592524800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004181825476429288, + "loss": 1.2541, + "theoretical_loss": 3.526130275288277, + "tokens_seen": 1592590336 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041816649949849544, + "loss": 1.4107, + "theoretical_loss": 3.5261177306538434, + "tokens_seen": 1592655872 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004181504513540622, + "loss": 1.294, + "theoretical_loss": 3.526105186680126, + "tokens_seen": 1592721408 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041813440320962884, + "loss": 1.3976, + "theoretical_loss": 3.526092643367061, + "tokens_seen": 1592786944 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004181183550651956, + "loss": 1.3058, + "theoretical_loss": 3.526080100714589, + "tokens_seen": 1592852480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041810230692076225, + "loss": 1.2832, + "theoretical_loss": 3.526067558722646, + "tokens_seen": 1592918016 + }, + { + "epoch": 0.48, + "learning_rate": 0.000418086258776329, + "loss": 1.3361, + "theoretical_loss": 3.5260550173911707, + "tokens_seen": 1592983552 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041807021063189565, + "loss": 1.2854, + "theoretical_loss": 3.526042476720101, + "tokens_seen": 1593049088 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004180541624874624, + "loss": 1.3805, + "theoretical_loss": 3.5260299367093753, + "tokens_seen": 1593114624 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041803811434302905, + "loss": 1.3232, + "theoretical_loss": 3.526017397358931, + "tokens_seen": 1593180160 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004180220661985958, + "loss": 1.3571, + "theoretical_loss": 3.526004858668707, + "tokens_seen": 1593245696 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041800601805416256, + "loss": 1.3235, + "theoretical_loss": 3.525992320638641, + "tokens_seen": 1593311232 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004179899699097292, + "loss": 1.0997, + "theoretical_loss": 3.525979783268671, + "tokens_seen": 1593376768 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041797392176529596, + "loss": 1.3121, + "theoretical_loss": 3.5259672465587353, + "tokens_seen": 1593442304 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004179578736208626, + "loss": 1.3656, + "theoretical_loss": 3.5259547105087714, + "tokens_seen": 1593507840 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041794182547642936, + "loss": 1.2168, + "theoretical_loss": 3.5259421751187183, + "tokens_seen": 1593573376 + }, + { + "epoch": 0.48, + "learning_rate": 0.000417925777331996, + "loss": 1.3795, + "theoretical_loss": 3.525929640388514, + "tokens_seen": 1593638912 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041790972918756276, + "loss": 1.3063, + "theoretical_loss": 3.5259171063180963, + "tokens_seen": 1593704448 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004178936810431294, + "loss": 1.2893, + "theoretical_loss": 3.525904572907403, + "tokens_seen": 1593769984 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041787763289869616, + "loss": 1.2935, + "theoretical_loss": 3.5258920401563727, + "tokens_seen": 1593835520 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004178615847542628, + "loss": 1.3267, + "theoretical_loss": 3.5258795080649437, + "tokens_seen": 1593901056 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041784553660982956, + "loss": 1.3593, + "theoretical_loss": 3.525866976633054, + "tokens_seen": 1593966592 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004178294884653962, + "loss": 1.319, + "theoretical_loss": 3.5258544458606416, + "tokens_seen": 1594032128 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041781344032096297, + "loss": 1.297, + "theoretical_loss": 3.5258419157476446, + "tokens_seen": 1594097664 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004177973921765296, + "loss": 1.2661, + "theoretical_loss": 3.525829386294002, + "tokens_seen": 1594163200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041778134403209637, + "loss": 1.2155, + "theoretical_loss": 3.5258168574996516, + "tokens_seen": 1594228736 + }, + { + "epoch": 0.48, + "learning_rate": 0.000417765295887663, + "loss": 1.4208, + "theoretical_loss": 3.5258043293645307, + "tokens_seen": 1594294272 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041774924774322977, + "loss": 1.3209, + "theoretical_loss": 3.5257918018885785, + "tokens_seen": 1594359808 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004177331995987964, + "loss": 1.3237, + "theoretical_loss": 3.5257792750717334, + "tokens_seen": 1594425344 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041771715145436317, + "loss": 1.3139, + "theoretical_loss": 3.5257667489139326, + "tokens_seen": 1594490880 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004177011033099298, + "loss": 1.3462, + "theoretical_loss": 3.525754223415116, + "tokens_seen": 1594556416 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004176850551654965, + "loss": 1.2605, + "theoretical_loss": 3.5257416985752195, + "tokens_seen": 1594621952 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004176690070210632, + "loss": 1.268, + "theoretical_loss": 3.525729174394183, + "tokens_seen": 1594687488 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004176529588766299, + "loss": 1.3794, + "theoretical_loss": 3.525716650871945, + "tokens_seen": 1594753024 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004176369107321966, + "loss": 1.4053, + "theoretical_loss": 3.525704128008443, + "tokens_seen": 1594818560 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004176208625877633, + "loss": 1.3663, + "theoretical_loss": 3.5256916058036154, + "tokens_seen": 1594884096 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041760481444333, + "loss": 1.3703, + "theoretical_loss": 3.5256790842574004, + "tokens_seen": 1594949632 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004175887662988967, + "loss": 1.286, + "theoretical_loss": 3.525666563369737, + "tokens_seen": 1595015168 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004175727181544634, + "loss": 1.3113, + "theoretical_loss": 3.5256540431405625, + "tokens_seen": 1595080704 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004175566700100301, + "loss": 1.3088, + "theoretical_loss": 3.5256415235698157, + "tokens_seen": 1595146240 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041754062186559677, + "loss": 1.2932, + "theoretical_loss": 3.5256290046574352, + "tokens_seen": 1595211776 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004175245737211635, + "loss": 1.3011, + "theoretical_loss": 3.525616486403359, + "tokens_seen": 1595277312 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041750852557673017, + "loss": 1.391, + "theoretical_loss": 3.525603968807525, + "tokens_seen": 1595342848 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004174924774322969, + "loss": 1.355, + "theoretical_loss": 3.5255914518698726, + "tokens_seen": 1595408384 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041747642928786357, + "loss": 1.3184, + "theoretical_loss": 3.5255789355903397, + "tokens_seen": 1595473920 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004174603811434303, + "loss": 1.2601, + "theoretical_loss": 3.5255664199688646, + "tokens_seen": 1595539456 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041744433299899697, + "loss": 1.3963, + "theoretical_loss": 3.5255539050053852, + "tokens_seen": 1595604992 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004174282848545637, + "loss": 1.2895, + "theoretical_loss": 3.5255413906998405, + "tokens_seen": 1595670528 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041741223671013037, + "loss": 1.3094, + "theoretical_loss": 3.525528877052169, + "tokens_seen": 1595736064 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 535018, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2892045974731445, + "objective/train/theoretical_loss": 3.5255163640623084, + "objective/train/tokens_used": 1616261600, + "theoretical_loss": 3.5255163640623084, + "tokens_seen": 1595801600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041739618856569713, + "loss": 1.3489, + "theoretical_loss": 3.5255163640623084, + "tokens_seen": 1595801600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004173801404212638, + "loss": 1.3358, + "theoretical_loss": 3.525503851730198, + "tokens_seen": 1595867136 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041736409227683053, + "loss": 1.3527, + "theoretical_loss": 3.5254913400557752, + "tokens_seen": 1595932672 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004173480441323972, + "loss": 1.3156, + "theoretical_loss": 3.5254788290389794, + "tokens_seen": 1595998208 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041733199598796393, + "loss": 1.2212, + "theoretical_loss": 3.5254663186797486, + "tokens_seen": 1596063744 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004173159478435306, + "loss": 1.2936, + "theoretical_loss": 3.525453808978021, + "tokens_seen": 1596129280 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041729989969909733, + "loss": 1.3786, + "theoretical_loss": 3.5254412999337355, + "tokens_seen": 1596194816 + }, + { + "epoch": 0.48, + "learning_rate": 0.000417283851554664, + "loss": 1.2774, + "theoretical_loss": 3.5254287915468305, + "tokens_seen": 1596260352 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041726780341023073, + "loss": 1.2742, + "theoretical_loss": 3.5254162838172443, + "tokens_seen": 1596325888 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004172517552657974, + "loss": 1.3532, + "theoretical_loss": 3.5254037767449153, + "tokens_seen": 1596391424 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041723570712136413, + "loss": 1.4497, + "theoretical_loss": 3.5253912703297816, + "tokens_seen": 1596456960 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004172196589769308, + "loss": 1.2876, + "theoretical_loss": 3.5253787645717827, + "tokens_seen": 1596522496 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041720361083249754, + "loss": 1.302, + "theoretical_loss": 3.5253662594708564, + "tokens_seen": 1596588032 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004171875626880642, + "loss": 1.3419, + "theoretical_loss": 3.5253537550269414, + "tokens_seen": 1596653568 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041717151454363094, + "loss": 1.3104, + "theoretical_loss": 3.5253412512399764, + "tokens_seen": 1596719104 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004171554663991976, + "loss": 1.2956, + "theoretical_loss": 3.5253287481098994, + "tokens_seen": 1596784640 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041713941825476434, + "loss": 1.3469, + "theoretical_loss": 3.52531624563665, + "tokens_seen": 1596850176 + }, + { + "epoch": 0.48, + "learning_rate": 0.000417123370110331, + "loss": 1.1238, + "theoretical_loss": 3.5253037438201646, + "tokens_seen": 1596915712 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041710732196589774, + "loss": 1.3157, + "theoretical_loss": 3.5252912426603844, + "tokens_seen": 1596981248 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004170912738214644, + "loss": 1.2143, + "theoretical_loss": 3.525278742157246, + "tokens_seen": 1597046784 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041707522567703114, + "loss": 1.2819, + "theoretical_loss": 3.525266242310689, + "tokens_seen": 1597112320 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004170591775325978, + "loss": 1.263, + "theoretical_loss": 3.5252537431206514, + "tokens_seen": 1597177856 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041704312938816454, + "loss": 1.3309, + "theoretical_loss": 3.5252412445870718, + "tokens_seen": 1597243392 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004170270812437312, + "loss": 1.1777, + "theoretical_loss": 3.5252287467098897, + "tokens_seen": 1597308928 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041701103309929794, + "loss": 1.2347, + "theoretical_loss": 3.525216249489042, + "tokens_seen": 1597374464 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004169949849548646, + "loss": 1.2991, + "theoretical_loss": 3.5252037529244693, + "tokens_seen": 1597440000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041697893681043134, + "loss": 1.3162, + "theoretical_loss": 3.525191257016109, + "tokens_seen": 1597505536 + }, + { + "epoch": 0.48, + "learning_rate": 0.000416962888665998, + "loss": 1.3729, + "theoretical_loss": 3.5251787617638994, + "tokens_seen": 1597571072 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041694684052156475, + "loss": 1.3111, + "theoretical_loss": 3.5251662671677804, + "tokens_seen": 1597636608 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004169307923771314, + "loss": 1.356, + "theoretical_loss": 3.5251537732276894, + "tokens_seen": 1597702144 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004169147442326981, + "loss": 1.2632, + "theoretical_loss": 3.5251412799435657, + "tokens_seen": 1597767680 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004168986960882648, + "loss": 1.3601, + "theoretical_loss": 3.525128787315348, + "tokens_seen": 1597833216 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004168826479438315, + "loss": 1.3514, + "theoretical_loss": 3.5251162953429747, + "tokens_seen": 1597898752 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004168665997993982, + "loss": 1.2357, + "theoretical_loss": 3.5251038040263847, + "tokens_seen": 1597964288 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004168505516549649, + "loss": 1.3505, + "theoretical_loss": 3.5250913133655164, + "tokens_seen": 1598029824 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004168345035105316, + "loss": 1.3869, + "theoretical_loss": 3.5250788233603085, + "tokens_seen": 1598095360 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004168184553660983, + "loss": 1.3275, + "theoretical_loss": 3.5250663340107, + "tokens_seen": 1598160896 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041680240722166505, + "loss": 1.3261, + "theoretical_loss": 3.525053845316629, + "tokens_seen": 1598226432 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004167863590772317, + "loss": 1.2504, + "theoretical_loss": 3.525041357278035, + "tokens_seen": 1598291968 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041677031093279845, + "loss": 1.2848, + "theoretical_loss": 3.5250288698948564, + "tokens_seen": 1598357504 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004167542627883651, + "loss": 1.2343, + "theoretical_loss": 3.525016383167032, + "tokens_seen": 1598423040 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041673821464393185, + "loss": 1.2793, + "theoretical_loss": 3.5250038970945, + "tokens_seen": 1598488576 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004167221664994985, + "loss": 1.2619, + "theoretical_loss": 3.5249914116772, + "tokens_seen": 1598554112 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041670611835506525, + "loss": 1.3503, + "theoretical_loss": 3.52497892691507, + "tokens_seen": 1598619648 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004166900702106319, + "loss": 1.2669, + "theoretical_loss": 3.5249664428080494, + "tokens_seen": 1598685184 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041667402206619866, + "loss": 1.2003, + "theoretical_loss": 3.5249539593560764, + "tokens_seen": 1598750720 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004166579739217653, + "loss": 1.2375, + "theoretical_loss": 3.52494147655909, + "tokens_seen": 1598816256 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041664192577733206, + "loss": 1.253, + "theoretical_loss": 3.5249289944170297, + "tokens_seen": 1598881792 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004166258776328987, + "loss": 1.2089, + "theoretical_loss": 3.524916512929833, + "tokens_seen": 1598947328 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041660982948846546, + "loss": 1.3491, + "theoretical_loss": 3.5249040320974396, + "tokens_seen": 1599012864 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 535874, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3526699542999268, + "objective/train/theoretical_loss": 3.524891551919788, + "objective/train/tokens_used": 1619538400, + "theoretical_loss": 3.524891551919788, + "tokens_seen": 1599078400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004165937813440321, + "loss": 1.2515, + "theoretical_loss": 3.524891551919788, + "tokens_seen": 1599078400 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041657773319959886, + "loss": 1.2148, + "theoretical_loss": 3.524879072396817, + "tokens_seen": 1599143936 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004165616850551655, + "loss": 1.3939, + "theoretical_loss": 3.5248665935284658, + "tokens_seen": 1599209472 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041654563691073226, + "loss": 1.2845, + "theoretical_loss": 3.5248541153146724, + "tokens_seen": 1599275008 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004165295887662989, + "loss": 1.2864, + "theoretical_loss": 3.5248416377553764, + "tokens_seen": 1599340544 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041651354062186566, + "loss": 1.1897, + "theoretical_loss": 3.5248291608505165, + "tokens_seen": 1599406080 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004164974924774323, + "loss": 1.3159, + "theoretical_loss": 3.5248166846000313, + "tokens_seen": 1599471616 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041648144433299906, + "loss": 1.2669, + "theoretical_loss": 3.52480420900386, + "tokens_seen": 1599537152 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004164653961885657, + "loss": 1.2628, + "theoretical_loss": 3.5247917340619415, + "tokens_seen": 1599602688 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041644934804413247, + "loss": 1.3442, + "theoretical_loss": 3.5247792597742147, + "tokens_seen": 1599668224 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004164332998996991, + "loss": 1.3347, + "theoretical_loss": 3.524766786140618, + "tokens_seen": 1599733760 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041641725175526587, + "loss": 1.2555, + "theoretical_loss": 3.5247543131610906, + "tokens_seen": 1599799296 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004164012036108325, + "loss": 1.2195, + "theoretical_loss": 3.5247418408355715, + "tokens_seen": 1599864832 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041638515546639927, + "loss": 1.25, + "theoretical_loss": 3.5247293691639996, + "tokens_seen": 1599930368 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004163691073219659, + "loss": 1.2788, + "theoretical_loss": 3.5247168981463144, + "tokens_seen": 1599995904 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041635305917753267, + "loss": 1.3879, + "theoretical_loss": 3.5247044277824533, + "tokens_seen": 1600061440 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004163370110330993, + "loss": 1.3778, + "theoretical_loss": 3.5246919580723564, + "tokens_seen": 1600126976 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041632096288866607, + "loss": 1.3168, + "theoretical_loss": 3.5246794890159627, + "tokens_seen": 1600192512 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004163049147442327, + "loss": 1.3215, + "theoretical_loss": 3.5246670206132107, + "tokens_seen": 1600258048 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004162888665997994, + "loss": 1.3128, + "theoretical_loss": 3.52465455286404, + "tokens_seen": 1600323584 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004162728184553661, + "loss": 1.249, + "theoretical_loss": 3.524642085768388, + "tokens_seen": 1600389120 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004162567703109328, + "loss": 1.2697, + "theoretical_loss": 3.5246296193261957, + "tokens_seen": 1600454656 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004162407221664995, + "loss": 1.3321, + "theoretical_loss": 3.524617153537401, + "tokens_seen": 1600520192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004162246740220662, + "loss": 1.4486, + "theoretical_loss": 3.524604688401943, + "tokens_seen": 1600585728 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004162086258776329, + "loss": 1.2488, + "theoretical_loss": 3.524592223919761, + "tokens_seen": 1600651264 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004161925777331996, + "loss": 1.2404, + "theoretical_loss": 3.524579760090794, + "tokens_seen": 1600716800 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004161765295887663, + "loss": 1.391, + "theoretical_loss": 3.524567296914981, + "tokens_seen": 1600782336 + }, + { + "epoch": 0.49, + "learning_rate": 0.000416160481444333, + "loss": 1.269, + "theoretical_loss": 3.5245548343922604, + "tokens_seen": 1600847872 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041614443329989967, + "loss": 1.2758, + "theoretical_loss": 3.5245423725225713, + "tokens_seen": 1600913408 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004161283851554664, + "loss": 1.2227, + "theoretical_loss": 3.524529911305854, + "tokens_seen": 1600978944 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041611233701103307, + "loss": 1.2996, + "theoretical_loss": 3.5245174507420467, + "tokens_seen": 1601044480 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004160962888665998, + "loss": 1.2715, + "theoretical_loss": 3.524504990831088, + "tokens_seen": 1601110016 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041608024072216647, + "loss": 1.3514, + "theoretical_loss": 3.5244925315729176, + "tokens_seen": 1601175552 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004160641925777332, + "loss": 1.3901, + "theoretical_loss": 3.5244800729674752, + "tokens_seen": 1601241088 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041604814443329987, + "loss": 1.2613, + "theoretical_loss": 3.5244676150146983, + "tokens_seen": 1601306624 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041603209628886663, + "loss": 1.27, + "theoretical_loss": 3.5244551577145273, + "tokens_seen": 1601372160 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004160160481444333, + "loss": 1.2595, + "theoretical_loss": 3.524442701066901, + "tokens_seen": 1601437696 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041600000000000003, + "loss": 1.3035, + "theoretical_loss": 3.524430245071758, + "tokens_seen": 1601503232 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004159839518555667, + "loss": 1.3108, + "theoretical_loss": 3.5244177897290383, + "tokens_seen": 1601568768 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041596790371113343, + "loss": 1.2935, + "theoretical_loss": 3.52440533503868, + "tokens_seen": 1601634304 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004159518555667001, + "loss": 1.1936, + "theoretical_loss": 3.524392881000623, + "tokens_seen": 1601699840 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041593580742226683, + "loss": 1.3264, + "theoretical_loss": 3.524380427614806, + "tokens_seen": 1601765376 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004159197592778335, + "loss": 1.4175, + "theoretical_loss": 3.5243679748811685, + "tokens_seen": 1601830912 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041590371113340023, + "loss": 1.3324, + "theoretical_loss": 3.5243555227996497, + "tokens_seen": 1601896448 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004158876629889669, + "loss": 1.2627, + "theoretical_loss": 3.524343071370189, + "tokens_seen": 1601961984 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041587161484453363, + "loss": 1.252, + "theoretical_loss": 3.5243306205927247, + "tokens_seen": 1602027520 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004158555667001003, + "loss": 1.286, + "theoretical_loss": 3.5243181704671964, + "tokens_seen": 1602093056 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041583951855566703, + "loss": 1.2345, + "theoretical_loss": 3.524305720993544, + "tokens_seen": 1602158592 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004158234704112337, + "loss": 1.2133, + "theoretical_loss": 3.5242932721717057, + "tokens_seen": 1602224128 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041580742226680044, + "loss": 1.2693, + "theoretical_loss": 3.524280824001621, + "tokens_seen": 1602289664 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 537090, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3611112833023071, + "objective/train/theoretical_loss": 3.524268376483229, + "objective/train/tokens_used": 1622815200, + "theoretical_loss": 3.524268376483229, + "tokens_seen": 1602355200 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004157913741223671, + "loss": 1.2988, + "theoretical_loss": 3.524268376483229, + "tokens_seen": 1602355200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041577532597793384, + "loss": 1.283, + "theoretical_loss": 3.5242559296164697, + "tokens_seen": 1602420736 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004157592778335005, + "loss": 1.403, + "theoretical_loss": 3.524243483401282, + "tokens_seen": 1602486272 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041574322968906724, + "loss": 1.2206, + "theoretical_loss": 3.5242310378376045, + "tokens_seen": 1602551808 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004157271815446339, + "loss": 1.2726, + "theoretical_loss": 3.5242185929253766, + "tokens_seen": 1602617344 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041571113340020064, + "loss": 1.351, + "theoretical_loss": 3.5242061486645384, + "tokens_seen": 1602682880 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004156950852557673, + "loss": 1.3489, + "theoretical_loss": 3.5241937050550285, + "tokens_seen": 1602748416 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041567903711133404, + "loss": 1.2575, + "theoretical_loss": 3.5241812620967865, + "tokens_seen": 1602813952 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004156629889669007, + "loss": 1.2989, + "theoretical_loss": 3.524168819789751, + "tokens_seen": 1602879488 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041564694082246744, + "loss": 1.2948, + "theoretical_loss": 3.5241563781338625, + "tokens_seen": 1602945024 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004156308926780341, + "loss": 1.3361, + "theoretical_loss": 3.5241439371290593, + "tokens_seen": 1603010560 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041561484453360084, + "loss": 1.2728, + "theoretical_loss": 3.524131496775281, + "tokens_seen": 1603076096 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041559879638916754, + "loss": 1.3738, + "theoretical_loss": 3.5241190570724665, + "tokens_seen": 1603141632 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041558274824473425, + "loss": 1.2662, + "theoretical_loss": 3.5241066180205562, + "tokens_seen": 1603207168 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041556670010030095, + "loss": 1.297, + "theoretical_loss": 3.524094179619489, + "tokens_seen": 1603272704 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041555065195586765, + "loss": 1.2055, + "theoretical_loss": 3.524081741869203, + "tokens_seen": 1603338240 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041553460381143435, + "loss": 1.2949, + "theoretical_loss": 3.5240693047696396, + "tokens_seen": 1603403776 + }, + { + "epoch": 0.49, + "learning_rate": 0.000415518555667001, + "loss": 1.357, + "theoretical_loss": 3.5240568683207365, + "tokens_seen": 1603469312 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041550250752256775, + "loss": 1.2265, + "theoretical_loss": 3.524044432522434, + "tokens_seen": 1603534848 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004154864593781344, + "loss": 1.3354, + "theoretical_loss": 3.524031997374671, + "tokens_seen": 1603600384 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041547041123370115, + "loss": 1.2541, + "theoretical_loss": 3.5240195628773874, + "tokens_seen": 1603665920 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004154543630892678, + "loss": 1.3173, + "theoretical_loss": 3.5240071290305224, + "tokens_seen": 1603731456 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041543831494483455, + "loss": 1.2055, + "theoretical_loss": 3.523994695834015, + "tokens_seen": 1603796992 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004154222668004012, + "loss": 1.2651, + "theoretical_loss": 3.523982263287805, + "tokens_seen": 1603862528 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041540621865596795, + "loss": 1.2806, + "theoretical_loss": 3.5239698313918315, + "tokens_seen": 1603928064 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004153901705115346, + "loss": 1.3639, + "theoretical_loss": 3.5239574001460348, + "tokens_seen": 1603993600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041537412236710135, + "loss": 1.2573, + "theoretical_loss": 3.5239449695503526, + "tokens_seen": 1604059136 + }, + { + "epoch": 0.49, + "learning_rate": 0.000415358074222668, + "loss": 1.2743, + "theoretical_loss": 3.523932539604726, + "tokens_seen": 1604124672 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041534202607823475, + "loss": 1.2824, + "theoretical_loss": 3.523920110309094, + "tokens_seen": 1604190208 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004153259779338014, + "loss": 1.1919, + "theoretical_loss": 3.5239076816633954, + "tokens_seen": 1604255744 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041530992978936816, + "loss": 1.3154, + "theoretical_loss": 3.523895253667571, + "tokens_seen": 1604321280 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004152938816449348, + "loss": 1.2457, + "theoretical_loss": 3.523882826321559, + "tokens_seen": 1604386816 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041527783350050156, + "loss": 1.2227, + "theoretical_loss": 3.523870399625299, + "tokens_seen": 1604452352 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004152617853560682, + "loss": 1.2721, + "theoretical_loss": 3.5238579735787314, + "tokens_seen": 1604517888 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041524573721163496, + "loss": 1.3266, + "theoretical_loss": 3.5238455481817947, + "tokens_seen": 1604583424 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004152296890672016, + "loss": 1.2273, + "theoretical_loss": 3.523833123434429, + "tokens_seen": 1604648960 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041521364092276836, + "loss": 1.2599, + "theoretical_loss": 3.5238206993365737, + "tokens_seen": 1604714496 + }, + { + "epoch": 0.49, + "learning_rate": 0.000415197592778335, + "loss": 1.3976, + "theoretical_loss": 3.5238082758881686, + "tokens_seen": 1604780032 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041518154463390176, + "loss": 1.2827, + "theoretical_loss": 3.5237958530891524, + "tokens_seen": 1604845568 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004151654964894684, + "loss": 1.3363, + "theoretical_loss": 3.5237834309394653, + "tokens_seen": 1604911104 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041514944834503516, + "loss": 1.2726, + "theoretical_loss": 3.523771009439047, + "tokens_seen": 1604976640 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004151334002006018, + "loss": 1.2827, + "theoretical_loss": 3.523758588587836, + "tokens_seen": 1605042176 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041511735205616856, + "loss": 1.2786, + "theoretical_loss": 3.5237461683857734, + "tokens_seen": 1605107712 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004151013039117352, + "loss": 1.2373, + "theoretical_loss": 3.5237337488327976, + "tokens_seen": 1605173248 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041508525576730196, + "loss": 1.2837, + "theoretical_loss": 3.523721329928849, + "tokens_seen": 1605238784 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004150692076228686, + "loss": 1.3342, + "theoretical_loss": 3.523708911673866, + "tokens_seen": 1605304320 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041505315947843537, + "loss": 1.2353, + "theoretical_loss": 3.523696494067789, + "tokens_seen": 1605369856 + }, + { + "epoch": 0.49, + "learning_rate": 0.000415037111334002, + "loss": 1.3092, + "theoretical_loss": 3.5236840771105578, + "tokens_seen": 1605435392 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041502106318956877, + "loss": 1.2923, + "theoretical_loss": 3.523671660802112, + "tokens_seen": 1605500928 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004150050150451354, + "loss": 1.2769, + "theoretical_loss": 3.5236592451423907, + "tokens_seen": 1605566464 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 538406, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2975478172302246, + "objective/train/theoretical_loss": 3.5236468301313337, + "objective/train/tokens_used": 1626092000, + "theoretical_loss": 3.5236468301313337, + "tokens_seen": 1605632000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041498896690070217, + "loss": 1.3045, + "theoretical_loss": 3.5236468301313337, + "tokens_seen": 1605632000 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004149729187562688, + "loss": 1.3784, + "theoretical_loss": 3.523634415768881, + "tokens_seen": 1605697536 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041495687061183557, + "loss": 1.2807, + "theoretical_loss": 3.523622002054972, + "tokens_seen": 1605763072 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004149408224674022, + "loss": 1.286, + "theoretical_loss": 3.5236095889895465, + "tokens_seen": 1605828608 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041492477432296897, + "loss": 1.3701, + "theoretical_loss": 3.523597176572544, + "tokens_seen": 1605894144 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004149087261785356, + "loss": 1.3172, + "theoretical_loss": 3.5235847648039034, + "tokens_seen": 1605959680 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004148926780341023, + "loss": 1.2887, + "theoretical_loss": 3.5235723536835657, + "tokens_seen": 1606025216 + }, + { + "epoch": 0.49, + "learning_rate": 0.000414876629889669, + "loss": 1.2816, + "theoretical_loss": 3.52355994321147, + "tokens_seen": 1606090752 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004148605817452357, + "loss": 1.2408, + "theoretical_loss": 3.523547533387556, + "tokens_seen": 1606156288 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004148445336008024, + "loss": 1.1945, + "theoretical_loss": 3.5235351242117634, + "tokens_seen": 1606221824 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004148284854563691, + "loss": 1.3582, + "theoretical_loss": 3.523522715684032, + "tokens_seen": 1606287360 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004148124373119358, + "loss": 1.3528, + "theoretical_loss": 3.523510307804301, + "tokens_seen": 1606352896 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004147963891675025, + "loss": 1.3063, + "theoretical_loss": 3.5234979005725116, + "tokens_seen": 1606418432 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004147803410230692, + "loss": 1.3183, + "theoretical_loss": 3.5234854939886016, + "tokens_seen": 1606483968 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004147642928786359, + "loss": 1.2798, + "theoretical_loss": 3.523473088052512, + "tokens_seen": 1606549504 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041474824473420257, + "loss": 1.3439, + "theoretical_loss": 3.523460682764182, + "tokens_seen": 1606615040 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004147321965897693, + "loss": 1.3044, + "theoretical_loss": 3.523448278123552, + "tokens_seen": 1606680576 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041471614844533597, + "loss": 1.2384, + "theoretical_loss": 3.523435874130561, + "tokens_seen": 1606746112 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004147001003009027, + "loss": 1.3326, + "theoretical_loss": 3.523423470785149, + "tokens_seen": 1606811648 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041468405215646937, + "loss": 1.3018, + "theoretical_loss": 3.523411068087256, + "tokens_seen": 1606877184 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004146680040120361, + "loss": 1.3179, + "theoretical_loss": 3.5233986660368215, + "tokens_seen": 1606942720 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004146519558676028, + "loss": 1.296, + "theoretical_loss": 3.5233862646337855, + "tokens_seen": 1607008256 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041463590772316953, + "loss": 1.3282, + "theoretical_loss": 3.523373863878088, + "tokens_seen": 1607073792 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004146198595787362, + "loss": 1.2659, + "theoretical_loss": 3.5233614637696684, + "tokens_seen": 1607139328 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041460381143430293, + "loss": 1.3368, + "theoretical_loss": 3.5233490643084666, + "tokens_seen": 1607204864 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004145877632898696, + "loss": 1.3547, + "theoretical_loss": 3.5233366654944227, + "tokens_seen": 1607270400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041457171514543633, + "loss": 1.3301, + "theoretical_loss": 3.5233242673274763, + "tokens_seen": 1607335936 + }, + { + "epoch": 0.49, + "learning_rate": 0.000414555667001003, + "loss": 1.2426, + "theoretical_loss": 3.5233118698075674, + "tokens_seen": 1607401472 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041453961885656973, + "loss": 1.3547, + "theoretical_loss": 3.523299472934636, + "tokens_seen": 1607467008 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004145235707121364, + "loss": 1.2979, + "theoretical_loss": 3.5232870767086215, + "tokens_seen": 1607532544 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041450752256770313, + "loss": 1.3082, + "theoretical_loss": 3.5232746811294637, + "tokens_seen": 1607598080 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004144914744232698, + "loss": 1.3847, + "theoretical_loss": 3.5232622861971032, + "tokens_seen": 1607663616 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041447542627883653, + "loss": 1.3044, + "theoretical_loss": 3.5232498919114787, + "tokens_seen": 1607729152 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004144593781344032, + "loss": 1.2915, + "theoretical_loss": 3.5232374982725316, + "tokens_seen": 1607794688 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041444332998996994, + "loss": 1.3189, + "theoretical_loss": 3.523225105280201, + "tokens_seen": 1607860224 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004144272818455366, + "loss": 1.3514, + "theoretical_loss": 3.523212712934427, + "tokens_seen": 1607925760 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041441123370110334, + "loss": 1.2246, + "theoretical_loss": 3.523200321235149, + "tokens_seen": 1607991296 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004143951855566701, + "loss": 1.3455, + "theoretical_loss": 3.5231879301823072, + "tokens_seen": 1608056832 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041437913741223674, + "loss": 1.2461, + "theoretical_loss": 3.523175539775842, + "tokens_seen": 1608122368 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004143630892678035, + "loss": 1.1826, + "theoretical_loss": 3.523163150015693, + "tokens_seen": 1608187904 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041434704112337014, + "loss": 1.2882, + "theoretical_loss": 3.5231507609018, + "tokens_seen": 1608253440 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004143309929789369, + "loss": 1.3647, + "theoretical_loss": 3.5231383724341034, + "tokens_seen": 1608318976 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041431494483450354, + "loss": 1.4542, + "theoretical_loss": 3.523125984612542, + "tokens_seen": 1608384512 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004142988966900703, + "loss": 1.4548, + "theoretical_loss": 3.5231135974370575, + "tokens_seen": 1608450048 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041428284854563694, + "loss": 1.223, + "theoretical_loss": 3.5231012109075888, + "tokens_seen": 1608515584 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041426680040120364, + "loss": 1.245, + "theoretical_loss": 3.523088825024076, + "tokens_seen": 1608581120 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041425075225677034, + "loss": 1.2947, + "theoretical_loss": 3.5230764397864593, + "tokens_seen": 1608646656 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041423470411233704, + "loss": 1.1774, + "theoretical_loss": 3.523064055194679, + "tokens_seen": 1608712192 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041421865596790374, + "loss": 1.3004, + "theoretical_loss": 3.523051671248674, + "tokens_seen": 1608777728 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041420260782347045, + "loss": 1.2679, + "theoretical_loss": 3.5230392879483854, + "tokens_seen": 1608843264 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 539279, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.215774416923523, + "objective/train/theoretical_loss": 3.5230269052937526, + "objective/train/tokens_used": 1629368800, + "theoretical_loss": 3.5230269052937526, + "tokens_seen": 1608908800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041418655967903715, + "loss": 1.2308, + "theoretical_loss": 3.5230269052937526, + "tokens_seen": 1608908800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041417051153460385, + "loss": 1.3234, + "theoretical_loss": 3.5230145232847163, + "tokens_seen": 1608974336 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041415446339017055, + "loss": 1.444, + "theoretical_loss": 3.523002141921216, + "tokens_seen": 1609039872 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041413841524573725, + "loss": 1.275, + "theoretical_loss": 3.522989761203192, + "tokens_seen": 1609105408 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004141223671013039, + "loss": 1.3311, + "theoretical_loss": 3.522977381130584, + "tokens_seen": 1609170944 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041410631895687065, + "loss": 1.3435, + "theoretical_loss": 3.5229650017033327, + "tokens_seen": 1609236480 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004140902708124373, + "loss": 1.2941, + "theoretical_loss": 3.5229526229213777, + "tokens_seen": 1609302016 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041407422266800405, + "loss": 1.2829, + "theoretical_loss": 3.5229402447846594, + "tokens_seen": 1609367552 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004140581745235707, + "loss": 1.2612, + "theoretical_loss": 3.5229278672931175, + "tokens_seen": 1609433088 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041404212637913745, + "loss": 1.2628, + "theoretical_loss": 3.5229154904466924, + "tokens_seen": 1609498624 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004140260782347041, + "loss": 1.2724, + "theoretical_loss": 3.522903114245324, + "tokens_seen": 1609564160 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041401003009027085, + "loss": 1.3474, + "theoretical_loss": 3.522890738688953, + "tokens_seen": 1609629696 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004139939819458375, + "loss": 1.2274, + "theoretical_loss": 3.5228783637775183, + "tokens_seen": 1609695232 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041397793380140425, + "loss": 1.3432, + "theoretical_loss": 3.5228659895109615, + "tokens_seen": 1609760768 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004139618856569709, + "loss": 1.2198, + "theoretical_loss": 3.522853615889222, + "tokens_seen": 1609826304 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041394583751253766, + "loss": 1.3051, + "theoretical_loss": 3.5228412429122393, + "tokens_seen": 1609891840 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004139297893681043, + "loss": 1.3384, + "theoretical_loss": 3.5228288705799544, + "tokens_seen": 1609957376 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041391374122367106, + "loss": 1.2414, + "theoretical_loss": 3.5228164988923076, + "tokens_seen": 1610022912 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004138976930792377, + "loss": 1.364, + "theoretical_loss": 3.5228041278492395, + "tokens_seen": 1610088448 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041388164493480446, + "loss": 1.3697, + "theoretical_loss": 3.5227917574506886, + "tokens_seen": 1610153984 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004138655967903711, + "loss": 1.2494, + "theoretical_loss": 3.5227793876965965, + "tokens_seen": 1610219520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041384954864593786, + "loss": 1.2935, + "theoretical_loss": 3.5227670185869027, + "tokens_seen": 1610285056 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004138335005015045, + "loss": 1.3039, + "theoretical_loss": 3.5227546501215476, + "tokens_seen": 1610350592 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041381745235707126, + "loss": 1.3171, + "theoretical_loss": 3.522742282300472, + "tokens_seen": 1610416128 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004138014042126379, + "loss": 1.1883, + "theoretical_loss": 3.522729915123615, + "tokens_seen": 1610481664 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041378535606820466, + "loss": 1.2982, + "theoretical_loss": 3.522717548590918, + "tokens_seen": 1610547200 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004137693079237713, + "loss": 1.2177, + "theoretical_loss": 3.5227051827023206, + "tokens_seen": 1610612736 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041375325977933806, + "loss": 1.3035, + "theoretical_loss": 3.522692817457763, + "tokens_seen": 1610678272 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004137372116349047, + "loss": 1.269, + "theoretical_loss": 3.522680452857186, + "tokens_seen": 1610743808 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041372116349047146, + "loss": 1.3868, + "theoretical_loss": 3.522668088900528, + "tokens_seen": 1610809344 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004137051153460381, + "loss": 1.3411, + "theoretical_loss": 3.5226557255877324, + "tokens_seen": 1610874880 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041368906720160487, + "loss": 1.2988, + "theoretical_loss": 3.5226433629187373, + "tokens_seen": 1610940416 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004136730190571715, + "loss": 1.1887, + "theoretical_loss": 3.522631000893483, + "tokens_seen": 1611005952 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041365697091273827, + "loss": 1.202, + "theoretical_loss": 3.5226186395119106, + "tokens_seen": 1611071488 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004136409227683049, + "loss": 1.2691, + "theoretical_loss": 3.52260627877396, + "tokens_seen": 1611137024 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041362487462387167, + "loss": 1.2519, + "theoretical_loss": 3.5225939186795716, + "tokens_seen": 1611202560 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004136088264794383, + "loss": 1.3052, + "theoretical_loss": 3.5225815592286853, + "tokens_seen": 1611268096 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041359277833500507, + "loss": 1.3524, + "theoretical_loss": 3.5225692004212426, + "tokens_seen": 1611333632 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004135767301905717, + "loss": 1.1798, + "theoretical_loss": 3.522556842257183, + "tokens_seen": 1611399168 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041356068204613847, + "loss": 1.2865, + "theoretical_loss": 3.5225444847364464, + "tokens_seen": 1611464704 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004135446339017051, + "loss": 1.3008, + "theoretical_loss": 3.5225321278589736, + "tokens_seen": 1611530240 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041352858575727187, + "loss": 1.3017, + "theoretical_loss": 3.5225197716247054, + "tokens_seen": 1611595776 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004135125376128385, + "loss": 1.3895, + "theoretical_loss": 3.522507416033582, + "tokens_seen": 1611661312 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004134964894684052, + "loss": 1.2815, + "theoretical_loss": 3.522495061085543, + "tokens_seen": 1611726848 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004134804413239719, + "loss": 1.3243, + "theoretical_loss": 3.5224827067805293, + "tokens_seen": 1611792384 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004134643931795386, + "loss": 1.3513, + "theoretical_loss": 3.5224703531184813, + "tokens_seen": 1611857920 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004134483450351053, + "loss": 1.3026, + "theoretical_loss": 3.5224580000993395, + "tokens_seen": 1611923456 + }, + { + "epoch": 0.49, + "learning_rate": 0.000413432296890672, + "loss": 1.3368, + "theoretical_loss": 3.522445647723045, + "tokens_seen": 1611988992 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004134162487462387, + "loss": 1.2917, + "theoretical_loss": 3.5224332959895364, + "tokens_seen": 1612054528 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004134002006018054, + "loss": 1.3536, + "theoretical_loss": 3.522420944898756, + "tokens_seen": 1612120064 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 540479, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2837307453155518, + "objective/train/theoretical_loss": 3.522408594450643, + "objective/train/tokens_used": 1632645600, + "theoretical_loss": 3.522408594450643, + "tokens_seen": 1612185600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004133841524573721, + "loss": 1.3376, + "theoretical_loss": 3.522408594450643, + "tokens_seen": 1612185600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004133681043129388, + "loss": 1.263, + "theoretical_loss": 3.5223962446451376, + "tokens_seen": 1612251136 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041335205616850547, + "loss": 1.2734, + "theoretical_loss": 3.522383895482182, + "tokens_seen": 1612316672 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004133360080240722, + "loss": 1.382, + "theoretical_loss": 3.5223715469617147, + "tokens_seen": 1612382208 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041331995987963887, + "loss": 1.2277, + "theoretical_loss": 3.5223591990836773, + "tokens_seen": 1612447744 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004133039117352056, + "loss": 1.3789, + "theoretical_loss": 3.5223468518480097, + "tokens_seen": 1612513280 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004132878635907723, + "loss": 1.2571, + "theoretical_loss": 3.522334505254653, + "tokens_seen": 1612578816 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041327181544633903, + "loss": 1.2769, + "theoretical_loss": 3.522322159303547, + "tokens_seen": 1612644352 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004132557673019057, + "loss": 1.3262, + "theoretical_loss": 3.5223098139946325, + "tokens_seen": 1612709888 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041323971915747243, + "loss": 1.3533, + "theoretical_loss": 3.5222974693278504, + "tokens_seen": 1612775424 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004132236710130391, + "loss": 1.1954, + "theoretical_loss": 3.5222851253031404, + "tokens_seen": 1612840960 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041320762286860583, + "loss": 1.3715, + "theoretical_loss": 3.522272781920444, + "tokens_seen": 1612906496 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004131915747241726, + "loss": 1.3869, + "theoretical_loss": 3.5222604391797008, + "tokens_seen": 1612972032 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041317552657973923, + "loss": 1.4016, + "theoretical_loss": 3.5222480970808516, + "tokens_seen": 1613037568 + }, + { + "epoch": 0.49, + "learning_rate": 0.000413159478435306, + "loss": 1.3609, + "theoretical_loss": 3.522235755623837, + "tokens_seen": 1613103104 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041314343029087263, + "loss": 1.2471, + "theoretical_loss": 3.5222234148085976, + "tokens_seen": 1613168640 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004131273821464394, + "loss": 1.3479, + "theoretical_loss": 3.522211074635074, + "tokens_seen": 1613234176 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041311133400200603, + "loss": 1.4156, + "theoretical_loss": 3.5221987351032067, + "tokens_seen": 1613299712 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004130952858575728, + "loss": 1.2331, + "theoretical_loss": 3.5221863962129367, + "tokens_seen": 1613365248 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041307923771313944, + "loss": 1.2116, + "theoretical_loss": 3.5221740579642034, + "tokens_seen": 1613430784 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004130631895687062, + "loss": 1.2599, + "theoretical_loss": 3.5221617203569484, + "tokens_seen": 1613496320 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041304714142427284, + "loss": 1.3422, + "theoretical_loss": 3.5221493833911124, + "tokens_seen": 1613561856 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004130310932798396, + "loss": 1.1559, + "theoretical_loss": 3.522137047066635, + "tokens_seen": 1613627392 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041301504513540624, + "loss": 1.2957, + "theoretical_loss": 3.5221247113834577, + "tokens_seen": 1613692928 + }, + { + "epoch": 0.49, + "learning_rate": 0.000412998996990973, + "loss": 1.2715, + "theoretical_loss": 3.5221123763415214, + "tokens_seen": 1613758464 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041298294884653964, + "loss": 1.3064, + "theoretical_loss": 3.5221000419407655, + "tokens_seen": 1613824000 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004129669007021064, + "loss": 1.2881, + "theoretical_loss": 3.5220877081811315, + "tokens_seen": 1613889536 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041295085255767304, + "loss": 1.3578, + "theoretical_loss": 3.52207537506256, + "tokens_seen": 1613955072 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004129348044132398, + "loss": 1.3039, + "theoretical_loss": 3.5220630425849917, + "tokens_seen": 1614020608 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041291875626880644, + "loss": 1.3073, + "theoretical_loss": 3.522050710748367, + "tokens_seen": 1614086144 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004129027081243732, + "loss": 1.3265, + "theoretical_loss": 3.522038379552626, + "tokens_seen": 1614151680 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041288665997993984, + "loss": 1.3409, + "theoretical_loss": 3.522026048997711, + "tokens_seen": 1614217216 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004128706118355066, + "loss": 1.273, + "theoretical_loss": 3.522013719083561, + "tokens_seen": 1614282752 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041285456369107324, + "loss": 1.2903, + "theoretical_loss": 3.522001389810118, + "tokens_seen": 1614348288 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041283851554663995, + "loss": 1.3074, + "theoretical_loss": 3.5219890611773215, + "tokens_seen": 1614413824 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041282246740220665, + "loss": 1.4072, + "theoretical_loss": 3.521976733185113, + "tokens_seen": 1614479360 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041280641925777335, + "loss": 1.2841, + "theoretical_loss": 3.521964405833433, + "tokens_seen": 1614544896 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041279037111334005, + "loss": 1.4054, + "theoretical_loss": 3.5219520791222223, + "tokens_seen": 1614610432 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041277432296890675, + "loss": 1.3945, + "theoretical_loss": 3.521939753051422, + "tokens_seen": 1614675968 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041275827482447345, + "loss": 1.3681, + "theoretical_loss": 3.521927427620972, + "tokens_seen": 1614741504 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041274222668004015, + "loss": 1.3696, + "theoretical_loss": 3.521915102830813, + "tokens_seen": 1614807040 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004127261785356068, + "loss": 1.3058, + "theoretical_loss": 3.521902778680887, + "tokens_seen": 1614872576 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041271013039117355, + "loss": 1.2813, + "theoretical_loss": 3.5218904551711336, + "tokens_seen": 1614938112 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004126940822467402, + "loss": 1.3238, + "theoretical_loss": 3.5218781323014943, + "tokens_seen": 1615003648 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041267803410230695, + "loss": 1.4571, + "theoretical_loss": 3.521865810071909, + "tokens_seen": 1615069184 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004126619859578736, + "loss": 1.3464, + "theoretical_loss": 3.521853488482319, + "tokens_seen": 1615134720 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041264593781344035, + "loss": 1.2765, + "theoretical_loss": 3.5218411675326653, + "tokens_seen": 1615200256 + }, + { + "epoch": 0.49, + "learning_rate": 0.000412629889669007, + "loss": 1.3163, + "theoretical_loss": 3.5218288472228885, + "tokens_seen": 1615265792 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041261384152457375, + "loss": 1.349, + "theoretical_loss": 3.5218165275529296, + "tokens_seen": 1615331328 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004125977933801404, + "loss": 1.3393, + "theoretical_loss": 3.5218042085227292, + "tokens_seen": 1615396864 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 541462, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3924484252929688, + "objective/train/theoretical_loss": 3.521791890132228, + "objective/train/tokens_used": 1635922400, + "theoretical_loss": 3.521791890132228, + "tokens_seen": 1615462400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041258174523570716, + "loss": 1.3301, + "theoretical_loss": 3.521791890132228, + "tokens_seen": 1615462400 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004125656970912738, + "loss": 1.2946, + "theoretical_loss": 3.5217795723813667, + "tokens_seen": 1615527936 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041254964894684056, + "loss": 1.2791, + "theoretical_loss": 3.521767255270087, + "tokens_seen": 1615593472 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004125336008024072, + "loss": 1.2704, + "theoretical_loss": 3.521754938798329, + "tokens_seen": 1615659008 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041251755265797396, + "loss": 1.2761, + "theoretical_loss": 3.521742622966033, + "tokens_seen": 1615724544 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004125015045135406, + "loss": 1.2553, + "theoretical_loss": 3.521730307773142, + "tokens_seen": 1615790080 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041248545636910736, + "loss": 1.3563, + "theoretical_loss": 3.5217179932195943, + "tokens_seen": 1615855616 + }, + { + "epoch": 0.49, + "learning_rate": 0.000412469408224674, + "loss": 1.3129, + "theoretical_loss": 3.521705679305332, + "tokens_seen": 1615921152 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041245336008024076, + "loss": 1.3465, + "theoretical_loss": 3.5216933660302967, + "tokens_seen": 1615986688 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004124373119358074, + "loss": 1.2953, + "theoretical_loss": 3.5216810533944276, + "tokens_seen": 1616052224 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041242126379137416, + "loss": 1.3918, + "theoretical_loss": 3.521668741397667, + "tokens_seen": 1616117760 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004124052156469408, + "loss": 1.3707, + "theoretical_loss": 3.5216564300399558, + "tokens_seen": 1616183296 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041238916750250756, + "loss": 1.3014, + "theoretical_loss": 3.5216441193212336, + "tokens_seen": 1616248832 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004123731193580742, + "loss": 1.2844, + "theoretical_loss": 3.5216318092414425, + "tokens_seen": 1616314368 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041235707121364096, + "loss": 1.2968, + "theoretical_loss": 3.5216194998005235, + "tokens_seen": 1616379904 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004123410230692076, + "loss": 1.2852, + "theoretical_loss": 3.5216071909984166, + "tokens_seen": 1616445440 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041232497492477437, + "loss": 1.2701, + "theoretical_loss": 3.5215948828350636, + "tokens_seen": 1616510976 + }, + { + "epoch": 0.49, + "learning_rate": 0.000412308926780341, + "loss": 1.306, + "theoretical_loss": 3.521582575310405, + "tokens_seen": 1616576512 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041229287863590777, + "loss": 1.3116, + "theoretical_loss": 3.521570268424382, + "tokens_seen": 1616642048 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004122768304914744, + "loss": 1.2724, + "theoretical_loss": 3.521557962176936, + "tokens_seen": 1616707584 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041226078234704117, + "loss": 1.2804, + "theoretical_loss": 3.5215456565680068, + "tokens_seen": 1616773120 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004122447342026078, + "loss": 1.232, + "theoretical_loss": 3.5215333515975367, + "tokens_seen": 1616838656 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041222868605817457, + "loss": 1.3332, + "theoretical_loss": 3.521521047265465, + "tokens_seen": 1616904192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004122126379137412, + "loss": 1.2532, + "theoretical_loss": 3.521508743571735, + "tokens_seen": 1616969728 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041219658976930797, + "loss": 1.2531, + "theoretical_loss": 3.521496440516286, + "tokens_seen": 1617035264 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004121805416248746, + "loss": 1.3661, + "theoretical_loss": 3.5214841380990594, + "tokens_seen": 1617100800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041216449348044137, + "loss": 1.3272, + "theoretical_loss": 3.5214718363199964, + "tokens_seen": 1617166336 + }, + { + "epoch": 0.49, + "learning_rate": 0.000412148445336008, + "loss": 1.2134, + "theoretical_loss": 3.521459535179038, + "tokens_seen": 1617231872 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004121323971915748, + "loss": 1.1857, + "theoretical_loss": 3.521447234676126, + "tokens_seen": 1617297408 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004121163490471414, + "loss": 1.4602, + "theoretical_loss": 3.5214349348111997, + "tokens_seen": 1617362944 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004121003009027082, + "loss": 1.303, + "theoretical_loss": 3.5214226355842015, + "tokens_seen": 1617428480 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004120842527582748, + "loss": 1.3819, + "theoretical_loss": 3.521410336995072, + "tokens_seen": 1617494016 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004120682046138415, + "loss": 1.3027, + "theoretical_loss": 3.5213980390437527, + "tokens_seen": 1617559552 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004120521564694082, + "loss": 1.2673, + "theoretical_loss": 3.521385741730184, + "tokens_seen": 1617625088 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004120361083249749, + "loss": 1.3548, + "theoretical_loss": 3.5213734450543077, + "tokens_seen": 1617690624 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004120200601805416, + "loss": 1.2943, + "theoretical_loss": 3.521361149016064, + "tokens_seen": 1617756160 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004120040120361083, + "loss": 1.3091, + "theoretical_loss": 3.521348853615395, + "tokens_seen": 1617821696 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004119879638916751, + "loss": 1.354, + "theoretical_loss": 3.5213365588522416, + "tokens_seen": 1617887232 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004119719157472417, + "loss": 1.3145, + "theoretical_loss": 3.5213242647265446, + "tokens_seen": 1617952768 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004119558676028085, + "loss": 1.2963, + "theoretical_loss": 3.521311971238245, + "tokens_seen": 1618018304 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004119398194583751, + "loss": 1.321, + "theoretical_loss": 3.521299678387284, + "tokens_seen": 1618083840 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004119237713139419, + "loss": 1.3095, + "theoretical_loss": 3.5212873861736034, + "tokens_seen": 1618149376 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041190772316950853, + "loss": 1.2735, + "theoretical_loss": 3.5212750945971436, + "tokens_seen": 1618214912 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004118916750250753, + "loss": 1.367, + "theoretical_loss": 3.521262803657846, + "tokens_seen": 1618280448 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041187562688064193, + "loss": 1.2877, + "theoretical_loss": 3.521250513355652, + "tokens_seen": 1618345984 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004118595787362087, + "loss": 1.306, + "theoretical_loss": 3.5212382236905024, + "tokens_seen": 1618411520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041184353059177533, + "loss": 1.2761, + "theoretical_loss": 3.521225934662339, + "tokens_seen": 1618477056 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004118274824473421, + "loss": 1.327, + "theoretical_loss": 3.521213646271102, + "tokens_seen": 1618542592 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041181143430290873, + "loss": 1.3305, + "theoretical_loss": 3.521201358516733, + "tokens_seen": 1618608128 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004117953861584755, + "loss": 1.2317, + "theoretical_loss": 3.5211890713991743, + "tokens_seen": 1618673664 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 542790, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.221559762954712, + "objective/train/theoretical_loss": 3.5211767849183655, + "objective/train/tokens_used": 1639199200, + "theoretical_loss": 3.5211767849183655, + "tokens_seen": 1618739200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041177933801404213, + "loss": 1.2886, + "theoretical_loss": 3.5211767849183655, + "tokens_seen": 1618739200 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004117632898696089, + "loss": 1.3146, + "theoretical_loss": 3.521164499074249, + "tokens_seen": 1618804736 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041174724172517553, + "loss": 1.2495, + "theoretical_loss": 3.521152213866765, + "tokens_seen": 1618870272 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004117311935807423, + "loss": 1.2029, + "theoretical_loss": 3.521139929295856, + "tokens_seen": 1618935808 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041171514543630894, + "loss": 1.2901, + "theoretical_loss": 3.521127645361462, + "tokens_seen": 1619001344 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004116990972918757, + "loss": 1.3533, + "theoretical_loss": 3.521115362063525, + "tokens_seen": 1619066880 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041168304914744234, + "loss": 1.3077, + "theoretical_loss": 3.5211030794019855, + "tokens_seen": 1619132416 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004116670010030091, + "loss": 1.4194, + "theoretical_loss": 3.521090797376786, + "tokens_seen": 1619197952 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041165095285857574, + "loss": 1.292, + "theoretical_loss": 3.521078515987867, + "tokens_seen": 1619263488 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004116349047141425, + "loss": 1.337, + "theoretical_loss": 3.5210662352351694, + "tokens_seen": 1619329024 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041161885656970914, + "loss": 1.2442, + "theoretical_loss": 3.521053955118636, + "tokens_seen": 1619394560 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004116028084252759, + "loss": 1.4051, + "theoretical_loss": 3.521041675638206, + "tokens_seen": 1619460096 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041158676028084254, + "loss": 1.329, + "theoretical_loss": 3.521029396793822, + "tokens_seen": 1619525632 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004115707121364093, + "loss": 1.345, + "theoretical_loss": 3.521017118585426, + "tokens_seen": 1619591168 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041155466399197594, + "loss": 1.2474, + "theoretical_loss": 3.5210048410129575, + "tokens_seen": 1619656704 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004115386158475427, + "loss": 1.294, + "theoretical_loss": 3.5209925640763595, + "tokens_seen": 1619722240 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041152256770310934, + "loss": 1.3307, + "theoretical_loss": 3.520980287775572, + "tokens_seen": 1619787776 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004115065195586761, + "loss": 1.2483, + "theoretical_loss": 3.5209680121105373, + "tokens_seen": 1619853312 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041149047141424274, + "loss": 1.2847, + "theoretical_loss": 3.520955737081197, + "tokens_seen": 1619918848 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004114744232698095, + "loss": 1.4058, + "theoretical_loss": 3.520943462687491, + "tokens_seen": 1619984384 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041145837512537615, + "loss": 1.2474, + "theoretical_loss": 3.5209311889293615, + "tokens_seen": 1620049920 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041144232698094285, + "loss": 1.3477, + "theoretical_loss": 3.5209189158067504, + "tokens_seen": 1620115456 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041142627883650955, + "loss": 1.2323, + "theoretical_loss": 3.5209066433195986, + "tokens_seen": 1620180992 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041141023069207625, + "loss": 1.3001, + "theoretical_loss": 3.5208943714678473, + "tokens_seen": 1620246528 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041139418254764295, + "loss": 1.3605, + "theoretical_loss": 3.5208821002514386, + "tokens_seen": 1620312064 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041137813440320965, + "loss": 1.3282, + "theoretical_loss": 3.520869829670313, + "tokens_seen": 1620377600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041136208625877635, + "loss": 1.2522, + "theoretical_loss": 3.5208575597244125, + "tokens_seen": 1620443136 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041134603811434305, + "loss": 1.2853, + "theoretical_loss": 3.5208452904136784, + "tokens_seen": 1620508672 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041132998996990975, + "loss": 1.2942, + "theoretical_loss": 3.520833021738052, + "tokens_seen": 1620574208 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041131394182547645, + "loss": 1.3175, + "theoretical_loss": 3.5208207536974747, + "tokens_seen": 1620639744 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004112978936810431, + "loss": 1.3737, + "theoretical_loss": 3.520808486291889, + "tokens_seen": 1620705280 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041128184553660985, + "loss": 1.1986, + "theoretical_loss": 3.520796219521235, + "tokens_seen": 1620770816 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004112657973921765, + "loss": 1.2954, + "theoretical_loss": 3.520783953385454, + "tokens_seen": 1620836352 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041124974924774325, + "loss": 1.2996, + "theoretical_loss": 3.520771687884489, + "tokens_seen": 1620901888 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004112337011033099, + "loss": 1.322, + "theoretical_loss": 3.52075942301828, + "tokens_seen": 1620967424 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041121765295887666, + "loss": 1.397, + "theoretical_loss": 3.5207471587867696, + "tokens_seen": 1621032960 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004112016048144433, + "loss": 1.2729, + "theoretical_loss": 3.5207348951898982, + "tokens_seen": 1621098496 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041118555667001006, + "loss": 1.3236, + "theoretical_loss": 3.520722632227608, + "tokens_seen": 1621164032 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004111695085255767, + "loss": 1.3361, + "theoretical_loss": 3.520710369899841, + "tokens_seen": 1621229568 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041115346038114346, + "loss": 1.3386, + "theoretical_loss": 3.520698108206538, + "tokens_seen": 1621295104 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004111374122367101, + "loss": 1.3073, + "theoretical_loss": 3.5206858471476403, + "tokens_seen": 1621360640 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041112136409227686, + "loss": 1.434, + "theoretical_loss": 3.52067358672309, + "tokens_seen": 1621426176 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004111053159478435, + "loss": 1.3304, + "theoretical_loss": 3.520661326932828, + "tokens_seen": 1621491712 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041108926780341026, + "loss": 1.2513, + "theoretical_loss": 3.5206490677767968, + "tokens_seen": 1621557248 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004110732196589769, + "loss": 1.2836, + "theoretical_loss": 3.5206368092549374, + "tokens_seen": 1621622784 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041105717151454366, + "loss": 1.2449, + "theoretical_loss": 3.5206245513671908, + "tokens_seen": 1621688320 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004110411233701103, + "loss": 1.2701, + "theoretical_loss": 3.5206122941134996, + "tokens_seen": 1621753856 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041102507522567706, + "loss": 1.33, + "theoretical_loss": 3.5206000374938053, + "tokens_seen": 1621819392 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004110090270812437, + "loss": 1.3547, + "theoretical_loss": 3.520587781508049, + "tokens_seen": 1621884928 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041099297893681046, + "loss": 1.3295, + "theoretical_loss": 3.520575526156172, + "tokens_seen": 1621950464 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 543728, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1965467929840088, + "objective/train/theoretical_loss": 3.5205632714381165, + "objective/train/tokens_used": 1642476000, + "theoretical_loss": 3.5205632714381165, + "tokens_seen": 1622016000 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004109769307923771, + "loss": 1.1936, + "theoretical_loss": 3.5205632714381165, + "tokens_seen": 1622016000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041096088264794387, + "loss": 1.2342, + "theoretical_loss": 3.5205510173538244, + "tokens_seen": 1622081536 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004109448345035105, + "loss": 1.3057, + "theoretical_loss": 3.520538763903237, + "tokens_seen": 1622147072 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041092878635907727, + "loss": 1.301, + "theoretical_loss": 3.5205265110862953, + "tokens_seen": 1622212608 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004109127382146439, + "loss": 1.3495, + "theoretical_loss": 3.5205142589029412, + "tokens_seen": 1622278144 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041089669007021067, + "loss": 1.3808, + "theoretical_loss": 3.5205020073531172, + "tokens_seen": 1622343680 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004108806419257773, + "loss": 1.3109, + "theoretical_loss": 3.5204897564367643, + "tokens_seen": 1622409216 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041086459378134407, + "loss": 1.265, + "theoretical_loss": 3.520477506153824, + "tokens_seen": 1622474752 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004108485456369107, + "loss": 1.3811, + "theoretical_loss": 3.520465256504238, + "tokens_seen": 1622540288 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041083249749247747, + "loss": 1.284, + "theoretical_loss": 3.5204530074879488, + "tokens_seen": 1622605824 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004108164493480441, + "loss": 1.2275, + "theoretical_loss": 3.520440759104897, + "tokens_seen": 1622671360 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041080040120361087, + "loss": 1.3408, + "theoretical_loss": 3.5204285113550244, + "tokens_seen": 1622736896 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041078435305917757, + "loss": 1.3113, + "theoretical_loss": 3.5204162642382735, + "tokens_seen": 1622802432 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041076830491474427, + "loss": 1.2345, + "theoretical_loss": 3.5204040177545854, + "tokens_seen": 1622867968 + }, + { + "epoch": 0.49, + "learning_rate": 0.000410752256770311, + "loss": 1.3309, + "theoretical_loss": 3.5203917719039017, + "tokens_seen": 1622933504 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004107362086258777, + "loss": 1.3104, + "theoretical_loss": 3.520379526686165, + "tokens_seen": 1622999040 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004107201604814444, + "loss": 1.3521, + "theoretical_loss": 3.520367282101316, + "tokens_seen": 1623064576 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004107041123370111, + "loss": 1.2375, + "theoretical_loss": 3.5203550381492965, + "tokens_seen": 1623130112 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004106880641925778, + "loss": 1.367, + "theoretical_loss": 3.520342794830049, + "tokens_seen": 1623195648 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004106720160481444, + "loss": 1.36, + "theoretical_loss": 3.520330552143515, + "tokens_seen": 1623261184 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004106559679037112, + "loss": 1.3016, + "theoretical_loss": 3.520318310089636, + "tokens_seen": 1623326720 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004106399197592778, + "loss": 1.2955, + "theoretical_loss": 3.5203060686683534, + "tokens_seen": 1623392256 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004106238716148446, + "loss": 1.3719, + "theoretical_loss": 3.52029382787961, + "tokens_seen": 1623457792 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004106078234704112, + "loss": 1.2813, + "theoretical_loss": 3.5202815877233467, + "tokens_seen": 1623523328 + }, + { + "epoch": 0.49, + "learning_rate": 0.000410591775325978, + "loss": 1.2415, + "theoretical_loss": 3.5202693481995055, + "tokens_seen": 1623588864 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004105757271815446, + "loss": 1.2761, + "theoretical_loss": 3.5202571093080284, + "tokens_seen": 1623654400 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004105596790371114, + "loss": 1.2857, + "theoretical_loss": 3.520244871048857, + "tokens_seen": 1623719936 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041054363089267803, + "loss": 1.3139, + "theoretical_loss": 3.5202326334219336, + "tokens_seen": 1623785472 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004105275827482448, + "loss": 1.2739, + "theoretical_loss": 3.520220396427199, + "tokens_seen": 1623851008 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041051153460381143, + "loss": 1.2938, + "theoretical_loss": 3.5202081600645965, + "tokens_seen": 1623916544 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004104954864593782, + "loss": 1.2656, + "theoretical_loss": 3.520195924334067, + "tokens_seen": 1623982080 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041047943831494483, + "loss": 1.3287, + "theoretical_loss": 3.5201836892355516, + "tokens_seen": 1624047616 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004104633901705116, + "loss": 1.315, + "theoretical_loss": 3.5201714547689935, + "tokens_seen": 1624113152 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041044734202607823, + "loss": 1.3726, + "theoretical_loss": 3.5201592209343344, + "tokens_seen": 1624178688 + }, + { + "epoch": 0.49, + "learning_rate": 0.000410431293881645, + "loss": 1.2545, + "theoretical_loss": 3.5201469877315157, + "tokens_seen": 1624244224 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041041524573721163, + "loss": 1.3051, + "theoretical_loss": 3.520134755160479, + "tokens_seen": 1624309760 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004103991975927784, + "loss": 1.297, + "theoretical_loss": 3.520122523221167, + "tokens_seen": 1624375296 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041038314944834503, + "loss": 1.3264, + "theoretical_loss": 3.520110291913521, + "tokens_seen": 1624440832 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004103671013039118, + "loss": 1.2975, + "theoretical_loss": 3.5200980612374835, + "tokens_seen": 1624506368 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041035105315947844, + "loss": 1.3084, + "theoretical_loss": 3.5200858311929952, + "tokens_seen": 1624571904 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004103350050150452, + "loss": 1.3527, + "theoretical_loss": 3.520073601779999, + "tokens_seen": 1624637440 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041031895687061184, + "loss": 1.203, + "theoretical_loss": 3.5200613729984376, + "tokens_seen": 1624702976 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004103029087261786, + "loss": 1.2778, + "theoretical_loss": 3.520049144848251, + "tokens_seen": 1624768512 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041028686058174524, + "loss": 1.3016, + "theoretical_loss": 3.5200369173293824, + "tokens_seen": 1624834048 + }, + { + "epoch": 0.49, + "learning_rate": 0.000410270812437312, + "loss": 1.2638, + "theoretical_loss": 3.5200246904417734, + "tokens_seen": 1624899584 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041025476429287864, + "loss": 1.2902, + "theoretical_loss": 3.5200124641853656, + "tokens_seen": 1624965120 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004102387161484454, + "loss": 1.3101, + "theoretical_loss": 3.520000238560102, + "tokens_seen": 1625030656 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041022266800401204, + "loss": 1.2553, + "theoretical_loss": 3.519988013565923, + "tokens_seen": 1625096192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004102066198595788, + "loss": 1.2852, + "theoretical_loss": 3.5199757892027725, + "tokens_seen": 1625161728 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041019057171514544, + "loss": 1.3299, + "theoretical_loss": 3.519963565470591, + "tokens_seen": 1625227264 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 545041, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.421590805053711, + "objective/train/theoretical_loss": 3.519951342369321, + "objective/train/tokens_used": 1645752800, + "theoretical_loss": 3.519951342369321, + "tokens_seen": 1625292800 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004101745235707122, + "loss": 1.434, + "theoretical_loss": 3.519951342369321, + "tokens_seen": 1625292800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041015847542627884, + "loss": 1.3512, + "theoretical_loss": 3.5199391198989045, + "tokens_seen": 1625358336 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004101424272818456, + "loss": 1.2329, + "theoretical_loss": 3.5199268980592837, + "tokens_seen": 1625423872 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041012637913741224, + "loss": 1.3568, + "theoretical_loss": 3.5199146768503997, + "tokens_seen": 1625489408 + }, + { + "epoch": 0.49, + "learning_rate": 0.000410110330992979, + "loss": 1.1842, + "theoretical_loss": 3.519902456272196, + "tokens_seen": 1625554944 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041009428284854565, + "loss": 1.3386, + "theoretical_loss": 3.5198902363246134, + "tokens_seen": 1625620480 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004100782347041124, + "loss": 1.3737, + "theoretical_loss": 3.5198780170075947, + "tokens_seen": 1625686016 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041006218655967905, + "loss": 1.3289, + "theoretical_loss": 3.519865798321081, + "tokens_seen": 1625751552 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041004613841524575, + "loss": 1.3245, + "theoretical_loss": 3.519853580265015, + "tokens_seen": 1625817088 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041003009027081245, + "loss": 1.2956, + "theoretical_loss": 3.5198413628393395, + "tokens_seen": 1625882624 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041001404212637915, + "loss": 1.2515, + "theoretical_loss": 3.519829146043995, + "tokens_seen": 1625948160 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040999799398194585, + "loss": 1.3645, + "theoretical_loss": 3.519816929878925, + "tokens_seen": 1626013696 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040998194583751255, + "loss": 1.2441, + "theoretical_loss": 3.5198047143440707, + "tokens_seen": 1626079232 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040996589769307925, + "loss": 1.2035, + "theoretical_loss": 3.519792499439374, + "tokens_seen": 1626144768 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040994984954864595, + "loss": 1.2841, + "theoretical_loss": 3.519780285164778, + "tokens_seen": 1626210304 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040993380140421265, + "loss": 1.3427, + "theoretical_loss": 3.5197680715202244, + "tokens_seen": 1626275840 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040991775325977935, + "loss": 1.2832, + "theoretical_loss": 3.519755858505655, + "tokens_seen": 1626341376 + }, + { + "epoch": 0.49, + "learning_rate": 0.000409901705115346, + "loss": 1.3184, + "theoretical_loss": 3.519743646121012, + "tokens_seen": 1626406912 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040988565697091275, + "loss": 1.3831, + "theoretical_loss": 3.5197314343662374, + "tokens_seen": 1626472448 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004098696088264794, + "loss": 1.3433, + "theoretical_loss": 3.5197192232412733, + "tokens_seen": 1626537984 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040985356068204615, + "loss": 1.2464, + "theoretical_loss": 3.5197070127460623, + "tokens_seen": 1626603520 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004098375125376128, + "loss": 1.2645, + "theoretical_loss": 3.519694802880547, + "tokens_seen": 1626669056 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040982146439317956, + "loss": 1.329, + "theoretical_loss": 3.519682593644668, + "tokens_seen": 1626734592 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004098054162487462, + "loss": 1.2149, + "theoretical_loss": 3.519670385038369, + "tokens_seen": 1626800128 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040978936810431296, + "loss": 1.191, + "theoretical_loss": 3.5196581770615913, + "tokens_seen": 1626865664 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004097733199598796, + "loss": 1.278, + "theoretical_loss": 3.5196459697142775, + "tokens_seen": 1626931200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040975727181544636, + "loss": 1.2842, + "theoretical_loss": 3.519633762996369, + "tokens_seen": 1626996736 + }, + { + "epoch": 0.49, + "learning_rate": 0.000409741223671013, + "loss": 1.2341, + "theoretical_loss": 3.5196215569078095, + "tokens_seen": 1627062272 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040972517552657976, + "loss": 1.1918, + "theoretical_loss": 3.5196093514485396, + "tokens_seen": 1627127808 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004097091273821464, + "loss": 1.3009, + "theoretical_loss": 3.5195971466185023, + "tokens_seen": 1627193344 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040969307923771316, + "loss": 1.2998, + "theoretical_loss": 3.51958494241764, + "tokens_seen": 1627258880 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004096770310932798, + "loss": 1.259, + "theoretical_loss": 3.519572738845895, + "tokens_seen": 1627324416 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040966098294884656, + "loss": 1.3007, + "theoretical_loss": 3.5195605359032087, + "tokens_seen": 1627389952 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004096449348044132, + "loss": 1.2726, + "theoretical_loss": 3.519548333589524, + "tokens_seen": 1627455488 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040962888665997996, + "loss": 1.3173, + "theoretical_loss": 3.5195361319047826, + "tokens_seen": 1627521024 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004096128385155466, + "loss": 1.1967, + "theoretical_loss": 3.519523930848928, + "tokens_seen": 1627586560 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040959679037111336, + "loss": 1.326, + "theoretical_loss": 3.5195117304219012, + "tokens_seen": 1627652096 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004095807422266801, + "loss": 1.2693, + "theoretical_loss": 3.5194995306236443, + "tokens_seen": 1627717632 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040956469408224677, + "loss": 1.37, + "theoretical_loss": 3.5194873314541013, + "tokens_seen": 1627783168 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004095486459378135, + "loss": 1.2091, + "theoretical_loss": 3.519475132913213, + "tokens_seen": 1627848704 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040953259779338017, + "loss": 1.3749, + "theoretical_loss": 3.5194629350009214, + "tokens_seen": 1627914240 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004095165496489469, + "loss": 1.1392, + "theoretical_loss": 3.51945073771717, + "tokens_seen": 1627979776 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040950050150451357, + "loss": 1.1767, + "theoretical_loss": 3.5194385410619002, + "tokens_seen": 1628045312 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004094844533600803, + "loss": 1.2558, + "theoretical_loss": 3.5194263450350554, + "tokens_seen": 1628110848 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040946840521564697, + "loss": 1.3181, + "theoretical_loss": 3.519414149636577, + "tokens_seen": 1628176384 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004094523570712137, + "loss": 1.3602, + "theoretical_loss": 3.5194019548664075, + "tokens_seen": 1628241920 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040943630892678037, + "loss": 1.267, + "theoretical_loss": 3.519389760724489, + "tokens_seen": 1628307456 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040942026078234707, + "loss": 1.2681, + "theoretical_loss": 3.5193775672107646, + "tokens_seen": 1628372992 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040940421263791377, + "loss": 1.2576, + "theoretical_loss": 3.519365374325176, + "tokens_seen": 1628438528 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004093881644934805, + "loss": 1.3498, + "theoretical_loss": 3.5193531820676656, + "tokens_seen": 1628504064 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 545970, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.416658878326416, + "objective/train/theoretical_loss": 3.5193409904381765, + "objective/train/tokens_used": 1649029600, + "theoretical_loss": 3.5193409904381765, + "tokens_seen": 1628569600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004093721163490472, + "loss": 1.3766, + "theoretical_loss": 3.5193409904381765, + "tokens_seen": 1628569600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004093560682046139, + "loss": 1.2934, + "theoretical_loss": 3.51932879943665, + "tokens_seen": 1628635136 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004093400200601806, + "loss": 1.3387, + "theoretical_loss": 3.5193166090630292, + "tokens_seen": 1628700672 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004093239719157473, + "loss": 1.3228, + "theoretical_loss": 3.5193044193172565, + "tokens_seen": 1628766208 + }, + { + "epoch": 0.49, + "learning_rate": 0.000409307923771314, + "loss": 1.2837, + "theoretical_loss": 3.519292230199274, + "tokens_seen": 1628831744 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004092918756268807, + "loss": 1.3637, + "theoretical_loss": 3.519280041709024, + "tokens_seen": 1628897280 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004092758274824473, + "loss": 1.2393, + "theoretical_loss": 3.5192678538464492, + "tokens_seen": 1628962816 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004092597793380141, + "loss": 1.3145, + "theoretical_loss": 3.5192556666114925, + "tokens_seen": 1629028352 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004092437311935807, + "loss": 1.3558, + "theoretical_loss": 3.5192434800040955, + "tokens_seen": 1629093888 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004092276830491475, + "loss": 1.2483, + "theoretical_loss": 3.5192312940242005, + "tokens_seen": 1629159424 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004092116349047141, + "loss": 1.2581, + "theoretical_loss": 3.519219108671751, + "tokens_seen": 1629224960 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004091955867602809, + "loss": 1.2863, + "theoretical_loss": 3.5192069239466885, + "tokens_seen": 1629290496 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040917953861584753, + "loss": 1.4091, + "theoretical_loss": 3.519194739848956, + "tokens_seen": 1629356032 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004091634904714143, + "loss": 1.2028, + "theoretical_loss": 3.5191825563784955, + "tokens_seen": 1629421568 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040914744232698093, + "loss": 1.23, + "theoretical_loss": 3.5191703735352498, + "tokens_seen": 1629487104 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004091313941825477, + "loss": 1.307, + "theoretical_loss": 3.5191581913191614, + "tokens_seen": 1629552640 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040911534603811433, + "loss": 1.3266, + "theoretical_loss": 3.519146009730173, + "tokens_seen": 1629618176 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004090992978936811, + "loss": 1.3664, + "theoretical_loss": 3.519133828768227, + "tokens_seen": 1629683712 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040908324974924773, + "loss": 1.2227, + "theoretical_loss": 3.5191216484332655, + "tokens_seen": 1629749248 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004090672016048145, + "loss": 1.2636, + "theoretical_loss": 3.5191094687252313, + "tokens_seen": 1629814784 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040905115346038113, + "loss": 1.2572, + "theoretical_loss": 3.519097289644067, + "tokens_seen": 1629880320 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004090351053159479, + "loss": 1.2375, + "theoretical_loss": 3.519085111189715, + "tokens_seen": 1629945856 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040901905717151453, + "loss": 1.2524, + "theoretical_loss": 3.519072933362118, + "tokens_seen": 1630011392 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004090030090270813, + "loss": 1.3324, + "theoretical_loss": 3.5190607561612177, + "tokens_seen": 1630076928 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040898696088264793, + "loss": 1.399, + "theoretical_loss": 3.5190485795869577, + "tokens_seen": 1630142464 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004089709127382147, + "loss": 1.3517, + "theoretical_loss": 3.519036403639281, + "tokens_seen": 1630208000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040895486459378134, + "loss": 1.2874, + "theoretical_loss": 3.5190242283181288, + "tokens_seen": 1630273536 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004089388164493481, + "loss": 1.1723, + "theoretical_loss": 3.519012053623444, + "tokens_seen": 1630339072 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040892276830491474, + "loss": 1.3063, + "theoretical_loss": 3.5189998795551696, + "tokens_seen": 1630404608 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004089067201604815, + "loss": 1.2878, + "theoretical_loss": 3.5189877061132484, + "tokens_seen": 1630470144 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040889067201604814, + "loss": 1.3226, + "theoretical_loss": 3.5189755332976222, + "tokens_seen": 1630535680 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004088746238716149, + "loss": 1.3535, + "theoretical_loss": 3.5189633611082347, + "tokens_seen": 1630601216 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040885857572718154, + "loss": 1.2736, + "theoretical_loss": 3.518951189545027, + "tokens_seen": 1630666752 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004088425275827483, + "loss": 1.2574, + "theoretical_loss": 3.5189390186079437, + "tokens_seen": 1630732288 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040882647943831494, + "loss": 1.3744, + "theoretical_loss": 3.5189268482969256, + "tokens_seen": 1630797824 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004088104312938817, + "loss": 1.3011, + "theoretical_loss": 3.5189146786119165, + "tokens_seen": 1630863360 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040879438314944834, + "loss": 1.3871, + "theoretical_loss": 3.5189025095528583, + "tokens_seen": 1630928896 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004087783350050151, + "loss": 1.3542, + "theoretical_loss": 3.518890341119694, + "tokens_seen": 1630994432 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040876228686058174, + "loss": 1.259, + "theoretical_loss": 3.518878173312366, + "tokens_seen": 1631059968 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004087462387161485, + "loss": 1.3816, + "theoretical_loss": 3.518866006130818, + "tokens_seen": 1631125504 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040873019057171515, + "loss": 1.3348, + "theoretical_loss": 3.5188538395749918, + "tokens_seen": 1631191040 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004087141424272819, + "loss": 1.2186, + "theoretical_loss": 3.5188416736448294, + "tokens_seen": 1631256576 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040869809428284855, + "loss": 1.2514, + "theoretical_loss": 3.518829508340275, + "tokens_seen": 1631322112 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004086820461384153, + "loss": 1.3279, + "theoretical_loss": 3.5188173436612695, + "tokens_seen": 1631387648 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040866599799398195, + "loss": 1.2843, + "theoretical_loss": 3.5188051796077575, + "tokens_seen": 1631453184 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040864994984954865, + "loss": 1.1867, + "theoretical_loss": 3.518793016179681, + "tokens_seen": 1631518720 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040863390170511535, + "loss": 1.2986, + "theoretical_loss": 3.518780853376982, + "tokens_seen": 1631584256 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040861785356068205, + "loss": 1.2974, + "theoretical_loss": 3.518768691199605, + "tokens_seen": 1631649792 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040860180541624875, + "loss": 1.2806, + "theoretical_loss": 3.51875652964749, + "tokens_seen": 1631715328 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040858575727181545, + "loss": 1.2822, + "theoretical_loss": 3.518744368720582, + "tokens_seen": 1631780864 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 547214, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2601799964904785, + "objective/train/theoretical_loss": 3.5187322084188235, + "objective/train/tokens_used": 1652306400, + "theoretical_loss": 3.5187322084188235, + "tokens_seen": 1631846400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040856970912738215, + "loss": 1.2906, + "theoretical_loss": 3.5187322084188235, + "tokens_seen": 1631846400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040855366098294885, + "loss": 1.3311, + "theoretical_loss": 3.518720048742156, + "tokens_seen": 1631911936 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040853761283851555, + "loss": 1.2151, + "theoretical_loss": 3.518707889690524, + "tokens_seen": 1631977472 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040852156469408225, + "loss": 1.2723, + "theoretical_loss": 3.5186957312638683, + "tokens_seen": 1632043008 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004085055165496489, + "loss": 1.3535, + "theoretical_loss": 3.5186835734621336, + "tokens_seen": 1632108544 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040848946840521565, + "loss": 1.2423, + "theoretical_loss": 3.5186714162852613, + "tokens_seen": 1632174080 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004084734202607823, + "loss": 1.3532, + "theoretical_loss": 3.518659259733195, + "tokens_seen": 1632239616 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040845737211634906, + "loss": 1.3451, + "theoretical_loss": 3.5186471038058773, + "tokens_seen": 1632305152 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004084413239719157, + "loss": 1.3717, + "theoretical_loss": 3.5186349485032506, + "tokens_seen": 1632370688 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040842527582748246, + "loss": 1.2511, + "theoretical_loss": 3.518622793825258, + "tokens_seen": 1632436224 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004084092276830491, + "loss": 1.2722, + "theoretical_loss": 3.518610639771843, + "tokens_seen": 1632501760 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040839317953861586, + "loss": 1.2873, + "theoretical_loss": 3.518598486342947, + "tokens_seen": 1632567296 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004083771313941826, + "loss": 1.3081, + "theoretical_loss": 3.518586333538514, + "tokens_seen": 1632632832 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040836108324974926, + "loss": 1.2417, + "theoretical_loss": 3.518574181358487, + "tokens_seen": 1632698368 + }, + { + "epoch": 0.49, + "learning_rate": 0.000408345035105316, + "loss": 1.399, + "theoretical_loss": 3.5185620298028075, + "tokens_seen": 1632763904 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040832898696088266, + "loss": 1.2732, + "theoretical_loss": 3.5185498788714202, + "tokens_seen": 1632829440 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004083129388164494, + "loss": 1.3301, + "theoretical_loss": 3.518537728564266, + "tokens_seen": 1632894976 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040829689067201606, + "loss": 1.3286, + "theoretical_loss": 3.51852557888129, + "tokens_seen": 1632960512 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004082808425275828, + "loss": 1.3132, + "theoretical_loss": 3.518513429822433, + "tokens_seen": 1633026048 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040826479438314946, + "loss": 1.3872, + "theoretical_loss": 3.518501281387639, + "tokens_seen": 1633091584 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004082487462387162, + "loss": 1.3259, + "theoretical_loss": 3.5184891335768507, + "tokens_seen": 1633157120 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040823269809428286, + "loss": 1.2861, + "theoretical_loss": 3.5184769863900103, + "tokens_seen": 1633222656 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004082166499498496, + "loss": 1.2396, + "theoretical_loss": 3.518464839827062, + "tokens_seen": 1633288192 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040820060180541627, + "loss": 1.3408, + "theoretical_loss": 3.5184526938879483, + "tokens_seen": 1633353728 + }, + { + "epoch": 0.49, + "learning_rate": 0.000408184553660983, + "loss": 1.3001, + "theoretical_loss": 3.518440548572612, + "tokens_seen": 1633419264 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040816850551654967, + "loss": 1.3264, + "theoretical_loss": 3.5184284038809954, + "tokens_seen": 1633484800 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004081524573721164, + "loss": 1.2697, + "theoretical_loss": 3.5184162598130424, + "tokens_seen": 1633550336 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040813640922768307, + "loss": 1.3854, + "theoretical_loss": 3.5184041163686954, + "tokens_seen": 1633615872 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004081203610832498, + "loss": 1.3554, + "theoretical_loss": 3.5183919735478977, + "tokens_seen": 1633681408 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040810431293881647, + "loss": 1.2912, + "theoretical_loss": 3.5183798313505923, + "tokens_seen": 1633746944 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004080882647943832, + "loss": 1.2618, + "theoretical_loss": 3.5183676897767215, + "tokens_seen": 1633812480 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040807221664994987, + "loss": 1.3732, + "theoretical_loss": 3.5183555488262295, + "tokens_seen": 1633878016 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004080561685055166, + "loss": 1.2545, + "theoretical_loss": 3.518343408499058, + "tokens_seen": 1633943552 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040804012036108327, + "loss": 1.3064, + "theoretical_loss": 3.5183312687951505, + "tokens_seen": 1634009088 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040802407221665, + "loss": 1.3529, + "theoretical_loss": 3.5183191297144507, + "tokens_seen": 1634074624 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004080080240722167, + "loss": 1.2502, + "theoretical_loss": 3.5183069912569005, + "tokens_seen": 1634140160 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004079919759277834, + "loss": 1.2531, + "theoretical_loss": 3.5182948534224434, + "tokens_seen": 1634205696 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004079759277833501, + "loss": 1.2107, + "theoretical_loss": 3.518282716211022, + "tokens_seen": 1634271232 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004079598796389168, + "loss": 1.3794, + "theoretical_loss": 3.518270579622581, + "tokens_seen": 1634336768 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004079438314944835, + "loss": 1.2892, + "theoretical_loss": 3.5182584436570616, + "tokens_seen": 1634402304 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004079277833500502, + "loss": 1.2906, + "theoretical_loss": 3.5182463083144073, + "tokens_seen": 1634467840 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004079117352056169, + "loss": 1.3662, + "theoretical_loss": 3.5182341735945615, + "tokens_seen": 1634533376 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004078956870611836, + "loss": 1.194, + "theoretical_loss": 3.5182220394974673, + "tokens_seen": 1634598912 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004078796389167502, + "loss": 1.2531, + "theoretical_loss": 3.5182099060230674, + "tokens_seen": 1634664448 + }, + { + "epoch": 0.5, + "learning_rate": 0.000407863590772317, + "loss": 1.3946, + "theoretical_loss": 3.518197773171305, + "tokens_seen": 1634729984 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004078475426278836, + "loss": 1.2397, + "theoretical_loss": 3.5181856409421233, + "tokens_seen": 1634795520 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004078314944834504, + "loss": 1.3523, + "theoretical_loss": 3.5181735093354654, + "tokens_seen": 1634861056 + }, + { + "epoch": 0.5, + "learning_rate": 0.000407815446339017, + "loss": 1.3561, + "theoretical_loss": 3.5181613783512744, + "tokens_seen": 1634926592 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004077993981945838, + "loss": 1.317, + "theoretical_loss": 3.518149247989493, + "tokens_seen": 1634992128 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040778335005015043, + "loss": 1.4313, + "theoretical_loss": 3.518137118250065, + "tokens_seen": 1635057664 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 548116, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2663007974624634, + "objective/train/theoretical_loss": 3.518124989132933, + "objective/train/tokens_used": 1655583200, + "theoretical_loss": 3.518124989132933, + "tokens_seen": 1635123200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004077673019057172, + "loss": 1.28, + "theoretical_loss": 3.518124989132933, + "tokens_seen": 1635123200 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040775125376128383, + "loss": 1.3439, + "theoretical_loss": 3.5181128606380407, + "tokens_seen": 1635188736 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004077352056168506, + "loss": 1.2223, + "theoretical_loss": 3.5181007327653306, + "tokens_seen": 1635254272 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040771915747241723, + "loss": 1.2541, + "theoretical_loss": 3.5180886055147464, + "tokens_seen": 1635319808 + }, + { + "epoch": 0.5, + "learning_rate": 0.000407703109327984, + "loss": 1.3701, + "theoretical_loss": 3.518076478886231, + "tokens_seen": 1635385344 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040768706118355063, + "loss": 1.2392, + "theoretical_loss": 3.5180643528797275, + "tokens_seen": 1635450880 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004076710130391174, + "loss": 1.313, + "theoretical_loss": 3.518052227495179, + "tokens_seen": 1635516416 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040765496489468403, + "loss": 1.3045, + "theoretical_loss": 3.5180401027325288, + "tokens_seen": 1635581952 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004076389167502508, + "loss": 1.3109, + "theoretical_loss": 3.51802797859172, + "tokens_seen": 1635647488 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040762286860581743, + "loss": 1.3282, + "theoretical_loss": 3.5180158550726963, + "tokens_seen": 1635713024 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004076068204613842, + "loss": 1.2817, + "theoretical_loss": 3.5180037321754, + "tokens_seen": 1635778560 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040759077231695084, + "loss": 1.507, + "theoretical_loss": 3.517991609899775, + "tokens_seen": 1635844096 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004075747241725176, + "loss": 1.2632, + "theoretical_loss": 3.5179794882457642, + "tokens_seen": 1635909632 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040755867602808424, + "loss": 1.371, + "theoretical_loss": 3.5179673672133114, + "tokens_seen": 1635975168 + }, + { + "epoch": 0.5, + "learning_rate": 0.000407542627883651, + "loss": 1.3235, + "theoretical_loss": 3.5179552468023587, + "tokens_seen": 1636040704 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040752657973921764, + "loss": 1.3234, + "theoretical_loss": 3.5179431270128507, + "tokens_seen": 1636106240 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004075105315947844, + "loss": 1.2365, + "theoretical_loss": 3.5179310078447297, + "tokens_seen": 1636171776 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040749448345035104, + "loss": 1.3097, + "theoretical_loss": 3.517918889297939, + "tokens_seen": 1636237312 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004074784353059178, + "loss": 1.3809, + "theoretical_loss": 3.517906771372422, + "tokens_seen": 1636302848 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040746238716148444, + "loss": 1.3125, + "theoretical_loss": 3.5178946540681224, + "tokens_seen": 1636368384 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004074463390170512, + "loss": 1.2686, + "theoretical_loss": 3.517882537384983, + "tokens_seen": 1636433920 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040743029087261784, + "loss": 1.3453, + "theoretical_loss": 3.5178704213229475, + "tokens_seen": 1636499456 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004074142427281846, + "loss": 1.2856, + "theoretical_loss": 3.5178583058819584, + "tokens_seen": 1636564992 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040739819458375124, + "loss": 1.2855, + "theoretical_loss": 3.5178461910619596, + "tokens_seen": 1636630528 + }, + { + "epoch": 0.5, + "learning_rate": 0.000407382146439318, + "loss": 1.3232, + "theoretical_loss": 3.5178340768628944, + "tokens_seen": 1636696064 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040736609829488464, + "loss": 1.2578, + "theoretical_loss": 3.517821963284706, + "tokens_seen": 1636761600 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004073500501504514, + "loss": 1.2877, + "theoretical_loss": 3.517809850327337, + "tokens_seen": 1636827136 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040733400200601805, + "loss": 1.3605, + "theoretical_loss": 3.5177977379907324, + "tokens_seen": 1636892672 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004073179538615848, + "loss": 1.2611, + "theoretical_loss": 3.5177856262748346, + "tokens_seen": 1636958208 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040730190571715145, + "loss": 1.2498, + "theoretical_loss": 3.5177735151795866, + "tokens_seen": 1637023744 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004072858575727182, + "loss": 1.4699, + "theoretical_loss": 3.517761404704932, + "tokens_seen": 1637089280 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040726980942828485, + "loss": 1.2863, + "theoretical_loss": 3.5177492948508147, + "tokens_seen": 1637154816 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040725376128385155, + "loss": 1.2188, + "theoretical_loss": 3.517737185617177, + "tokens_seen": 1637220352 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040723771313941825, + "loss": 1.3205, + "theoretical_loss": 3.5177250770039636, + "tokens_seen": 1637285888 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040722166499498495, + "loss": 1.2454, + "theoretical_loss": 3.5177129690111166, + "tokens_seen": 1637351424 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004072056168505517, + "loss": 1.3755, + "theoretical_loss": 3.5177008616385805, + "tokens_seen": 1637416960 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040718956870611835, + "loss": 1.2752, + "theoretical_loss": 3.517688754886297, + "tokens_seen": 1637482496 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004071735205616851, + "loss": 1.3169, + "theoretical_loss": 3.517676648754212, + "tokens_seen": 1637548032 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040715747241725175, + "loss": 1.2806, + "theoretical_loss": 3.517664543242267, + "tokens_seen": 1637613568 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004071414242728185, + "loss": 1.2345, + "theoretical_loss": 3.5176524383504058, + "tokens_seen": 1637679104 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040712537612838515, + "loss": 1.2848, + "theoretical_loss": 3.517640334078572, + "tokens_seen": 1637744640 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004071093279839519, + "loss": 1.2902, + "theoretical_loss": 3.5176282304267095, + "tokens_seen": 1637810176 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040709327983951856, + "loss": 1.3012, + "theoretical_loss": 3.517616127394761, + "tokens_seen": 1637875712 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004070772316950853, + "loss": 1.3301, + "theoretical_loss": 3.51760402498267, + "tokens_seen": 1637941248 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040706118355065196, + "loss": 1.2662, + "theoretical_loss": 3.5175919231903805, + "tokens_seen": 1638006784 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004070451354062187, + "loss": 1.3013, + "theoretical_loss": 3.5175798220178356, + "tokens_seen": 1638072320 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040702908726178536, + "loss": 1.2981, + "theoretical_loss": 3.5175677214649785, + "tokens_seen": 1638137856 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004070130391173521, + "loss": 1.2958, + "theoretical_loss": 3.5175556215317534, + "tokens_seen": 1638203392 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040699699097291876, + "loss": 1.348, + "theoretical_loss": 3.5175435222181033, + "tokens_seen": 1638268928 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004069809428284855, + "loss": 1.3486, + "theoretical_loss": 3.5175314235239714, + "tokens_seen": 1638334464 + }, + { + "debugging/Compilability": 0.0, + "debugging/distinct-1-grams": 0.7322549377064853, + "debugging/entropy-1-grams": 5.133293776725962, + "debugging/length": 605.75, + "debugging/num_segments": 8, + "debugging/score": 0.044954849742907824, + "debugging/score_std": 0.025932746393242948, + "epoch": 0.5, + "objective/train/docs_used": 549338, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.18026864528656, + "objective/train/theoretical_loss": 3.517519325449302, + "objective/train/tokens_used": 1658860000, + "theoretical_loss": 3.517519325449302, + "tokens_seen": 1638400000 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040696489468405216, + "loss": 1.1928, + "theoretical_loss": 3.517519325449302, + "tokens_seen": 1638400000 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004069488465396189, + "loss": 1.3697, + "theoretical_loss": 3.517507227994038, + "tokens_seen": 1638465536 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040693279839518556, + "loss": 1.3529, + "theoretical_loss": 3.517495131158123, + "tokens_seen": 1638531072 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004069167502507523, + "loss": 1.1634, + "theoretical_loss": 3.5174830349415007, + "tokens_seen": 1638596608 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040690070210631896, + "loss": 1.3304, + "theoretical_loss": 3.517470939344114, + "tokens_seen": 1638662144 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004068846539618857, + "loss": 1.1961, + "theoretical_loss": 3.5174588443659074, + "tokens_seen": 1638727680 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040686860581745236, + "loss": 1.3084, + "theoretical_loss": 3.5174467500068243, + "tokens_seen": 1638793216 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004068525576730191, + "loss": 1.3103, + "theoretical_loss": 3.5174346562668077, + "tokens_seen": 1638858752 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040683650952858577, + "loss": 1.2826, + "theoretical_loss": 3.5174225631458014, + "tokens_seen": 1638924288 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004068204613841525, + "loss": 1.3464, + "theoretical_loss": 3.517410470643749, + "tokens_seen": 1638989824 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040680441323971917, + "loss": 1.2669, + "theoretical_loss": 3.5173983787605945, + "tokens_seen": 1639055360 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004067883650952859, + "loss": 1.2666, + "theoretical_loss": 3.5173862874962802, + "tokens_seen": 1639120896 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040677231695085257, + "loss": 1.2279, + "theoretical_loss": 3.517374196850751, + "tokens_seen": 1639186432 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004067562688064193, + "loss": 1.1905, + "theoretical_loss": 3.51736210682395, + "tokens_seen": 1639251968 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040674022066198597, + "loss": 1.3524, + "theoretical_loss": 3.5173500174158208, + "tokens_seen": 1639317504 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004067241725175527, + "loss": 1.3141, + "theoretical_loss": 3.517337928626307, + "tokens_seen": 1639383040 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040670812437311937, + "loss": 1.3674, + "theoretical_loss": 3.517325840455352, + "tokens_seen": 1639448576 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004066920762286861, + "loss": 1.3128, + "theoretical_loss": 3.5173137529029006, + "tokens_seen": 1639514112 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040667602808425277, + "loss": 1.2473, + "theoretical_loss": 3.5173016659688945, + "tokens_seen": 1639579648 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004066599799398195, + "loss": 1.3721, + "theoretical_loss": 3.517289579653279, + "tokens_seen": 1639645184 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004066439317953862, + "loss": 1.2469, + "theoretical_loss": 3.5172774939559965, + "tokens_seen": 1639710720 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040662788365095293, + "loss": 1.2926, + "theoretical_loss": 3.517265408876992, + "tokens_seen": 1639776256 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004066118355065196, + "loss": 1.3442, + "theoretical_loss": 3.517253324416208, + "tokens_seen": 1639841792 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004065957873620863, + "loss": 1.3684, + "theoretical_loss": 3.517241240573588, + "tokens_seen": 1639907328 + }, + { + "epoch": 0.5, + "learning_rate": 0.000406579739217653, + "loss": 1.2832, + "theoretical_loss": 3.5172291573490773, + "tokens_seen": 1639972864 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004065636910732197, + "loss": 1.2527, + "theoretical_loss": 3.517217074742618, + "tokens_seen": 1640038400 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004065476429287864, + "loss": 1.3512, + "theoretical_loss": 3.517204992754154, + "tokens_seen": 1640103936 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004065315947843531, + "loss": 1.3242, + "theoretical_loss": 3.5171929113836295, + "tokens_seen": 1640169472 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004065155466399198, + "loss": 1.3171, + "theoretical_loss": 3.5171808306309886, + "tokens_seen": 1640235008 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004064994984954865, + "loss": 1.2756, + "theoretical_loss": 3.5171687504961744, + "tokens_seen": 1640300544 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004064834503510531, + "loss": 1.2805, + "theoretical_loss": 3.51715667097913, + "tokens_seen": 1640366080 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004064674022066199, + "loss": 1.311, + "theoretical_loss": 3.5171445920798003, + "tokens_seen": 1640431616 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004064513540621865, + "loss": 1.267, + "theoretical_loss": 3.5171325137981286, + "tokens_seen": 1640497152 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004064353059177533, + "loss": 1.2189, + "theoretical_loss": 3.517120436134058, + "tokens_seen": 1640562688 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040641925777331993, + "loss": 1.4002, + "theoretical_loss": 3.5171083590875334, + "tokens_seen": 1640628224 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004064032096288867, + "loss": 1.3636, + "theoretical_loss": 3.517096282658498, + "tokens_seen": 1640693760 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040638716148445333, + "loss": 1.3992, + "theoretical_loss": 3.5170842068468953, + "tokens_seen": 1640759296 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004063711133400201, + "loss": 1.3882, + "theoretical_loss": 3.5170721316526694, + "tokens_seen": 1640824832 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040635506519558673, + "loss": 1.2632, + "theoretical_loss": 3.517060057075764, + "tokens_seen": 1640890368 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004063390170511535, + "loss": 1.2589, + "theoretical_loss": 3.517047983116123, + "tokens_seen": 1640955904 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040632296890672013, + "loss": 1.3114, + "theoretical_loss": 3.51703590977369, + "tokens_seen": 1641021440 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004063069207622869, + "loss": 1.2914, + "theoretical_loss": 3.517023837048409, + "tokens_seen": 1641086976 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040629087261785353, + "loss": 1.2638, + "theoretical_loss": 3.5170117649402233, + "tokens_seen": 1641152512 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004062748244734203, + "loss": 1.2527, + "theoretical_loss": 3.5169996934490775, + "tokens_seen": 1641218048 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040625877632898693, + "loss": 1.2254, + "theoretical_loss": 3.516987622574915, + "tokens_seen": 1641283584 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004062427281845537, + "loss": 1.3077, + "theoretical_loss": 3.51697555231768, + "tokens_seen": 1641349120 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040622668004012034, + "loss": 1.4039, + "theoretical_loss": 3.5169634826773155, + "tokens_seen": 1641414656 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004062106318956871, + "loss": 1.3378, + "theoretical_loss": 3.516951413653766, + "tokens_seen": 1641480192 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040619458375125374, + "loss": 1.2719, + "theoretical_loss": 3.516939345246975, + "tokens_seen": 1641545728 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004061785356068205, + "loss": 1.2856, + "theoretical_loss": 3.5169272774568867, + "tokens_seen": 1641611264 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 550613, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3520652055740356, + "objective/train/theoretical_loss": 3.516915210283445, + "objective/train/tokens_used": 1662136800, + "theoretical_loss": 3.516915210283445, + "tokens_seen": 1641676800 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040616248746238714, + "loss": 1.2895, + "theoretical_loss": 3.516915210283445, + "tokens_seen": 1641676800 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004061464393179539, + "loss": 1.312, + "theoretical_loss": 3.5169031437265934, + "tokens_seen": 1641742336 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040613039117352054, + "loss": 1.2866, + "theoretical_loss": 3.5168910777862763, + "tokens_seen": 1641807872 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004061143430290873, + "loss": 1.2786, + "theoretical_loss": 3.516879012462437, + "tokens_seen": 1641873408 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040609829488465394, + "loss": 1.3183, + "theoretical_loss": 3.5168669477550196, + "tokens_seen": 1641938944 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004060822467402207, + "loss": 1.254, + "theoretical_loss": 3.5168548836639686, + "tokens_seen": 1642004480 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040606619859578734, + "loss": 1.3026, + "theoretical_loss": 3.5168428201892272, + "tokens_seen": 1642070016 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004060501504513541, + "loss": 1.4237, + "theoretical_loss": 3.516830757330739, + "tokens_seen": 1642135552 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040603410230692074, + "loss": 1.2869, + "theoretical_loss": 3.5168186950884492, + "tokens_seen": 1642201088 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004060180541624875, + "loss": 1.2253, + "theoretical_loss": 3.5168066334623003, + "tokens_seen": 1642266624 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040600200601805425, + "loss": 1.2861, + "theoretical_loss": 3.5167945724522376, + "tokens_seen": 1642332160 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004059859578736209, + "loss": 1.3845, + "theoretical_loss": 3.516782512058204, + "tokens_seen": 1642397696 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004059699097291876, + "loss": 1.2079, + "theoretical_loss": 3.516770452280144, + "tokens_seen": 1642463232 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004059538615847543, + "loss": 1.3241, + "theoretical_loss": 3.5167583931180015, + "tokens_seen": 1642528768 + }, + { + "epoch": 0.5, + "learning_rate": 0.000405937813440321, + "loss": 1.2881, + "theoretical_loss": 3.51674633457172, + "tokens_seen": 1642594304 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004059217652958877, + "loss": 1.2206, + "theoretical_loss": 3.5167342766412437, + "tokens_seen": 1642659840 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004059057171514544, + "loss": 1.3041, + "theoretical_loss": 3.516722219326517, + "tokens_seen": 1642725376 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004058896690070211, + "loss": 1.527, + "theoretical_loss": 3.5167101626274837, + "tokens_seen": 1642790912 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004058736208625878, + "loss": 1.2195, + "theoretical_loss": 3.5166981065440877, + "tokens_seen": 1642856448 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004058575727181545, + "loss": 1.2176, + "theoretical_loss": 3.516686051076273, + "tokens_seen": 1642921984 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004058415245737212, + "loss": 1.4429, + "theoretical_loss": 3.5166739962239832, + "tokens_seen": 1642987520 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040582547642928785, + "loss": 1.4121, + "theoretical_loss": 3.516661941987164, + "tokens_seen": 1643053056 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004058094282848546, + "loss": 1.2556, + "theoretical_loss": 3.516649888365757, + "tokens_seen": 1643118592 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040579338014042125, + "loss": 1.1572, + "theoretical_loss": 3.516637835359708, + "tokens_seen": 1643184128 + }, + { + "epoch": 0.5, + "learning_rate": 0.000405777331995988, + "loss": 1.1544, + "theoretical_loss": 3.5166257829689602, + "tokens_seen": 1643249664 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040576128385155465, + "loss": 1.2845, + "theoretical_loss": 3.516613731193458, + "tokens_seen": 1643315200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004057452357071214, + "loss": 1.165, + "theoretical_loss": 3.5166016800331454, + "tokens_seen": 1643380736 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040572918756268806, + "loss": 1.332, + "theoretical_loss": 3.5165896294879664, + "tokens_seen": 1643446272 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004057131394182548, + "loss": 1.3102, + "theoretical_loss": 3.5165775795578655, + "tokens_seen": 1643511808 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040569709127382146, + "loss": 1.2905, + "theoretical_loss": 3.516565530242786, + "tokens_seen": 1643577344 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004056810431293882, + "loss": 1.3407, + "theoretical_loss": 3.5165534815426724, + "tokens_seen": 1643642880 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040566499498495486, + "loss": 1.3558, + "theoretical_loss": 3.516541433457469, + "tokens_seen": 1643708416 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004056489468405216, + "loss": 1.2896, + "theoretical_loss": 3.51652938598712, + "tokens_seen": 1643773952 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040563289869608826, + "loss": 1.281, + "theoretical_loss": 3.5165173391315685, + "tokens_seen": 1643839488 + }, + { + "epoch": 0.5, + "learning_rate": 0.000405616850551655, + "loss": 1.2944, + "theoretical_loss": 3.516505292890759, + "tokens_seen": 1643905024 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040560080240722166, + "loss": 1.2427, + "theoretical_loss": 3.516493247264637, + "tokens_seen": 1643970560 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004055847542627884, + "loss": 1.2297, + "theoretical_loss": 3.516481202253145, + "tokens_seen": 1644036096 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040556870611835506, + "loss": 1.2446, + "theoretical_loss": 3.5164691578562284, + "tokens_seen": 1644101632 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004055526579739218, + "loss": 1.3117, + "theoretical_loss": 3.51645711407383, + "tokens_seen": 1644167168 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040553660982948846, + "loss": 1.2907, + "theoretical_loss": 3.5164450709058945, + "tokens_seen": 1644232704 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004055205616850552, + "loss": 1.3179, + "theoretical_loss": 3.516433028352367, + "tokens_seen": 1644298240 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040550451354062186, + "loss": 1.4179, + "theoretical_loss": 3.51642098641319, + "tokens_seen": 1644363776 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004054884653961886, + "loss": 1.324, + "theoretical_loss": 3.5164089450883087, + "tokens_seen": 1644429312 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040547241725175527, + "loss": 1.1669, + "theoretical_loss": 3.5163969043776673, + "tokens_seen": 1644494848 + }, + { + "epoch": 0.5, + "learning_rate": 0.000405456369107322, + "loss": 1.3281, + "theoretical_loss": 3.5163848642812097, + "tokens_seen": 1644560384 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040544032096288867, + "loss": 1.2432, + "theoretical_loss": 3.5163728247988804, + "tokens_seen": 1644625920 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004054242728184554, + "loss": 1.2673, + "theoretical_loss": 3.516360785930623, + "tokens_seen": 1644691456 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040540822467402207, + "loss": 1.2448, + "theoretical_loss": 3.516348747676382, + "tokens_seen": 1644756992 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004053921765295888, + "loss": 1.3363, + "theoretical_loss": 3.5163367100361023, + "tokens_seen": 1644822528 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040537612838515547, + "loss": 1.2778, + "theoretical_loss": 3.516324673009727, + "tokens_seen": 1644888064 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 551496, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3743598461151123, + "objective/train/theoretical_loss": 3.5163126365972017, + "objective/train/tokens_used": 1665413600, + "theoretical_loss": 3.5163126365972017, + "tokens_seen": 1644953600 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004053600802407222, + "loss": 1.3564, + "theoretical_loss": 3.5163126365972017, + "tokens_seen": 1644953600 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040534403209628887, + "loss": 1.2824, + "theoretical_loss": 3.516300600798469, + "tokens_seen": 1645019136 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004053279839518556, + "loss": 1.2393, + "theoretical_loss": 3.5162885656134746, + "tokens_seen": 1645084672 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040531193580742227, + "loss": 1.4257, + "theoretical_loss": 3.5162765310421618, + "tokens_seen": 1645150208 + }, + { + "epoch": 0.5, + "learning_rate": 0.000405295887662989, + "loss": 1.2824, + "theoretical_loss": 3.5162644970844754, + "tokens_seen": 1645215744 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004052798395185557, + "loss": 1.3281, + "theoretical_loss": 3.516252463740359, + "tokens_seen": 1645281280 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040526379137412243, + "loss": 1.2545, + "theoretical_loss": 3.516240431009758, + "tokens_seen": 1645346816 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004052477432296891, + "loss": 1.2819, + "theoretical_loss": 3.5162283988926157, + "tokens_seen": 1645412352 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040523169508525583, + "loss": 1.3082, + "theoretical_loss": 3.5162163673888767, + "tokens_seen": 1645477888 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004052156469408225, + "loss": 1.2894, + "theoretical_loss": 3.516204336498485, + "tokens_seen": 1645543424 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004051995987963892, + "loss": 1.2994, + "theoretical_loss": 3.516192306221386, + "tokens_seen": 1645608960 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004051835506519559, + "loss": 1.3521, + "theoretical_loss": 3.516180276557523, + "tokens_seen": 1645674496 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004051675025075226, + "loss": 1.3075, + "theoretical_loss": 3.5161682475068403, + "tokens_seen": 1645740032 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004051514543630893, + "loss": 1.2642, + "theoretical_loss": 3.5161562190692823, + "tokens_seen": 1645805568 + }, + { + "epoch": 0.5, + "learning_rate": 0.000405135406218656, + "loss": 1.2422, + "theoretical_loss": 3.516144191244794, + "tokens_seen": 1645871104 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004051193580742227, + "loss": 1.3944, + "theoretical_loss": 3.516132164033319, + "tokens_seen": 1645936640 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004051033099297894, + "loss": 1.2794, + "theoretical_loss": 3.516120137434802, + "tokens_seen": 1646002176 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004050872617853561, + "loss": 1.3183, + "theoretical_loss": 3.5161081114491877, + "tokens_seen": 1646067712 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004050712136409228, + "loss": 1.3001, + "theoretical_loss": 3.51609608607642, + "tokens_seen": 1646133248 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040505516549648943, + "loss": 1.2354, + "theoretical_loss": 3.516084061316443, + "tokens_seen": 1646198784 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004050391173520562, + "loss": 1.1901, + "theoretical_loss": 3.5160720371692014, + "tokens_seen": 1646264320 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040502306920762283, + "loss": 1.3966, + "theoretical_loss": 3.5160600136346396, + "tokens_seen": 1646329856 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004050070210631896, + "loss": 1.3421, + "theoretical_loss": 3.5160479907127016, + "tokens_seen": 1646395392 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040499097291875623, + "loss": 1.3397, + "theoretical_loss": 3.516035968403333, + "tokens_seen": 1646460928 + }, + { + "epoch": 0.5, + "learning_rate": 0.000404974924774323, + "loss": 1.4182, + "theoretical_loss": 3.516023946706477, + "tokens_seen": 1646526464 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040495887662988963, + "loss": 1.3631, + "theoretical_loss": 3.5160119256220783, + "tokens_seen": 1646592000 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004049428284854564, + "loss": 1.2533, + "theoretical_loss": 3.515999905150082, + "tokens_seen": 1646657536 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040492678034102303, + "loss": 1.3535, + "theoretical_loss": 3.515987885290431, + "tokens_seen": 1646723072 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004049107321965898, + "loss": 1.3533, + "theoretical_loss": 3.515975866043071, + "tokens_seen": 1646788608 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040489468405215643, + "loss": 1.3462, + "theoretical_loss": 3.5159638474079467, + "tokens_seen": 1646854144 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004048786359077232, + "loss": 1.4025, + "theoretical_loss": 3.515951829385002, + "tokens_seen": 1646919680 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040486258776328984, + "loss": 1.2922, + "theoretical_loss": 3.5159398119741807, + "tokens_seen": 1646985216 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004048465396188566, + "loss": 1.2653, + "theoretical_loss": 3.515927795175428, + "tokens_seen": 1647050752 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040483049147442324, + "loss": 1.1922, + "theoretical_loss": 3.5159157789886892, + "tokens_seen": 1647116288 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040481444332999, + "loss": 1.3475, + "theoretical_loss": 3.515903763413907, + "tokens_seen": 1647181824 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040479839518555675, + "loss": 1.2216, + "theoretical_loss": 3.515891748451027, + "tokens_seen": 1647247360 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004047823470411234, + "loss": 1.2698, + "theoretical_loss": 3.515879734099993, + "tokens_seen": 1647312896 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040476629889669015, + "loss": 1.3232, + "theoretical_loss": 3.5158677203607507, + "tokens_seen": 1647378432 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004047502507522568, + "loss": 1.3189, + "theoretical_loss": 3.5158557072332437, + "tokens_seen": 1647443968 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040473420260782355, + "loss": 1.2758, + "theoretical_loss": 3.5158436947174168, + "tokens_seen": 1647509504 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004047181544633902, + "loss": 1.3033, + "theoretical_loss": 3.5158316828132143, + "tokens_seen": 1647575040 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040470210631895695, + "loss": 1.3382, + "theoretical_loss": 3.5158196715205805, + "tokens_seen": 1647640576 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004046860581745236, + "loss": 1.2258, + "theoretical_loss": 3.5158076608394606, + "tokens_seen": 1647706112 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040467001003009035, + "loss": 1.3701, + "theoretical_loss": 3.5157956507697987, + "tokens_seen": 1647771648 + }, + { + "epoch": 0.5, + "learning_rate": 0.000404653961885657, + "loss": 1.313, + "theoretical_loss": 3.5157836413115398, + "tokens_seen": 1647837184 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040463791374122375, + "loss": 1.3229, + "theoretical_loss": 3.515771632464628, + "tokens_seen": 1647902720 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004046218655967904, + "loss": 1.3026, + "theoretical_loss": 3.515759624229008, + "tokens_seen": 1647968256 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040460581745235715, + "loss": 1.2709, + "theoretical_loss": 3.5157476166046244, + "tokens_seen": 1648033792 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004045897693079238, + "loss": 1.3436, + "theoretical_loss": 3.5157356095914216, + "tokens_seen": 1648099328 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004045737211634905, + "loss": 1.3777, + "theoretical_loss": 3.5157236031893446, + "tokens_seen": 1648164864 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 552711, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2650483846664429, + "objective/train/theoretical_loss": 3.5157115973983375, + "objective/train/tokens_used": 1668690400, + "theoretical_loss": 3.5157115973983375, + "tokens_seen": 1648230400 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004045576730190572, + "loss": 1.3071, + "theoretical_loss": 3.5157115973983375, + "tokens_seen": 1648230400 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004045416248746239, + "loss": 1.2882, + "theoretical_loss": 3.5156995922183456, + "tokens_seen": 1648295936 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004045255767301906, + "loss": 1.3203, + "theoretical_loss": 3.515687587649313, + "tokens_seen": 1648361472 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004045095285857573, + "loss": 1.3522, + "theoretical_loss": 3.515675583691184, + "tokens_seen": 1648427008 + }, + { + "epoch": 0.5, + "learning_rate": 0.000404493480441324, + "loss": 1.3041, + "theoretical_loss": 3.515663580343904, + "tokens_seen": 1648492544 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004044774322968907, + "loss": 1.319, + "theoretical_loss": 3.515651577607417, + "tokens_seen": 1648558080 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004044613841524574, + "loss": 1.4277, + "theoretical_loss": 3.515639575481668, + "tokens_seen": 1648623616 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004044453360080241, + "loss": 1.439, + "theoretical_loss": 3.5156275739666016, + "tokens_seen": 1648689152 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040442928786359075, + "loss": 1.2819, + "theoretical_loss": 3.5156155730621625, + "tokens_seen": 1648754688 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004044132397191575, + "loss": 1.2366, + "theoretical_loss": 3.5156035727682946, + "tokens_seen": 1648820224 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040439719157472415, + "loss": 1.2742, + "theoretical_loss": 3.515591573084944, + "tokens_seen": 1648885760 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004043811434302909, + "loss": 1.1554, + "theoretical_loss": 3.5155795740120546, + "tokens_seen": 1648951296 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040436509528585755, + "loss": 1.2962, + "theoretical_loss": 3.5155675755495706, + "tokens_seen": 1649016832 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004043490471414243, + "loss": 1.3395, + "theoretical_loss": 3.5155555776974374, + "tokens_seen": 1649082368 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040433299899699096, + "loss": 1.3604, + "theoretical_loss": 3.5155435804555997, + "tokens_seen": 1649147904 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004043169508525577, + "loss": 1.2655, + "theoretical_loss": 3.5155315838240018, + "tokens_seen": 1649213440 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040430090270812436, + "loss": 1.2242, + "theoretical_loss": 3.5155195878025887, + "tokens_seen": 1649278976 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004042848545636911, + "loss": 1.3321, + "theoretical_loss": 3.515507592391305, + "tokens_seen": 1649344512 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040426880641925776, + "loss": 1.3798, + "theoretical_loss": 3.5154955975900952, + "tokens_seen": 1649410048 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004042527582748245, + "loss": 1.3723, + "theoretical_loss": 3.5154836033989048, + "tokens_seen": 1649475584 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040423671013039116, + "loss": 1.307, + "theoretical_loss": 3.5154716098176775, + "tokens_seen": 1649541120 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004042206619859579, + "loss": 1.3331, + "theoretical_loss": 3.5154596168463588, + "tokens_seen": 1649606656 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040420461384152456, + "loss": 1.3565, + "theoretical_loss": 3.515447624484893, + "tokens_seen": 1649672192 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004041885656970913, + "loss": 1.2387, + "theoretical_loss": 3.515435632733225, + "tokens_seen": 1649737728 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040417251755265796, + "loss": 1.3197, + "theoretical_loss": 3.5154236415913003, + "tokens_seen": 1649803264 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004041564694082247, + "loss": 1.2871, + "theoretical_loss": 3.5154116510590625, + "tokens_seen": 1649868800 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040414042126379136, + "loss": 1.2499, + "theoretical_loss": 3.515399661136457, + "tokens_seen": 1649934336 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004041243731193581, + "loss": 1.3688, + "theoretical_loss": 3.5153876718234285, + "tokens_seen": 1649999872 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040410832497492477, + "loss": 1.2212, + "theoretical_loss": 3.515375683119922, + "tokens_seen": 1650065408 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004040922768304915, + "loss": 1.4593, + "theoretical_loss": 3.515363695025882, + "tokens_seen": 1650130944 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040407622868605817, + "loss": 1.2195, + "theoretical_loss": 3.515351707541253, + "tokens_seen": 1650196480 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004040601805416249, + "loss": 1.2406, + "theoretical_loss": 3.515339720665981, + "tokens_seen": 1650262016 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040404413239719157, + "loss": 1.2425, + "theoretical_loss": 3.51532773440001, + "tokens_seen": 1650327552 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004040280842527583, + "loss": 1.3901, + "theoretical_loss": 3.5153157487432845, + "tokens_seen": 1650393088 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040401203610832497, + "loss": 1.3186, + "theoretical_loss": 3.5153037636957496, + "tokens_seen": 1650458624 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004039959879638917, + "loss": 1.2464, + "theoretical_loss": 3.5152917792573506, + "tokens_seen": 1650524160 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040397993981945837, + "loss": 1.2504, + "theoretical_loss": 3.5152797954280324, + "tokens_seen": 1650589696 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004039638916750251, + "loss": 1.3325, + "theoretical_loss": 3.515267812207739, + "tokens_seen": 1650655232 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040394784353059177, + "loss": 1.2496, + "theoretical_loss": 3.5152558295964162, + "tokens_seen": 1650720768 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004039317953861585, + "loss": 1.2919, + "theoretical_loss": 3.515243847594008, + "tokens_seen": 1650786304 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040391574724172517, + "loss": 1.237, + "theoretical_loss": 3.5152318662004602, + "tokens_seen": 1650851840 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040389969909729193, + "loss": 1.2685, + "theoretical_loss": 3.515219885415717, + "tokens_seen": 1650917376 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004038836509528586, + "loss": 1.2956, + "theoretical_loss": 3.515207905239724, + "tokens_seen": 1650982912 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040386760280842533, + "loss": 1.2047, + "theoretical_loss": 3.5151959256724252, + "tokens_seen": 1651048448 + }, + { + "epoch": 0.5, + "learning_rate": 0.000403851554663992, + "loss": 1.3302, + "theoretical_loss": 3.5151839467137656, + "tokens_seen": 1651113984 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040383550651955873, + "loss": 1.2759, + "theoretical_loss": 3.515171968363691, + "tokens_seen": 1651179520 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004038194583751254, + "loss": 1.3293, + "theoretical_loss": 3.515159990622146, + "tokens_seen": 1651245056 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004038034102306921, + "loss": 1.1966, + "theoretical_loss": 3.5151480134890747, + "tokens_seen": 1651310592 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004037873620862588, + "loss": 1.262, + "theoretical_loss": 3.515136036964423, + "tokens_seen": 1651376128 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004037713139418255, + "loss": 1.2622, + "theoretical_loss": 3.5151240610481356, + "tokens_seen": 1651441664 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 553645, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1858956813812256, + "objective/train/theoretical_loss": 3.5151120857401574, + "objective/train/tokens_used": 1671967200, + "theoretical_loss": 3.5151120857401574, + "tokens_seen": 1651507200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004037552657973922, + "loss": 1.194, + "theoretical_loss": 3.5151120857401574, + "tokens_seen": 1651507200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004037392176529589, + "loss": 1.3638, + "theoretical_loss": 3.5151001110404336, + "tokens_seen": 1651572736 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004037231695085256, + "loss": 1.2681, + "theoretical_loss": 3.5150881369489086, + "tokens_seen": 1651638272 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004037071213640923, + "loss": 1.2787, + "theoretical_loss": 3.515076163465528, + "tokens_seen": 1651703808 + }, + { + "epoch": 0.5, + "learning_rate": 0.000403691073219659, + "loss": 1.2985, + "theoretical_loss": 3.5150641905902367, + "tokens_seen": 1651769344 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004036750250752257, + "loss": 1.2556, + "theoretical_loss": 3.515052218322979, + "tokens_seen": 1651834880 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040365897693079233, + "loss": 1.2907, + "theoretical_loss": 3.5150402466637, + "tokens_seen": 1651900416 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004036429287863591, + "loss": 1.2359, + "theoretical_loss": 3.515028275612346, + "tokens_seen": 1651965952 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040362688064192573, + "loss": 1.3341, + "theoretical_loss": 3.5150163051688605, + "tokens_seen": 1652031488 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004036108324974925, + "loss": 1.2262, + "theoretical_loss": 3.5150043353331895, + "tokens_seen": 1652097024 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040359478435305924, + "loss": 1.4032, + "theoretical_loss": 3.514992366105277, + "tokens_seen": 1652162560 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004035787362086259, + "loss": 1.2673, + "theoretical_loss": 3.5149803974850697, + "tokens_seen": 1652228096 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040356268806419264, + "loss": 1.2494, + "theoretical_loss": 3.5149684294725114, + "tokens_seen": 1652293632 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004035466399197593, + "loss": 1.3535, + "theoretical_loss": 3.5149564620675466, + "tokens_seen": 1652359168 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040353059177532604, + "loss": 1.3012, + "theoretical_loss": 3.514944495270122, + "tokens_seen": 1652424704 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004035145436308927, + "loss": 1.4052, + "theoretical_loss": 3.5149325290801814, + "tokens_seen": 1652490240 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040349849548645944, + "loss": 1.3336, + "theoretical_loss": 3.5149205634976703, + "tokens_seen": 1652555776 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004034824473420261, + "loss": 1.3278, + "theoretical_loss": 3.514908598522534, + "tokens_seen": 1652621312 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040346639919759284, + "loss": 1.2521, + "theoretical_loss": 3.5148966341547174, + "tokens_seen": 1652686848 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004034503510531595, + "loss": 1.2629, + "theoretical_loss": 3.5148846703941654, + "tokens_seen": 1652752384 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040343430290872625, + "loss": 1.2761, + "theoretical_loss": 3.514872707240823, + "tokens_seen": 1652817920 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004034182547642929, + "loss": 1.4867, + "theoretical_loss": 3.5148607446946354, + "tokens_seen": 1652883456 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040340220661985965, + "loss": 1.3313, + "theoretical_loss": 3.514848782755548, + "tokens_seen": 1652948992 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004033861584754263, + "loss": 1.2909, + "theoretical_loss": 3.514836821423506, + "tokens_seen": 1653014528 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040337011033099305, + "loss": 1.2385, + "theoretical_loss": 3.5148248606984547, + "tokens_seen": 1653080064 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004033540621865597, + "loss": 1.2123, + "theoretical_loss": 3.5148129005803384, + "tokens_seen": 1653145600 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040333801404212645, + "loss": 1.1998, + "theoretical_loss": 3.514800941069103, + "tokens_seen": 1653211136 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004033219658976931, + "loss": 1.2844, + "theoretical_loss": 3.5147889821646925, + "tokens_seen": 1653276672 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040330591775325985, + "loss": 1.2154, + "theoretical_loss": 3.5147770238670537, + "tokens_seen": 1653342208 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004032898696088265, + "loss": 1.279, + "theoretical_loss": 3.51476506617613, + "tokens_seen": 1653407744 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040327382146439325, + "loss": 1.2855, + "theoretical_loss": 3.5147531090918687, + "tokens_seen": 1653473280 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004032577733199599, + "loss": 1.2695, + "theoretical_loss": 3.514741152614213, + "tokens_seen": 1653538816 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040324172517552665, + "loss": 1.2805, + "theoretical_loss": 3.5147291967431094, + "tokens_seen": 1653604352 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004032256770310933, + "loss": 1.2239, + "theoretical_loss": 3.514717241478502, + "tokens_seen": 1653669888 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040320962888666005, + "loss": 1.2177, + "theoretical_loss": 3.5147052868203374, + "tokens_seen": 1653735424 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004031935807422267, + "loss": 1.3711, + "theoretical_loss": 3.5146933327685597, + "tokens_seen": 1653800960 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004031775325977934, + "loss": 1.3291, + "theoretical_loss": 3.5146813793231138, + "tokens_seen": 1653866496 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004031614844533601, + "loss": 1.2799, + "theoretical_loss": 3.5146694264839464, + "tokens_seen": 1653932032 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004031454363089268, + "loss": 1.2196, + "theoretical_loss": 3.5146574742510013, + "tokens_seen": 1653997568 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004031293881644935, + "loss": 1.2649, + "theoretical_loss": 3.5146455226242246, + "tokens_seen": 1654063104 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004031133400200602, + "loss": 1.2337, + "theoretical_loss": 3.514633571603561, + "tokens_seen": 1654128640 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004030972918756269, + "loss": 1.2707, + "theoretical_loss": 3.514621621188956, + "tokens_seen": 1654194176 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004030812437311936, + "loss": 1.4027, + "theoretical_loss": 3.514609671380355, + "tokens_seen": 1654259712 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004030651955867603, + "loss": 1.2711, + "theoretical_loss": 3.5145977221777027, + "tokens_seen": 1654325248 + }, + { + "epoch": 0.5, + "learning_rate": 0.000403049147442327, + "loss": 1.3762, + "theoretical_loss": 3.5145857735809454, + "tokens_seen": 1654390784 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040303309929789365, + "loss": 1.2744, + "theoretical_loss": 3.514573825590027, + "tokens_seen": 1654456320 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004030170511534604, + "loss": 1.2372, + "theoretical_loss": 3.5145618782048937, + "tokens_seen": 1654521856 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040300100300902705, + "loss": 1.3249, + "theoretical_loss": 3.514549931425491, + "tokens_seen": 1654587392 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004029849548645938, + "loss": 1.2668, + "theoretical_loss": 3.5145379852517635, + "tokens_seen": 1654652928 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040296890672016046, + "loss": 1.3107, + "theoretical_loss": 3.5145260396836573, + "tokens_seen": 1654718464 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 554730, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0662038326263428, + "objective/train/theoretical_loss": 3.514514094721117, + "objective/train/tokens_used": 1675244000, + "theoretical_loss": 3.514514094721117, + "tokens_seen": 1654784000 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004029528585757272, + "loss": 1.2191, + "theoretical_loss": 3.514514094721117, + "tokens_seen": 1654784000 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040293681043129386, + "loss": 1.2078, + "theoretical_loss": 3.514502150364088, + "tokens_seen": 1654849536 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004029207622868606, + "loss": 1.2636, + "theoretical_loss": 3.514490206612516, + "tokens_seen": 1654915072 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040290471414242726, + "loss": 1.2449, + "theoretical_loss": 3.514478263466346, + "tokens_seen": 1654980608 + }, + { + "epoch": 0.5, + "learning_rate": 0.000402888665997994, + "loss": 1.2674, + "theoretical_loss": 3.5144663209255236, + "tokens_seen": 1655046144 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040287261785356066, + "loss": 1.3643, + "theoretical_loss": 3.514454378989994, + "tokens_seen": 1655111680 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004028565697091274, + "loss": 1.3403, + "theoretical_loss": 3.514442437659702, + "tokens_seen": 1655177216 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040284052156469406, + "loss": 1.3899, + "theoretical_loss": 3.5144304969345947, + "tokens_seen": 1655242752 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004028244734202608, + "loss": 1.3013, + "theoretical_loss": 3.5144185568146153, + "tokens_seen": 1655308288 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040280842527582746, + "loss": 1.3463, + "theoretical_loss": 3.5144066172997106, + "tokens_seen": 1655373824 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004027923771313942, + "loss": 1.4002, + "theoretical_loss": 3.5143946783898254, + "tokens_seen": 1655439360 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040277632898696086, + "loss": 1.2795, + "theoretical_loss": 3.5143827400849057, + "tokens_seen": 1655504896 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004027602808425276, + "loss": 1.3027, + "theoretical_loss": 3.514370802384896, + "tokens_seen": 1655570432 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040274423269809426, + "loss": 1.3428, + "theoretical_loss": 3.514358865289742, + "tokens_seen": 1655635968 + }, + { + "epoch": 0.5, + "learning_rate": 0.000402728184553661, + "loss": 1.2307, + "theoretical_loss": 3.51434692879939, + "tokens_seen": 1655701504 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040271213640922767, + "loss": 1.2406, + "theoretical_loss": 3.514334992913784, + "tokens_seen": 1655767040 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004026960882647944, + "loss": 1.3787, + "theoretical_loss": 3.5143230576328706, + "tokens_seen": 1655832576 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040268004012036107, + "loss": 1.271, + "theoretical_loss": 3.5143111229565944, + "tokens_seen": 1655898112 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004026639919759278, + "loss": 1.2659, + "theoretical_loss": 3.514299188884902, + "tokens_seen": 1655963648 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040264794383149447, + "loss": 1.2307, + "theoretical_loss": 3.514287255417737, + "tokens_seen": 1656029184 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004026318956870612, + "loss": 1.2847, + "theoretical_loss": 3.5142753225550463, + "tokens_seen": 1656094720 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040261584754262787, + "loss": 1.2566, + "theoretical_loss": 3.514263390296775, + "tokens_seen": 1656160256 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004025997993981946, + "loss": 1.2322, + "theoretical_loss": 3.5142514586428684, + "tokens_seen": 1656225792 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040258375125376127, + "loss": 1.2389, + "theoretical_loss": 3.5142395275932725, + "tokens_seen": 1656291328 + }, + { + "epoch": 0.5, + "learning_rate": 0.000402567703109328, + "loss": 1.3053, + "theoretical_loss": 3.514227597147932, + "tokens_seen": 1656356864 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040255165496489467, + "loss": 1.2863, + "theoretical_loss": 3.514215667306793, + "tokens_seen": 1656422400 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040253560682046143, + "loss": 1.2838, + "theoretical_loss": 3.514203738069801, + "tokens_seen": 1656487936 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004025195586760281, + "loss": 1.308, + "theoretical_loss": 3.514191809436901, + "tokens_seen": 1656553472 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040250351053159483, + "loss": 1.2878, + "theoretical_loss": 3.5141798814080385, + "tokens_seen": 1656619008 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004024874623871615, + "loss": 1.3471, + "theoretical_loss": 3.5141679539831596, + "tokens_seen": 1656684544 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040247141424272823, + "loss": 1.4304, + "theoretical_loss": 3.5141560271622097, + "tokens_seen": 1656750080 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004024553660982949, + "loss": 1.233, + "theoretical_loss": 3.514144100945134, + "tokens_seen": 1656815616 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040243931795386163, + "loss": 1.3339, + "theoretical_loss": 3.5141321753318784, + "tokens_seen": 1656881152 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004024232698094283, + "loss": 1.1742, + "theoretical_loss": 3.514120250322388, + "tokens_seen": 1656946688 + }, + { + "epoch": 0.5, + "learning_rate": 0.000402407221664995, + "loss": 1.2235, + "theoretical_loss": 3.5141083259166086, + "tokens_seen": 1657012224 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040239117352056173, + "loss": 1.2988, + "theoretical_loss": 3.514096402114486, + "tokens_seen": 1657077760 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004023751253761284, + "loss": 1.3111, + "theoretical_loss": 3.514084478915966, + "tokens_seen": 1657143296 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040235907723169513, + "loss": 1.2148, + "theoretical_loss": 3.514072556320993, + "tokens_seen": 1657208832 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004023430290872618, + "loss": 1.34, + "theoretical_loss": 3.5140606343295135, + "tokens_seen": 1657274368 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040232698094282854, + "loss": 1.1875, + "theoretical_loss": 3.514048712941473, + "tokens_seen": 1657339904 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004023109327983952, + "loss": 1.2747, + "theoretical_loss": 3.514036792156817, + "tokens_seen": 1657405440 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040229488465396194, + "loss": 1.2087, + "theoretical_loss": 3.514024871975491, + "tokens_seen": 1657470976 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004022788365095286, + "loss": 1.3389, + "theoretical_loss": 3.514012952397441, + "tokens_seen": 1657536512 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040226278836509534, + "loss": 1.3444, + "theoretical_loss": 3.5140010334226117, + "tokens_seen": 1657602048 + }, + { + "epoch": 0.5, + "learning_rate": 0.000402246740220662, + "loss": 1.2374, + "theoretical_loss": 3.5139891150509497, + "tokens_seen": 1657667584 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040223069207622874, + "loss": 1.372, + "theoretical_loss": 3.5139771972824008, + "tokens_seen": 1657733120 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004022146439317954, + "loss": 1.2957, + "theoretical_loss": 3.513965280116909, + "tokens_seen": 1657798656 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040219859578736214, + "loss": 1.3287, + "theoretical_loss": 3.513953363554422, + "tokens_seen": 1657864192 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004021825476429288, + "loss": 1.2013, + "theoretical_loss": 3.5139414475948842, + "tokens_seen": 1657929728 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040216649949849554, + "loss": 1.2265, + "theoretical_loss": 3.513929532238242, + "tokens_seen": 1657995264 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 555622, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.201681137084961, + "objective/train/theoretical_loss": 3.5139176174844398, + "objective/train/tokens_used": 1678520800, + "theoretical_loss": 3.5139176174844398, + "tokens_seen": 1658060800 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004021504513540622, + "loss": 1.2452, + "theoretical_loss": 3.5139176174844398, + "tokens_seen": 1658060800 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040213440320962894, + "loss": 1.2517, + "theoretical_loss": 3.5139057033334247, + "tokens_seen": 1658126336 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004021183550651956, + "loss": 1.2518, + "theoretical_loss": 3.5138937897851417, + "tokens_seen": 1658191872 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004021183550651956, + "loss": 1.2347, + "theoretical_loss": 3.5138818768395366, + "tokens_seen": 1658257408 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040210230692076234, + "loss": 1.213, + "theoretical_loss": 3.513869964496555, + "tokens_seen": 1658322944 + }, + { + "epoch": 0.5, + "learning_rate": 0.000402086258776329, + "loss": 1.385, + "theoretical_loss": 3.513858052756143, + "tokens_seen": 1658388480 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040207021063189575, + "loss": 1.3498, + "theoretical_loss": 3.513846141618246, + "tokens_seen": 1658454016 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004020541624874624, + "loss": 1.3418, + "theoretical_loss": 3.513834231082809, + "tokens_seen": 1658519552 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040203811434302915, + "loss": 1.2901, + "theoretical_loss": 3.513822321149779, + "tokens_seen": 1658585088 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004020220661985958, + "loss": 1.2277, + "theoretical_loss": 3.5138104118191014, + "tokens_seen": 1658650624 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040200601805416255, + "loss": 1.2502, + "theoretical_loss": 3.5137985030907215, + "tokens_seen": 1658716160 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004019899699097292, + "loss": 1.328, + "theoretical_loss": 3.513786594964585, + "tokens_seen": 1658781696 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040197392176529595, + "loss": 1.3518, + "theoretical_loss": 3.5137746874406384, + "tokens_seen": 1658847232 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004019578736208626, + "loss": 1.3002, + "theoretical_loss": 3.5137627805188263, + "tokens_seen": 1658912768 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040194182547642935, + "loss": 1.2317, + "theoretical_loss": 3.5137508741990953, + "tokens_seen": 1658978304 + }, + { + "epoch": 0.5, + "learning_rate": 0.000401925777331996, + "loss": 1.2738, + "theoretical_loss": 3.5137389684813916, + "tokens_seen": 1659043840 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040190972918756275, + "loss": 1.3161, + "theoretical_loss": 3.5137270633656597, + "tokens_seen": 1659109376 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004018936810431294, + "loss": 1.2779, + "theoretical_loss": 3.5137151588518467, + "tokens_seen": 1659174912 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040187763289869615, + "loss": 1.3001, + "theoretical_loss": 3.513703254939897, + "tokens_seen": 1659240448 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004018615847542628, + "loss": 1.2651, + "theoretical_loss": 3.513691351629758, + "tokens_seen": 1659305984 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040184553660982955, + "loss": 1.2889, + "theoretical_loss": 3.5136794489213736, + "tokens_seen": 1659371520 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004018294884653962, + "loss": 1.287, + "theoretical_loss": 3.5136675468146916, + "tokens_seen": 1659437056 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040181344032096296, + "loss": 1.2384, + "theoretical_loss": 3.513655645309657, + "tokens_seen": 1659502592 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004017973921765296, + "loss": 1.3519, + "theoretical_loss": 3.513643744406215, + "tokens_seen": 1659568128 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004017813440320963, + "loss": 1.3794, + "theoretical_loss": 3.513631844104312, + "tokens_seen": 1659633664 + }, + { + "epoch": 0.5, + "learning_rate": 0.000401765295887663, + "loss": 1.2307, + "theoretical_loss": 3.513619944403894, + "tokens_seen": 1659699200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004017492477432297, + "loss": 1.3276, + "theoretical_loss": 3.5136080453049066, + "tokens_seen": 1659764736 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004017331995987964, + "loss": 1.1935, + "theoretical_loss": 3.513596146807296, + "tokens_seen": 1659830272 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004017171514543631, + "loss": 1.2553, + "theoretical_loss": 3.513584248911007, + "tokens_seen": 1659895808 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004017011033099298, + "loss": 1.2243, + "theoretical_loss": 3.513572351615987, + "tokens_seen": 1659961344 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004016850551654965, + "loss": 1.328, + "theoretical_loss": 3.513560454922181, + "tokens_seen": 1660026880 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004016690070210632, + "loss": 1.2381, + "theoretical_loss": 3.5135485588295348, + "tokens_seen": 1660092416 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004016529588766299, + "loss": 1.2497, + "theoretical_loss": 3.513536663337995, + "tokens_seen": 1660157952 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040163691073219655, + "loss": 1.3023, + "theoretical_loss": 3.513524768447507, + "tokens_seen": 1660223488 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004016208625877633, + "loss": 1.2794, + "theoretical_loss": 3.5135128741580157, + "tokens_seen": 1660289024 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040160481444332996, + "loss": 1.2542, + "theoretical_loss": 3.513500980469469, + "tokens_seen": 1660354560 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004015887662988967, + "loss": 1.308, + "theoretical_loss": 3.513489087381812, + "tokens_seen": 1660420096 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040157271815446336, + "loss": 1.3863, + "theoretical_loss": 3.51347719489499, + "tokens_seen": 1660485632 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004015566700100301, + "loss": 1.302, + "theoretical_loss": 3.513465303008949, + "tokens_seen": 1660551168 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040154062186559676, + "loss": 1.3184, + "theoretical_loss": 3.513453411723636, + "tokens_seen": 1660616704 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004015245737211635, + "loss": 1.3139, + "theoretical_loss": 3.513441521038996, + "tokens_seen": 1660682240 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040150852557673016, + "loss": 1.2198, + "theoretical_loss": 3.5134296309549757, + "tokens_seen": 1660747776 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004014924774322969, + "loss": 1.2535, + "theoretical_loss": 3.51341774147152, + "tokens_seen": 1660813312 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040147642928786356, + "loss": 1.2408, + "theoretical_loss": 3.513405852588576, + "tokens_seen": 1660878848 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004014603811434303, + "loss": 1.3916, + "theoretical_loss": 3.513393964306089, + "tokens_seen": 1660944384 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040144433299899696, + "loss": 1.3197, + "theoretical_loss": 3.513382076624005, + "tokens_seen": 1661009920 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004014282848545637, + "loss": 1.1997, + "theoretical_loss": 3.51337018954227, + "tokens_seen": 1661075456 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040141223671013036, + "loss": 1.2186, + "theoretical_loss": 3.5133583030608304, + "tokens_seen": 1661140992 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004013961885656971, + "loss": 1.3315, + "theoretical_loss": 3.513346417179632, + "tokens_seen": 1661206528 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040138014042126376, + "loss": 1.3006, + "theoretical_loss": 3.5133345318986207, + "tokens_seen": 1661272064 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 556798, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1186680793762207, + "objective/train/theoretical_loss": 3.513322647217742, + "objective/train/tokens_used": 1681797600, + "theoretical_loss": 3.513322647217742, + "tokens_seen": 1661337600 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004013640922768305, + "loss": 1.1569, + "theoretical_loss": 3.513322647217742, + "tokens_seen": 1661337600 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040134804413239717, + "loss": 1.2233, + "theoretical_loss": 3.5133107631369436, + "tokens_seen": 1661403136 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004013319959879639, + "loss": 1.3372, + "theoretical_loss": 3.51329887965617, + "tokens_seen": 1661468672 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040131594784353057, + "loss": 1.1737, + "theoretical_loss": 3.513286996775367, + "tokens_seen": 1661534208 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004012998996990973, + "loss": 1.3467, + "theoretical_loss": 3.513275114494482, + "tokens_seen": 1661599744 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040128385155466397, + "loss": 1.327, + "theoretical_loss": 3.5132632328134603, + "tokens_seen": 1661665280 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004012678034102307, + "loss": 1.333, + "theoretical_loss": 3.5132513517322477, + "tokens_seen": 1661730816 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040125175526579737, + "loss": 1.3408, + "theoretical_loss": 3.513239471250791, + "tokens_seen": 1661796352 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004012357071213641, + "loss": 1.2169, + "theoretical_loss": 3.513227591369035, + "tokens_seen": 1661861888 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040121965897693077, + "loss": 1.2781, + "theoretical_loss": 3.513215712086928, + "tokens_seen": 1661927424 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004012036108324975, + "loss": 1.3716, + "theoretical_loss": 3.513203833404414, + "tokens_seen": 1661992960 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004011875626880643, + "loss": 1.299, + "theoretical_loss": 3.5131919553214397, + "tokens_seen": 1662058496 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004011715145436309, + "loss": 1.2452, + "theoretical_loss": 3.513180077837951, + "tokens_seen": 1662124032 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040115546639919763, + "loss": 1.2357, + "theoretical_loss": 3.5131682009538947, + "tokens_seen": 1662189568 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040113941825476433, + "loss": 1.3702, + "theoretical_loss": 3.513156324669217, + "tokens_seen": 1662255104 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040112337011033103, + "loss": 1.2116, + "theoretical_loss": 3.513144448983863, + "tokens_seen": 1662320640 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040110732196589773, + "loss": 1.3076, + "theoretical_loss": 3.5131325738977797, + "tokens_seen": 1662386176 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040109127382146443, + "loss": 1.2959, + "theoretical_loss": 3.513120699410913, + "tokens_seen": 1662451712 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040107522567703113, + "loss": 1.3756, + "theoretical_loss": 3.513108825523209, + "tokens_seen": 1662517248 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040105917753259783, + "loss": 1.2802, + "theoretical_loss": 3.513096952234613, + "tokens_seen": 1662582784 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040104312938816453, + "loss": 1.3151, + "theoretical_loss": 3.513085079545073, + "tokens_seen": 1662648320 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040102708124373123, + "loss": 1.3457, + "theoretical_loss": 3.513073207454534, + "tokens_seen": 1662713856 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004010110330992979, + "loss": 1.36, + "theoretical_loss": 3.5130613359629415, + "tokens_seen": 1662779392 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040099498495486463, + "loss": 1.2871, + "theoretical_loss": 3.5130494650702433, + "tokens_seen": 1662844928 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004009789368104313, + "loss": 1.3447, + "theoretical_loss": 3.5130375947763843, + "tokens_seen": 1662910464 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040096288866599804, + "loss": 1.3518, + "theoretical_loss": 3.513025725081312, + "tokens_seen": 1662976000 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004009468405215647, + "loss": 1.3244, + "theoretical_loss": 3.513013855984971, + "tokens_seen": 1663041536 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040093079237713144, + "loss": 1.2671, + "theoretical_loss": 3.513001987487309, + "tokens_seen": 1663107072 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004009147442326981, + "loss": 1.2597, + "theoretical_loss": 3.5129901195882707, + "tokens_seen": 1663172608 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040089869608826484, + "loss": 1.3389, + "theoretical_loss": 3.5129782522878035, + "tokens_seen": 1663238144 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004008826479438315, + "loss": 1.4299, + "theoretical_loss": 3.512966385585853, + "tokens_seen": 1663303680 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040086659979939824, + "loss": 1.2636, + "theoretical_loss": 3.512954519482366, + "tokens_seen": 1663369216 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004008505516549649, + "loss": 1.3397, + "theoretical_loss": 3.512942653977288, + "tokens_seen": 1663434752 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040083450351053164, + "loss": 1.2753, + "theoretical_loss": 3.512930789070566, + "tokens_seen": 1663500288 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004008184553660983, + "loss": 1.3264, + "theoretical_loss": 3.512918924762146, + "tokens_seen": 1663565824 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040080240722166504, + "loss": 1.2388, + "theoretical_loss": 3.5129070610519744, + "tokens_seen": 1663631360 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004007863590772317, + "loss": 1.2611, + "theoretical_loss": 3.5128951979399967, + "tokens_seen": 1663696896 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040077031093279844, + "loss": 1.3229, + "theoretical_loss": 3.51288333542616, + "tokens_seen": 1663762432 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004007542627883651, + "loss": 1.3436, + "theoretical_loss": 3.51287147351041, + "tokens_seen": 1663827968 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040073821464393184, + "loss": 1.3225, + "theoretical_loss": 3.5128596121926936, + "tokens_seen": 1663893504 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004007221664994985, + "loss": 1.3475, + "theoretical_loss": 3.512847751472957, + "tokens_seen": 1663959040 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040070611835506525, + "loss": 1.3899, + "theoretical_loss": 3.5128358913511457, + "tokens_seen": 1664024576 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004006900702106319, + "loss": 1.2378, + "theoretical_loss": 3.5128240318272073, + "tokens_seen": 1664090112 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040067402206619865, + "loss": 1.2986, + "theoretical_loss": 3.512812172901087, + "tokens_seen": 1664155648 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004006579739217653, + "loss": 1.2335, + "theoretical_loss": 3.512800314572732, + "tokens_seen": 1664221184 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040064192577733205, + "loss": 1.2466, + "theoretical_loss": 3.5127884568420873, + "tokens_seen": 1664286720 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004006258776328987, + "loss": 1.3123, + "theoretical_loss": 3.5127765997091007, + "tokens_seen": 1664352256 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040060982948846545, + "loss": 1.3587, + "theoretical_loss": 3.512764743173718, + "tokens_seen": 1664417792 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004005937813440321, + "loss": 1.2871, + "theoretical_loss": 3.5127528872358855, + "tokens_seen": 1664483328 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040057773319959885, + "loss": 1.2303, + "theoretical_loss": 3.5127410318955494, + "tokens_seen": 1664548864 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 558020, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1752697229385376, + "objective/train/theoretical_loss": 3.5127291771526563, + "objective/train/tokens_used": 1685074400, + "theoretical_loss": 3.5127291771526563, + "tokens_seen": 1664614400 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004005616850551655, + "loss": 1.1725, + "theoretical_loss": 3.5127291771526563, + "tokens_seen": 1664614400 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040054563691073225, + "loss": 1.2851, + "theoretical_loss": 3.512717323007153, + "tokens_seen": 1664679936 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004005295887662989, + "loss": 1.2361, + "theoretical_loss": 3.512705469458985, + "tokens_seen": 1664745472 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040051354062186565, + "loss": 1.2552, + "theoretical_loss": 3.512693616508099, + "tokens_seen": 1664811008 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004004974924774323, + "loss": 1.366, + "theoretical_loss": 3.512681764154441, + "tokens_seen": 1664876544 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040048144433299905, + "loss": 1.2966, + "theoretical_loss": 3.512669912397959, + "tokens_seen": 1664942080 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004004653961885657, + "loss": 1.2526, + "theoretical_loss": 3.5126580612385974, + "tokens_seen": 1665007616 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040044934804413246, + "loss": 1.2695, + "theoretical_loss": 3.5126462106763037, + "tokens_seen": 1665073152 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004004332998996991, + "loss": 1.224, + "theoretical_loss": 3.512634360711024, + "tokens_seen": 1665138688 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040041725175526586, + "loss": 1.3352, + "theoretical_loss": 3.5126225113427054, + "tokens_seen": 1665204224 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004004012036108325, + "loss": 1.3257, + "theoretical_loss": 3.5126106625712934, + "tokens_seen": 1665269760 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004003851554663992, + "loss": 1.2553, + "theoretical_loss": 3.5125988143967346, + "tokens_seen": 1665335296 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004003691073219659, + "loss": 1.279, + "theoretical_loss": 3.5125869668189758, + "tokens_seen": 1665400832 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004003530591775326, + "loss": 1.2295, + "theoretical_loss": 3.5125751198379636, + "tokens_seen": 1665466368 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004003370110330993, + "loss": 1.2436, + "theoretical_loss": 3.512563273453644, + "tokens_seen": 1665531904 + }, + { + "epoch": 0.5, + "learning_rate": 0.000400320962888666, + "loss": 1.169, + "theoretical_loss": 3.5125514276659637, + "tokens_seen": 1665597440 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004003049147442327, + "loss": 1.2532, + "theoretical_loss": 3.512539582474869, + "tokens_seen": 1665662976 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004002888665997994, + "loss": 1.3368, + "theoretical_loss": 3.5125277378803066, + "tokens_seen": 1665728512 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004002728184553661, + "loss": 1.3694, + "theoretical_loss": 3.512515893882223, + "tokens_seen": 1665794048 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004002567703109328, + "loss": 1.251, + "theoretical_loss": 3.5125040504805645, + "tokens_seen": 1665859584 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040024072216649946, + "loss": 1.3366, + "theoretical_loss": 3.512492207675278, + "tokens_seen": 1665925120 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004002246740220662, + "loss": 1.2784, + "theoretical_loss": 3.5124803654663093, + "tokens_seen": 1665990656 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040020862587763286, + "loss": 1.3781, + "theoretical_loss": 3.5124685238536055, + "tokens_seen": 1666056192 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004001925777331996, + "loss": 1.3585, + "theoretical_loss": 3.512456682837113, + "tokens_seen": 1666121728 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040017652958876626, + "loss": 1.2931, + "theoretical_loss": 3.5124448424167785, + "tokens_seen": 1666187264 + }, + { + "epoch": 0.5, + "learning_rate": 0.000400160481444333, + "loss": 1.259, + "theoretical_loss": 3.5124330025925476, + "tokens_seen": 1666252800 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040014443329989966, + "loss": 1.2293, + "theoretical_loss": 3.512421163364368, + "tokens_seen": 1666318336 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004001283851554664, + "loss": 1.2357, + "theoretical_loss": 3.512409324732186, + "tokens_seen": 1666383872 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040011233701103306, + "loss": 1.3073, + "theoretical_loss": 3.512397486695948, + "tokens_seen": 1666449408 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004000962888665998, + "loss": 1.2175, + "theoretical_loss": 3.5123856492556005, + "tokens_seen": 1666514944 + }, + { + "epoch": 0.51, + "learning_rate": 0.00040008024072216646, + "loss": 1.246, + "theoretical_loss": 3.5123738124110897, + "tokens_seen": 1666580480 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004000641925777332, + "loss": 1.2875, + "theoretical_loss": 3.5123619761623632, + "tokens_seen": 1666646016 + }, + { + "epoch": 0.51, + "learning_rate": 0.00040004814443329986, + "loss": 1.2551, + "theoretical_loss": 3.5123501405093664, + "tokens_seen": 1666711552 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004000320962888666, + "loss": 1.3639, + "theoretical_loss": 3.5123383054520474, + "tokens_seen": 1666777088 + }, + { + "epoch": 0.51, + "learning_rate": 0.00040001604814443326, + "loss": 1.2718, + "theoretical_loss": 3.5123264709903514, + "tokens_seen": 1666842624 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004, + "loss": 1.317, + "theoretical_loss": 3.5123146371242253, + "tokens_seen": 1666908160 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003999839518555667, + "loss": 1.2167, + "theoretical_loss": 3.5123028038536157, + "tokens_seen": 1666973696 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003999679037111334, + "loss": 1.2887, + "theoretical_loss": 3.51229097117847, + "tokens_seen": 1667039232 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003999518555667001, + "loss": 1.3884, + "theoretical_loss": 3.512279139098734, + "tokens_seen": 1667104768 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003999358074222668, + "loss": 1.247, + "theoretical_loss": 3.512267307614355, + "tokens_seen": 1667170304 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003999197592778335, + "loss": 1.2982, + "theoretical_loss": 3.5122554767252785, + "tokens_seen": 1667235840 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003999037111334002, + "loss": 1.3163, + "theoretical_loss": 3.512243646431452, + "tokens_seen": 1667301376 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003998876629889669, + "loss": 1.2875, + "theoretical_loss": 3.5122318167328226, + "tokens_seen": 1667366912 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003998716148445336, + "loss": 1.2666, + "theoretical_loss": 3.512219987629336, + "tokens_seen": 1667432448 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003998555667001003, + "loss": 1.2264, + "theoretical_loss": 3.5122081591209393, + "tokens_seen": 1667497984 + }, + { + "epoch": 0.51, + "learning_rate": 0.000399839518555667, + "loss": 1.2592, + "theoretical_loss": 3.512196331207579, + "tokens_seen": 1667563520 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003998234704112337, + "loss": 1.3014, + "theoretical_loss": 3.5121845038892023, + "tokens_seen": 1667629056 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003998074222668004, + "loss": 1.3068, + "theoretical_loss": 3.5121726771657555, + "tokens_seen": 1667694592 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039979137412236713, + "loss": 1.2329, + "theoretical_loss": 3.512160851037185, + "tokens_seen": 1667760128 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039977532597793383, + "loss": 1.3256, + "theoretical_loss": 3.512149025503438, + "tokens_seen": 1667825664 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 558924, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3271944522857666, + "objective/train/theoretical_loss": 3.512137200564461, + "objective/train/tokens_used": 1688351200, + "theoretical_loss": 3.512137200564461, + "tokens_seen": 1667891200 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039975927783350053, + "loss": 1.3634, + "theoretical_loss": 3.512137200564461, + "tokens_seen": 1667891200 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039974322968906723, + "loss": 1.3186, + "theoretical_loss": 3.512125376220201, + "tokens_seen": 1667956736 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039972718154463393, + "loss": 1.2646, + "theoretical_loss": 3.5121135524706038, + "tokens_seen": 1668022272 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039971113340020063, + "loss": 1.3485, + "theoretical_loss": 3.5121017293156176, + "tokens_seen": 1668087808 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039969508525576733, + "loss": 1.3102, + "theoretical_loss": 3.512089906755188, + "tokens_seen": 1668153344 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039967903711133403, + "loss": 1.328, + "theoretical_loss": 3.512078084789262, + "tokens_seen": 1668218880 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039966298896690073, + "loss": 1.2124, + "theoretical_loss": 3.5120662634177866, + "tokens_seen": 1668284416 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039964694082246743, + "loss": 1.2866, + "theoretical_loss": 3.512054442640708, + "tokens_seen": 1668349952 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039963089267803413, + "loss": 1.2512, + "theoretical_loss": 3.512042622457974, + "tokens_seen": 1668415488 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039961484453360083, + "loss": 1.3261, + "theoretical_loss": 3.51203080286953, + "tokens_seen": 1668481024 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003995987963891675, + "loss": 1.2457, + "theoretical_loss": 3.512018983875324, + "tokens_seen": 1668546560 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003995827482447342, + "loss": 1.3881, + "theoretical_loss": 3.5120071654753025, + "tokens_seen": 1668612096 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003995667001003009, + "loss": 1.3635, + "theoretical_loss": 3.511995347669412, + "tokens_seen": 1668677632 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003995506519558676, + "loss": 1.3337, + "theoretical_loss": 3.511983530457599, + "tokens_seen": 1668743168 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003995346038114343, + "loss": 1.3834, + "theoretical_loss": 3.511971713839811, + "tokens_seen": 1668808704 + }, + { + "epoch": 0.51, + "learning_rate": 0.000399518555667001, + "loss": 1.3068, + "theoretical_loss": 3.5119598978159945, + "tokens_seen": 1668874240 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003995025075225677, + "loss": 1.2736, + "theoretical_loss": 3.5119480823860965, + "tokens_seen": 1668939776 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003994864593781344, + "loss": 1.2634, + "theoretical_loss": 3.5119362675500634, + "tokens_seen": 1669005312 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003994704112337011, + "loss": 1.2757, + "theoretical_loss": 3.5119244533078424, + "tokens_seen": 1669070848 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003994543630892678, + "loss": 1.2338, + "theoretical_loss": 3.51191263965938, + "tokens_seen": 1669136384 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003994383149448345, + "loss": 1.1499, + "theoretical_loss": 3.511900826604623, + "tokens_seen": 1669201920 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003994222668004012, + "loss": 1.3141, + "theoretical_loss": 3.5118890141435193, + "tokens_seen": 1669267456 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003994062186559679, + "loss": 1.3507, + "theoretical_loss": 3.5118772022760147, + "tokens_seen": 1669332992 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039939017051153464, + "loss": 1.252, + "theoretical_loss": 3.5118653910020563, + "tokens_seen": 1669398528 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039937412236710134, + "loss": 1.2746, + "theoretical_loss": 3.511853580321591, + "tokens_seen": 1669464064 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039935807422266804, + "loss": 1.3705, + "theoretical_loss": 3.511841770234566, + "tokens_seen": 1669529600 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039934202607823475, + "loss": 1.2305, + "theoretical_loss": 3.511829960740928, + "tokens_seen": 1669595136 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039932597793380145, + "loss": 1.266, + "theoretical_loss": 3.5118181518406235, + "tokens_seen": 1669660672 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039930992978936815, + "loss": 1.3566, + "theoretical_loss": 3.5118063435335998, + "tokens_seen": 1669726208 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039929388164493485, + "loss": 1.3382, + "theoretical_loss": 3.5117945358198037, + "tokens_seen": 1669791744 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039927783350050155, + "loss": 1.321, + "theoretical_loss": 3.511782728699182, + "tokens_seen": 1669857280 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039926178535606825, + "loss": 1.2712, + "theoretical_loss": 3.5117709221716815, + "tokens_seen": 1669922816 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039924573721163495, + "loss": 1.2721, + "theoretical_loss": 3.5117591162372497, + "tokens_seen": 1669988352 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039922968906720165, + "loss": 1.3125, + "theoretical_loss": 3.5117473108958337, + "tokens_seen": 1670053888 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039921364092276835, + "loss": 1.3768, + "theoretical_loss": 3.5117355061473794, + "tokens_seen": 1670119424 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039919759277833505, + "loss": 1.2725, + "theoretical_loss": 3.5117237019918344, + "tokens_seen": 1670184960 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039918154463390175, + "loss": 1.2311, + "theoretical_loss": 3.5117118984291458, + "tokens_seen": 1670250496 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039916549648946845, + "loss": 1.2508, + "theoretical_loss": 3.5117000954592603, + "tokens_seen": 1670316032 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039914944834503515, + "loss": 1.2023, + "theoretical_loss": 3.5116882930821247, + "tokens_seen": 1670381568 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039913340020060185, + "loss": 1.2607, + "theoretical_loss": 3.5116764912976866, + "tokens_seen": 1670447104 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039911735205616855, + "loss": 1.4079, + "theoretical_loss": 3.511664690105892, + "tokens_seen": 1670512640 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039910130391173525, + "loss": 1.2866, + "theoretical_loss": 3.5116528895066885, + "tokens_seen": 1670578176 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039908525576730196, + "loss": 1.3169, + "theoretical_loss": 3.5116410895000234, + "tokens_seen": 1670643712 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039906920762286866, + "loss": 1.3179, + "theoretical_loss": 3.5116292900858435, + "tokens_seen": 1670709248 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039905315947843536, + "loss": 1.3108, + "theoretical_loss": 3.5116174912640954, + "tokens_seen": 1670774784 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039903711133400206, + "loss": 1.3305, + "theoretical_loss": 3.5116056930347264, + "tokens_seen": 1670840320 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039902106318956876, + "loss": 1.2188, + "theoretical_loss": 3.5115938953976835, + "tokens_seen": 1670905856 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039900501504513546, + "loss": 1.2628, + "theoretical_loss": 3.511582098352914, + "tokens_seen": 1670971392 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039898896690070216, + "loss": 1.2478, + "theoretical_loss": 3.5115703019003646, + "tokens_seen": 1671036928 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003989729187562688, + "loss": 1.3259, + "theoretical_loss": 3.5115585060399823, + "tokens_seen": 1671102464 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 560108, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.264222502708435, + "objective/train/theoretical_loss": 3.5115467107717144, + "objective/train/tokens_used": 1691628000, + "theoretical_loss": 3.5115467107717144, + "tokens_seen": 1671168000 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003989568706118355, + "loss": 1.215, + "theoretical_loss": 3.5115467107717144, + "tokens_seen": 1671168000 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003989408224674022, + "loss": 1.2237, + "theoretical_loss": 3.5115349160955076, + "tokens_seen": 1671233536 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003989247743229689, + "loss": 1.2811, + "theoretical_loss": 3.5115231220113095, + "tokens_seen": 1671299072 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003989087261785356, + "loss": 1.2688, + "theoretical_loss": 3.5115113285190667, + "tokens_seen": 1671364608 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003988926780341023, + "loss": 1.3173, + "theoretical_loss": 3.511499535618727, + "tokens_seen": 1671430144 + }, + { + "epoch": 0.51, + "learning_rate": 0.000398876629889669, + "loss": 1.3643, + "theoretical_loss": 3.5114877433102363, + "tokens_seen": 1671495680 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003988605817452357, + "loss": 1.3867, + "theoretical_loss": 3.5114759515935425, + "tokens_seen": 1671561216 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003988445336008024, + "loss": 1.4443, + "theoretical_loss": 3.5114641604685928, + "tokens_seen": 1671626752 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003988284854563691, + "loss": 1.2469, + "theoretical_loss": 3.5114523699353337, + "tokens_seen": 1671692288 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003988124373119358, + "loss": 1.299, + "theoretical_loss": 3.511440579993713, + "tokens_seen": 1671757824 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003987963891675025, + "loss": 1.3172, + "theoretical_loss": 3.511428790643677, + "tokens_seen": 1671823360 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003987803410230692, + "loss": 1.4323, + "theoretical_loss": 3.5114170018851736, + "tokens_seen": 1671888896 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003987642928786359, + "loss": 1.3876, + "theoretical_loss": 3.51140521371815, + "tokens_seen": 1671954432 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003987482447342026, + "loss": 1.2385, + "theoretical_loss": 3.511393426142553, + "tokens_seen": 1672019968 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003987321965897693, + "loss": 1.3177, + "theoretical_loss": 3.511381639158329, + "tokens_seen": 1672085504 + }, + { + "epoch": 0.51, + "learning_rate": 0.000398716148445336, + "loss": 1.3065, + "theoretical_loss": 3.5113698527654265, + "tokens_seen": 1672151040 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003987001003009027, + "loss": 1.29, + "theoretical_loss": 3.5113580669637923, + "tokens_seen": 1672216576 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003986840521564694, + "loss": 1.2656, + "theoretical_loss": 3.5113462817533727, + "tokens_seen": 1672282112 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003986680040120361, + "loss": 1.2402, + "theoretical_loss": 3.511334497134116, + "tokens_seen": 1672347648 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003986519558676028, + "loss": 1.3029, + "theoretical_loss": 3.5113227131059688, + "tokens_seen": 1672413184 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003986359077231695, + "loss": 1.2444, + "theoretical_loss": 3.511310929668878, + "tokens_seen": 1672478720 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003986198595787362, + "loss": 1.435, + "theoretical_loss": 3.5112991468227914, + "tokens_seen": 1672544256 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003986038114343029, + "loss": 1.2414, + "theoretical_loss": 3.5112873645676554, + "tokens_seen": 1672609792 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003985877632898696, + "loss": 1.314, + "theoretical_loss": 3.511275582903419, + "tokens_seen": 1672675328 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003985717151454363, + "loss": 1.3716, + "theoretical_loss": 3.5112638018300273, + "tokens_seen": 1672740864 + }, + { + "epoch": 0.51, + "learning_rate": 0.000398555667001003, + "loss": 1.2773, + "theoretical_loss": 3.5112520213474285, + "tokens_seen": 1672806400 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003985396188565697, + "loss": 1.2805, + "theoretical_loss": 3.51124024145557, + "tokens_seen": 1672871936 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003985235707121364, + "loss": 1.2475, + "theoretical_loss": 3.5112284621543983, + "tokens_seen": 1672937472 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003985075225677031, + "loss": 1.3827, + "theoretical_loss": 3.5112166834438616, + "tokens_seen": 1673003008 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003984914744232698, + "loss": 1.3495, + "theoretical_loss": 3.5112049053239067, + "tokens_seen": 1673068544 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003984754262788365, + "loss": 1.2697, + "theoretical_loss": 3.511193127794481, + "tokens_seen": 1673134080 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003984593781344032, + "loss": 1.1476, + "theoretical_loss": 3.511181350855531, + "tokens_seen": 1673199616 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003984433299899699, + "loss": 1.2646, + "theoretical_loss": 3.511169574507005, + "tokens_seen": 1673265152 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039842728184553663, + "loss": 1.2672, + "theoretical_loss": 3.5111577987488496, + "tokens_seen": 1673330688 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039841123370110333, + "loss": 1.2964, + "theoretical_loss": 3.5111460235810124, + "tokens_seen": 1673396224 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039839518555667003, + "loss": 1.305, + "theoretical_loss": 3.5111342490034407, + "tokens_seen": 1673461760 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039837913741223673, + "loss": 1.3364, + "theoretical_loss": 3.511122475016082, + "tokens_seen": 1673527296 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039836308926780343, + "loss": 1.2294, + "theoretical_loss": 3.5111107016188825, + "tokens_seen": 1673592832 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039834704112337013, + "loss": 1.3126, + "theoretical_loss": 3.5110989288117906, + "tokens_seen": 1673658368 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039833099297893683, + "loss": 1.315, + "theoretical_loss": 3.511087156594754, + "tokens_seen": 1673723904 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039831494483450353, + "loss": 1.3515, + "theoretical_loss": 3.5110753849677185, + "tokens_seen": 1673789440 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039829889669007023, + "loss": 1.3381, + "theoretical_loss": 3.5110636139306326, + "tokens_seen": 1673854976 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039828284854563693, + "loss": 1.4816, + "theoretical_loss": 3.5110518434834432, + "tokens_seen": 1673920512 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039826680040120363, + "loss": 1.2884, + "theoretical_loss": 3.5110400736260985, + "tokens_seen": 1673986048 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039825075225677033, + "loss": 1.1816, + "theoretical_loss": 3.511028304358544, + "tokens_seen": 1674051584 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039823470411233703, + "loss": 1.4133, + "theoretical_loss": 3.5110165356807292, + "tokens_seen": 1674117120 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039821865596790374, + "loss": 1.3351, + "theoretical_loss": 3.5110047675926, + "tokens_seen": 1674182656 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003982026078234704, + "loss": 1.3084, + "theoretical_loss": 3.5109930000941043, + "tokens_seen": 1674248192 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039818655967903714, + "loss": 1.3028, + "theoretical_loss": 3.5109812331851895, + "tokens_seen": 1674313728 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039817051153460384, + "loss": 1.2348, + "theoretical_loss": 3.5109694668658027, + "tokens_seen": 1674379264 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 561113, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4016382694244385, + "objective/train/theoretical_loss": 3.5109577011358915, + "objective/train/tokens_used": 1694904800, + "theoretical_loss": 3.5109577011358915, + "tokens_seen": 1674444800 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039815446339017054, + "loss": 1.3322, + "theoretical_loss": 3.5109577011358915, + "tokens_seen": 1674444800 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039813841524573724, + "loss": 1.4314, + "theoretical_loss": 3.5109459359954034, + "tokens_seen": 1674510336 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039812236710130394, + "loss": 1.3244, + "theoretical_loss": 3.5109341714442857, + "tokens_seen": 1674575872 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039810631895687064, + "loss": 1.3995, + "theoretical_loss": 3.510922407482486, + "tokens_seen": 1674641408 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039809027081243734, + "loss": 1.2781, + "theoretical_loss": 3.510910644109951, + "tokens_seen": 1674706944 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039807422266800404, + "loss": 1.5037, + "theoretical_loss": 3.5108988813266286, + "tokens_seen": 1674772480 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039805817452357074, + "loss": 1.2534, + "theoretical_loss": 3.510887119132467, + "tokens_seen": 1674838016 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039804212637913744, + "loss": 1.4516, + "theoretical_loss": 3.510875357527412, + "tokens_seen": 1674903552 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039802607823470414, + "loss": 1.2991, + "theoretical_loss": 3.5108635965114123, + "tokens_seen": 1674969088 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039801003009027084, + "loss": 1.2832, + "theoretical_loss": 3.510851836084415, + "tokens_seen": 1675034624 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039799398194583754, + "loss": 1.2692, + "theoretical_loss": 3.510840076246368, + "tokens_seen": 1675100160 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039797793380140424, + "loss": 1.3077, + "theoretical_loss": 3.510828316997218, + "tokens_seen": 1675165696 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039796188565697095, + "loss": 1.3315, + "theoretical_loss": 3.510816558336913, + "tokens_seen": 1675231232 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039794583751253765, + "loss": 1.3603, + "theoretical_loss": 3.5108048002654, + "tokens_seen": 1675296768 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039792978936810435, + "loss": 1.4642, + "theoretical_loss": 3.510793042782627, + "tokens_seen": 1675362304 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039791374122367105, + "loss": 1.2901, + "theoretical_loss": 3.510781285888541, + "tokens_seen": 1675427840 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039789769307923775, + "loss": 1.2233, + "theoretical_loss": 3.5107695295830896, + "tokens_seen": 1675493376 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039788164493480445, + "loss": 1.2658, + "theoretical_loss": 3.510757773866221, + "tokens_seen": 1675558912 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039786559679037115, + "loss": 1.2616, + "theoretical_loss": 3.5107460187378816, + "tokens_seen": 1675624448 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039784954864593785, + "loss": 1.3663, + "theoretical_loss": 3.51073426419802, + "tokens_seen": 1675689984 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039783350050150455, + "loss": 1.3221, + "theoretical_loss": 3.510722510246583, + "tokens_seen": 1675755520 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039781745235707125, + "loss": 1.4027, + "theoretical_loss": 3.5107107568835185, + "tokens_seen": 1675821056 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039780140421263795, + "loss": 1.333, + "theoretical_loss": 3.5106990041087736, + "tokens_seen": 1675886592 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039778535606820465, + "loss": 1.286, + "theoretical_loss": 3.5106872519222962, + "tokens_seen": 1675952128 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039776930792377135, + "loss": 1.3235, + "theoretical_loss": 3.510675500324034, + "tokens_seen": 1676017664 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039775325977933805, + "loss": 1.2732, + "theoretical_loss": 3.5106637493139345, + "tokens_seen": 1676083200 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039773721163490475, + "loss": 1.3502, + "theoretical_loss": 3.5106519988919453, + "tokens_seen": 1676148736 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039772116349047146, + "loss": 1.2631, + "theoretical_loss": 3.510640249058013, + "tokens_seen": 1676214272 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039770511534603816, + "loss": 1.4787, + "theoretical_loss": 3.5106284998120865, + "tokens_seen": 1676279808 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039768906720160486, + "loss": 1.3185, + "theoretical_loss": 3.510616751154113, + "tokens_seen": 1676345344 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039767301905717156, + "loss": 1.3165, + "theoretical_loss": 3.5106050030840397, + "tokens_seen": 1676410880 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039765697091273826, + "loss": 1.2874, + "theoretical_loss": 3.5105932556018145, + "tokens_seen": 1676476416 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039764092276830496, + "loss": 1.3481, + "theoretical_loss": 3.510581508707385, + "tokens_seen": 1676541952 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039762487462387166, + "loss": 1.1868, + "theoretical_loss": 3.510569762400699, + "tokens_seen": 1676607488 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039760882647943836, + "loss": 1.4, + "theoretical_loss": 3.5105580166817036, + "tokens_seen": 1676673024 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039759277833500506, + "loss": 1.374, + "theoretical_loss": 3.510546271550347, + "tokens_seen": 1676738560 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003975767301905717, + "loss": 1.3006, + "theoretical_loss": 3.510534527006576, + "tokens_seen": 1676804096 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003975606820461384, + "loss": 1.2967, + "theoretical_loss": 3.510522783050339, + "tokens_seen": 1676869632 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003975446339017051, + "loss": 1.3701, + "theoretical_loss": 3.5105110396815844, + "tokens_seen": 1676935168 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003975285857572718, + "loss": 1.5421, + "theoretical_loss": 3.5104992969002575, + "tokens_seen": 1677000704 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003975125376128385, + "loss": 1.4495, + "theoretical_loss": 3.510487554706309, + "tokens_seen": 1677066240 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003974964894684052, + "loss": 1.33, + "theoretical_loss": 3.5104758130996836, + "tokens_seen": 1677131776 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003974804413239719, + "loss": 1.2582, + "theoretical_loss": 3.5104640720803304, + "tokens_seen": 1677197312 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003974643931795386, + "loss": 1.3215, + "theoretical_loss": 3.5104523316481973, + "tokens_seen": 1677262848 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003974483450351053, + "loss": 1.2341, + "theoretical_loss": 3.5104405918032318, + "tokens_seen": 1677328384 + }, + { + "epoch": 0.51, + "learning_rate": 0.000397432296890672, + "loss": 1.3402, + "theoretical_loss": 3.510428852545381, + "tokens_seen": 1677393920 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003974162487462387, + "loss": 1.3536, + "theoretical_loss": 3.5104171138745937, + "tokens_seen": 1677459456 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003974002006018054, + "loss": 1.2893, + "theoretical_loss": 3.5104053757908167, + "tokens_seen": 1677524992 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003973841524573721, + "loss": 1.3752, + "theoretical_loss": 3.510393638293998, + "tokens_seen": 1677590528 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003973681043129388, + "loss": 1.4102, + "theoretical_loss": 3.5103819013840853, + "tokens_seen": 1677656064 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 562346, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.505894422531128, + "objective/train/theoretical_loss": 3.5103701650610266, + "objective/train/tokens_used": 1698181600, + "theoretical_loss": 3.5103701650610266, + "tokens_seen": 1677721600 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003973520561685055, + "loss": 1.391, + "theoretical_loss": 3.5103701650610266, + "tokens_seen": 1677721600 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003973360080240722, + "loss": 1.2973, + "theoretical_loss": 3.5103584293247687, + "tokens_seen": 1677787136 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003973199598796389, + "loss": 1.3318, + "theoretical_loss": 3.5103466941752606, + "tokens_seen": 1677852672 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003973039117352056, + "loss": 1.4684, + "theoretical_loss": 3.5103349596124493, + "tokens_seen": 1677918208 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003972878635907723, + "loss": 1.3585, + "theoretical_loss": 3.510323225636283, + "tokens_seen": 1677983744 + }, + { + "epoch": 0.51, + "learning_rate": 0.000397271815446339, + "loss": 1.2919, + "theoretical_loss": 3.510311492246709, + "tokens_seen": 1678049280 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003972557673019057, + "loss": 1.4674, + "theoretical_loss": 3.5102997594436753, + "tokens_seen": 1678114816 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003972397191574724, + "loss": 1.2753, + "theoretical_loss": 3.5102880272271295, + "tokens_seen": 1678180352 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003972236710130391, + "loss": 1.2766, + "theoretical_loss": 3.51027629559702, + "tokens_seen": 1678245888 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003972076228686058, + "loss": 1.266, + "theoretical_loss": 3.5102645645532933, + "tokens_seen": 1678311424 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003971915747241725, + "loss": 1.2545, + "theoretical_loss": 3.5102528340958985, + "tokens_seen": 1678376960 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003971755265797392, + "loss": 1.3106, + "theoretical_loss": 3.5102411042247823, + "tokens_seen": 1678442496 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003971594784353059, + "loss": 1.2323, + "theoretical_loss": 3.510229374939894, + "tokens_seen": 1678508032 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003971434302908726, + "loss": 1.3667, + "theoretical_loss": 3.51021764624118, + "tokens_seen": 1678573568 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003971273821464393, + "loss": 1.3376, + "theoretical_loss": 3.5102059181285887, + "tokens_seen": 1678639104 + }, + { + "epoch": 0.51, + "learning_rate": 0.000397111334002006, + "loss": 1.3192, + "theoretical_loss": 3.5101941906020677, + "tokens_seen": 1678704640 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003970952858575727, + "loss": 1.3216, + "theoretical_loss": 3.5101824636615655, + "tokens_seen": 1678770176 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003970792377131394, + "loss": 1.3051, + "theoretical_loss": 3.510170737307029, + "tokens_seen": 1678835712 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003970631895687061, + "loss": 1.3591, + "theoretical_loss": 3.510159011538407, + "tokens_seen": 1678901248 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039704714142427283, + "loss": 1.2935, + "theoretical_loss": 3.510147286355646, + "tokens_seen": 1678966784 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039703109327983953, + "loss": 1.395, + "theoretical_loss": 3.5101355617586956, + "tokens_seen": 1679032320 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039701504513540623, + "loss": 1.348, + "theoretical_loss": 3.5101238377475026, + "tokens_seen": 1679097856 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039699899699097293, + "loss": 1.2281, + "theoretical_loss": 3.5101121143220144, + "tokens_seen": 1679163392 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003969829488465397, + "loss": 1.4958, + "theoretical_loss": 3.5101003914821804, + "tokens_seen": 1679228928 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003969669007021064, + "loss": 1.2386, + "theoretical_loss": 3.5100886692279474, + "tokens_seen": 1679294464 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003969508525576731, + "loss": 1.3555, + "theoretical_loss": 3.510076947559263, + "tokens_seen": 1679360000 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039693480441323973, + "loss": 1.3538, + "theoretical_loss": 3.510065226476076, + "tokens_seen": 1679425536 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039691875626880643, + "loss": 1.3413, + "theoretical_loss": 3.510053505978334, + "tokens_seen": 1679491072 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039690270812437313, + "loss": 1.1577, + "theoretical_loss": 3.5100417860659845, + "tokens_seen": 1679556608 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039688665997993983, + "loss": 1.3217, + "theoretical_loss": 3.510030066738976, + "tokens_seen": 1679622144 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039687061183550653, + "loss": 1.3729, + "theoretical_loss": 3.5100183479972564, + "tokens_seen": 1679687680 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039685456369107324, + "loss": 1.4063, + "theoretical_loss": 3.510006629840773, + "tokens_seen": 1679753216 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039683851554663994, + "loss": 1.3822, + "theoretical_loss": 3.5099949122694745, + "tokens_seen": 1679818752 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039682246740220664, + "loss": 1.4432, + "theoretical_loss": 3.5099831952833087, + "tokens_seen": 1679884288 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039680641925777334, + "loss": 1.3221, + "theoretical_loss": 3.509971478882223, + "tokens_seen": 1679949824 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039679037111334004, + "loss": 1.27, + "theoretical_loss": 3.5099597630661656, + "tokens_seen": 1680015360 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039677432296890674, + "loss": 1.3166, + "theoretical_loss": 3.509948047835085, + "tokens_seen": 1680080896 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039675827482447344, + "loss": 1.3011, + "theoretical_loss": 3.5099363331889286, + "tokens_seen": 1680146432 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039674222668004014, + "loss": 1.3736, + "theoretical_loss": 3.5099246191276445, + "tokens_seen": 1680211968 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039672617853560684, + "loss": 1.3727, + "theoretical_loss": 3.5099129056511806, + "tokens_seen": 1680277504 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039671013039117354, + "loss": 1.3683, + "theoretical_loss": 3.509901192759486, + "tokens_seen": 1680343040 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039669408224674024, + "loss": 1.3507, + "theoretical_loss": 3.5098894804525065, + "tokens_seen": 1680408576 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039667803410230694, + "loss": 1.3361, + "theoretical_loss": 3.509877768730192, + "tokens_seen": 1680474112 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039666198595787364, + "loss": 1.3515, + "theoretical_loss": 3.5098660575924896, + "tokens_seen": 1680539648 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039664593781344034, + "loss": 1.3914, + "theoretical_loss": 3.509854347039348, + "tokens_seen": 1680605184 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039662988966900704, + "loss": 1.3152, + "theoretical_loss": 3.5098426370707143, + "tokens_seen": 1680670720 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039661384152457374, + "loss": 1.3061, + "theoretical_loss": 3.5098309276865374, + "tokens_seen": 1680736256 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039659779338014045, + "loss": 1.3654, + "theoretical_loss": 3.509819218886765, + "tokens_seen": 1680801792 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039658174523570715, + "loss": 1.2333, + "theoretical_loss": 3.509807510671345, + "tokens_seen": 1680867328 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039656569709127385, + "loss": 1.3129, + "theoretical_loss": 3.509795803040226, + "tokens_seen": 1680932864 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 563295, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3253703117370605, + "objective/train/theoretical_loss": 3.5097840959933553, + "objective/train/tokens_used": 1701458400, + "theoretical_loss": 3.5097840959933553, + "tokens_seen": 1680998400 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039654964894684055, + "loss": 1.2798, + "theoretical_loss": 3.5097840959933553, + "tokens_seen": 1680998400 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039653360080240725, + "loss": 1.3603, + "theoretical_loss": 3.5097723895306814, + "tokens_seen": 1681063936 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039651755265797395, + "loss": 1.3603, + "theoretical_loss": 3.5097606836521518, + "tokens_seen": 1681129472 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039650150451354065, + "loss": 1.2203, + "theoretical_loss": 3.509748978357716, + "tokens_seen": 1681195008 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039648545636910735, + "loss": 1.3285, + "theoretical_loss": 3.5097372736473207, + "tokens_seen": 1681260544 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039646940822467405, + "loss": 1.274, + "theoretical_loss": 3.5097255695209144, + "tokens_seen": 1681326080 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039645336008024075, + "loss": 1.335, + "theoretical_loss": 3.509713865978445, + "tokens_seen": 1681391616 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039643731193580745, + "loss": 1.2972, + "theoretical_loss": 3.5097021630198615, + "tokens_seen": 1681457152 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039642126379137415, + "loss": 1.3608, + "theoretical_loss": 3.5096904606451114, + "tokens_seen": 1681522688 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039640521564694085, + "loss": 1.3304, + "theoretical_loss": 3.5096787588541423, + "tokens_seen": 1681588224 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039638916750250755, + "loss": 1.3501, + "theoretical_loss": 3.509667057646903, + "tokens_seen": 1681653760 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039637311935807425, + "loss": 1.2682, + "theoretical_loss": 3.509655357023342, + "tokens_seen": 1681719296 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039635707121364095, + "loss": 1.4212, + "theoretical_loss": 3.509643656983406, + "tokens_seen": 1681784832 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039634102306920766, + "loss": 1.2548, + "theoretical_loss": 3.509631957527045, + "tokens_seen": 1681850368 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039632497492477436, + "loss": 1.3027, + "theoretical_loss": 3.5096202586542056, + "tokens_seen": 1681915904 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039630892678034106, + "loss": 1.3055, + "theoretical_loss": 3.5096085603648364, + "tokens_seen": 1681981440 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039629287863590776, + "loss": 1.3005, + "theoretical_loss": 3.5095968626588867, + "tokens_seen": 1682046976 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039627683049147446, + "loss": 1.2608, + "theoretical_loss": 3.509585165536303, + "tokens_seen": 1682112512 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039626078234704116, + "loss": 1.3951, + "theoretical_loss": 3.509573468997034, + "tokens_seen": 1682178048 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039624473420260786, + "loss": 1.2975, + "theoretical_loss": 3.5095617730410282, + "tokens_seen": 1682243584 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039622868605817456, + "loss": 1.3445, + "theoretical_loss": 3.5095500776682345, + "tokens_seen": 1682309120 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039621263791374126, + "loss": 1.3088, + "theoretical_loss": 3.509538382878599, + "tokens_seen": 1682374656 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039619658976930796, + "loss": 1.1954, + "theoretical_loss": 3.5095266886720724, + "tokens_seen": 1682440192 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039618054162487466, + "loss": 1.2492, + "theoretical_loss": 3.509514995048601, + "tokens_seen": 1682505728 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003961644934804413, + "loss": 1.4055, + "theoretical_loss": 3.5095033020081337, + "tokens_seen": 1682571264 + }, + { + "epoch": 0.51, + "learning_rate": 0.000396148445336008, + "loss": 1.4714, + "theoretical_loss": 3.5094916095506195, + "tokens_seen": 1682636800 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003961323971915747, + "loss": 1.3832, + "theoretical_loss": 3.509479917676005, + "tokens_seen": 1682702336 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003961163490471414, + "loss": 1.395, + "theoretical_loss": 3.5094682263842394, + "tokens_seen": 1682767872 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003961003009027081, + "loss": 1.2958, + "theoretical_loss": 3.5094565356752714, + "tokens_seen": 1682833408 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003960842527582748, + "loss": 1.3726, + "theoretical_loss": 3.509444845549049, + "tokens_seen": 1682898944 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003960682046138415, + "loss": 1.3026, + "theoretical_loss": 3.509433156005519, + "tokens_seen": 1682964480 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003960521564694082, + "loss": 1.358, + "theoretical_loss": 3.5094214670446315, + "tokens_seen": 1683030016 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003960361083249749, + "loss": 1.4805, + "theoretical_loss": 3.5094097786663343, + "tokens_seen": 1683095552 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003960200601805416, + "loss": 1.2693, + "theoretical_loss": 3.509398090870575, + "tokens_seen": 1683161088 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003960040120361083, + "loss": 1.3693, + "theoretical_loss": 3.5093864036573033, + "tokens_seen": 1683226624 + }, + { + "epoch": 0.51, + "learning_rate": 0.000395987963891675, + "loss": 1.2932, + "theoretical_loss": 3.509374717026466, + "tokens_seen": 1683292160 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003959719157472417, + "loss": 1.2604, + "theoretical_loss": 3.5093630309780117, + "tokens_seen": 1683357696 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003959558676028084, + "loss": 1.3242, + "theoretical_loss": 3.509351345511889, + "tokens_seen": 1683423232 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003959398194583751, + "loss": 1.4339, + "theoretical_loss": 3.509339660628047, + "tokens_seen": 1683488768 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003959237713139418, + "loss": 1.5202, + "theoretical_loss": 3.509327976326433, + "tokens_seen": 1683554304 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003959077231695085, + "loss": 1.3499, + "theoretical_loss": 3.5093162926069947, + "tokens_seen": 1683619840 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003958916750250752, + "loss": 1.3759, + "theoretical_loss": 3.509304609469682, + "tokens_seen": 1683685376 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003958756268806419, + "loss": 1.4145, + "theoretical_loss": 3.5092929269144424, + "tokens_seen": 1683750912 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003958595787362086, + "loss": 1.2288, + "theoretical_loss": 3.5092812449412243, + "tokens_seen": 1683816448 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003958435305917753, + "loss": 1.3088, + "theoretical_loss": 3.509269563549976, + "tokens_seen": 1683881984 + }, + { + "epoch": 0.51, + "learning_rate": 0.000395827482447342, + "loss": 1.3354, + "theoretical_loss": 3.5092578827406467, + "tokens_seen": 1683947520 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003958114343029087, + "loss": 1.4618, + "theoretical_loss": 3.509246202513183, + "tokens_seen": 1684013056 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003957953861584755, + "loss": 1.4285, + "theoretical_loss": 3.5092345228675352, + "tokens_seen": 1684078592 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003957793380140422, + "loss": 1.3331, + "theoretical_loss": 3.5092228438036503, + "tokens_seen": 1684144128 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003957632898696089, + "loss": 1.395, + "theoretical_loss": 3.5092111653214775, + "tokens_seen": 1684209664 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 564537, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2556380033493042, + "objective/train/theoretical_loss": 3.5091994874209647, + "objective/train/tokens_used": 1704735200, + "theoretical_loss": 3.5091994874209647, + "tokens_seen": 1684275200 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003957472417251756, + "loss": 1.2579, + "theoretical_loss": 3.5091994874209647, + "tokens_seen": 1684275200 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003957311935807423, + "loss": 1.328, + "theoretical_loss": 3.509187810102061, + "tokens_seen": 1684340736 + }, + { + "epoch": 0.51, + "learning_rate": 0.000395715145436309, + "loss": 1.3016, + "theoretical_loss": 3.509176133364713, + "tokens_seen": 1684406272 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003956990972918757, + "loss": 1.3205, + "theoretical_loss": 3.509164457208872, + "tokens_seen": 1684471808 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003956830491474424, + "loss": 1.3491, + "theoretical_loss": 3.5091527816344836, + "tokens_seen": 1684537344 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003956670010030091, + "loss": 1.294, + "theoretical_loss": 3.509141106641498, + "tokens_seen": 1684602880 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003956509528585758, + "loss": 1.3225, + "theoretical_loss": 3.5091294322298623, + "tokens_seen": 1684668416 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003956349047141425, + "loss": 1.4362, + "theoretical_loss": 3.5091177583995266, + "tokens_seen": 1684733952 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003956188565697092, + "loss": 1.3623, + "theoretical_loss": 3.509106085150438, + "tokens_seen": 1684799488 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003956028084252759, + "loss": 1.3761, + "theoretical_loss": 3.5090944124825456, + "tokens_seen": 1684865024 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003955867602808426, + "loss": 1.3485, + "theoretical_loss": 3.509082740395798, + "tokens_seen": 1684930560 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003955707121364093, + "loss": 1.3722, + "theoretical_loss": 3.509071068890143, + "tokens_seen": 1684996096 + }, + { + "epoch": 0.51, + "learning_rate": 0.000395554663991976, + "loss": 1.2775, + "theoretical_loss": 3.5090593979655287, + "tokens_seen": 1685061632 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039553861584754263, + "loss": 1.3034, + "theoretical_loss": 3.509047727621905, + "tokens_seen": 1685127168 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039552256770310933, + "loss": 1.2756, + "theoretical_loss": 3.5090360578592197, + "tokens_seen": 1685192704 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039550651955867603, + "loss": 1.3566, + "theoretical_loss": 3.509024388677421, + "tokens_seen": 1685258240 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039549047141424273, + "loss": 1.3254, + "theoretical_loss": 3.5090127200764583, + "tokens_seen": 1685323776 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039547442326980944, + "loss": 1.3804, + "theoretical_loss": 3.509001052056279, + "tokens_seen": 1685389312 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039545837512537614, + "loss": 1.2246, + "theoretical_loss": 3.5089893846168314, + "tokens_seen": 1685454848 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039544232698094284, + "loss": 1.2429, + "theoretical_loss": 3.508977717758065, + "tokens_seen": 1685520384 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039542627883650954, + "loss": 1.2984, + "theoretical_loss": 3.508966051479929, + "tokens_seen": 1685585920 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039541023069207624, + "loss": 1.2696, + "theoretical_loss": 3.5089543857823697, + "tokens_seen": 1685651456 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039539418254764294, + "loss": 1.3574, + "theoretical_loss": 3.5089427206653374, + "tokens_seen": 1685716992 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039537813440320964, + "loss": 1.2712, + "theoretical_loss": 3.5089310561287803, + "tokens_seen": 1685782528 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039536208625877634, + "loss": 1.3783, + "theoretical_loss": 3.5089193921726465, + "tokens_seen": 1685848064 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039534603811434304, + "loss": 1.2313, + "theoretical_loss": 3.508907728796885, + "tokens_seen": 1685913600 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039532998996990974, + "loss": 1.3936, + "theoretical_loss": 3.5088960660014443, + "tokens_seen": 1685979136 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039531394182547644, + "loss": 1.4377, + "theoretical_loss": 3.5088844037862725, + "tokens_seen": 1686044672 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039529789368104314, + "loss": 1.2427, + "theoretical_loss": 3.5088727421513184, + "tokens_seen": 1686110208 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039528184553660984, + "loss": 1.3432, + "theoretical_loss": 3.508861081096531, + "tokens_seen": 1686175744 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039526579739217654, + "loss": 1.2598, + "theoretical_loss": 3.508849420621859, + "tokens_seen": 1686241280 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039524974924774324, + "loss": 1.337, + "theoretical_loss": 3.50883776072725, + "tokens_seen": 1686306816 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039523370110330995, + "loss": 1.2391, + "theoretical_loss": 3.5088261014126534, + "tokens_seen": 1686372352 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039521765295887665, + "loss": 1.2831, + "theoretical_loss": 3.508814442678017, + "tokens_seen": 1686437888 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039520160481444335, + "loss": 1.2825, + "theoretical_loss": 3.508802784523291, + "tokens_seen": 1686503424 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039518555667001005, + "loss": 1.303, + "theoretical_loss": 3.508791126948423, + "tokens_seen": 1686568960 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039516950852557675, + "loss": 1.4422, + "theoretical_loss": 3.5087794699533608, + "tokens_seen": 1686634496 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039515346038114345, + "loss": 1.3851, + "theoretical_loss": 3.5087678135380544, + "tokens_seen": 1686700032 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039513741223671015, + "loss": 1.2958, + "theoretical_loss": 3.508756157702452, + "tokens_seen": 1686765568 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039512136409227685, + "loss": 1.22, + "theoretical_loss": 3.5087445024465023, + "tokens_seen": 1686831104 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039510531594784355, + "loss": 1.2511, + "theoretical_loss": 3.508732847770154, + "tokens_seen": 1686896640 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039508926780341025, + "loss": 1.2633, + "theoretical_loss": 3.508721193673355, + "tokens_seen": 1686962176 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039507321965897695, + "loss": 1.3662, + "theoretical_loss": 3.5087095401560546, + "tokens_seen": 1687027712 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039505717151454365, + "loss": 1.3562, + "theoretical_loss": 3.5086978872182017, + "tokens_seen": 1687093248 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039504112337011035, + "loss": 1.3871, + "theoretical_loss": 3.508686234859745, + "tokens_seen": 1687158784 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039502507522567705, + "loss": 1.2493, + "theoretical_loss": 3.5086745830806327, + "tokens_seen": 1687224320 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039500902708124375, + "loss": 1.3574, + "theoretical_loss": 3.5086629318808136, + "tokens_seen": 1687289856 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039499297893681045, + "loss": 1.3345, + "theoretical_loss": 3.508651281260237, + "tokens_seen": 1687355392 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039497693079237716, + "loss": 1.439, + "theoretical_loss": 3.5086396312188506, + "tokens_seen": 1687420928 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039496088264794386, + "loss": 1.3993, + "theoretical_loss": 3.5086279817566037, + "tokens_seen": 1687486464 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 565497, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2637591361999512, + "objective/train/theoretical_loss": 3.508616332873445, + "objective/train/tokens_used": 1708012000, + "theoretical_loss": 3.508616332873445, + "tokens_seen": 1687552000 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039494483450351056, + "loss": 1.2331, + "theoretical_loss": 3.508616332873445, + "tokens_seen": 1687552000 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039492878635907726, + "loss": 1.3628, + "theoretical_loss": 3.508604684569324, + "tokens_seen": 1687617536 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039491273821464396, + "loss": 1.2634, + "theoretical_loss": 3.5085930368441876, + "tokens_seen": 1687683072 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039489669007021066, + "loss": 1.3334, + "theoretical_loss": 3.508581389697986, + "tokens_seen": 1687748608 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039488064192577736, + "loss": 1.275, + "theoretical_loss": 3.5085697431306673, + "tokens_seen": 1687814144 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039486459378134406, + "loss": 1.21, + "theoretical_loss": 3.5085580971421804, + "tokens_seen": 1687879680 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039484854563691076, + "loss": 1.2891, + "theoretical_loss": 3.5085464517324745, + "tokens_seen": 1687945216 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039483249749247746, + "loss": 1.392, + "theoretical_loss": 3.5085348069014977, + "tokens_seen": 1688010752 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039481644934804416, + "loss": 1.3625, + "theoretical_loss": 3.508523162649199, + "tokens_seen": 1688076288 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039480040120361086, + "loss": 1.3794, + "theoretical_loss": 3.5085115189755274, + "tokens_seen": 1688141824 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039478435305917756, + "loss": 1.3969, + "theoretical_loss": 3.5084998758804313, + "tokens_seen": 1688207360 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003947683049147442, + "loss": 1.3288, + "theoretical_loss": 3.5084882333638596, + "tokens_seen": 1688272896 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003947522567703109, + "loss": 1.3619, + "theoretical_loss": 3.5084765914257616, + "tokens_seen": 1688338432 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003947362086258776, + "loss": 1.4312, + "theoretical_loss": 3.508464950066086, + "tokens_seen": 1688403968 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003947201604814443, + "loss": 1.2946, + "theoretical_loss": 3.5084533092847807, + "tokens_seen": 1688469504 + }, + { + "epoch": 0.51, + "learning_rate": 0.000394704112337011, + "loss": 1.3366, + "theoretical_loss": 3.5084416690817957, + "tokens_seen": 1688535040 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003946880641925777, + "loss": 1.2482, + "theoretical_loss": 3.5084300294570787, + "tokens_seen": 1688600576 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003946720160481444, + "loss": 1.3815, + "theoretical_loss": 3.5084183904105792, + "tokens_seen": 1688666112 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003946559679037111, + "loss": 1.4278, + "theoretical_loss": 3.508406751942246, + "tokens_seen": 1688731648 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003946399197592778, + "loss": 1.3412, + "theoretical_loss": 3.5083951140520284, + "tokens_seen": 1688797184 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003946238716148445, + "loss": 1.4112, + "theoretical_loss": 3.508383476739874, + "tokens_seen": 1688862720 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003946078234704112, + "loss": 1.2653, + "theoretical_loss": 3.508371840005733, + "tokens_seen": 1688928256 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039459177532597797, + "loss": 1.3189, + "theoretical_loss": 3.508360203849553, + "tokens_seen": 1688993792 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039457572718154467, + "loss": 1.331, + "theoretical_loss": 3.508348568271284, + "tokens_seen": 1689059328 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039455967903711137, + "loss": 1.2246, + "theoretical_loss": 3.5083369332708747, + "tokens_seen": 1689124864 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039454363089267807, + "loss": 1.3353, + "theoretical_loss": 3.508325298848273, + "tokens_seen": 1689190400 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003945275827482448, + "loss": 1.3473, + "theoretical_loss": 3.5083136650034286, + "tokens_seen": 1689255936 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003945115346038115, + "loss": 1.2427, + "theoretical_loss": 3.5083020317362905, + "tokens_seen": 1689321472 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003944954864593782, + "loss": 1.3452, + "theoretical_loss": 3.5082903990468073, + "tokens_seen": 1689387008 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003944794383149449, + "loss": 1.3949, + "theoretical_loss": 3.508278766934928, + "tokens_seen": 1689452544 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003944633901705116, + "loss": 1.2256, + "theoretical_loss": 3.5082671354006014, + "tokens_seen": 1689518080 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003944473420260783, + "loss": 1.3816, + "theoretical_loss": 3.5082555044437767, + "tokens_seen": 1689583616 + }, + { + "epoch": 0.51, + "learning_rate": 0.000394431293881645, + "loss": 1.2993, + "theoretical_loss": 3.5082438740644024, + "tokens_seen": 1689649152 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003944152457372117, + "loss": 1.3085, + "theoretical_loss": 3.5082322442624276, + "tokens_seen": 1689714688 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003943991975927784, + "loss": 1.3416, + "theoretical_loss": 3.5082206150378017, + "tokens_seen": 1689780224 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003943831494483451, + "loss": 1.2564, + "theoretical_loss": 3.508208986390473, + "tokens_seen": 1689845760 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003943671013039118, + "loss": 1.2665, + "theoretical_loss": 3.508197358320391, + "tokens_seen": 1689911296 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003943510531594785, + "loss": 1.3752, + "theoretical_loss": 3.5081857308275044, + "tokens_seen": 1689976832 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003943350050150452, + "loss": 1.3116, + "theoretical_loss": 3.5081741039117618, + "tokens_seen": 1690042368 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003943189568706119, + "loss": 1.324, + "theoretical_loss": 3.5081624775731126, + "tokens_seen": 1690107904 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003943029087261786, + "loss": 1.2645, + "theoretical_loss": 3.508150851811506, + "tokens_seen": 1690173440 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003942868605817453, + "loss": 1.3925, + "theoretical_loss": 3.5081392266268905, + "tokens_seen": 1690238976 + }, + { + "epoch": 0.51, + "learning_rate": 0.000394270812437312, + "loss": 1.2388, + "theoretical_loss": 3.508127602019215, + "tokens_seen": 1690304512 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003942547642928787, + "loss": 1.3048, + "theoretical_loss": 3.508115977988429, + "tokens_seen": 1690370048 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003942387161484454, + "loss": 1.2891, + "theoretical_loss": 3.5081043545344817, + "tokens_seen": 1690435584 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003942226680040121, + "loss": 1.1727, + "theoretical_loss": 3.5080927316573214, + "tokens_seen": 1690501120 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003942066198595788, + "loss": 1.3343, + "theoretical_loss": 3.5080811093568975, + "tokens_seen": 1690566656 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003941905717151455, + "loss": 1.3054, + "theoretical_loss": 3.508069487633159, + "tokens_seen": 1690632192 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003941745235707122, + "loss": 1.2212, + "theoretical_loss": 3.5080578664860544, + "tokens_seen": 1690697728 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003941584754262789, + "loss": 1.3057, + "theoretical_loss": 3.508046245915534, + "tokens_seen": 1690763264 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 566850, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2949119806289673, + "objective/train/theoretical_loss": 3.5080346259215456, + "objective/train/tokens_used": 1711288800, + "theoretical_loss": 3.5080346259215456, + "tokens_seen": 1690828800 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039414242728184553, + "loss": 1.263, + "theoretical_loss": 3.5080346259215456, + "tokens_seen": 1690828800 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039412637913741223, + "loss": 1.2198, + "theoretical_loss": 3.5080230065040388, + "tokens_seen": 1690894336 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039411033099297894, + "loss": 1.3229, + "theoretical_loss": 3.508011387662963, + "tokens_seen": 1690959872 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039409428284854564, + "loss": 1.301, + "theoretical_loss": 3.5079997693982663, + "tokens_seen": 1691025408 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039407823470411234, + "loss": 1.2693, + "theoretical_loss": 3.5079881517098985, + "tokens_seen": 1691090944 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039406218655967904, + "loss": 1.2663, + "theoretical_loss": 3.507976534597808, + "tokens_seen": 1691156480 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039404613841524574, + "loss": 1.2388, + "theoretical_loss": 3.507964918061945, + "tokens_seen": 1691222016 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039403009027081244, + "loss": 1.3611, + "theoretical_loss": 3.507953302102258, + "tokens_seen": 1691287552 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039401404212637914, + "loss": 1.3171, + "theoretical_loss": 3.507941686718696, + "tokens_seen": 1691353088 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039399799398194584, + "loss": 1.2994, + "theoretical_loss": 3.507930071911208, + "tokens_seen": 1691418624 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039398194583751254, + "loss": 1.28, + "theoretical_loss": 3.5079184576797435, + "tokens_seen": 1691484160 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039396589769307924, + "loss": 1.3806, + "theoretical_loss": 3.507906844024251, + "tokens_seen": 1691549696 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039394984954864594, + "loss": 1.2292, + "theoretical_loss": 3.5078952309446807, + "tokens_seen": 1691615232 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039393380140421264, + "loss": 1.3192, + "theoretical_loss": 3.5078836184409803, + "tokens_seen": 1691680768 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039391775325977934, + "loss": 1.3907, + "theoretical_loss": 3.5078720065131, + "tokens_seen": 1691746304 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039390170511534604, + "loss": 1.3506, + "theoretical_loss": 3.507860395160989, + "tokens_seen": 1691811840 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039388565697091274, + "loss": 1.2145, + "theoretical_loss": 3.5078487843845956, + "tokens_seen": 1691877376 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039386960882647944, + "loss": 1.316, + "theoretical_loss": 3.5078371741838694, + "tokens_seen": 1691942912 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039385356068204615, + "loss": 1.417, + "theoretical_loss": 3.5078255645587597, + "tokens_seen": 1692008448 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039383751253761285, + "loss": 1.2959, + "theoretical_loss": 3.507813955509216, + "tokens_seen": 1692073984 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039382146439317955, + "loss": 1.2502, + "theoretical_loss": 3.5078023470351862, + "tokens_seen": 1692139520 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039380541624874625, + "loss": 1.3025, + "theoretical_loss": 3.507790739136621, + "tokens_seen": 1692205056 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039378936810431295, + "loss": 1.3568, + "theoretical_loss": 3.507779131813469, + "tokens_seen": 1692270592 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039377331995987965, + "loss": 1.3236, + "theoretical_loss": 3.507767525065679, + "tokens_seen": 1692336128 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039375727181544635, + "loss": 1.3854, + "theoretical_loss": 3.5077559188932, + "tokens_seen": 1692401664 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039374122367101305, + "loss": 1.3598, + "theoretical_loss": 3.5077443132959827, + "tokens_seen": 1692467200 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039372517552657975, + "loss": 1.2831, + "theoretical_loss": 3.5077327082739744, + "tokens_seen": 1692532736 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039370912738214645, + "loss": 1.3579, + "theoretical_loss": 3.5077211038271257, + "tokens_seen": 1692598272 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039369307923771315, + "loss": 1.3286, + "theoretical_loss": 3.5077094999553857, + "tokens_seen": 1692663808 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039367703109327985, + "loss": 1.4154, + "theoretical_loss": 3.507697896658703, + "tokens_seen": 1692729344 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039366098294884655, + "loss": 1.2917, + "theoretical_loss": 3.5076862939370272, + "tokens_seen": 1692794880 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039364493480441325, + "loss": 1.3786, + "theoretical_loss": 3.5076746917903074, + "tokens_seen": 1692860416 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039362888665997995, + "loss": 1.3908, + "theoretical_loss": 3.507663090218493, + "tokens_seen": 1692925952 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039361283851554665, + "loss": 1.3052, + "theoretical_loss": 3.5076514892215327, + "tokens_seen": 1692991488 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039359679037111336, + "loss": 1.3047, + "theoretical_loss": 3.507639888799377, + "tokens_seen": 1693057024 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039358074222668006, + "loss": 1.2796, + "theoretical_loss": 3.507628288951974, + "tokens_seen": 1693122560 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039356469408224676, + "loss": 1.2555, + "theoretical_loss": 3.5076166896792733, + "tokens_seen": 1693188096 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039354864593781346, + "loss": 1.3208, + "theoretical_loss": 3.507605090981225, + "tokens_seen": 1693253632 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039353259779338016, + "loss": 1.3149, + "theoretical_loss": 3.507593492857777, + "tokens_seen": 1693319168 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039351654964894686, + "loss": 1.2571, + "theoretical_loss": 3.5075818953088795, + "tokens_seen": 1693384704 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039350050150451356, + "loss": 1.2692, + "theoretical_loss": 3.507570298334482, + "tokens_seen": 1693450240 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039348445336008026, + "loss": 1.3489, + "theoretical_loss": 3.507558701934533, + "tokens_seen": 1693515776 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039346840521564696, + "loss": 1.3062, + "theoretical_loss": 3.507547106108982, + "tokens_seen": 1693581312 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039345235707121366, + "loss": 1.184, + "theoretical_loss": 3.507535510857779, + "tokens_seen": 1693646848 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039343630892678036, + "loss": 1.2422, + "theoretical_loss": 3.5075239161808724, + "tokens_seen": 1693712384 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039342026078234706, + "loss": 1.342, + "theoretical_loss": 3.5075123220782123, + "tokens_seen": 1693777920 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039340421263791376, + "loss": 1.3779, + "theoretical_loss": 3.507500728549748, + "tokens_seen": 1693843456 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039338816449348046, + "loss": 1.2936, + "theoretical_loss": 3.507489135595428, + "tokens_seen": 1693908992 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039337211634904716, + "loss": 1.2804, + "theoretical_loss": 3.5074775432152023, + "tokens_seen": 1693974528 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039335606820461387, + "loss": 1.3327, + "theoretical_loss": 3.5074659514090207, + "tokens_seen": 1694040064 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 567680, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2184815406799316, + "objective/train/theoretical_loss": 3.507454360176832, + "objective/train/tokens_used": 1714565600, + "theoretical_loss": 3.507454360176832, + "tokens_seen": 1694105600 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039334002006018057, + "loss": 1.3059, + "theoretical_loss": 3.507454360176832, + "tokens_seen": 1694105600 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039332397191574727, + "loss": 1.4208, + "theoretical_loss": 3.5074427695185855, + "tokens_seen": 1694171136 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039330792377131397, + "loss": 1.3137, + "theoretical_loss": 3.5074311794342306, + "tokens_seen": 1694236672 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039329187562688067, + "loss": 1.103, + "theoretical_loss": 3.507419589923717, + "tokens_seen": 1694302208 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039327582748244737, + "loss": 1.2381, + "theoretical_loss": 3.507408000986994, + "tokens_seen": 1694367744 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039325977933801407, + "loss": 1.3529, + "theoretical_loss": 3.5073964126240105, + "tokens_seen": 1694433280 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039324373119358077, + "loss": 1.2204, + "theoretical_loss": 3.507384824834717, + "tokens_seen": 1694498816 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039322768304914747, + "loss": 1.2338, + "theoretical_loss": 3.5073732376190616, + "tokens_seen": 1694564352 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039321163490471417, + "loss": 1.3662, + "theoretical_loss": 3.5073616509769945, + "tokens_seen": 1694629888 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039319558676028087, + "loss": 1.3137, + "theoretical_loss": 3.507350064908465, + "tokens_seen": 1694695424 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039317953861584757, + "loss": 1.3784, + "theoretical_loss": 3.5073384794134226, + "tokens_seen": 1694760960 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039316349047141427, + "loss": 1.2652, + "theoretical_loss": 3.507326894491817, + "tokens_seen": 1694826496 + }, + { + "epoch": 0.51, + "learning_rate": 0.000393147442326981, + "loss": 1.2707, + "theoretical_loss": 3.507315310143597, + "tokens_seen": 1694892032 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003931313941825477, + "loss": 1.3725, + "theoretical_loss": 3.507303726368712, + "tokens_seen": 1694957568 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003931153460381144, + "loss": 1.234, + "theoretical_loss": 3.5072921431671125, + "tokens_seen": 1695023104 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003930992978936811, + "loss": 1.2371, + "theoretical_loss": 3.5072805605387467, + "tokens_seen": 1695088640 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003930832497492478, + "loss": 1.3322, + "theoretical_loss": 3.507268978483565, + "tokens_seen": 1695154176 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003930672016048145, + "loss": 1.2113, + "theoretical_loss": 3.507257397001516, + "tokens_seen": 1695219712 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003930511534603812, + "loss": 1.2818, + "theoretical_loss": 3.50724581609255, + "tokens_seen": 1695285248 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003930351053159479, + "loss": 1.2946, + "theoretical_loss": 3.5072342357566164, + "tokens_seen": 1695350784 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003930190571715146, + "loss": 1.2973, + "theoretical_loss": 3.5072226559936643, + "tokens_seen": 1695416320 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003930030090270813, + "loss": 1.3219, + "theoretical_loss": 3.5072110768036433, + "tokens_seen": 1695481856 + }, + { + "epoch": 0.51, + "learning_rate": 0.000392986960882648, + "loss": 1.3231, + "theoretical_loss": 3.5071994981865036, + "tokens_seen": 1695547392 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003929709127382147, + "loss": 1.2811, + "theoretical_loss": 3.507187920142193, + "tokens_seen": 1695612928 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003929548645937814, + "loss": 1.2201, + "theoretical_loss": 3.507176342670663, + "tokens_seen": 1695678464 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003929388164493481, + "loss": 1.2676, + "theoretical_loss": 3.5071647657718623, + "tokens_seen": 1695744000 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003929227683049148, + "loss": 1.2043, + "theoretical_loss": 3.5071531894457397, + "tokens_seen": 1695809536 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003929067201604815, + "loss": 1.2442, + "theoretical_loss": 3.507141613692246, + "tokens_seen": 1695875072 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003928906720160482, + "loss": 1.4047, + "theoretical_loss": 3.50713003851133, + "tokens_seen": 1695940608 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003928746238716149, + "loss": 1.3448, + "theoretical_loss": 3.507118463902942, + "tokens_seen": 1696006144 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003928585757271816, + "loss": 1.3224, + "theoretical_loss": 3.50710688986703, + "tokens_seen": 1696071680 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003928425275827483, + "loss": 1.3006, + "theoretical_loss": 3.507095316403545, + "tokens_seen": 1696137216 + }, + { + "epoch": 0.51, + "learning_rate": 0.000392826479438315, + "loss": 1.4223, + "theoretical_loss": 3.5070837435124362, + "tokens_seen": 1696202752 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003928104312938817, + "loss": 1.3044, + "theoretical_loss": 3.5070721711936534, + "tokens_seen": 1696268288 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003927943831494484, + "loss": 1.2839, + "theoretical_loss": 3.5070605994471453, + "tokens_seen": 1696333824 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003927783350050151, + "loss": 1.2511, + "theoretical_loss": 3.5070490282728626, + "tokens_seen": 1696399360 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003927622868605818, + "loss": 1.2513, + "theoretical_loss": 3.507037457670754, + "tokens_seen": 1696464896 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003927462387161485, + "loss": 1.3928, + "theoretical_loss": 3.5070258876407703, + "tokens_seen": 1696530432 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039273019057171514, + "loss": 1.2776, + "theoretical_loss": 3.5070143181828595, + "tokens_seen": 1696595968 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039271414242728184, + "loss": 1.3406, + "theoretical_loss": 3.507002749296973, + "tokens_seen": 1696661504 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039269809428284854, + "loss": 1.3934, + "theoretical_loss": 3.506991180983058, + "tokens_seen": 1696727040 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039268204613841524, + "loss": 1.3359, + "theoretical_loss": 3.5069796132410667, + "tokens_seen": 1696792576 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039266599799398194, + "loss": 1.3247, + "theoretical_loss": 3.506968046070947, + "tokens_seen": 1696858112 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039264994984954864, + "loss": 1.2309, + "theoretical_loss": 3.5069564794726498, + "tokens_seen": 1696923648 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039263390170511534, + "loss": 1.416, + "theoretical_loss": 3.5069449134461235, + "tokens_seen": 1696989184 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039261785356068204, + "loss": 1.285, + "theoretical_loss": 3.5069333479913185, + "tokens_seen": 1697054720 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039260180541624874, + "loss": 1.2694, + "theoretical_loss": 3.5069217831081847, + "tokens_seen": 1697120256 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039258575727181544, + "loss": 1.39, + "theoretical_loss": 3.506910218796671, + "tokens_seen": 1697185792 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039256970912738214, + "loss": 1.2365, + "theoretical_loss": 3.5068986550567276, + "tokens_seen": 1697251328 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039255366098294884, + "loss": 1.2729, + "theoretical_loss": 3.506887091888304, + "tokens_seen": 1697316864 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 569072, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3679840564727783, + "objective/train/theoretical_loss": 3.5068755292913503, + "objective/train/tokens_used": 1717842400, + "theoretical_loss": 3.5068755292913503, + "tokens_seen": 1697382400 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039253761283851554, + "loss": 1.2892, + "theoretical_loss": 3.5068755292913503, + "tokens_seen": 1697382400 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039252156469408224, + "loss": 1.3705, + "theoretical_loss": 3.5068639672658155, + "tokens_seen": 1697447936 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039250551654964894, + "loss": 1.2891, + "theoretical_loss": 3.50685240581165, + "tokens_seen": 1697513472 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039248946840521565, + "loss": 1.1874, + "theoretical_loss": 3.506840844928803, + "tokens_seen": 1697579008 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039247342026078235, + "loss": 1.366, + "theoretical_loss": 3.5068292846172238, + "tokens_seen": 1697644544 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039245737211634905, + "loss": 1.4041, + "theoretical_loss": 3.5068177248768633, + "tokens_seen": 1697710080 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039244132397191575, + "loss": 1.3315, + "theoretical_loss": 3.506806165707671, + "tokens_seen": 1697775616 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039242527582748245, + "loss": 1.2505, + "theoretical_loss": 3.506794607109595, + "tokens_seen": 1697841152 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039240922768304915, + "loss": 1.2652, + "theoretical_loss": 3.5067830490825873, + "tokens_seen": 1697906688 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039239317953861585, + "loss": 1.2623, + "theoretical_loss": 3.506771491626596, + "tokens_seen": 1697972224 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039237713139418255, + "loss": 1.2927, + "theoretical_loss": 3.506759934741572, + "tokens_seen": 1698037760 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039236108324974925, + "loss": 1.2347, + "theoretical_loss": 3.5067483784274645, + "tokens_seen": 1698103296 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039234503510531595, + "loss": 1.2806, + "theoretical_loss": 3.506736822684223, + "tokens_seen": 1698168832 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039232898696088265, + "loss": 1.3313, + "theoretical_loss": 3.5067252675117975, + "tokens_seen": 1698234368 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039231293881644935, + "loss": 1.3127, + "theoretical_loss": 3.506713712910138, + "tokens_seen": 1698299904 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039229689067201605, + "loss": 1.3687, + "theoretical_loss": 3.506702158879194, + "tokens_seen": 1698365440 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039228084252758275, + "loss": 1.294, + "theoretical_loss": 3.5066906054189158, + "tokens_seen": 1698430976 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039226479438314945, + "loss": 1.3141, + "theoretical_loss": 3.5066790525292526, + "tokens_seen": 1698496512 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039224874623871615, + "loss": 1.2798, + "theoretical_loss": 3.5066675002101544, + "tokens_seen": 1698562048 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039223269809428286, + "loss": 1.2686, + "theoretical_loss": 3.506655948461571, + "tokens_seen": 1698627584 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039221664994984956, + "loss": 1.2018, + "theoretical_loss": 3.506644397283452, + "tokens_seen": 1698693120 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039220060180541626, + "loss": 1.2765, + "theoretical_loss": 3.506632846675748, + "tokens_seen": 1698758656 + }, + { + "epoch": 0.51, + "learning_rate": 0.000392184553660983, + "loss": 1.2743, + "theoretical_loss": 3.506621296638408, + "tokens_seen": 1698824192 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003921685055165497, + "loss": 1.2716, + "theoretical_loss": 3.5066097471713826, + "tokens_seen": 1698889728 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003921524573721164, + "loss": 1.3958, + "theoretical_loss": 3.50659819827462, + "tokens_seen": 1698955264 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003921364092276831, + "loss": 1.3092, + "theoretical_loss": 3.5065866499480727, + "tokens_seen": 1699020800 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003921203610832498, + "loss": 1.2211, + "theoretical_loss": 3.5065751021916878, + "tokens_seen": 1699086336 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039210431293881646, + "loss": 1.3246, + "theoretical_loss": 3.5065635550054175, + "tokens_seen": 1699151872 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039208826479438316, + "loss": 1.2878, + "theoretical_loss": 3.50655200838921, + "tokens_seen": 1699217408 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039207221664994986, + "loss": 1.305, + "theoretical_loss": 3.5065404623430156, + "tokens_seen": 1699282944 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039205616850551656, + "loss": 1.3833, + "theoretical_loss": 3.5065289168667846, + "tokens_seen": 1699348480 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039204012036108326, + "loss": 1.2665, + "theoretical_loss": 3.5065173719604665, + "tokens_seen": 1699414016 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039202407221664996, + "loss": 1.2285, + "theoretical_loss": 3.5065058276240113, + "tokens_seen": 1699479552 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039200802407221666, + "loss": 1.3226, + "theoretical_loss": 3.506494283857369, + "tokens_seen": 1699545088 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039199197592778336, + "loss": 1.2431, + "theoretical_loss": 3.5064827406604895, + "tokens_seen": 1699610624 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039197592778335007, + "loss": 1.1, + "theoretical_loss": 3.5064711980333225, + "tokens_seen": 1699676160 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039195987963891677, + "loss": 1.2415, + "theoretical_loss": 3.506459655975818, + "tokens_seen": 1699741696 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039194383149448347, + "loss": 1.2683, + "theoretical_loss": 3.5064481144879265, + "tokens_seen": 1699807232 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039192778335005017, + "loss": 1.3517, + "theoretical_loss": 3.506436573569597, + "tokens_seen": 1699872768 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039191173520561687, + "loss": 1.2321, + "theoretical_loss": 3.5064250332207796, + "tokens_seen": 1699938304 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039189568706118357, + "loss": 1.3446, + "theoretical_loss": 3.506413493441425, + "tokens_seen": 1700003840 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039187963891675027, + "loss": 1.3141, + "theoretical_loss": 3.506401954231482, + "tokens_seen": 1700069376 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039186359077231697, + "loss": 1.3589, + "theoretical_loss": 3.506390415590902, + "tokens_seen": 1700134912 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039184754262788367, + "loss": 1.2161, + "theoretical_loss": 3.5063788775196336, + "tokens_seen": 1700200448 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039183149448345037, + "loss": 1.2506, + "theoretical_loss": 3.506367340017627, + "tokens_seen": 1700265984 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039181544633901707, + "loss": 1.3, + "theoretical_loss": 3.5063558030848325, + "tokens_seen": 1700331520 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039179939819458377, + "loss": 1.2518, + "theoretical_loss": 3.506344266721201, + "tokens_seen": 1700397056 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003917833500501505, + "loss": 1.342, + "theoretical_loss": 3.5063327309266805, + "tokens_seen": 1700462592 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003917673019057172, + "loss": 1.3103, + "theoretical_loss": 3.5063211957012226, + "tokens_seen": 1700528128 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003917512537612839, + "loss": 1.3478, + "theoretical_loss": 3.506309661044777, + "tokens_seen": 1700593664 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 569913, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1845080852508545, + "objective/train/theoretical_loss": 3.506298126957293, + "objective/train/tokens_used": 1721119200, + "theoretical_loss": 3.506298126957293, + "tokens_seen": 1700659200 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003917352056168506, + "loss": 1.2229, + "theoretical_loss": 3.506298126957293, + "tokens_seen": 1700659200 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003917191574724173, + "loss": 1.3233, + "theoretical_loss": 3.506286593438721, + "tokens_seen": 1700724736 + }, + { + "epoch": 0.52, + "learning_rate": 0.000391703109327984, + "loss": 1.3259, + "theoretical_loss": 3.5062750604890116, + "tokens_seen": 1700790272 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003916870611835507, + "loss": 1.4017, + "theoretical_loss": 3.5062635281081134, + "tokens_seen": 1700855808 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003916710130391174, + "loss": 1.4651, + "theoretical_loss": 3.506251996295978, + "tokens_seen": 1700921344 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003916549648946841, + "loss": 1.3008, + "theoretical_loss": 3.5062404650525547, + "tokens_seen": 1700986880 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003916389167502508, + "loss": 1.3168, + "theoretical_loss": 3.5062289343777935, + "tokens_seen": 1701052416 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003916228686058175, + "loss": 1.2484, + "theoretical_loss": 3.5062174042716445, + "tokens_seen": 1701117952 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003916068204613842, + "loss": 1.3394, + "theoretical_loss": 3.506205874734058, + "tokens_seen": 1701183488 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003915907723169509, + "loss": 1.3757, + "theoretical_loss": 3.5061943457649836, + "tokens_seen": 1701249024 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003915747241725176, + "loss": 1.4251, + "theoretical_loss": 3.5061828173643717, + "tokens_seen": 1701314560 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003915586760280843, + "loss": 1.2473, + "theoretical_loss": 3.5061712895321726, + "tokens_seen": 1701380096 + }, + { + "epoch": 0.52, + "learning_rate": 0.000391542627883651, + "loss": 1.2597, + "theoretical_loss": 3.506159762268336, + "tokens_seen": 1701445632 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003915265797392177, + "loss": 1.2608, + "theoretical_loss": 3.506148235572812, + "tokens_seen": 1701511168 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003915105315947844, + "loss": 1.1998, + "theoretical_loss": 3.506136709445551, + "tokens_seen": 1701576704 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003914944834503511, + "loss": 1.3601, + "theoretical_loss": 3.5061251838865024, + "tokens_seen": 1701642240 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003914784353059178, + "loss": 1.3088, + "theoretical_loss": 3.506113658895617, + "tokens_seen": 1701707776 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003914623871614845, + "loss": 1.2491, + "theoretical_loss": 3.506102134472845, + "tokens_seen": 1701773312 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003914463390170512, + "loss": 1.2246, + "theoretical_loss": 3.506090610618136, + "tokens_seen": 1701838848 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003914302908726179, + "loss": 1.2195, + "theoretical_loss": 3.5060790873314405, + "tokens_seen": 1701904384 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003914142427281846, + "loss": 1.2233, + "theoretical_loss": 3.5060675646127084, + "tokens_seen": 1701969920 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003913981945837513, + "loss": 1.2162, + "theoretical_loss": 3.506056042461889, + "tokens_seen": 1702035456 + }, + { + "epoch": 0.52, + "learning_rate": 0.000391382146439318, + "loss": 1.2986, + "theoretical_loss": 3.506044520878935, + "tokens_seen": 1702100992 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003913660982948847, + "loss": 1.3525, + "theoretical_loss": 3.506032999863794, + "tokens_seen": 1702166528 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003913500501504514, + "loss": 1.2059, + "theoretical_loss": 3.506021479416417, + "tokens_seen": 1702232064 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039133400200601804, + "loss": 1.3543, + "theoretical_loss": 3.506009959536754, + "tokens_seen": 1702297600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039131795386158474, + "loss": 1.2637, + "theoretical_loss": 3.505998440224756, + "tokens_seen": 1702363136 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039130190571715144, + "loss": 1.3314, + "theoretical_loss": 3.5059869214803725, + "tokens_seen": 1702428672 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039128585757271814, + "loss": 1.2534, + "theoretical_loss": 3.5059754033035535, + "tokens_seen": 1702494208 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039126980942828484, + "loss": 1.2815, + "theoretical_loss": 3.50596388569425, + "tokens_seen": 1702559744 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039125376128385154, + "loss": 1.2362, + "theoretical_loss": 3.5059523686524106, + "tokens_seen": 1702625280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039123771313941824, + "loss": 1.3542, + "theoretical_loss": 3.5059408521779876, + "tokens_seen": 1702690816 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039122166499498494, + "loss": 1.2336, + "theoretical_loss": 3.5059293362709294, + "tokens_seen": 1702756352 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039120561685055164, + "loss": 1.4013, + "theoretical_loss": 3.505917820931187, + "tokens_seen": 1702821888 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039118956870611834, + "loss": 1.2276, + "theoretical_loss": 3.5059063061587112, + "tokens_seen": 1702887424 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039117352056168504, + "loss": 1.1717, + "theoretical_loss": 3.505894791953451, + "tokens_seen": 1702952960 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039115747241725174, + "loss": 1.2855, + "theoretical_loss": 3.5058832783153573, + "tokens_seen": 1703018496 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039114142427281844, + "loss": 1.3067, + "theoretical_loss": 3.5058717652443803, + "tokens_seen": 1703084032 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039112537612838514, + "loss": 1.2898, + "theoretical_loss": 3.5058602527404705, + "tokens_seen": 1703149568 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039110932798395185, + "loss": 1.2778, + "theoretical_loss": 3.5058487408035774, + "tokens_seen": 1703215104 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039109327983951855, + "loss": 1.201, + "theoretical_loss": 3.505837229433652, + "tokens_seen": 1703280640 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039107723169508525, + "loss": 1.2255, + "theoretical_loss": 3.5058257186306445, + "tokens_seen": 1703346176 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039106118355065195, + "loss": 1.3764, + "theoretical_loss": 3.5058142083945043, + "tokens_seen": 1703411712 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039104513540621865, + "loss": 1.3237, + "theoretical_loss": 3.505802698725183, + "tokens_seen": 1703477248 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039102908726178535, + "loss": 1.3422, + "theoretical_loss": 3.50579118962263, + "tokens_seen": 1703542784 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039101303911735205, + "loss": 1.2957, + "theoretical_loss": 3.505779681086796, + "tokens_seen": 1703608320 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039099699097291875, + "loss": 1.221, + "theoretical_loss": 3.5057681731176307, + "tokens_seen": 1703673856 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003909809428284855, + "loss": 1.2782, + "theoretical_loss": 3.5057566657150847, + "tokens_seen": 1703739392 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003909648946840522, + "loss": 1.2679, + "theoretical_loss": 3.505745158879109, + "tokens_seen": 1703804928 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003909488465396189, + "loss": 1.3089, + "theoretical_loss": 3.5057336526096528, + "tokens_seen": 1703870464 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 571215, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2172174453735352, + "objective/train/theoretical_loss": 3.5057221469066673, + "objective/train/tokens_used": 1724396000, + "theoretical_loss": 3.5057221469066673, + "tokens_seen": 1703936000 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003909327983951856, + "loss": 1.1634, + "theoretical_loss": 3.5057221469066673, + "tokens_seen": 1703936000 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003909167502507523, + "loss": 1.2943, + "theoretical_loss": 3.5057106417701025, + "tokens_seen": 1704001536 + }, + { + "epoch": 0.52, + "learning_rate": 0.000390900702106319, + "loss": 1.3314, + "theoretical_loss": 3.505699137199908, + "tokens_seen": 1704067072 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003908846539618857, + "loss": 1.272, + "theoretical_loss": 3.5056876331960356, + "tokens_seen": 1704132608 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003908686058174524, + "loss": 1.3367, + "theoretical_loss": 3.5056761297584345, + "tokens_seen": 1704198144 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003908525576730191, + "loss": 1.349, + "theoretical_loss": 3.5056646268870555, + "tokens_seen": 1704263680 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003908365095285858, + "loss": 1.1898, + "theoretical_loss": 3.5056531245818494, + "tokens_seen": 1704329216 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003908204613841525, + "loss": 1.3588, + "theoretical_loss": 3.5056416228427656, + "tokens_seen": 1704394752 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003908044132397192, + "loss": 1.4023, + "theoretical_loss": 3.5056301216697547, + "tokens_seen": 1704460288 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003907883650952859, + "loss": 1.275, + "theoretical_loss": 3.5056186210627676, + "tokens_seen": 1704525824 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003907723169508526, + "loss": 1.2603, + "theoretical_loss": 3.5056071210217543, + "tokens_seen": 1704591360 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003907562688064193, + "loss": 1.3336, + "theoretical_loss": 3.5055956215466653, + "tokens_seen": 1704656896 + }, + { + "epoch": 0.52, + "learning_rate": 0.000390740220661986, + "loss": 1.2997, + "theoretical_loss": 3.505584122637451, + "tokens_seen": 1704722432 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003907241725175527, + "loss": 1.3568, + "theoretical_loss": 3.5055726242940617, + "tokens_seen": 1704787968 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003907081243731194, + "loss": 1.2819, + "theoretical_loss": 3.5055611265164477, + "tokens_seen": 1704853504 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039069207622868606, + "loss": 1.3309, + "theoretical_loss": 3.50554962930456, + "tokens_seen": 1704919040 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039067602808425276, + "loss": 1.2358, + "theoretical_loss": 3.5055381326583483, + "tokens_seen": 1704984576 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039065997993981946, + "loss": 1.3174, + "theoretical_loss": 3.5055266365777635, + "tokens_seen": 1705050112 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039064393179538616, + "loss": 1.2773, + "theoretical_loss": 3.5055151410627556, + "tokens_seen": 1705115648 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039062788365095286, + "loss": 1.3895, + "theoretical_loss": 3.505503646113276, + "tokens_seen": 1705181184 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039061183550651957, + "loss": 1.2752, + "theoretical_loss": 3.505492151729274, + "tokens_seen": 1705246720 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039059578736208627, + "loss": 1.3243, + "theoretical_loss": 3.5054806579107, + "tokens_seen": 1705312256 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039057973921765297, + "loss": 1.3128, + "theoretical_loss": 3.5054691646575056, + "tokens_seen": 1705377792 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039056369107321967, + "loss": 1.2306, + "theoretical_loss": 3.50545767196964, + "tokens_seen": 1705443328 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039054764292878637, + "loss": 1.3383, + "theoretical_loss": 3.505446179847055, + "tokens_seen": 1705508864 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039053159478435307, + "loss": 1.2969, + "theoretical_loss": 3.5054346882897, + "tokens_seen": 1705574400 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039051554663991977, + "loss": 1.2773, + "theoretical_loss": 3.5054231972975263, + "tokens_seen": 1705639936 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039049949849548647, + "loss": 1.3803, + "theoretical_loss": 3.5054117068704835, + "tokens_seen": 1705705472 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039048345035105317, + "loss": 1.2678, + "theoretical_loss": 3.5054002170085226, + "tokens_seen": 1705771008 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039046740220661987, + "loss": 1.3976, + "theoretical_loss": 3.505388727711594, + "tokens_seen": 1705836544 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039045135406218657, + "loss": 1.3176, + "theoretical_loss": 3.5053772389796487, + "tokens_seen": 1705902080 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039043530591775327, + "loss": 1.2333, + "theoretical_loss": 3.505365750812636, + "tokens_seen": 1705967616 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039041925777331997, + "loss": 1.271, + "theoretical_loss": 3.5053542632105077, + "tokens_seen": 1706033152 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003904032096288867, + "loss": 1.196, + "theoretical_loss": 3.5053427761732134, + "tokens_seen": 1706098688 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003903871614844534, + "loss": 1.25, + "theoretical_loss": 3.5053312897007043, + "tokens_seen": 1706164224 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003903711133400201, + "loss": 1.2192, + "theoretical_loss": 3.505319803792931, + "tokens_seen": 1706229760 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003903550651955868, + "loss": 1.3528, + "theoretical_loss": 3.5053083184498433, + "tokens_seen": 1706295296 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003903390170511535, + "loss": 1.3391, + "theoretical_loss": 3.5052968336713923, + "tokens_seen": 1706360832 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003903229689067202, + "loss": 1.317, + "theoretical_loss": 3.5052853494575285, + "tokens_seen": 1706426368 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003903069207622869, + "loss": 1.2589, + "theoretical_loss": 3.505273865808202, + "tokens_seen": 1706491904 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003902908726178536, + "loss": 1.2602, + "theoretical_loss": 3.505262382723364, + "tokens_seen": 1706557440 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003902748244734203, + "loss": 1.2766, + "theoretical_loss": 3.505250900202965, + "tokens_seen": 1706622976 + }, + { + "epoch": 0.52, + "learning_rate": 0.000390258776328987, + "loss": 1.2648, + "theoretical_loss": 3.5052394182469553, + "tokens_seen": 1706688512 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003902427281845537, + "loss": 1.3225, + "theoretical_loss": 3.5052279368552854, + "tokens_seen": 1706754048 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003902266800401204, + "loss": 1.2978, + "theoretical_loss": 3.505216456027907, + "tokens_seen": 1706819584 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003902106318956871, + "loss": 1.22, + "theoretical_loss": 3.5052049757647685, + "tokens_seen": 1706885120 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003901945837512538, + "loss": 1.2294, + "theoretical_loss": 3.5051934960658224, + "tokens_seen": 1706950656 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003901785356068205, + "loss": 1.2286, + "theoretical_loss": 3.505182016931019, + "tokens_seen": 1707016192 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003901624874623872, + "loss": 1.3281, + "theoretical_loss": 3.5051705383603085, + "tokens_seen": 1707081728 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003901464393179539, + "loss": 1.2442, + "theoretical_loss": 3.5051590603536416, + "tokens_seen": 1707147264 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 572081, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.278428077697754, + "objective/train/theoretical_loss": 3.505147582910969, + "objective/train/tokens_used": 1727672800, + "theoretical_loss": 3.505147582910969, + "tokens_seen": 1707212800 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003901303911735206, + "loss": 1.2695, + "theoretical_loss": 3.505147582910969, + "tokens_seen": 1707212800 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003901143430290873, + "loss": 1.2812, + "theoretical_loss": 3.505136106032241, + "tokens_seen": 1707278336 + }, + { + "epoch": 0.52, + "learning_rate": 0.000390098294884654, + "loss": 1.3015, + "theoretical_loss": 3.5051246297174092, + "tokens_seen": 1707343872 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003900822467402207, + "loss": 1.254, + "theoretical_loss": 3.5051131539664233, + "tokens_seen": 1707409408 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003900661985957874, + "loss": 1.2437, + "theoretical_loss": 3.5051016787792344, + "tokens_seen": 1707474944 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003900501504513541, + "loss": 1.3657, + "theoretical_loss": 3.5050902041557928, + "tokens_seen": 1707540480 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003900341023069208, + "loss": 1.2889, + "theoretical_loss": 3.5050787300960495, + "tokens_seen": 1707606016 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003900180541624875, + "loss": 1.2741, + "theoretical_loss": 3.5050672565999554, + "tokens_seen": 1707671552 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003900020060180542, + "loss": 1.224, + "theoretical_loss": 3.5050557836674607, + "tokens_seen": 1707737088 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003899859578736209, + "loss": 1.2895, + "theoretical_loss": 3.5050443112985166, + "tokens_seen": 1707802624 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003899699097291876, + "loss": 1.3661, + "theoretical_loss": 3.505032839493073, + "tokens_seen": 1707868160 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003899538615847543, + "loss": 1.2699, + "theoretical_loss": 3.5050213682510813, + "tokens_seen": 1707933696 + }, + { + "epoch": 0.52, + "learning_rate": 0.000389937813440321, + "loss": 1.3541, + "theoretical_loss": 3.5050098975724917, + "tokens_seen": 1707999232 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038992176529588764, + "loss": 1.1876, + "theoretical_loss": 3.504998427457255, + "tokens_seen": 1708064768 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038990571715145434, + "loss": 1.3763, + "theoretical_loss": 3.5049869579053228, + "tokens_seen": 1708130304 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038988966900702104, + "loss": 1.3298, + "theoretical_loss": 3.504975488916645, + "tokens_seen": 1708195840 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038987362086258774, + "loss": 1.2195, + "theoretical_loss": 3.504964020491172, + "tokens_seen": 1708261376 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038985757271815444, + "loss": 1.2675, + "theoretical_loss": 3.504952552628855, + "tokens_seen": 1708326912 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038984152457372114, + "loss": 1.3184, + "theoretical_loss": 3.504941085329645, + "tokens_seen": 1708392448 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038982547642928784, + "loss": 1.2692, + "theoretical_loss": 3.5049296185934926, + "tokens_seen": 1708457984 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038980942828485454, + "loss": 1.1954, + "theoretical_loss": 3.5049181524203483, + "tokens_seen": 1708523520 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038979338014042124, + "loss": 1.2956, + "theoretical_loss": 3.504906686810163, + "tokens_seen": 1708589056 + }, + { + "epoch": 0.52, + "learning_rate": 0.000389777331995988, + "loss": 1.2719, + "theoretical_loss": 3.5048952217628875, + "tokens_seen": 1708654592 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003897612838515547, + "loss": 1.2865, + "theoretical_loss": 3.504883757278473, + "tokens_seen": 1708720128 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003897452357071214, + "loss": 1.3534, + "theoretical_loss": 3.5048722933568692, + "tokens_seen": 1708785664 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003897291875626881, + "loss": 1.411, + "theoretical_loss": 3.5048608299980275, + "tokens_seen": 1708851200 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003897131394182548, + "loss": 1.3395, + "theoretical_loss": 3.5048493672018988, + "tokens_seen": 1708916736 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003896970912738215, + "loss": 1.2185, + "theoretical_loss": 3.5048379049684337, + "tokens_seen": 1708982272 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003896810431293882, + "loss": 1.1767, + "theoretical_loss": 3.504826443297584, + "tokens_seen": 1709047808 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003896649949849549, + "loss": 1.2424, + "theoretical_loss": 3.5048149821892984, + "tokens_seen": 1709113344 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003896489468405216, + "loss": 1.2404, + "theoretical_loss": 3.50480352164353, + "tokens_seen": 1709178880 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003896328986960883, + "loss": 1.192, + "theoretical_loss": 3.504792061660228, + "tokens_seen": 1709244416 + }, + { + "epoch": 0.52, + "learning_rate": 0.000389616850551655, + "loss": 1.3171, + "theoretical_loss": 3.504780602239344, + "tokens_seen": 1709309952 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003896008024072217, + "loss": 1.1987, + "theoretical_loss": 3.5047691433808286, + "tokens_seen": 1709375488 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003895847542627884, + "loss": 1.4319, + "theoretical_loss": 3.5047576850846323, + "tokens_seen": 1709441024 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003895687061183551, + "loss": 1.2345, + "theoretical_loss": 3.5047462273507066, + "tokens_seen": 1709506560 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003895526579739218, + "loss": 1.3256, + "theoretical_loss": 3.5047347701790024, + "tokens_seen": 1709572096 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003895366098294885, + "loss": 1.3368, + "theoretical_loss": 3.5047233135694698, + "tokens_seen": 1709637632 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003895205616850552, + "loss": 1.3262, + "theoretical_loss": 3.50471185752206, + "tokens_seen": 1709703168 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003895045135406219, + "loss": 1.3029, + "theoretical_loss": 3.5047004020367245, + "tokens_seen": 1709768704 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003894884653961886, + "loss": 1.2998, + "theoretical_loss": 3.5046889471134133, + "tokens_seen": 1709834240 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003894724172517553, + "loss": 1.222, + "theoretical_loss": 3.504677492752078, + "tokens_seen": 1709899776 + }, + { + "epoch": 0.52, + "learning_rate": 0.000389456369107322, + "loss": 1.3711, + "theoretical_loss": 3.504666038952669, + "tokens_seen": 1709965312 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003894403209628887, + "loss": 1.2662, + "theoretical_loss": 3.504654585715137, + "tokens_seen": 1710030848 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003894242728184554, + "loss": 1.3747, + "theoretical_loss": 3.5046431330394334, + "tokens_seen": 1710096384 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003894082246740221, + "loss": 1.1992, + "theoretical_loss": 3.504631680925509, + "tokens_seen": 1710161920 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003893921765295888, + "loss": 1.2252, + "theoretical_loss": 3.504620229373315, + "tokens_seen": 1710227456 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003893761283851555, + "loss": 1.2379, + "theoretical_loss": 3.504608778382802, + "tokens_seen": 1710292992 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003893600802407222, + "loss": 1.2257, + "theoretical_loss": 3.5045973279539204, + "tokens_seen": 1710358528 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003893440320962889, + "loss": 1.2381, + "theoretical_loss": 3.5045858780866217, + "tokens_seen": 1710424064 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 573219, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2893404960632324, + "objective/train/theoretical_loss": 3.5045744287808573, + "objective/train/tokens_used": 1730949600, + "theoretical_loss": 3.5045744287808573, + "tokens_seen": 1710489600 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003893279839518556, + "loss": 1.2964, + "theoretical_loss": 3.5045744287808573, + "tokens_seen": 1710489600 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003893119358074223, + "loss": 1.2419, + "theoretical_loss": 3.504562980036577, + "tokens_seen": 1710555136 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038929588766298896, + "loss": 1.3192, + "theoretical_loss": 3.5045515318537332, + "tokens_seen": 1710620672 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038927983951855566, + "loss": 1.3724, + "theoretical_loss": 3.504540084232276, + "tokens_seen": 1710686208 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038926379137412236, + "loss": 1.281, + "theoretical_loss": 3.5045286371721556, + "tokens_seen": 1710751744 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038924774322968906, + "loss": 1.1835, + "theoretical_loss": 3.5045171906733246, + "tokens_seen": 1710817280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038923169508525577, + "loss": 1.1356, + "theoretical_loss": 3.5045057447357326, + "tokens_seen": 1710882816 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038921564694082247, + "loss": 1.3201, + "theoretical_loss": 3.5044942993593318, + "tokens_seen": 1710948352 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038919959879638917, + "loss": 1.2701, + "theoretical_loss": 3.5044828545440723, + "tokens_seen": 1711013888 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038918355065195587, + "loss": 1.2679, + "theoretical_loss": 3.504471410289905, + "tokens_seen": 1711079424 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038916750250752257, + "loss": 1.2908, + "theoretical_loss": 3.504459966596782, + "tokens_seen": 1711144960 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038915145436308927, + "loss": 1.1841, + "theoretical_loss": 3.504448523464653, + "tokens_seen": 1711210496 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038913540621865597, + "loss": 1.2049, + "theoretical_loss": 3.5044370808934695, + "tokens_seen": 1711276032 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038911935807422267, + "loss": 1.2412, + "theoretical_loss": 3.5044256388831827, + "tokens_seen": 1711341568 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038910330992978937, + "loss": 1.2963, + "theoretical_loss": 3.5044141974337437, + "tokens_seen": 1711407104 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038908726178535607, + "loss": 1.2257, + "theoretical_loss": 3.5044027565451032, + "tokens_seen": 1711472640 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038907121364092277, + "loss": 1.3079, + "theoretical_loss": 3.5043913162172124, + "tokens_seen": 1711538176 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038905516549648947, + "loss": 1.222, + "theoretical_loss": 3.5043798764500225, + "tokens_seen": 1711603712 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003890391173520562, + "loss": 1.241, + "theoretical_loss": 3.5043684372434845, + "tokens_seen": 1711669248 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003890230692076229, + "loss": 1.3066, + "theoretical_loss": 3.504356998597549, + "tokens_seen": 1711734784 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003890070210631896, + "loss": 1.1662, + "theoretical_loss": 3.504345560512168, + "tokens_seen": 1711800320 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003889909729187563, + "loss": 1.3688, + "theoretical_loss": 3.5043341229872915, + "tokens_seen": 1711865856 + }, + { + "epoch": 0.52, + "learning_rate": 0.000388974924774323, + "loss": 1.3321, + "theoretical_loss": 3.504322686022871, + "tokens_seen": 1711931392 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003889588766298897, + "loss": 1.3109, + "theoretical_loss": 3.504311249618858, + "tokens_seen": 1711996928 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003889428284854564, + "loss": 1.3121, + "theoretical_loss": 3.5042998137752033, + "tokens_seen": 1712062464 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003889267803410231, + "loss": 1.2468, + "theoretical_loss": 3.5042883784918577, + "tokens_seen": 1712128000 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003889107321965898, + "loss": 1.2557, + "theoretical_loss": 3.5042769437687724, + "tokens_seen": 1712193536 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003888946840521565, + "loss": 1.344, + "theoretical_loss": 3.504265509605899, + "tokens_seen": 1712259072 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003888786359077232, + "loss": 1.1765, + "theoretical_loss": 3.504254076003188, + "tokens_seen": 1712324608 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003888625877632899, + "loss": 1.3774, + "theoretical_loss": 3.504242642960591, + "tokens_seen": 1712390144 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003888465396188566, + "loss": 1.3669, + "theoretical_loss": 3.504231210478059, + "tokens_seen": 1712455680 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003888304914744233, + "loss": 1.1514, + "theoretical_loss": 3.5042197785555427, + "tokens_seen": 1712521216 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038881444332999, + "loss": 1.286, + "theoretical_loss": 3.5042083471929937, + "tokens_seen": 1712586752 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003887983951855567, + "loss": 1.2841, + "theoretical_loss": 3.504196916390363, + "tokens_seen": 1712652288 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003887823470411234, + "loss": 1.3139, + "theoretical_loss": 3.504185486147602, + "tokens_seen": 1712717824 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003887662988966901, + "loss": 1.262, + "theoretical_loss": 3.5041740564646613, + "tokens_seen": 1712783360 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003887502507522568, + "loss": 1.2651, + "theoretical_loss": 3.5041626273414925, + "tokens_seen": 1712848896 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003887342026078235, + "loss": 1.2129, + "theoretical_loss": 3.504151198778047, + "tokens_seen": 1712914432 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003887181544633902, + "loss": 1.2808, + "theoretical_loss": 3.504139770774276, + "tokens_seen": 1712979968 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003887021063189569, + "loss": 1.1952, + "theoretical_loss": 3.5041283433301293, + "tokens_seen": 1713045504 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003886860581745236, + "loss": 1.3395, + "theoretical_loss": 3.5041169164455592, + "tokens_seen": 1713111040 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003886700100300903, + "loss": 1.2485, + "theoretical_loss": 3.5041054901205175, + "tokens_seen": 1713176576 + }, + { + "epoch": 0.52, + "learning_rate": 0.000388653961885657, + "loss": 1.3806, + "theoretical_loss": 3.504094064354954, + "tokens_seen": 1713242112 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003886379137412237, + "loss": 1.2293, + "theoretical_loss": 3.5040826391488213, + "tokens_seen": 1713307648 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003886218655967904, + "loss": 1.2129, + "theoretical_loss": 3.504071214502069, + "tokens_seen": 1713373184 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003886058174523571, + "loss": 1.3016, + "theoretical_loss": 3.50405979041465, + "tokens_seen": 1713438720 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003885897693079238, + "loss": 1.2437, + "theoretical_loss": 3.504048366886515, + "tokens_seen": 1713504256 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003885737211634905, + "loss": 1.3001, + "theoretical_loss": 3.5040369439176144, + "tokens_seen": 1713569792 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003885576730190572, + "loss": 1.2468, + "theoretical_loss": 3.5040255215079004, + "tokens_seen": 1713635328 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003885416248746239, + "loss": 1.3741, + "theoretical_loss": 3.504014099657323, + "tokens_seen": 1713700864 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 574441, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.194473147392273, + "objective/train/theoretical_loss": 3.5040026783658353, + "objective/train/tokens_used": 1734226400, + "theoretical_loss": 3.5040026783658353, + "tokens_seen": 1713766400 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003885255767301906, + "loss": 1.274, + "theoretical_loss": 3.5040026783658353, + "tokens_seen": 1713766400 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003885095285857573, + "loss": 1.1716, + "theoretical_loss": 3.5039912576333876, + "tokens_seen": 1713831936 + }, + { + "epoch": 0.52, + "learning_rate": 0.000388493480441324, + "loss": 1.3378, + "theoretical_loss": 3.5039798374599305, + "tokens_seen": 1713897472 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003884774322968907, + "loss": 1.2533, + "theoretical_loss": 3.5039684178454165, + "tokens_seen": 1713963008 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003884613841524574, + "loss": 1.2971, + "theoretical_loss": 3.5039569987897954, + "tokens_seen": 1714028544 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003884453360080241, + "loss": 1.2658, + "theoretical_loss": 3.50394558029302, + "tokens_seen": 1714094080 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003884292878635908, + "loss": 1.2133, + "theoretical_loss": 3.5039341623550406, + "tokens_seen": 1714159616 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003884132397191575, + "loss": 1.3748, + "theoretical_loss": 3.5039227449758092, + "tokens_seen": 1714225152 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003883971915747242, + "loss": 1.2349, + "theoretical_loss": 3.5039113281552767, + "tokens_seen": 1714290688 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003883811434302909, + "loss": 1.1814, + "theoretical_loss": 3.5038999118933942, + "tokens_seen": 1714356224 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003883650952858576, + "loss": 1.1674, + "theoretical_loss": 3.5038884961901138, + "tokens_seen": 1714421760 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003883490471414243, + "loss": 1.2864, + "theoretical_loss": 3.5038770810453856, + "tokens_seen": 1714487296 + }, + { + "epoch": 0.52, + "learning_rate": 0.000388332998996991, + "loss": 1.2061, + "theoretical_loss": 3.503865666459162, + "tokens_seen": 1714552832 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003883169508525577, + "loss": 1.2857, + "theoretical_loss": 3.5038542524313936, + "tokens_seen": 1714618368 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003883009027081244, + "loss": 1.248, + "theoretical_loss": 3.503842838962032, + "tokens_seen": 1714683904 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003882848545636911, + "loss": 1.291, + "theoretical_loss": 3.503831426051029, + "tokens_seen": 1714749440 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003882688064192578, + "loss": 1.2367, + "theoretical_loss": 3.503820013698335, + "tokens_seen": 1714814976 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003882527582748245, + "loss": 1.3266, + "theoretical_loss": 3.5038086019039025, + "tokens_seen": 1714880512 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003882367101303912, + "loss": 1.2368, + "theoretical_loss": 3.503797190667682, + "tokens_seen": 1714946048 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003882206619859579, + "loss": 1.3412, + "theoretical_loss": 3.5037857799896246, + "tokens_seen": 1715011584 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003882046138415246, + "loss": 1.2556, + "theoretical_loss": 3.503774369869683, + "tokens_seen": 1715077120 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003881885656970913, + "loss": 1.2661, + "theoretical_loss": 3.503762960307807, + "tokens_seen": 1715142656 + }, + { + "epoch": 0.52, + "learning_rate": 0.000388172517552658, + "loss": 1.1965, + "theoretical_loss": 3.503751551303949, + "tokens_seen": 1715208192 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003881564694082247, + "loss": 1.2574, + "theoretical_loss": 3.50374014285806, + "tokens_seen": 1715273728 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003881404212637914, + "loss": 1.1964, + "theoretical_loss": 3.503728734970092, + "tokens_seen": 1715339264 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003881243731193581, + "loss": 1.3806, + "theoretical_loss": 3.5037173276399955, + "tokens_seen": 1715404800 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003881083249749248, + "loss": 1.2613, + "theoretical_loss": 3.5037059208677226, + "tokens_seen": 1715470336 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003880922768304915, + "loss": 1.2779, + "theoretical_loss": 3.5036945146532243, + "tokens_seen": 1715535872 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003880762286860582, + "loss": 1.2052, + "theoretical_loss": 3.503683108996452, + "tokens_seen": 1715601408 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003880601805416249, + "loss": 1.2645, + "theoretical_loss": 3.503671703897357, + "tokens_seen": 1715666944 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003880441323971916, + "loss": 1.3068, + "theoretical_loss": 3.503660299355892, + "tokens_seen": 1715732480 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003880280842527583, + "loss": 1.2085, + "theoretical_loss": 3.503648895372007, + "tokens_seen": 1715798016 + }, + { + "epoch": 0.52, + "learning_rate": 0.000388012036108325, + "loss": 1.2277, + "theoretical_loss": 3.5036374919456534, + "tokens_seen": 1715863552 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003879959879638917, + "loss": 1.2956, + "theoretical_loss": 3.503626089076784, + "tokens_seen": 1715929088 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003879799398194584, + "loss": 1.2481, + "theoretical_loss": 3.5036146867653484, + "tokens_seen": 1715994624 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003879638916750251, + "loss": 1.3012, + "theoretical_loss": 3.5036032850113, + "tokens_seen": 1716060160 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003879478435305918, + "loss": 1.3273, + "theoretical_loss": 3.5035918838145887, + "tokens_seen": 1716125696 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003879317953861585, + "loss": 1.2891, + "theoretical_loss": 3.503580483175167, + "tokens_seen": 1716191232 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003879157472417252, + "loss": 1.2986, + "theoretical_loss": 3.5035690830929855, + "tokens_seen": 1716256768 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038789969909729186, + "loss": 1.1123, + "theoretical_loss": 3.5035576835679967, + "tokens_seen": 1716322304 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038788365095285856, + "loss": 1.346, + "theoretical_loss": 3.503546284600151, + "tokens_seen": 1716387840 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038786760280842527, + "loss": 1.233, + "theoretical_loss": 3.503534886189401, + "tokens_seen": 1716453376 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038785155466399197, + "loss": 1.2708, + "theoretical_loss": 3.5035234883356976, + "tokens_seen": 1716518912 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038783550651955867, + "loss": 1.2776, + "theoretical_loss": 3.503512091038992, + "tokens_seen": 1716584448 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038781945837512537, + "loss": 1.3255, + "theoretical_loss": 3.5035006942992366, + "tokens_seen": 1716649984 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038780341023069207, + "loss": 1.3007, + "theoretical_loss": 3.503489298116382, + "tokens_seen": 1716715520 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038778736208625877, + "loss": 1.2798, + "theoretical_loss": 3.5034779024903804, + "tokens_seen": 1716781056 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038777131394182547, + "loss": 1.3119, + "theoretical_loss": 3.503466507421183, + "tokens_seen": 1716846592 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038775526579739217, + "loss": 1.2688, + "theoretical_loss": 3.503455112908741, + "tokens_seen": 1716912128 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038773921765295887, + "loss": 1.1861, + "theoretical_loss": 3.5034437189530068, + "tokens_seen": 1716977664 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 575414, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4047168493270874, + "objective/train/theoretical_loss": 3.5034323255539315, + "objective/train/tokens_used": 1737503200, + "theoretical_loss": 3.5034323255539315, + "tokens_seen": 1717043200 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038772316950852557, + "loss": 1.3359, + "theoretical_loss": 3.5034323255539315, + "tokens_seen": 1717043200 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038770712136409227, + "loss": 1.1252, + "theoretical_loss": 3.5034209327114665, + "tokens_seen": 1717108736 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038769107321965897, + "loss": 1.2256, + "theoretical_loss": 3.503409540425564, + "tokens_seen": 1717174272 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038767502507522567, + "loss": 1.3175, + "theoretical_loss": 3.5033981486961747, + "tokens_seen": 1717239808 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003876589769307924, + "loss": 1.1878, + "theoretical_loss": 3.503386757523251, + "tokens_seen": 1717305344 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003876429287863591, + "loss": 1.2558, + "theoretical_loss": 3.5033753669067433, + "tokens_seen": 1717370880 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003876268806419258, + "loss": 1.2643, + "theoretical_loss": 3.5033639768466047, + "tokens_seen": 1717436416 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003876108324974925, + "loss": 1.3296, + "theoretical_loss": 3.5033525873427855, + "tokens_seen": 1717501952 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003875947843530592, + "loss": 1.3068, + "theoretical_loss": 3.5033411983952387, + "tokens_seen": 1717567488 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003875787362086259, + "loss": 1.2233, + "theoretical_loss": 3.5033298100039145, + "tokens_seen": 1717633024 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003875626880641926, + "loss": 1.2723, + "theoretical_loss": 3.503318422168765, + "tokens_seen": 1717698560 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003875466399197593, + "loss": 1.2527, + "theoretical_loss": 3.503307034889742, + "tokens_seen": 1717764096 + }, + { + "epoch": 0.52, + "learning_rate": 0.000387530591775326, + "loss": 1.2452, + "theoretical_loss": 3.503295648166797, + "tokens_seen": 1717829632 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003875145436308927, + "loss": 1.2603, + "theoretical_loss": 3.503284261999882, + "tokens_seen": 1717895168 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003874984954864594, + "loss": 1.284, + "theoretical_loss": 3.503272876388948, + "tokens_seen": 1717960704 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003874824473420261, + "loss": 1.3345, + "theoretical_loss": 3.503261491333947, + "tokens_seen": 1718026240 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003874663991975928, + "loss": 1.3111, + "theoretical_loss": 3.5032501068348307, + "tokens_seen": 1718091776 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003874503510531595, + "loss": 1.2987, + "theoretical_loss": 3.5032387228915507, + "tokens_seen": 1718157312 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003874343029087262, + "loss": 1.1232, + "theoretical_loss": 3.503227339504058, + "tokens_seen": 1718222848 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003874182547642929, + "loss": 1.2476, + "theoretical_loss": 3.503215956672306, + "tokens_seen": 1718288384 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003874022066198596, + "loss": 1.2676, + "theoretical_loss": 3.503204574396244, + "tokens_seen": 1718353920 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038738615847542634, + "loss": 1.3392, + "theoretical_loss": 3.5031931926758255, + "tokens_seen": 1718419456 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038737011033099304, + "loss": 1.0821, + "theoretical_loss": 3.5031818115110016, + "tokens_seen": 1718484992 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038735406218655974, + "loss": 1.2518, + "theoretical_loss": 3.503170430901724, + "tokens_seen": 1718550528 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038733801404212644, + "loss": 1.3108, + "theoretical_loss": 3.5031590508479447, + "tokens_seen": 1718616064 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038732196589769314, + "loss": 1.3032, + "theoretical_loss": 3.503147671349615, + "tokens_seen": 1718681600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038730591775325984, + "loss": 1.2388, + "theoretical_loss": 3.5031362924066864, + "tokens_seen": 1718747136 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038728986960882654, + "loss": 1.2785, + "theoretical_loss": 3.503124914019111, + "tokens_seen": 1718812672 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038727382146439324, + "loss": 1.2557, + "theoretical_loss": 3.5031135361868406, + "tokens_seen": 1718878208 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003872577733199599, + "loss": 1.295, + "theoretical_loss": 3.503102158909827, + "tokens_seen": 1718943744 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003872417251755266, + "loss": 1.2851, + "theoretical_loss": 3.5030907821880213, + "tokens_seen": 1719009280 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003872256770310933, + "loss": 1.2851, + "theoretical_loss": 3.503079406021376, + "tokens_seen": 1719074816 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038720962888666, + "loss": 1.2895, + "theoretical_loss": 3.5030680304098425, + "tokens_seen": 1719140352 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003871935807422267, + "loss": 1.3034, + "theoretical_loss": 3.5030566553533724, + "tokens_seen": 1719205888 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003871775325977934, + "loss": 1.3101, + "theoretical_loss": 3.503045280851918, + "tokens_seen": 1719271424 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003871614844533601, + "loss": 1.3319, + "theoretical_loss": 3.50303390690543, + "tokens_seen": 1719336960 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003871454363089268, + "loss": 1.2764, + "theoretical_loss": 3.5030225335138616, + "tokens_seen": 1719402496 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003871293881644935, + "loss": 1.2565, + "theoretical_loss": 3.503011160677164, + "tokens_seen": 1719468032 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003871133400200602, + "loss": 1.2869, + "theoretical_loss": 3.5029997883952877, + "tokens_seen": 1719533568 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003870972918756269, + "loss": 1.3573, + "theoretical_loss": 3.5029884166681864, + "tokens_seen": 1719599104 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003870812437311936, + "loss": 1.307, + "theoretical_loss": 3.502977045495811, + "tokens_seen": 1719664640 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003870651955867603, + "loss": 1.2413, + "theoretical_loss": 3.502965674878113, + "tokens_seen": 1719730176 + }, + { + "epoch": 0.52, + "learning_rate": 0.000387049147442327, + "loss": 1.393, + "theoretical_loss": 3.502954304815045, + "tokens_seen": 1719795712 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003870330992978937, + "loss": 1.2674, + "theoretical_loss": 3.5029429353065584, + "tokens_seen": 1719861248 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003870170511534604, + "loss": 1.2347, + "theoretical_loss": 3.502931566352605, + "tokens_seen": 1719926784 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003870010030090271, + "loss": 1.3603, + "theoretical_loss": 3.502920197953137, + "tokens_seen": 1719992320 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003869849548645938, + "loss": 1.3203, + "theoretical_loss": 3.5029088301081055, + "tokens_seen": 1720057856 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003869689067201605, + "loss": 1.2947, + "theoretical_loss": 3.502897462817463, + "tokens_seen": 1720123392 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003869528585757272, + "loss": 1.2552, + "theoretical_loss": 3.502886096081161, + "tokens_seen": 1720188928 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003869368104312939, + "loss": 1.3952, + "theoretical_loss": 3.502874729899151, + "tokens_seen": 1720254464 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 576723, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3283004760742188, + "objective/train/theoretical_loss": 3.502863364271386, + "objective/train/tokens_used": 1740780000, + "theoretical_loss": 3.502863364271386, + "tokens_seen": 1720320000 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003869207622868606, + "loss": 1.2018, + "theoretical_loss": 3.502863364271386, + "tokens_seen": 1720320000 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003869047141424273, + "loss": 1.1755, + "theoretical_loss": 3.5028519991978166, + "tokens_seen": 1720385536 + }, + { + "epoch": 0.52, + "learning_rate": 0.000386888665997994, + "loss": 1.2339, + "theoretical_loss": 3.5028406346783956, + "tokens_seen": 1720451072 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003868726178535607, + "loss": 1.2661, + "theoretical_loss": 3.5028292707130744, + "tokens_seen": 1720516608 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003868565697091274, + "loss": 1.263, + "theoretical_loss": 3.5028179073018046, + "tokens_seen": 1720582144 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003868405215646941, + "loss": 1.2141, + "theoretical_loss": 3.502806544444539, + "tokens_seen": 1720647680 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003868244734202608, + "loss": 1.3295, + "theoretical_loss": 3.5027951821412286, + "tokens_seen": 1720713216 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003868084252758275, + "loss": 1.2162, + "theoretical_loss": 3.5027838203918256, + "tokens_seen": 1720778752 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003867923771313942, + "loss": 1.1668, + "theoretical_loss": 3.502772459196282, + "tokens_seen": 1720844288 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003867763289869609, + "loss": 1.2905, + "theoretical_loss": 3.50276109855455, + "tokens_seen": 1720909824 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003867602808425276, + "loss": 1.2931, + "theoretical_loss": 3.502749738466581, + "tokens_seen": 1720975360 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003867442326980943, + "loss": 1.263, + "theoretical_loss": 3.5027383789323263, + "tokens_seen": 1721040896 + }, + { + "epoch": 0.52, + "learning_rate": 0.000386728184553661, + "loss": 1.2885, + "theoretical_loss": 3.5027270199517395, + "tokens_seen": 1721106432 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003867121364092277, + "loss": 1.2198, + "theoretical_loss": 3.502715661524771, + "tokens_seen": 1721171968 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003866960882647944, + "loss": 1.2981, + "theoretical_loss": 3.5027043036513743, + "tokens_seen": 1721237504 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003866800401203611, + "loss": 1.2177, + "theoretical_loss": 3.5026929463315, + "tokens_seen": 1721303040 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003866639919759278, + "loss": 1.3437, + "theoretical_loss": 3.5026815895651002, + "tokens_seen": 1721368576 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003866479438314945, + "loss": 1.36, + "theoretical_loss": 3.5026702333521276, + "tokens_seen": 1721434112 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003866318956870612, + "loss": 1.2333, + "theoretical_loss": 3.502658877692533, + "tokens_seen": 1721499648 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003866158475426279, + "loss": 1.2444, + "theoretical_loss": 3.50264752258627, + "tokens_seen": 1721565184 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003865997993981946, + "loss": 1.1374, + "theoretical_loss": 3.5026361680332894, + "tokens_seen": 1721630720 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003865837512537613, + "loss": 1.2729, + "theoretical_loss": 3.5026248140335428, + "tokens_seen": 1721696256 + }, + { + "epoch": 0.52, + "learning_rate": 0.000386567703109328, + "loss": 1.3754, + "theoretical_loss": 3.5026134605869834, + "tokens_seen": 1721761792 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003865516549648947, + "loss": 1.2265, + "theoretical_loss": 3.5026021076935625, + "tokens_seen": 1721827328 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003865356068204614, + "loss": 1.4701, + "theoretical_loss": 3.502590755353232, + "tokens_seen": 1721892864 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003865195586760281, + "loss": 1.3204, + "theoretical_loss": 3.502579403565944, + "tokens_seen": 1721958400 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003865035105315948, + "loss": 1.3213, + "theoretical_loss": 3.502568052331651, + "tokens_seen": 1722023936 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038648746238716147, + "loss": 1.2029, + "theoretical_loss": 3.502556701650305, + "tokens_seen": 1722089472 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038647141424272817, + "loss": 1.3054, + "theoretical_loss": 3.502545351521857, + "tokens_seen": 1722155008 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038645536609829487, + "loss": 1.2125, + "theoretical_loss": 3.50253400194626, + "tokens_seen": 1722220544 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038643931795386157, + "loss": 1.2697, + "theoretical_loss": 3.5025226529234654, + "tokens_seen": 1722286080 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038642326980942827, + "loss": 1.2667, + "theoretical_loss": 3.502511304453426, + "tokens_seen": 1722351616 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038640722166499497, + "loss": 1.3068, + "theoretical_loss": 3.5024999565360933, + "tokens_seen": 1722417152 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038639117352056167, + "loss": 1.2916, + "theoretical_loss": 3.5024886091714196, + "tokens_seen": 1722482688 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038637512537612837, + "loss": 1.3041, + "theoretical_loss": 3.5024772623593563, + "tokens_seen": 1722548224 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038635907723169507, + "loss": 1.2427, + "theoretical_loss": 3.5024659160998564, + "tokens_seen": 1722613760 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038634302908726177, + "loss": 1.2501, + "theoretical_loss": 3.502454570392872, + "tokens_seen": 1722679296 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038632698094282847, + "loss": 1.3302, + "theoretical_loss": 3.5024432252383537, + "tokens_seen": 1722744832 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038631093279839517, + "loss": 1.3459, + "theoretical_loss": 3.502431880636255, + "tokens_seen": 1722810368 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003862948846539619, + "loss": 1.2801, + "theoretical_loss": 3.502420536586528, + "tokens_seen": 1722875904 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003862788365095286, + "loss": 1.3317, + "theoretical_loss": 3.5024091930891244, + "tokens_seen": 1722941440 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003862627883650953, + "loss": 1.2373, + "theoretical_loss": 3.5023978501439963, + "tokens_seen": 1723006976 + }, + { + "epoch": 0.52, + "learning_rate": 0.000386246740220662, + "loss": 1.3406, + "theoretical_loss": 3.5023865077510954, + "tokens_seen": 1723072512 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003862306920762287, + "loss": 1.2907, + "theoretical_loss": 3.502375165910374, + "tokens_seen": 1723138048 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003862146439317954, + "loss": 1.2452, + "theoretical_loss": 3.502363824621785, + "tokens_seen": 1723203584 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003861985957873621, + "loss": 1.2664, + "theoretical_loss": 3.5023524838852804, + "tokens_seen": 1723269120 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038618254764292883, + "loss": 1.2562, + "theoretical_loss": 3.5023411437008116, + "tokens_seen": 1723334656 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038616649949849553, + "loss": 1.2671, + "theoretical_loss": 3.5023298040683306, + "tokens_seen": 1723400192 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038615045135406223, + "loss": 1.2465, + "theoretical_loss": 3.5023184649877903, + "tokens_seen": 1723465728 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038613440320962893, + "loss": 1.2813, + "theoretical_loss": 3.5023071264591423, + "tokens_seen": 1723531264 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 577498, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2661242485046387, + "objective/train/theoretical_loss": 3.502295788482339, + "objective/train/tokens_used": 1744056800, + "theoretical_loss": 3.502295788482339, + "tokens_seen": 1723596800 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038611835506519563, + "loss": 1.2756, + "theoretical_loss": 3.502295788482339, + "tokens_seen": 1723596800 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038610230692076234, + "loss": 1.2662, + "theoretical_loss": 3.5022844510573328, + "tokens_seen": 1723662336 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038608625877632904, + "loss": 1.2964, + "theoretical_loss": 3.5022731141840753, + "tokens_seen": 1723727872 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038607021063189574, + "loss": 1.32, + "theoretical_loss": 3.502261777862519, + "tokens_seen": 1723793408 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038605416248746244, + "loss": 1.3984, + "theoretical_loss": 3.502250442092617, + "tokens_seen": 1723858944 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038603811434302914, + "loss": 1.2503, + "theoretical_loss": 3.5022391068743195, + "tokens_seen": 1723924480 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038602206619859584, + "loss": 1.4484, + "theoretical_loss": 3.50222777220758, + "tokens_seen": 1723990016 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038600601805416254, + "loss": 1.2121, + "theoretical_loss": 3.5022164380923506, + "tokens_seen": 1724055552 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038598996990972924, + "loss": 1.3436, + "theoretical_loss": 3.502205104528583, + "tokens_seen": 1724121088 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038597392176529594, + "loss": 1.2334, + "theoretical_loss": 3.5021937715162306, + "tokens_seen": 1724186624 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038595787362086264, + "loss": 1.2394, + "theoretical_loss": 3.502182439055244, + "tokens_seen": 1724252160 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038594182547642934, + "loss": 1.3184, + "theoretical_loss": 3.5021711071455766, + "tokens_seen": 1724317696 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038592577733199604, + "loss": 1.351, + "theoretical_loss": 3.5021597757871796, + "tokens_seen": 1724383232 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038590972918756274, + "loss": 1.2574, + "theoretical_loss": 3.502148444980006, + "tokens_seen": 1724448768 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038589368104312944, + "loss": 1.3638, + "theoretical_loss": 3.5021371147240083, + "tokens_seen": 1724514304 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038587763289869614, + "loss": 1.2326, + "theoretical_loss": 3.502125785019138, + "tokens_seen": 1724579840 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003858615847542628, + "loss": 1.3309, + "theoretical_loss": 3.502114455865348, + "tokens_seen": 1724645376 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003858455366098295, + "loss": 1.2836, + "theoretical_loss": 3.5021031272625898, + "tokens_seen": 1724710912 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003858294884653962, + "loss": 1.2931, + "theoretical_loss": 3.5020917992108167, + "tokens_seen": 1724776448 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003858134403209629, + "loss": 1.2563, + "theoretical_loss": 3.50208047170998, + "tokens_seen": 1724841984 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003857973921765296, + "loss": 1.2966, + "theoretical_loss": 3.502069144760032, + "tokens_seen": 1724907520 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003857813440320963, + "loss": 1.2582, + "theoretical_loss": 3.502057818360926, + "tokens_seen": 1724973056 + }, + { + "epoch": 0.52, + "learning_rate": 0.000385765295887663, + "loss": 1.1705, + "theoretical_loss": 3.502046492512613, + "tokens_seen": 1725038592 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003857492477432297, + "loss": 1.383, + "theoretical_loss": 3.502035167215046, + "tokens_seen": 1725104128 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003857331995987964, + "loss": 1.3285, + "theoretical_loss": 3.5020238424681773, + "tokens_seen": 1725169664 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003857171514543631, + "loss": 1.2179, + "theoretical_loss": 3.5020125182719593, + "tokens_seen": 1725235200 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003857011033099298, + "loss": 1.2932, + "theoretical_loss": 3.502001194626344, + "tokens_seen": 1725300736 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003856850551654965, + "loss": 1.3432, + "theoretical_loss": 3.5019898715312836, + "tokens_seen": 1725366272 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003856690070210632, + "loss": 1.2372, + "theoretical_loss": 3.5019785489867306, + "tokens_seen": 1725431808 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003856529588766299, + "loss": 1.2005, + "theoretical_loss": 3.5019672269926376, + "tokens_seen": 1725497344 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003856369107321966, + "loss": 1.2437, + "theoretical_loss": 3.5019559055489564, + "tokens_seen": 1725562880 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003856208625877633, + "loss": 1.3594, + "theoretical_loss": 3.5019445846556403, + "tokens_seen": 1725628416 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038560481444333, + "loss": 1.3102, + "theoretical_loss": 3.5019332643126404, + "tokens_seen": 1725693952 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003855887662988967, + "loss": 1.377, + "theoretical_loss": 3.5019219445199097, + "tokens_seen": 1725759488 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003855727181544634, + "loss": 1.2909, + "theoretical_loss": 3.5019106252774006, + "tokens_seen": 1725825024 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003855566700100301, + "loss": 1.2693, + "theoretical_loss": 3.5018993065850657, + "tokens_seen": 1725890560 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003855406218655968, + "loss": 1.2746, + "theoretical_loss": 3.5018879884428564, + "tokens_seen": 1725956096 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003855245737211635, + "loss": 1.282, + "theoretical_loss": 3.501876670850726, + "tokens_seen": 1726021632 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003855085255767302, + "loss": 1.3583, + "theoretical_loss": 3.5018653538086264, + "tokens_seen": 1726087168 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003854924774322969, + "loss": 1.3132, + "theoretical_loss": 3.50185403731651, + "tokens_seen": 1726152704 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003854764292878636, + "loss": 1.3157, + "theoretical_loss": 3.50184272137433, + "tokens_seen": 1726218240 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003854603811434303, + "loss": 1.3115, + "theoretical_loss": 3.501831405982038, + "tokens_seen": 1726283776 + }, + { + "epoch": 0.52, + "learning_rate": 0.000385444332998997, + "loss": 1.1602, + "theoretical_loss": 3.501820091139586, + "tokens_seen": 1726349312 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003854282848545637, + "loss": 1.2871, + "theoretical_loss": 3.5018087768469277, + "tokens_seen": 1726414848 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003854122367101304, + "loss": 1.3348, + "theoretical_loss": 3.5017974631040145, + "tokens_seen": 1726480384 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003853961885656971, + "loss": 1.3088, + "theoretical_loss": 3.5017861499107985, + "tokens_seen": 1726545920 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003853801404212638, + "loss": 1.2737, + "theoretical_loss": 3.5017748372672335, + "tokens_seen": 1726611456 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003853640922768305, + "loss": 1.2174, + "theoretical_loss": 3.501763525173271, + "tokens_seen": 1726676992 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003853480441323972, + "loss": 1.2302, + "theoretical_loss": 3.5017522136288637, + "tokens_seen": 1726742528 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003853319959879639, + "loss": 1.2866, + "theoretical_loss": 3.501740902633964, + "tokens_seen": 1726808064 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 578865, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2408570051193237, + "objective/train/theoretical_loss": 3.501729592188524, + "objective/train/tokens_used": 1747333600, + "theoretical_loss": 3.501729592188524, + "tokens_seen": 1726873600 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003853159478435306, + "loss": 1.2528, + "theoretical_loss": 3.501729592188524, + "tokens_seen": 1726873600 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003852998996990973, + "loss": 1.2242, + "theoretical_loss": 3.5017182822924964, + "tokens_seen": 1726939136 + }, + { + "epoch": 0.52, + "learning_rate": 0.000385283851554664, + "loss": 1.3313, + "theoretical_loss": 3.5017069729458337, + "tokens_seen": 1727004672 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003852678034102307, + "loss": 1.3475, + "theoretical_loss": 3.501695664148489, + "tokens_seen": 1727070208 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003852517552657974, + "loss": 1.2876, + "theoretical_loss": 3.501684355900414, + "tokens_seen": 1727135744 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003852357071213641, + "loss": 1.2614, + "theoretical_loss": 3.5016730482015612, + "tokens_seen": 1727201280 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003852196589769308, + "loss": 1.303, + "theoretical_loss": 3.5016617410518833, + "tokens_seen": 1727266816 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003852036108324975, + "loss": 1.2216, + "theoretical_loss": 3.5016504344513324, + "tokens_seen": 1727332352 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003851875626880642, + "loss": 1.3225, + "theoretical_loss": 3.5016391283998614, + "tokens_seen": 1727397888 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003851715145436309, + "loss": 1.3322, + "theoretical_loss": 3.501627822897423, + "tokens_seen": 1727463424 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003851554663991976, + "loss": 1.3411, + "theoretical_loss": 3.5016165179439698, + "tokens_seen": 1727528960 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003851394182547643, + "loss": 1.3269, + "theoretical_loss": 3.5016052135394533, + "tokens_seen": 1727594496 + }, + { + "epoch": 0.52, + "learning_rate": 0.000385123370110331, + "loss": 1.2784, + "theoretical_loss": 3.5015939096838267, + "tokens_seen": 1727660032 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003851073219658977, + "loss": 1.1286, + "theoretical_loss": 3.501582606377043, + "tokens_seen": 1727725568 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038509127382146437, + "loss": 1.1784, + "theoretical_loss": 3.501571303619054, + "tokens_seen": 1727791104 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038507522567703107, + "loss": 1.3183, + "theoretical_loss": 3.5015600014098127, + "tokens_seen": 1727856640 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038505917753259777, + "loss": 1.1559, + "theoretical_loss": 3.5015486997492715, + "tokens_seen": 1727922176 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038504312938816447, + "loss": 1.2435, + "theoretical_loss": 3.5015373986373826, + "tokens_seen": 1727987712 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038502708124373117, + "loss": 1.2908, + "theoretical_loss": 3.5015260980740988, + "tokens_seen": 1728053248 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038501103309929787, + "loss": 1.2526, + "theoretical_loss": 3.5015147980593726, + "tokens_seen": 1728118784 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038499498495486457, + "loss": 1.2501, + "theoretical_loss": 3.5015034985931575, + "tokens_seen": 1728184320 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003849789368104313, + "loss": 1.1863, + "theoretical_loss": 3.5014921996754045, + "tokens_seen": 1728249856 + }, + { + "epoch": 0.52, + "learning_rate": 0.000384962888665998, + "loss": 1.2358, + "theoretical_loss": 3.501480901306067, + "tokens_seen": 1728315392 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003849468405215647, + "loss": 1.2084, + "theoretical_loss": 3.5014696034850976, + "tokens_seen": 1728380928 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038493079237713143, + "loss": 1.255, + "theoretical_loss": 3.501458306212449, + "tokens_seen": 1728446464 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038491474423269813, + "loss": 1.3228, + "theoretical_loss": 3.5014470094880736, + "tokens_seen": 1728512000 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038489869608826483, + "loss": 1.214, + "theoretical_loss": 3.501435713311924, + "tokens_seen": 1728577536 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038488264794383153, + "loss": 1.3664, + "theoretical_loss": 3.501424417683953, + "tokens_seen": 1728643072 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038486659979939823, + "loss": 1.4088, + "theoretical_loss": 3.501413122604113, + "tokens_seen": 1728708608 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038485055165496493, + "loss": 1.2379, + "theoretical_loss": 3.501401828072356, + "tokens_seen": 1728774144 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038483450351053163, + "loss": 1.3197, + "theoretical_loss": 3.501390534088636, + "tokens_seen": 1728839680 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038481845536609833, + "loss": 1.3221, + "theoretical_loss": 3.5013792406529047, + "tokens_seen": 1728905216 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038480240722166503, + "loss": 1.3249, + "theoretical_loss": 3.5013679477651154, + "tokens_seen": 1728970752 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038478635907723173, + "loss": 1.3494, + "theoretical_loss": 3.5013566554252202, + "tokens_seen": 1729036288 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038477031093279843, + "loss": 1.2786, + "theoretical_loss": 3.5013453636331717, + "tokens_seen": 1729101824 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038475426278836513, + "loss": 1.3348, + "theoretical_loss": 3.501334072388923, + "tokens_seen": 1729167360 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038473821464393183, + "loss": 1.2591, + "theoretical_loss": 3.501322781692426, + "tokens_seen": 1729232896 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038472216649949854, + "loss": 1.4162, + "theoretical_loss": 3.5013114915436345, + "tokens_seen": 1729298432 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038470611835506524, + "loss": 1.3602, + "theoretical_loss": 3.5013002019424997, + "tokens_seen": 1729363968 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038469007021063194, + "loss": 1.2954, + "theoretical_loss": 3.5012889128889757, + "tokens_seen": 1729429504 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038467402206619864, + "loss": 1.2467, + "theoretical_loss": 3.5012776243830146, + "tokens_seen": 1729495040 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038465797392176534, + "loss": 1.3075, + "theoretical_loss": 3.5012663364245693, + "tokens_seen": 1729560576 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038464192577733204, + "loss": 1.2362, + "theoretical_loss": 3.5012550490135923, + "tokens_seen": 1729626112 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038462587763289874, + "loss": 1.351, + "theoretical_loss": 3.501243762150036, + "tokens_seen": 1729691648 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038460982948846544, + "loss": 1.2804, + "theoretical_loss": 3.501232475833854, + "tokens_seen": 1729757184 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038459378134403214, + "loss": 1.3178, + "theoretical_loss": 3.5012211900649977, + "tokens_seen": 1729822720 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038457773319959884, + "loss": 1.2581, + "theoretical_loss": 3.501209904843421, + "tokens_seen": 1729888256 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038456168505516554, + "loss": 1.2255, + "theoretical_loss": 3.501198620169076, + "tokens_seen": 1729953792 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038454563691073224, + "loss": 1.2716, + "theoretical_loss": 3.501187336041916, + "tokens_seen": 1730019328 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038452958876629894, + "loss": 1.332, + "theoretical_loss": 3.501176052461893, + "tokens_seen": 1730084864 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 579798, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.325954556465149, + "objective/train/theoretical_loss": 3.5011647694289607, + "objective/train/tokens_used": 1750610400, + "theoretical_loss": 3.5011647694289607, + "tokens_seen": 1730150400 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038451354062186564, + "loss": 1.3125, + "theoretical_loss": 3.5011647694289607, + "tokens_seen": 1730150400 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038449749247743234, + "loss": 1.2824, + "theoretical_loss": 3.5011534869430703, + "tokens_seen": 1730215936 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038448144433299904, + "loss": 1.3664, + "theoretical_loss": 3.501142205004176, + "tokens_seen": 1730281472 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038446539618856575, + "loss": 1.2695, + "theoretical_loss": 3.5011309236122305, + "tokens_seen": 1730347008 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003844493480441324, + "loss": 1.2399, + "theoretical_loss": 3.5011196427671862, + "tokens_seen": 1730412544 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003844332998996991, + "loss": 1.185, + "theoretical_loss": 3.5011083624689956, + "tokens_seen": 1730478080 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003844172517552658, + "loss": 1.1577, + "theoretical_loss": 3.501097082717611, + "tokens_seen": 1730543616 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003844012036108325, + "loss": 1.3516, + "theoretical_loss": 3.501085803512987, + "tokens_seen": 1730609152 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003843851554663992, + "loss": 1.2924, + "theoretical_loss": 3.501074524855075, + "tokens_seen": 1730674688 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003843691073219659, + "loss": 1.3261, + "theoretical_loss": 3.5010632467438283, + "tokens_seen": 1730740224 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003843530591775326, + "loss": 1.191, + "theoretical_loss": 3.501051969179199, + "tokens_seen": 1730805760 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003843370110330993, + "loss": 1.2456, + "theoretical_loss": 3.5010406921611406, + "tokens_seen": 1730871296 + }, + { + "epoch": 0.52, + "learning_rate": 0.000384320962888666, + "loss": 1.2673, + "theoretical_loss": 3.501029415689606, + "tokens_seen": 1730936832 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003843049147442327, + "loss": 1.2671, + "theoretical_loss": 3.5010181397645472, + "tokens_seen": 1731002368 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003842888665997994, + "loss": 1.2298, + "theoretical_loss": 3.5010068643859182, + "tokens_seen": 1731067904 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003842728184553661, + "loss": 1.3162, + "theoretical_loss": 3.500995589553671, + "tokens_seen": 1731133440 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003842567703109328, + "loss": 1.2446, + "theoretical_loss": 3.5009843152677584, + "tokens_seen": 1731198976 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003842407221664995, + "loss": 1.2342, + "theoretical_loss": 3.500973041528134, + "tokens_seen": 1731264512 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003842246740220662, + "loss": 1.2531, + "theoretical_loss": 3.50096176833475, + "tokens_seen": 1731330048 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003842086258776329, + "loss": 1.255, + "theoretical_loss": 3.5009504956875594, + "tokens_seen": 1731395584 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003841925777331996, + "loss": 1.2469, + "theoretical_loss": 3.500939223586515, + "tokens_seen": 1731461120 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003841765295887663, + "loss": 1.27, + "theoretical_loss": 3.50092795203157, + "tokens_seen": 1731526656 + }, + { + "epoch": 0.52, + "learning_rate": 0.000384160481444333, + "loss": 1.2564, + "theoretical_loss": 3.5009166810226766, + "tokens_seen": 1731592192 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003841444332998997, + "loss": 1.2429, + "theoretical_loss": 3.5009054105597883, + "tokens_seen": 1731657728 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003841283851554664, + "loss": 1.2264, + "theoretical_loss": 3.500894140642858, + "tokens_seen": 1731723264 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003841123370110331, + "loss": 1.1761, + "theoretical_loss": 3.5008828712718385, + "tokens_seen": 1731788800 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003840962888665998, + "loss": 1.2289, + "theoretical_loss": 3.5008716024466824, + "tokens_seen": 1731854336 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003840802407221665, + "loss": 1.3899, + "theoretical_loss": 3.500860334167343, + "tokens_seen": 1731919872 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003840641925777332, + "loss": 1.2143, + "theoretical_loss": 3.500849066433773, + "tokens_seen": 1731985408 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003840481444332999, + "loss": 1.2238, + "theoretical_loss": 3.5008377992459248, + "tokens_seen": 1732050944 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003840320962888666, + "loss": 1.2567, + "theoretical_loss": 3.500826532603752, + "tokens_seen": 1732116480 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003840160481444333, + "loss": 1.2777, + "theoretical_loss": 3.5008152665072076, + "tokens_seen": 1732182016 + }, + { + "epoch": 0.52, + "learning_rate": 0.000384, + "loss": 1.3072, + "theoretical_loss": 3.5008040009562444, + "tokens_seen": 1732247552 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003839839518555667, + "loss": 1.162, + "theoretical_loss": 3.500792735950815, + "tokens_seen": 1732313088 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003839679037111334, + "loss": 1.3281, + "theoretical_loss": 3.500781471490873, + "tokens_seen": 1732378624 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003839518555667001, + "loss": 1.2472, + "theoretical_loss": 3.5007702075763705, + "tokens_seen": 1732444160 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003839358074222668, + "loss": 1.344, + "theoretical_loss": 3.500758944207261, + "tokens_seen": 1732509696 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003839197592778335, + "loss": 1.2549, + "theoretical_loss": 3.5007476813834977, + "tokens_seen": 1732575232 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003839037111334002, + "loss": 1.2809, + "theoretical_loss": 3.5007364191050327, + "tokens_seen": 1732640768 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003838876629889669, + "loss": 1.2261, + "theoretical_loss": 3.5007251573718197, + "tokens_seen": 1732706304 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003838716148445336, + "loss": 1.2648, + "theoretical_loss": 3.500713896183812, + "tokens_seen": 1732771840 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003838555667001003, + "loss": 1.2785, + "theoretical_loss": 3.5007026355409616, + "tokens_seen": 1732837376 + }, + { + "epoch": 0.53, + "learning_rate": 0.000383839518555667, + "loss": 1.339, + "theoretical_loss": 3.500691375443222, + "tokens_seen": 1732902912 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003838234704112337, + "loss": 1.1722, + "theoretical_loss": 3.500680115890546, + "tokens_seen": 1732968448 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003838074222668004, + "loss": 1.282, + "theoretical_loss": 3.5006688568828874, + "tokens_seen": 1733033984 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003837913741223671, + "loss": 1.2174, + "theoretical_loss": 3.500657598420198, + "tokens_seen": 1733099520 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003837753259779338, + "loss": 1.2379, + "theoretical_loss": 3.5006463405024313, + "tokens_seen": 1733165056 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003837592778335005, + "loss": 1.3383, + "theoretical_loss": 3.5006350831295414, + "tokens_seen": 1733230592 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003837432296890672, + "loss": 1.2755, + "theoretical_loss": 3.5006238263014793, + "tokens_seen": 1733296128 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003837271815446339, + "loss": 1.3077, + "theoretical_loss": 3.5006125700181996, + "tokens_seen": 1733361664 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 580942, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1904053688049316, + "objective/train/theoretical_loss": 3.5006013142796544, + "objective/train/tokens_used": 1753887200, + "theoretical_loss": 3.5006013142796544, + "tokens_seen": 1733427200 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003837111334002006, + "loss": 1.2738, + "theoretical_loss": 3.5006013142796544, + "tokens_seen": 1733427200 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003836950852557673, + "loss": 1.2081, + "theoretical_loss": 3.5005900590857975, + "tokens_seen": 1733492736 + }, + { + "epoch": 0.53, + "learning_rate": 0.000383679037111334, + "loss": 1.2873, + "theoretical_loss": 3.5005788044365818, + "tokens_seen": 1733558272 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003836629889669007, + "loss": 1.2783, + "theoretical_loss": 3.5005675503319598, + "tokens_seen": 1733623808 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003836469408224674, + "loss": 1.151, + "theoretical_loss": 3.5005562967718853, + "tokens_seen": 1733689344 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003836308926780341, + "loss": 1.3054, + "theoretical_loss": 3.5005450437563104, + "tokens_seen": 1733754880 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003836148445336008, + "loss": 1.2496, + "theoretical_loss": 3.5005337912851893, + "tokens_seen": 1733820416 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003835987963891675, + "loss": 1.3286, + "theoretical_loss": 3.5005225393584745, + "tokens_seen": 1733885952 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003835827482447342, + "loss": 1.2651, + "theoretical_loss": 3.500511287976119, + "tokens_seen": 1733951488 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003835667001003009, + "loss": 1.1749, + "theoretical_loss": 3.5005000371380763, + "tokens_seen": 1734017024 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038355065195586763, + "loss": 1.3233, + "theoretical_loss": 3.500488786844299, + "tokens_seen": 1734082560 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038353460381143433, + "loss": 1.2113, + "theoretical_loss": 3.5004775370947403, + "tokens_seen": 1734148096 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038351855566700103, + "loss": 1.3278, + "theoretical_loss": 3.5004662878893535, + "tokens_seen": 1734213632 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038350250752256773, + "loss": 1.2055, + "theoretical_loss": 3.500455039228092, + "tokens_seen": 1734279168 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038348645937813443, + "loss": 1.3052, + "theoretical_loss": 3.500443791110908, + "tokens_seen": 1734344704 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038347041123370113, + "loss": 1.3052, + "theoretical_loss": 3.500432543537756, + "tokens_seen": 1734410240 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038345436308926783, + "loss": 1.321, + "theoretical_loss": 3.5004212965085877, + "tokens_seen": 1734475776 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038343831494483453, + "loss": 1.308, + "theoretical_loss": 3.5004100500233575, + "tokens_seen": 1734541312 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038342226680040123, + "loss": 1.3135, + "theoretical_loss": 3.500398804082017, + "tokens_seen": 1734606848 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038340621865596793, + "loss": 1.2704, + "theoretical_loss": 3.500387558684521, + "tokens_seen": 1734672384 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038339017051153463, + "loss": 1.3219, + "theoretical_loss": 3.500376313830822, + "tokens_seen": 1734737920 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038337412236710133, + "loss": 1.2833, + "theoretical_loss": 3.500365069520872, + "tokens_seen": 1734803456 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038335807422266804, + "loss": 1.1523, + "theoretical_loss": 3.500353825754626, + "tokens_seen": 1734868992 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038334202607823474, + "loss": 1.2502, + "theoretical_loss": 3.5003425825320367, + "tokens_seen": 1734934528 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038332597793380144, + "loss": 1.2206, + "theoretical_loss": 3.500331339853057, + "tokens_seen": 1735000064 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038330992978936814, + "loss": 1.2775, + "theoretical_loss": 3.5003200977176396, + "tokens_seen": 1735065600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038329388164493484, + "loss": 1.2141, + "theoretical_loss": 3.5003088561257387, + "tokens_seen": 1735131136 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038327783350050154, + "loss": 1.1558, + "theoretical_loss": 3.500297615077306, + "tokens_seen": 1735196672 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038326178535606824, + "loss": 1.4139, + "theoretical_loss": 3.5002863745722967, + "tokens_seen": 1735262208 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038324573721163494, + "loss": 1.2256, + "theoretical_loss": 3.5002751346106624, + "tokens_seen": 1735327744 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038322968906720164, + "loss": 1.2166, + "theoretical_loss": 3.5002638951923575, + "tokens_seen": 1735393280 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038321364092276834, + "loss": 1.2284, + "theoretical_loss": 3.5002526563173344, + "tokens_seen": 1735458816 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038319759277833504, + "loss": 1.2755, + "theoretical_loss": 3.500241417985546, + "tokens_seen": 1735524352 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038318154463390174, + "loss": 1.2979, + "theoretical_loss": 3.5002301801969464, + "tokens_seen": 1735589888 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038316549648946844, + "loss": 1.1886, + "theoretical_loss": 3.5002189429514887, + "tokens_seen": 1735655424 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038314944834503514, + "loss": 1.1812, + "theoretical_loss": 3.500207706249126, + "tokens_seen": 1735720960 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038313340020060184, + "loss": 1.3195, + "theoretical_loss": 3.5001964700898114, + "tokens_seen": 1735786496 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038311735205616854, + "loss": 1.1482, + "theoretical_loss": 3.500185234473498, + "tokens_seen": 1735852032 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038310130391173525, + "loss": 1.2689, + "theoretical_loss": 3.5001739994001397, + "tokens_seen": 1735917568 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038308525576730195, + "loss": 1.2669, + "theoretical_loss": 3.500162764869689, + "tokens_seen": 1735983104 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038306920762286865, + "loss": 1.3127, + "theoretical_loss": 3.5001515308821, + "tokens_seen": 1736048640 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003830531594784353, + "loss": 1.2754, + "theoretical_loss": 3.5001402974373246, + "tokens_seen": 1736114176 + }, + { + "epoch": 0.53, + "learning_rate": 0.000383037111334002, + "loss": 1.2392, + "theoretical_loss": 3.500129064535318, + "tokens_seen": 1736179712 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003830210631895687, + "loss": 1.2436, + "theoretical_loss": 3.500117832176032, + "tokens_seen": 1736245248 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003830050150451354, + "loss": 1.3301, + "theoretical_loss": 3.5001066003594206, + "tokens_seen": 1736310784 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003829889669007021, + "loss": 1.2287, + "theoretical_loss": 3.500095369085437, + "tokens_seen": 1736376320 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003829729187562688, + "loss": 1.1851, + "theoretical_loss": 3.500084138354034, + "tokens_seen": 1736441856 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003829568706118355, + "loss": 1.3487, + "theoretical_loss": 3.5000729081651656, + "tokens_seen": 1736507392 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003829408224674022, + "loss": 1.3696, + "theoretical_loss": 3.5000616785187844, + "tokens_seen": 1736572928 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003829247743229689, + "loss": 1.2771, + "theoretical_loss": 3.5000504494148448, + "tokens_seen": 1736638464 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 581741, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3218003511428833, + "objective/train/theoretical_loss": 3.5000392208532984, + "objective/train/tokens_used": 1757164000, + "theoretical_loss": 3.5000392208532984, + "tokens_seen": 1736704000 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003829087261785356, + "loss": 1.2328, + "theoretical_loss": 3.5000392208532984, + "tokens_seen": 1736704000 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003828926780341023, + "loss": 1.2375, + "theoretical_loss": 3.5000279928341005, + "tokens_seen": 1736769536 + }, + { + "epoch": 0.53, + "learning_rate": 0.000382876629889669, + "loss": 1.2961, + "theoretical_loss": 3.500016765357204, + "tokens_seen": 1736835072 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003828605817452357, + "loss": 1.2916, + "theoretical_loss": 3.5000055384225606, + "tokens_seen": 1736900608 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003828445336008024, + "loss": 1.2144, + "theoretical_loss": 3.499994312030126, + "tokens_seen": 1736966144 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003828284854563691, + "loss": 1.2668, + "theoretical_loss": 3.4999830861798515, + "tokens_seen": 1737031680 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003828124373119358, + "loss": 1.2562, + "theoretical_loss": 3.4999718608716917, + "tokens_seen": 1737097216 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003827963891675025, + "loss": 1.3202, + "theoretical_loss": 3.4999606361056, + "tokens_seen": 1737162752 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003827803410230692, + "loss": 1.1997, + "theoretical_loss": 3.499949411881529, + "tokens_seen": 1737228288 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003827642928786359, + "loss": 1.1846, + "theoretical_loss": 3.4999381881994323, + "tokens_seen": 1737293824 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003827482447342026, + "loss": 1.332, + "theoretical_loss": 3.499926965059264, + "tokens_seen": 1737359360 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003827321965897693, + "loss": 1.3656, + "theoretical_loss": 3.4999157424609764, + "tokens_seen": 1737424896 + }, + { + "epoch": 0.53, + "learning_rate": 0.000382716148445336, + "loss": 1.2244, + "theoretical_loss": 3.499904520404524, + "tokens_seen": 1737490432 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003827001003009027, + "loss": 1.3175, + "theoretical_loss": 3.4998932988898597, + "tokens_seen": 1737555968 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003826840521564694, + "loss": 1.1929, + "theoretical_loss": 3.4998820779169364, + "tokens_seen": 1737621504 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003826680040120361, + "loss": 1.2498, + "theoretical_loss": 3.4998708574857083, + "tokens_seen": 1737687040 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003826519558676028, + "loss": 1.2214, + "theoretical_loss": 3.4998596375961286, + "tokens_seen": 1737752576 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003826359077231695, + "loss": 1.2156, + "theoretical_loss": 3.4998484182481504, + "tokens_seen": 1737818112 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003826198595787362, + "loss": 1.1236, + "theoretical_loss": 3.4998371994417274, + "tokens_seen": 1737883648 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003826038114343029, + "loss": 1.3355, + "theoretical_loss": 3.499825981176813, + "tokens_seen": 1737949184 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003825877632898696, + "loss": 1.2855, + "theoretical_loss": 3.4998147634533607, + "tokens_seen": 1738014720 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038257171514543637, + "loss": 1.1721, + "theoretical_loss": 3.499803546271324, + "tokens_seen": 1738080256 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038255566700100307, + "loss": 1.3256, + "theoretical_loss": 3.499792329630657, + "tokens_seen": 1738145792 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038253961885656977, + "loss": 1.4245, + "theoretical_loss": 3.4997811135313115, + "tokens_seen": 1738211328 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038252357071213647, + "loss": 1.3257, + "theoretical_loss": 3.499769897973242, + "tokens_seen": 1738276864 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038250752256770317, + "loss": 1.2368, + "theoretical_loss": 3.4997586829564025, + "tokens_seen": 1738342400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038249147442326987, + "loss": 1.2761, + "theoretical_loss": 3.499747468480745, + "tokens_seen": 1738407936 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038247542627883657, + "loss": 1.1754, + "theoretical_loss": 3.4997362545462245, + "tokens_seen": 1738473472 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038245937813440327, + "loss": 1.1587, + "theoretical_loss": 3.499725041152793, + "tokens_seen": 1738539008 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038244332998996997, + "loss": 1.3838, + "theoretical_loss": 3.499713828300406, + "tokens_seen": 1738604544 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003824272818455366, + "loss": 1.2256, + "theoretical_loss": 3.4997026159890146, + "tokens_seen": 1738670080 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003824112337011033, + "loss": 1.2441, + "theoretical_loss": 3.4996914042185745, + "tokens_seen": 1738735616 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038239518555667, + "loss": 1.2126, + "theoretical_loss": 3.4996801929890378, + "tokens_seen": 1738801152 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003823791374122367, + "loss": 1.2905, + "theoretical_loss": 3.4996689823003586, + "tokens_seen": 1738866688 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003823630892678034, + "loss": 1.2692, + "theoretical_loss": 3.4996577721524904, + "tokens_seen": 1738932224 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003823470411233701, + "loss": 1.2691, + "theoretical_loss": 3.4996465625453865, + "tokens_seen": 1738997760 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003823309929789368, + "loss": 1.301, + "theoretical_loss": 3.4996353534790003, + "tokens_seen": 1739063296 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003823149448345035, + "loss": 1.2231, + "theoretical_loss": 3.4996241449532857, + "tokens_seen": 1739128832 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003822988966900702, + "loss": 1.2752, + "theoretical_loss": 3.4996129369681963, + "tokens_seen": 1739194368 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003822828485456369, + "loss": 1.1965, + "theoretical_loss": 3.4996017295236856, + "tokens_seen": 1739259904 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003822668004012036, + "loss": 1.1893, + "theoretical_loss": 3.499590522619707, + "tokens_seen": 1739325440 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003822507522567703, + "loss": 1.2183, + "theoretical_loss": 3.499579316256214, + "tokens_seen": 1739390976 + }, + { + "epoch": 0.53, + "learning_rate": 0.000382234704112337, + "loss": 1.2983, + "theoretical_loss": 3.4995681104331604, + "tokens_seen": 1739456512 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003822186559679037, + "loss": 1.2755, + "theoretical_loss": 3.4995569051504996, + "tokens_seen": 1739522048 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003822026078234704, + "loss": 1.1588, + "theoretical_loss": 3.499545700408185, + "tokens_seen": 1739587584 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038218655967903713, + "loss": 1.2943, + "theoretical_loss": 3.499534496206171, + "tokens_seen": 1739653120 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038217051153460383, + "loss": 1.2899, + "theoretical_loss": 3.49952329254441, + "tokens_seen": 1739718656 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038215446339017053, + "loss": 1.3464, + "theoretical_loss": 3.499512089422857, + "tokens_seen": 1739784192 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038213841524573723, + "loss": 1.2774, + "theoretical_loss": 3.4995008868414645, + "tokens_seen": 1739849728 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038212236710130393, + "loss": 1.334, + "theoretical_loss": 3.499489684800186, + "tokens_seen": 1739915264 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 583122, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2249778509140015, + "objective/train/theoretical_loss": 3.4994784832989763, + "objective/train/tokens_used": 1760440800, + "theoretical_loss": 3.4994784832989763, + "tokens_seen": 1739980800 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038210631895687063, + "loss": 1.2123, + "theoretical_loss": 3.4994784832989763, + "tokens_seen": 1739980800 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038209027081243733, + "loss": 1.2504, + "theoretical_loss": 3.499467282337788, + "tokens_seen": 1740046336 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038207422266800403, + "loss": 1.1921, + "theoretical_loss": 3.499456081916575, + "tokens_seen": 1740111872 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038205817452357073, + "loss": 1.3552, + "theoretical_loss": 3.4994448820352906, + "tokens_seen": 1740177408 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038204212637913743, + "loss": 1.1832, + "theoretical_loss": 3.4994336826938897, + "tokens_seen": 1740242944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038202607823470413, + "loss": 1.2714, + "theoretical_loss": 3.499422483892324, + "tokens_seen": 1740308480 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038201003009027083, + "loss": 1.2785, + "theoretical_loss": 3.499411285630549, + "tokens_seen": 1740374016 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038199398194583753, + "loss": 1.2713, + "theoretical_loss": 3.499400087908517, + "tokens_seen": 1740439552 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038197793380140424, + "loss": 1.2904, + "theoretical_loss": 3.4993888907261823, + "tokens_seen": 1740505088 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038196188565697094, + "loss": 1.2503, + "theoretical_loss": 3.4993776940834986, + "tokens_seen": 1740570624 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038194583751253764, + "loss": 1.2595, + "theoretical_loss": 3.4993664979804198, + "tokens_seen": 1740636160 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038192978936810434, + "loss": 1.2244, + "theoretical_loss": 3.4993553024168986, + "tokens_seen": 1740701696 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038191374122367104, + "loss": 1.3059, + "theoretical_loss": 3.49934410739289, + "tokens_seen": 1740767232 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038189769307923774, + "loss": 1.2428, + "theoretical_loss": 3.499332912908346, + "tokens_seen": 1740832768 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038189769307923774, + "loss": 1.169, + "theoretical_loss": 3.4993217189632224, + "tokens_seen": 1740898304 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038188164493480444, + "loss": 1.2514, + "theoretical_loss": 3.499310525557471, + "tokens_seen": 1740963840 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038186559679037114, + "loss": 1.2729, + "theoretical_loss": 3.499299332691047, + "tokens_seen": 1741029376 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038184954864593784, + "loss": 1.1428, + "theoretical_loss": 3.4992881403639027, + "tokens_seen": 1741094912 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038183350050150454, + "loss": 1.2252, + "theoretical_loss": 3.499276948575993, + "tokens_seen": 1741160448 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038181745235707124, + "loss": 1.1769, + "theoretical_loss": 3.499265757327271, + "tokens_seen": 1741225984 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038180140421263794, + "loss": 1.2602, + "theoretical_loss": 3.499254566617691, + "tokens_seen": 1741291520 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038178535606820464, + "loss": 1.2667, + "theoretical_loss": 3.4992433764472057, + "tokens_seen": 1741357056 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038176930792377134, + "loss": 1.4109, + "theoretical_loss": 3.49923218681577, + "tokens_seen": 1741422592 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038175325977933804, + "loss": 1.2601, + "theoretical_loss": 3.499220997723337, + "tokens_seen": 1741488128 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038173721163490474, + "loss": 1.3341, + "theoretical_loss": 3.4992098091698605, + "tokens_seen": 1741553664 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038172116349047145, + "loss": 1.3085, + "theoretical_loss": 3.499198621155294, + "tokens_seen": 1741619200 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038170511534603815, + "loss": 1.2852, + "theoretical_loss": 3.4991874336795923, + "tokens_seen": 1741684736 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038168906720160485, + "loss": 1.2356, + "theoretical_loss": 3.4991762467427083, + "tokens_seen": 1741750272 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038167301905717155, + "loss": 1.287, + "theoretical_loss": 3.4991650603445956, + "tokens_seen": 1741815808 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003816569709127382, + "loss": 1.3658, + "theoretical_loss": 3.4991538744852084, + "tokens_seen": 1741881344 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003816409227683049, + "loss": 1.3557, + "theoretical_loss": 3.499142689164501, + "tokens_seen": 1741946880 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003816248746238716, + "loss": 1.282, + "theoretical_loss": 3.499131504382426, + "tokens_seen": 1742012416 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003816088264794383, + "loss": 1.1449, + "theoretical_loss": 3.499120320138938, + "tokens_seen": 1742077952 + }, + { + "epoch": 0.53, + "learning_rate": 0.000381592778335005, + "loss": 1.2108, + "theoretical_loss": 3.499109136433991, + "tokens_seen": 1742143488 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003815767301905717, + "loss": 1.2244, + "theoretical_loss": 3.499097953267538, + "tokens_seen": 1742209024 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003815606820461384, + "loss": 1.1925, + "theoretical_loss": 3.4990867706395337, + "tokens_seen": 1742274560 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003815446339017051, + "loss": 1.306, + "theoretical_loss": 3.499075588549931, + "tokens_seen": 1742340096 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003815285857572718, + "loss": 1.1186, + "theoretical_loss": 3.4990644069986843, + "tokens_seen": 1742405632 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003815125376128385, + "loss": 1.2629, + "theoretical_loss": 3.4990532259857474, + "tokens_seen": 1742471168 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003814964894684052, + "loss": 1.229, + "theoretical_loss": 3.4990420455110742, + "tokens_seen": 1742536704 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003814804413239719, + "loss": 1.1506, + "theoretical_loss": 3.4990308655746185, + "tokens_seen": 1742602240 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003814643931795386, + "loss": 1.2068, + "theoretical_loss": 3.4990196861763336, + "tokens_seen": 1742667776 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003814483450351053, + "loss": 1.2987, + "theoretical_loss": 3.499008507316174, + "tokens_seen": 1742733312 + }, + { + "epoch": 0.53, + "learning_rate": 0.000381432296890672, + "loss": 1.2061, + "theoretical_loss": 3.4989973289940934, + "tokens_seen": 1742798848 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003814162487462387, + "loss": 1.1686, + "theoretical_loss": 3.4989861512100457, + "tokens_seen": 1742864384 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003814002006018054, + "loss": 1.3715, + "theoretical_loss": 3.498974973963985, + "tokens_seen": 1742929920 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003813841524573721, + "loss": 1.2624, + "theoretical_loss": 3.4989637972558647, + "tokens_seen": 1742995456 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038136810431293886, + "loss": 1.229, + "theoretical_loss": 3.498952621085639, + "tokens_seen": 1743060992 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038135205616850556, + "loss": 1.2645, + "theoretical_loss": 3.498941445453262, + "tokens_seen": 1743126528 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038133600802407226, + "loss": 1.272, + "theoretical_loss": 3.4989302703586866, + "tokens_seen": 1743192064 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 584079, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4145455360412598, + "objective/train/theoretical_loss": 3.4989190958018677, + "objective/train/tokens_used": 1763717600, + "theoretical_loss": 3.4989190958018677, + "tokens_seen": 1743257600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038131995987963896, + "loss": 1.3625, + "theoretical_loss": 3.4989190958018677, + "tokens_seen": 1743257600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038130391173520566, + "loss": 1.1762, + "theoretical_loss": 3.498907921782759, + "tokens_seen": 1743323136 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038128786359077236, + "loss": 1.3146, + "theoretical_loss": 3.4988967483013145, + "tokens_seen": 1743388672 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038127181544633906, + "loss": 1.3495, + "theoretical_loss": 3.4988855753574875, + "tokens_seen": 1743454208 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038125576730190576, + "loss": 1.3109, + "theoretical_loss": 3.498874402951232, + "tokens_seen": 1743519744 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038123971915747246, + "loss": 1.3115, + "theoretical_loss": 3.4988632310825034, + "tokens_seen": 1743585280 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038122367101303917, + "loss": 1.3352, + "theoretical_loss": 3.498852059751254, + "tokens_seen": 1743650816 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038120762286860587, + "loss": 1.3012, + "theoretical_loss": 3.498840888957438, + "tokens_seen": 1743716352 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038119157472417257, + "loss": 1.3218, + "theoretical_loss": 3.4988297187010096, + "tokens_seen": 1743781888 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038117552657973927, + "loss": 1.4101, + "theoretical_loss": 3.498818548981923, + "tokens_seen": 1743847424 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038115947843530597, + "loss": 1.2624, + "theoretical_loss": 3.498807379800132, + "tokens_seen": 1743912960 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038114343029087267, + "loss": 1.2683, + "theoretical_loss": 3.4987962111555904, + "tokens_seen": 1743978496 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038112738214643937, + "loss": 1.3151, + "theoretical_loss": 3.498785043048252, + "tokens_seen": 1744044032 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038111133400200607, + "loss": 1.3446, + "theoretical_loss": 3.4987738754780713, + "tokens_seen": 1744109568 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038109528585757277, + "loss": 1.3277, + "theoretical_loss": 3.498762708445002, + "tokens_seen": 1744175104 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038107923771313947, + "loss": 1.2807, + "theoretical_loss": 3.498751541948998, + "tokens_seen": 1744240640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038106318956870617, + "loss": 1.2471, + "theoretical_loss": 3.4987403759900135, + "tokens_seen": 1744306176 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038104714142427287, + "loss": 1.253, + "theoretical_loss": 3.498729210568002, + "tokens_seen": 1744371712 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003810310932798396, + "loss": 1.1891, + "theoretical_loss": 3.4987180456829186, + "tokens_seen": 1744437248 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003810150451354062, + "loss": 1.2817, + "theoretical_loss": 3.498706881334716, + "tokens_seen": 1744502784 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003809989969909729, + "loss": 1.2293, + "theoretical_loss": 3.4986957175233493, + "tokens_seen": 1744568320 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003809829488465396, + "loss": 1.1663, + "theoretical_loss": 3.498684554248772, + "tokens_seen": 1744633856 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003809669007021063, + "loss": 1.2083, + "theoretical_loss": 3.4986733915109376, + "tokens_seen": 1744699392 + }, + { + "epoch": 0.53, + "learning_rate": 0.000380950852557673, + "loss": 1.2162, + "theoretical_loss": 3.4986622293098013, + "tokens_seen": 1744764928 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003809348044132397, + "loss": 1.2886, + "theoretical_loss": 3.498651067645316, + "tokens_seen": 1744830464 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003809187562688064, + "loss": 1.2427, + "theoretical_loss": 3.498639906517437, + "tokens_seen": 1744896000 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003809027081243731, + "loss": 1.2386, + "theoretical_loss": 3.4986287459261174, + "tokens_seen": 1744961536 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003808866599799398, + "loss": 1.2453, + "theoretical_loss": 3.498617585871311, + "tokens_seen": 1745027072 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003808706118355065, + "loss": 1.1829, + "theoretical_loss": 3.498606426352973, + "tokens_seen": 1745092608 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003808545636910732, + "loss": 1.2011, + "theoretical_loss": 3.4985952673710567, + "tokens_seen": 1745158144 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003808385155466399, + "loss": 1.3406, + "theoretical_loss": 3.498584108925516, + "tokens_seen": 1745223680 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003808224674022066, + "loss": 1.1948, + "theoretical_loss": 3.498572951016305, + "tokens_seen": 1745289216 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038080641925777333, + "loss": 1.2913, + "theoretical_loss": 3.498561793643379, + "tokens_seen": 1745354752 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038079037111334003, + "loss": 1.243, + "theoretical_loss": 3.4985506368066903, + "tokens_seen": 1745420288 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038077432296890673, + "loss": 1.2639, + "theoretical_loss": 3.498539480506194, + "tokens_seen": 1745485824 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038075827482447343, + "loss": 1.2929, + "theoretical_loss": 3.4985283247418444, + "tokens_seen": 1745551360 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038074222668004013, + "loss": 1.2458, + "theoretical_loss": 3.498517169513595, + "tokens_seen": 1745616896 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038072617853560683, + "loss": 1.2286, + "theoretical_loss": 3.4985060148214, + "tokens_seen": 1745682432 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038071013039117353, + "loss": 1.1307, + "theoretical_loss": 3.4984948606652138, + "tokens_seen": 1745747968 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038069408224674023, + "loss": 1.2929, + "theoretical_loss": 3.4984837070449903, + "tokens_seen": 1745813504 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038067803410230693, + "loss": 1.2665, + "theoretical_loss": 3.498472553960684, + "tokens_seen": 1745879040 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038066198595787363, + "loss": 1.2547, + "theoretical_loss": 3.498461401412248, + "tokens_seen": 1745944576 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038064593781344033, + "loss": 1.2641, + "theoretical_loss": 3.4984502493996383, + "tokens_seen": 1746010112 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038062988966900703, + "loss": 1.212, + "theoretical_loss": 3.498439097922807, + "tokens_seen": 1746075648 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038061384152457374, + "loss": 1.2216, + "theoretical_loss": 3.49842794698171, + "tokens_seen": 1746141184 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038059779338014044, + "loss": 1.2021, + "theoretical_loss": 3.4984167965763002, + "tokens_seen": 1746206720 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038058174523570714, + "loss": 1.2376, + "theoretical_loss": 3.4984056467065323, + "tokens_seen": 1746272256 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038056569709127384, + "loss": 1.2614, + "theoretical_loss": 3.4983944973723604, + "tokens_seen": 1746337792 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038054964894684054, + "loss": 1.2343, + "theoretical_loss": 3.4983833485737383, + "tokens_seen": 1746403328 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038053360080240724, + "loss": 1.2116, + "theoretical_loss": 3.4983722003106212, + "tokens_seen": 1746468864 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 585249, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2488512992858887, + "objective/train/theoretical_loss": 3.498361052582962, + "objective/train/tokens_used": 1766994400, + "theoretical_loss": 3.498361052582962, + "tokens_seen": 1746534400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038051755265797394, + "loss": 1.2671, + "theoretical_loss": 3.498361052582962, + "tokens_seen": 1746534400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038050150451354064, + "loss": 1.2579, + "theoretical_loss": 3.498349905390716, + "tokens_seen": 1746599936 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038048545636910734, + "loss": 1.3079, + "theoretical_loss": 3.4983387587338366, + "tokens_seen": 1746665472 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038046940822467404, + "loss": 1.3475, + "theoretical_loss": 3.4983276126122784, + "tokens_seen": 1746731008 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038045336008024074, + "loss": 1.1422, + "theoretical_loss": 3.4983164670259956, + "tokens_seen": 1746796544 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038043731193580744, + "loss": 1.3277, + "theoretical_loss": 3.498305321974942, + "tokens_seen": 1746862080 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038042126379137414, + "loss": 1.3043, + "theoretical_loss": 3.498294177459073, + "tokens_seen": 1746927616 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038040521564694084, + "loss": 1.1626, + "theoretical_loss": 3.498283033478341, + "tokens_seen": 1746993152 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038038916750250754, + "loss": 1.2902, + "theoretical_loss": 3.4982718900327017, + "tokens_seen": 1747058688 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038037311935807424, + "loss": 1.2238, + "theoretical_loss": 3.4982607471221083, + "tokens_seen": 1747124224 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038035707121364095, + "loss": 1.3565, + "theoretical_loss": 3.4982496047465164, + "tokens_seen": 1747189760 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038034102306920765, + "loss": 1.1619, + "theoretical_loss": 3.4982384629058787, + "tokens_seen": 1747255296 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038032497492477435, + "loss": 1.2325, + "theoretical_loss": 3.498227321600151, + "tokens_seen": 1747320832 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038030892678034105, + "loss": 1.3565, + "theoretical_loss": 3.498216180829286, + "tokens_seen": 1747386368 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038029287863590775, + "loss": 1.1512, + "theoretical_loss": 3.498205040593239, + "tokens_seen": 1747451904 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038027683049147445, + "loss": 1.2248, + "theoretical_loss": 3.498193900891964, + "tokens_seen": 1747517440 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038026078234704115, + "loss": 1.2392, + "theoretical_loss": 3.498182761725415, + "tokens_seen": 1747582976 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003802447342026078, + "loss": 1.368, + "theoretical_loss": 3.4981716230935467, + "tokens_seen": 1747648512 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003802286860581745, + "loss": 1.2008, + "theoretical_loss": 3.4981604849963133, + "tokens_seen": 1747714048 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003802126379137412, + "loss": 1.3921, + "theoretical_loss": 3.498149347433669, + "tokens_seen": 1747779584 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003801965897693079, + "loss": 1.3668, + "theoretical_loss": 3.498138210405568, + "tokens_seen": 1747845120 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003801805416248746, + "loss": 1.2624, + "theoretical_loss": 3.4981270739119648, + "tokens_seen": 1747910656 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038016449348044135, + "loss": 1.1999, + "theoretical_loss": 3.4981159379528135, + "tokens_seen": 1747976192 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038014844533600805, + "loss": 1.2464, + "theoretical_loss": 3.4981048025280685, + "tokens_seen": 1748041728 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038013239719157475, + "loss": 1.2299, + "theoretical_loss": 3.4980936676376846, + "tokens_seen": 1748107264 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038011634904714145, + "loss": 1.3647, + "theoretical_loss": 3.498082533281615, + "tokens_seen": 1748172800 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038010030090270816, + "loss": 1.2543, + "theoretical_loss": 3.498071399459815, + "tokens_seen": 1748238336 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038008425275827486, + "loss": 1.2363, + "theoretical_loss": 3.4980602661722386, + "tokens_seen": 1748303872 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038006820461384156, + "loss": 1.3479, + "theoretical_loss": 3.4980491334188404, + "tokens_seen": 1748369408 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038005215646940826, + "loss": 1.2316, + "theoretical_loss": 3.498038001199574, + "tokens_seen": 1748434944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038003610832497496, + "loss": 1.3637, + "theoretical_loss": 3.498026869514395, + "tokens_seen": 1748500480 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038002006018054166, + "loss": 1.2802, + "theoretical_loss": 3.4980157383632564, + "tokens_seen": 1748566016 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038000401203610836, + "loss": 1.1717, + "theoretical_loss": 3.4980046077461138, + "tokens_seen": 1748631552 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037998796389167506, + "loss": 1.1947, + "theoretical_loss": 3.497993477662921, + "tokens_seen": 1748697088 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037997191574724176, + "loss": 1.2212, + "theoretical_loss": 3.497982348113632, + "tokens_seen": 1748762624 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037995586760280846, + "loss": 1.2998, + "theoretical_loss": 3.4979712190982015, + "tokens_seen": 1748828160 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037993981945837516, + "loss": 1.265, + "theoretical_loss": 3.4979600906165844, + "tokens_seen": 1748893696 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037992377131394186, + "loss": 1.2805, + "theoretical_loss": 3.497948962668734, + "tokens_seen": 1748959232 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037990772316950856, + "loss": 1.2182, + "theoretical_loss": 3.4979378352546058, + "tokens_seen": 1749024768 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037989167502507526, + "loss": 1.2204, + "theoretical_loss": 3.4979267083741536, + "tokens_seen": 1749090304 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037987562688064196, + "loss": 1.3641, + "theoretical_loss": 3.497915582027332, + "tokens_seen": 1749155840 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037985957873620867, + "loss": 1.3524, + "theoretical_loss": 3.497904456214095, + "tokens_seen": 1749221376 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037984353059177537, + "loss": 1.178, + "theoretical_loss": 3.4978933309343976, + "tokens_seen": 1749286912 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037982748244734207, + "loss": 1.1753, + "theoretical_loss": 3.4978822061881942, + "tokens_seen": 1749352448 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037981143430290877, + "loss": 1.2949, + "theoretical_loss": 3.497871081975439, + "tokens_seen": 1749417984 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037979538615847547, + "loss": 1.216, + "theoretical_loss": 3.497859958296086, + "tokens_seen": 1749483520 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037977933801404217, + "loss": 1.2135, + "theoretical_loss": 3.4978488351500907, + "tokens_seen": 1749549056 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037976328986960887, + "loss": 1.3318, + "theoretical_loss": 3.4978377125374065, + "tokens_seen": 1749614592 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037974724172517557, + "loss": 1.3575, + "theoretical_loss": 3.497826590457989, + "tokens_seen": 1749680128 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037973119358074227, + "loss": 1.2399, + "theoretical_loss": 3.4978154689117913, + "tokens_seen": 1749745664 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 586265, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2173155546188354, + "objective/train/theoretical_loss": 3.497804347898769, + "objective/train/tokens_used": 1770271200, + "theoretical_loss": 3.497804347898769, + "tokens_seen": 1749811200 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037971514543630897, + "loss": 1.2606, + "theoretical_loss": 3.497804347898769, + "tokens_seen": 1749811200 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037969909729187567, + "loss": 1.2775, + "theoretical_loss": 3.4977932274188754, + "tokens_seen": 1749876736 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037968304914744237, + "loss": 1.2137, + "theoretical_loss": 3.4977821074720663, + "tokens_seen": 1749942272 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037966700100300907, + "loss": 1.2222, + "theoretical_loss": 3.4977709880582957, + "tokens_seen": 1750007808 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003796509528585758, + "loss": 1.3053, + "theoretical_loss": 3.497759869177518, + "tokens_seen": 1750073344 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003796349047141425, + "loss": 1.2095, + "theoretical_loss": 3.497748750829687, + "tokens_seen": 1750138880 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003796188565697091, + "loss": 1.2132, + "theoretical_loss": 3.4977376330147583, + "tokens_seen": 1750204416 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003796028084252758, + "loss": 1.3074, + "theoretical_loss": 3.497726515732686, + "tokens_seen": 1750269952 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003795867602808425, + "loss": 1.2981, + "theoretical_loss": 3.4977153989834244, + "tokens_seen": 1750335488 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003795707121364092, + "loss": 1.2309, + "theoretical_loss": 3.4977042827669282, + "tokens_seen": 1750401024 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003795546639919759, + "loss": 1.2332, + "theoretical_loss": 3.497693167083152, + "tokens_seen": 1750466560 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003795386158475426, + "loss": 1.1881, + "theoretical_loss": 3.4976820519320504, + "tokens_seen": 1750532096 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003795225677031093, + "loss": 1.2304, + "theoretical_loss": 3.4976709373135773, + "tokens_seen": 1750597632 + }, + { + "epoch": 0.53, + "learning_rate": 0.000379506519558676, + "loss": 1.2441, + "theoretical_loss": 3.4976598232276883, + "tokens_seen": 1750663168 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003794904714142427, + "loss": 1.3871, + "theoretical_loss": 3.497648709674337, + "tokens_seen": 1750728704 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003794744232698094, + "loss": 1.1394, + "theoretical_loss": 3.4976375966534787, + "tokens_seen": 1750794240 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003794583751253761, + "loss": 1.2654, + "theoretical_loss": 3.497626484165067, + "tokens_seen": 1750859776 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037944232698094283, + "loss": 1.3217, + "theoretical_loss": 3.497615372209057, + "tokens_seen": 1750925312 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037942627883650953, + "loss": 1.2868, + "theoretical_loss": 3.497604260785404, + "tokens_seen": 1750990848 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037941023069207623, + "loss": 1.3097, + "theoretical_loss": 3.4975931498940613, + "tokens_seen": 1751056384 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037939418254764293, + "loss": 1.2723, + "theoretical_loss": 3.497582039534984, + "tokens_seen": 1751121920 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037937813440320963, + "loss": 1.2866, + "theoretical_loss": 3.497570929708127, + "tokens_seen": 1751187456 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037936208625877633, + "loss": 1.3044, + "theoretical_loss": 3.4975598204134446, + "tokens_seen": 1751252992 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037934603811434303, + "loss": 1.2763, + "theoretical_loss": 3.497548711650891, + "tokens_seen": 1751318528 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037932998996990973, + "loss": 1.2528, + "theoretical_loss": 3.4975376034204215, + "tokens_seen": 1751384064 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037931394182547643, + "loss": 1.3591, + "theoretical_loss": 3.4975264957219907, + "tokens_seen": 1751449600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037929789368104313, + "loss": 1.3523, + "theoretical_loss": 3.497515388555553, + "tokens_seen": 1751515136 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037928184553660983, + "loss": 1.2446, + "theoretical_loss": 3.4975042819210627, + "tokens_seen": 1751580672 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037926579739217653, + "loss": 1.2758, + "theoretical_loss": 3.497493175818474, + "tokens_seen": 1751646208 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037924974924774323, + "loss": 1.4203, + "theoretical_loss": 3.497482070247743, + "tokens_seen": 1751711744 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037923370110330994, + "loss": 1.2685, + "theoretical_loss": 3.497470965208823, + "tokens_seen": 1751777280 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037921765295887664, + "loss": 1.2143, + "theoretical_loss": 3.4974598607016696, + "tokens_seen": 1751842816 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037920160481444334, + "loss": 1.3125, + "theoretical_loss": 3.4974487567262367, + "tokens_seen": 1751908352 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037918555667001004, + "loss": 1.2584, + "theoretical_loss": 3.4974376532824794, + "tokens_seen": 1751973888 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037916950852557674, + "loss": 1.225, + "theoretical_loss": 3.4974265503703523, + "tokens_seen": 1752039424 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037915346038114344, + "loss": 1.2427, + "theoretical_loss": 3.49741544798981, + "tokens_seen": 1752104960 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037913741223671014, + "loss": 1.3059, + "theoretical_loss": 3.497404346140807, + "tokens_seen": 1752170496 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037912136409227684, + "loss": 1.2717, + "theoretical_loss": 3.4973932448232983, + "tokens_seen": 1752236032 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037910531594784354, + "loss": 1.3706, + "theoretical_loss": 3.497382144037238, + "tokens_seen": 1752301568 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037908926780341024, + "loss": 1.2548, + "theoretical_loss": 3.4973710437825813, + "tokens_seen": 1752367104 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037907321965897694, + "loss": 1.3386, + "theoretical_loss": 3.4973599440592826, + "tokens_seen": 1752432640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037905717151454364, + "loss": 1.2317, + "theoretical_loss": 3.497348844867297, + "tokens_seen": 1752498176 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037904112337011034, + "loss": 1.344, + "theoretical_loss": 3.497337746206579, + "tokens_seen": 1752563712 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037902507522567704, + "loss": 1.2787, + "theoretical_loss": 3.4973266480770837, + "tokens_seen": 1752629248 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037900902708124374, + "loss": 1.2631, + "theoretical_loss": 3.4973155504787643, + "tokens_seen": 1752694784 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037899297893681045, + "loss": 1.3797, + "theoretical_loss": 3.4973044534115774, + "tokens_seen": 1752760320 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037897693079237715, + "loss": 1.2981, + "theoretical_loss": 3.4972933568754767, + "tokens_seen": 1752825856 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037896088264794385, + "loss": 1.2052, + "theoretical_loss": 3.497282260870417, + "tokens_seen": 1752891392 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037894483450351055, + "loss": 1.2827, + "theoretical_loss": 3.4972711653963535, + "tokens_seen": 1752956928 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037892878635907725, + "loss": 1.2338, + "theoretical_loss": 3.4972600704532404, + "tokens_seen": 1753022464 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 587536, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2830181121826172, + "objective/train/theoretical_loss": 3.497248976041033, + "objective/train/tokens_used": 1773548000, + "theoretical_loss": 3.497248976041033, + "tokens_seen": 1753088000 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037891273821464395, + "loss": 1.3373, + "theoretical_loss": 3.497248976041033, + "tokens_seen": 1753088000 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037889669007021065, + "loss": 1.3406, + "theoretical_loss": 3.497237882159685, + "tokens_seen": 1753153536 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037888064192577735, + "loss": 1.3413, + "theoretical_loss": 3.4972267888091526, + "tokens_seen": 1753219072 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037886459378134405, + "loss": 1.3309, + "theoretical_loss": 3.4972156959893894, + "tokens_seen": 1753284608 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037884854563691075, + "loss": 1.3091, + "theoretical_loss": 3.49720460370035, + "tokens_seen": 1753350144 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037883249749247745, + "loss": 1.3314, + "theoretical_loss": 3.497193511941991, + "tokens_seen": 1753415680 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037881644934804415, + "loss": 1.3395, + "theoretical_loss": 3.4971824207142657, + "tokens_seen": 1753481216 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037880040120361085, + "loss": 1.3523, + "theoretical_loss": 3.497171330017128, + "tokens_seen": 1753546752 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037878435305917755, + "loss": 1.338, + "theoretical_loss": 3.497160239850535, + "tokens_seen": 1753612288 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037876830491474425, + "loss": 1.2973, + "theoretical_loss": 3.49714915021444, + "tokens_seen": 1753677824 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037875225677031095, + "loss": 1.2021, + "theoretical_loss": 3.4971380611087977, + "tokens_seen": 1753743360 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037873620862587766, + "loss": 1.2977, + "theoretical_loss": 3.4971269725335636, + "tokens_seen": 1753808896 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037872016048144436, + "loss": 1.3104, + "theoretical_loss": 3.4971158844886925, + "tokens_seen": 1753874432 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037870411233701106, + "loss": 1.3159, + "theoretical_loss": 3.4971047969741385, + "tokens_seen": 1753939968 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037868806419257776, + "loss": 1.3875, + "theoretical_loss": 3.4970937099898576, + "tokens_seen": 1754005504 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037867201604814446, + "loss": 1.3549, + "theoretical_loss": 3.4970826235358032, + "tokens_seen": 1754071040 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037865596790371116, + "loss": 1.2536, + "theoretical_loss": 3.497071537611931, + "tokens_seen": 1754136576 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037863991975927786, + "loss": 1.195, + "theoretical_loss": 3.4970604522181956, + "tokens_seen": 1754202112 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037862387161484456, + "loss": 1.2748, + "theoretical_loss": 3.4970493673545517, + "tokens_seen": 1754267648 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037860782347041126, + "loss": 1.234, + "theoretical_loss": 3.497038283020955, + "tokens_seen": 1754333184 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037859177532597796, + "loss": 1.2261, + "theoretical_loss": 3.497027199217359, + "tokens_seen": 1754398720 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037857572718154466, + "loss": 1.19, + "theoretical_loss": 3.49701611594372, + "tokens_seen": 1754464256 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037855967903711136, + "loss": 1.2683, + "theoretical_loss": 3.497005033199992, + "tokens_seen": 1754529792 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037854363089267806, + "loss": 1.3002, + "theoretical_loss": 3.496993950986129, + "tokens_seen": 1754595328 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037852758274824476, + "loss": 1.3702, + "theoretical_loss": 3.496982869302088, + "tokens_seen": 1754660864 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037851153460381146, + "loss": 1.3319, + "theoretical_loss": 3.496971788147822, + "tokens_seen": 1754726400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037849548645937816, + "loss": 1.3352, + "theoretical_loss": 3.4969607075232876, + "tokens_seen": 1754791936 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037847943831494487, + "loss": 1.2652, + "theoretical_loss": 3.496949627428438, + "tokens_seen": 1754857472 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037846339017051157, + "loss": 1.3375, + "theoretical_loss": 3.4969385478632287, + "tokens_seen": 1754923008 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037844734202607827, + "loss": 1.3323, + "theoretical_loss": 3.4969274688276153, + "tokens_seen": 1754988544 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037843129388164497, + "loss": 1.2569, + "theoretical_loss": 3.496916390321552, + "tokens_seen": 1755054080 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037841524573721167, + "loss": 1.3081, + "theoretical_loss": 3.4969053123449942, + "tokens_seen": 1755119616 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037839919759277837, + "loss": 1.322, + "theoretical_loss": 3.496894234897896, + "tokens_seen": 1755185152 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037838314944834507, + "loss": 1.292, + "theoretical_loss": 3.4968831579802133, + "tokens_seen": 1755250688 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037836710130391177, + "loss": 1.3255, + "theoretical_loss": 3.4968720815919, + "tokens_seen": 1755316224 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037835105315947847, + "loss": 1.2901, + "theoretical_loss": 3.496861005732912, + "tokens_seen": 1755381760 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037833500501504517, + "loss": 1.3479, + "theoretical_loss": 3.4968499304032035, + "tokens_seen": 1755447296 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037831895687061187, + "loss": 1.118, + "theoretical_loss": 3.4968388556027303, + "tokens_seen": 1755512832 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037830290872617857, + "loss": 1.3388, + "theoretical_loss": 3.4968277813314463, + "tokens_seen": 1755578368 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003782868605817453, + "loss": 1.2706, + "theoretical_loss": 3.4968167075893075, + "tokens_seen": 1755643904 + }, + { + "epoch": 0.53, + "learning_rate": 0.000378270812437312, + "loss": 1.2414, + "theoretical_loss": 3.496805634376268, + "tokens_seen": 1755709440 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003782547642928787, + "loss": 1.4021, + "theoretical_loss": 3.4967945616922838, + "tokens_seen": 1755774976 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003782387161484454, + "loss": 1.3017, + "theoretical_loss": 3.4967834895373087, + "tokens_seen": 1755840512 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003782226680040121, + "loss": 1.2743, + "theoretical_loss": 3.4967724179112984, + "tokens_seen": 1755906048 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003782066198595787, + "loss": 1.2816, + "theoretical_loss": 3.4967613468142074, + "tokens_seen": 1755971584 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003781905717151454, + "loss": 1.2562, + "theoretical_loss": 3.496750276245991, + "tokens_seen": 1756037120 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003781745235707121, + "loss": 1.3066, + "theoretical_loss": 3.4967392062066045, + "tokens_seen": 1756102656 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003781584754262788, + "loss": 1.2172, + "theoretical_loss": 3.4967281366960026, + "tokens_seen": 1756168192 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003781424272818455, + "loss": 1.2646, + "theoretical_loss": 3.4967170677141404, + "tokens_seen": 1756233728 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003781263791374122, + "loss": 1.1734, + "theoretical_loss": 3.496705999260972, + "tokens_seen": 1756299264 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 588499, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2977731227874756, + "objective/train/theoretical_loss": 3.496694931336454, + "objective/train/tokens_used": 1776824800, + "theoretical_loss": 3.496694931336454, + "tokens_seen": 1756364800 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003781103309929789, + "loss": 1.3492, + "theoretical_loss": 3.496694931336454, + "tokens_seen": 1756364800 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003780942828485456, + "loss": 1.2438, + "theoretical_loss": 3.4966838639405404, + "tokens_seen": 1756430336 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003780782347041123, + "loss": 1.3504, + "theoretical_loss": 3.496672797073187, + "tokens_seen": 1756495872 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037806218655967903, + "loss": 1.2986, + "theoretical_loss": 3.4966617307343477, + "tokens_seen": 1756561408 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037804613841524573, + "loss": 1.2992, + "theoretical_loss": 3.4966506649239784, + "tokens_seen": 1756626944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037803009027081243, + "loss": 1.2713, + "theoretical_loss": 3.496639599642034, + "tokens_seen": 1756692480 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037801404212637913, + "loss": 1.3161, + "theoretical_loss": 3.4966285348884694, + "tokens_seen": 1756758016 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037799799398194583, + "loss": 1.3771, + "theoretical_loss": 3.4966174706632396, + "tokens_seen": 1756823552 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037798194583751253, + "loss": 1.2327, + "theoretical_loss": 3.4966064069663, + "tokens_seen": 1756889088 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037796589769307923, + "loss": 1.2138, + "theoretical_loss": 3.4965953437976056, + "tokens_seen": 1756954624 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037794984954864593, + "loss": 1.2982, + "theoretical_loss": 3.4965842811571113, + "tokens_seen": 1757020160 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037793380140421263, + "loss": 1.2421, + "theoretical_loss": 3.4965732190447723, + "tokens_seen": 1757085696 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037791775325977933, + "loss": 1.159, + "theoretical_loss": 3.496562157460543, + "tokens_seen": 1757151232 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037790170511534603, + "loss": 1.3358, + "theoretical_loss": 3.4965510964043798, + "tokens_seen": 1757216768 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037788565697091273, + "loss": 1.2656, + "theoretical_loss": 3.4965400358762366, + "tokens_seen": 1757282304 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037786960882647944, + "loss": 1.3325, + "theoretical_loss": 3.4965289758760694, + "tokens_seen": 1757347840 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037785356068204614, + "loss": 1.3204, + "theoretical_loss": 3.4965179164038327, + "tokens_seen": 1757413376 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037783751253761284, + "loss": 1.2431, + "theoretical_loss": 3.496506857459482, + "tokens_seen": 1757478912 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037782146439317954, + "loss": 1.2312, + "theoretical_loss": 3.4964957990429726, + "tokens_seen": 1757544448 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037780541624874624, + "loss": 1.3462, + "theoretical_loss": 3.4964847411542586, + "tokens_seen": 1757609984 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037778936810431294, + "loss": 1.2712, + "theoretical_loss": 3.496473683793296, + "tokens_seen": 1757675520 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003777733199598797, + "loss": 1.3506, + "theoretical_loss": 3.49646262696004, + "tokens_seen": 1757741056 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003777572718154464, + "loss": 1.3224, + "theoretical_loss": 3.4964515706544455, + "tokens_seen": 1757806592 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003777412236710131, + "loss": 1.2802, + "theoretical_loss": 3.4964405148764675, + "tokens_seen": 1757872128 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003777251755265798, + "loss": 1.2428, + "theoretical_loss": 3.4964294596260608, + "tokens_seen": 1757937664 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003777091273821465, + "loss": 1.3147, + "theoretical_loss": 3.496418404903182, + "tokens_seen": 1758003200 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003776930792377132, + "loss": 1.1837, + "theoretical_loss": 3.496407350707785, + "tokens_seen": 1758068736 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003776770310932799, + "loss": 1.2977, + "theoretical_loss": 3.4963962970398246, + "tokens_seen": 1758134272 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003776609829488466, + "loss": 1.3655, + "theoretical_loss": 3.4963852438992573, + "tokens_seen": 1758199808 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003776449348044133, + "loss": 1.169, + "theoretical_loss": 3.4963741912860375, + "tokens_seen": 1758265344 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037762888665998, + "loss": 1.1813, + "theoretical_loss": 3.4963631392001204, + "tokens_seen": 1758330880 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003776128385155467, + "loss": 1.2342, + "theoretical_loss": 3.496352087641462, + "tokens_seen": 1758396416 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003775967903711134, + "loss": 1.3225, + "theoretical_loss": 3.496341036610016, + "tokens_seen": 1758461952 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037758074222668005, + "loss": 1.3467, + "theoretical_loss": 3.4963299861057386, + "tokens_seen": 1758527488 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037756469408224675, + "loss": 1.2901, + "theoretical_loss": 3.496318936128585, + "tokens_seen": 1758593024 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037754864593781345, + "loss": 1.2335, + "theoretical_loss": 3.49630788667851, + "tokens_seen": 1758658560 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037753259779338015, + "loss": 1.3257, + "theoretical_loss": 3.4962968377554695, + "tokens_seen": 1758724096 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037751654964894685, + "loss": 1.2647, + "theoretical_loss": 3.4962857893594177, + "tokens_seen": 1758789632 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037750050150451355, + "loss": 1.3047, + "theoretical_loss": 3.4962747414903106, + "tokens_seen": 1758855168 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037748445336008025, + "loss": 1.3836, + "theoretical_loss": 3.4962636941481033, + "tokens_seen": 1758920704 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037746840521564695, + "loss": 1.3101, + "theoretical_loss": 3.496252647332751, + "tokens_seen": 1758986240 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037745235707121365, + "loss": 1.257, + "theoretical_loss": 3.496241601044209, + "tokens_seen": 1759051776 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037743630892678035, + "loss": 1.2424, + "theoretical_loss": 3.496230555282432, + "tokens_seen": 1759117312 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037742026078234705, + "loss": 1.2928, + "theoretical_loss": 3.4962195100473767, + "tokens_seen": 1759182848 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037740421263791375, + "loss": 1.333, + "theoretical_loss": 3.496208465338997, + "tokens_seen": 1759248384 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037738816449348045, + "loss": 1.1944, + "theoretical_loss": 3.496197421157248, + "tokens_seen": 1759313920 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037737211634904715, + "loss": 1.2373, + "theoretical_loss": 3.4961863775020863, + "tokens_seen": 1759379456 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037735606820461386, + "loss": 1.3181, + "theoretical_loss": 3.496175334373466, + "tokens_seen": 1759444992 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037734002006018056, + "loss": 1.2687, + "theoretical_loss": 3.4961642917713425, + "tokens_seen": 1759510528 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037732397191574726, + "loss": 1.2005, + "theoretical_loss": 3.4961532496956718, + "tokens_seen": 1759576064 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 589734, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2679030895233154, + "objective/train/theoretical_loss": 3.496142208146409, + "objective/train/tokens_used": 1780101600, + "theoretical_loss": 3.496142208146409, + "tokens_seen": 1759641600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037730792377131396, + "loss": 1.3017, + "theoretical_loss": 3.496142208146409, + "tokens_seen": 1759641600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037729187562688066, + "loss": 1.1657, + "theoretical_loss": 3.496131167123509, + "tokens_seen": 1759707136 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037727582748244736, + "loss": 1.2008, + "theoretical_loss": 3.496120126626927, + "tokens_seen": 1759772672 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037725977933801406, + "loss": 1.1927, + "theoretical_loss": 3.4961090866566185, + "tokens_seen": 1759838208 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037724373119358076, + "loss": 1.4921, + "theoretical_loss": 3.4960980472125396, + "tokens_seen": 1759903744 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037722768304914746, + "loss": 1.3269, + "theoretical_loss": 3.4960870082946442, + "tokens_seen": 1759969280 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037721163490471416, + "loss": 1.2817, + "theoretical_loss": 3.496075969902889, + "tokens_seen": 1760034816 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037719558676028086, + "loss": 1.1871, + "theoretical_loss": 3.4960649320372283, + "tokens_seen": 1760100352 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037717953861584756, + "loss": 1.3425, + "theoretical_loss": 3.496053894697618, + "tokens_seen": 1760165888 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037716349047141426, + "loss": 1.224, + "theoretical_loss": 3.4960428578840137, + "tokens_seen": 1760231424 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037714744232698096, + "loss": 1.2625, + "theoretical_loss": 3.49603182159637, + "tokens_seen": 1760296960 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037713139418254766, + "loss": 1.3932, + "theoretical_loss": 3.4960207858346424, + "tokens_seen": 1760362496 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037711534603811437, + "loss": 1.2925, + "theoretical_loss": 3.4960097505987866, + "tokens_seen": 1760428032 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037709929789368107, + "loss": 1.2564, + "theoretical_loss": 3.4959987158887573, + "tokens_seen": 1760493568 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037708324974924777, + "loss": 1.2739, + "theoretical_loss": 3.495987681704511, + "tokens_seen": 1760559104 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037706720160481447, + "loss": 1.292, + "theoretical_loss": 3.4959766480460024, + "tokens_seen": 1760624640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037705115346038117, + "loss": 1.3586, + "theoretical_loss": 3.495965614913187, + "tokens_seen": 1760690176 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037703510531594787, + "loss": 1.3021, + "theoretical_loss": 3.4959545823060196, + "tokens_seen": 1760755712 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037701905717151457, + "loss": 1.2686, + "theoretical_loss": 3.4959435502244567, + "tokens_seen": 1760821248 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037700300902708127, + "loss": 1.2292, + "theoretical_loss": 3.4959325186684533, + "tokens_seen": 1760886784 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037698696088264797, + "loss": 1.3947, + "theoretical_loss": 3.495921487637964, + "tokens_seen": 1760952320 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037697091273821467, + "loss": 1.285, + "theoretical_loss": 3.495910457132945, + "tokens_seen": 1761017856 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037695486459378137, + "loss": 1.2282, + "theoretical_loss": 3.4958994271533523, + "tokens_seen": 1761083392 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037693881644934807, + "loss": 1.2148, + "theoretical_loss": 3.4958883976991397, + "tokens_seen": 1761148928 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037692276830491477, + "loss": 1.3451, + "theoretical_loss": 3.495877368770264, + "tokens_seen": 1761214464 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003769067201604815, + "loss": 1.1869, + "theoretical_loss": 3.49586634036668, + "tokens_seen": 1761280000 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003768906720160482, + "loss": 1.2616, + "theoretical_loss": 3.495855312488343, + "tokens_seen": 1761345536 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003768746238716149, + "loss": 1.2516, + "theoretical_loss": 3.4958442851352087, + "tokens_seen": 1761411072 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003768585757271816, + "loss": 1.3719, + "theoretical_loss": 3.4958332583072327, + "tokens_seen": 1761476608 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003768425275827483, + "loss": 1.3195, + "theoretical_loss": 3.495822232004371, + "tokens_seen": 1761542144 + }, + { + "epoch": 0.53, + "learning_rate": 0.000376826479438315, + "loss": 1.2961, + "theoretical_loss": 3.4958112062265774, + "tokens_seen": 1761607680 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003768104312938816, + "loss": 1.3393, + "theoretical_loss": 3.495800180973809, + "tokens_seen": 1761673216 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003767943831494483, + "loss": 1.3445, + "theoretical_loss": 3.49578915624602, + "tokens_seen": 1761738752 + }, + { + "epoch": 0.53, + "learning_rate": 0.000376778335005015, + "loss": 1.3252, + "theoretical_loss": 3.495778132043167, + "tokens_seen": 1761804288 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003767622868605817, + "loss": 1.2419, + "theoretical_loss": 3.4957671083652047, + "tokens_seen": 1761869824 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003767462387161484, + "loss": 1.4053, + "theoretical_loss": 3.495756085212089, + "tokens_seen": 1761935360 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003767301905717151, + "loss": 1.3721, + "theoretical_loss": 3.495745062583775, + "tokens_seen": 1762000896 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003767141424272818, + "loss": 1.2116, + "theoretical_loss": 3.495734040480219, + "tokens_seen": 1762066432 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037669809428284853, + "loss": 1.2582, + "theoretical_loss": 3.495723018901375, + "tokens_seen": 1762131968 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037668204613841523, + "loss": 1.3481, + "theoretical_loss": 3.4957119978472, + "tokens_seen": 1762197504 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037666599799398193, + "loss": 1.3263, + "theoretical_loss": 3.4957009773176493, + "tokens_seen": 1762263040 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037664994984954863, + "loss": 1.2597, + "theoretical_loss": 3.4956899573126776, + "tokens_seen": 1762328576 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037663390170511533, + "loss": 1.2317, + "theoretical_loss": 3.4956789378322406, + "tokens_seen": 1762394112 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037661785356068203, + "loss": 1.17, + "theoretical_loss": 3.4956679188762947, + "tokens_seen": 1762459648 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037660180541624873, + "loss": 1.3709, + "theoretical_loss": 3.4956569004447946, + "tokens_seen": 1762525184 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037658575727181543, + "loss": 1.3024, + "theoretical_loss": 3.495645882537696, + "tokens_seen": 1762590720 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003765697091273822, + "loss": 1.1902, + "theoretical_loss": 3.4956348651549547, + "tokens_seen": 1762656256 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003765536609829489, + "loss": 1.2605, + "theoretical_loss": 3.4956238482965265, + "tokens_seen": 1762721792 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003765376128385156, + "loss": 1.2986, + "theoretical_loss": 3.495612831962366, + "tokens_seen": 1762787328 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003765215646940823, + "loss": 1.2204, + "theoretical_loss": 3.49560181615243, + "tokens_seen": 1762852864 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 591063, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.269951343536377, + "objective/train/theoretical_loss": 3.4955908008666725, + "objective/train/tokens_used": 1783378400, + "theoretical_loss": 3.4955908008666725, + "tokens_seen": 1762918400 + }, + { + "epoch": 0.53, + "learning_rate": 0.000376505516549649, + "loss": 1.2915, + "theoretical_loss": 3.4955908008666725, + "tokens_seen": 1762918400 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003764894684052157, + "loss": 1.3602, + "theoretical_loss": 3.4955797861050506, + "tokens_seen": 1762983936 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003764734202607824, + "loss": 1.2533, + "theoretical_loss": 3.495568771867519, + "tokens_seen": 1763049472 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003764573721163491, + "loss": 1.2646, + "theoretical_loss": 3.495557758154033, + "tokens_seen": 1763115008 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003764413239719158, + "loss": 1.3023, + "theoretical_loss": 3.4955467449645496, + "tokens_seen": 1763180544 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003764252758274825, + "loss": 1.3067, + "theoretical_loss": 3.4955357322990226, + "tokens_seen": 1763246080 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003764092276830492, + "loss": 1.315, + "theoretical_loss": 3.495524720157409, + "tokens_seen": 1763311616 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003763931795386159, + "loss": 1.3536, + "theoretical_loss": 3.495513708539664, + "tokens_seen": 1763377152 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003763771313941826, + "loss": 1.3975, + "theoretical_loss": 3.495502697445743, + "tokens_seen": 1763442688 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003763610832497493, + "loss": 1.1657, + "theoretical_loss": 3.495491686875602, + "tokens_seen": 1763508224 + }, + { + "epoch": 0.53, + "learning_rate": 0.000376345035105316, + "loss": 1.2885, + "theoretical_loss": 3.495480676829196, + "tokens_seen": 1763573760 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003763289869608827, + "loss": 1.3532, + "theoretical_loss": 3.495469667306481, + "tokens_seen": 1763639296 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003763129388164494, + "loss": 1.3095, + "theoretical_loss": 3.495458658307413, + "tokens_seen": 1763704832 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003762968906720161, + "loss": 1.2785, + "theoretical_loss": 3.495447649831947, + "tokens_seen": 1763770368 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003762808425275828, + "loss": 1.2835, + "theoretical_loss": 3.4954366418800387, + "tokens_seen": 1763835904 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003762647943831495, + "loss": 1.2842, + "theoretical_loss": 3.495425634451644, + "tokens_seen": 1763901440 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003762487462387162, + "loss": 1.3379, + "theoretical_loss": 3.495414627546719, + "tokens_seen": 1763966976 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003762326980942829, + "loss": 1.2861, + "theoretical_loss": 3.495403621165218, + "tokens_seen": 1764032512 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003762166499498496, + "loss": 1.2767, + "theoretical_loss": 3.495392615307098, + "tokens_seen": 1764098048 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003762006018054163, + "loss": 1.3185, + "theoretical_loss": 3.495381609972314, + "tokens_seen": 1764163584 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037618455366098295, + "loss": 1.2642, + "theoretical_loss": 3.495370605160822, + "tokens_seen": 1764229120 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037616850551654965, + "loss": 1.2175, + "theoretical_loss": 3.4953596008725776, + "tokens_seen": 1764294656 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037615245737211635, + "loss": 1.2559, + "theoretical_loss": 3.4953485971075366, + "tokens_seen": 1764360192 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037613640922768305, + "loss": 1.2036, + "theoretical_loss": 3.4953375938656537, + "tokens_seen": 1764425728 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037612036108324975, + "loss": 1.2947, + "theoretical_loss": 3.4953265911468865, + "tokens_seen": 1764491264 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037610431293881645, + "loss": 1.2635, + "theoretical_loss": 3.495315588951189, + "tokens_seen": 1764556800 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037608826479438315, + "loss": 1.1946, + "theoretical_loss": 3.4953045872785173, + "tokens_seen": 1764622336 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037607221664994985, + "loss": 1.3941, + "theoretical_loss": 3.495293586128828, + "tokens_seen": 1764687872 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037605616850551655, + "loss": 1.2831, + "theoretical_loss": 3.4952825855020757, + "tokens_seen": 1764753408 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037604012036108325, + "loss": 1.2313, + "theoretical_loss": 3.495271585398217, + "tokens_seen": 1764818944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037602407221664995, + "loss": 1.2768, + "theoretical_loss": 3.4952605858172068, + "tokens_seen": 1764884480 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037600802407221665, + "loss": 1.3326, + "theoretical_loss": 3.495249586759001, + "tokens_seen": 1764950016 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037599197592778336, + "loss": 1.2739, + "theoretical_loss": 3.495238588223556, + "tokens_seen": 1765015552 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037597592778335006, + "loss": 1.2575, + "theoretical_loss": 3.495227590210827, + "tokens_seen": 1765081088 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037595987963891676, + "loss": 1.2384, + "theoretical_loss": 3.4952165927207703, + "tokens_seen": 1765146624 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037594383149448346, + "loss": 1.3129, + "theoretical_loss": 3.4952055957533408, + "tokens_seen": 1765212160 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037592778335005016, + "loss": 1.248, + "theoretical_loss": 3.495194599308495, + "tokens_seen": 1765277696 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037591173520561686, + "loss": 1.2642, + "theoretical_loss": 3.495183603386188, + "tokens_seen": 1765343232 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037589568706118356, + "loss": 1.3172, + "theoretical_loss": 3.495172607986376, + "tokens_seen": 1765408768 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037587963891675026, + "loss": 1.2373, + "theoretical_loss": 3.4951616131090146, + "tokens_seen": 1765474304 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037586359077231696, + "loss": 1.3126, + "theoretical_loss": 3.49515061875406, + "tokens_seen": 1765539840 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037584754262788366, + "loss": 1.3983, + "theoretical_loss": 3.495139624921468, + "tokens_seen": 1765605376 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037583149448345036, + "loss": 1.24, + "theoretical_loss": 3.495128631611194, + "tokens_seen": 1765670912 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037581544633901706, + "loss": 1.3613, + "theoretical_loss": 3.495117638823193, + "tokens_seen": 1765736448 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037579939819458376, + "loss": 1.2363, + "theoretical_loss": 3.4951066465574225, + "tokens_seen": 1765801984 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037578335005015046, + "loss": 1.218, + "theoretical_loss": 3.495095654813837, + "tokens_seen": 1765867520 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037576730190571716, + "loss": 1.2626, + "theoretical_loss": 3.495084663592393, + "tokens_seen": 1765933056 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037575125376128386, + "loss": 1.2772, + "theoretical_loss": 3.4950736728930467, + "tokens_seen": 1765998592 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037573520561685057, + "loss": 1.2854, + "theoretical_loss": 3.4950626827157523, + "tokens_seen": 1766064128 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037571915747241727, + "loss": 1.2305, + "theoretical_loss": 3.4950516930604674, + "tokens_seen": 1766129664 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 591956, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2556519508361816, + "objective/train/theoretical_loss": 3.495040703927147, + "objective/train/tokens_used": 1786655200, + "theoretical_loss": 3.495040703927147, + "tokens_seen": 1766195200 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037570310932798397, + "loss": 1.3152, + "theoretical_loss": 3.495040703927147, + "tokens_seen": 1766195200 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037568706118355067, + "loss": 1.3164, + "theoretical_loss": 3.495029715315747, + "tokens_seen": 1766260736 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037567101303911737, + "loss": 1.259, + "theoretical_loss": 3.495018727226223, + "tokens_seen": 1766326272 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037565496489468407, + "loss": 1.2204, + "theoretical_loss": 3.495007739658532, + "tokens_seen": 1766391808 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037563891675025077, + "loss": 1.2158, + "theoretical_loss": 3.4949967526126287, + "tokens_seen": 1766457344 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037562286860581747, + "loss": 1.2531, + "theoretical_loss": 3.494985766088469, + "tokens_seen": 1766522880 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037560682046138417, + "loss": 1.3381, + "theoretical_loss": 3.494974780086009, + "tokens_seen": 1766588416 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037559077231695087, + "loss": 1.4084, + "theoretical_loss": 3.494963794605205, + "tokens_seen": 1766653952 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037557472417251757, + "loss": 1.332, + "theoretical_loss": 3.494952809646012, + "tokens_seen": 1766719488 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037555867602808427, + "loss": 1.3173, + "theoretical_loss": 3.494941825208387, + "tokens_seen": 1766785024 + }, + { + "epoch": 0.54, + "learning_rate": 0.000375542627883651, + "loss": 1.3249, + "theoretical_loss": 3.494930841292285, + "tokens_seen": 1766850560 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003755265797392177, + "loss": 1.2097, + "theoretical_loss": 3.4949198578976626, + "tokens_seen": 1766916096 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003755105315947844, + "loss": 1.3193, + "theoretical_loss": 3.494908875024475, + "tokens_seen": 1766981632 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003754944834503511, + "loss": 1.2514, + "theoretical_loss": 3.494897892672678, + "tokens_seen": 1767047168 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003754784353059178, + "loss": 1.2798, + "theoretical_loss": 3.4948869108422285, + "tokens_seen": 1767112704 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003754623871614845, + "loss": 1.3326, + "theoretical_loss": 3.494875929533081, + "tokens_seen": 1767178240 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003754463390170512, + "loss": 1.184, + "theoretical_loss": 3.4948649487451933, + "tokens_seen": 1767243776 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003754302908726179, + "loss": 1.2614, + "theoretical_loss": 3.4948539684785196, + "tokens_seen": 1767309312 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003754142427281845, + "loss": 1.3068, + "theoretical_loss": 3.4948429887330166, + "tokens_seen": 1767374848 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003753981945837512, + "loss": 1.1678, + "theoretical_loss": 3.4948320095086403, + "tokens_seen": 1767440384 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003753821464393179, + "loss": 1.3387, + "theoretical_loss": 3.4948210308053462, + "tokens_seen": 1767505920 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003753660982948847, + "loss": 1.396, + "theoretical_loss": 3.494810052623091, + "tokens_seen": 1767571456 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003753500501504514, + "loss": 1.2773, + "theoretical_loss": 3.49479907496183, + "tokens_seen": 1767636992 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003753340020060181, + "loss": 1.306, + "theoretical_loss": 3.494788097821519, + "tokens_seen": 1767702528 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003753179538615848, + "loss": 1.2512, + "theoretical_loss": 3.494777121202115, + "tokens_seen": 1767768064 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003753019057171515, + "loss": 1.3901, + "theoretical_loss": 3.494766145103573, + "tokens_seen": 1767833600 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003752858575727182, + "loss": 1.3101, + "theoretical_loss": 3.494755169525849, + "tokens_seen": 1767899136 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003752698094282849, + "loss": 1.2967, + "theoretical_loss": 3.4947441944688995, + "tokens_seen": 1767964672 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003752537612838516, + "loss": 1.3844, + "theoretical_loss": 3.49473321993268, + "tokens_seen": 1768030208 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003752377131394183, + "loss": 1.2835, + "theoretical_loss": 3.494722245917147, + "tokens_seen": 1768095744 + }, + { + "epoch": 0.54, + "learning_rate": 0.000375221664994985, + "loss": 1.2095, + "theoretical_loss": 3.494711272422256, + "tokens_seen": 1768161280 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003752056168505517, + "loss": 1.2166, + "theoretical_loss": 3.4947002994479632, + "tokens_seen": 1768226816 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003751895687061184, + "loss": 1.2557, + "theoretical_loss": 3.4946893269942247, + "tokens_seen": 1768292352 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003751735205616851, + "loss": 1.256, + "theoretical_loss": 3.494678355060997, + "tokens_seen": 1768357888 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003751574724172518, + "loss": 1.2208, + "theoretical_loss": 3.4946673836482347, + "tokens_seen": 1768423424 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003751414242728185, + "loss": 1.3012, + "theoretical_loss": 3.4946564127558952, + "tokens_seen": 1768488960 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003751253761283852, + "loss": 1.3499, + "theoretical_loss": 3.494645442383934, + "tokens_seen": 1768554496 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003751093279839519, + "loss": 1.3661, + "theoretical_loss": 3.494634472532307, + "tokens_seen": 1768620032 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003750932798395186, + "loss": 1.2057, + "theoretical_loss": 3.494623503200971, + "tokens_seen": 1768685568 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003750772316950853, + "loss": 1.2916, + "theoretical_loss": 3.494612534389881, + "tokens_seen": 1768751104 + }, + { + "epoch": 0.54, + "learning_rate": 0.000375061183550652, + "loss": 1.3139, + "theoretical_loss": 3.4946015660989933, + "tokens_seen": 1768816640 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003750451354062187, + "loss": 1.3618, + "theoretical_loss": 3.494590598328264, + "tokens_seen": 1768882176 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003750290872617854, + "loss": 1.3754, + "theoretical_loss": 3.49457963107765, + "tokens_seen": 1768947712 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003750130391173521, + "loss": 1.2143, + "theoretical_loss": 3.4945686643471063, + "tokens_seen": 1769013248 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003749969909729188, + "loss": 1.3679, + "theoretical_loss": 3.4945576981365893, + "tokens_seen": 1769078784 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003749809428284855, + "loss": 1.3123, + "theoretical_loss": 3.4945467324460555, + "tokens_seen": 1769144320 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003749648946840522, + "loss": 1.2931, + "theoretical_loss": 3.49453576727546, + "tokens_seen": 1769209856 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003749488465396189, + "loss": 1.3804, + "theoretical_loss": 3.4945248026247597, + "tokens_seen": 1769275392 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003749327983951856, + "loss": 1.2771, + "theoretical_loss": 3.4945138384939107, + "tokens_seen": 1769340928 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003749167502507523, + "loss": 1.2594, + "theoretical_loss": 3.494502874882869, + "tokens_seen": 1769406464 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 593210, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3526359796524048, + "objective/train/theoretical_loss": 3.4944919117915907, + "objective/train/tokens_used": 1789932000, + "theoretical_loss": 3.4944919117915907, + "tokens_seen": 1769472000 + }, + { + "epoch": 0.54, + "learning_rate": 0.000374900702106319, + "loss": 1.2735, + "theoretical_loss": 3.4944919117915907, + "tokens_seen": 1769472000 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003748846539618857, + "loss": 1.3899, + "theoretical_loss": 3.4944809492200317, + "tokens_seen": 1769537536 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003748686058174524, + "loss": 1.2759, + "theoretical_loss": 3.4944699871681477, + "tokens_seen": 1769603072 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003748525576730191, + "loss": 1.2225, + "theoretical_loss": 3.494459025635896, + "tokens_seen": 1769668608 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003748365095285858, + "loss": 1.3233, + "theoretical_loss": 3.4944480646232314, + "tokens_seen": 1769734144 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003748204613841525, + "loss": 1.2751, + "theoretical_loss": 3.494437104130111, + "tokens_seen": 1769799680 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003748044132397192, + "loss": 1.2445, + "theoretical_loss": 3.494426144156491, + "tokens_seen": 1769865216 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003747883650952859, + "loss": 1.2962, + "theoretical_loss": 3.4944151847023273, + "tokens_seen": 1769930752 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037477231695085255, + "loss": 1.3272, + "theoretical_loss": 3.4944042257675756, + "tokens_seen": 1769996288 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037475626880641925, + "loss": 1.2421, + "theoretical_loss": 3.4943932673521925, + "tokens_seen": 1770061824 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037474022066198595, + "loss": 1.301, + "theoretical_loss": 3.4943823094561335, + "tokens_seen": 1770127360 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037472417251755265, + "loss": 1.3186, + "theoretical_loss": 3.494371352079356, + "tokens_seen": 1770192896 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037470812437311935, + "loss": 1.2732, + "theoretical_loss": 3.494360395221815, + "tokens_seen": 1770258432 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037469207622868605, + "loss": 1.2608, + "theoretical_loss": 3.4943494388834675, + "tokens_seen": 1770323968 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037467602808425275, + "loss": 1.2879, + "theoretical_loss": 3.4943384830642694, + "tokens_seen": 1770389504 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037465997993981945, + "loss": 1.4281, + "theoretical_loss": 3.4943275277641765, + "tokens_seen": 1770455040 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037464393179538615, + "loss": 1.246, + "theoretical_loss": 3.4943165729831454, + "tokens_seen": 1770520576 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037462788365095286, + "loss": 1.2865, + "theoretical_loss": 3.494305618721132, + "tokens_seen": 1770586112 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037461183550651956, + "loss": 1.2833, + "theoretical_loss": 3.494294664978093, + "tokens_seen": 1770651648 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037459578736208626, + "loss": 1.3122, + "theoretical_loss": 3.4942837117539844, + "tokens_seen": 1770717184 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037457973921765296, + "loss": 1.3477, + "theoretical_loss": 3.494272759048762, + "tokens_seen": 1770782720 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037456369107321966, + "loss": 1.2849, + "theoretical_loss": 3.494261806862383, + "tokens_seen": 1770848256 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037454764292878636, + "loss": 1.2498, + "theoretical_loss": 3.494250855194802, + "tokens_seen": 1770913792 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037453159478435306, + "loss": 1.3322, + "theoretical_loss": 3.4942399040459766, + "tokens_seen": 1770979328 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037451554663991976, + "loss": 1.2993, + "theoretical_loss": 3.4942289534158624, + "tokens_seen": 1771044864 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037449949849548646, + "loss": 1.2036, + "theoretical_loss": 3.4942180033044163, + "tokens_seen": 1771110400 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037448345035105316, + "loss": 1.2881, + "theoretical_loss": 3.4942070537115937, + "tokens_seen": 1771175936 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037446740220661986, + "loss": 1.2977, + "theoretical_loss": 3.4941961046373518, + "tokens_seen": 1771241472 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037445135406218656, + "loss": 1.3372, + "theoretical_loss": 3.4941851560816457, + "tokens_seen": 1771307008 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037443530591775326, + "loss": 1.2596, + "theoretical_loss": 3.4941742080444325, + "tokens_seen": 1771372544 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037441925777331996, + "loss": 1.2261, + "theoretical_loss": 3.494163260525668, + "tokens_seen": 1771438080 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037440320962888666, + "loss": 1.2539, + "theoretical_loss": 3.494152313525309, + "tokens_seen": 1771503616 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037438716148445336, + "loss": 1.2532, + "theoretical_loss": 3.4941413670433112, + "tokens_seen": 1771569152 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037437111334002007, + "loss": 1.2389, + "theoretical_loss": 3.494130421079632, + "tokens_seen": 1771634688 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037435506519558677, + "loss": 1.3776, + "theoretical_loss": 3.4941194756342258, + "tokens_seen": 1771700224 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037433901705115347, + "loss": 1.3169, + "theoretical_loss": 3.4941085307070496, + "tokens_seen": 1771765760 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037432296890672017, + "loss": 1.2748, + "theoretical_loss": 3.4940975862980608, + "tokens_seen": 1771831296 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037430692076228687, + "loss": 1.3159, + "theoretical_loss": 3.494086642407215, + "tokens_seen": 1771896832 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037429087261785357, + "loss": 1.3105, + "theoretical_loss": 3.494075699034468, + "tokens_seen": 1771962368 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037427482447342027, + "loss": 1.2734, + "theoretical_loss": 3.494064756179777, + "tokens_seen": 1772027904 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037425877632898697, + "loss": 1.3476, + "theoretical_loss": 3.4940538138430974, + "tokens_seen": 1772093440 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037424272818455367, + "loss": 1.3377, + "theoretical_loss": 3.4940428720243863, + "tokens_seen": 1772158976 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037422668004012037, + "loss": 1.366, + "theoretical_loss": 3.494031930723599, + "tokens_seen": 1772224512 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037421063189568707, + "loss": 1.2121, + "theoretical_loss": 3.4940209899406933, + "tokens_seen": 1772290048 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037419458375125377, + "loss": 1.3367, + "theoretical_loss": 3.494010049675625, + "tokens_seen": 1772355584 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037417853560682047, + "loss": 1.2784, + "theoretical_loss": 3.4939991099283496, + "tokens_seen": 1772421120 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037416248746238723, + "loss": 1.215, + "theoretical_loss": 3.493988170698824, + "tokens_seen": 1772486656 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003741464393179539, + "loss": 1.2512, + "theoretical_loss": 3.493977231987005, + "tokens_seen": 1772552192 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003741303911735206, + "loss": 1.3701, + "theoretical_loss": 3.493966293792848, + "tokens_seen": 1772617728 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003741143430290873, + "loss": 1.2991, + "theoretical_loss": 3.493955356116311, + "tokens_seen": 1772683264 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 594137, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2289804220199585, + "objective/train/theoretical_loss": 3.4939444189573483, + "objective/train/tokens_used": 1793208800, + "theoretical_loss": 3.4939444189573483, + "tokens_seen": 1772748800 + }, + { + "epoch": 0.54, + "learning_rate": 0.000374098294884654, + "loss": 1.235, + "theoretical_loss": 3.4939444189573483, + "tokens_seen": 1772748800 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003740822467402207, + "loss": 1.2525, + "theoretical_loss": 3.493933482315918, + "tokens_seen": 1772814336 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003740661985957874, + "loss": 1.2766, + "theoretical_loss": 3.4939225461919756, + "tokens_seen": 1772879872 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003740501504513541, + "loss": 1.3094, + "theoretical_loss": 3.493911610585477, + "tokens_seen": 1772945408 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003740341023069208, + "loss": 1.3317, + "theoretical_loss": 3.49390067549638, + "tokens_seen": 1773010944 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003740180541624875, + "loss": 1.3131, + "theoretical_loss": 3.49388974092464, + "tokens_seen": 1773076480 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003740020060180542, + "loss": 1.1886, + "theoretical_loss": 3.493878806870214, + "tokens_seen": 1773142016 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003739859578736209, + "loss": 1.2711, + "theoretical_loss": 3.4938678733330573, + "tokens_seen": 1773207552 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003739699097291876, + "loss": 1.3242, + "theoretical_loss": 3.493856940313128, + "tokens_seen": 1773273088 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003739538615847543, + "loss": 1.2552, + "theoretical_loss": 3.493846007810381, + "tokens_seen": 1773338624 + }, + { + "epoch": 0.54, + "learning_rate": 0.000373937813440321, + "loss": 1.2606, + "theoretical_loss": 3.493835075824773, + "tokens_seen": 1773404160 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003739217652958877, + "loss": 1.2418, + "theoretical_loss": 3.493824144356261, + "tokens_seen": 1773469696 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003739057171514544, + "loss": 1.3952, + "theoretical_loss": 3.4938132134048017, + "tokens_seen": 1773535232 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003738896690070211, + "loss": 1.295, + "theoretical_loss": 3.4938022829703503, + "tokens_seen": 1773600768 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003738736208625878, + "loss": 1.2371, + "theoretical_loss": 3.493791353052864, + "tokens_seen": 1773666304 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003738575727181545, + "loss": 1.2186, + "theoretical_loss": 3.4937804236523, + "tokens_seen": 1773731840 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003738415245737212, + "loss": 1.2767, + "theoretical_loss": 3.493769494768613, + "tokens_seen": 1773797376 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003738254764292879, + "loss": 1.286, + "theoretical_loss": 3.493758566401761, + "tokens_seen": 1773862912 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003738094282848546, + "loss": 1.3555, + "theoretical_loss": 3.4937476385516995, + "tokens_seen": 1773928448 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003737933801404213, + "loss": 1.162, + "theoretical_loss": 3.4937367112183857, + "tokens_seen": 1773993984 + }, + { + "epoch": 0.54, + "learning_rate": 0.000373777331995988, + "loss": 1.2317, + "theoretical_loss": 3.493725784401776, + "tokens_seen": 1774059520 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003737612838515547, + "loss": 1.2598, + "theoretical_loss": 3.493714858101826, + "tokens_seen": 1774125056 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003737452357071214, + "loss": 1.2244, + "theoretical_loss": 3.4937039323184926, + "tokens_seen": 1774190592 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003737291875626881, + "loss": 1.2337, + "theoretical_loss": 3.493693007051733, + "tokens_seen": 1774256128 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003737131394182548, + "loss": 1.3792, + "theoretical_loss": 3.493682082301503, + "tokens_seen": 1774321664 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003736970912738215, + "loss": 1.3639, + "theoretical_loss": 3.4936711580677597, + "tokens_seen": 1774387200 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003736810431293882, + "loss": 1.3068, + "theoretical_loss": 3.493660234350459, + "tokens_seen": 1774452736 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003736649949849549, + "loss": 1.2667, + "theoretical_loss": 3.4936493111495572, + "tokens_seen": 1774518272 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003736489468405216, + "loss": 1.324, + "theoretical_loss": 3.493638388465012, + "tokens_seen": 1774583808 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003736328986960883, + "loss": 1.3445, + "theoretical_loss": 3.493627466296778, + "tokens_seen": 1774649344 + }, + { + "epoch": 0.54, + "learning_rate": 0.000373616850551655, + "loss": 1.2381, + "theoretical_loss": 3.493616544644814, + "tokens_seen": 1774714880 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003736008024072217, + "loss": 1.391, + "theoretical_loss": 3.4936056235090747, + "tokens_seen": 1774780416 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003735847542627884, + "loss": 1.2758, + "theoretical_loss": 3.4935947028895176, + "tokens_seen": 1774845952 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003735687061183551, + "loss": 1.2259, + "theoretical_loss": 3.493583782786099, + "tokens_seen": 1774911488 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003735526579739218, + "loss": 1.3042, + "theoretical_loss": 3.4935728631987755, + "tokens_seen": 1774977024 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003735366098294885, + "loss": 1.2185, + "theoretical_loss": 3.4935619441275034, + "tokens_seen": 1775042560 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003735205616850552, + "loss": 1.3384, + "theoretical_loss": 3.4935510255722395, + "tokens_seen": 1775108096 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003735045135406219, + "loss": 1.3412, + "theoretical_loss": 3.4935401075329406, + "tokens_seen": 1775173632 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003734884653961886, + "loss": 1.2276, + "theoretical_loss": 3.4935291900095624, + "tokens_seen": 1775239168 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003734724172517553, + "loss": 1.2023, + "theoretical_loss": 3.4935182730020626, + "tokens_seen": 1775304704 + }, + { + "epoch": 0.54, + "learning_rate": 0.000373456369107322, + "loss": 1.375, + "theoretical_loss": 3.4935073565103973, + "tokens_seen": 1775370240 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003734403209628887, + "loss": 1.3523, + "theoretical_loss": 3.493496440534523, + "tokens_seen": 1775435776 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003734242728184554, + "loss": 1.2747, + "theoretical_loss": 3.493485525074396, + "tokens_seen": 1775501312 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003734082246740221, + "loss": 1.2721, + "theoretical_loss": 3.493474610129973, + "tokens_seen": 1775566848 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003733921765295888, + "loss": 1.2859, + "theoretical_loss": 3.4934636957012115, + "tokens_seen": 1775632384 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037337612838515545, + "loss": 1.3189, + "theoretical_loss": 3.4934527817880676, + "tokens_seen": 1775697920 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037336008024072215, + "loss": 1.4163, + "theoretical_loss": 3.493441868390497, + "tokens_seen": 1775763456 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037334403209628885, + "loss": 1.4387, + "theoretical_loss": 3.4934309555084573, + "tokens_seen": 1775828992 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037332798395185555, + "loss": 1.1661, + "theoretical_loss": 3.493420043141905, + "tokens_seen": 1775894528 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037331193580742225, + "loss": 1.3508, + "theoretical_loss": 3.4934091312907967, + "tokens_seen": 1775960064 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 595419, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3804264068603516, + "objective/train/theoretical_loss": 3.493398219955089, + "objective/train/tokens_used": 1796485600, + "theoretical_loss": 3.493398219955089, + "tokens_seen": 1776025600 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037329588766298895, + "loss": 1.3509, + "theoretical_loss": 3.493398219955089, + "tokens_seen": 1776025600 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037327983951855565, + "loss": 1.3068, + "theoretical_loss": 3.4933873091347385, + "tokens_seen": 1776091136 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037326379137412235, + "loss": 1.1843, + "theoretical_loss": 3.4933763988297013, + "tokens_seen": 1776156672 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037324774322968906, + "loss": 1.1732, + "theoretical_loss": 3.493365489039935, + "tokens_seen": 1776222208 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037323169508525576, + "loss": 1.2681, + "theoretical_loss": 3.4933545797653958, + "tokens_seen": 1776287744 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037321564694082246, + "loss": 1.3126, + "theoretical_loss": 3.4933436710060404, + "tokens_seen": 1776353280 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037319959879638916, + "loss": 1.3525, + "theoretical_loss": 3.493332762761826, + "tokens_seen": 1776418816 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037318355065195586, + "loss": 1.3163, + "theoretical_loss": 3.4933218550327076, + "tokens_seen": 1776484352 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037316750250752256, + "loss": 1.3326, + "theoretical_loss": 3.493310947818644, + "tokens_seen": 1776549888 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037315145436308926, + "loss": 1.2645, + "theoretical_loss": 3.49330004111959, + "tokens_seen": 1776615424 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037313540621865596, + "loss": 1.2377, + "theoretical_loss": 3.493289134935504, + "tokens_seen": 1776680960 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037311935807422266, + "loss": 1.3154, + "theoretical_loss": 3.4932782292663416, + "tokens_seen": 1776746496 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037310330992978936, + "loss": 1.3308, + "theoretical_loss": 3.4932673241120593, + "tokens_seen": 1776812032 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037308726178535606, + "loss": 1.1598, + "theoretical_loss": 3.4932564194726154, + "tokens_seen": 1776877568 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037307121364092276, + "loss": 1.2539, + "theoretical_loss": 3.4932455153479642, + "tokens_seen": 1776943104 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037305516549648946, + "loss": 1.2541, + "theoretical_loss": 3.493234611738065, + "tokens_seen": 1777008640 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037303911735205616, + "loss": 1.2887, + "theoretical_loss": 3.493223708642872, + "tokens_seen": 1777074176 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037302306920762286, + "loss": 1.2249, + "theoretical_loss": 3.4932128060623437, + "tokens_seen": 1777139712 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037300702106318956, + "loss": 1.2834, + "theoretical_loss": 3.493201903996436, + "tokens_seen": 1777205248 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037299097291875627, + "loss": 1.339, + "theoretical_loss": 3.493191002445106, + "tokens_seen": 1777270784 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037297492477432297, + "loss": 1.163, + "theoretical_loss": 3.4931801014083104, + "tokens_seen": 1777336320 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003729588766298897, + "loss": 1.3499, + "theoretical_loss": 3.4931692008860056, + "tokens_seen": 1777401856 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003729428284854564, + "loss": 1.3439, + "theoretical_loss": 3.493158300878149, + "tokens_seen": 1777467392 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003729267803410231, + "loss": 1.3143, + "theoretical_loss": 3.4931474013846966, + "tokens_seen": 1777532928 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003729107321965898, + "loss": 1.2063, + "theoretical_loss": 3.493136502405606, + "tokens_seen": 1777598464 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003728946840521565, + "loss": 1.3282, + "theoretical_loss": 3.4931256039408334, + "tokens_seen": 1777664000 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003728786359077232, + "loss": 1.2519, + "theoretical_loss": 3.4931147059903354, + "tokens_seen": 1777729536 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003728625877632899, + "loss": 1.2361, + "theoretical_loss": 3.493103808554069, + "tokens_seen": 1777795072 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003728465396188566, + "loss": 1.3067, + "theoretical_loss": 3.493092911631991, + "tokens_seen": 1777860608 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003728304914744233, + "loss": 1.3457, + "theoretical_loss": 3.4930820152240587, + "tokens_seen": 1777926144 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037281444332999, + "loss": 1.3067, + "theoretical_loss": 3.493071119330228, + "tokens_seen": 1777991680 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037279839518555673, + "loss": 1.2777, + "theoretical_loss": 3.4930602239504562, + "tokens_seen": 1778057216 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037278234704112343, + "loss": 1.2514, + "theoretical_loss": 3.4930493290847, + "tokens_seen": 1778122752 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037276629889669013, + "loss": 1.312, + "theoretical_loss": 3.493038434732916, + "tokens_seen": 1778188288 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003727502507522568, + "loss": 1.2231, + "theoretical_loss": 3.4930275408950617, + "tokens_seen": 1778253824 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003727342026078235, + "loss": 1.3043, + "theoretical_loss": 3.4930166475710926, + "tokens_seen": 1778319360 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003727181544633902, + "loss": 1.2251, + "theoretical_loss": 3.493005754760967, + "tokens_seen": 1778384896 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003727021063189569, + "loss": 1.1795, + "theoretical_loss": 3.4929948624646414, + "tokens_seen": 1778450432 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003726860581745236, + "loss": 1.2545, + "theoretical_loss": 3.4929839706820713, + "tokens_seen": 1778515968 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003726700100300903, + "loss": 1.3244, + "theoretical_loss": 3.4929730794132157, + "tokens_seen": 1778581504 + }, + { + "epoch": 0.54, + "learning_rate": 0.000372653961885657, + "loss": 1.3092, + "theoretical_loss": 3.4929621886580295, + "tokens_seen": 1778647040 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003726379137412237, + "loss": 1.2789, + "theoretical_loss": 3.492951298416471, + "tokens_seen": 1778712576 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003726218655967904, + "loss": 1.3034, + "theoretical_loss": 3.4929404086884954, + "tokens_seen": 1778778112 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003726058174523571, + "loss": 1.3272, + "theoretical_loss": 3.4929295194740613, + "tokens_seen": 1778843648 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003725897693079238, + "loss": 1.2807, + "theoretical_loss": 3.4929186307731244, + "tokens_seen": 1778909184 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003725737211634905, + "loss": 1.288, + "theoretical_loss": 3.4929077425856425, + "tokens_seen": 1778974720 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003725576730190572, + "loss": 1.2739, + "theoretical_loss": 3.4928968549115713, + "tokens_seen": 1779040256 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003725416248746239, + "loss": 1.2927, + "theoretical_loss": 3.492885967750869, + "tokens_seen": 1779105792 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003725255767301906, + "loss": 1.3796, + "theoretical_loss": 3.4928750811034917, + "tokens_seen": 1779171328 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003725095285857573, + "loss": 1.375, + "theoretical_loss": 3.492864194969396, + "tokens_seen": 1779236864 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 596357, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4457015991210938, + "objective/train/theoretical_loss": 3.4928533093485394, + "objective/train/tokens_used": 1799762400, + "theoretical_loss": 3.4928533093485394, + "tokens_seen": 1779302400 + }, + { + "epoch": 0.54, + "learning_rate": 0.000372493480441324, + "loss": 1.3579, + "theoretical_loss": 3.4928533093485394, + "tokens_seen": 1779302400 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003724774322968907, + "loss": 1.2273, + "theoretical_loss": 3.4928424242408784, + "tokens_seen": 1779367936 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003724613841524574, + "loss": 1.4106, + "theoretical_loss": 3.492831539646371, + "tokens_seen": 1779433472 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003724453360080241, + "loss": 1.2412, + "theoretical_loss": 3.492820655564972, + "tokens_seen": 1779499008 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003724292878635908, + "loss": 1.3766, + "theoretical_loss": 3.4928097719966402, + "tokens_seen": 1779564544 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003724132397191575, + "loss": 1.2539, + "theoretical_loss": 3.4927988889413317, + "tokens_seen": 1779630080 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003723971915747242, + "loss": 1.348, + "theoretical_loss": 3.492788006399004, + "tokens_seen": 1779695616 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003723811434302909, + "loss": 1.2678, + "theoretical_loss": 3.4927771243696135, + "tokens_seen": 1779761152 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003723650952858576, + "loss": 1.281, + "theoretical_loss": 3.4927662428531168, + "tokens_seen": 1779826688 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003723490471414243, + "loss": 1.2367, + "theoretical_loss": 3.492755361849472, + "tokens_seen": 1779892224 + }, + { + "epoch": 0.54, + "learning_rate": 0.000372332998996991, + "loss": 1.3321, + "theoretical_loss": 3.492744481358635, + "tokens_seen": 1779957760 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003723169508525577, + "loss": 1.2936, + "theoretical_loss": 3.4927336013805625, + "tokens_seen": 1780023296 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003723009027081244, + "loss": 1.343, + "theoretical_loss": 3.4927227219152126, + "tokens_seen": 1780088832 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003722848545636911, + "loss": 1.3221, + "theoretical_loss": 3.492711842962542, + "tokens_seen": 1780154368 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003722688064192578, + "loss": 1.2065, + "theoretical_loss": 3.4927009645225073, + "tokens_seen": 1780219904 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003722527582748245, + "loss": 1.3838, + "theoretical_loss": 3.492690086595065, + "tokens_seen": 1780285440 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003722367101303912, + "loss": 1.2958, + "theoretical_loss": 3.492679209180173, + "tokens_seen": 1780350976 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003722206619859579, + "loss": 1.2921, + "theoretical_loss": 3.492668332277788, + "tokens_seen": 1780416512 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003722046138415246, + "loss": 1.2606, + "theoretical_loss": 3.492657455887867, + "tokens_seen": 1780482048 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003721885656970913, + "loss": 1.3063, + "theoretical_loss": 3.4926465800103665, + "tokens_seen": 1780547584 + }, + { + "epoch": 0.54, + "learning_rate": 0.000372172517552658, + "loss": 1.2809, + "theoretical_loss": 3.4926357046452443, + "tokens_seen": 1780613120 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003721564694082247, + "loss": 1.2915, + "theoretical_loss": 3.4926248297924567, + "tokens_seen": 1780678656 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003721404212637914, + "loss": 1.2651, + "theoretical_loss": 3.4926139554519615, + "tokens_seen": 1780744192 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003721243731193581, + "loss": 1.2603, + "theoretical_loss": 3.492603081623715, + "tokens_seen": 1780809728 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003721083249749248, + "loss": 1.2487, + "theoretical_loss": 3.492592208307674, + "tokens_seen": 1780875264 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003720922768304915, + "loss": 1.337, + "theoretical_loss": 3.4925813355037967, + "tokens_seen": 1780940800 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003720762286860582, + "loss": 1.3738, + "theoretical_loss": 3.4925704632120387, + "tokens_seen": 1781006336 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003720601805416249, + "loss": 1.2783, + "theoretical_loss": 3.492559591432358, + "tokens_seen": 1781071872 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003720441323971916, + "loss": 1.2845, + "theoretical_loss": 3.4925487201647116, + "tokens_seen": 1781137408 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003720280842527583, + "loss": 1.2604, + "theoretical_loss": 3.4925378494090564, + "tokens_seen": 1781202944 + }, + { + "epoch": 0.54, + "learning_rate": 0.000372012036108325, + "loss": 1.3961, + "theoretical_loss": 3.4925269791653486, + "tokens_seen": 1781268480 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003719959879638917, + "loss": 1.2661, + "theoretical_loss": 3.492516109433547, + "tokens_seen": 1781334016 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003719799398194584, + "loss": 1.2576, + "theoretical_loss": 3.4925052402136076, + "tokens_seen": 1781399552 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037196389167502505, + "loss": 1.2348, + "theoretical_loss": 3.4924943715054866, + "tokens_seen": 1781465088 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037194784353059175, + "loss": 1.2802, + "theoretical_loss": 3.492483503309143, + "tokens_seen": 1781530624 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037193179538615845, + "loss": 1.2739, + "theoretical_loss": 3.492472635624533, + "tokens_seen": 1781596160 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037191574724172515, + "loss": 1.3493, + "theoretical_loss": 3.492461768451613, + "tokens_seen": 1781661696 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037189969909729185, + "loss": 1.2575, + "theoretical_loss": 3.492450901790341, + "tokens_seen": 1781727232 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037188365095285856, + "loss": 1.322, + "theoretical_loss": 3.4924400356406737, + "tokens_seen": 1781792768 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037186760280842526, + "loss": 1.1625, + "theoretical_loss": 3.492429170002568, + "tokens_seen": 1781858304 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037185155466399196, + "loss": 1.2551, + "theoretical_loss": 3.492418304875981, + "tokens_seen": 1781923840 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037183550651955866, + "loss": 1.2944, + "theoretical_loss": 3.4924074402608714, + "tokens_seen": 1781989376 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037181945837512536, + "loss": 1.3602, + "theoretical_loss": 3.4923965761571942, + "tokens_seen": 1782054912 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037180341023069206, + "loss": 1.1933, + "theoretical_loss": 3.492385712564907, + "tokens_seen": 1782120448 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037178736208625876, + "loss": 1.3251, + "theoretical_loss": 3.4923748494839675, + "tokens_seen": 1782185984 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003717713139418255, + "loss": 1.29, + "theoretical_loss": 3.4923639869143326, + "tokens_seen": 1782251520 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003717552657973922, + "loss": 1.3486, + "theoretical_loss": 3.4923531248559594, + "tokens_seen": 1782317056 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003717392176529589, + "loss": 1.1895, + "theoretical_loss": 3.4923422633088053, + "tokens_seen": 1782382592 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003717231695085256, + "loss": 1.229, + "theoretical_loss": 3.4923314022728267, + "tokens_seen": 1782448128 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003717071213640923, + "loss": 1.3417, + "theoretical_loss": 3.492320541747982, + "tokens_seen": 1782513664 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 597594, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.43216872215271, + "objective/train/theoretical_loss": 3.4923096817342265, + "objective/train/tokens_used": 1803039200, + "theoretical_loss": 3.4923096817342265, + "tokens_seen": 1782579200 + }, + { + "epoch": 0.54, + "learning_rate": 0.000371691073219659, + "loss": 1.3708, + "theoretical_loss": 3.4923096817342265, + "tokens_seen": 1782579200 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003716750250752257, + "loss": 1.2503, + "theoretical_loss": 3.492298822231519, + "tokens_seen": 1782644736 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003716589769307924, + "loss": 1.2937, + "theoretical_loss": 3.492287963239816, + "tokens_seen": 1782710272 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003716429287863591, + "loss": 1.287, + "theoretical_loss": 3.492277104759075, + "tokens_seen": 1782775808 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003716268806419258, + "loss": 1.2429, + "theoretical_loss": 3.492266246789253, + "tokens_seen": 1782841344 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003716108324974925, + "loss": 1.2975, + "theoretical_loss": 3.4922553893303068, + "tokens_seen": 1782906880 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003715947843530592, + "loss": 1.2665, + "theoretical_loss": 3.4922445323821942, + "tokens_seen": 1782972416 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003715787362086259, + "loss": 1.2447, + "theoretical_loss": 3.492233675944872, + "tokens_seen": 1783037952 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003715626880641926, + "loss": 1.2945, + "theoretical_loss": 3.4922228200182976, + "tokens_seen": 1783103488 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003715466399197593, + "loss": 1.2915, + "theoretical_loss": 3.4922119646024283, + "tokens_seen": 1783169024 + }, + { + "epoch": 0.54, + "learning_rate": 0.000371530591775326, + "loss": 1.3223, + "theoretical_loss": 3.4922011096972208, + "tokens_seen": 1783234560 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003715145436308927, + "loss": 1.2997, + "theoretical_loss": 3.4921902553026327, + "tokens_seen": 1783300096 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003714984954864594, + "loss": 1.2471, + "theoretical_loss": 3.492179401418621, + "tokens_seen": 1783365632 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003714824473420261, + "loss": 1.2847, + "theoretical_loss": 3.492168548045143, + "tokens_seen": 1783431168 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003714663991975928, + "loss": 1.2639, + "theoretical_loss": 3.492157695182156, + "tokens_seen": 1783496704 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003714503510531595, + "loss": 1.1435, + "theoretical_loss": 3.4921468428296176, + "tokens_seen": 1783562240 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037143430290872623, + "loss": 1.2784, + "theoretical_loss": 3.4921359909874847, + "tokens_seen": 1783627776 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037141825476429293, + "loss": 1.3278, + "theoretical_loss": 3.4921251396557142, + "tokens_seen": 1783693312 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037140220661985963, + "loss": 1.2553, + "theoretical_loss": 3.492114288834264, + "tokens_seen": 1783758848 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037138615847542633, + "loss": 1.3382, + "theoretical_loss": 3.4921034385230905, + "tokens_seen": 1783824384 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037137011033099303, + "loss": 1.2681, + "theoretical_loss": 3.4920925887221514, + "tokens_seen": 1783889920 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037135406218655973, + "loss": 1.2504, + "theoretical_loss": 3.4920817394314048, + "tokens_seen": 1783955456 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003713380140421264, + "loss": 1.203, + "theoretical_loss": 3.4920708906508064, + "tokens_seen": 1784020992 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003713219658976931, + "loss": 1.3214, + "theoretical_loss": 3.4920600423803148, + "tokens_seen": 1784086528 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003713059177532598, + "loss": 1.2705, + "theoretical_loss": 3.492049194619886, + "tokens_seen": 1784152064 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003712898696088265, + "loss": 1.3463, + "theoretical_loss": 3.492038347369479, + "tokens_seen": 1784217600 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003712738214643932, + "loss": 1.312, + "theoretical_loss": 3.4920275006290495, + "tokens_seen": 1784283136 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003712577733199599, + "loss": 1.2994, + "theoretical_loss": 3.4920166543985554, + "tokens_seen": 1784348672 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003712417251755266, + "loss": 1.2465, + "theoretical_loss": 3.492005808677954, + "tokens_seen": 1784414208 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003712256770310933, + "loss": 1.2741, + "theoretical_loss": 3.491994963467203, + "tokens_seen": 1784479744 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037120962888666, + "loss": 1.2434, + "theoretical_loss": 3.491984118766259, + "tokens_seen": 1784545280 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003711935807422267, + "loss": 1.2654, + "theoretical_loss": 3.4919732745750798, + "tokens_seen": 1784610816 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003711775325977934, + "loss": 1.3727, + "theoretical_loss": 3.491962430893622, + "tokens_seen": 1784676352 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003711614844533601, + "loss": 1.2808, + "theoretical_loss": 3.4919515877218443, + "tokens_seen": 1784741888 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003711454363089268, + "loss": 1.2759, + "theoretical_loss": 3.491940745059703, + "tokens_seen": 1784807424 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003711293881644935, + "loss": 1.3148, + "theoretical_loss": 3.491929902907155, + "tokens_seen": 1784872960 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003711133400200602, + "loss": 1.2479, + "theoretical_loss": 3.4919190612641593, + "tokens_seen": 1784938496 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003710972918756269, + "loss": 1.3257, + "theoretical_loss": 3.4919082201306715, + "tokens_seen": 1785004032 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003710812437311936, + "loss": 1.3539, + "theoretical_loss": 3.49189737950665, + "tokens_seen": 1785069568 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003710651955867603, + "loss": 1.2847, + "theoretical_loss": 3.491886539392052, + "tokens_seen": 1785135104 + }, + { + "epoch": 0.54, + "learning_rate": 0.000371049147442327, + "loss": 1.2675, + "theoretical_loss": 3.491875699786834, + "tokens_seen": 1785200640 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003710330992978937, + "loss": 1.3089, + "theoretical_loss": 3.4918648606909546, + "tokens_seen": 1785266176 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003710170511534604, + "loss": 1.2881, + "theoretical_loss": 3.4918540221043703, + "tokens_seen": 1785331712 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003710010030090271, + "loss": 1.2959, + "theoretical_loss": 3.491843184027039, + "tokens_seen": 1785397248 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003709849548645938, + "loss": 1.4386, + "theoretical_loss": 3.491832346458918, + "tokens_seen": 1785462784 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003709689067201605, + "loss": 1.3127, + "theoretical_loss": 3.4918215093999643, + "tokens_seen": 1785528320 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003709528585757272, + "loss": 1.2915, + "theoretical_loss": 3.4918106728501357, + "tokens_seen": 1785593856 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003709368104312939, + "loss": 1.2447, + "theoretical_loss": 3.4917998368093897, + "tokens_seen": 1785659392 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003709207622868606, + "loss": 1.1769, + "theoretical_loss": 3.491789001277683, + "tokens_seen": 1785724928 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003709047141424273, + "loss": 1.2448, + "theoretical_loss": 3.4917781662549734, + "tokens_seen": 1785790464 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 598857, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3066961765289307, + "objective/train/theoretical_loss": 3.491767331741219, + "objective/train/tokens_used": 1806316000, + "theoretical_loss": 3.491767331741219, + "tokens_seen": 1785856000 + }, + { + "epoch": 0.54, + "learning_rate": 0.000370888665997994, + "loss": 1.2125, + "theoretical_loss": 3.491767331741219, + "tokens_seen": 1785856000 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003708726178535607, + "loss": 1.3361, + "theoretical_loss": 3.491756497736376, + "tokens_seen": 1785921536 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003708565697091274, + "loss": 1.1691, + "theoretical_loss": 3.491745664240403, + "tokens_seen": 1785987072 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003708405215646941, + "loss": 1.1888, + "theoretical_loss": 3.4917348312532566, + "tokens_seen": 1786052608 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003708244734202608, + "loss": 1.2728, + "theoretical_loss": 3.4917239987748943, + "tokens_seen": 1786118144 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003708084252758275, + "loss": 1.2376, + "theoretical_loss": 3.491713166805274, + "tokens_seen": 1786183680 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003707923771313942, + "loss": 1.2818, + "theoretical_loss": 3.4917023353443524, + "tokens_seen": 1786249216 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003707763289869609, + "loss": 1.3009, + "theoretical_loss": 3.4916915043920875, + "tokens_seen": 1786314752 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003707602808425276, + "loss": 1.3365, + "theoretical_loss": 3.491680673948437, + "tokens_seen": 1786380288 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003707442326980943, + "loss": 1.2533, + "theoretical_loss": 3.491669844013358, + "tokens_seen": 1786445824 + }, + { + "epoch": 0.54, + "learning_rate": 0.000370728184553661, + "loss": 1.1861, + "theoretical_loss": 3.4916590145868076, + "tokens_seen": 1786511360 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003707121364092277, + "loss": 1.1946, + "theoretical_loss": 3.491648185668744, + "tokens_seen": 1786576896 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003706960882647944, + "loss": 1.3617, + "theoretical_loss": 3.4916373572591244, + "tokens_seen": 1786642432 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003706800401203611, + "loss": 1.2532, + "theoretical_loss": 3.491626529357906, + "tokens_seen": 1786707968 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003706639919759278, + "loss": 1.2814, + "theoretical_loss": 3.4916157019650464, + "tokens_seen": 1786773504 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003706479438314945, + "loss": 1.311, + "theoretical_loss": 3.4916048750805033, + "tokens_seen": 1786839040 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003706318956870612, + "loss": 1.2634, + "theoretical_loss": 3.4915940487042345, + "tokens_seen": 1786904576 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003706158475426279, + "loss": 1.3685, + "theoretical_loss": 3.491583222836196, + "tokens_seen": 1786970112 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003705997993981946, + "loss": 1.2525, + "theoretical_loss": 3.491572397476347, + "tokens_seen": 1787035648 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003705837512537613, + "loss": 1.3504, + "theoretical_loss": 3.4915615726246445, + "tokens_seen": 1787101184 + }, + { + "epoch": 0.54, + "learning_rate": 0.000370567703109328, + "loss": 1.2234, + "theoretical_loss": 3.4915507482810457, + "tokens_seen": 1787166720 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003705516549648947, + "loss": 1.2395, + "theoretical_loss": 3.4915399244455085, + "tokens_seen": 1787232256 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003705356068204614, + "loss": 1.2051, + "theoretical_loss": 3.4915291011179903, + "tokens_seen": 1787297792 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003705195586760281, + "loss": 1.2563, + "theoretical_loss": 3.491518278298448, + "tokens_seen": 1787363328 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003705035105315948, + "loss": 1.229, + "theoretical_loss": 3.4915074559868398, + "tokens_seen": 1787428864 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003704874623871615, + "loss": 1.3566, + "theoretical_loss": 3.4914966341831235, + "tokens_seen": 1787494400 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003704714142427282, + "loss": 1.2101, + "theoretical_loss": 3.491485812887256, + "tokens_seen": 1787559936 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003704553660982949, + "loss": 1.3985, + "theoretical_loss": 3.491474992099196, + "tokens_seen": 1787625472 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003704393179538616, + "loss": 1.269, + "theoretical_loss": 3.4914641718188992, + "tokens_seen": 1787691008 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003704232698094283, + "loss": 1.2999, + "theoretical_loss": 3.4914533520463245, + "tokens_seen": 1787756544 + }, + { + "epoch": 0.54, + "learning_rate": 0.000370407221664995, + "loss": 1.2251, + "theoretical_loss": 3.491442532781429, + "tokens_seen": 1787822080 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003703911735205617, + "loss": 1.2227, + "theoretical_loss": 3.49143171402417, + "tokens_seen": 1787887616 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003703751253761284, + "loss": 1.2596, + "theoretical_loss": 3.491420895774506, + "tokens_seen": 1787953152 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003703590772316951, + "loss": 1.2984, + "theoretical_loss": 3.491410078032394, + "tokens_seen": 1788018688 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003703430290872618, + "loss": 1.2125, + "theoretical_loss": 3.4913992607977917, + "tokens_seen": 1788084224 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003703269809428285, + "loss": 1.2306, + "theoretical_loss": 3.4913884440706564, + "tokens_seen": 1788149760 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003703109327983952, + "loss": 1.3528, + "theoretical_loss": 3.4913776278509463, + "tokens_seen": 1788215296 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003702948846539619, + "loss": 1.2936, + "theoretical_loss": 3.491366812138618, + "tokens_seen": 1788280832 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003702788365095286, + "loss": 1.2766, + "theoretical_loss": 3.49135599693363, + "tokens_seen": 1788346368 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003702627883650953, + "loss": 1.2442, + "theoretical_loss": 3.4913451822359396, + "tokens_seen": 1788411904 + }, + { + "epoch": 0.54, + "learning_rate": 0.000370246740220662, + "loss": 1.2768, + "theoretical_loss": 3.4913343680455045, + "tokens_seen": 1788477440 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003702306920762287, + "loss": 1.2349, + "theoretical_loss": 3.4913235543622823, + "tokens_seen": 1788542976 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003702146439317954, + "loss": 1.2915, + "theoretical_loss": 3.4913127411862304, + "tokens_seen": 1788608512 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003701985957873621, + "loss": 1.2762, + "theoretical_loss": 3.4913019285173066, + "tokens_seen": 1788674048 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003701825476429288, + "loss": 1.3148, + "theoretical_loss": 3.4912911163554687, + "tokens_seen": 1788739584 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003701664994984955, + "loss": 1.303, + "theoretical_loss": 3.491280304700674, + "tokens_seen": 1788805120 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003701504513540622, + "loss": 1.269, + "theoretical_loss": 3.491269493552881, + "tokens_seen": 1788870656 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003701344032096289, + "loss": 1.2141, + "theoretical_loss": 3.491258682912046, + "tokens_seen": 1788936192 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003701183550651956, + "loss": 1.2542, + "theoretical_loss": 3.491247872778128, + "tokens_seen": 1789001728 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003701023069207623, + "loss": 1.2917, + "theoretical_loss": 3.4912370631510834, + "tokens_seen": 1789067264 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 599715, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.152665138244629, + "objective/train/theoretical_loss": 3.491226254030871, + "objective/train/tokens_used": 1809592800, + "theoretical_loss": 3.491226254030871, + "tokens_seen": 1789132800 + }, + { + "epoch": 0.54, + "learning_rate": 0.000370086258776329, + "loss": 1.1846, + "theoretical_loss": 3.491226254030871, + "tokens_seen": 1789132800 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003700702106318957, + "loss": 1.3383, + "theoretical_loss": 3.4912154454174478, + "tokens_seen": 1789198336 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037005416248746243, + "loss": 1.215, + "theoretical_loss": 3.4912046373107715, + "tokens_seen": 1789263872 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037003811434302913, + "loss": 1.2956, + "theoretical_loss": 3.4911938297108, + "tokens_seen": 1789329408 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037002206619859583, + "loss": 1.248, + "theoretical_loss": 3.491183022617491, + "tokens_seen": 1789394944 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037000601805416253, + "loss": 1.2806, + "theoretical_loss": 3.491172216030802, + "tokens_seen": 1789460480 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036998996990972923, + "loss": 1.2622, + "theoretical_loss": 3.4911614099506907, + "tokens_seen": 1789526016 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036997392176529593, + "loss": 1.2419, + "theoretical_loss": 3.4911506043771148, + "tokens_seen": 1789591552 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036995787362086263, + "loss": 1.2245, + "theoretical_loss": 3.4911397993100324, + "tokens_seen": 1789657088 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003699418254764293, + "loss": 1.3144, + "theoretical_loss": 3.491128994749401, + "tokens_seen": 1789722624 + }, + { + "epoch": 0.54, + "learning_rate": 0.000369925777331996, + "loss": 1.4211, + "theoretical_loss": 3.491118190695178, + "tokens_seen": 1789788160 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003699097291875627, + "loss": 1.366, + "theoretical_loss": 3.491107387147321, + "tokens_seen": 1789853696 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003698936810431294, + "loss": 1.3039, + "theoretical_loss": 3.4910965841057884, + "tokens_seen": 1789919232 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003698776328986961, + "loss": 1.4293, + "theoretical_loss": 3.491085781570538, + "tokens_seen": 1789984768 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003698615847542628, + "loss": 1.2793, + "theoretical_loss": 3.491074979541527, + "tokens_seen": 1790050304 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003698455366098295, + "loss": 1.3377, + "theoretical_loss": 3.4910641780187133, + "tokens_seen": 1790115840 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003698294884653962, + "loss": 1.2425, + "theoretical_loss": 3.4910533770020544, + "tokens_seen": 1790181376 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003698134403209629, + "loss": 1.289, + "theoretical_loss": 3.491042576491508, + "tokens_seen": 1790246912 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003697973921765296, + "loss": 1.3521, + "theoretical_loss": 3.491031776487033, + "tokens_seen": 1790312448 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003697813440320963, + "loss": 1.3469, + "theoretical_loss": 3.491020976988586, + "tokens_seen": 1790377984 + }, + { + "epoch": 0.54, + "learning_rate": 0.000369765295887663, + "loss": 1.2454, + "theoretical_loss": 3.4910101779961247, + "tokens_seen": 1790443520 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003697492477432297, + "loss": 1.1692, + "theoretical_loss": 3.4909993795096077, + "tokens_seen": 1790509056 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003697331995987964, + "loss": 1.3322, + "theoretical_loss": 3.4909885815289923, + "tokens_seen": 1790574592 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003697171514543631, + "loss": 1.2742, + "theoretical_loss": 3.490977784054236, + "tokens_seen": 1790640128 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003697011033099298, + "loss": 1.3179, + "theoretical_loss": 3.4909669870852973, + "tokens_seen": 1790705664 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003696850551654965, + "loss": 1.2333, + "theoretical_loss": 3.490956190622134, + "tokens_seen": 1790771200 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003696690070210632, + "loss": 1.3423, + "theoretical_loss": 3.4909453946647027, + "tokens_seen": 1790836736 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003696529588766299, + "loss": 1.3452, + "theoretical_loss": 3.490934599212962, + "tokens_seen": 1790902272 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003696369107321966, + "loss": 1.2635, + "theoretical_loss": 3.4909238042668704, + "tokens_seen": 1790967808 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003696208625877633, + "loss": 1.3571, + "theoretical_loss": 3.490913009826385, + "tokens_seen": 1791033344 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036960481444333, + "loss": 1.3289, + "theoretical_loss": 3.490902215891463, + "tokens_seen": 1791098880 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003695887662988967, + "loss": 1.2266, + "theoretical_loss": 3.4908914224620635, + "tokens_seen": 1791164416 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003695727181544634, + "loss": 1.3161, + "theoretical_loss": 3.490880629538143, + "tokens_seen": 1791229952 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003695566700100301, + "loss": 1.2959, + "theoretical_loss": 3.4908698371196607, + "tokens_seen": 1791295488 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003695406218655968, + "loss": 1.1958, + "theoretical_loss": 3.490859045206574, + "tokens_seen": 1791361024 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003695245737211635, + "loss": 1.3658, + "theoretical_loss": 3.4908482537988403, + "tokens_seen": 1791426560 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003695085255767302, + "loss": 1.3027, + "theoretical_loss": 3.490837462896417, + "tokens_seen": 1791492096 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003694924774322969, + "loss": 1.3298, + "theoretical_loss": 3.4908266724992636, + "tokens_seen": 1791557632 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003694764292878636, + "loss": 1.2188, + "theoretical_loss": 3.4908158826073366, + "tokens_seen": 1791623168 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003694603811434303, + "loss": 1.2184, + "theoretical_loss": 3.4908050932205943, + "tokens_seen": 1791688704 + }, + { + "epoch": 0.54, + "learning_rate": 0.000369444332998997, + "loss": 1.2216, + "theoretical_loss": 3.490794304338994, + "tokens_seen": 1791754240 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003694282848545637, + "loss": 1.2462, + "theoretical_loss": 3.490783515962495, + "tokens_seen": 1791819776 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003694122367101304, + "loss": 1.2614, + "theoretical_loss": 3.490772728091054, + "tokens_seen": 1791885312 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003693961885656971, + "loss": 1.2888, + "theoretical_loss": 3.490761940724629, + "tokens_seen": 1791950848 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003693801404212638, + "loss": 1.3756, + "theoretical_loss": 3.4907511538631777, + "tokens_seen": 1792016384 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036936409227683055, + "loss": 1.1894, + "theoretical_loss": 3.4907403675066595, + "tokens_seen": 1792081920 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036934804413239726, + "loss": 1.4534, + "theoretical_loss": 3.49072958165503, + "tokens_seen": 1792147456 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036933199598796396, + "loss": 1.3039, + "theoretical_loss": 3.490718796308249, + "tokens_seen": 1792212992 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036931594784353066, + "loss": 1.2747, + "theoretical_loss": 3.490708011466273, + "tokens_seen": 1792278528 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003692998996990973, + "loss": 1.3275, + "theoretical_loss": 3.4906972271290613, + "tokens_seen": 1792344064 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 601014, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2282359600067139, + "objective/train/theoretical_loss": 3.4906864432965707, + "objective/train/tokens_used": 1812869600, + "theoretical_loss": 3.4906864432965707, + "tokens_seen": 1792409600 + }, + { + "epoch": 0.54, + "learning_rate": 0.000369283851554664, + "loss": 1.2805, + "theoretical_loss": 3.4906864432965707, + "tokens_seen": 1792409600 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003692678034102307, + "loss": 1.4108, + "theoretical_loss": 3.4906756599687596, + "tokens_seen": 1792475136 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003692517552657974, + "loss": 1.2863, + "theoretical_loss": 3.490664877145586, + "tokens_seen": 1792540672 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003692357071213641, + "loss": 1.329, + "theoretical_loss": 3.4906540948270077, + "tokens_seen": 1792606208 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003692196589769308, + "loss": 1.3037, + "theoretical_loss": 3.490643313012982, + "tokens_seen": 1792671744 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003692036108324975, + "loss": 1.2022, + "theoretical_loss": 3.4906325317034685, + "tokens_seen": 1792737280 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003691875626880642, + "loss": 1.3753, + "theoretical_loss": 3.4906217508984234, + "tokens_seen": 1792802816 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003691715145436309, + "loss": 1.3744, + "theoretical_loss": 3.490610970597806, + "tokens_seen": 1792868352 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003691554663991976, + "loss": 1.3385, + "theoretical_loss": 3.4906001908015734, + "tokens_seen": 1792933888 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003691394182547643, + "loss": 1.4016, + "theoretical_loss": 3.490589411509683, + "tokens_seen": 1792999424 + }, + { + "epoch": 0.54, + "learning_rate": 0.000369123370110331, + "loss": 1.2883, + "theoretical_loss": 3.490578632722095, + "tokens_seen": 1793064960 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003691073219658977, + "loss": 1.3234, + "theoretical_loss": 3.490567854438765, + "tokens_seen": 1793130496 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003690912738214644, + "loss": 1.3384, + "theoretical_loss": 3.4905570766596523, + "tokens_seen": 1793196032 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003690752256770311, + "loss": 1.4166, + "theoretical_loss": 3.4905462993847145, + "tokens_seen": 1793261568 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003690591775325978, + "loss": 1.2816, + "theoretical_loss": 3.49053552261391, + "tokens_seen": 1793327104 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003690431293881645, + "loss": 1.2996, + "theoretical_loss": 3.4905247463471953, + "tokens_seen": 1793392640 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003690270812437312, + "loss": 1.3015, + "theoretical_loss": 3.490513970584531, + "tokens_seen": 1793458176 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003690110330992979, + "loss": 1.3316, + "theoretical_loss": 3.4905031953258723, + "tokens_seen": 1793523712 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003689949849548646, + "loss": 1.2759, + "theoretical_loss": 3.4904924205711794, + "tokens_seen": 1793589248 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003689789368104313, + "loss": 1.2784, + "theoretical_loss": 3.490481646320409, + "tokens_seen": 1793654784 + }, + { + "epoch": 0.54, + "learning_rate": 0.000368962888665998, + "loss": 1.2901, + "theoretical_loss": 3.4904708725735203, + "tokens_seen": 1793720320 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003689468405215647, + "loss": 1.3297, + "theoretical_loss": 3.4904600993304697, + "tokens_seen": 1793785856 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003689307923771314, + "loss": 1.3194, + "theoretical_loss": 3.4904493265912166, + "tokens_seen": 1793851392 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003689147442326981, + "loss": 1.3115, + "theoretical_loss": 3.4904385543557184, + "tokens_seen": 1793916928 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003688986960882648, + "loss": 1.311, + "theoretical_loss": 3.490427782623933, + "tokens_seen": 1793982464 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003688826479438315, + "loss": 1.2727, + "theoretical_loss": 3.4904170113958193, + "tokens_seen": 1794048000 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003688665997993982, + "loss": 1.2672, + "theoretical_loss": 3.490406240671335, + "tokens_seen": 1794113536 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003688505516549649, + "loss": 1.2585, + "theoretical_loss": 3.4903954704504376, + "tokens_seen": 1794179072 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003688345035105316, + "loss": 1.3117, + "theoretical_loss": 3.4903847007330855, + "tokens_seen": 1794244608 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003688184553660983, + "loss": 1.2732, + "theoretical_loss": 3.490373931519237, + "tokens_seen": 1794310144 + }, + { + "epoch": 0.54, + "learning_rate": 0.000368802407221665, + "loss": 1.2919, + "theoretical_loss": 3.49036316280885, + "tokens_seen": 1794375680 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003687863590772317, + "loss": 1.2599, + "theoretical_loss": 3.4903523946018824, + "tokens_seen": 1794441216 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003687703109327984, + "loss": 1.3624, + "theoretical_loss": 3.4903416268982923, + "tokens_seen": 1794506752 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003687542627883651, + "loss": 1.1819, + "theoretical_loss": 3.490330859698038, + "tokens_seen": 1794572288 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003687382146439318, + "loss": 1.3663, + "theoretical_loss": 3.4903200930010776, + "tokens_seen": 1794637824 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003687221664994985, + "loss": 1.1524, + "theoretical_loss": 3.4903093268073686, + "tokens_seen": 1794703360 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003687061183550652, + "loss": 1.3235, + "theoretical_loss": 3.4902985611168704, + "tokens_seen": 1794768896 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036869007021063193, + "loss": 1.3086, + "theoretical_loss": 3.4902877959295395, + "tokens_seen": 1794834432 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036867402206619863, + "loss": 1.3833, + "theoretical_loss": 3.4902770312453355, + "tokens_seen": 1794899968 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036865797392176533, + "loss": 1.2455, + "theoretical_loss": 3.4902662670642157, + "tokens_seen": 1794965504 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036864192577733203, + "loss": 1.3019, + "theoretical_loss": 3.490255503386138, + "tokens_seen": 1795031040 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036862587763289873, + "loss": 1.3645, + "theoretical_loss": 3.490244740211061, + "tokens_seen": 1795096576 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036860982948846543, + "loss": 1.3092, + "theoretical_loss": 3.490233977538943, + "tokens_seen": 1795162112 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036859378134403213, + "loss": 1.3018, + "theoretical_loss": 3.490223215369742, + "tokens_seen": 1795227648 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036857773319959883, + "loss": 1.2883, + "theoretical_loss": 3.4902124537034154, + "tokens_seen": 1795293184 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036856168505516553, + "loss": 1.2532, + "theoretical_loss": 3.490201692539922, + "tokens_seen": 1795358720 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036854563691073223, + "loss": 1.2823, + "theoretical_loss": 3.49019093187922, + "tokens_seen": 1795424256 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003685295887662989, + "loss": 1.3471, + "theoretical_loss": 3.490180171721268, + "tokens_seen": 1795489792 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003685135406218656, + "loss": 1.3271, + "theoretical_loss": 3.490169412066023, + "tokens_seen": 1795555328 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003684974924774323, + "loss": 1.3352, + "theoretical_loss": 3.4901586529134443, + "tokens_seen": 1795620864 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 601905, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3266921043395996, + "objective/train/theoretical_loss": 3.490147894263489, + "objective/train/tokens_used": 1816146400, + "theoretical_loss": 3.490147894263489, + "tokens_seen": 1795686400 + }, + { + "epoch": 0.54, + "learning_rate": 0.000368481444332999, + "loss": 1.2901, + "theoretical_loss": 3.490147894263489, + "tokens_seen": 1795686400 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003684653961885657, + "loss": 1.1648, + "theoretical_loss": 3.490137136116116, + "tokens_seen": 1795751936 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003684493480441324, + "loss": 1.2923, + "theoretical_loss": 3.4901263784712837, + "tokens_seen": 1795817472 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003684332998996991, + "loss": 1.3094, + "theoretical_loss": 3.4901156213289495, + "tokens_seen": 1795883008 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003684172517552658, + "loss": 1.3039, + "theoretical_loss": 3.490104864689072, + "tokens_seen": 1795948544 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003684012036108325, + "loss": 1.3145, + "theoretical_loss": 3.4900941085516095, + "tokens_seen": 1796014080 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003683851554663992, + "loss": 1.2464, + "theoretical_loss": 3.4900833529165194, + "tokens_seen": 1796079616 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003683691073219659, + "loss": 1.3541, + "theoretical_loss": 3.4900725977837617, + "tokens_seen": 1796145152 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003683530591775326, + "loss": 1.3685, + "theoretical_loss": 3.490061843153293, + "tokens_seen": 1796210688 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003683370110330993, + "loss": 1.3124, + "theoretical_loss": 3.490051089025072, + "tokens_seen": 1796276224 + }, + { + "epoch": 0.54, + "learning_rate": 0.000368320962888666, + "loss": 1.2261, + "theoretical_loss": 3.4900403353990566, + "tokens_seen": 1796341760 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003683049147442327, + "loss": 1.2574, + "theoretical_loss": 3.490029582275206, + "tokens_seen": 1796407296 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003682888665997994, + "loss": 1.1785, + "theoretical_loss": 3.4900188296534775, + "tokens_seen": 1796472832 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003682728184553661, + "loss": 1.2935, + "theoretical_loss": 3.4900080775338296, + "tokens_seen": 1796538368 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003682567703109328, + "loss": 1.2904, + "theoretical_loss": 3.489997325916221, + "tokens_seen": 1796603904 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003682407221664995, + "loss": 1.3337, + "theoretical_loss": 3.489986574800609, + "tokens_seen": 1796669440 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003682246740220662, + "loss": 1.2769, + "theoretical_loss": 3.4899758241869527, + "tokens_seen": 1796734976 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003682086258776329, + "loss": 1.3017, + "theoretical_loss": 3.4899650740752097, + "tokens_seen": 1796800512 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003681925777331996, + "loss": 1.3803, + "theoretical_loss": 3.489954324465339, + "tokens_seen": 1796866048 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003681765295887663, + "loss": 1.3225, + "theoretical_loss": 3.489943575357298, + "tokens_seen": 1796931584 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036816048144433305, + "loss": 1.2369, + "theoretical_loss": 3.489932826751046, + "tokens_seen": 1796997120 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036814443329989975, + "loss": 1.3214, + "theoretical_loss": 3.4899220786465404, + "tokens_seen": 1797062656 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036812838515546645, + "loss": 1.2718, + "theoretical_loss": 3.4899113310437397, + "tokens_seen": 1797128192 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036811233701103315, + "loss": 1.363, + "theoretical_loss": 3.4899005839426023, + "tokens_seen": 1797193728 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036809628886659985, + "loss": 1.227, + "theoretical_loss": 3.4898898373430867, + "tokens_seen": 1797259264 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036808024072216655, + "loss": 1.319, + "theoretical_loss": 3.489879091245151, + "tokens_seen": 1797324800 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036806419257773325, + "loss": 1.3035, + "theoretical_loss": 3.489868345648753, + "tokens_seen": 1797390336 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036804814443329995, + "loss": 1.2971, + "theoretical_loss": 3.4898576005538517, + "tokens_seen": 1797455872 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036803209628886665, + "loss": 1.3193, + "theoretical_loss": 3.4898468559604057, + "tokens_seen": 1797521408 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036801604814443335, + "loss": 1.3089, + "theoretical_loss": 3.4898361118683723, + "tokens_seen": 1797586944 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036800000000000005, + "loss": 1.2996, + "theoretical_loss": 3.4898253682777103, + "tokens_seen": 1797652480 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036798395185556676, + "loss": 1.2571, + "theoretical_loss": 3.4898146251883784, + "tokens_seen": 1797718016 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036796790371113346, + "loss": 1.3091, + "theoretical_loss": 3.4898038826003344, + "tokens_seen": 1797783552 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036795185556670016, + "loss": 1.2822, + "theoretical_loss": 3.4897931405135365, + "tokens_seen": 1797849088 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036793580742226686, + "loss": 1.3579, + "theoretical_loss": 3.489782398927944, + "tokens_seen": 1797914624 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036791975927783356, + "loss": 1.3209, + "theoretical_loss": 3.489771657843514, + "tokens_seen": 1797980160 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003679037111334002, + "loss": 1.2417, + "theoretical_loss": 3.489760917260206, + "tokens_seen": 1798045696 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003678876629889669, + "loss": 1.2935, + "theoretical_loss": 3.489750177177977, + "tokens_seen": 1798111232 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003678716148445336, + "loss": 1.2485, + "theoretical_loss": 3.4897394375967874, + "tokens_seen": 1798176768 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003678555667001003, + "loss": 1.2728, + "theoretical_loss": 3.4897286985165934, + "tokens_seen": 1798242304 + }, + { + "epoch": 0.54, + "learning_rate": 0.000367839518555667, + "loss": 1.4074, + "theoretical_loss": 3.4897179599373547, + "tokens_seen": 1798307840 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003678234704112337, + "loss": 1.3086, + "theoretical_loss": 3.489707221859029, + "tokens_seen": 1798373376 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003678074222668004, + "loss": 1.3222, + "theoretical_loss": 3.489696484281575, + "tokens_seen": 1798438912 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003677913741223671, + "loss": 1.2609, + "theoretical_loss": 3.4896857472049514, + "tokens_seen": 1798504448 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003677753259779338, + "loss": 1.2677, + "theoretical_loss": 3.4896750106291163, + "tokens_seen": 1798569984 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003677592778335005, + "loss": 1.3476, + "theoretical_loss": 3.489664274554028, + "tokens_seen": 1798635520 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003677432296890672, + "loss": 1.3758, + "theoretical_loss": 3.4896535389796446, + "tokens_seen": 1798701056 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003677271815446339, + "loss": 1.2741, + "theoretical_loss": 3.489642803905925, + "tokens_seen": 1798766592 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003677111334002006, + "loss": 1.3326, + "theoretical_loss": 3.4896320693328278, + "tokens_seen": 1798832128 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003676950852557673, + "loss": 1.3004, + "theoretical_loss": 3.4896213352603107, + "tokens_seen": 1798897664 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 603132, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2088148593902588, + "objective/train/theoretical_loss": 3.489610601688333, + "objective/train/tokens_used": 1819423200, + "theoretical_loss": 3.489610601688333, + "tokens_seen": 1798963200 + }, + { + "epoch": 0.55, + "learning_rate": 0.000367679037111334, + "loss": 1.2736, + "theoretical_loss": 3.489610601688333, + "tokens_seen": 1798963200 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003676629889669007, + "loss": 1.2503, + "theoretical_loss": 3.4895998686168515, + "tokens_seen": 1799028736 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003676469408224674, + "loss": 1.4063, + "theoretical_loss": 3.4895891360458267, + "tokens_seen": 1799094272 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003676308926780341, + "loss": 1.3199, + "theoretical_loss": 3.489578403975216, + "tokens_seen": 1799159808 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003676148445336008, + "loss": 1.2766, + "theoretical_loss": 3.489567672404978, + "tokens_seen": 1799225344 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003675987963891675, + "loss": 1.2983, + "theoretical_loss": 3.4895569413350707, + "tokens_seen": 1799290880 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003675827482447342, + "loss": 1.3561, + "theoretical_loss": 3.4895462107654533, + "tokens_seen": 1799356416 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003675667001003009, + "loss": 1.2474, + "theoretical_loss": 3.489535480696084, + "tokens_seen": 1799421952 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003675506519558676, + "loss": 1.3403, + "theoretical_loss": 3.489524751126921, + "tokens_seen": 1799487488 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003675346038114343, + "loss": 1.2994, + "theoretical_loss": 3.4895140220579224, + "tokens_seen": 1799553024 + }, + { + "epoch": 0.55, + "learning_rate": 0.000367518555667001, + "loss": 1.1748, + "theoretical_loss": 3.4895032934890478, + "tokens_seen": 1799618560 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003675025075225677, + "loss": 1.2566, + "theoretical_loss": 3.489492565420255, + "tokens_seen": 1799684096 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003674864593781344, + "loss": 1.2915, + "theoretical_loss": 3.4894818378515025, + "tokens_seen": 1799749632 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003674704112337011, + "loss": 1.2793, + "theoretical_loss": 3.489471110782749, + "tokens_seen": 1799815168 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003674543630892678, + "loss": 1.2856, + "theoretical_loss": 3.4894603842139524, + "tokens_seen": 1799880704 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003674383149448345, + "loss": 1.2609, + "theoretical_loss": 3.489449658145072, + "tokens_seen": 1799946240 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003674222668004012, + "loss": 1.2692, + "theoretical_loss": 3.489438932576066, + "tokens_seen": 1800011776 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003674062186559679, + "loss": 1.2663, + "theoretical_loss": 3.4894282075068923, + "tokens_seen": 1800077312 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003673901705115346, + "loss": 1.2696, + "theoretical_loss": 3.48941748293751, + "tokens_seen": 1800142848 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003673741223671013, + "loss": 1.2318, + "theoretical_loss": 3.489406758867878, + "tokens_seen": 1800208384 + }, + { + "epoch": 0.55, + "learning_rate": 0.000367358074222668, + "loss": 1.2533, + "theoretical_loss": 3.489396035297954, + "tokens_seen": 1800273920 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003673420260782347, + "loss": 1.1788, + "theoretical_loss": 3.4893853122276974, + "tokens_seen": 1800339456 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003673259779338014, + "loss": 1.1501, + "theoretical_loss": 3.489374589657066, + "tokens_seen": 1800404992 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036730992978936813, + "loss": 1.2524, + "theoretical_loss": 3.489363867586018, + "tokens_seen": 1800470528 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036729388164493483, + "loss": 1.3698, + "theoretical_loss": 3.4893531460145133, + "tokens_seen": 1800536064 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036727783350050153, + "loss": 1.2802, + "theoretical_loss": 3.4893424249425093, + "tokens_seen": 1800601600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036726178535606823, + "loss": 1.2505, + "theoretical_loss": 3.489331704369965, + "tokens_seen": 1800667136 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036724573721163493, + "loss": 1.2847, + "theoretical_loss": 3.489320984296839, + "tokens_seen": 1800732672 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036722968906720163, + "loss": 1.299, + "theoretical_loss": 3.4893102647230894, + "tokens_seen": 1800798208 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036721364092276833, + "loss": 1.2856, + "theoretical_loss": 3.4892995456486746, + "tokens_seen": 1800863744 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036719759277833503, + "loss": 1.2903, + "theoretical_loss": 3.4892888270735547, + "tokens_seen": 1800929280 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036718154463390173, + "loss": 1.2459, + "theoretical_loss": 3.4892781089976865, + "tokens_seen": 1800994816 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036716549648946843, + "loss": 1.3398, + "theoretical_loss": 3.4892673914210297, + "tokens_seen": 1801060352 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036714944834503513, + "loss": 1.2801, + "theoretical_loss": 3.489256674343542, + "tokens_seen": 1801125888 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003671334002006018, + "loss": 1.3084, + "theoretical_loss": 3.489245957765183, + "tokens_seen": 1801191424 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003671173520561685, + "loss": 1.357, + "theoretical_loss": 3.4892352416859107, + "tokens_seen": 1801256960 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003671013039117352, + "loss": 1.3455, + "theoretical_loss": 3.4892245261056836, + "tokens_seen": 1801322496 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003670852557673019, + "loss": 1.2824, + "theoretical_loss": 3.4892138110244604, + "tokens_seen": 1801388032 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003670692076228686, + "loss": 1.3372, + "theoretical_loss": 3.4892030964422003, + "tokens_seen": 1801453568 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003670531594784353, + "loss": 1.3567, + "theoretical_loss": 3.489192382358861, + "tokens_seen": 1801519104 + }, + { + "epoch": 0.55, + "learning_rate": 0.000367037111334002, + "loss": 1.3404, + "theoretical_loss": 3.4891816687744015, + "tokens_seen": 1801584640 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003670210631895687, + "loss": 1.4004, + "theoretical_loss": 3.48917095568878, + "tokens_seen": 1801650176 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003670050150451354, + "loss": 1.3492, + "theoretical_loss": 3.4891602431019564, + "tokens_seen": 1801715712 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003669889669007021, + "loss": 1.2796, + "theoretical_loss": 3.4891495310138883, + "tokens_seen": 1801781248 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003669729187562688, + "loss": 1.2911, + "theoretical_loss": 3.4891388194245345, + "tokens_seen": 1801846784 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036695687061183554, + "loss": 1.2793, + "theoretical_loss": 3.489128108333854, + "tokens_seen": 1801912320 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036694082246740224, + "loss": 1.2687, + "theoretical_loss": 3.489117397741804, + "tokens_seen": 1801977856 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036692477432296894, + "loss": 1.3156, + "theoretical_loss": 3.4891066876483454, + "tokens_seen": 1802043392 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036690872617853564, + "loss": 1.2252, + "theoretical_loss": 3.4890959780534354, + "tokens_seen": 1802108928 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036689267803410234, + "loss": 1.2341, + "theoretical_loss": 3.489085268957033, + "tokens_seen": 1802174464 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 604041, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2103506326675415, + "objective/train/theoretical_loss": 3.489074560359097, + "objective/train/tokens_used": 1822700000, + "theoretical_loss": 3.489074560359097, + "tokens_seen": 1802240000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036687662988966904, + "loss": 1.221, + "theoretical_loss": 3.489074560359097, + "tokens_seen": 1802240000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036686058174523575, + "loss": 1.2555, + "theoretical_loss": 3.489063852259586, + "tokens_seen": 1802305536 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036684453360080245, + "loss": 1.3352, + "theoretical_loss": 3.4890531446584583, + "tokens_seen": 1802371072 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036682848545636915, + "loss": 1.3494, + "theoretical_loss": 3.489042437555673, + "tokens_seen": 1802436608 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036681243731193585, + "loss": 1.3403, + "theoretical_loss": 3.489031730951189, + "tokens_seen": 1802502144 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036679638916750255, + "loss": 1.2712, + "theoretical_loss": 3.4890210248449645, + "tokens_seen": 1802567680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036678034102306925, + "loss": 1.2242, + "theoretical_loss": 3.489010319236958, + "tokens_seen": 1802633216 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036676429287863595, + "loss": 1.2683, + "theoretical_loss": 3.4889996141271293, + "tokens_seen": 1802698752 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036674824473420265, + "loss": 1.2561, + "theoretical_loss": 3.4889889095154363, + "tokens_seen": 1802764288 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036673219658976935, + "loss": 1.3386, + "theoretical_loss": 3.4889782054018377, + "tokens_seen": 1802829824 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036671614844533605, + "loss": 1.1877, + "theoretical_loss": 3.4889675017862922, + "tokens_seen": 1802895360 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036670010030090275, + "loss": 1.4449, + "theoretical_loss": 3.4889567986687586, + "tokens_seen": 1802960896 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036668405215646945, + "loss": 1.2252, + "theoretical_loss": 3.488946096049196, + "tokens_seen": 1803026432 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036666800401203615, + "loss": 1.3736, + "theoretical_loss": 3.488935393927563, + "tokens_seen": 1803091968 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036665195586760285, + "loss": 1.3421, + "theoretical_loss": 3.4889246923038177, + "tokens_seen": 1803157504 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036663590772316955, + "loss": 1.2742, + "theoretical_loss": 3.4889139911779194, + "tokens_seen": 1803223040 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036661985957873625, + "loss": 1.3049, + "theoretical_loss": 3.4889032905498274, + "tokens_seen": 1803288576 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036660381143430296, + "loss": 1.2682, + "theoretical_loss": 3.488892590419499, + "tokens_seen": 1803354112 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036658776328986966, + "loss": 1.2359, + "theoretical_loss": 3.488881890786894, + "tokens_seen": 1803419648 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036657171514543636, + "loss": 1.2045, + "theoretical_loss": 3.488871191651971, + "tokens_seen": 1803485184 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036655566700100306, + "loss": 1.2634, + "theoretical_loss": 3.4888604930146885, + "tokens_seen": 1803550720 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036653961885656976, + "loss": 1.2235, + "theoretical_loss": 3.4888497948750055, + "tokens_seen": 1803616256 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036652357071213646, + "loss": 1.2785, + "theoretical_loss": 3.488839097232881, + "tokens_seen": 1803681792 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003665075225677031, + "loss": 1.2603, + "theoretical_loss": 3.4888284000882734, + "tokens_seen": 1803747328 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003664914744232698, + "loss": 1.2322, + "theoretical_loss": 3.488817703441141, + "tokens_seen": 1803812864 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003664754262788365, + "loss": 1.1507, + "theoretical_loss": 3.4888070072914443, + "tokens_seen": 1803878400 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003664593781344032, + "loss": 1.2894, + "theoretical_loss": 3.48879631163914, + "tokens_seen": 1803943936 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003664433299899699, + "loss": 1.296, + "theoretical_loss": 3.4887856164841886, + "tokens_seen": 1804009472 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003664272818455366, + "loss": 1.2533, + "theoretical_loss": 3.488774921826548, + "tokens_seen": 1804075008 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003664112337011033, + "loss": 1.2278, + "theoretical_loss": 3.488764227666177, + "tokens_seen": 1804140544 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036639518555667, + "loss": 1.1999, + "theoretical_loss": 3.4887535340030347, + "tokens_seen": 1804206080 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003663791374122367, + "loss": 1.3648, + "theoretical_loss": 3.48874284083708, + "tokens_seen": 1804271616 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003663630892678034, + "loss": 1.2293, + "theoretical_loss": 3.488732148168272, + "tokens_seen": 1804337152 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003663470411233701, + "loss": 1.2551, + "theoretical_loss": 3.4887214559965685, + "tokens_seen": 1804402688 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003663309929789368, + "loss": 1.2348, + "theoretical_loss": 3.488710764321929, + "tokens_seen": 1804468224 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003663149448345035, + "loss": 1.3269, + "theoretical_loss": 3.4887000731443125, + "tokens_seen": 1804533760 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003662988966900702, + "loss": 1.1643, + "theoretical_loss": 3.4886893824636775, + "tokens_seen": 1804599296 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003662828485456369, + "loss": 1.2309, + "theoretical_loss": 3.488678692279983, + "tokens_seen": 1804664832 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003662668004012036, + "loss": 1.1484, + "theoretical_loss": 3.4886680025931875, + "tokens_seen": 1804730368 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003662507522567703, + "loss": 1.2447, + "theoretical_loss": 3.488657313403251, + "tokens_seen": 1804795904 + }, + { + "epoch": 0.55, + "learning_rate": 0.000366234704112337, + "loss": 1.2493, + "theoretical_loss": 3.488646624710131, + "tokens_seen": 1804861440 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003662186559679037, + "loss": 1.2746, + "theoretical_loss": 3.488635936513787, + "tokens_seen": 1804926976 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003662026078234704, + "loss": 1.2255, + "theoretical_loss": 3.488625248814178, + "tokens_seen": 1804992512 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003661865596790371, + "loss": 1.2239, + "theoretical_loss": 3.4886145616112625, + "tokens_seen": 1805058048 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003661705115346038, + "loss": 1.2817, + "theoretical_loss": 3.4886038749049995, + "tokens_seen": 1805123584 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003661544633901705, + "loss": 1.1934, + "theoretical_loss": 3.488593188695348, + "tokens_seen": 1805189120 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003661384152457372, + "loss": 1.165, + "theoretical_loss": 3.488582502982267, + "tokens_seen": 1805254656 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003661223671013039, + "loss": 1.3509, + "theoretical_loss": 3.4885718177657155, + "tokens_seen": 1805320192 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003661063189568706, + "loss": 1.2895, + "theoretical_loss": 3.4885611330456516, + "tokens_seen": 1805385728 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003660902708124373, + "loss": 1.3409, + "theoretical_loss": 3.488550448822035, + "tokens_seen": 1805451264 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 605221, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3439946174621582, + "objective/train/theoretical_loss": 3.4885397650948247, + "objective/train/tokens_used": 1825976800, + "theoretical_loss": 3.4885397650948247, + "tokens_seen": 1805516800 + }, + { + "epoch": 0.55, + "learning_rate": 0.000366074222668004, + "loss": 1.3831, + "theoretical_loss": 3.4885397650948247, + "tokens_seen": 1805516800 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003660581745235707, + "loss": 1.1929, + "theoretical_loss": 3.488529081863979, + "tokens_seen": 1805582336 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003660421263791374, + "loss": 1.1917, + "theoretical_loss": 3.488518399129457, + "tokens_seen": 1805647872 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003660260782347041, + "loss": 1.2, + "theoretical_loss": 3.4885077168912177, + "tokens_seen": 1805713408 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003660100300902708, + "loss": 1.172, + "theoretical_loss": 3.4884970351492206, + "tokens_seen": 1805778944 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003659939819458375, + "loss": 1.1592, + "theoretical_loss": 3.488486353903424, + "tokens_seen": 1805844480 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003659779338014042, + "loss": 1.2067, + "theoretical_loss": 3.4884756731537867, + "tokens_seen": 1805910016 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003659618856569709, + "loss": 1.2015, + "theoretical_loss": 3.4884649929002682, + "tokens_seen": 1805975552 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036594583751253763, + "loss": 1.3029, + "theoretical_loss": 3.488454313142827, + "tokens_seen": 1806041088 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036592978936810433, + "loss": 1.2363, + "theoretical_loss": 3.488443633881422, + "tokens_seen": 1806106624 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036591374122367103, + "loss": 1.3062, + "theoretical_loss": 3.488432955116013, + "tokens_seen": 1806172160 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036589769307923773, + "loss": 1.2663, + "theoretical_loss": 3.488422276846558, + "tokens_seen": 1806237696 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036588164493480443, + "loss": 1.2229, + "theoretical_loss": 3.4884115990730162, + "tokens_seen": 1806303232 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036586559679037113, + "loss": 1.2928, + "theoretical_loss": 3.488400921795347, + "tokens_seen": 1806368768 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036584954864593783, + "loss": 1.3814, + "theoretical_loss": 3.488390245013509, + "tokens_seen": 1806434304 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036583350050150453, + "loss": 1.3392, + "theoretical_loss": 3.488379568727461, + "tokens_seen": 1806499840 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036581745235707123, + "loss": 1.2796, + "theoretical_loss": 3.488368892937163, + "tokens_seen": 1806565376 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036580140421263793, + "loss": 1.3161, + "theoretical_loss": 3.4883582176425723, + "tokens_seen": 1806630912 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036578535606820463, + "loss": 1.2214, + "theoretical_loss": 3.4883475428436492, + "tokens_seen": 1806696448 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036576930792377133, + "loss": 1.3398, + "theoretical_loss": 3.488336868540353, + "tokens_seen": 1806761984 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036575325977933803, + "loss": 1.1897, + "theoretical_loss": 3.4883261947326414, + "tokens_seen": 1806827520 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036573721163490474, + "loss": 1.2651, + "theoretical_loss": 3.488315521420475, + "tokens_seen": 1806893056 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036572116349047144, + "loss": 1.2436, + "theoretical_loss": 3.4883048486038106, + "tokens_seen": 1806958592 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036570511534603814, + "loss": 1.2633, + "theoretical_loss": 3.4882941762826096, + "tokens_seen": 1807024128 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036568906720160484, + "loss": 1.3836, + "theoretical_loss": 3.4882835044568297, + "tokens_seen": 1807089664 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036567301905717154, + "loss": 1.2462, + "theoretical_loss": 3.4882728331264303, + "tokens_seen": 1807155200 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036565697091273824, + "loss": 1.2729, + "theoretical_loss": 3.4882621622913703, + "tokens_seen": 1807220736 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036564092276830494, + "loss": 1.208, + "theoretical_loss": 3.4882514919516088, + "tokens_seen": 1807286272 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036562487462387164, + "loss": 1.2772, + "theoretical_loss": 3.488240822107105, + "tokens_seen": 1807351808 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036560882647943834, + "loss": 1.258, + "theoretical_loss": 3.4882301527578177, + "tokens_seen": 1807417344 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036559277833500504, + "loss": 1.2259, + "theoretical_loss": 3.4882194839037055, + "tokens_seen": 1807482880 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036557673019057174, + "loss": 1.1876, + "theoretical_loss": 3.488208815544729, + "tokens_seen": 1807548416 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036556068204613844, + "loss": 1.2081, + "theoretical_loss": 3.488198147680846, + "tokens_seen": 1807613952 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036554463390170514, + "loss": 1.3262, + "theoretical_loss": 3.488187480312016, + "tokens_seen": 1807679488 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036552858575727184, + "loss": 1.3382, + "theoretical_loss": 3.488176813438198, + "tokens_seen": 1807745024 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036551253761283854, + "loss": 1.1967, + "theoretical_loss": 3.48816614705935, + "tokens_seen": 1807810560 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036549648946840525, + "loss": 1.3265, + "theoretical_loss": 3.4881554811754336, + "tokens_seen": 1807876096 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036548044132397195, + "loss": 1.2275, + "theoretical_loss": 3.4881448157864057, + "tokens_seen": 1807941632 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036546439317953865, + "loss": 1.1881, + "theoretical_loss": 3.4881341508922263, + "tokens_seen": 1808007168 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036544834503510535, + "loss": 1.2415, + "theoretical_loss": 3.4881234864928543, + "tokens_seen": 1808072704 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036543229689067205, + "loss": 1.2662, + "theoretical_loss": 3.488112822588249, + "tokens_seen": 1808138240 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036541624874623875, + "loss": 1.3863, + "theoretical_loss": 3.488102159178369, + "tokens_seen": 1808203776 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036540020060180545, + "loss": 1.3339, + "theoretical_loss": 3.4880914962631744, + "tokens_seen": 1808269312 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036538415245737215, + "loss": 1.2851, + "theoretical_loss": 3.4880808338426235, + "tokens_seen": 1808334848 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036536810431293885, + "loss": 1.142, + "theoretical_loss": 3.488070171916675, + "tokens_seen": 1808400384 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036535205616850555, + "loss": 1.3827, + "theoretical_loss": 3.4880595104852894, + "tokens_seen": 1808465920 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036533600802407225, + "loss": 1.3717, + "theoretical_loss": 3.488048849548425, + "tokens_seen": 1808531456 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036531995987963895, + "loss": 1.3093, + "theoretical_loss": 3.488038189106041, + "tokens_seen": 1808596992 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036530391173520565, + "loss": 1.3356, + "theoretical_loss": 3.4880275291580967, + "tokens_seen": 1808662528 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036528786359077235, + "loss": 1.254, + "theoretical_loss": 3.488016869704551, + "tokens_seen": 1808728064 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 606156, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4112968444824219, + "objective/train/theoretical_loss": 3.488006210745363, + "objective/train/tokens_used": 1829253600, + "theoretical_loss": 3.488006210745363, + "tokens_seen": 1808793600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036527181544633905, + "loss": 1.3484, + "theoretical_loss": 3.488006210745363, + "tokens_seen": 1808793600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036525576730190575, + "loss": 1.2864, + "theoretical_loss": 3.4879955522804926, + "tokens_seen": 1808859136 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036523971915747246, + "loss": 1.3229, + "theoretical_loss": 3.487984894309898, + "tokens_seen": 1808924672 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036522367101303916, + "loss": 1.2487, + "theoretical_loss": 3.487974236833539, + "tokens_seen": 1808990208 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036520762286860586, + "loss": 1.1661, + "theoretical_loss": 3.4879635798513746, + "tokens_seen": 1809055744 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036519157472417256, + "loss": 1.1866, + "theoretical_loss": 3.4879529233633644, + "tokens_seen": 1809121280 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036517552657973926, + "loss": 1.2133, + "theoretical_loss": 3.4879422673694664, + "tokens_seen": 1809186816 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036515947843530596, + "loss": 1.2459, + "theoretical_loss": 3.487931611869641, + "tokens_seen": 1809252352 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036514343029087266, + "loss": 1.2336, + "theoretical_loss": 3.4879209568638467, + "tokens_seen": 1809317888 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036512738214643936, + "loss": 1.1863, + "theoretical_loss": 3.4879103023520432, + "tokens_seen": 1809383424 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036511133400200606, + "loss": 1.2961, + "theoretical_loss": 3.4878996483341895, + "tokens_seen": 1809448960 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003650952858575727, + "loss": 1.1978, + "theoretical_loss": 3.4878889948102447, + "tokens_seen": 1809514496 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003650792377131394, + "loss": 1.2247, + "theoretical_loss": 3.487878341780168, + "tokens_seen": 1809580032 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003650631895687061, + "loss": 1.1912, + "theoretical_loss": 3.4878676892439184, + "tokens_seen": 1809645568 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003650471414242728, + "loss": 1.2955, + "theoretical_loss": 3.487857037201456, + "tokens_seen": 1809711104 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003650310932798395, + "loss": 1.192, + "theoretical_loss": 3.4878463856527393, + "tokens_seen": 1809776640 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003650150451354062, + "loss": 1.3272, + "theoretical_loss": 3.487835734597728, + "tokens_seen": 1809842176 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003649989969909729, + "loss": 1.3164, + "theoretical_loss": 3.4878250840363805, + "tokens_seen": 1809907712 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003649829488465396, + "loss": 1.2639, + "theoretical_loss": 3.4878144339686568, + "tokens_seen": 1809973248 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003649669007021063, + "loss": 1.36, + "theoretical_loss": 3.487803784394516, + "tokens_seen": 1810038784 + }, + { + "epoch": 0.55, + "learning_rate": 0.000364950852557673, + "loss": 1.3253, + "theoretical_loss": 3.4877931353139173, + "tokens_seen": 1810104320 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003649348044132397, + "loss": 1.2444, + "theoretical_loss": 3.4877824867268195, + "tokens_seen": 1810169856 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003649187562688064, + "loss": 1.3384, + "theoretical_loss": 3.487771838633183, + "tokens_seen": 1810235392 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003649027081243731, + "loss": 1.3045, + "theoretical_loss": 3.4877611910329662, + "tokens_seen": 1810300928 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003648866599799398, + "loss": 1.3564, + "theoretical_loss": 3.4877505439261283, + "tokens_seen": 1810366464 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003648706118355065, + "loss": 1.2878, + "theoretical_loss": 3.487739897312629, + "tokens_seen": 1810432000 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003648545636910732, + "loss": 1.2484, + "theoretical_loss": 3.487729251192427, + "tokens_seen": 1810497536 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003648385155466399, + "loss": 1.2649, + "theoretical_loss": 3.4877186055654823, + "tokens_seen": 1810563072 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003648224674022066, + "loss": 1.2481, + "theoretical_loss": 3.487707960431754, + "tokens_seen": 1810628608 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003648064192577733, + "loss": 1.3836, + "theoretical_loss": 3.487697315791202, + "tokens_seen": 1810694144 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036479037111334, + "loss": 1.3128, + "theoretical_loss": 3.4876866716437838, + "tokens_seen": 1810759680 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003647743229689067, + "loss": 1.3066, + "theoretical_loss": 3.48767602798946, + "tokens_seen": 1810825216 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003647582748244734, + "loss": 1.2664, + "theoretical_loss": 3.4876653848281904, + "tokens_seen": 1810890752 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003647422266800401, + "loss": 1.3305, + "theoretical_loss": 3.4876547421599327, + "tokens_seen": 1810956288 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003647261785356068, + "loss": 1.368, + "theoretical_loss": 3.4876440999846476, + "tokens_seen": 1811021824 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003647101303911735, + "loss": 1.2373, + "theoretical_loss": 3.4876334583022945, + "tokens_seen": 1811087360 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003646940822467402, + "loss": 1.2326, + "theoretical_loss": 3.4876228171128316, + "tokens_seen": 1811152896 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003646780341023069, + "loss": 1.3704, + "theoretical_loss": 3.487612176416219, + "tokens_seen": 1811218432 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003646619859578736, + "loss": 1.3057, + "theoretical_loss": 3.487601536212416, + "tokens_seen": 1811283968 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003646459378134403, + "loss": 1.2593, + "theoretical_loss": 3.4875908965013815, + "tokens_seen": 1811349504 + }, + { + "epoch": 0.55, + "learning_rate": 0.000364629889669007, + "loss": 1.316, + "theoretical_loss": 3.487580257283076, + "tokens_seen": 1811415040 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003646138415245737, + "loss": 1.3584, + "theoretical_loss": 3.4875696185574574, + "tokens_seen": 1811480576 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003645977933801404, + "loss": 1.2604, + "theoretical_loss": 3.4875589803244855, + "tokens_seen": 1811546112 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003645817452357071, + "loss": 1.2751, + "theoretical_loss": 3.4875483425841205, + "tokens_seen": 1811611648 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036456569709127383, + "loss": 1.2514, + "theoretical_loss": 3.487537705336321, + "tokens_seen": 1811677184 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003645496489468406, + "loss": 1.2563, + "theoretical_loss": 3.4875270685810467, + "tokens_seen": 1811742720 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003645336008024073, + "loss": 1.3224, + "theoretical_loss": 3.487516432318256, + "tokens_seen": 1811808256 + }, + { + "epoch": 0.55, + "learning_rate": 0.000364517552657974, + "loss": 1.2943, + "theoretical_loss": 3.48750579654791, + "tokens_seen": 1811873792 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003645015045135407, + "loss": 1.2754, + "theoretical_loss": 3.4874951612699667, + "tokens_seen": 1811939328 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003644854563691074, + "loss": 1.1284, + "theoretical_loss": 3.487484526484386, + "tokens_seen": 1812004864 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 607421, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2716153860092163, + "objective/train/theoretical_loss": 3.4874738921911277, + "objective/train/tokens_used": 1832530400, + "theoretical_loss": 3.4874738921911277, + "tokens_seen": 1812070400 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036446940822467403, + "loss": 1.3402, + "theoretical_loss": 3.4874738921911277, + "tokens_seen": 1812070400 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036445336008024073, + "loss": 1.2999, + "theoretical_loss": 3.48746325839015, + "tokens_seen": 1812135936 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036443731193580743, + "loss": 1.1687, + "theoretical_loss": 3.4874526250814144, + "tokens_seen": 1812201472 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036442126379137413, + "loss": 1.2976, + "theoretical_loss": 3.487441992264878, + "tokens_seen": 1812267008 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036440521564694083, + "loss": 1.311, + "theoretical_loss": 3.4874313599405014, + "tokens_seen": 1812332544 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036438916750250753, + "loss": 1.2737, + "theoretical_loss": 3.4874207281082437, + "tokens_seen": 1812398080 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036437311935807424, + "loss": 1.2597, + "theoretical_loss": 3.4874100967680643, + "tokens_seen": 1812463616 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036435707121364094, + "loss": 1.3513, + "theoretical_loss": 3.487399465919924, + "tokens_seen": 1812529152 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036434102306920764, + "loss": 1.3616, + "theoretical_loss": 3.4873888355637797, + "tokens_seen": 1812594688 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036432497492477434, + "loss": 1.2454, + "theoretical_loss": 3.487378205699593, + "tokens_seen": 1812660224 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036430892678034104, + "loss": 1.2765, + "theoretical_loss": 3.487367576327322, + "tokens_seen": 1812725760 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036429287863590774, + "loss": 1.2213, + "theoretical_loss": 3.4873569474469273, + "tokens_seen": 1812791296 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036427683049147444, + "loss": 1.3066, + "theoretical_loss": 3.487346319058367, + "tokens_seen": 1812856832 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036426078234704114, + "loss": 1.2707, + "theoretical_loss": 3.4873356911616016, + "tokens_seen": 1812922368 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036424473420260784, + "loss": 1.247, + "theoretical_loss": 3.4873250637565905, + "tokens_seen": 1812987904 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036422868605817454, + "loss": 1.3099, + "theoretical_loss": 3.4873144368432927, + "tokens_seen": 1813053440 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036421263791374124, + "loss": 1.255, + "theoretical_loss": 3.4873038104216683, + "tokens_seen": 1813118976 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036419658976930794, + "loss": 1.1848, + "theoretical_loss": 3.4872931844916764, + "tokens_seen": 1813184512 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036418054162487464, + "loss": 1.247, + "theoretical_loss": 3.487282559053276, + "tokens_seen": 1813250048 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036416449348044134, + "loss": 1.2147, + "theoretical_loss": 3.4872719341064276, + "tokens_seen": 1813315584 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036414844533600804, + "loss": 1.2911, + "theoretical_loss": 3.48726130965109, + "tokens_seen": 1813381120 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036413239719157474, + "loss": 1.3377, + "theoretical_loss": 3.487250685687223, + "tokens_seen": 1813446656 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036411634904714145, + "loss": 1.2274, + "theoretical_loss": 3.4872400622147857, + "tokens_seen": 1813512192 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036410030090270815, + "loss": 1.3628, + "theoretical_loss": 3.487229439233738, + "tokens_seen": 1813577728 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036408425275827485, + "loss": 1.1811, + "theoretical_loss": 3.487218816744039, + "tokens_seen": 1813643264 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036406820461384155, + "loss": 1.3189, + "theoretical_loss": 3.4872081947456492, + "tokens_seen": 1813708800 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036405215646940825, + "loss": 1.1791, + "theoretical_loss": 3.487197573238527, + "tokens_seen": 1813774336 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036403610832497495, + "loss": 1.4073, + "theoretical_loss": 3.4871869522226326, + "tokens_seen": 1813839872 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036402006018054165, + "loss": 1.268, + "theoretical_loss": 3.487176331697925, + "tokens_seen": 1813905408 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036400401203610835, + "loss": 1.3516, + "theoretical_loss": 3.487165711664365, + "tokens_seen": 1813970944 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036398796389167505, + "loss": 1.2671, + "theoretical_loss": 3.48715509212191, + "tokens_seen": 1814036480 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036397191574724175, + "loss": 1.2873, + "theoretical_loss": 3.4871444730705212, + "tokens_seen": 1814102016 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036395586760280845, + "loss": 1.2495, + "theoretical_loss": 3.487133854510158, + "tokens_seen": 1814167552 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036393981945837515, + "loss": 1.2243, + "theoretical_loss": 3.4871232364407794, + "tokens_seen": 1814233088 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036392377131394185, + "loss": 1.256, + "theoretical_loss": 3.487112618862345, + "tokens_seen": 1814298624 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036390772316950855, + "loss": 1.2645, + "theoretical_loss": 3.487102001774815, + "tokens_seen": 1814364160 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036389167502507525, + "loss": 1.2762, + "theoretical_loss": 3.4870913851781484, + "tokens_seen": 1814429696 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036387562688064195, + "loss": 1.2648, + "theoretical_loss": 3.487080769072305, + "tokens_seen": 1814495232 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036385957873620866, + "loss": 1.2338, + "theoretical_loss": 3.4870701534572444, + "tokens_seen": 1814560768 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036384353059177536, + "loss": 1.3548, + "theoretical_loss": 3.487059538332926, + "tokens_seen": 1814626304 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036382748244734206, + "loss": 1.3588, + "theoretical_loss": 3.4870489236993096, + "tokens_seen": 1814691840 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036381143430290876, + "loss": 1.3121, + "theoretical_loss": 3.4870383095563544, + "tokens_seen": 1814757376 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036379538615847546, + "loss": 1.3529, + "theoretical_loss": 3.487027695904021, + "tokens_seen": 1814822912 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036377933801404216, + "loss": 1.3046, + "theoretical_loss": 3.4870170827422675, + "tokens_seen": 1814888448 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036376328986960886, + "loss": 1.3314, + "theoretical_loss": 3.487006470071055, + "tokens_seen": 1814953984 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036374724172517556, + "loss": 1.188, + "theoretical_loss": 3.486995857890342, + "tokens_seen": 1815019520 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036373119358074226, + "loss": 1.2663, + "theoretical_loss": 3.4869852462000885, + "tokens_seen": 1815085056 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036371514543630896, + "loss": 1.2313, + "theoretical_loss": 3.486974635000254, + "tokens_seen": 1815150592 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003636990972918756, + "loss": 1.2599, + "theoretical_loss": 3.486964024290799, + "tokens_seen": 1815216128 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003636830491474423, + "loss": 1.3257, + "theoretical_loss": 3.486953414071682, + "tokens_seen": 1815281664 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 608441, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3015462160110474, + "objective/train/theoretical_loss": 3.4869428043428634, + "objective/train/tokens_used": 1835807200, + "theoretical_loss": 3.4869428043428634, + "tokens_seen": 1815347200 + }, + { + "epoch": 0.55, + "learning_rate": 0.000363667001003009, + "loss": 1.3048, + "theoretical_loss": 3.4869428043428634, + "tokens_seen": 1815347200 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003636509528585757, + "loss": 1.3369, + "theoretical_loss": 3.486932195104302, + "tokens_seen": 1815412736 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003636349047141424, + "loss": 1.2625, + "theoretical_loss": 3.4869215863559586, + "tokens_seen": 1815478272 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003636188565697091, + "loss": 1.3374, + "theoretical_loss": 3.486910978097792, + "tokens_seen": 1815543808 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003636028084252758, + "loss": 1.2534, + "theoretical_loss": 3.486900370329762, + "tokens_seen": 1815609344 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003635867602808425, + "loss": 1.2444, + "theoretical_loss": 3.4868897630518285, + "tokens_seen": 1815674880 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003635707121364092, + "loss": 1.3043, + "theoretical_loss": 3.4868791562639507, + "tokens_seen": 1815740416 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003635546639919759, + "loss": 1.3031, + "theoretical_loss": 3.486868549966089, + "tokens_seen": 1815805952 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003635386158475426, + "loss": 1.2724, + "theoretical_loss": 3.486857944158202, + "tokens_seen": 1815871488 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003635225677031093, + "loss": 1.3081, + "theoretical_loss": 3.4868473388402506, + "tokens_seen": 1815937024 + }, + { + "epoch": 0.55, + "learning_rate": 0.000363506519558676, + "loss": 1.2642, + "theoretical_loss": 3.486836734012194, + "tokens_seen": 1816002560 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003634904714142427, + "loss": 1.3277, + "theoretical_loss": 3.4868261296739917, + "tokens_seen": 1816068096 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003634744232698094, + "loss": 1.2766, + "theoretical_loss": 3.4868155258256035, + "tokens_seen": 1816133632 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003634583751253761, + "loss": 1.2914, + "theoretical_loss": 3.4868049224669893, + "tokens_seen": 1816199168 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003634423269809428, + "loss": 1.2828, + "theoretical_loss": 3.4867943195981086, + "tokens_seen": 1816264704 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003634262788365095, + "loss": 1.2143, + "theoretical_loss": 3.4867837172189207, + "tokens_seen": 1816330240 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003634102306920762, + "loss": 1.2123, + "theoretical_loss": 3.4867731153293864, + "tokens_seen": 1816395776 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003633941825476429, + "loss": 1.271, + "theoretical_loss": 3.486762513929464, + "tokens_seen": 1816461312 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003633781344032096, + "loss": 1.194, + "theoretical_loss": 3.486751913019115, + "tokens_seen": 1816526848 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003633620862587764, + "loss": 1.256, + "theoretical_loss": 3.486741312598298, + "tokens_seen": 1816592384 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003633460381143431, + "loss": 1.2416, + "theoretical_loss": 3.486730712666972, + "tokens_seen": 1816657920 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003633299899699098, + "loss": 1.2039, + "theoretical_loss": 3.4867201132250982, + "tokens_seen": 1816723456 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003633139418254765, + "loss": 1.2122, + "theoretical_loss": 3.486709514272636, + "tokens_seen": 1816788992 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003632978936810432, + "loss": 1.1513, + "theoretical_loss": 3.4866989158095443, + "tokens_seen": 1816854528 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003632818455366099, + "loss": 1.2599, + "theoretical_loss": 3.486688317835784, + "tokens_seen": 1816920064 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003632657973921766, + "loss": 1.256, + "theoretical_loss": 3.486677720351314, + "tokens_seen": 1816985600 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003632497492477433, + "loss": 1.2604, + "theoretical_loss": 3.486667123356095, + "tokens_seen": 1817051136 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036323370110331, + "loss": 1.2827, + "theoretical_loss": 3.4866565268500853, + "tokens_seen": 1817116672 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003632176529588767, + "loss": 1.2261, + "theoretical_loss": 3.486645930833246, + "tokens_seen": 1817182208 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003632016048144434, + "loss": 1.3218, + "theoretical_loss": 3.486635335305536, + "tokens_seen": 1817247744 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003631855566700101, + "loss": 1.2733, + "theoretical_loss": 3.486624740266916, + "tokens_seen": 1817313280 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003631695085255768, + "loss": 1.2521, + "theoretical_loss": 3.486614145717345, + "tokens_seen": 1817378816 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003631534603811435, + "loss": 1.2266, + "theoretical_loss": 3.486603551656783, + "tokens_seen": 1817444352 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003631374122367102, + "loss": 1.251, + "theoretical_loss": 3.48659295808519, + "tokens_seen": 1817509888 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003631213640922769, + "loss": 1.3789, + "theoretical_loss": 3.486582365002526, + "tokens_seen": 1817575424 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003631053159478436, + "loss": 1.3212, + "theoretical_loss": 3.48657177240875, + "tokens_seen": 1817640960 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003630892678034103, + "loss": 1.2854, + "theoretical_loss": 3.4865611803038226, + "tokens_seen": 1817706496 + }, + { + "epoch": 0.55, + "learning_rate": 0.000363073219658977, + "loss": 1.3285, + "theoretical_loss": 3.4865505886877033, + "tokens_seen": 1817772032 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036305717151454363, + "loss": 1.3461, + "theoretical_loss": 3.4865399975603513, + "tokens_seen": 1817837568 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036304112337011033, + "loss": 1.3236, + "theoretical_loss": 3.4865294069217274, + "tokens_seen": 1817903104 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036302507522567703, + "loss": 1.3279, + "theoretical_loss": 3.4865188167717913, + "tokens_seen": 1817968640 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036300902708124374, + "loss": 1.3376, + "theoretical_loss": 3.4865082271105026, + "tokens_seen": 1818034176 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036299297893681044, + "loss": 1.273, + "theoretical_loss": 3.4864976379378207, + "tokens_seen": 1818099712 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036297693079237714, + "loss": 1.2072, + "theoretical_loss": 3.4864870492537063, + "tokens_seen": 1818165248 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036296088264794384, + "loss": 1.2285, + "theoretical_loss": 3.4864764610581194, + "tokens_seen": 1818230784 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036294483450351054, + "loss": 1.2893, + "theoretical_loss": 3.486465873351018, + "tokens_seen": 1818296320 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036292878635907724, + "loss": 1.3307, + "theoretical_loss": 3.4864552861323643, + "tokens_seen": 1818361856 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036291273821464394, + "loss": 1.2505, + "theoretical_loss": 3.4864446994021168, + "tokens_seen": 1818427392 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036289669007021064, + "loss": 1.2763, + "theoretical_loss": 3.486434113160236, + "tokens_seen": 1818492928 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036288064192577734, + "loss": 1.3113, + "theoretical_loss": 3.4864235274066804, + "tokens_seen": 1818558464 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 609685, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2688941955566406, + "objective/train/theoretical_loss": 3.486412942141412, + "objective/train/tokens_used": 1839084000, + "theoretical_loss": 3.486412942141412, + "tokens_seen": 1818624000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036286459378134404, + "loss": 1.2658, + "theoretical_loss": 3.486412942141412, + "tokens_seen": 1818624000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036284854563691074, + "loss": 1.2564, + "theoretical_loss": 3.486402357364389, + "tokens_seen": 1818689536 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036283249749247744, + "loss": 1.3313, + "theoretical_loss": 3.4863917730755727, + "tokens_seen": 1818755072 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036281644934804414, + "loss": 1.2624, + "theoretical_loss": 3.4863811892749212, + "tokens_seen": 1818820608 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036280040120361084, + "loss": 1.2665, + "theoretical_loss": 3.486370605962396, + "tokens_seen": 1818886144 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036278435305917754, + "loss": 1.3522, + "theoretical_loss": 3.4863600231379563, + "tokens_seen": 1818951680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036276830491474424, + "loss": 1.38, + "theoretical_loss": 3.486349440801562, + "tokens_seen": 1819017216 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036275225677031095, + "loss": 1.3221, + "theoretical_loss": 3.4863388589531734, + "tokens_seen": 1819082752 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036273620862587765, + "loss": 1.2921, + "theoretical_loss": 3.48632827759275, + "tokens_seen": 1819148288 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036272016048144435, + "loss": 1.3046, + "theoretical_loss": 3.4863176967202514, + "tokens_seen": 1819213824 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036270411233701105, + "loss": 1.3125, + "theoretical_loss": 3.4863071163356385, + "tokens_seen": 1819279360 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036268806419257775, + "loss": 1.3842, + "theoretical_loss": 3.4862965364388705, + "tokens_seen": 1819344896 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036267201604814445, + "loss": 1.2527, + "theoretical_loss": 3.4862859570299074, + "tokens_seen": 1819410432 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036265596790371115, + "loss": 1.3334, + "theoretical_loss": 3.4862753781087097, + "tokens_seen": 1819475968 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036263991975927785, + "loss": 1.359, + "theoretical_loss": 3.4862647996752365, + "tokens_seen": 1819541504 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036262387161484455, + "loss": 1.3526, + "theoretical_loss": 3.4862542217294483, + "tokens_seen": 1819607040 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036260782347041125, + "loss": 1.2612, + "theoretical_loss": 3.4862436442713047, + "tokens_seen": 1819672576 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036259177532597795, + "loss": 1.227, + "theoretical_loss": 3.486233067300766, + "tokens_seen": 1819738112 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036257572718154465, + "loss": 1.2963, + "theoretical_loss": 3.486222490817793, + "tokens_seen": 1819803648 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036255967903711135, + "loss": 1.3204, + "theoretical_loss": 3.486211914822344, + "tokens_seen": 1819869184 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036254363089267805, + "loss": 1.2492, + "theoretical_loss": 3.486201339314379, + "tokens_seen": 1819934720 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036252758274824475, + "loss": 1.2459, + "theoretical_loss": 3.4861907642938594, + "tokens_seen": 1820000256 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036251153460381145, + "loss": 1.3057, + "theoretical_loss": 3.486180189760744, + "tokens_seen": 1820065792 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036249548645937816, + "loss": 1.1972, + "theoretical_loss": 3.486169615714994, + "tokens_seen": 1820131328 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036247943831494486, + "loss": 1.1669, + "theoretical_loss": 3.486159042156568, + "tokens_seen": 1820196864 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036246339017051156, + "loss": 1.3157, + "theoretical_loss": 3.4861484690854265, + "tokens_seen": 1820262400 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036244734202607826, + "loss": 1.269, + "theoretical_loss": 3.48613789650153, + "tokens_seen": 1820327936 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036243129388164496, + "loss": 1.3196, + "theoretical_loss": 3.4861273244048383, + "tokens_seen": 1820393472 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036241524573721166, + "loss": 1.3406, + "theoretical_loss": 3.4861167527953105, + "tokens_seen": 1820459008 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036239919759277836, + "loss": 1.2703, + "theoretical_loss": 3.486106181672908, + "tokens_seen": 1820524544 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036238314944834506, + "loss": 1.2174, + "theoretical_loss": 3.48609561103759, + "tokens_seen": 1820590080 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036236710130391176, + "loss": 1.2678, + "theoretical_loss": 3.4860850408893165, + "tokens_seen": 1820655616 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036235105315947846, + "loss": 1.3515, + "theoretical_loss": 3.486074471228048, + "tokens_seen": 1820721152 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036233500501504516, + "loss": 1.3061, + "theoretical_loss": 3.4860639020537443, + "tokens_seen": 1820786688 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036231895687061186, + "loss": 1.2274, + "theoretical_loss": 3.486053333366365, + "tokens_seen": 1820852224 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036230290872617856, + "loss": 1.2016, + "theoretical_loss": 3.4860427651658705, + "tokens_seen": 1820917760 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003622868605817452, + "loss": 1.2892, + "theoretical_loss": 3.4860321974522215, + "tokens_seen": 1820983296 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003622708124373119, + "loss": 1.3436, + "theoretical_loss": 3.486021630225377, + "tokens_seen": 1821048832 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003622547642928786, + "loss": 1.235, + "theoretical_loss": 3.4860110634852974, + "tokens_seen": 1821114368 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003622387161484453, + "loss": 1.3445, + "theoretical_loss": 3.486000497231943, + "tokens_seen": 1821179904 + }, + { + "epoch": 0.55, + "learning_rate": 0.000362222668004012, + "loss": 1.2943, + "theoretical_loss": 3.485989931465274, + "tokens_seen": 1821245440 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003622066198595787, + "loss": 1.3383, + "theoretical_loss": 3.4859793661852496, + "tokens_seen": 1821310976 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003621905717151454, + "loss": 1.2164, + "theoretical_loss": 3.485968801391831, + "tokens_seen": 1821376512 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003621745235707121, + "loss": 1.3033, + "theoretical_loss": 3.4859582370849775, + "tokens_seen": 1821442048 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036215847542627887, + "loss": 1.2653, + "theoretical_loss": 3.4859476732646493, + "tokens_seen": 1821507584 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036214242728184557, + "loss": 1.2512, + "theoretical_loss": 3.4859371099308065, + "tokens_seen": 1821573120 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036212637913741227, + "loss": 1.2727, + "theoretical_loss": 3.4859265470834098, + "tokens_seen": 1821638656 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036211033099297897, + "loss": 1.1472, + "theoretical_loss": 3.4859159847224186, + "tokens_seen": 1821704192 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036209428284854567, + "loss": 1.1873, + "theoretical_loss": 3.485905422847793, + "tokens_seen": 1821769728 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036207823470411237, + "loss": 1.2393, + "theoretical_loss": 3.485894861459494, + "tokens_seen": 1821835264 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 610785, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2064809799194336, + "objective/train/theoretical_loss": 3.4858843005574807, + "objective/train/tokens_used": 1842360800, + "theoretical_loss": 3.4858843005574807, + "tokens_seen": 1821900800 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036206218655967907, + "loss": 1.223, + "theoretical_loss": 3.4858843005574807, + "tokens_seen": 1821900800 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003620461384152458, + "loss": 1.2586, + "theoretical_loss": 3.485873740141713, + "tokens_seen": 1821966336 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003620300902708125, + "loss": 1.334, + "theoretical_loss": 3.4858631802121525, + "tokens_seen": 1822031872 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003620140421263792, + "loss": 1.2901, + "theoretical_loss": 3.4858526207687577, + "tokens_seen": 1822097408 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003619979939819459, + "loss": 1.2435, + "theoretical_loss": 3.48584206181149, + "tokens_seen": 1822162944 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003619819458375126, + "loss": 1.1528, + "theoretical_loss": 3.4858315033403087, + "tokens_seen": 1822228480 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003619658976930793, + "loss": 1.3139, + "theoretical_loss": 3.485820945355174, + "tokens_seen": 1822294016 + }, + { + "epoch": 0.55, + "learning_rate": 0.000361949849548646, + "loss": 1.2167, + "theoretical_loss": 3.4858103878560467, + "tokens_seen": 1822359552 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003619338014042127, + "loss": 1.2569, + "theoretical_loss": 3.485799830842886, + "tokens_seen": 1822425088 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003619177532597794, + "loss": 1.1894, + "theoretical_loss": 3.4857892743156533, + "tokens_seen": 1822490624 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003619017051153461, + "loss": 1.1715, + "theoretical_loss": 3.4857787182743074, + "tokens_seen": 1822556160 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003618856569709128, + "loss": 1.3059, + "theoretical_loss": 3.4857681627188097, + "tokens_seen": 1822621696 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003618696088264795, + "loss": 1.2832, + "theoretical_loss": 3.4857576076491195, + "tokens_seen": 1822687232 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003618535606820462, + "loss": 1.2599, + "theoretical_loss": 3.4857470530651975, + "tokens_seen": 1822752768 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003618375125376129, + "loss": 1.2569, + "theoretical_loss": 3.485736498967003, + "tokens_seen": 1822818304 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003618214643931796, + "loss": 1.2071, + "theoretical_loss": 3.4857259453544973, + "tokens_seen": 1822883840 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003618054162487463, + "loss": 1.2447, + "theoretical_loss": 3.4857153922276396, + "tokens_seen": 1822949376 + }, + { + "epoch": 0.55, + "learning_rate": 0.000361789368104313, + "loss": 1.4541, + "theoretical_loss": 3.485704839586391, + "tokens_seen": 1823014912 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003617733199598797, + "loss": 1.2774, + "theoretical_loss": 3.4856942874307117, + "tokens_seen": 1823080448 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003617572718154464, + "loss": 1.166, + "theoretical_loss": 3.485683735760561, + "tokens_seen": 1823145984 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003617412236710131, + "loss": 1.2274, + "theoretical_loss": 3.4856731845758997, + "tokens_seen": 1823211520 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003617251755265798, + "loss": 1.1972, + "theoretical_loss": 3.4856626338766876, + "tokens_seen": 1823277056 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003617091273821465, + "loss": 1.3944, + "theoretical_loss": 3.4856520836628855, + "tokens_seen": 1823342592 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003616930792377132, + "loss": 1.2454, + "theoretical_loss": 3.4856415339344533, + "tokens_seen": 1823408128 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003616770310932799, + "loss": 1.2749, + "theoretical_loss": 3.485630984691352, + "tokens_seen": 1823473664 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036166098294884653, + "loss": 1.316, + "theoretical_loss": 3.48562043593354, + "tokens_seen": 1823539200 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036164493480441323, + "loss": 1.2725, + "theoretical_loss": 3.4856098876609796, + "tokens_seen": 1823604736 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036162888665997994, + "loss": 1.2374, + "theoretical_loss": 3.4855993398736294, + "tokens_seen": 1823670272 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036161283851554664, + "loss": 1.3047, + "theoretical_loss": 3.4855887925714506, + "tokens_seen": 1823735808 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036159679037111334, + "loss": 1.3376, + "theoretical_loss": 3.485578245754403, + "tokens_seen": 1823801344 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036158074222668004, + "loss": 1.294, + "theoretical_loss": 3.4855676994224476, + "tokens_seen": 1823866880 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036156469408224674, + "loss": 1.2798, + "theoretical_loss": 3.4855571535755434, + "tokens_seen": 1823932416 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036154864593781344, + "loss": 1.1986, + "theoretical_loss": 3.485546608213652, + "tokens_seen": 1823997952 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036153259779338014, + "loss": 1.2344, + "theoretical_loss": 3.4855360633367325, + "tokens_seen": 1824063488 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036151654964894684, + "loss": 1.268, + "theoretical_loss": 3.485525518944746, + "tokens_seen": 1824129024 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036150050150451354, + "loss": 1.4095, + "theoretical_loss": 3.4855149750376526, + "tokens_seen": 1824194560 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036148445336008024, + "loss": 1.2657, + "theoretical_loss": 3.4855044316154125, + "tokens_seen": 1824260096 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036146840521564694, + "loss": 1.1598, + "theoretical_loss": 3.4854938886779854, + "tokens_seen": 1824325632 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036145235707121364, + "loss": 1.2525, + "theoretical_loss": 3.4854833462253323, + "tokens_seen": 1824391168 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036143630892678034, + "loss": 1.4092, + "theoretical_loss": 3.4854728042574137, + "tokens_seen": 1824456704 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036142026078234704, + "loss": 1.1669, + "theoretical_loss": 3.485462262774189, + "tokens_seen": 1824522240 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036140421263791374, + "loss": 1.2821, + "theoretical_loss": 3.48545172177562, + "tokens_seen": 1824587776 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036138816449348044, + "loss": 1.169, + "theoretical_loss": 3.4854411812616655, + "tokens_seen": 1824653312 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036137211634904715, + "loss": 1.2792, + "theoretical_loss": 3.485430641232286, + "tokens_seen": 1824718848 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036135606820461385, + "loss": 1.1355, + "theoretical_loss": 3.485420101687443, + "tokens_seen": 1824784384 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036134002006018055, + "loss": 1.1121, + "theoretical_loss": 3.485409562627096, + "tokens_seen": 1824849920 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036132397191574725, + "loss": 1.2295, + "theoretical_loss": 3.4853990240512047, + "tokens_seen": 1824915456 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036130792377131395, + "loss": 1.2294, + "theoretical_loss": 3.48538848595973, + "tokens_seen": 1824980992 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036129187562688065, + "loss": 1.2275, + "theoretical_loss": 3.485377948352633, + "tokens_seen": 1825046528 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036127582748244735, + "loss": 1.3045, + "theoretical_loss": 3.4853674112298734, + "tokens_seen": 1825112064 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 611941, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2710858583450317, + "objective/train/theoretical_loss": 3.4853568745914107, + "objective/train/tokens_used": 1845637600, + "theoretical_loss": 3.4853568745914107, + "tokens_seen": 1825177600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036125977933801405, + "loss": 1.2599, + "theoretical_loss": 3.4853568745914107, + "tokens_seen": 1825177600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036124373119358075, + "loss": 1.2478, + "theoretical_loss": 3.485346338437207, + "tokens_seen": 1825243136 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036122768304914745, + "loss": 1.1488, + "theoretical_loss": 3.4853358027672208, + "tokens_seen": 1825308672 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036121163490471415, + "loss": 1.2829, + "theoretical_loss": 3.485325267581414, + "tokens_seen": 1825374208 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036119558676028085, + "loss": 1.2769, + "theoretical_loss": 3.485314732879746, + "tokens_seen": 1825439744 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036117953861584755, + "loss": 1.2999, + "theoretical_loss": 3.485304198662178, + "tokens_seen": 1825505280 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036116349047141425, + "loss": 1.2482, + "theoretical_loss": 3.48529366492867, + "tokens_seen": 1825570816 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036114744232698095, + "loss": 1.2877, + "theoretical_loss": 3.4852831316791817, + "tokens_seen": 1825636352 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036113139418254766, + "loss": 1.3272, + "theoretical_loss": 3.4852725989136744, + "tokens_seen": 1825701888 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036111534603811436, + "loss": 1.2646, + "theoretical_loss": 3.485262066632108, + "tokens_seen": 1825767424 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036109929789368106, + "loss": 1.2557, + "theoretical_loss": 3.4852515348344433, + "tokens_seen": 1825832960 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036108324974924776, + "loss": 1.2524, + "theoretical_loss": 3.48524100352064, + "tokens_seen": 1825898496 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036106720160481446, + "loss": 1.2494, + "theoretical_loss": 3.4852304726906596, + "tokens_seen": 1825964032 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036105115346038116, + "loss": 1.2649, + "theoretical_loss": 3.4852199423444614, + "tokens_seen": 1826029568 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036103510531594786, + "loss": 1.3101, + "theoretical_loss": 3.485209412482006, + "tokens_seen": 1826095104 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036101905717151456, + "loss": 1.3538, + "theoretical_loss": 3.485198883103255, + "tokens_seen": 1826160640 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036100300902708126, + "loss": 1.2657, + "theoretical_loss": 3.4851883542081676, + "tokens_seen": 1826226176 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036098696088264796, + "loss": 1.1941, + "theoretical_loss": 3.485177825796704, + "tokens_seen": 1826291712 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036097091273821466, + "loss": 1.3217, + "theoretical_loss": 3.4851672978688257, + "tokens_seen": 1826357248 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036095486459378136, + "loss": 1.2495, + "theoretical_loss": 3.4851567704244926, + "tokens_seen": 1826422784 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036093881644934806, + "loss": 1.1728, + "theoretical_loss": 3.485146243463665, + "tokens_seen": 1826488320 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036092276830491476, + "loss": 1.1705, + "theoretical_loss": 3.4851357169863038, + "tokens_seen": 1826553856 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036090672016048146, + "loss": 1.2066, + "theoretical_loss": 3.4851251909923686, + "tokens_seen": 1826619392 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036089067201604816, + "loss": 1.3087, + "theoretical_loss": 3.485114665481821, + "tokens_seen": 1826684928 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036087462387161487, + "loss": 1.2307, + "theoretical_loss": 3.4851041404546206, + "tokens_seen": 1826750464 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036085857572718157, + "loss": 1.2749, + "theoretical_loss": 3.485093615910728, + "tokens_seen": 1826816000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036084252758274827, + "loss": 1.1746, + "theoretical_loss": 3.485083091850104, + "tokens_seen": 1826881536 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036082647943831497, + "loss": 1.2735, + "theoretical_loss": 3.485072568272709, + "tokens_seen": 1826947072 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036081043129388167, + "loss": 1.1848, + "theoretical_loss": 3.4850620451785033, + "tokens_seen": 1827012608 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036079438314944837, + "loss": 1.2691, + "theoretical_loss": 3.4850515225674474, + "tokens_seen": 1827078144 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036077833500501507, + "loss": 1.335, + "theoretical_loss": 3.4850410004395016, + "tokens_seen": 1827143680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036076228686058177, + "loss": 1.3148, + "theoretical_loss": 3.485030478794627, + "tokens_seen": 1827209216 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036074623871614847, + "loss": 1.3177, + "theoretical_loss": 3.4850199576327836, + "tokens_seen": 1827274752 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036073019057171517, + "loss": 1.3288, + "theoretical_loss": 3.4850094369539315, + "tokens_seen": 1827340288 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036071414242728187, + "loss": 1.2212, + "theoretical_loss": 3.4849989167580326, + "tokens_seen": 1827405824 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036069809428284857, + "loss": 1.3141, + "theoretical_loss": 3.484988397045046, + "tokens_seen": 1827471360 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036068204613841527, + "loss": 1.4041, + "theoretical_loss": 3.484977877814933, + "tokens_seen": 1827536896 + }, + { + "epoch": 0.55, + "learning_rate": 0.000360665997993982, + "loss": 1.2151, + "theoretical_loss": 3.484967359067654, + "tokens_seen": 1827602432 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003606499498495487, + "loss": 1.2378, + "theoretical_loss": 3.484956840803169, + "tokens_seen": 1827667968 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003606339017051154, + "loss": 1.2024, + "theoretical_loss": 3.484946323021439, + "tokens_seen": 1827733504 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003606178535606821, + "loss": 1.2173, + "theoretical_loss": 3.484935805722425, + "tokens_seen": 1827799040 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003606018054162488, + "loss": 1.349, + "theoretical_loss": 3.484925288906086, + "tokens_seen": 1827864576 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003605857572718155, + "loss": 1.2604, + "theoretical_loss": 3.4849147725723846, + "tokens_seen": 1827930112 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003605697091273822, + "loss": 1.2905, + "theoretical_loss": 3.48490425672128, + "tokens_seen": 1827995648 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003605536609829489, + "loss": 1.2654, + "theoretical_loss": 3.4848937413527326, + "tokens_seen": 1828061184 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003605376128385156, + "loss": 1.2722, + "theoretical_loss": 3.484883226466704, + "tokens_seen": 1828126720 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003605215646940823, + "loss": 1.2965, + "theoretical_loss": 3.4848727120631535, + "tokens_seen": 1828192256 + }, + { + "epoch": 0.55, + "learning_rate": 0.000360505516549649, + "loss": 1.2509, + "theoretical_loss": 3.484862198142043, + "tokens_seen": 1828257792 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003604894684052157, + "loss": 1.2392, + "theoretical_loss": 3.484851684703332, + "tokens_seen": 1828323328 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003604734202607824, + "loss": 1.299, + "theoretical_loss": 3.4848411717469814, + "tokens_seen": 1828388864 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 612914, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2961459159851074, + "objective/train/theoretical_loss": 3.4848306592729523, + "objective/train/tokens_used": 1848914400, + "theoretical_loss": 3.4848306592729523, + "tokens_seen": 1828454400 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003604573721163491, + "loss": 1.2261, + "theoretical_loss": 3.4848306592729523, + "tokens_seen": 1828454400 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003604413239719158, + "loss": 1.2769, + "theoretical_loss": 3.4848201472812046, + "tokens_seen": 1828519936 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003604252758274825, + "loss": 1.2933, + "theoretical_loss": 3.484809635771699, + "tokens_seen": 1828585472 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003604092276830492, + "loss": 1.2791, + "theoretical_loss": 3.4847991247443963, + "tokens_seen": 1828651008 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003603931795386159, + "loss": 1.2587, + "theoretical_loss": 3.4847886141992572, + "tokens_seen": 1828716544 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003603771313941826, + "loss": 1.2012, + "theoretical_loss": 3.4847781041362422, + "tokens_seen": 1828782080 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003603610832497493, + "loss": 1.2621, + "theoretical_loss": 3.484767594555312, + "tokens_seen": 1828847616 + }, + { + "epoch": 0.55, + "learning_rate": 0.000360345035105316, + "loss": 1.1882, + "theoretical_loss": 3.484757085456427, + "tokens_seen": 1828913152 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003603289869608827, + "loss": 1.3032, + "theoretical_loss": 3.484746576839547, + "tokens_seen": 1828978688 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003603129388164494, + "loss": 1.1647, + "theoretical_loss": 3.4847360687046347, + "tokens_seen": 1829044224 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003602968906720161, + "loss": 1.2247, + "theoretical_loss": 3.4847255610516488, + "tokens_seen": 1829109760 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003602808425275828, + "loss": 1.2256, + "theoretical_loss": 3.484715053880551, + "tokens_seen": 1829175296 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036026479438314944, + "loss": 1.3092, + "theoretical_loss": 3.484704547191301, + "tokens_seen": 1829240832 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036024874623871614, + "loss": 1.1803, + "theoretical_loss": 3.484694040983861, + "tokens_seen": 1829306368 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036023269809428284, + "loss": 1.2245, + "theoretical_loss": 3.48468353525819, + "tokens_seen": 1829371904 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036021664994984954, + "loss": 1.3079, + "theoretical_loss": 3.4846730300142497, + "tokens_seen": 1829437440 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036020060180541624, + "loss": 1.2751, + "theoretical_loss": 3.4846625252520003, + "tokens_seen": 1829502976 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036018455366098294, + "loss": 1.3587, + "theoretical_loss": 3.484652020971402, + "tokens_seen": 1829568512 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036016850551654964, + "loss": 1.2593, + "theoretical_loss": 3.4846415171724168, + "tokens_seen": 1829634048 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036015245737211634, + "loss": 1.0759, + "theoretical_loss": 3.484631013855004, + "tokens_seen": 1829699584 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036013640922768304, + "loss": 1.2275, + "theoretical_loss": 3.4846205110191253, + "tokens_seen": 1829765120 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036012036108324974, + "loss": 1.3465, + "theoretical_loss": 3.484610008664741, + "tokens_seen": 1829830656 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036010431293881644, + "loss": 1.2459, + "theoretical_loss": 3.4845995067918114, + "tokens_seen": 1829896192 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036008826479438314, + "loss": 1.2123, + "theoretical_loss": 3.4845890054002973, + "tokens_seen": 1829961728 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036007221664994984, + "loss": 1.2271, + "theoretical_loss": 3.48457850449016, + "tokens_seen": 1830027264 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036005616850551654, + "loss": 1.175, + "theoretical_loss": 3.4845680040613596, + "tokens_seen": 1830092800 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036004012036108324, + "loss": 1.3492, + "theoretical_loss": 3.484557504113857, + "tokens_seen": 1830158336 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036002407221664994, + "loss": 1.3292, + "theoretical_loss": 3.484547004647613, + "tokens_seen": 1830223872 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036000802407221665, + "loss": 1.2915, + "theoretical_loss": 3.484536505662588, + "tokens_seen": 1830289408 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035999197592778335, + "loss": 1.3461, + "theoretical_loss": 3.484526007158743, + "tokens_seen": 1830354944 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035997592778335005, + "loss": 1.3139, + "theoretical_loss": 3.4845155091360387, + "tokens_seen": 1830420480 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035995987963891675, + "loss": 1.2514, + "theoretical_loss": 3.484505011594436, + "tokens_seen": 1830486016 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035994383149448345, + "loss": 1.2276, + "theoretical_loss": 3.4844945145338952, + "tokens_seen": 1830551552 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035992778335005015, + "loss": 1.3074, + "theoretical_loss": 3.484484017954377, + "tokens_seen": 1830617088 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035991173520561685, + "loss": 1.3158, + "theoretical_loss": 3.4844735218558425, + "tokens_seen": 1830682624 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035989568706118355, + "loss": 1.2718, + "theoretical_loss": 3.4844630262382523, + "tokens_seen": 1830748160 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035987963891675025, + "loss": 1.2012, + "theoretical_loss": 3.4844525311015673, + "tokens_seen": 1830813696 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035986359077231695, + "loss": 1.33, + "theoretical_loss": 3.4844420364457482, + "tokens_seen": 1830879232 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035984754262788365, + "loss": 1.2363, + "theoretical_loss": 3.4844315422707552, + "tokens_seen": 1830944768 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035983149448345035, + "loss": 1.2533, + "theoretical_loss": 3.48442104857655, + "tokens_seen": 1831010304 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035981544633901705, + "loss": 1.2156, + "theoretical_loss": 3.484410555363093, + "tokens_seen": 1831075840 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035979939819458375, + "loss": 1.2119, + "theoretical_loss": 3.4844000626303444, + "tokens_seen": 1831141376 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035978335005015045, + "loss": 1.2532, + "theoretical_loss": 3.4843895703782657, + "tokens_seen": 1831206912 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035976730190571715, + "loss": 1.2062, + "theoretical_loss": 3.4843790786068176, + "tokens_seen": 1831272448 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003597512537612839, + "loss": 1.1707, + "theoretical_loss": 3.48436858731596, + "tokens_seen": 1831337984 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003597352056168506, + "loss": 1.2169, + "theoretical_loss": 3.484358096505655, + "tokens_seen": 1831403520 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003597191574724173, + "loss": 1.4065, + "theoretical_loss": 3.4843476061758625, + "tokens_seen": 1831469056 + }, + { + "epoch": 0.56, + "learning_rate": 0.000359703109327984, + "loss": 1.3943, + "theoretical_loss": 3.484337116326544, + "tokens_seen": 1831534592 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003596870611835507, + "loss": 1.2586, + "theoretical_loss": 3.4843266269576594, + "tokens_seen": 1831600128 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003596710130391174, + "loss": 1.1794, + "theoretical_loss": 3.48431613806917, + "tokens_seen": 1831665664 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 614178, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1981984376907349, + "objective/train/theoretical_loss": 3.484305649661037, + "objective/train/tokens_used": 1852191200, + "theoretical_loss": 3.484305649661037, + "tokens_seen": 1831731200 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003596549648946841, + "loss": 1.1518, + "theoretical_loss": 3.484305649661037, + "tokens_seen": 1831731200 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003596389167502508, + "loss": 1.19, + "theoretical_loss": 3.48429516173322, + "tokens_seen": 1831796736 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035962286860581746, + "loss": 1.3193, + "theoretical_loss": 3.4842846742856812, + "tokens_seen": 1831862272 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035960682046138416, + "loss": 1.2772, + "theoretical_loss": 3.4842741873183813, + "tokens_seen": 1831927808 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035959077231695086, + "loss": 1.2874, + "theoretical_loss": 3.48426370083128, + "tokens_seen": 1831993344 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035957472417251756, + "loss": 1.3171, + "theoretical_loss": 3.4842532148243395, + "tokens_seen": 1832058880 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035955867602808426, + "loss": 1.3326, + "theoretical_loss": 3.4842427292975193, + "tokens_seen": 1832124416 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035954262788365096, + "loss": 1.2284, + "theoretical_loss": 3.484232244250781, + "tokens_seen": 1832189952 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035952657973921766, + "loss": 1.2879, + "theoretical_loss": 3.4842217596840857, + "tokens_seen": 1832255488 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035951053159478436, + "loss": 1.3435, + "theoretical_loss": 3.4842112755973935, + "tokens_seen": 1832321024 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035949448345035107, + "loss": 1.2232, + "theoretical_loss": 3.484200791990666, + "tokens_seen": 1832386560 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035947843530591777, + "loss": 1.2788, + "theoretical_loss": 3.4841903088638637, + "tokens_seen": 1832452096 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035946238716148447, + "loss": 1.2972, + "theoretical_loss": 3.4841798262169474, + "tokens_seen": 1832517632 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035944633901705117, + "loss": 1.3475, + "theoretical_loss": 3.484169344049878, + "tokens_seen": 1832583168 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035943029087261787, + "loss": 1.3467, + "theoretical_loss": 3.4841588623626167, + "tokens_seen": 1832648704 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035941424272818457, + "loss": 1.29, + "theoretical_loss": 3.4841483811551237, + "tokens_seen": 1832714240 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035939819458375127, + "loss": 1.1779, + "theoretical_loss": 3.4841379004273607, + "tokens_seen": 1832779776 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035938214643931797, + "loss": 1.3041, + "theoretical_loss": 3.4841274201792882, + "tokens_seen": 1832845312 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035936609829488467, + "loss": 1.176, + "theoretical_loss": 3.4841169404108667, + "tokens_seen": 1832910848 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035935005015045137, + "loss": 1.333, + "theoretical_loss": 3.484106461122058, + "tokens_seen": 1832976384 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035933400200601807, + "loss": 1.3407, + "theoretical_loss": 3.484095982312822, + "tokens_seen": 1833041920 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035931795386158477, + "loss": 1.2646, + "theoretical_loss": 3.4840855039831204, + "tokens_seen": 1833107456 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003593019057171515, + "loss": 1.2558, + "theoretical_loss": 3.484075026132914, + "tokens_seen": 1833172992 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003592858575727182, + "loss": 1.2761, + "theoretical_loss": 3.4840645487621633, + "tokens_seen": 1833238528 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003592698094282849, + "loss": 1.248, + "theoretical_loss": 3.4840540718708297, + "tokens_seen": 1833304064 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003592537612838516, + "loss": 1.2802, + "theoretical_loss": 3.484043595458873, + "tokens_seen": 1833369600 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003592377131394183, + "loss": 1.3597, + "theoretical_loss": 3.484033119526256, + "tokens_seen": 1833435136 + }, + { + "epoch": 0.56, + "learning_rate": 0.000359221664994985, + "loss": 1.3109, + "theoretical_loss": 3.4840226440729385, + "tokens_seen": 1833500672 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003592056168505517, + "loss": 1.3115, + "theoretical_loss": 3.4840121690988815, + "tokens_seen": 1833566208 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003591895687061184, + "loss": 1.3127, + "theoretical_loss": 3.4840016946040455, + "tokens_seen": 1833631744 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003591735205616851, + "loss": 1.3082, + "theoretical_loss": 3.4839912205883925, + "tokens_seen": 1833697280 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003591574724172518, + "loss": 1.3108, + "theoretical_loss": 3.4839807470518833, + "tokens_seen": 1833762816 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003591414242728185, + "loss": 1.2422, + "theoretical_loss": 3.483970273994478, + "tokens_seen": 1833828352 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003591253761283852, + "loss": 1.3497, + "theoretical_loss": 3.483959801416138, + "tokens_seen": 1833893888 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003591093279839519, + "loss": 1.3792, + "theoretical_loss": 3.4839493293168244, + "tokens_seen": 1833959424 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003590932798395186, + "loss": 1.2936, + "theoretical_loss": 3.483938857696498, + "tokens_seen": 1834024960 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003590772316950853, + "loss": 1.3189, + "theoretical_loss": 3.48392838655512, + "tokens_seen": 1834090496 + }, + { + "epoch": 0.56, + "learning_rate": 0.000359061183550652, + "loss": 1.2818, + "theoretical_loss": 3.4839179158926514, + "tokens_seen": 1834156032 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003590451354062187, + "loss": 1.3387, + "theoretical_loss": 3.483907445709053, + "tokens_seen": 1834221568 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003590290872617854, + "loss": 1.2675, + "theoretical_loss": 3.483896976004286, + "tokens_seen": 1834287104 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003590130391173521, + "loss": 1.3338, + "theoretical_loss": 3.483886506778311, + "tokens_seen": 1834352640 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003589969909729188, + "loss": 1.3693, + "theoretical_loss": 3.483876038031089, + "tokens_seen": 1834418176 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003589809428284855, + "loss": 1.2749, + "theoretical_loss": 3.4838655697625813, + "tokens_seen": 1834483712 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003589648946840522, + "loss": 1.1919, + "theoretical_loss": 3.483855101972749, + "tokens_seen": 1834549248 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003589488465396189, + "loss": 1.3126, + "theoretical_loss": 3.483844634661553, + "tokens_seen": 1834614784 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003589327983951856, + "loss": 1.3354, + "theoretical_loss": 3.483834167828954, + "tokens_seen": 1834680320 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003589167502507523, + "loss": 1.2303, + "theoretical_loss": 3.483823701474914, + "tokens_seen": 1834745856 + }, + { + "epoch": 0.56, + "learning_rate": 0.000358900702106319, + "loss": 1.3366, + "theoretical_loss": 3.4838132355993925, + "tokens_seen": 1834811392 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003588846539618857, + "loss": 1.1893, + "theoretical_loss": 3.4838027702023515, + "tokens_seen": 1834876928 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003588686058174524, + "loss": 1.2126, + "theoretical_loss": 3.4837923052837523, + "tokens_seen": 1834942464 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 615428, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3236737251281738, + "objective/train/theoretical_loss": 3.4837818408435552, + "objective/train/tokens_used": 1855468000, + "theoretical_loss": 3.4837818408435552, + "tokens_seen": 1835008000 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035885255767301904, + "loss": 1.3667, + "theoretical_loss": 3.4837818408435552, + "tokens_seen": 1835008000 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035883650952858574, + "loss": 1.2132, + "theoretical_loss": 3.483771376881722, + "tokens_seen": 1835073536 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035882046138415244, + "loss": 1.3665, + "theoretical_loss": 3.483760913398213, + "tokens_seen": 1835139072 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035880441323971914, + "loss": 1.2977, + "theoretical_loss": 3.4837504503929893, + "tokens_seen": 1835204608 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035878836509528584, + "loss": 1.1866, + "theoretical_loss": 3.483739987866013, + "tokens_seen": 1835270144 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035877231695085254, + "loss": 1.218, + "theoretical_loss": 3.4837295258172434, + "tokens_seen": 1835335680 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035875626880641924, + "loss": 1.3062, + "theoretical_loss": 3.4837190642466433, + "tokens_seen": 1835401216 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035874022066198594, + "loss": 1.2915, + "theoretical_loss": 3.483708603154173, + "tokens_seen": 1835466752 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035872417251755264, + "loss": 1.321, + "theoretical_loss": 3.4836981425397933, + "tokens_seen": 1835532288 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035870812437311934, + "loss": 1.1974, + "theoretical_loss": 3.483687682403466, + "tokens_seen": 1835597824 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035869207622868604, + "loss": 1.2398, + "theoretical_loss": 3.4836772227451513, + "tokens_seen": 1835663360 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035867602808425274, + "loss": 1.3121, + "theoretical_loss": 3.483666763564811, + "tokens_seen": 1835728896 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035865997993981944, + "loss": 1.2651, + "theoretical_loss": 3.483656304862406, + "tokens_seen": 1835794432 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035864393179538614, + "loss": 1.3312, + "theoretical_loss": 3.4836458466378972, + "tokens_seen": 1835859968 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035862788365095285, + "loss": 1.3201, + "theoretical_loss": 3.483635388891246, + "tokens_seen": 1835925504 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035861183550651955, + "loss": 1.2883, + "theoretical_loss": 3.4836249316224137, + "tokens_seen": 1835991040 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035859578736208625, + "loss": 1.2329, + "theoretical_loss": 3.4836144748313607, + "tokens_seen": 1836056576 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035857973921765295, + "loss": 1.4045, + "theoretical_loss": 3.4836040185180486, + "tokens_seen": 1836122112 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035856369107321965, + "loss": 1.2736, + "theoretical_loss": 3.4835935626824384, + "tokens_seen": 1836187648 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003585476429287864, + "loss": 1.247, + "theoretical_loss": 3.483583107324492, + "tokens_seen": 1836253184 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003585315947843531, + "loss": 1.2594, + "theoretical_loss": 3.4835726524441686, + "tokens_seen": 1836318720 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003585155466399198, + "loss": 1.2779, + "theoretical_loss": 3.4835621980414313, + "tokens_seen": 1836384256 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003584994984954865, + "loss": 1.3443, + "theoretical_loss": 3.48355174411624, + "tokens_seen": 1836449792 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003584834503510532, + "loss": 1.2156, + "theoretical_loss": 3.4835412906685566, + "tokens_seen": 1836515328 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003584674022066199, + "loss": 1.3157, + "theoretical_loss": 3.4835308376983423, + "tokens_seen": 1836580864 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003584513540621866, + "loss": 1.2937, + "theoretical_loss": 3.4835203852055576, + "tokens_seen": 1836646400 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003584353059177533, + "loss": 1.2619, + "theoretical_loss": 3.483509933190164, + "tokens_seen": 1836711936 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035841925777332, + "loss": 1.2072, + "theoretical_loss": 3.4834994816521228, + "tokens_seen": 1836777472 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003584032096288867, + "loss": 1.2671, + "theoretical_loss": 3.483489030591395, + "tokens_seen": 1836843008 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003583871614844534, + "loss": 1.2276, + "theoretical_loss": 3.483478580007941, + "tokens_seen": 1836908544 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003583711133400201, + "loss": 1.2143, + "theoretical_loss": 3.4834681299017234, + "tokens_seen": 1836974080 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003583550651955868, + "loss": 1.2861, + "theoretical_loss": 3.483457680272703, + "tokens_seen": 1837039616 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003583390170511535, + "loss": 1.3095, + "theoretical_loss": 3.4834472311208406, + "tokens_seen": 1837105152 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003583229689067202, + "loss": 1.4022, + "theoretical_loss": 3.4834367824460974, + "tokens_seen": 1837170688 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003583069207622869, + "loss": 1.3521, + "theoretical_loss": 3.483426334248435, + "tokens_seen": 1837236224 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003582908726178536, + "loss": 1.3364, + "theoretical_loss": 3.4834158865278138, + "tokens_seen": 1837301760 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003582748244734203, + "loss": 1.2759, + "theoretical_loss": 3.483405439284196, + "tokens_seen": 1837367296 + }, + { + "epoch": 0.56, + "learning_rate": 0.000358258776328987, + "loss": 1.3857, + "theoretical_loss": 3.483394992517542, + "tokens_seen": 1837432832 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003582427281845537, + "loss": 1.303, + "theoretical_loss": 3.4833845462278137, + "tokens_seen": 1837498368 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035822668004012036, + "loss": 1.3633, + "theoretical_loss": 3.483374100414972, + "tokens_seen": 1837563904 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035821063189568706, + "loss": 1.2501, + "theoretical_loss": 3.4833636550789775, + "tokens_seen": 1837629440 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035819458375125376, + "loss": 1.14, + "theoretical_loss": 3.483353210219793, + "tokens_seen": 1837694976 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035817853560682046, + "loss": 1.2763, + "theoretical_loss": 3.483342765837378, + "tokens_seen": 1837760512 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035816248746238716, + "loss": 1.315, + "theoretical_loss": 3.4833323219316945, + "tokens_seen": 1837826048 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035814643931795386, + "loss": 1.2777, + "theoretical_loss": 3.4833218785027036, + "tokens_seen": 1837891584 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035813039117352057, + "loss": 1.3048, + "theoretical_loss": 3.483311435550367, + "tokens_seen": 1837957120 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035811434302908727, + "loss": 1.226, + "theoretical_loss": 3.4833009930746455, + "tokens_seen": 1838022656 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035809829488465397, + "loss": 1.3317, + "theoretical_loss": 3.483290551075501, + "tokens_seen": 1838088192 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035808224674022067, + "loss": 1.2434, + "theoretical_loss": 3.4832801095528936, + "tokens_seen": 1838153728 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035806619859578737, + "loss": 1.3287, + "theoretical_loss": 3.483269668506785, + "tokens_seen": 1838219264 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 616389, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1942389011383057, + "objective/train/theoretical_loss": 3.483259227937137, + "objective/train/tokens_used": 1858744800, + "theoretical_loss": 3.483259227937137, + "tokens_seen": 1838284800 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035805015045135407, + "loss": 1.2035, + "theoretical_loss": 3.483259227937137, + "tokens_seen": 1838284800 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035803410230692077, + "loss": 1.3313, + "theoretical_loss": 3.4832487878439107, + "tokens_seen": 1838350336 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035801805416248747, + "loss": 1.279, + "theoretical_loss": 3.4832383482270672, + "tokens_seen": 1838415872 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035800200601805417, + "loss": 1.2452, + "theoretical_loss": 3.483227909086568, + "tokens_seen": 1838481408 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035798595787362087, + "loss": 1.2942, + "theoretical_loss": 3.4832174704223737, + "tokens_seen": 1838546944 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035796990972918757, + "loss": 1.2314, + "theoretical_loss": 3.4832070322344464, + "tokens_seen": 1838612480 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035795386158475427, + "loss": 1.2063, + "theoretical_loss": 3.483196594522747, + "tokens_seen": 1838678016 + }, + { + "epoch": 0.56, + "learning_rate": 0.000357937813440321, + "loss": 1.3389, + "theoretical_loss": 3.4831861572872365, + "tokens_seen": 1838743552 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003579217652958877, + "loss": 1.3447, + "theoretical_loss": 3.483175720527877, + "tokens_seen": 1838809088 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003579057171514544, + "loss": 1.3295, + "theoretical_loss": 3.4831652842446292, + "tokens_seen": 1838874624 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003578896690070211, + "loss": 1.2889, + "theoretical_loss": 3.483154848437455, + "tokens_seen": 1838940160 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003578736208625878, + "loss": 1.2895, + "theoretical_loss": 3.4831444131063147, + "tokens_seen": 1839005696 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003578575727181545, + "loss": 1.251, + "theoretical_loss": 3.4831339782511703, + "tokens_seen": 1839071232 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003578415245737212, + "loss": 1.2106, + "theoretical_loss": 3.483123543871984, + "tokens_seen": 1839136768 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003578254764292879, + "loss": 1.2716, + "theoretical_loss": 3.4831131099687154, + "tokens_seen": 1839202304 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003578094282848546, + "loss": 1.296, + "theoretical_loss": 3.4831026765413267, + "tokens_seen": 1839267840 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003577933801404213, + "loss": 1.2149, + "theoretical_loss": 3.4830922435897786, + "tokens_seen": 1839333376 + }, + { + "epoch": 0.56, + "learning_rate": 0.000357777331995988, + "loss": 1.2302, + "theoretical_loss": 3.483081811114034, + "tokens_seen": 1839398912 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003577612838515547, + "loss": 1.3389, + "theoretical_loss": 3.483071379114053, + "tokens_seen": 1839464448 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003577452357071214, + "loss": 1.4062, + "theoretical_loss": 3.483060947589797, + "tokens_seen": 1839529984 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003577291875626881, + "loss": 1.253, + "theoretical_loss": 3.483050516541228, + "tokens_seen": 1839595520 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003577131394182548, + "loss": 1.2078, + "theoretical_loss": 3.483040085968306, + "tokens_seen": 1839661056 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003576970912738215, + "loss": 1.2078, + "theoretical_loss": 3.483029655870994, + "tokens_seen": 1839726592 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003576810431293882, + "loss": 1.1787, + "theoretical_loss": 3.4830192262492528, + "tokens_seen": 1839792128 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003576649949849549, + "loss": 1.2702, + "theoretical_loss": 3.483008797103044, + "tokens_seen": 1839857664 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003576489468405216, + "loss": 1.3047, + "theoretical_loss": 3.4829983684323276, + "tokens_seen": 1839923200 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003576328986960883, + "loss": 1.3219, + "theoretical_loss": 3.4829879402370665, + "tokens_seen": 1839988736 + }, + { + "epoch": 0.56, + "learning_rate": 0.000357616850551655, + "loss": 1.2892, + "theoretical_loss": 3.4829775125172215, + "tokens_seen": 1840054272 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003576008024072217, + "loss": 1.2746, + "theoretical_loss": 3.4829670852727546, + "tokens_seen": 1840119808 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003575847542627884, + "loss": 1.3876, + "theoretical_loss": 3.4829566585036265, + "tokens_seen": 1840185344 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003575687061183551, + "loss": 1.2227, + "theoretical_loss": 3.4829462322097986, + "tokens_seen": 1840250880 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003575526579739218, + "loss": 1.2704, + "theoretical_loss": 3.4829358063912323, + "tokens_seen": 1840316416 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003575366098294885, + "loss": 1.2178, + "theoretical_loss": 3.48292538104789, + "tokens_seen": 1840381952 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003575205616850552, + "loss": 1.1665, + "theoretical_loss": 3.482914956179732, + "tokens_seen": 1840447488 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003575045135406219, + "loss": 1.2546, + "theoretical_loss": 3.48290453178672, + "tokens_seen": 1840513024 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003574884653961886, + "loss": 1.3041, + "theoretical_loss": 3.482894107868815, + "tokens_seen": 1840578560 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003574724172517553, + "loss": 1.2707, + "theoretical_loss": 3.4828836844259796, + "tokens_seen": 1840644096 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035745636910732194, + "loss": 1.3621, + "theoretical_loss": 3.4828732614581743, + "tokens_seen": 1840709632 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035744032096288864, + "loss": 1.3637, + "theoretical_loss": 3.482862838965361, + "tokens_seen": 1840775168 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035742427281845534, + "loss": 1.2813, + "theoretical_loss": 3.482852416947501, + "tokens_seen": 1840840704 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035740822467402204, + "loss": 1.3091, + "theoretical_loss": 3.4828419954045553, + "tokens_seen": 1840906240 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035739217652958874, + "loss": 1.2187, + "theoretical_loss": 3.482831574336486, + "tokens_seen": 1840971776 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035737612838515544, + "loss": 1.1802, + "theoretical_loss": 3.482821153743254, + "tokens_seen": 1841037312 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035736008024072214, + "loss": 1.3933, + "theoretical_loss": 3.4828107336248215, + "tokens_seen": 1841102848 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003573440320962889, + "loss": 1.3383, + "theoretical_loss": 3.4828003139811496, + "tokens_seen": 1841168384 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003573279839518556, + "loss": 1.2194, + "theoretical_loss": 3.4827898948121994, + "tokens_seen": 1841233920 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003573119358074223, + "loss": 1.3819, + "theoretical_loss": 3.4827794761179325, + "tokens_seen": 1841299456 + }, + { + "epoch": 0.56, + "learning_rate": 0.000357295887662989, + "loss": 1.2321, + "theoretical_loss": 3.4827690578983113, + "tokens_seen": 1841364992 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003572798395185557, + "loss": 1.3872, + "theoretical_loss": 3.4827586401532957, + "tokens_seen": 1841430528 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003572637913741224, + "loss": 1.2951, + "theoretical_loss": 3.4827482228828486, + "tokens_seen": 1841496064 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 617553, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2685306072235107, + "objective/train/theoretical_loss": 3.4827378060869307, + "objective/train/tokens_used": 1862021600, + "theoretical_loss": 3.4827378060869307, + "tokens_seen": 1841561600 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003572477432296891, + "loss": 1.2613, + "theoretical_loss": 3.4827378060869307, + "tokens_seen": 1841561600 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003572316950852558, + "loss": 1.2346, + "theoretical_loss": 3.482727389765504, + "tokens_seen": 1841627136 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003572156469408225, + "loss": 1.2575, + "theoretical_loss": 3.4827169739185293, + "tokens_seen": 1841692672 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003571995987963892, + "loss": 1.3158, + "theoretical_loss": 3.482706558545969, + "tokens_seen": 1841758208 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003571835506519559, + "loss": 1.3707, + "theoretical_loss": 3.4826961436477837, + "tokens_seen": 1841823744 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003571675025075226, + "loss": 1.1855, + "theoretical_loss": 3.4826857292239355, + "tokens_seen": 1841889280 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003571514543630893, + "loss": 1.2667, + "theoretical_loss": 3.4826753152743857, + "tokens_seen": 1841954816 + }, + { + "epoch": 0.56, + "learning_rate": 0.000357135406218656, + "loss": 1.2439, + "theoretical_loss": 3.4826649017990965, + "tokens_seen": 1842020352 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003571193580742227, + "loss": 1.2445, + "theoretical_loss": 3.482654488798028, + "tokens_seen": 1842085888 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003571033099297894, + "loss": 1.2524, + "theoretical_loss": 3.482644076271143, + "tokens_seen": 1842151424 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003570872617853561, + "loss": 1.1124, + "theoretical_loss": 3.482633664218403, + "tokens_seen": 1842216960 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003570712136409228, + "loss": 1.2624, + "theoretical_loss": 3.4826232526397685, + "tokens_seen": 1842282496 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003570551654964895, + "loss": 1.3294, + "theoretical_loss": 3.482612841535202, + "tokens_seen": 1842348032 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003570391173520562, + "loss": 1.2951, + "theoretical_loss": 3.482602430904665, + "tokens_seen": 1842413568 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003570230692076229, + "loss": 1.2645, + "theoretical_loss": 3.4825920207481182, + "tokens_seen": 1842479104 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003570070210631896, + "loss": 1.2925, + "theoretical_loss": 3.482581611065524, + "tokens_seen": 1842544640 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003569909729187563, + "loss": 1.4029, + "theoretical_loss": 3.482571201856844, + "tokens_seen": 1842610176 + }, + { + "epoch": 0.56, + "learning_rate": 0.000356974924774323, + "loss": 1.3022, + "theoretical_loss": 3.4825607931220395, + "tokens_seen": 1842675712 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003569588766298897, + "loss": 1.2768, + "theoretical_loss": 3.4825503848610717, + "tokens_seen": 1842741248 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003569428284854564, + "loss": 1.2332, + "theoretical_loss": 3.4825399770739027, + "tokens_seen": 1842806784 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003569267803410231, + "loss": 1.3129, + "theoretical_loss": 3.4825295697604943, + "tokens_seen": 1842872320 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003569107321965898, + "loss": 1.2704, + "theoretical_loss": 3.4825191629208074, + "tokens_seen": 1842937856 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003568946840521565, + "loss": 1.3705, + "theoretical_loss": 3.482508756554804, + "tokens_seen": 1843003392 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003568786359077232, + "loss": 1.2985, + "theoretical_loss": 3.4824983506624454, + "tokens_seen": 1843068928 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003568625877632899, + "loss": 1.4798, + "theoretical_loss": 3.482487945243694, + "tokens_seen": 1843134464 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003568465396188566, + "loss": 1.2527, + "theoretical_loss": 3.48247754029851, + "tokens_seen": 1843200000 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003568304914744233, + "loss": 1.1568, + "theoretical_loss": 3.4824671358268566, + "tokens_seen": 1843265536 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035681444332998996, + "loss": 1.2164, + "theoretical_loss": 3.4824567318286945, + "tokens_seen": 1843331072 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035679839518555666, + "loss": 1.2176, + "theoretical_loss": 3.4824463283039853, + "tokens_seen": 1843396608 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035678234704112336, + "loss": 1.3636, + "theoretical_loss": 3.482435925252691, + "tokens_seen": 1843462144 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035676629889669007, + "loss": 1.2504, + "theoretical_loss": 3.482425522674773, + "tokens_seen": 1843527680 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035675025075225677, + "loss": 1.384, + "theoretical_loss": 3.4824151205701925, + "tokens_seen": 1843593216 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035673420260782347, + "loss": 1.2453, + "theoretical_loss": 3.482404718938912, + "tokens_seen": 1843658752 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035671815446339017, + "loss": 1.2501, + "theoretical_loss": 3.4823943177808925, + "tokens_seen": 1843724288 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035670210631895687, + "loss": 1.1321, + "theoretical_loss": 3.4823839170960964, + "tokens_seen": 1843789824 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035668605817452357, + "loss": 1.3099, + "theoretical_loss": 3.4823735168844845, + "tokens_seen": 1843855360 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035667001003009027, + "loss": 1.2082, + "theoretical_loss": 3.4823631171460185, + "tokens_seen": 1843920896 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035665396188565697, + "loss": 1.273, + "theoretical_loss": 3.482352717880661, + "tokens_seen": 1843986432 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035663791374122367, + "loss": 1.288, + "theoretical_loss": 3.4823423190883727, + "tokens_seen": 1844051968 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035662186559679037, + "loss": 1.3538, + "theoretical_loss": 3.4823319207691155, + "tokens_seen": 1844117504 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035660581745235707, + "loss": 1.3, + "theoretical_loss": 3.482321522922851, + "tokens_seen": 1844183040 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035658976930792377, + "loss": 1.2148, + "theoretical_loss": 3.482311125549541, + "tokens_seen": 1844248576 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035657372116349047, + "loss": 1.3326, + "theoretical_loss": 3.4823007286491476, + "tokens_seen": 1844314112 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003565576730190572, + "loss": 1.2638, + "theoretical_loss": 3.482290332221632, + "tokens_seen": 1844379648 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003565416248746239, + "loss": 1.3407, + "theoretical_loss": 3.4822799362669556, + "tokens_seen": 1844445184 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003565255767301906, + "loss": 1.3225, + "theoretical_loss": 3.482269540785081, + "tokens_seen": 1844510720 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003565095285857573, + "loss": 1.2919, + "theoretical_loss": 3.482259145775969, + "tokens_seen": 1844576256 + }, + { + "epoch": 0.56, + "learning_rate": 0.000356493480441324, + "loss": 1.3169, + "theoretical_loss": 3.482248751239582, + "tokens_seen": 1844641792 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003564774322968907, + "loss": 1.2356, + "theoretical_loss": 3.4822383571758815, + "tokens_seen": 1844707328 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003564613841524574, + "loss": 1.2596, + "theoretical_loss": 3.482227963584829, + "tokens_seen": 1844772864 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 618553, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.335506558418274, + "objective/train/theoretical_loss": 3.4822175704663865, + "objective/train/tokens_used": 1865298400, + "theoretical_loss": 3.4822175704663865, + "tokens_seen": 1844838400 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003564453360080241, + "loss": 1.2785, + "theoretical_loss": 3.4822175704663865, + "tokens_seen": 1844838400 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003564292878635908, + "loss": 1.2987, + "theoretical_loss": 3.4822071778205146, + "tokens_seen": 1844903936 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003564132397191575, + "loss": 1.2812, + "theoretical_loss": 3.4821967856471767, + "tokens_seen": 1844969472 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003563971915747242, + "loss": 1.2096, + "theoretical_loss": 3.482186393946334, + "tokens_seen": 1845035008 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003563811434302909, + "loss": 1.2482, + "theoretical_loss": 3.4821760027179476, + "tokens_seen": 1845100544 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003563650952858576, + "loss": 1.2864, + "theoretical_loss": 3.48216561196198, + "tokens_seen": 1845166080 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003563490471414243, + "loss": 1.3675, + "theoretical_loss": 3.4821552216783926, + "tokens_seen": 1845231616 + }, + { + "epoch": 0.56, + "learning_rate": 0.000356332998996991, + "loss": 1.2629, + "theoretical_loss": 3.482144831867147, + "tokens_seen": 1845297152 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003563169508525577, + "loss": 1.2266, + "theoretical_loss": 3.4821344425282055, + "tokens_seen": 1845362688 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003563009027081244, + "loss": 1.3251, + "theoretical_loss": 3.482124053661529, + "tokens_seen": 1845428224 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003562848545636911, + "loss": 1.27, + "theoretical_loss": 3.48211366526708, + "tokens_seen": 1845493760 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003562688064192578, + "loss": 1.2866, + "theoretical_loss": 3.48210327734482, + "tokens_seen": 1845559296 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003562527582748245, + "loss": 1.1667, + "theoretical_loss": 3.4820928898947106, + "tokens_seen": 1845624832 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003562367101303912, + "loss": 1.319, + "theoretical_loss": 3.482082502916714, + "tokens_seen": 1845690368 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003562206619859579, + "loss": 1.2643, + "theoretical_loss": 3.4820721164107917, + "tokens_seen": 1845755904 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003562046138415246, + "loss": 1.2869, + "theoretical_loss": 3.4820617303769055, + "tokens_seen": 1845821440 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003561885656970913, + "loss": 1.3225, + "theoretical_loss": 3.4820513448150177, + "tokens_seen": 1845886976 + }, + { + "epoch": 0.56, + "learning_rate": 0.000356172517552658, + "loss": 1.3266, + "theoretical_loss": 3.4820409597250888, + "tokens_seen": 1845952512 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003561564694082247, + "loss": 1.2993, + "theoretical_loss": 3.482030575107082, + "tokens_seen": 1846018048 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003561404212637914, + "loss": 1.3679, + "theoretical_loss": 3.482020190960958, + "tokens_seen": 1846083584 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003561243731193581, + "loss": 1.2383, + "theoretical_loss": 3.482009807286679, + "tokens_seen": 1846149120 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003561083249749248, + "loss": 1.3407, + "theoretical_loss": 3.481999424084208, + "tokens_seen": 1846214656 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003560922768304915, + "loss": 1.2877, + "theoretical_loss": 3.481989041353504, + "tokens_seen": 1846280192 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003560762286860582, + "loss": 1.2945, + "theoretical_loss": 3.481978659094532, + "tokens_seen": 1846345728 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003560601805416249, + "loss": 1.343, + "theoretical_loss": 3.481968277307252, + "tokens_seen": 1846411264 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003560441323971916, + "loss": 1.2717, + "theoretical_loss": 3.481957895991626, + "tokens_seen": 1846476800 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003560280842527583, + "loss": 1.3299, + "theoretical_loss": 3.481947515147617, + "tokens_seen": 1846542336 + }, + { + "epoch": 0.56, + "learning_rate": 0.000356012036108325, + "loss": 1.3658, + "theoretical_loss": 3.4819371347751846, + "tokens_seen": 1846607872 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003559959879638917, + "loss": 1.305, + "theoretical_loss": 3.4819267548742925, + "tokens_seen": 1846673408 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003559799398194584, + "loss": 1.3971, + "theoretical_loss": 3.481916375444902, + "tokens_seen": 1846738944 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003559638916750251, + "loss": 1.2732, + "theoretical_loss": 3.481905996486975, + "tokens_seen": 1846804480 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003559478435305918, + "loss": 1.2556, + "theoretical_loss": 3.4818956180004728, + "tokens_seen": 1846870016 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003559317953861585, + "loss": 1.2968, + "theoretical_loss": 3.481885239985358, + "tokens_seen": 1846935552 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003559157472417252, + "loss": 1.2193, + "theoretical_loss": 3.4818748624415923, + "tokens_seen": 1847001088 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003558996990972919, + "loss": 1.2138, + "theoretical_loss": 3.4818644853691376, + "tokens_seen": 1847066624 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003558836509528586, + "loss": 1.2683, + "theoretical_loss": 3.4818541087679558, + "tokens_seen": 1847132160 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003558676028084253, + "loss": 1.3527, + "theoretical_loss": 3.481843732638008, + "tokens_seen": 1847197696 + }, + { + "epoch": 0.56, + "learning_rate": 0.000355851554663992, + "loss": 1.2651, + "theoretical_loss": 3.4818333569792568, + "tokens_seen": 1847263232 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003558355065195587, + "loss": 1.2958, + "theoretical_loss": 3.4818229817916646, + "tokens_seen": 1847328768 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003558194583751254, + "loss": 1.189, + "theoretical_loss": 3.4818126070751925, + "tokens_seen": 1847394304 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003558034102306921, + "loss": 1.289, + "theoretical_loss": 3.481802232829802, + "tokens_seen": 1847459840 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003557873620862588, + "loss": 1.217, + "theoretical_loss": 3.4817918590554564, + "tokens_seen": 1847525376 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003557713139418255, + "loss": 1.3107, + "theoretical_loss": 3.481781485752116, + "tokens_seen": 1847590912 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003557552657973922, + "loss": 1.3368, + "theoretical_loss": 3.481771112919744, + "tokens_seen": 1847656448 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003557392176529589, + "loss": 1.2753, + "theoretical_loss": 3.4817607405583013, + "tokens_seen": 1847721984 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003557231695085256, + "loss": 1.2219, + "theoretical_loss": 3.481750368667751, + "tokens_seen": 1847787520 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003557071213640923, + "loss": 1.2997, + "theoretical_loss": 3.481739997248054, + "tokens_seen": 1847853056 + }, + { + "epoch": 0.56, + "learning_rate": 0.000355691073219659, + "loss": 1.3898, + "theoretical_loss": 3.4817296262991726, + "tokens_seen": 1847918592 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003556750250752257, + "loss": 1.2225, + "theoretical_loss": 3.481719255821069, + "tokens_seen": 1847984128 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003556589769307924, + "loss": 1.291, + "theoretical_loss": 3.4817088858137044, + "tokens_seen": 1848049664 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 619773, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3261522054672241, + "objective/train/theoretical_loss": 3.4816985162770413, + "objective/train/tokens_used": 1868575200, + "theoretical_loss": 3.4816985162770413, + "tokens_seen": 1848115200 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003556429287863591, + "loss": 1.2588, + "theoretical_loss": 3.4816985162770413, + "tokens_seen": 1848115200 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003556268806419258, + "loss": 1.2595, + "theoretical_loss": 3.4816881472110417, + "tokens_seen": 1848180736 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003556108324974925, + "loss": 1.2753, + "theoretical_loss": 3.481677778615667, + "tokens_seen": 1848246272 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003555947843530592, + "loss": 1.2621, + "theoretical_loss": 3.48166741049088, + "tokens_seen": 1848311808 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003555787362086259, + "loss": 1.167, + "theoretical_loss": 3.481657042836642, + "tokens_seen": 1848377344 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003555626880641926, + "loss": 1.2358, + "theoretical_loss": 3.4816466756529145, + "tokens_seen": 1848442880 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003555466399197593, + "loss": 1.176, + "theoretical_loss": 3.481636308939661, + "tokens_seen": 1848508416 + }, + { + "epoch": 0.56, + "learning_rate": 0.000355530591775326, + "loss": 1.3824, + "theoretical_loss": 3.481625942696842, + "tokens_seen": 1848573952 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003555145436308927, + "loss": 1.3081, + "theoretical_loss": 3.481615576924421, + "tokens_seen": 1848639488 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003554984954864594, + "loss": 1.3239, + "theoretical_loss": 3.481605211622358, + "tokens_seen": 1848705024 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003554824473420261, + "loss": 1.2946, + "theoretical_loss": 3.481594846790616, + "tokens_seen": 1848770560 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003554663991975928, + "loss": 1.2138, + "theoretical_loss": 3.4815844824291577, + "tokens_seen": 1848836096 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003554503510531595, + "loss": 1.1801, + "theoretical_loss": 3.481574118537944, + "tokens_seen": 1848901632 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003554343029087262, + "loss": 1.2425, + "theoretical_loss": 3.4815637551169374, + "tokens_seen": 1848967168 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035541825476429286, + "loss": 1.467, + "theoretical_loss": 3.4815533921660995, + "tokens_seen": 1849032704 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035540220661985956, + "loss": 1.1877, + "theoretical_loss": 3.481543029685393, + "tokens_seen": 1849098240 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035538615847542627, + "loss": 1.2843, + "theoretical_loss": 3.4815326676747795, + "tokens_seen": 1849163776 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035537011033099297, + "loss": 1.3301, + "theoretical_loss": 3.481522306134221, + "tokens_seen": 1849229312 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035535406218655967, + "loss": 1.3068, + "theoretical_loss": 3.481511945063679, + "tokens_seen": 1849294848 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035533801404212637, + "loss": 1.1361, + "theoretical_loss": 3.4815015844631167, + "tokens_seen": 1849360384 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035532196589769307, + "loss": 1.3086, + "theoretical_loss": 3.481491224332496, + "tokens_seen": 1849425920 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035530591775325977, + "loss": 1.1684, + "theoretical_loss": 3.481480864671777, + "tokens_seen": 1849491456 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035528986960882647, + "loss": 1.3715, + "theoretical_loss": 3.4814705054809245, + "tokens_seen": 1849556992 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035527382146439317, + "loss": 1.322, + "theoretical_loss": 3.4814601467598987, + "tokens_seen": 1849622528 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035525777331995987, + "loss": 1.1257, + "theoretical_loss": 3.4814497885086624, + "tokens_seen": 1849688064 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035524172517552657, + "loss": 1.2464, + "theoretical_loss": 3.481439430727177, + "tokens_seen": 1849753600 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035522567703109327, + "loss": 1.1682, + "theoretical_loss": 3.4814290734154048, + "tokens_seen": 1849819136 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035520962888665997, + "loss": 1.2991, + "theoretical_loss": 3.4814187165733084, + "tokens_seen": 1849884672 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003551935807422267, + "loss": 1.1955, + "theoretical_loss": 3.4814083602008496, + "tokens_seen": 1849950208 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003551775325977934, + "loss": 1.326, + "theoretical_loss": 3.48139800429799, + "tokens_seen": 1850015744 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003551614844533601, + "loss": 1.1748, + "theoretical_loss": 3.4813876488646924, + "tokens_seen": 1850081280 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003551454363089268, + "loss": 1.3308, + "theoretical_loss": 3.481377293900918, + "tokens_seen": 1850146816 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003551293881644935, + "loss": 1.2837, + "theoretical_loss": 3.48136693940663, + "tokens_seen": 1850212352 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003551133400200602, + "loss": 1.1984, + "theoretical_loss": 3.481356585381789, + "tokens_seen": 1850277888 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003550972918756269, + "loss": 1.3163, + "theoretical_loss": 3.481346231826359, + "tokens_seen": 1850343424 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003550812437311936, + "loss": 1.2126, + "theoretical_loss": 3.4813358787403006, + "tokens_seen": 1850408960 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003550651955867603, + "loss": 1.2265, + "theoretical_loss": 3.4813255261235763, + "tokens_seen": 1850474496 + }, + { + "epoch": 0.56, + "learning_rate": 0.000355049147442327, + "loss": 1.2722, + "theoretical_loss": 3.481315173976148, + "tokens_seen": 1850540032 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003550330992978937, + "loss": 1.136, + "theoretical_loss": 3.4813048222979783, + "tokens_seen": 1850605568 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003550170511534604, + "loss": 1.2749, + "theoretical_loss": 3.4812944710890292, + "tokens_seen": 1850671104 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003550010030090271, + "loss": 1.3021, + "theoretical_loss": 3.4812841203492626, + "tokens_seen": 1850736640 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003549849548645938, + "loss": 1.2612, + "theoretical_loss": 3.4812737700786407, + "tokens_seen": 1850802176 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003549689067201605, + "loss": 1.273, + "theoretical_loss": 3.4812634202771258, + "tokens_seen": 1850867712 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035495285857572724, + "loss": 1.2622, + "theoretical_loss": 3.4812530709446796, + "tokens_seen": 1850933248 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035493681043129394, + "loss": 1.2257, + "theoretical_loss": 3.481242722081265, + "tokens_seen": 1850998784 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035492076228686064, + "loss": 1.2262, + "theoretical_loss": 3.481232373686843, + "tokens_seen": 1851064320 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035490471414242734, + "loss": 1.2075, + "theoretical_loss": 3.481222025761377, + "tokens_seen": 1851129856 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035488866599799404, + "loss": 1.2839, + "theoretical_loss": 3.4812116783048275, + "tokens_seen": 1851195392 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035487261785356074, + "loss": 1.1574, + "theoretical_loss": 3.4812013313171586, + "tokens_seen": 1851260928 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035485656970912744, + "loss": 1.1607, + "theoretical_loss": 3.4811909847983316, + "tokens_seen": 1851326464 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 620645, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.138369083404541, + "objective/train/theoretical_loss": 3.4811806387483077, + "objective/train/tokens_used": 1871852000, + "theoretical_loss": 3.4811806387483077, + "tokens_seen": 1851392000 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035484052156469414, + "loss": 1.2111, + "theoretical_loss": 3.4811806387483077, + "tokens_seen": 1851392000 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035482447342026084, + "loss": 1.2684, + "theoretical_loss": 3.4811702931670507, + "tokens_seen": 1851457536 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035480842527582754, + "loss": 1.2848, + "theoretical_loss": 3.481159948054522, + "tokens_seen": 1851523072 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003547923771313942, + "loss": 1.185, + "theoretical_loss": 3.4811496034106835, + "tokens_seen": 1851588608 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003547763289869609, + "loss": 1.227, + "theoretical_loss": 3.4811392592354977, + "tokens_seen": 1851654144 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003547602808425276, + "loss": 1.1717, + "theoretical_loss": 3.4811289155289273, + "tokens_seen": 1851719680 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003547442326980943, + "loss": 1.2226, + "theoretical_loss": 3.4811185722909332, + "tokens_seen": 1851785216 + }, + { + "epoch": 0.56, + "learning_rate": 0.000354728184553661, + "loss": 1.3275, + "theoretical_loss": 3.481108229521479, + "tokens_seen": 1851850752 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003547121364092277, + "loss": 1.3113, + "theoretical_loss": 3.4810978872205256, + "tokens_seen": 1851916288 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003546960882647944, + "loss": 1.2839, + "theoretical_loss": 3.4810875453880366, + "tokens_seen": 1851981824 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003546800401203611, + "loss": 1.1094, + "theoretical_loss": 3.481077204023973, + "tokens_seen": 1852047360 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003546639919759278, + "loss": 1.2206, + "theoretical_loss": 3.481066863128297, + "tokens_seen": 1852112896 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003546479438314945, + "loss": 1.1729, + "theoretical_loss": 3.481056522700972, + "tokens_seen": 1852178432 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003546318956870612, + "loss": 1.2434, + "theoretical_loss": 3.4810461827419594, + "tokens_seen": 1852243968 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003546158475426279, + "loss": 1.1684, + "theoretical_loss": 3.481035843251221, + "tokens_seen": 1852309504 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003545997993981946, + "loss": 1.1849, + "theoretical_loss": 3.48102550422872, + "tokens_seen": 1852375040 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003545837512537613, + "loss": 1.1773, + "theoretical_loss": 3.481015165674418, + "tokens_seen": 1852440576 + }, + { + "epoch": 0.56, + "learning_rate": 0.000354567703109328, + "loss": 1.2843, + "theoretical_loss": 3.4810048275882775, + "tokens_seen": 1852506112 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003545516549648947, + "loss": 1.1171, + "theoretical_loss": 3.4809944899702603, + "tokens_seen": 1852571648 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003545356068204614, + "loss": 1.2333, + "theoretical_loss": 3.480984152820329, + "tokens_seen": 1852637184 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003545195586760281, + "loss": 1.3347, + "theoretical_loss": 3.4809738161384463, + "tokens_seen": 1852702720 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003545035105315948, + "loss": 1.3383, + "theoretical_loss": 3.4809634799245734, + "tokens_seen": 1852768256 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003544874623871615, + "loss": 1.3042, + "theoretical_loss": 3.4809531441786734, + "tokens_seen": 1852833792 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003544714142427282, + "loss": 1.3029, + "theoretical_loss": 3.4809428089007084, + "tokens_seen": 1852899328 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003544553660982949, + "loss": 1.2386, + "theoretical_loss": 3.4809324740906407, + "tokens_seen": 1852964864 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003544393179538616, + "loss": 1.2548, + "theoretical_loss": 3.480922139748432, + "tokens_seen": 1853030400 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003544232698094283, + "loss": 1.3125, + "theoretical_loss": 3.480911805874045, + "tokens_seen": 1853095936 + }, + { + "epoch": 0.56, + "learning_rate": 0.000354407221664995, + "loss": 1.3323, + "theoretical_loss": 3.4809014724674423, + "tokens_seen": 1853161472 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003543911735205617, + "loss": 1.3229, + "theoretical_loss": 3.4808911395285858, + "tokens_seen": 1853227008 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003543751253761284, + "loss": 1.2603, + "theoretical_loss": 3.4808808070574377, + "tokens_seen": 1853292544 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003543590772316951, + "loss": 1.2726, + "theoretical_loss": 3.48087047505396, + "tokens_seen": 1853358080 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003543430290872618, + "loss": 1.2289, + "theoretical_loss": 3.4808601435181163, + "tokens_seen": 1853423616 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003543269809428285, + "loss": 1.1585, + "theoretical_loss": 3.480849812449868, + "tokens_seen": 1853489152 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003543109327983952, + "loss": 1.1852, + "theoretical_loss": 3.480839481849177, + "tokens_seen": 1853554688 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003542948846539619, + "loss": 1.2325, + "theoretical_loss": 3.480829151716006, + "tokens_seen": 1853620224 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003542788365095286, + "loss": 1.3029, + "theoretical_loss": 3.480818822050318, + "tokens_seen": 1853685760 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003542627883650953, + "loss": 1.2026, + "theoretical_loss": 3.480808492852074, + "tokens_seen": 1853751296 + }, + { + "epoch": 0.56, + "learning_rate": 0.000354246740220662, + "loss": 1.3862, + "theoretical_loss": 3.4807981641212375, + "tokens_seen": 1853816832 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003542306920762287, + "loss": 1.2485, + "theoretical_loss": 3.48078783585777, + "tokens_seen": 1853882368 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003542146439317954, + "loss": 1.1972, + "theoretical_loss": 3.4807775080616343, + "tokens_seen": 1853947904 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003541985957873621, + "loss": 1.1841, + "theoretical_loss": 3.4807671807327925, + "tokens_seen": 1854013440 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003541825476429288, + "loss": 1.272, + "theoretical_loss": 3.480756853871207, + "tokens_seen": 1854078976 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003541664994984955, + "loss": 1.521, + "theoretical_loss": 3.48074652747684, + "tokens_seen": 1854144512 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003541504513540622, + "loss": 1.2621, + "theoretical_loss": 3.4807362015496546, + "tokens_seen": 1854210048 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003541344032096289, + "loss": 1.3607, + "theoretical_loss": 3.480725876089612, + "tokens_seen": 1854275584 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003541183550651956, + "loss": 1.2133, + "theoretical_loss": 3.480715551096676, + "tokens_seen": 1854341120 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003541023069207623, + "loss": 1.2679, + "theoretical_loss": 3.480705226570807, + "tokens_seen": 1854406656 + }, + { + "epoch": 0.56, + "learning_rate": 0.000354086258776329, + "loss": 1.1958, + "theoretical_loss": 3.4806949025119693, + "tokens_seen": 1854472192 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003540702106318957, + "loss": 1.33, + "theoretical_loss": 3.480684578920124, + "tokens_seen": 1854537728 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003540541624874624, + "loss": 1.2586, + "theoretical_loss": 3.4806742557952344, + "tokens_seen": 1854603264 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 621939, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3597110509872437, + "objective/train/theoretical_loss": 3.4806639331372615, + "objective/train/tokens_used": 1875128800, + "theoretical_loss": 3.4806639331372615, + "tokens_seen": 1854668800 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003540381143430291, + "loss": 1.2847, + "theoretical_loss": 3.4806639331372615, + "tokens_seen": 1854668800 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035402206619859577, + "loss": 1.2603, + "theoretical_loss": 3.4806536109461694, + "tokens_seen": 1854734336 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035400601805416247, + "loss": 1.1308, + "theoretical_loss": 3.4806432892219195, + "tokens_seen": 1854799872 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035398996990972917, + "loss": 1.1318, + "theoretical_loss": 3.4806329679644743, + "tokens_seen": 1854865408 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035397392176529587, + "loss": 1.3945, + "theoretical_loss": 3.480622647173796, + "tokens_seen": 1854930944 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035395787362086257, + "loss": 1.1835, + "theoretical_loss": 3.4806123268498474, + "tokens_seen": 1854996480 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035394182547642927, + "loss": 1.2976, + "theoretical_loss": 3.480602006992591, + "tokens_seen": 1855062016 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035392577733199597, + "loss": 1.4108, + "theoretical_loss": 3.480591687601989, + "tokens_seen": 1855127552 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035390972918756267, + "loss": 1.2807, + "theoretical_loss": 3.4805813686780036, + "tokens_seen": 1855193088 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035389368104312937, + "loss": 1.1922, + "theoretical_loss": 3.4805710502205973, + "tokens_seen": 1855258624 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035387763289869607, + "loss": 1.3072, + "theoretical_loss": 3.4805607322297325, + "tokens_seen": 1855324160 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035386158475426277, + "loss": 1.2357, + "theoretical_loss": 3.4805504147053723, + "tokens_seen": 1855389696 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035384553660982947, + "loss": 1.3204, + "theoretical_loss": 3.480540097647478, + "tokens_seen": 1855455232 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035382948846539617, + "loss": 1.2103, + "theoretical_loss": 3.4805297810560134, + "tokens_seen": 1855520768 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003538134403209629, + "loss": 1.2275, + "theoretical_loss": 3.480519464930939, + "tokens_seen": 1855586304 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003537973921765296, + "loss": 1.2977, + "theoretical_loss": 3.4805091492722195, + "tokens_seen": 1855651840 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003537813440320963, + "loss": 1.2677, + "theoretical_loss": 3.4804988340798158, + "tokens_seen": 1855717376 + }, + { + "epoch": 0.56, + "learning_rate": 0.000353765295887663, + "loss": 1.1013, + "theoretical_loss": 3.480488519353691, + "tokens_seen": 1855782912 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035374924774322973, + "loss": 1.1932, + "theoretical_loss": 3.480478205093807, + "tokens_seen": 1855848448 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035373319959879643, + "loss": 1.3038, + "theoretical_loss": 3.480467891300127, + "tokens_seen": 1855913984 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035371715145436313, + "loss": 1.3431, + "theoretical_loss": 3.4804575779726123, + "tokens_seen": 1855979520 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035370110330992983, + "loss": 1.1593, + "theoretical_loss": 3.480447265111227, + "tokens_seen": 1856045056 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035368505516549653, + "loss": 1.2414, + "theoretical_loss": 3.480436952715933, + "tokens_seen": 1856110592 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035366900702106323, + "loss": 1.2544, + "theoretical_loss": 3.4804266407866917, + "tokens_seen": 1856176128 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035365295887662993, + "loss": 1.2539, + "theoretical_loss": 3.4804163293234667, + "tokens_seen": 1856241664 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035363691073219663, + "loss": 1.2375, + "theoretical_loss": 3.48040601832622, + "tokens_seen": 1856307200 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035362086258776334, + "loss": 1.3539, + "theoretical_loss": 3.480395707794915, + "tokens_seen": 1856372736 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035360481444333004, + "loss": 1.1772, + "theoretical_loss": 3.4803853977295125, + "tokens_seen": 1856438272 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035358876629889674, + "loss": 1.3211, + "theoretical_loss": 3.4803750881299766, + "tokens_seen": 1856503808 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035357271815446344, + "loss": 1.1062, + "theoretical_loss": 3.480364778996269, + "tokens_seen": 1856569344 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035355667001003014, + "loss": 1.2429, + "theoretical_loss": 3.4803544703283524, + "tokens_seen": 1856634880 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035354062186559684, + "loss": 1.3015, + "theoretical_loss": 3.4803441621261895, + "tokens_seen": 1856700416 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035352457372116354, + "loss": 1.1434, + "theoretical_loss": 3.4803338543897424, + "tokens_seen": 1856765952 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035350852557673024, + "loss": 1.2892, + "theoretical_loss": 3.4803235471189735, + "tokens_seen": 1856831488 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035349247743229694, + "loss": 1.3286, + "theoretical_loss": 3.4803132403138464, + "tokens_seen": 1856897024 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035347642928786364, + "loss": 1.2758, + "theoretical_loss": 3.4803029339743228, + "tokens_seen": 1856962560 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035346038114343034, + "loss": 1.1949, + "theoretical_loss": 3.480292628100365, + "tokens_seen": 1857028096 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035344433299899704, + "loss": 1.2342, + "theoretical_loss": 3.480282322691936, + "tokens_seen": 1857093632 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035342828485456374, + "loss": 1.2792, + "theoretical_loss": 3.480272017748998, + "tokens_seen": 1857159168 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035341223671013044, + "loss": 1.2919, + "theoretical_loss": 3.4802617132715143, + "tokens_seen": 1857224704 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035339618856569714, + "loss": 1.3163, + "theoretical_loss": 3.4802514092594468, + "tokens_seen": 1857290240 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003533801404212638, + "loss": 1.1526, + "theoretical_loss": 3.4802411057127576, + "tokens_seen": 1857355776 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003533640922768305, + "loss": 1.3376, + "theoretical_loss": 3.4802308026314104, + "tokens_seen": 1857421312 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003533480441323972, + "loss": 1.2372, + "theoretical_loss": 3.480220500015367, + "tokens_seen": 1857486848 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003533319959879639, + "loss": 1.1392, + "theoretical_loss": 3.48021019786459, + "tokens_seen": 1857552384 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003533159478435306, + "loss": 1.2869, + "theoretical_loss": 3.480199896179043, + "tokens_seen": 1857617920 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003532998996990973, + "loss": 1.0969, + "theoretical_loss": 3.480189594958687, + "tokens_seen": 1857683456 + }, + { + "epoch": 0.56, + "learning_rate": 0.000353283851554664, + "loss": 1.2201, + "theoretical_loss": 3.4801792942034853, + "tokens_seen": 1857748992 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003532678034102307, + "loss": 1.2125, + "theoretical_loss": 3.480168993913401, + "tokens_seen": 1857814528 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003532517552657974, + "loss": 1.2873, + "theoretical_loss": 3.480158694088396, + "tokens_seen": 1857880064 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 622851, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1701698303222656, + "objective/train/theoretical_loss": 3.480148394728433, + "objective/train/tokens_used": 1878405600, + "theoretical_loss": 3.480148394728433, + "tokens_seen": 1857945600 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003532357071213641, + "loss": 1.2884, + "theoretical_loss": 3.480148394728433, + "tokens_seen": 1857945600 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003532196589769308, + "loss": 1.2617, + "theoretical_loss": 3.480138095833474, + "tokens_seen": 1858011136 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003532036108324975, + "loss": 1.2999, + "theoretical_loss": 3.480127797403483, + "tokens_seen": 1858076672 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003531875626880642, + "loss": 1.3064, + "theoretical_loss": 3.480117499438422, + "tokens_seen": 1858142208 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003531715145436309, + "loss": 1.313, + "theoretical_loss": 3.480107201938254, + "tokens_seen": 1858207744 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003531554663991976, + "loss": 1.2608, + "theoretical_loss": 3.48009690490294, + "tokens_seen": 1858273280 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003531394182547643, + "loss": 1.2381, + "theoretical_loss": 3.4800866083324444, + "tokens_seen": 1858338816 + }, + { + "epoch": 0.56, + "learning_rate": 0.000353123370110331, + "loss": 1.2377, + "theoretical_loss": 3.4800763122267293, + "tokens_seen": 1858404352 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003531073219658977, + "loss": 1.2618, + "theoretical_loss": 3.480066016585757, + "tokens_seen": 1858469888 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003530912738214644, + "loss": 1.2694, + "theoretical_loss": 3.4800557214094905, + "tokens_seen": 1858535424 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003530752256770311, + "loss": 1.1255, + "theoretical_loss": 3.4800454266978917, + "tokens_seen": 1858600960 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003530591775325978, + "loss": 1.2449, + "theoretical_loss": 3.480035132450925, + "tokens_seen": 1858666496 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003530431293881645, + "loss": 1.2354, + "theoretical_loss": 3.480024838668551, + "tokens_seen": 1858732032 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003530270812437312, + "loss": 1.3882, + "theoretical_loss": 3.480014545350733, + "tokens_seen": 1858797568 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003530110330992979, + "loss": 1.364, + "theoretical_loss": 3.4800042524974346, + "tokens_seen": 1858863104 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003529949849548646, + "loss": 1.2397, + "theoretical_loss": 3.479993960108618, + "tokens_seen": 1858928640 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003529789368104313, + "loss": 1.246, + "theoretical_loss": 3.4799836681842446, + "tokens_seen": 1858994176 + }, + { + "epoch": 0.56, + "learning_rate": 0.000352962888665998, + "loss": 1.1804, + "theoretical_loss": 3.4799733767242786, + "tokens_seen": 1859059712 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003529468405215647, + "loss": 1.2561, + "theoretical_loss": 3.4799630857286825, + "tokens_seen": 1859125248 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003529307923771314, + "loss": 1.2491, + "theoretical_loss": 3.479952795197418, + "tokens_seen": 1859190784 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003529147442326981, + "loss": 1.2596, + "theoretical_loss": 3.479942505130449, + "tokens_seen": 1859256320 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003528986960882648, + "loss": 1.2859, + "theoretical_loss": 3.4799322155277377, + "tokens_seen": 1859321856 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003528826479438315, + "loss": 1.2669, + "theoretical_loss": 3.4799219263892462, + "tokens_seen": 1859387392 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003528665997993982, + "loss": 1.2556, + "theoretical_loss": 3.479911637714938, + "tokens_seen": 1859452928 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003528505516549649, + "loss": 1.326, + "theoretical_loss": 3.4799013495047753, + "tokens_seen": 1859518464 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003528345035105316, + "loss": 1.2371, + "theoretical_loss": 3.479891061758721, + "tokens_seen": 1859584000 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003528184553660983, + "loss": 1.2856, + "theoretical_loss": 3.479880774476738, + "tokens_seen": 1859649536 + }, + { + "epoch": 0.56, + "learning_rate": 0.000352802407221665, + "loss": 1.2215, + "theoretical_loss": 3.479870487658789, + "tokens_seen": 1859715072 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003527863590772317, + "loss": 1.1979, + "theoretical_loss": 3.4798602013048363, + "tokens_seen": 1859780608 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003527703109327984, + "loss": 1.2099, + "theoretical_loss": 3.479849915414843, + "tokens_seen": 1859846144 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003527542627883651, + "loss": 1.2611, + "theoretical_loss": 3.4798396299887715, + "tokens_seen": 1859911680 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003527382146439318, + "loss": 1.2559, + "theoretical_loss": 3.4798293450265847, + "tokens_seen": 1859977216 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003527221664994985, + "loss": 1.2405, + "theoretical_loss": 3.4798190605282455, + "tokens_seen": 1860042752 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003527061183550652, + "loss": 1.207, + "theoretical_loss": 3.4798087764937167, + "tokens_seen": 1860108288 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003526900702106319, + "loss": 1.27, + "theoretical_loss": 3.4797984929229604, + "tokens_seen": 1860173824 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003526740220661986, + "loss": 1.2256, + "theoretical_loss": 3.47978820981594, + "tokens_seen": 1860239360 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003526579739217653, + "loss": 1.2341, + "theoretical_loss": 3.479777927172618, + "tokens_seen": 1860304896 + }, + { + "epoch": 0.56, + "learning_rate": 0.000352641925777332, + "loss": 1.3621, + "theoretical_loss": 3.479767644992957, + "tokens_seen": 1860370432 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003526258776328987, + "loss": 1.1453, + "theoretical_loss": 3.4797573632769203, + "tokens_seen": 1860435968 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035260982948846537, + "loss": 1.2651, + "theoretical_loss": 3.4797470820244705, + "tokens_seen": 1860501504 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035259378134403207, + "loss": 1.2691, + "theoretical_loss": 3.4797368012355694, + "tokens_seen": 1860567040 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035257773319959877, + "loss": 1.298, + "theoretical_loss": 3.479726520910181, + "tokens_seen": 1860632576 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035256168505516547, + "loss": 1.2245, + "theoretical_loss": 3.4797162410482683, + "tokens_seen": 1860698112 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003525456369107322, + "loss": 1.2671, + "theoretical_loss": 3.4797059616497927, + "tokens_seen": 1860763648 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003525295887662989, + "loss": 1.1978, + "theoretical_loss": 3.4796956827147176, + "tokens_seen": 1860829184 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003525135406218656, + "loss": 1.1468, + "theoretical_loss": 3.479685404243006, + "tokens_seen": 1860894720 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003524974924774323, + "loss": 1.2522, + "theoretical_loss": 3.479675126234621, + "tokens_seen": 1860960256 + }, + { + "epoch": 0.56, + "learning_rate": 0.000352481444332999, + "loss": 1.2432, + "theoretical_loss": 3.4796648486895245, + "tokens_seen": 1861025792 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003524653961885657, + "loss": 1.1668, + "theoretical_loss": 3.4796545716076803, + "tokens_seen": 1861091328 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035244934804413243, + "loss": 1.1882, + "theoretical_loss": 3.4796442949890505, + "tokens_seen": 1861156864 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 624035, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2446659803390503, + "objective/train/theoretical_loss": 3.4796340188335977, + "objective/train/tokens_used": 1881682400, + "theoretical_loss": 3.4796340188335977, + "tokens_seen": 1861222400 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035243329989969913, + "loss": 1.1809, + "theoretical_loss": 3.4796340188335977, + "tokens_seen": 1861222400 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035241725175526583, + "loss": 1.1594, + "theoretical_loss": 3.4796237431412855, + "tokens_seen": 1861287936 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035240120361083253, + "loss": 1.2682, + "theoretical_loss": 3.4796134679120767, + "tokens_seen": 1861353472 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035238515546639923, + "loss": 1.4019, + "theoretical_loss": 3.479603193145933, + "tokens_seen": 1861419008 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035236910732196593, + "loss": 1.3315, + "theoretical_loss": 3.4795929188428185, + "tokens_seen": 1861484544 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035235305917753263, + "loss": 1.2562, + "theoretical_loss": 3.4795826450026954, + "tokens_seen": 1861550080 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035233701103309933, + "loss": 1.2403, + "theoretical_loss": 3.479572371625527, + "tokens_seen": 1861615616 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035232096288866603, + "loss": 1.1111, + "theoretical_loss": 3.479562098711275, + "tokens_seen": 1861681152 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035230491474423273, + "loss": 1.2603, + "theoretical_loss": 3.479551826259904, + "tokens_seen": 1861746688 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035228886659979943, + "loss": 1.2011, + "theoretical_loss": 3.4795415542713757, + "tokens_seen": 1861812224 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035227281845536613, + "loss": 1.2842, + "theoretical_loss": 3.479531282745653, + "tokens_seen": 1861877760 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035225677031093283, + "loss": 1.2741, + "theoretical_loss": 3.4795210116826985, + "tokens_seen": 1861943296 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035224072216649954, + "loss": 1.2592, + "theoretical_loss": 3.4795107410824757, + "tokens_seen": 1862008832 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035222467402206624, + "loss": 1.202, + "theoretical_loss": 3.4795004709449477, + "tokens_seen": 1862074368 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035220862587763294, + "loss": 1.3344, + "theoretical_loss": 3.479490201270077, + "tokens_seen": 1862139904 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035219257773319964, + "loss": 1.1294, + "theoretical_loss": 3.479479932057826, + "tokens_seen": 1862205440 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035217652958876634, + "loss": 1.3079, + "theoretical_loss": 3.479469663308158, + "tokens_seen": 1862270976 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035216048144433304, + "loss": 1.3409, + "theoretical_loss": 3.4794593950210357, + "tokens_seen": 1862336512 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035214443329989974, + "loss": 1.1971, + "theoretical_loss": 3.479449127196423, + "tokens_seen": 1862402048 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035212838515546644, + "loss": 1.1932, + "theoretical_loss": 3.479438859834281, + "tokens_seen": 1862467584 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035211233701103314, + "loss": 1.204, + "theoretical_loss": 3.479428592934574, + "tokens_seen": 1862533120 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035209628886659984, + "loss": 1.1678, + "theoretical_loss": 3.4794183264972647, + "tokens_seen": 1862598656 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035208024072216654, + "loss": 1.3539, + "theoretical_loss": 3.4794080605223154, + "tokens_seen": 1862664192 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035206419257773324, + "loss": 1.3683, + "theoretical_loss": 3.4793977950096893, + "tokens_seen": 1862729728 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035204814443329994, + "loss": 1.3091, + "theoretical_loss": 3.4793875299593493, + "tokens_seen": 1862795264 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035203209628886664, + "loss": 1.2677, + "theoretical_loss": 3.4793772653712587, + "tokens_seen": 1862860800 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035201604814443334, + "loss": 1.2411, + "theoretical_loss": 3.47936700124538, + "tokens_seen": 1862926336 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035200000000000005, + "loss": 1.2611, + "theoretical_loss": 3.4793567375816767, + "tokens_seen": 1862991872 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003519839518555667, + "loss": 1.2394, + "theoretical_loss": 3.479346474380111, + "tokens_seen": 1863057408 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003519679037111334, + "loss": 1.245, + "theoretical_loss": 3.4793362116406454, + "tokens_seen": 1863122944 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003519518555667001, + "loss": 1.2949, + "theoretical_loss": 3.4793259493632442, + "tokens_seen": 1863188480 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003519358074222668, + "loss": 1.3093, + "theoretical_loss": 3.47931568754787, + "tokens_seen": 1863254016 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003519197592778335, + "loss": 1.3423, + "theoretical_loss": 3.479305426194485, + "tokens_seen": 1863319552 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003519037111334002, + "loss": 1.3179, + "theoretical_loss": 3.4792951653030526, + "tokens_seen": 1863385088 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003518876629889669, + "loss": 1.3124, + "theoretical_loss": 3.4792849048735364, + "tokens_seen": 1863450624 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003518716148445336, + "loss": 1.2778, + "theoretical_loss": 3.479274644905898, + "tokens_seen": 1863516160 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003518555667001003, + "loss": 1.2661, + "theoretical_loss": 3.4792643854001017, + "tokens_seen": 1863581696 + }, + { + "epoch": 0.56, + "learning_rate": 0.000351839518555667, + "loss": 1.3245, + "theoretical_loss": 3.4792541263561096, + "tokens_seen": 1863647232 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003518234704112337, + "loss": 1.2322, + "theoretical_loss": 3.4792438677738846, + "tokens_seen": 1863712768 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003518074222668004, + "loss": 1.1972, + "theoretical_loss": 3.4792336096533907, + "tokens_seen": 1863778304 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003517913741223671, + "loss": 1.1831, + "theoretical_loss": 3.479223351994589, + "tokens_seen": 1863843840 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003517753259779338, + "loss": 1.2943, + "theoretical_loss": 3.479213094797445, + "tokens_seen": 1863909376 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003517592778335005, + "loss": 1.3168, + "theoretical_loss": 3.4792028380619198, + "tokens_seen": 1863974912 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003517432296890672, + "loss": 1.1766, + "theoretical_loss": 3.479192581787977, + "tokens_seen": 1864040448 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003517271815446339, + "loss": 1.2784, + "theoretical_loss": 3.4791823259755796, + "tokens_seen": 1864105984 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003517111334002006, + "loss": 1.302, + "theoretical_loss": 3.479172070624691, + "tokens_seen": 1864171520 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003516950852557673, + "loss": 1.2858, + "theoretical_loss": 3.4791618157352735, + "tokens_seen": 1864237056 + }, + { + "epoch": 0.56, + "learning_rate": 0.000351679037111334, + "loss": 1.2243, + "theoretical_loss": 3.4791515613072903, + "tokens_seen": 1864302592 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003516629889669007, + "loss": 1.3262, + "theoretical_loss": 3.4791413073407043, + "tokens_seen": 1864368128 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003516469408224674, + "loss": 1.2562, + "theoretical_loss": 3.479131053835479, + "tokens_seen": 1864433664 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 625017, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2231545448303223, + "objective/train/theoretical_loss": 3.479120800791577, + "objective/train/tokens_used": 1884959200, + "theoretical_loss": 3.479120800791577, + "tokens_seen": 1864499200 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003516308926780341, + "loss": 1.2109, + "theoretical_loss": 3.479120800791577, + "tokens_seen": 1864499200 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003516148445336008, + "loss": 1.2546, + "theoretical_loss": 3.479110548208962, + "tokens_seen": 1864564736 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003515987963891675, + "loss": 1.2542, + "theoretical_loss": 3.479100296087596, + "tokens_seen": 1864630272 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003515827482447342, + "loss": 1.1893, + "theoretical_loss": 3.479090044427443, + "tokens_seen": 1864695808 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003515667001003009, + "loss": 1.2244, + "theoretical_loss": 3.4790797932284656, + "tokens_seen": 1864761344 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003515506519558676, + "loss": 1.3097, + "theoretical_loss": 3.4790695424906266, + "tokens_seen": 1864826880 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003515346038114343, + "loss": 1.1156, + "theoretical_loss": 3.479059292213889, + "tokens_seen": 1864892416 + }, + { + "epoch": 0.57, + "learning_rate": 0.000351518555667001, + "loss": 1.2433, + "theoretical_loss": 3.479049042398217, + "tokens_seen": 1864957952 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003515025075225677, + "loss": 1.2961, + "theoretical_loss": 3.479038793043572, + "tokens_seen": 1865023488 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003514864593781344, + "loss": 1.1828, + "theoretical_loss": 3.4790285441499185, + "tokens_seen": 1865089024 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003514704112337011, + "loss": 1.1852, + "theoretical_loss": 3.4790182957172187, + "tokens_seen": 1865154560 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003514543630892678, + "loss": 1.2386, + "theoretical_loss": 3.479008047745436, + "tokens_seen": 1865220096 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003514383149448345, + "loss": 1.3009, + "theoretical_loss": 3.4789978002345334, + "tokens_seen": 1865285632 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003514222668004012, + "loss": 1.309, + "theoretical_loss": 3.4789875531844743, + "tokens_seen": 1865351168 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003514062186559679, + "loss": 1.3046, + "theoretical_loss": 3.4789773065952208, + "tokens_seen": 1865416704 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003513901705115346, + "loss": 1.2765, + "theoretical_loss": 3.4789670604667373, + "tokens_seen": 1865482240 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003513741223671013, + "loss": 1.2994, + "theoretical_loss": 3.4789568147989858, + "tokens_seen": 1865547776 + }, + { + "epoch": 0.57, + "learning_rate": 0.000351358074222668, + "loss": 1.3446, + "theoretical_loss": 3.47894656959193, + "tokens_seen": 1865613312 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003513420260782347, + "loss": 1.173, + "theoretical_loss": 3.478936324845533, + "tokens_seen": 1865678848 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003513259779338014, + "loss": 1.3353, + "theoretical_loss": 3.478926080559758, + "tokens_seen": 1865744384 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003513099297893681, + "loss": 1.2428, + "theoretical_loss": 3.4789158367345676, + "tokens_seen": 1865809920 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003512938816449348, + "loss": 1.2132, + "theoretical_loss": 3.478905593369925, + "tokens_seen": 1865875456 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003512778335005015, + "loss": 1.1979, + "theoretical_loss": 3.478895350465794, + "tokens_seen": 1865940992 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003512617853560682, + "loss": 1.3416, + "theoretical_loss": 3.478885108022137, + "tokens_seen": 1866006528 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003512457372116349, + "loss": 1.4879, + "theoretical_loss": 3.4788748660389173, + "tokens_seen": 1866072064 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003512296890672016, + "loss": 1.3334, + "theoretical_loss": 3.478864624516098, + "tokens_seen": 1866137600 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003512136409227683, + "loss": 1.2625, + "theoretical_loss": 3.4788543834536423, + "tokens_seen": 1866203136 + }, + { + "epoch": 0.57, + "learning_rate": 0.000351197592778335, + "loss": 1.1902, + "theoretical_loss": 3.4788441428515133, + "tokens_seen": 1866268672 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003511815446339017, + "loss": 1.1718, + "theoretical_loss": 3.478833902709675, + "tokens_seen": 1866334208 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003511654964894684, + "loss": 1.3965, + "theoretical_loss": 3.4788236630280887, + "tokens_seen": 1866399744 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003511494483450351, + "loss": 1.2303, + "theoretical_loss": 3.478813423806719, + "tokens_seen": 1866465280 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003511334002006018, + "loss": 1.1672, + "theoretical_loss": 3.478803185045529, + "tokens_seen": 1866530816 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003511173520561685, + "loss": 1.2076, + "theoretical_loss": 3.4787929467444814, + "tokens_seen": 1866596352 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003511013039117352, + "loss": 1.2809, + "theoretical_loss": 3.4787827089035397, + "tokens_seen": 1866661888 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003510852557673019, + "loss": 1.2007, + "theoretical_loss": 3.4787724715226664, + "tokens_seen": 1866727424 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035106920762286863, + "loss": 1.1515, + "theoretical_loss": 3.4787622346018257, + "tokens_seen": 1866792960 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035105315947843533, + "loss": 1.2986, + "theoretical_loss": 3.47875199814098, + "tokens_seen": 1866858496 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035103711133400203, + "loss": 1.2965, + "theoretical_loss": 3.4787417621400927, + "tokens_seen": 1866924032 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035102106318956873, + "loss": 1.3908, + "theoretical_loss": 3.4787315265991268, + "tokens_seen": 1866989568 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035100501504513543, + "loss": 1.2961, + "theoretical_loss": 3.4787212915180463, + "tokens_seen": 1867055104 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035098896690070213, + "loss": 1.1641, + "theoretical_loss": 3.478711056896813, + "tokens_seen": 1867120640 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035097291875626883, + "loss": 1.2166, + "theoretical_loss": 3.4787008227353917, + "tokens_seen": 1867186176 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035095687061183553, + "loss": 1.3288, + "theoretical_loss": 3.478690589033744, + "tokens_seen": 1867251712 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035094082246740223, + "loss": 1.2192, + "theoretical_loss": 3.4786803557918344, + "tokens_seen": 1867317248 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035092477432296893, + "loss": 1.2304, + "theoretical_loss": 3.4786701230096257, + "tokens_seen": 1867382784 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035090872617853563, + "loss": 1.2213, + "theoretical_loss": 3.478659890687081, + "tokens_seen": 1867448320 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035089267803410233, + "loss": 1.3573, + "theoretical_loss": 3.478649658824163, + "tokens_seen": 1867513856 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035087662988966904, + "loss": 1.2319, + "theoretical_loss": 3.478639427420836, + "tokens_seen": 1867579392 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035086058174523574, + "loss": 1.2222, + "theoretical_loss": 3.478629196477063, + "tokens_seen": 1867644928 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035084453360080244, + "loss": 1.3038, + "theoretical_loss": 3.4786189659928057, + "tokens_seen": 1867710464 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 626305, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3878669738769531, + "objective/train/theoretical_loss": 3.47860873596803, + "objective/train/tokens_used": 1888236000, + "theoretical_loss": 3.47860873596803, + "tokens_seen": 1867776000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035082848545636914, + "loss": 1.3074, + "theoretical_loss": 3.47860873596803, + "tokens_seen": 1867776000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035081243731193584, + "loss": 1.2039, + "theoretical_loss": 3.478598506402697, + "tokens_seen": 1867841536 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035079638916750254, + "loss": 1.3566, + "theoretical_loss": 3.4785882772967707, + "tokens_seen": 1867907072 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035078034102306924, + "loss": 1.3228, + "theoretical_loss": 3.4785780486502142, + "tokens_seen": 1867972608 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035076429287863594, + "loss": 1.2506, + "theoretical_loss": 3.4785678204629917, + "tokens_seen": 1868038144 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035074824473420264, + "loss": 1.3418, + "theoretical_loss": 3.478557592735065, + "tokens_seen": 1868103680 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035073219658976934, + "loss": 1.2865, + "theoretical_loss": 3.478547365466398, + "tokens_seen": 1868169216 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035071614844533604, + "loss": 1.239, + "theoretical_loss": 3.478537138656954, + "tokens_seen": 1868234752 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035070010030090274, + "loss": 1.2603, + "theoretical_loss": 3.4785269123066964, + "tokens_seen": 1868300288 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035068405215646944, + "loss": 1.1591, + "theoretical_loss": 3.4785166864155883, + "tokens_seen": 1868365824 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035066800401203614, + "loss": 1.2985, + "theoretical_loss": 3.478506460983593, + "tokens_seen": 1868431360 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035065195586760284, + "loss": 1.2762, + "theoretical_loss": 3.478496236010673, + "tokens_seen": 1868496896 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035063590772316954, + "loss": 1.1663, + "theoretical_loss": 3.4784860114967935, + "tokens_seen": 1868562432 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035061985957873625, + "loss": 1.2888, + "theoretical_loss": 3.478475787441916, + "tokens_seen": 1868627968 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035060381143430295, + "loss": 1.2325, + "theoretical_loss": 3.478465563846005, + "tokens_seen": 1868693504 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035058776328986965, + "loss": 1.3075, + "theoretical_loss": 3.478455340709023, + "tokens_seen": 1868759040 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003505717151454363, + "loss": 1.2535, + "theoretical_loss": 3.4784451180309333, + "tokens_seen": 1868824576 + }, + { + "epoch": 0.57, + "learning_rate": 0.000350555667001003, + "loss": 1.2611, + "theoretical_loss": 3.4784348958116995, + "tokens_seen": 1868890112 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003505396188565697, + "loss": 1.2416, + "theoretical_loss": 3.4784246740512854, + "tokens_seen": 1868955648 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003505235707121364, + "loss": 1.2366, + "theoretical_loss": 3.4784144527496537, + "tokens_seen": 1869021184 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003505075225677031, + "loss": 1.3414, + "theoretical_loss": 3.4784042319067674, + "tokens_seen": 1869086720 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003504914744232698, + "loss": 1.1661, + "theoretical_loss": 3.4783940115225906, + "tokens_seen": 1869152256 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003504754262788365, + "loss": 1.1315, + "theoretical_loss": 3.478383791597086, + "tokens_seen": 1869217792 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003504593781344032, + "loss": 1.2743, + "theoretical_loss": 3.4783735721302174, + "tokens_seen": 1869283328 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003504433299899699, + "loss": 1.2474, + "theoretical_loss": 3.4783633531219484, + "tokens_seen": 1869348864 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003504272818455366, + "loss": 1.2987, + "theoretical_loss": 3.478353134572241, + "tokens_seen": 1869414400 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003504112337011033, + "loss": 1.3324, + "theoretical_loss": 3.4783429164810604, + "tokens_seen": 1869479936 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035039518555667, + "loss": 1.2863, + "theoretical_loss": 3.4783326988483685, + "tokens_seen": 1869545472 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003503791374122367, + "loss": 1.3066, + "theoretical_loss": 3.4783224816741294, + "tokens_seen": 1869611008 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003503630892678034, + "loss": 1.2684, + "theoretical_loss": 3.478312264958306, + "tokens_seen": 1869676544 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003503470411233701, + "loss": 1.3193, + "theoretical_loss": 3.478302048700862, + "tokens_seen": 1869742080 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003503309929789368, + "loss": 1.3287, + "theoretical_loss": 3.4782918329017605, + "tokens_seen": 1869807616 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003503149448345035, + "loss": 1.3643, + "theoretical_loss": 3.478281617560965, + "tokens_seen": 1869873152 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003502988966900702, + "loss": 1.3551, + "theoretical_loss": 3.4782714026784394, + "tokens_seen": 1869938688 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003502828485456369, + "loss": 1.2436, + "theoretical_loss": 3.4782611882541463, + "tokens_seen": 1870004224 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003502668004012036, + "loss": 1.3334, + "theoretical_loss": 3.4782509742880494, + "tokens_seen": 1870069760 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003502507522567703, + "loss": 1.3374, + "theoretical_loss": 3.478240760780112, + "tokens_seen": 1870135296 + }, + { + "epoch": 0.57, + "learning_rate": 0.000350234704112337, + "loss": 1.2015, + "theoretical_loss": 3.4782305477302975, + "tokens_seen": 1870200832 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003502186559679037, + "loss": 1.2048, + "theoretical_loss": 3.4782203351385697, + "tokens_seen": 1870266368 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003502026078234704, + "loss": 1.2261, + "theoretical_loss": 3.4782101230048914, + "tokens_seen": 1870331904 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003501865596790371, + "loss": 1.2746, + "theoretical_loss": 3.4781999113292263, + "tokens_seen": 1870397440 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003501705115346038, + "loss": 1.3215, + "theoretical_loss": 3.4781897001115376, + "tokens_seen": 1870462976 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003501544633901705, + "loss": 1.3354, + "theoretical_loss": 3.478179489351789, + "tokens_seen": 1870528512 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035013841524573726, + "loss": 1.2014, + "theoretical_loss": 3.4781692790499443, + "tokens_seen": 1870594048 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035012236710130397, + "loss": 1.3876, + "theoretical_loss": 3.478159069205966, + "tokens_seen": 1870659584 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035010631895687067, + "loss": 1.2212, + "theoretical_loss": 3.478148859819818, + "tokens_seen": 1870725120 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035009027081243737, + "loss": 1.2918, + "theoretical_loss": 3.478138650891464, + "tokens_seen": 1870790656 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035007422266800407, + "loss": 1.3017, + "theoretical_loss": 3.4781284424208665, + "tokens_seen": 1870856192 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035005817452357077, + "loss": 1.1694, + "theoretical_loss": 3.47811823440799, + "tokens_seen": 1870921728 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035004212637913747, + "loss": 1.23, + "theoretical_loss": 3.478108026852797, + "tokens_seen": 1870987264 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 627508, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2596744298934937, + "objective/train/theoretical_loss": 3.4780978197552526, + "objective/train/tokens_used": 1891512800, + "theoretical_loss": 3.4780978197552526, + "tokens_seen": 1871052800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035002607823470417, + "loss": 1.2767, + "theoretical_loss": 3.4780978197552526, + "tokens_seen": 1871052800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035001003009027087, + "loss": 1.1887, + "theoretical_loss": 3.4780876131153184, + "tokens_seen": 1871118336 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034999398194583757, + "loss": 1.2981, + "theoretical_loss": 3.4780774069329583, + "tokens_seen": 1871183872 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034997793380140427, + "loss": 1.2167, + "theoretical_loss": 3.4780672012081366, + "tokens_seen": 1871249408 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034996188565697097, + "loss": 1.307, + "theoretical_loss": 3.478056995940816, + "tokens_seen": 1871314944 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003499458375125376, + "loss": 1.2567, + "theoretical_loss": 3.4780467911309603, + "tokens_seen": 1871380480 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003499297893681043, + "loss": 1.2335, + "theoretical_loss": 3.478036586778533, + "tokens_seen": 1871446016 + }, + { + "epoch": 0.57, + "learning_rate": 0.000349913741223671, + "loss": 1.1905, + "theoretical_loss": 3.478026382883497, + "tokens_seen": 1871511552 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003498976930792377, + "loss": 1.3345, + "theoretical_loss": 3.4780161794458166, + "tokens_seen": 1871577088 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003498816449348044, + "loss": 1.2942, + "theoretical_loss": 3.4780059764654547, + "tokens_seen": 1871642624 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003498655967903711, + "loss": 1.2912, + "theoretical_loss": 3.477995773942375, + "tokens_seen": 1871708160 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003498495486459378, + "loss": 1.2644, + "theoretical_loss": 3.477985571876541, + "tokens_seen": 1871773696 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003498335005015045, + "loss": 1.3195, + "theoretical_loss": 3.477975370267916, + "tokens_seen": 1871839232 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003498174523570712, + "loss": 1.3754, + "theoretical_loss": 3.477965169116464, + "tokens_seen": 1871904768 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003498014042126379, + "loss": 1.4017, + "theoretical_loss": 3.4779549684221482, + "tokens_seen": 1871970304 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003497853560682046, + "loss": 1.223, + "theoretical_loss": 3.477944768184932, + "tokens_seen": 1872035840 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003497693079237713, + "loss": 1.3074, + "theoretical_loss": 3.477934568404779, + "tokens_seen": 1872101376 + }, + { + "epoch": 0.57, + "learning_rate": 0.000349753259779338, + "loss": 1.2329, + "theoretical_loss": 3.477924369081653, + "tokens_seen": 1872166912 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003497372116349047, + "loss": 1.335, + "theoretical_loss": 3.477914170215517, + "tokens_seen": 1872232448 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003497211634904714, + "loss": 1.3182, + "theoretical_loss": 3.4779039718063345, + "tokens_seen": 1872297984 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034970511534603813, + "loss": 1.3578, + "theoretical_loss": 3.4778937738540696, + "tokens_seen": 1872363520 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034968906720160483, + "loss": 1.2558, + "theoretical_loss": 3.477883576358686, + "tokens_seen": 1872429056 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034967301905717153, + "loss": 1.2519, + "theoretical_loss": 3.4778733793201466, + "tokens_seen": 1872494592 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034965697091273823, + "loss": 1.2095, + "theoretical_loss": 3.4778631827384148, + "tokens_seen": 1872560128 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034964092276830493, + "loss": 1.341, + "theoretical_loss": 3.477852986613455, + "tokens_seen": 1872625664 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034962487462387163, + "loss": 1.3427, + "theoretical_loss": 3.47784279094523, + "tokens_seen": 1872691200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034960882647943833, + "loss": 1.3541, + "theoretical_loss": 3.4778325957337035, + "tokens_seen": 1872756736 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034959277833500503, + "loss": 1.2472, + "theoretical_loss": 3.477822400978839, + "tokens_seen": 1872822272 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034957673019057173, + "loss": 1.2933, + "theoretical_loss": 3.4778122066806008, + "tokens_seen": 1872887808 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034956068204613843, + "loss": 1.1452, + "theoretical_loss": 3.4778020128389517, + "tokens_seen": 1872953344 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034954463390170513, + "loss": 1.2915, + "theoretical_loss": 3.4777918194538557, + "tokens_seen": 1873018880 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034952858575727183, + "loss": 1.1957, + "theoretical_loss": 3.477781626525276, + "tokens_seen": 1873084416 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034951253761283853, + "loss": 1.2944, + "theoretical_loss": 3.4777714340531762, + "tokens_seen": 1873149952 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034949648946840524, + "loss": 1.2395, + "theoretical_loss": 3.47776124203752, + "tokens_seen": 1873215488 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034948044132397194, + "loss": 1.2769, + "theoretical_loss": 3.477751050478271, + "tokens_seen": 1873281024 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034946439317953864, + "loss": 1.2623, + "theoretical_loss": 3.477740859375393, + "tokens_seen": 1873346560 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034944834503510534, + "loss": 1.3131, + "theoretical_loss": 3.4777306687288494, + "tokens_seen": 1873412096 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034943229689067204, + "loss": 1.2359, + "theoretical_loss": 3.477720478538604, + "tokens_seen": 1873477632 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034941624874623874, + "loss": 1.2749, + "theoretical_loss": 3.4777102888046203, + "tokens_seen": 1873543168 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034940020060180544, + "loss": 1.2203, + "theoretical_loss": 3.4777000995268614, + "tokens_seen": 1873608704 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034938415245737214, + "loss": 1.296, + "theoretical_loss": 3.477689910705292, + "tokens_seen": 1873674240 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034936810431293884, + "loss": 1.2472, + "theoretical_loss": 3.4776797223398743, + "tokens_seen": 1873739776 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034935205616850554, + "loss": 1.3447, + "theoretical_loss": 3.4776695344305733, + "tokens_seen": 1873805312 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034933600802407224, + "loss": 1.3039, + "theoretical_loss": 3.477659346977352, + "tokens_seen": 1873870848 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034931995987963894, + "loss": 1.2965, + "theoretical_loss": 3.4776491599801735, + "tokens_seen": 1873936384 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034930391173520564, + "loss": 1.2527, + "theoretical_loss": 3.477638973439003, + "tokens_seen": 1874001920 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034928786359077234, + "loss": 1.2741, + "theoretical_loss": 3.4776287873538023, + "tokens_seen": 1874067456 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034927181544633904, + "loss": 1.3203, + "theoretical_loss": 3.4776186017245365, + "tokens_seen": 1874132992 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034925576730190575, + "loss": 1.1955, + "theoretical_loss": 3.4776084165511683, + "tokens_seen": 1874198528 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034923971915747245, + "loss": 1.2311, + "theoretical_loss": 3.4775982318336616, + "tokens_seen": 1874264064 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 628392, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2881674766540527, + "objective/train/theoretical_loss": 3.47758804757198, + "objective/train/tokens_used": 1894789600, + "theoretical_loss": 3.47758804757198, + "tokens_seen": 1874329600 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034922367101303915, + "loss": 1.2322, + "theoretical_loss": 3.47758804757198, + "tokens_seen": 1874329600 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034920762286860585, + "loss": 1.2592, + "theoretical_loss": 3.477577863766088, + "tokens_seen": 1874395136 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034919157472417255, + "loss": 1.3634, + "theoretical_loss": 3.477567680415948, + "tokens_seen": 1874460672 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003491755265797392, + "loss": 1.2855, + "theoretical_loss": 3.477557497521525, + "tokens_seen": 1874526208 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003491594784353059, + "loss": 1.2787, + "theoretical_loss": 3.477547315082781, + "tokens_seen": 1874591744 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003491434302908726, + "loss": 1.2652, + "theoretical_loss": 3.4775371330996814, + "tokens_seen": 1874657280 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003491273821464393, + "loss": 1.3414, + "theoretical_loss": 3.4775269515721887, + "tokens_seen": 1874722816 + }, + { + "epoch": 0.57, + "learning_rate": 0.000349111334002006, + "loss": 1.2574, + "theoretical_loss": 3.4775167705002668, + "tokens_seen": 1874788352 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003490952858575727, + "loss": 1.3128, + "theoretical_loss": 3.47750658988388, + "tokens_seen": 1874853888 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003490792377131394, + "loss": 1.2863, + "theoretical_loss": 3.4774964097229915, + "tokens_seen": 1874919424 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003490631895687061, + "loss": 1.3146, + "theoretical_loss": 3.477486230017565, + "tokens_seen": 1874984960 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003490471414242728, + "loss": 1.263, + "theoretical_loss": 3.477476050767564, + "tokens_seen": 1875050496 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003490310932798395, + "loss": 1.3015, + "theoretical_loss": 3.477465871972953, + "tokens_seen": 1875116032 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003490150451354062, + "loss": 1.2515, + "theoretical_loss": 3.477455693633695, + "tokens_seen": 1875181568 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003489989969909729, + "loss": 1.3155, + "theoretical_loss": 3.4774455157497535, + "tokens_seen": 1875247104 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003489829488465396, + "loss": 1.2219, + "theoretical_loss": 3.477435338321093, + "tokens_seen": 1875312640 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003489669007021063, + "loss": 1.2636, + "theoretical_loss": 3.4774251613476768, + "tokens_seen": 1875378176 + }, + { + "epoch": 0.57, + "learning_rate": 0.000348950852557673, + "loss": 1.316, + "theoretical_loss": 3.477414984829469, + "tokens_seen": 1875443712 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034893480441323976, + "loss": 1.2967, + "theoretical_loss": 3.4774048087664324, + "tokens_seen": 1875509248 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034891875626880646, + "loss": 1.3149, + "theoretical_loss": 3.477394633158532, + "tokens_seen": 1875574784 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034890270812437316, + "loss": 1.2952, + "theoretical_loss": 3.4773844580057305, + "tokens_seen": 1875640320 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034888665997993986, + "loss": 1.2977, + "theoretical_loss": 3.4773742833079924, + "tokens_seen": 1875705856 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034887061183550656, + "loss": 1.2053, + "theoretical_loss": 3.477364109065281, + "tokens_seen": 1875771392 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034885456369107326, + "loss": 1.3631, + "theoretical_loss": 3.4773539352775593, + "tokens_seen": 1875836928 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034883851554663996, + "loss": 1.3677, + "theoretical_loss": 3.477343761944793, + "tokens_seen": 1875902464 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034882246740220666, + "loss": 1.3194, + "theoretical_loss": 3.4773335890669443, + "tokens_seen": 1875968000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034880641925777336, + "loss": 1.2171, + "theoretical_loss": 3.4773234166439773, + "tokens_seen": 1876033536 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034879037111334006, + "loss": 1.2656, + "theoretical_loss": 3.477313244675856, + "tokens_seen": 1876099072 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034877432296890676, + "loss": 1.2395, + "theoretical_loss": 3.4773030731625445, + "tokens_seen": 1876164608 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034875827482447346, + "loss": 1.3173, + "theoretical_loss": 3.477292902104006, + "tokens_seen": 1876230144 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034874222668004017, + "loss": 1.3607, + "theoretical_loss": 3.477282731500204, + "tokens_seen": 1876295680 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034872617853560687, + "loss": 1.2865, + "theoretical_loss": 3.477272561351103, + "tokens_seen": 1876361216 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034871013039117357, + "loss": 1.3007, + "theoretical_loss": 3.4772623916566667, + "tokens_seen": 1876426752 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034869408224674027, + "loss": 1.2685, + "theoretical_loss": 3.4772522224168583, + "tokens_seen": 1876492288 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034867803410230697, + "loss": 1.333, + "theoretical_loss": 3.4772420536316426, + "tokens_seen": 1876557824 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034866198595787367, + "loss": 1.3772, + "theoretical_loss": 3.4772318853009825, + "tokens_seen": 1876623360 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034864593781344037, + "loss": 1.1597, + "theoretical_loss": 3.477221717424842, + "tokens_seen": 1876688896 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034862988966900707, + "loss": 1.2317, + "theoretical_loss": 3.477211550003185, + "tokens_seen": 1876754432 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034861384152457377, + "loss": 1.2302, + "theoretical_loss": 3.4772013830359754, + "tokens_seen": 1876819968 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034859779338014047, + "loss": 1.2797, + "theoretical_loss": 3.4771912165231775, + "tokens_seen": 1876885504 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034858174523570717, + "loss": 1.2687, + "theoretical_loss": 3.477181050464754, + "tokens_seen": 1876951040 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034856569709127387, + "loss": 1.3463, + "theoretical_loss": 3.4771708848606693, + "tokens_seen": 1877016576 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003485496489468405, + "loss": 1.308, + "theoretical_loss": 3.477160719710888, + "tokens_seen": 1877082112 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003485336008024072, + "loss": 1.3132, + "theoretical_loss": 3.4771505550153723, + "tokens_seen": 1877147648 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003485175526579739, + "loss": 1.2976, + "theoretical_loss": 3.477140390774087, + "tokens_seen": 1877213184 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003485015045135406, + "loss": 1.3134, + "theoretical_loss": 3.477130226986996, + "tokens_seen": 1877278720 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003484854563691073, + "loss": 1.257, + "theoretical_loss": 3.4771200636540636, + "tokens_seen": 1877344256 + }, + { + "epoch": 0.57, + "learning_rate": 0.000348469408224674, + "loss": 1.2723, + "theoretical_loss": 3.4771099007752526, + "tokens_seen": 1877409792 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003484533600802407, + "loss": 1.3667, + "theoretical_loss": 3.4770997383505273, + "tokens_seen": 1877475328 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003484373119358074, + "loss": 1.2546, + "theoretical_loss": 3.4770895763798517, + "tokens_seen": 1877540864 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 629623, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.36664879322052, + "objective/train/theoretical_loss": 3.47707941486319, + "objective/train/tokens_used": 1898066400, + "theoretical_loss": 3.47707941486319, + "tokens_seen": 1877606400 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003484212637913741, + "loss": 1.3118, + "theoretical_loss": 3.47707941486319, + "tokens_seen": 1877606400 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003484052156469408, + "loss": 1.2411, + "theoretical_loss": 3.4770692538005052, + "tokens_seen": 1877671936 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003483891675025075, + "loss": 1.2114, + "theoretical_loss": 3.4770590931917615, + "tokens_seen": 1877737472 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003483731193580742, + "loss": 1.2308, + "theoretical_loss": 3.477048933036923, + "tokens_seen": 1877803008 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003483570712136409, + "loss": 1.3513, + "theoretical_loss": 3.477038773335954, + "tokens_seen": 1877868544 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034834102306920763, + "loss": 1.2517, + "theoretical_loss": 3.477028614088817, + "tokens_seen": 1877934080 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034832497492477433, + "loss": 1.3276, + "theoretical_loss": 3.477018455295477, + "tokens_seen": 1877999616 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034830892678034103, + "loss": 1.2847, + "theoretical_loss": 3.4770082969558977, + "tokens_seen": 1878065152 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034829287863590773, + "loss": 1.304, + "theoretical_loss": 3.4769981390700435, + "tokens_seen": 1878130688 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034827683049147443, + "loss": 1.3372, + "theoretical_loss": 3.476987981637877, + "tokens_seen": 1878196224 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034826078234704113, + "loss": 1.322, + "theoretical_loss": 3.4769778246593632, + "tokens_seen": 1878261760 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034824473420260783, + "loss": 1.1953, + "theoretical_loss": 3.4769676681344652, + "tokens_seen": 1878327296 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034822868605817453, + "loss": 1.2815, + "theoretical_loss": 3.476957512063148, + "tokens_seen": 1878392832 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034821263791374123, + "loss": 1.2241, + "theoretical_loss": 3.476947356445375, + "tokens_seen": 1878458368 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034819658976930793, + "loss": 1.1834, + "theoretical_loss": 3.4769372012811095, + "tokens_seen": 1878523904 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034818054162487463, + "loss": 1.3603, + "theoretical_loss": 3.4769270465703164, + "tokens_seen": 1878589440 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034816449348044133, + "loss": 1.3119, + "theoretical_loss": 3.4769168923129588, + "tokens_seen": 1878654976 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034814844533600803, + "loss": 1.2568, + "theoretical_loss": 3.476906738509001, + "tokens_seen": 1878720512 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034813239719157474, + "loss": 1.3474, + "theoretical_loss": 3.476896585158407, + "tokens_seen": 1878786048 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034811634904714144, + "loss": 1.2602, + "theoretical_loss": 3.476886432261141, + "tokens_seen": 1878851584 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034810030090270814, + "loss": 1.264, + "theoretical_loss": 3.4768762798171666, + "tokens_seen": 1878917120 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034808425275827484, + "loss": 1.2258, + "theoretical_loss": 3.4768661278264474, + "tokens_seen": 1878982656 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034806820461384154, + "loss": 1.239, + "theoretical_loss": 3.4768559762889484, + "tokens_seen": 1879048192 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034805215646940824, + "loss": 1.2603, + "theoretical_loss": 3.4768458252046326, + "tokens_seen": 1879113728 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034803610832497494, + "loss": 1.3197, + "theoretical_loss": 3.476835674573464, + "tokens_seen": 1879179264 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034802006018054164, + "loss": 1.3999, + "theoretical_loss": 3.476825524395407, + "tokens_seen": 1879244800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034800401203610834, + "loss": 1.2971, + "theoretical_loss": 3.4768153746704256, + "tokens_seen": 1879310336 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034798796389167504, + "loss": 1.3798, + "theoretical_loss": 3.4768052253984836, + "tokens_seen": 1879375872 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034797191574724174, + "loss": 1.2581, + "theoretical_loss": 3.4767950765795455, + "tokens_seen": 1879441408 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034795586760280844, + "loss": 1.2281, + "theoretical_loss": 3.476784928213574, + "tokens_seen": 1879506944 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034793981945837514, + "loss": 1.2776, + "theoretical_loss": 3.4767747803005338, + "tokens_seen": 1879572480 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034792377131394184, + "loss": 1.3545, + "theoretical_loss": 3.476764632840389, + "tokens_seen": 1879638016 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034790772316950854, + "loss": 1.2976, + "theoretical_loss": 3.476754485833104, + "tokens_seen": 1879703552 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034789167502507524, + "loss": 1.2647, + "theoretical_loss": 3.4767443392786426, + "tokens_seen": 1879769088 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034787562688064195, + "loss": 1.3606, + "theoretical_loss": 3.4767341931769677, + "tokens_seen": 1879834624 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034785957873620865, + "loss": 1.2684, + "theoretical_loss": 3.4767240475280445, + "tokens_seen": 1879900160 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034784353059177535, + "loss": 1.2395, + "theoretical_loss": 3.476713902331837, + "tokens_seen": 1879965696 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034782748244734205, + "loss": 1.2505, + "theoretical_loss": 3.476703757588308, + "tokens_seen": 1880031232 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034781143430290875, + "loss": 1.308, + "theoretical_loss": 3.4766936132974235, + "tokens_seen": 1880096768 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034779538615847545, + "loss": 1.2212, + "theoretical_loss": 3.476683469459146, + "tokens_seen": 1880162304 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003477793380140421, + "loss": 1.2989, + "theoretical_loss": 3.47667332607344, + "tokens_seen": 1880227840 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003477632898696088, + "loss": 1.2363, + "theoretical_loss": 3.47666318314027, + "tokens_seen": 1880293376 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034774724172517555, + "loss": 1.3316, + "theoretical_loss": 3.476653040659599, + "tokens_seen": 1880358912 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034773119358074225, + "loss": 1.2643, + "theoretical_loss": 3.4766428986313915, + "tokens_seen": 1880424448 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034771514543630895, + "loss": 1.4198, + "theoretical_loss": 3.476632757055612, + "tokens_seen": 1880489984 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034769909729187565, + "loss": 1.3403, + "theoretical_loss": 3.476622615932224, + "tokens_seen": 1880555520 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034768304914744235, + "loss": 1.3469, + "theoretical_loss": 3.476612475261192, + "tokens_seen": 1880621056 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034766700100300905, + "loss": 1.2963, + "theoretical_loss": 3.4766023350424797, + "tokens_seen": 1880686592 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034765095285857575, + "loss": 1.254, + "theoretical_loss": 3.476592195276051, + "tokens_seen": 1880752128 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034763490471414246, + "loss": 1.2565, + "theoretical_loss": 3.4765820559618703, + "tokens_seen": 1880817664 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 630537, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3395367860794067, + "objective/train/theoretical_loss": 3.476571917099902, + "objective/train/tokens_used": 1901343200, + "theoretical_loss": 3.476571917099902, + "tokens_seen": 1880883200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034761885656970916, + "loss": 1.2829, + "theoretical_loss": 3.476571917099902, + "tokens_seen": 1880883200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034760280842527586, + "loss": 1.2508, + "theoretical_loss": 3.47656177869011, + "tokens_seen": 1880948736 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034758676028084256, + "loss": 1.3241, + "theoretical_loss": 3.4765516407324575, + "tokens_seen": 1881014272 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034757071213640926, + "loss": 1.2649, + "theoretical_loss": 3.476541503226909, + "tokens_seen": 1881079808 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034755466399197596, + "loss": 1.2795, + "theoretical_loss": 3.47653136617343, + "tokens_seen": 1881145344 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034753861584754266, + "loss": 1.3853, + "theoretical_loss": 3.4765212295719827, + "tokens_seen": 1881210880 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034752256770310936, + "loss": 1.2557, + "theoretical_loss": 3.4765110934225323, + "tokens_seen": 1881276416 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034750651955867606, + "loss": 1.2602, + "theoretical_loss": 3.476500957725042, + "tokens_seen": 1881341952 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034749047141424276, + "loss": 1.2826, + "theoretical_loss": 3.4764908224794766, + "tokens_seen": 1881407488 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034747442326980946, + "loss": 1.2976, + "theoretical_loss": 3.4764806876858003, + "tokens_seen": 1881473024 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034745837512537616, + "loss": 1.2569, + "theoretical_loss": 3.476470553343977, + "tokens_seen": 1881538560 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034744232698094286, + "loss": 1.3674, + "theoretical_loss": 3.4764604194539706, + "tokens_seen": 1881604096 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034742627883650956, + "loss": 1.2636, + "theoretical_loss": 3.476450286015745, + "tokens_seen": 1881669632 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034741023069207626, + "loss": 1.3229, + "theoretical_loss": 3.4764401530292655, + "tokens_seen": 1881735168 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034739418254764296, + "loss": 1.1782, + "theoretical_loss": 3.476430020494495, + "tokens_seen": 1881800704 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034737813440320967, + "loss": 1.237, + "theoretical_loss": 3.476419888411398, + "tokens_seen": 1881866240 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034736208625877637, + "loss": 1.2918, + "theoretical_loss": 3.4764097567799395, + "tokens_seen": 1881931776 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034734603811434307, + "loss": 1.244, + "theoretical_loss": 3.4763996256000818, + "tokens_seen": 1881997312 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034732998996990977, + "loss": 1.305, + "theoretical_loss": 3.476389494871791, + "tokens_seen": 1882062848 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034731394182547647, + "loss": 1.1724, + "theoretical_loss": 3.47637936459503, + "tokens_seen": 1882128384 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034729789368104317, + "loss": 1.1619, + "theoretical_loss": 3.476369234769763, + "tokens_seen": 1882193920 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034728184553660987, + "loss": 1.2412, + "theoretical_loss": 3.4763591053959546, + "tokens_seen": 1882259456 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034726579739217657, + "loss": 1.2919, + "theoretical_loss": 3.4763489764735693, + "tokens_seen": 1882324992 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034724974924774327, + "loss": 1.1972, + "theoretical_loss": 3.4763388480025705, + "tokens_seen": 1882390528 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034723370110330997, + "loss": 1.2253, + "theoretical_loss": 3.4763287199829227, + "tokens_seen": 1882456064 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034721765295887667, + "loss": 1.3869, + "theoretical_loss": 3.47631859241459, + "tokens_seen": 1882521600 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034720160481444337, + "loss": 1.2571, + "theoretical_loss": 3.4763084652975365, + "tokens_seen": 1882587136 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034718555667001007, + "loss": 1.3585, + "theoretical_loss": 3.4762983386317265, + "tokens_seen": 1882652672 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003471695085255768, + "loss": 1.3123, + "theoretical_loss": 3.476288212417124, + "tokens_seen": 1882718208 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003471534603811435, + "loss": 1.3165, + "theoretical_loss": 3.476278086653694, + "tokens_seen": 1882783744 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003471374122367101, + "loss": 1.1979, + "theoretical_loss": 3.4762679613414003, + "tokens_seen": 1882849280 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003471213640922768, + "loss": 1.2957, + "theoretical_loss": 3.4762578364802064, + "tokens_seen": 1882914816 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003471053159478435, + "loss": 1.3283, + "theoretical_loss": 3.4762477120700765, + "tokens_seen": 1882980352 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003470892678034102, + "loss": 1.4514, + "theoretical_loss": 3.476237588110976, + "tokens_seen": 1883045888 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003470732196589769, + "loss": 1.3323, + "theoretical_loss": 3.476227464602868, + "tokens_seen": 1883111424 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003470571715145436, + "loss": 1.2501, + "theoretical_loss": 3.4762173415457176, + "tokens_seen": 1883176960 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003470411233701103, + "loss": 1.2769, + "theoretical_loss": 3.4762072189394884, + "tokens_seen": 1883242496 + }, + { + "epoch": 0.57, + "learning_rate": 0.000347025075225677, + "loss": 1.2319, + "theoretical_loss": 3.4761970967841442, + "tokens_seen": 1883308032 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003470090270812437, + "loss": 1.315, + "theoretical_loss": 3.47618697507965, + "tokens_seen": 1883373568 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003469929789368104, + "loss": 1.176, + "theoretical_loss": 3.47617685382597, + "tokens_seen": 1883439104 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003469769307923771, + "loss": 1.3379, + "theoretical_loss": 3.476166733023068, + "tokens_seen": 1883504640 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034696088264794383, + "loss": 1.3245, + "theoretical_loss": 3.476156612670909, + "tokens_seen": 1883570176 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034694483450351053, + "loss": 1.2619, + "theoretical_loss": 3.4761464927694563, + "tokens_seen": 1883635712 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034692878635907723, + "loss": 1.2807, + "theoretical_loss": 3.476136373318675, + "tokens_seen": 1883701248 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034691273821464393, + "loss": 1.418, + "theoretical_loss": 3.4761262543185287, + "tokens_seen": 1883766784 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034689669007021063, + "loss": 1.2756, + "theoretical_loss": 3.476116135768982, + "tokens_seen": 1883832320 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034688064192577733, + "loss": 1.3639, + "theoretical_loss": 3.476106017669999, + "tokens_seen": 1883897856 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034686459378134403, + "loss": 1.2701, + "theoretical_loss": 3.476095900021544, + "tokens_seen": 1883963392 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034684854563691073, + "loss": 1.214, + "theoretical_loss": 3.4760857828235814, + "tokens_seen": 1884028928 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034683249749247743, + "loss": 1.4156, + "theoretical_loss": 3.476075666076075, + "tokens_seen": 1884094464 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 631766, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2157416343688965, + "objective/train/theoretical_loss": 3.4760655497789896, + "objective/train/tokens_used": 1904620000, + "theoretical_loss": 3.4760655497789896, + "tokens_seen": 1884160000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034681644934804413, + "loss": 1.1797, + "theoretical_loss": 3.4760655497789896, + "tokens_seen": 1884160000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034680040120361083, + "loss": 1.2719, + "theoretical_loss": 3.4760554339322893, + "tokens_seen": 1884225536 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034678435305917753, + "loss": 1.2336, + "theoretical_loss": 3.476045318535938, + "tokens_seen": 1884291072 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034676830491474424, + "loss": 1.3025, + "theoretical_loss": 3.476035203589901, + "tokens_seen": 1884356608 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034675225677031094, + "loss": 1.2623, + "theoretical_loss": 3.4760250890941418, + "tokens_seen": 1884422144 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034673620862587764, + "loss": 1.4002, + "theoretical_loss": 3.476014975048625, + "tokens_seen": 1884487680 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034672016048144434, + "loss": 1.3848, + "theoretical_loss": 3.4760048614533146, + "tokens_seen": 1884553216 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034670411233701104, + "loss": 1.3453, + "theoretical_loss": 3.4759947483081755, + "tokens_seen": 1884618752 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034668806419257774, + "loss": 1.2601, + "theoretical_loss": 3.475984635613171, + "tokens_seen": 1884684288 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034667201604814444, + "loss": 1.3033, + "theoretical_loss": 3.475974523368266, + "tokens_seen": 1884749824 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034665596790371114, + "loss": 1.302, + "theoretical_loss": 3.4759644115734254, + "tokens_seen": 1884815360 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034663991975927784, + "loss": 1.3104, + "theoretical_loss": 3.4759543002286124, + "tokens_seen": 1884880896 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034662387161484454, + "loss": 1.2814, + "theoretical_loss": 3.4759441893337923, + "tokens_seen": 1884946432 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034660782347041124, + "loss": 1.3162, + "theoretical_loss": 3.475934078888929, + "tokens_seen": 1885011968 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034659177532597794, + "loss": 1.2729, + "theoretical_loss": 3.4759239688939862, + "tokens_seen": 1885077504 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034657572718154464, + "loss": 1.3725, + "theoretical_loss": 3.475913859348929, + "tokens_seen": 1885143040 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034655967903711134, + "loss": 1.3049, + "theoretical_loss": 3.475903750253722, + "tokens_seen": 1885208576 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003465436308926781, + "loss": 1.3137, + "theoretical_loss": 3.4758936416083293, + "tokens_seen": 1885274112 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003465275827482448, + "loss": 1.2189, + "theoretical_loss": 3.4758835334127145, + "tokens_seen": 1885339648 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034651153460381145, + "loss": 1.3198, + "theoretical_loss": 3.475873425666843, + "tokens_seen": 1885405184 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034649548645937815, + "loss": 1.2949, + "theoretical_loss": 3.4758633183706786, + "tokens_seen": 1885470720 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034647943831494485, + "loss": 1.3653, + "theoretical_loss": 3.4758532115241856, + "tokens_seen": 1885536256 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034646339017051155, + "loss": 1.1303, + "theoretical_loss": 3.4758431051273293, + "tokens_seen": 1885601792 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034644734202607825, + "loss": 1.2017, + "theoretical_loss": 3.4758329991800725, + "tokens_seen": 1885667328 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034643129388164495, + "loss": 1.1794, + "theoretical_loss": 3.4758228936823805, + "tokens_seen": 1885732864 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034641524573721165, + "loss": 1.3534, + "theoretical_loss": 3.475812788634218, + "tokens_seen": 1885798400 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034639919759277835, + "loss": 1.3524, + "theoretical_loss": 3.475802684035548, + "tokens_seen": 1885863936 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034638314944834505, + "loss": 1.1679, + "theoretical_loss": 3.475792579886337, + "tokens_seen": 1885929472 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034636710130391175, + "loss": 1.3444, + "theoretical_loss": 3.4757824761865477, + "tokens_seen": 1885995008 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034635105315947845, + "loss": 1.237, + "theoretical_loss": 3.475772372936145, + "tokens_seen": 1886060544 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034633500501504515, + "loss": 1.3152, + "theoretical_loss": 3.4757622701350934, + "tokens_seen": 1886126080 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034631895687061185, + "loss": 1.2485, + "theoretical_loss": 3.475752167783357, + "tokens_seen": 1886191616 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034630290872617855, + "loss": 1.242, + "theoretical_loss": 3.4757420658809006, + "tokens_seen": 1886257152 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034628686058174525, + "loss": 1.2349, + "theoretical_loss": 3.4757319644276885, + "tokens_seen": 1886322688 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034627081243731195, + "loss": 1.2666, + "theoretical_loss": 3.4757218634236846, + "tokens_seen": 1886388224 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034625476429287866, + "loss": 1.1945, + "theoretical_loss": 3.4757117628688543, + "tokens_seen": 1886453760 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034623871614844536, + "loss": 1.2792, + "theoretical_loss": 3.4757016627631616, + "tokens_seen": 1886519296 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034622266800401206, + "loss": 1.2926, + "theoretical_loss": 3.4756915631065706, + "tokens_seen": 1886584832 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034620661985957876, + "loss": 1.1737, + "theoretical_loss": 3.4756814638990456, + "tokens_seen": 1886650368 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034619057171514546, + "loss": 1.2689, + "theoretical_loss": 3.4756713651405517, + "tokens_seen": 1886715904 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034617452357071216, + "loss": 1.3873, + "theoretical_loss": 3.475661266831053, + "tokens_seen": 1886781440 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034615847542627886, + "loss": 1.2714, + "theoretical_loss": 3.475651168970514, + "tokens_seen": 1886846976 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034614242728184556, + "loss": 1.2713, + "theoretical_loss": 3.4756410715588992, + "tokens_seen": 1886912512 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034612637913741226, + "loss": 1.311, + "theoretical_loss": 3.475630974596173, + "tokens_seen": 1886978048 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034611033099297896, + "loss": 1.2887, + "theoretical_loss": 3.4756208780822995, + "tokens_seen": 1887043584 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034609428284854566, + "loss": 1.143, + "theoretical_loss": 3.4756107820172435, + "tokens_seen": 1887109120 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034607823470411236, + "loss": 1.3259, + "theoretical_loss": 3.47560068640097, + "tokens_seen": 1887174656 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034606218655967906, + "loss": 1.2593, + "theoretical_loss": 3.475590591233442, + "tokens_seen": 1887240192 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034604613841524576, + "loss": 1.3756, + "theoretical_loss": 3.4755804965146258, + "tokens_seen": 1887305728 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034603009027081246, + "loss": 1.1932, + "theoretical_loss": 3.475570402244484, + "tokens_seen": 1887371264 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 632651, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3401379585266113, + "objective/train/theoretical_loss": 3.475560308422983, + "objective/train/tokens_used": 1907896800, + "theoretical_loss": 3.475560308422983, + "tokens_seen": 1887436800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034601404212637916, + "loss": 1.3055, + "theoretical_loss": 3.475560308422983, + "tokens_seen": 1887436800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034599799398194587, + "loss": 1.2112, + "theoretical_loss": 3.4755502150500863, + "tokens_seen": 1887502336 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034598194583751257, + "loss": 1.2654, + "theoretical_loss": 3.475540122125758, + "tokens_seen": 1887567872 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034596589769307927, + "loss": 1.2081, + "theoretical_loss": 3.4755300296499625, + "tokens_seen": 1887633408 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034594984954864597, + "loss": 1.3585, + "theoretical_loss": 3.4755199376226655, + "tokens_seen": 1887698944 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034593380140421267, + "loss": 1.2937, + "theoretical_loss": 3.4755098460438307, + "tokens_seen": 1887764480 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034591775325977937, + "loss": 1.209, + "theoretical_loss": 3.4754997549134226, + "tokens_seen": 1887830016 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034590170511534607, + "loss": 1.2338, + "theoretical_loss": 3.475489664231406, + "tokens_seen": 1887895552 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034588565697091277, + "loss": 1.2702, + "theoretical_loss": 3.475479573997745, + "tokens_seen": 1887961088 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034586960882647947, + "loss": 1.3039, + "theoretical_loss": 3.475469484212404, + "tokens_seen": 1888026624 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034585356068204617, + "loss": 1.2108, + "theoretical_loss": 3.4754593948753483, + "tokens_seen": 1888092160 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034583751253761287, + "loss": 1.1778, + "theoretical_loss": 3.475449305986542, + "tokens_seen": 1888157696 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034582146439317957, + "loss": 1.2504, + "theoretical_loss": 3.4754392175459494, + "tokens_seen": 1888223232 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003458054162487463, + "loss": 1.206, + "theoretical_loss": 3.475429129553535, + "tokens_seen": 1888288768 + }, + { + "epoch": 0.57, + "learning_rate": 0.000345789368104313, + "loss": 1.264, + "theoretical_loss": 3.475419042009264, + "tokens_seen": 1888354304 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003457733199598797, + "loss": 1.2604, + "theoretical_loss": 3.475408954913101, + "tokens_seen": 1888419840 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003457572718154464, + "loss": 1.2237, + "theoretical_loss": 3.4753988682650094, + "tokens_seen": 1888485376 + }, + { + "epoch": 0.57, + "learning_rate": 0.000345741223671013, + "loss": 1.2801, + "theoretical_loss": 3.4753887820649547, + "tokens_seen": 1888550912 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003457251755265797, + "loss": 1.2739, + "theoretical_loss": 3.4753786963129008, + "tokens_seen": 1888616448 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003457091273821464, + "loss": 1.2609, + "theoretical_loss": 3.4753686110088124, + "tokens_seen": 1888681984 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003456930792377131, + "loss": 1.2537, + "theoretical_loss": 3.4753585261526547, + "tokens_seen": 1888747520 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003456770310932798, + "loss": 1.2924, + "theoretical_loss": 3.4753484417443916, + "tokens_seen": 1888813056 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003456609829488465, + "loss": 1.4141, + "theoretical_loss": 3.475338357783988, + "tokens_seen": 1888878592 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003456449348044132, + "loss": 1.3914, + "theoretical_loss": 3.475328274271409, + "tokens_seen": 1888944128 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003456288866599799, + "loss": 1.3289, + "theoretical_loss": 3.475318191206618, + "tokens_seen": 1889009664 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003456128385155466, + "loss": 1.169, + "theoretical_loss": 3.47530810858958, + "tokens_seen": 1889075200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034559679037111333, + "loss": 1.2749, + "theoretical_loss": 3.4752980264202598, + "tokens_seen": 1889140736 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034558074222668003, + "loss": 1.3492, + "theoretical_loss": 3.4752879446986222, + "tokens_seen": 1889206272 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034556469408224673, + "loss": 1.3428, + "theoretical_loss": 3.475277863424631, + "tokens_seen": 1889271808 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034554864593781343, + "loss": 1.3233, + "theoretical_loss": 3.4752677825982516, + "tokens_seen": 1889337344 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034553259779338013, + "loss": 1.2268, + "theoretical_loss": 3.475257702219448, + "tokens_seen": 1889402880 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034551654964894683, + "loss": 1.266, + "theoretical_loss": 3.475247622288186, + "tokens_seen": 1889468416 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034550050150451353, + "loss": 1.3253, + "theoretical_loss": 3.4752375428044284, + "tokens_seen": 1889533952 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034548445336008023, + "loss": 1.2741, + "theoretical_loss": 3.475227463768141, + "tokens_seen": 1889599488 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034546840521564693, + "loss": 1.2559, + "theoretical_loss": 3.4752173851792882, + "tokens_seen": 1889665024 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034545235707121363, + "loss": 1.1909, + "theoretical_loss": 3.4752073070378344, + "tokens_seen": 1889730560 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034543630892678033, + "loss": 1.3639, + "theoretical_loss": 3.4751972293437445, + "tokens_seen": 1889796096 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034542026078234703, + "loss": 1.2577, + "theoretical_loss": 3.475187152096983, + "tokens_seen": 1889861632 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034540421263791373, + "loss": 1.2106, + "theoretical_loss": 3.4751770752975144, + "tokens_seen": 1889927168 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034538816449348044, + "loss": 1.2934, + "theoretical_loss": 3.475166998945304, + "tokens_seen": 1889992704 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034537211634904714, + "loss": 1.2602, + "theoretical_loss": 3.4751569230403154, + "tokens_seen": 1890058240 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034535606820461384, + "loss": 1.3608, + "theoretical_loss": 3.475146847582514, + "tokens_seen": 1890123776 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003453400200601806, + "loss": 1.2963, + "theoretical_loss": 3.4751367725718643, + "tokens_seen": 1890189312 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003453239719157473, + "loss": 1.398, + "theoretical_loss": 3.4751266980083306, + "tokens_seen": 1890254848 + }, + { + "epoch": 0.57, + "learning_rate": 0.000345307923771314, + "loss": 1.1694, + "theoretical_loss": 3.4751166238918776, + "tokens_seen": 1890320384 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003452918756268807, + "loss": 1.2798, + "theoretical_loss": 3.4751065502224705, + "tokens_seen": 1890385920 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003452758274824474, + "loss": 1.2935, + "theoretical_loss": 3.475096477000074, + "tokens_seen": 1890451456 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003452597793380141, + "loss": 1.2158, + "theoretical_loss": 3.475086404224652, + "tokens_seen": 1890516992 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003452437311935808, + "loss": 1.2658, + "theoretical_loss": 3.4750763318961697, + "tokens_seen": 1890582528 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003452276830491475, + "loss": 1.238, + "theoretical_loss": 3.4750662600145916, + "tokens_seen": 1890648064 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 633930, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3091156482696533, + "objective/train/theoretical_loss": 3.4750561885798827, + "objective/train/tokens_used": 1911173600, + "theoretical_loss": 3.4750561885798827, + "tokens_seen": 1890713600 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003452116349047142, + "loss": 1.3026, + "theoretical_loss": 3.4750561885798827, + "tokens_seen": 1890713600 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003451955867602809, + "loss": 1.3184, + "theoretical_loss": 3.475046117592007, + "tokens_seen": 1890779136 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003451795386158476, + "loss": 1.3288, + "theoretical_loss": 3.47503604705093, + "tokens_seen": 1890844672 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003451634904714143, + "loss": 1.2822, + "theoretical_loss": 3.4750259769566156, + "tokens_seen": 1890910208 + }, + { + "epoch": 0.57, + "learning_rate": 0.000345147442326981, + "loss": 1.17, + "theoretical_loss": 3.4750159073090297, + "tokens_seen": 1890975744 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003451313941825477, + "loss": 1.1888, + "theoretical_loss": 3.4750058381081357, + "tokens_seen": 1891041280 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034511534603811435, + "loss": 1.2295, + "theoretical_loss": 3.4749957693538986, + "tokens_seen": 1891106816 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034509929789368105, + "loss": 1.3152, + "theoretical_loss": 3.4749857010462835, + "tokens_seen": 1891172352 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034508324974924775, + "loss": 1.295, + "theoretical_loss": 3.474975633185255, + "tokens_seen": 1891237888 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034506720160481445, + "loss": 1.2137, + "theoretical_loss": 3.4749655657707783, + "tokens_seen": 1891303424 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034505115346038115, + "loss": 1.2384, + "theoretical_loss": 3.474955498802817, + "tokens_seen": 1891368960 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034503510531594785, + "loss": 1.3459, + "theoretical_loss": 3.4749454322813365, + "tokens_seen": 1891434496 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034501905717151455, + "loss": 1.3558, + "theoretical_loss": 3.474935366206301, + "tokens_seen": 1891500032 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034500300902708125, + "loss": 1.2696, + "theoretical_loss": 3.4749253005776763, + "tokens_seen": 1891565568 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034498696088264795, + "loss": 1.2851, + "theoretical_loss": 3.4749152353954265, + "tokens_seen": 1891631104 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034497091273821465, + "loss": 1.3241, + "theoretical_loss": 3.4749051706595164, + "tokens_seen": 1891696640 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034495486459378135, + "loss": 1.2803, + "theoretical_loss": 3.4748951063699103, + "tokens_seen": 1891762176 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034493881644934805, + "loss": 1.2139, + "theoretical_loss": 3.474885042526574, + "tokens_seen": 1891827712 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034492276830491475, + "loss": 1.3661, + "theoretical_loss": 3.4748749791294706, + "tokens_seen": 1891893248 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034490672016048145, + "loss": 1.2944, + "theoretical_loss": 3.474864916178566, + "tokens_seen": 1891958784 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034489067201604816, + "loss": 1.31, + "theoretical_loss": 3.4748548536738255, + "tokens_seen": 1892024320 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034487462387161486, + "loss": 1.3213, + "theoretical_loss": 3.474844791615213, + "tokens_seen": 1892089856 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034485857572718156, + "loss": 1.2517, + "theoretical_loss": 3.4748347300026934, + "tokens_seen": 1892155392 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034484252758274826, + "loss": 1.3111, + "theoretical_loss": 3.474824668836231, + "tokens_seen": 1892220928 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034482647943831496, + "loss": 1.3167, + "theoretical_loss": 3.474814608115792, + "tokens_seen": 1892286464 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034481043129388166, + "loss": 1.2661, + "theoretical_loss": 3.474804547841339, + "tokens_seen": 1892352000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034479438314944836, + "loss": 1.3023, + "theoretical_loss": 3.474794488012839, + "tokens_seen": 1892417536 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034477833500501506, + "loss": 1.2426, + "theoretical_loss": 3.4747844286302563, + "tokens_seen": 1892483072 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034476228686058176, + "loss": 1.2107, + "theoretical_loss": 3.4747743696935545, + "tokens_seen": 1892548608 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034474623871614846, + "loss": 1.3656, + "theoretical_loss": 3.4747643112026996, + "tokens_seen": 1892614144 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034473019057171516, + "loss": 1.3224, + "theoretical_loss": 3.4747542531576556, + "tokens_seen": 1892679680 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034471414242728186, + "loss": 1.3857, + "theoretical_loss": 3.4747441955583875, + "tokens_seen": 1892745216 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034469809428284856, + "loss": 1.3303, + "theoretical_loss": 3.4747341384048607, + "tokens_seen": 1892810752 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034468204613841526, + "loss": 1.2032, + "theoretical_loss": 3.474724081697039, + "tokens_seen": 1892876288 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034466599799398196, + "loss": 1.2383, + "theoretical_loss": 3.4747140254348885, + "tokens_seen": 1892941824 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034464994984954866, + "loss": 1.2251, + "theoretical_loss": 3.4747039696183726, + "tokens_seen": 1893007360 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034463390170511537, + "loss": 1.1538, + "theoretical_loss": 3.4746939142474575, + "tokens_seen": 1893072896 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034461785356068207, + "loss": 1.2843, + "theoretical_loss": 3.4746838593221065, + "tokens_seen": 1893138432 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034460180541624877, + "loss": 1.2768, + "theoretical_loss": 3.4746738048422863, + "tokens_seen": 1893203968 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034458575727181547, + "loss": 1.3885, + "theoretical_loss": 3.4746637508079603, + "tokens_seen": 1893269504 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034456970912738217, + "loss": 1.3518, + "theoretical_loss": 3.4746536972190936, + "tokens_seen": 1893335040 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034455366098294887, + "loss": 1.3294, + "theoretical_loss": 3.4746436440756514, + "tokens_seen": 1893400576 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034453761283851557, + "loss": 1.2387, + "theoretical_loss": 3.4746335913775983, + "tokens_seen": 1893466112 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034452156469408227, + "loss": 1.3303, + "theoretical_loss": 3.474623539124899, + "tokens_seen": 1893531648 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034450551654964897, + "loss": 1.3307, + "theoretical_loss": 3.4746134873175194, + "tokens_seen": 1893597184 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034448946840521567, + "loss": 1.3185, + "theoretical_loss": 3.474603435955423, + "tokens_seen": 1893662720 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034447342026078237, + "loss": 1.3028, + "theoretical_loss": 3.474593385038575, + "tokens_seen": 1893728256 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034445737211634907, + "loss": 1.3192, + "theoretical_loss": 3.474583334566941, + "tokens_seen": 1893793792 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034444132397191577, + "loss": 1.342, + "theoretical_loss": 3.474573284540485, + "tokens_seen": 1893859328 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003444252758274825, + "loss": 1.1973, + "theoretical_loss": 3.474563234959172, + "tokens_seen": 1893924864 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 635239, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4342564344406128, + "objective/train/theoretical_loss": 3.474553185822967, + "objective/train/tokens_used": 1914450400, + "theoretical_loss": 3.474553185822967, + "tokens_seen": 1893990400 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003444092276830492, + "loss": 1.3198, + "theoretical_loss": 3.474553185822967, + "tokens_seen": 1893990400 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003443931795386159, + "loss": 1.2737, + "theoretical_loss": 3.474543137131836, + "tokens_seen": 1894055936 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003443771313941826, + "loss": 1.318, + "theoretical_loss": 3.474533088885742, + "tokens_seen": 1894121472 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003443610832497493, + "loss": 1.2231, + "theoretical_loss": 3.474523041084651, + "tokens_seen": 1894187008 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003443450351053159, + "loss": 1.2762, + "theoretical_loss": 3.4745129937285277, + "tokens_seen": 1894252544 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003443289869608826, + "loss": 1.3682, + "theoretical_loss": 3.4745029468173367, + "tokens_seen": 1894318080 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003443129388164493, + "loss": 1.314, + "theoretical_loss": 3.4744929003510436, + "tokens_seen": 1894383616 + }, + { + "epoch": 0.57, + "learning_rate": 0.000344296890672016, + "loss": 1.2886, + "theoretical_loss": 3.474482854329613, + "tokens_seen": 1894449152 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003442808425275827, + "loss": 1.3542, + "theoretical_loss": 3.474472808753009, + "tokens_seen": 1894514688 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003442647943831494, + "loss": 1.1662, + "theoretical_loss": 3.474462763621198, + "tokens_seen": 1894580224 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003442487462387161, + "loss": 1.2887, + "theoretical_loss": 3.4744527189341436, + "tokens_seen": 1894645760 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003442326980942828, + "loss": 1.2968, + "theoretical_loss": 3.4744426746918116, + "tokens_seen": 1894711296 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034421664994984953, + "loss": 1.3127, + "theoretical_loss": 3.4744326308941664, + "tokens_seen": 1894776832 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034420060180541623, + "loss": 1.1862, + "theoretical_loss": 3.474422587541173, + "tokens_seen": 1894842368 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034418455366098293, + "loss": 1.2971, + "theoretical_loss": 3.4744125446327967, + "tokens_seen": 1894907904 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034416850551654963, + "loss": 1.3183, + "theoretical_loss": 3.474402502169002, + "tokens_seen": 1894973440 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034415245737211633, + "loss": 1.1865, + "theoretical_loss": 3.4743924601497542, + "tokens_seen": 1895038976 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003441364092276831, + "loss": 1.274, + "theoretical_loss": 3.474382418575018, + "tokens_seen": 1895104512 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003441203610832498, + "loss": 1.2753, + "theoretical_loss": 3.474372377444759, + "tokens_seen": 1895170048 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003441043129388165, + "loss": 1.1921, + "theoretical_loss": 3.474362336758941, + "tokens_seen": 1895235584 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003440882647943832, + "loss": 1.2538, + "theoretical_loss": 3.4743522965175293, + "tokens_seen": 1895301120 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003440722166499499, + "loss": 1.2322, + "theoretical_loss": 3.47434225672049, + "tokens_seen": 1895366656 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003440561685055166, + "loss": 1.2942, + "theoretical_loss": 3.4743322173677864, + "tokens_seen": 1895432192 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003440401203610833, + "loss": 1.2598, + "theoretical_loss": 3.4743221784593845, + "tokens_seen": 1895497728 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034402407221665, + "loss": 1.2917, + "theoretical_loss": 3.4743121399952495, + "tokens_seen": 1895563264 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003440080240722167, + "loss": 1.3415, + "theoretical_loss": 3.4743021019753453, + "tokens_seen": 1895628800 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003439919759277834, + "loss": 1.2572, + "theoretical_loss": 3.474292064399638, + "tokens_seen": 1895694336 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003439759277833501, + "loss": 1.3347, + "theoretical_loss": 3.4742820272680914, + "tokens_seen": 1895759872 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003439598796389168, + "loss": 1.2674, + "theoretical_loss": 3.4742719905806716, + "tokens_seen": 1895825408 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003439438314944835, + "loss": 1.3307, + "theoretical_loss": 3.4742619543373436, + "tokens_seen": 1895890944 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003439277833500502, + "loss": 1.3106, + "theoretical_loss": 3.4742519185380716, + "tokens_seen": 1895956480 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003439117352056169, + "loss": 1.2808, + "theoretical_loss": 3.4742418831828212, + "tokens_seen": 1896022016 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003438956870611836, + "loss": 1.3894, + "theoretical_loss": 3.4742318482715566, + "tokens_seen": 1896087552 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003438796389167503, + "loss": 1.333, + "theoretical_loss": 3.474221813804244, + "tokens_seen": 1896153088 + }, + { + "epoch": 0.57, + "learning_rate": 0.000343863590772317, + "loss": 1.2456, + "theoretical_loss": 3.4742117797808474, + "tokens_seen": 1896218624 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003438475426278837, + "loss": 1.2233, + "theoretical_loss": 3.4742017462013326, + "tokens_seen": 1896284160 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003438314944834504, + "loss": 1.2812, + "theoretical_loss": 3.4741917130656645, + "tokens_seen": 1896349696 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003438154463390171, + "loss": 1.3476, + "theoretical_loss": 3.4741816803738077, + "tokens_seen": 1896415232 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003437993981945838, + "loss": 1.3916, + "theoretical_loss": 3.4741716481257274, + "tokens_seen": 1896480768 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003437833500501505, + "loss": 1.3828, + "theoretical_loss": 3.474161616321388, + "tokens_seen": 1896546304 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003437673019057172, + "loss": 1.2785, + "theoretical_loss": 3.4741515849607563, + "tokens_seen": 1896611840 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003437512537612839, + "loss": 1.2298, + "theoretical_loss": 3.474141554043795, + "tokens_seen": 1896677376 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003437352056168506, + "loss": 1.3062, + "theoretical_loss": 3.4741315235704717, + "tokens_seen": 1896742912 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003437191574724173, + "loss": 1.2785, + "theoretical_loss": 3.47412149354075, + "tokens_seen": 1896808448 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034370310932798395, + "loss": 1.3084, + "theoretical_loss": 3.4741114639545945, + "tokens_seen": 1896873984 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034368706118355065, + "loss": 1.2558, + "theoretical_loss": 3.474101434811971, + "tokens_seen": 1896939520 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034367101303911735, + "loss": 1.2023, + "theoretical_loss": 3.4740914061128443, + "tokens_seen": 1897005056 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034365496489468405, + "loss": 1.3064, + "theoretical_loss": 3.47408137785718, + "tokens_seen": 1897070592 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034363891675025075, + "loss": 1.2492, + "theoretical_loss": 3.4740713500449427, + "tokens_seen": 1897136128 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034362286860581745, + "loss": 1.1774, + "theoretical_loss": 3.4740613226760972, + "tokens_seen": 1897201664 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 636243, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.247380018234253, + "objective/train/theoretical_loss": 3.474051295750609, + "objective/train/tokens_used": 1917727200, + "theoretical_loss": 3.474051295750609, + "tokens_seen": 1897267200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034360682046138415, + "loss": 1.2707, + "theoretical_loss": 3.474051295750609, + "tokens_seen": 1897267200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034359077231695085, + "loss": 1.2482, + "theoretical_loss": 3.4740412692684437, + "tokens_seen": 1897332736 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034357472417251755, + "loss": 1.2457, + "theoretical_loss": 3.4740312432295655, + "tokens_seen": 1897398272 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034355867602808425, + "loss": 1.2609, + "theoretical_loss": 3.47402121763394, + "tokens_seen": 1897463808 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034354262788365095, + "loss": 1.3008, + "theoretical_loss": 3.474011192481531, + "tokens_seen": 1897529344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034352657973921765, + "loss": 1.2505, + "theoretical_loss": 3.4740011677723057, + "tokens_seen": 1897594880 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034351053159478436, + "loss": 1.1974, + "theoretical_loss": 3.473991143506228, + "tokens_seen": 1897660416 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034349448345035106, + "loss": 1.3438, + "theoretical_loss": 3.4739811196832626, + "tokens_seen": 1897725952 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034347843530591776, + "loss": 1.3501, + "theoretical_loss": 3.473971096303376, + "tokens_seen": 1897791488 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034346238716148446, + "loss": 1.2011, + "theoretical_loss": 3.4739610733665325, + "tokens_seen": 1897857024 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034344633901705116, + "loss": 1.3031, + "theoretical_loss": 3.4739510508726967, + "tokens_seen": 1897922560 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034343029087261786, + "loss": 1.2359, + "theoretical_loss": 3.4739410288218346, + "tokens_seen": 1897988096 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034341424272818456, + "loss": 1.3224, + "theoretical_loss": 3.473931007213911, + "tokens_seen": 1898053632 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034339819458375126, + "loss": 1.3595, + "theoretical_loss": 3.473920986048891, + "tokens_seen": 1898119168 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034338214643931796, + "loss": 1.3369, + "theoretical_loss": 3.4739109653267395, + "tokens_seen": 1898184704 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034336609829488466, + "loss": 1.2238, + "theoretical_loss": 3.473900945047422, + "tokens_seen": 1898250240 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034335005015045136, + "loss": 1.3535, + "theoretical_loss": 3.4738909252109043, + "tokens_seen": 1898315776 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034333400200601806, + "loss": 1.3003, + "theoretical_loss": 3.4738809058171496, + "tokens_seen": 1898381312 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034331795386158476, + "loss": 1.3546, + "theoretical_loss": 3.473870886866125, + "tokens_seen": 1898446848 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034330190571715146, + "loss": 1.2407, + "theoretical_loss": 3.4738608683577947, + "tokens_seen": 1898512384 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034328585757271816, + "loss": 1.3227, + "theoretical_loss": 3.473850850292124, + "tokens_seen": 1898577920 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034326980942828487, + "loss": 1.2954, + "theoretical_loss": 3.473840832669078, + "tokens_seen": 1898643456 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034325376128385157, + "loss": 1.1862, + "theoretical_loss": 3.473830815488622, + "tokens_seen": 1898708992 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034323771313941827, + "loss": 1.3211, + "theoretical_loss": 3.4738207987507215, + "tokens_seen": 1898774528 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034322166499498497, + "loss": 1.3274, + "theoretical_loss": 3.473810782455341, + "tokens_seen": 1898840064 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034320561685055167, + "loss": 1.265, + "theoretical_loss": 3.473800766602446, + "tokens_seen": 1898905600 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034318956870611837, + "loss": 1.3108, + "theoretical_loss": 3.473790751192002, + "tokens_seen": 1898971136 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034317352056168507, + "loss": 1.2596, + "theoretical_loss": 3.4737807362239734, + "tokens_seen": 1899036672 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034315747241725177, + "loss": 1.3156, + "theoretical_loss": 3.473770721698326, + "tokens_seen": 1899102208 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034314142427281847, + "loss": 1.2043, + "theoretical_loss": 3.473760707615025, + "tokens_seen": 1899167744 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034312537612838517, + "loss": 1.2432, + "theoretical_loss": 3.4737506939740355, + "tokens_seen": 1899233280 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034310932798395187, + "loss": 1.2747, + "theoretical_loss": 3.4737406807753226, + "tokens_seen": 1899298816 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034309327983951857, + "loss": 1.3753, + "theoretical_loss": 3.4737306680188516, + "tokens_seen": 1899364352 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034307723169508527, + "loss": 1.3495, + "theoretical_loss": 3.4737206557045877, + "tokens_seen": 1899429888 + }, + { + "epoch": 0.58, + "learning_rate": 0.000343061183550652, + "loss": 1.2598, + "theoretical_loss": 3.473710643832496, + "tokens_seen": 1899495424 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003430451354062187, + "loss": 1.1686, + "theoretical_loss": 3.4737006324025415, + "tokens_seen": 1899560960 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003430290872617854, + "loss": 1.2165, + "theoretical_loss": 3.47369062141469, + "tokens_seen": 1899626496 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003430130391173521, + "loss": 1.248, + "theoretical_loss": 3.4736806108689064, + "tokens_seen": 1899692032 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003429969909729188, + "loss": 1.2529, + "theoretical_loss": 3.473670600765156, + "tokens_seen": 1899757568 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003429809428284855, + "loss": 1.1621, + "theoretical_loss": 3.473660591103404, + "tokens_seen": 1899823104 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003429648946840522, + "loss": 1.3636, + "theoretical_loss": 3.4736505818836156, + "tokens_seen": 1899888640 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003429488465396189, + "loss": 1.3198, + "theoretical_loss": 3.4736405731057562, + "tokens_seen": 1899954176 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003429327983951856, + "loss": 1.2778, + "theoretical_loss": 3.473630564769791, + "tokens_seen": 1900019712 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003429167502507523, + "loss": 1.3051, + "theoretical_loss": 3.473620556875685, + "tokens_seen": 1900085248 + }, + { + "epoch": 0.58, + "learning_rate": 0.000342900702106319, + "loss": 1.2498, + "theoretical_loss": 3.4736105494234035, + "tokens_seen": 1900150784 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003428846539618857, + "loss": 1.1991, + "theoretical_loss": 3.473600542412912, + "tokens_seen": 1900216320 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003428686058174524, + "loss": 1.3323, + "theoretical_loss": 3.473590535844176, + "tokens_seen": 1900281856 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003428525576730191, + "loss": 1.2312, + "theoretical_loss": 3.47358052971716, + "tokens_seen": 1900347392 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003428365095285858, + "loss": 1.2788, + "theoretical_loss": 3.47357052403183, + "tokens_seen": 1900412928 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003428204613841525, + "loss": 1.284, + "theoretical_loss": 3.473560518788151, + "tokens_seen": 1900478464 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 637457, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2931511402130127, + "objective/train/theoretical_loss": 3.473550513986088, + "objective/train/tokens_used": 1921004000, + "theoretical_loss": 3.473550513986088, + "tokens_seen": 1900544000 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003428044132397192, + "loss": 1.3519, + "theoretical_loss": 3.473550513986088, + "tokens_seen": 1900544000 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003427883650952859, + "loss": 1.4324, + "theoretical_loss": 3.4735405096256065, + "tokens_seen": 1900609536 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003427723169508526, + "loss": 1.3575, + "theoretical_loss": 3.473530505706672, + "tokens_seen": 1900675072 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003427562688064193, + "loss": 1.203, + "theoretical_loss": 3.4735205022292495, + "tokens_seen": 1900740608 + }, + { + "epoch": 0.58, + "learning_rate": 0.000342740220661986, + "loss": 1.2644, + "theoretical_loss": 3.473510499193304, + "tokens_seen": 1900806144 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003427241725175527, + "loss": 1.2503, + "theoretical_loss": 3.4735004965988017, + "tokens_seen": 1900871680 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003427081243731194, + "loss": 1.3334, + "theoretical_loss": 3.4734904944457075, + "tokens_seen": 1900937216 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003426920762286861, + "loss": 1.2499, + "theoretical_loss": 3.473480492733986, + "tokens_seen": 1901002752 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003426760280842528, + "loss": 1.4125, + "theoretical_loss": 3.4734704914636034, + "tokens_seen": 1901068288 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003426599799398195, + "loss": 1.2694, + "theoretical_loss": 3.4734604906345248, + "tokens_seen": 1901133824 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003426439317953862, + "loss": 1.3579, + "theoretical_loss": 3.4734504902467154, + "tokens_seen": 1901199360 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003426278836509529, + "loss": 1.3578, + "theoretical_loss": 3.47344049030014, + "tokens_seen": 1901264896 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003426118355065196, + "loss": 1.1838, + "theoretical_loss": 3.4734304907947653, + "tokens_seen": 1901330432 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003425957873620863, + "loss": 1.2824, + "theoretical_loss": 3.4734204917305553, + "tokens_seen": 1901395968 + }, + { + "epoch": 0.58, + "learning_rate": 0.000342579739217653, + "loss": 1.2379, + "theoretical_loss": 3.473410493107476, + "tokens_seen": 1901461504 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003425636910732197, + "loss": 1.2518, + "theoretical_loss": 3.4734004949254924, + "tokens_seen": 1901527040 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003425476429287864, + "loss": 1.4048, + "theoretical_loss": 3.47339049718457, + "tokens_seen": 1901592576 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003425315947843531, + "loss": 1.2727, + "theoretical_loss": 3.4733804998846747, + "tokens_seen": 1901658112 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003425155466399198, + "loss": 1.2697, + "theoretical_loss": 3.4733705030257704, + "tokens_seen": 1901723648 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003424994984954865, + "loss": 1.2625, + "theoretical_loss": 3.4733605066078237, + "tokens_seen": 1901789184 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003424834503510532, + "loss": 1.1387, + "theoretical_loss": 3.4733505106308, + "tokens_seen": 1901854720 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003424674022066199, + "loss": 1.35, + "theoretical_loss": 3.473340515094664, + "tokens_seen": 1901920256 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003424513540621866, + "loss": 1.1771, + "theoretical_loss": 3.473330519999381, + "tokens_seen": 1901985792 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003424353059177533, + "loss": 1.2577, + "theoretical_loss": 3.473320525344917, + "tokens_seen": 1902051328 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034241925777332, + "loss": 1.3879, + "theoretical_loss": 3.473310531131237, + "tokens_seen": 1902116864 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003424032096288867, + "loss": 1.2248, + "theoretical_loss": 3.4733005373583064, + "tokens_seen": 1902182400 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003423871614844534, + "loss": 1.3157, + "theoretical_loss": 3.4732905440260904, + "tokens_seen": 1902247936 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003423711133400201, + "loss": 1.1326, + "theoretical_loss": 3.473280551134555, + "tokens_seen": 1902313472 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003423550651955868, + "loss": 1.2457, + "theoretical_loss": 3.473270558683665, + "tokens_seen": 1902379008 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003423390170511535, + "loss": 1.2991, + "theoretical_loss": 3.473260566673386, + "tokens_seen": 1902444544 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003423229689067202, + "loss": 1.3326, + "theoretical_loss": 3.473250575103683, + "tokens_seen": 1902510080 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034230692076228685, + "loss": 1.2935, + "theoretical_loss": 3.473240583974522, + "tokens_seen": 1902575616 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034229087261785355, + "loss": 1.2272, + "theoretical_loss": 3.4732305932858685, + "tokens_seen": 1902641152 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034227482447342025, + "loss": 1.2277, + "theoretical_loss": 3.473220603037687, + "tokens_seen": 1902706688 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034225877632898695, + "loss": 1.203, + "theoretical_loss": 3.473210613229944, + "tokens_seen": 1902772224 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034224272818455365, + "loss": 1.3093, + "theoretical_loss": 3.473200623862604, + "tokens_seen": 1902837760 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034222668004012035, + "loss": 1.2145, + "theoretical_loss": 3.473190634935633, + "tokens_seen": 1902903296 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034221063189568705, + "loss": 1.2696, + "theoretical_loss": 3.4731806464489963, + "tokens_seen": 1902968832 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034219458375125375, + "loss": 1.2956, + "theoretical_loss": 3.473170658402659, + "tokens_seen": 1903034368 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034217853560682045, + "loss": 1.3289, + "theoretical_loss": 3.473160670796587, + "tokens_seen": 1903099904 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034216248746238715, + "loss": 1.2844, + "theoretical_loss": 3.4731506836307453, + "tokens_seen": 1903165440 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034214643931795386, + "loss": 1.2704, + "theoretical_loss": 3.4731406969050997, + "tokens_seen": 1903230976 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034213039117352056, + "loss": 1.2736, + "theoretical_loss": 3.473130710619616, + "tokens_seen": 1903296512 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034211434302908726, + "loss": 1.2114, + "theoretical_loss": 3.473120724774258, + "tokens_seen": 1903362048 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034209829488465396, + "loss": 1.2031, + "theoretical_loss": 3.4731107393689933, + "tokens_seen": 1903427584 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034208224674022066, + "loss": 1.297, + "theoretical_loss": 3.473100754403786, + "tokens_seen": 1903493120 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034206619859578736, + "loss": 1.3535, + "theoretical_loss": 3.4730907698786018, + "tokens_seen": 1903558656 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034205015045135406, + "loss": 1.2116, + "theoretical_loss": 3.4730807857934063, + "tokens_seen": 1903624192 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034203410230692076, + "loss": 1.336, + "theoretical_loss": 3.473070802148165, + "tokens_seen": 1903689728 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034201805416248746, + "loss": 1.2639, + "theoretical_loss": 3.4730608189428436, + "tokens_seen": 1903755264 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 638270, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2350528240203857, + "objective/train/theoretical_loss": 3.4730508361774066, + "objective/train/tokens_used": 1924280800, + "theoretical_loss": 3.4730508361774066, + "tokens_seen": 1903820800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034200200601805416, + "loss": 1.2642, + "theoretical_loss": 3.4730508361774066, + "tokens_seen": 1903820800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034198595787362086, + "loss": 1.1995, + "theoretical_loss": 3.4730408538518205, + "tokens_seen": 1903886336 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034196990972918756, + "loss": 1.2251, + "theoretical_loss": 3.47303087196605, + "tokens_seen": 1903951872 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034195386158475426, + "loss": 1.2938, + "theoretical_loss": 3.473020890520062, + "tokens_seen": 1904017408 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034193781344032096, + "loss": 1.2223, + "theoretical_loss": 3.47301090951382, + "tokens_seen": 1904082944 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034192176529588766, + "loss": 1.3467, + "theoretical_loss": 3.4730009289472914, + "tokens_seen": 1904148480 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034190571715145436, + "loss": 1.3448, + "theoretical_loss": 3.4729909488204402, + "tokens_seen": 1904214016 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034188966900702107, + "loss": 1.2684, + "theoretical_loss": 3.472980969133232, + "tokens_seen": 1904279552 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034187362086258777, + "loss": 1.2732, + "theoretical_loss": 3.4729709898856336, + "tokens_seen": 1904345088 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034185757271815447, + "loss": 1.3134, + "theoretical_loss": 3.4729610110776097, + "tokens_seen": 1904410624 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034184152457372117, + "loss": 1.277, + "theoretical_loss": 3.4729510327091253, + "tokens_seen": 1904476160 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034182547642928787, + "loss": 1.3276, + "theoretical_loss": 3.4729410547801463, + "tokens_seen": 1904541696 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034180942828485457, + "loss": 1.2706, + "theoretical_loss": 3.4729310772906388, + "tokens_seen": 1904607232 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034179338014042127, + "loss": 1.2142, + "theoretical_loss": 3.472921100240568, + "tokens_seen": 1904672768 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034177733199598797, + "loss": 1.3635, + "theoretical_loss": 3.472911123629899, + "tokens_seen": 1904738304 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034176128385155467, + "loss": 1.2714, + "theoretical_loss": 3.4729011474585976, + "tokens_seen": 1904803840 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034174523570712137, + "loss": 1.3285, + "theoretical_loss": 3.4728911717266295, + "tokens_seen": 1904869376 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003417291875626881, + "loss": 1.3742, + "theoretical_loss": 3.47288119643396, + "tokens_seen": 1904934912 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003417131394182548, + "loss": 1.1989, + "theoretical_loss": 3.4728712215805544, + "tokens_seen": 1905000448 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034169709127382153, + "loss": 1.4007, + "theoretical_loss": 3.472861247166379, + "tokens_seen": 1905065984 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034168104312938823, + "loss": 1.3081, + "theoretical_loss": 3.4728512731913987, + "tokens_seen": 1905131520 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003416649949849549, + "loss": 1.3103, + "theoretical_loss": 3.4728412996555797, + "tokens_seen": 1905197056 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003416489468405216, + "loss": 1.2175, + "theoretical_loss": 3.472831326558887, + "tokens_seen": 1905262592 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003416328986960883, + "loss": 1.2998, + "theoretical_loss": 3.4728213539012858, + "tokens_seen": 1905328128 + }, + { + "epoch": 0.58, + "learning_rate": 0.000341616850551655, + "loss": 1.3657, + "theoretical_loss": 3.472811381682743, + "tokens_seen": 1905393664 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003416008024072217, + "loss": 1.1165, + "theoretical_loss": 3.472801409903223, + "tokens_seen": 1905459200 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003415847542627884, + "loss": 1.2717, + "theoretical_loss": 3.472791438562691, + "tokens_seen": 1905524736 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003415687061183551, + "loss": 1.346, + "theoretical_loss": 3.472781467661114, + "tokens_seen": 1905590272 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003415526579739218, + "loss": 1.3452, + "theoretical_loss": 3.472771497198457, + "tokens_seen": 1905655808 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003415366098294885, + "loss": 1.3512, + "theoretical_loss": 3.472761527174685, + "tokens_seen": 1905721344 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003415205616850552, + "loss": 1.2892, + "theoretical_loss": 3.4727515575897643, + "tokens_seen": 1905786880 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003415045135406219, + "loss": 1.3, + "theoretical_loss": 3.47274158844366, + "tokens_seen": 1905852416 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003414884653961886, + "loss": 1.2162, + "theoretical_loss": 3.4727316197363383, + "tokens_seen": 1905917952 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003414724172517553, + "loss": 1.2441, + "theoretical_loss": 3.472721651467764, + "tokens_seen": 1905983488 + }, + { + "epoch": 0.58, + "learning_rate": 0.000341456369107322, + "loss": 1.3087, + "theoretical_loss": 3.472711683637904, + "tokens_seen": 1906049024 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003414403209628887, + "loss": 1.2881, + "theoretical_loss": 3.472701716246722, + "tokens_seen": 1906114560 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003414242728184554, + "loss": 1.2606, + "theoretical_loss": 3.4726917492941856, + "tokens_seen": 1906180096 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003414082246740221, + "loss": 1.3042, + "theoretical_loss": 3.472681782780259, + "tokens_seen": 1906245632 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003413921765295888, + "loss": 1.2833, + "theoretical_loss": 3.472671816704908, + "tokens_seen": 1906311168 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003413761283851555, + "loss": 1.3791, + "theoretical_loss": 3.472661851068099, + "tokens_seen": 1906376704 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003413600802407222, + "loss": 1.3409, + "theoretical_loss": 3.472651885869797, + "tokens_seen": 1906442240 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003413440320962889, + "loss": 1.2872, + "theoretical_loss": 3.4726419211099677, + "tokens_seen": 1906507776 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003413279839518556, + "loss": 1.2778, + "theoretical_loss": 3.472631956788577, + "tokens_seen": 1906573312 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003413119358074223, + "loss": 1.266, + "theoretical_loss": 3.47262199290559, + "tokens_seen": 1906638848 + }, + { + "epoch": 0.58, + "learning_rate": 0.000341295887662989, + "loss": 1.369, + "theoretical_loss": 3.472612029460973, + "tokens_seen": 1906704384 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003412798395185557, + "loss": 1.2039, + "theoretical_loss": 3.4726020664546913, + "tokens_seen": 1906769920 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003412637913741224, + "loss": 1.2679, + "theoretical_loss": 3.4725921038867105, + "tokens_seen": 1906835456 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003412477432296891, + "loss": 1.2744, + "theoretical_loss": 3.4725821417569964, + "tokens_seen": 1906900992 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003412316950852558, + "loss": 1.3848, + "theoretical_loss": 3.472572180065515, + "tokens_seen": 1906966528 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003412156469408225, + "loss": 1.2444, + "theoretical_loss": 3.472562218812231, + "tokens_seen": 1907032064 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 639529, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.321748971939087, + "objective/train/theoretical_loss": 3.4725522579971106, + "objective/train/tokens_used": 1927557600, + "theoretical_loss": 3.4725522579971106, + "tokens_seen": 1907097600 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003411995987963892, + "loss": 1.3124, + "theoretical_loss": 3.4725522579971106, + "tokens_seen": 1907097600 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003411835506519559, + "loss": 1.3336, + "theoretical_loss": 3.47254229762012, + "tokens_seen": 1907163136 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003411675025075226, + "loss": 1.2696, + "theoretical_loss": 3.4725323376812236, + "tokens_seen": 1907228672 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003411514543630893, + "loss": 1.1987, + "theoretical_loss": 3.472522378180389, + "tokens_seen": 1907294208 + }, + { + "epoch": 0.58, + "learning_rate": 0.000341135406218656, + "loss": 1.3674, + "theoretical_loss": 3.47251241911758, + "tokens_seen": 1907359744 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003411193580742227, + "loss": 1.3064, + "theoretical_loss": 3.472502460492763, + "tokens_seen": 1907425280 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003411033099297894, + "loss": 1.2679, + "theoretical_loss": 3.4724925023059043, + "tokens_seen": 1907490816 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003410872617853561, + "loss": 1.3269, + "theoretical_loss": 3.4724825445569687, + "tokens_seen": 1907556352 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003410712136409228, + "loss": 1.3637, + "theoretical_loss": 3.472472587245922, + "tokens_seen": 1907621888 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003410551654964895, + "loss": 1.3615, + "theoretical_loss": 3.47246263037273, + "tokens_seen": 1907687424 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003410391173520562, + "loss": 1.2879, + "theoretical_loss": 3.472452673937359, + "tokens_seen": 1907752960 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003410230692076229, + "loss": 1.2683, + "theoretical_loss": 3.4724427179397734, + "tokens_seen": 1907818496 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003410070210631896, + "loss": 1.3401, + "theoretical_loss": 3.472432762379941, + "tokens_seen": 1907884032 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003409909729187563, + "loss": 1.3418, + "theoretical_loss": 3.4724228072578254, + "tokens_seen": 1907949568 + }, + { + "epoch": 0.58, + "learning_rate": 0.000340974924774323, + "loss": 1.2865, + "theoretical_loss": 3.4724128525733935, + "tokens_seen": 1908015104 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003409588766298897, + "loss": 1.3213, + "theoretical_loss": 3.47240289832661, + "tokens_seen": 1908080640 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003409428284854564, + "loss": 1.2519, + "theoretical_loss": 3.472392944517442, + "tokens_seen": 1908146176 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003409267803410231, + "loss": 1.3364, + "theoretical_loss": 3.472382991145855, + "tokens_seen": 1908211712 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003409107321965898, + "loss": 1.2792, + "theoretical_loss": 3.4723730382118134, + "tokens_seen": 1908277248 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034089468405215645, + "loss": 1.2777, + "theoretical_loss": 3.4723630857152843, + "tokens_seen": 1908342784 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034087863590772315, + "loss": 1.253, + "theoretical_loss": 3.4723531336562328, + "tokens_seen": 1908408320 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034086258776328985, + "loss": 1.1897, + "theoretical_loss": 3.472343182034625, + "tokens_seen": 1908473856 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034084653961885655, + "loss": 1.2611, + "theoretical_loss": 3.4723332308504267, + "tokens_seen": 1908539392 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034083049147442325, + "loss": 1.3273, + "theoretical_loss": 3.472323280103603, + "tokens_seen": 1908604928 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034081444332998995, + "loss": 1.3726, + "theoretical_loss": 3.4723133297941198, + "tokens_seen": 1908670464 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034079839518555665, + "loss": 1.1786, + "theoretical_loss": 3.472303379921944, + "tokens_seen": 1908736000 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034078234704112335, + "loss": 1.3043, + "theoretical_loss": 3.47229343048704, + "tokens_seen": 1908801536 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034076629889669006, + "loss": 1.376, + "theoretical_loss": 3.472283481489374, + "tokens_seen": 1908867072 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034075025075225676, + "loss": 1.342, + "theoretical_loss": 3.472273532928912, + "tokens_seen": 1908932608 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034073420260782346, + "loss": 1.2983, + "theoretical_loss": 3.47226358480562, + "tokens_seen": 1908998144 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034071815446339016, + "loss": 1.241, + "theoretical_loss": 3.472253637119463, + "tokens_seen": 1909063680 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034070210631895686, + "loss": 1.3002, + "theoretical_loss": 3.4722436898704077, + "tokens_seen": 1909129216 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034068605817452356, + "loss": 1.2919, + "theoretical_loss": 3.4722337430584194, + "tokens_seen": 1909194752 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034067001003009026, + "loss": 1.3391, + "theoretical_loss": 3.4722237966834633, + "tokens_seen": 1909260288 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034065396188565696, + "loss": 1.3131, + "theoretical_loss": 3.472213850745506, + "tokens_seen": 1909325824 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034063791374122366, + "loss": 1.2926, + "theoretical_loss": 3.4722039052445135, + "tokens_seen": 1909391360 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034062186559679036, + "loss": 1.3557, + "theoretical_loss": 3.4721939601804506, + "tokens_seen": 1909456896 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034060581745235706, + "loss": 1.3762, + "theoretical_loss": 3.4721840155532844, + "tokens_seen": 1909522432 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034058976930792376, + "loss": 1.2973, + "theoretical_loss": 3.4721740713629794, + "tokens_seen": 1909587968 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034057372116349046, + "loss": 1.3813, + "theoretical_loss": 3.4721641276095023, + "tokens_seen": 1909653504 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034055767301905716, + "loss": 1.2666, + "theoretical_loss": 3.472154184292819, + "tokens_seen": 1909719040 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034054162487462386, + "loss": 1.1721, + "theoretical_loss": 3.4721442414128942, + "tokens_seen": 1909784576 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003405255767301906, + "loss": 1.3085, + "theoretical_loss": 3.4721342989696953, + "tokens_seen": 1909850112 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003405095285857573, + "loss": 1.2772, + "theoretical_loss": 3.472124356963187, + "tokens_seen": 1909915648 + }, + { + "epoch": 0.58, + "learning_rate": 0.000340493480441324, + "loss": 1.2309, + "theoretical_loss": 3.4721144153933357, + "tokens_seen": 1909981184 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003404774322968907, + "loss": 1.292, + "theoretical_loss": 3.4721044742601066, + "tokens_seen": 1910046720 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003404613841524574, + "loss": 1.2997, + "theoretical_loss": 3.4720945335634665, + "tokens_seen": 1910112256 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003404453360080241, + "loss": 1.2661, + "theoretical_loss": 3.4720845933033804, + "tokens_seen": 1910177792 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003404292878635908, + "loss": 1.2563, + "theoretical_loss": 3.4720746534798144, + "tokens_seen": 1910243328 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003404132397191575, + "loss": 1.2148, + "theoretical_loss": 3.4720647140927348, + "tokens_seen": 1910308864 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 640490, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3794310092926025, + "objective/train/theoretical_loss": 3.472054775142107, + "objective/train/tokens_used": 1930834400, + "theoretical_loss": 3.472054775142107, + "tokens_seen": 1910374400 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003403971915747242, + "loss": 1.2561, + "theoretical_loss": 3.472054775142107, + "tokens_seen": 1910374400 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003403811434302909, + "loss": 1.3067, + "theoretical_loss": 3.4720448366278966, + "tokens_seen": 1910439936 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003403650952858576, + "loss": 1.2907, + "theoretical_loss": 3.47203489855007, + "tokens_seen": 1910505472 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003403490471414243, + "loss": 1.2845, + "theoretical_loss": 3.472024960908593, + "tokens_seen": 1910571008 + }, + { + "epoch": 0.58, + "learning_rate": 0.000340332998996991, + "loss": 1.2744, + "theoretical_loss": 3.4720150237034315, + "tokens_seen": 1910636544 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034031695085255773, + "loss": 1.4365, + "theoretical_loss": 3.4720050869345513, + "tokens_seen": 1910702080 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034030090270812443, + "loss": 1.264, + "theoretical_loss": 3.4719951506019178, + "tokens_seen": 1910767616 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034028485456369113, + "loss": 1.2367, + "theoretical_loss": 3.471985214705497, + "tokens_seen": 1910833152 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003402688064192578, + "loss": 1.3216, + "theoretical_loss": 3.471975279245256, + "tokens_seen": 1910898688 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003402527582748245, + "loss": 1.2689, + "theoretical_loss": 3.471965344221159, + "tokens_seen": 1910964224 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003402367101303912, + "loss": 1.2882, + "theoretical_loss": 3.4719554096331735, + "tokens_seen": 1911029760 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003402206619859579, + "loss": 1.3626, + "theoretical_loss": 3.471945475481264, + "tokens_seen": 1911095296 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003402046138415246, + "loss": 1.2561, + "theoretical_loss": 3.471935541765397, + "tokens_seen": 1911160832 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003401885656970913, + "loss": 1.3408, + "theoretical_loss": 3.471925608485539, + "tokens_seen": 1911226368 + }, + { + "epoch": 0.58, + "learning_rate": 0.000340172517552658, + "loss": 1.2421, + "theoretical_loss": 3.4719156756416547, + "tokens_seen": 1911291904 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003401564694082247, + "loss": 1.2793, + "theoretical_loss": 3.471905743233711, + "tokens_seen": 1911357440 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003401404212637914, + "loss": 1.3307, + "theoretical_loss": 3.4718958112616733, + "tokens_seen": 1911422976 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003401243731193581, + "loss": 1.3562, + "theoretical_loss": 3.471885879725508, + "tokens_seen": 1911488512 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003401083249749248, + "loss": 1.3743, + "theoretical_loss": 3.47187594862518, + "tokens_seen": 1911554048 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003400922768304915, + "loss": 1.2788, + "theoretical_loss": 3.4718660179606564, + "tokens_seen": 1911619584 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003400762286860582, + "loss": 1.2732, + "theoretical_loss": 3.4718560877319025, + "tokens_seen": 1911685120 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003400601805416249, + "loss": 1.2543, + "theoretical_loss": 3.471846157938885, + "tokens_seen": 1911750656 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003400441323971916, + "loss": 1.3305, + "theoretical_loss": 3.4718362285815685, + "tokens_seen": 1911816192 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003400280842527583, + "loss": 1.1654, + "theoretical_loss": 3.47182629965992, + "tokens_seen": 1911881728 + }, + { + "epoch": 0.58, + "learning_rate": 0.000340012036108325, + "loss": 1.2997, + "theoretical_loss": 3.4718163711739054, + "tokens_seen": 1911947264 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003399959879638917, + "loss": 1.2786, + "theoretical_loss": 3.4718064431234903, + "tokens_seen": 1912012800 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003399799398194584, + "loss": 1.2651, + "theoretical_loss": 3.4717965155086405, + "tokens_seen": 1912078336 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003399638916750251, + "loss": 1.2186, + "theoretical_loss": 3.471786588329323, + "tokens_seen": 1912143872 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003399478435305918, + "loss": 1.366, + "theoretical_loss": 3.471776661585502, + "tokens_seen": 1912209408 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003399317953861585, + "loss": 1.2173, + "theoretical_loss": 3.471766735277145, + "tokens_seen": 1912274944 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003399157472417252, + "loss": 1.177, + "theoretical_loss": 3.4717568094042175, + "tokens_seen": 1912340480 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003398996990972919, + "loss": 1.3297, + "theoretical_loss": 3.471746883966685, + "tokens_seen": 1912406016 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003398836509528586, + "loss": 1.3362, + "theoretical_loss": 3.471736958964515, + "tokens_seen": 1912471552 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003398676028084253, + "loss": 1.2236, + "theoretical_loss": 3.471727034397671, + "tokens_seen": 1912537088 + }, + { + "epoch": 0.58, + "learning_rate": 0.000339851554663992, + "loss": 1.314, + "theoretical_loss": 3.4717171102661215, + "tokens_seen": 1912602624 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003398355065195587, + "loss": 1.2637, + "theoretical_loss": 3.471707186569831, + "tokens_seen": 1912668160 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003398194583751254, + "loss": 1.1917, + "theoretical_loss": 3.4716972633087657, + "tokens_seen": 1912733696 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003398034102306921, + "loss": 1.1985, + "theoretical_loss": 3.471687340482892, + "tokens_seen": 1912799232 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003397873620862588, + "loss": 1.2209, + "theoretical_loss": 3.4716774180921752, + "tokens_seen": 1912864768 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003397713139418255, + "loss": 1.303, + "theoretical_loss": 3.4716674961365825, + "tokens_seen": 1912930304 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003397552657973922, + "loss": 1.3331, + "theoretical_loss": 3.4716575746160787, + "tokens_seen": 1912995840 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003397392176529589, + "loss": 1.2819, + "theoretical_loss": 3.4716476535306304, + "tokens_seen": 1913061376 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003397231695085256, + "loss": 1.2822, + "theoretical_loss": 3.471637732880204, + "tokens_seen": 1913126912 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003397071213640923, + "loss": 1.3185, + "theoretical_loss": 3.4716278126647646, + "tokens_seen": 1913192448 + }, + { + "epoch": 0.58, + "learning_rate": 0.000339691073219659, + "loss": 1.2822, + "theoretical_loss": 3.4716178928842787, + "tokens_seen": 1913257984 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003396750250752257, + "loss": 1.289, + "theoretical_loss": 3.4716079735387124, + "tokens_seen": 1913323520 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003396589769307924, + "loss": 1.2929, + "theoretical_loss": 3.471598054628031, + "tokens_seen": 1913389056 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003396429287863591, + "loss": 1.4238, + "theoretical_loss": 3.471588136152202, + "tokens_seen": 1913454592 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003396268806419258, + "loss": 1.3002, + "theoretical_loss": 3.471578218111191, + "tokens_seen": 1913520128 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003396108324974925, + "loss": 1.3253, + "theoretical_loss": 3.471568300504963, + "tokens_seen": 1913585664 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 641663, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2823185920715332, + "objective/train/theoretical_loss": 3.471558383333485, + "objective/train/tokens_used": 1934111200, + "theoretical_loss": 3.471558383333485, + "tokens_seen": 1913651200 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003395947843530592, + "loss": 1.3023, + "theoretical_loss": 3.471558383333485, + "tokens_seen": 1913651200 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003395787362086259, + "loss": 1.2016, + "theoretical_loss": 3.471548466596723, + "tokens_seen": 1913716736 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003395626880641926, + "loss": 1.3205, + "theoretical_loss": 3.4715385502946425, + "tokens_seen": 1913782272 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003395466399197593, + "loss": 1.2488, + "theoretical_loss": 3.47152863442721, + "tokens_seen": 1913847808 + }, + { + "epoch": 0.58, + "learning_rate": 0.000339530591775326, + "loss": 1.4029, + "theoretical_loss": 3.4715187189943917, + "tokens_seen": 1913913344 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003395145436308927, + "loss": 1.3087, + "theoretical_loss": 3.4715088039961532, + "tokens_seen": 1913978880 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033949849548645935, + "loss": 1.2972, + "theoretical_loss": 3.4714988894324605, + "tokens_seen": 1914044416 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033948244734202605, + "loss": 1.3646, + "theoretical_loss": 3.4714889753032807, + "tokens_seen": 1914109952 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033946639919759275, + "loss": 1.3149, + "theoretical_loss": 3.471479061608579, + "tokens_seen": 1914175488 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033945035105315945, + "loss": 1.3548, + "theoretical_loss": 3.4714691483483215, + "tokens_seen": 1914241024 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033943430290872615, + "loss": 1.3736, + "theoretical_loss": 3.4714592355224747, + "tokens_seen": 1914306560 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033941825476429285, + "loss": 1.2342, + "theoretical_loss": 3.4714493231310044, + "tokens_seen": 1914372096 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033940220661985956, + "loss": 1.3392, + "theoretical_loss": 3.4714394111738764, + "tokens_seen": 1914437632 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033938615847542626, + "loss": 1.2515, + "theoretical_loss": 3.471429499651057, + "tokens_seen": 1914503168 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033937011033099296, + "loss": 1.3416, + "theoretical_loss": 3.4714195885625134, + "tokens_seen": 1914568704 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033935406218655966, + "loss": 1.2121, + "theoretical_loss": 3.47140967790821, + "tokens_seen": 1914634240 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003393380140421264, + "loss": 1.3935, + "theoretical_loss": 3.471399767688114, + "tokens_seen": 1914699776 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003393219658976931, + "loss": 1.1634, + "theoretical_loss": 3.471389857902191, + "tokens_seen": 1914765312 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003393059177532598, + "loss": 1.2524, + "theoretical_loss": 3.471379948550407, + "tokens_seen": 1914830848 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003392898696088265, + "loss": 1.1771, + "theoretical_loss": 3.471370039632729, + "tokens_seen": 1914896384 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003392738214643932, + "loss": 1.3699, + "theoretical_loss": 3.4713601311491225, + "tokens_seen": 1914961920 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003392577733199599, + "loss": 1.3535, + "theoretical_loss": 3.471350223099553, + "tokens_seen": 1915027456 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003392417251755266, + "loss": 1.2448, + "theoretical_loss": 3.471340315483988, + "tokens_seen": 1915092992 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003392256770310933, + "loss": 1.2565, + "theoretical_loss": 3.471330408302393, + "tokens_seen": 1915158528 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033920962888666, + "loss": 1.2465, + "theoretical_loss": 3.471320501554734, + "tokens_seen": 1915224064 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003391935807422267, + "loss": 1.3965, + "theoretical_loss": 3.4713105952409773, + "tokens_seen": 1915289600 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003391775325977934, + "loss": 1.3081, + "theoretical_loss": 3.4713006893610885, + "tokens_seen": 1915355136 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003391614844533601, + "loss": 1.2785, + "theoretical_loss": 3.471290783915035, + "tokens_seen": 1915420672 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003391454363089268, + "loss": 1.2244, + "theoretical_loss": 3.471280878902782, + "tokens_seen": 1915486208 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003391293881644935, + "loss": 1.2579, + "theoretical_loss": 3.471270974324295, + "tokens_seen": 1915551744 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003391133400200602, + "loss": 1.2709, + "theoretical_loss": 3.471261070179542, + "tokens_seen": 1915617280 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003390972918756269, + "loss": 1.3413, + "theoretical_loss": 3.4712511664684875, + "tokens_seen": 1915682816 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003390812437311936, + "loss": 1.2658, + "theoretical_loss": 3.471241263191099, + "tokens_seen": 1915748352 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003390651955867603, + "loss": 1.2981, + "theoretical_loss": 3.4712313603473417, + "tokens_seen": 1915813888 + }, + { + "epoch": 0.58, + "learning_rate": 0.000339049147442327, + "loss": 1.2708, + "theoretical_loss": 3.4712214579371823, + "tokens_seen": 1915879424 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003390330992978937, + "loss": 1.3324, + "theoretical_loss": 3.471211555960587, + "tokens_seen": 1915944960 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003390170511534604, + "loss": 1.2945, + "theoretical_loss": 3.4712016544175217, + "tokens_seen": 1916010496 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003390010030090271, + "loss": 1.4641, + "theoretical_loss": 3.4711917533079526, + "tokens_seen": 1916076032 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003389849548645938, + "loss": 1.2335, + "theoretical_loss": 3.471181852631846, + "tokens_seen": 1916141568 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003389689067201605, + "loss": 1.2682, + "theoretical_loss": 3.4711719523891684, + "tokens_seen": 1916207104 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033895285857572723, + "loss": 1.2513, + "theoretical_loss": 3.471162052579885, + "tokens_seen": 1916272640 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033893681043129393, + "loss": 1.3361, + "theoretical_loss": 3.4711521532039633, + "tokens_seen": 1916338176 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033892076228686063, + "loss": 1.2282, + "theoretical_loss": 3.471142254261369, + "tokens_seen": 1916403712 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033890471414242733, + "loss": 1.3111, + "theoretical_loss": 3.4711323557520677, + "tokens_seen": 1916469248 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033888866599799403, + "loss": 1.3891, + "theoretical_loss": 3.4711224576760262, + "tokens_seen": 1916534784 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003388726178535607, + "loss": 1.178, + "theoretical_loss": 3.4711125600332107, + "tokens_seen": 1916600320 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003388565697091274, + "loss": 1.3536, + "theoretical_loss": 3.4711026628235873, + "tokens_seen": 1916665856 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003388405215646941, + "loss": 1.349, + "theoretical_loss": 3.4710927660471227, + "tokens_seen": 1916731392 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003388244734202608, + "loss": 1.3321, + "theoretical_loss": 3.4710828697037828, + "tokens_seen": 1916796928 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003388084252758275, + "loss": 1.3324, + "theoretical_loss": 3.4710729737935333, + "tokens_seen": 1916862464 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 642590, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3644001483917236, + "objective/train/theoretical_loss": 3.471063078316341, + "objective/train/tokens_used": 1937388000, + "theoretical_loss": 3.471063078316341, + "tokens_seen": 1916928000 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003387923771313942, + "loss": 1.2952, + "theoretical_loss": 3.471063078316341, + "tokens_seen": 1916928000 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003387763289869609, + "loss": 1.3196, + "theoretical_loss": 3.4710531832721725, + "tokens_seen": 1916993536 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003387602808425276, + "loss": 1.3215, + "theoretical_loss": 3.4710432886609937, + "tokens_seen": 1917059072 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003387442326980943, + "loss": 1.3273, + "theoretical_loss": 3.4710333944827703, + "tokens_seen": 1917124608 + }, + { + "epoch": 0.58, + "learning_rate": 0.000338728184553661, + "loss": 1.3059, + "theoretical_loss": 3.471023500737469, + "tokens_seen": 1917190144 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003387121364092277, + "loss": 1.3575, + "theoretical_loss": 3.471013607425056, + "tokens_seen": 1917255680 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003386960882647944, + "loss": 1.3217, + "theoretical_loss": 3.4710037145454984, + "tokens_seen": 1917321216 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003386800401203611, + "loss": 1.1912, + "theoretical_loss": 3.470993822098761, + "tokens_seen": 1917386752 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003386639919759278, + "loss": 1.2745, + "theoretical_loss": 3.4709839300848113, + "tokens_seen": 1917452288 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003386479438314945, + "loss": 1.1026, + "theoretical_loss": 3.4709740385036145, + "tokens_seen": 1917517824 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003386318956870612, + "loss": 1.3516, + "theoretical_loss": 3.470964147355138, + "tokens_seen": 1917583360 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003386158475426279, + "loss": 1.2604, + "theoretical_loss": 3.470954256639347, + "tokens_seen": 1917648896 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003385997993981946, + "loss": 1.255, + "theoretical_loss": 3.4709443663562087, + "tokens_seen": 1917714432 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003385837512537613, + "loss": 1.3005, + "theoretical_loss": 3.470934476505689, + "tokens_seen": 1917779968 + }, + { + "epoch": 0.58, + "learning_rate": 0.000338567703109328, + "loss": 1.2588, + "theoretical_loss": 3.470924587087754, + "tokens_seen": 1917845504 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003385516549648947, + "loss": 1.197, + "theoretical_loss": 3.47091469810237, + "tokens_seen": 1917911040 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003385356068204614, + "loss": 1.2088, + "theoretical_loss": 3.470904809549504, + "tokens_seen": 1917976576 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003385195586760281, + "loss": 1.2335, + "theoretical_loss": 3.470894921429122, + "tokens_seen": 1918042112 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003385035105315948, + "loss": 1.3654, + "theoretical_loss": 3.4708850337411894, + "tokens_seen": 1918107648 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003384874623871615, + "loss": 1.2825, + "theoretical_loss": 3.4708751464856737, + "tokens_seen": 1918173184 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003384714142427282, + "loss": 1.3037, + "theoretical_loss": 3.4708652596625402, + "tokens_seen": 1918238720 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003384553660982949, + "loss": 1.2987, + "theoretical_loss": 3.470855373271756, + "tokens_seen": 1918304256 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003384393179538616, + "loss": 1.2438, + "theoretical_loss": 3.4708454873132872, + "tokens_seen": 1918369792 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003384232698094283, + "loss": 1.2161, + "theoretical_loss": 3.4708356017871003, + "tokens_seen": 1918435328 + }, + { + "epoch": 0.58, + "learning_rate": 0.000338407221664995, + "loss": 1.2209, + "theoretical_loss": 3.4708257166931613, + "tokens_seen": 1918500864 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003383911735205617, + "loss": 1.2574, + "theoretical_loss": 3.4708158320314366, + "tokens_seen": 1918566400 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003383751253761284, + "loss": 1.3434, + "theoretical_loss": 3.470805947801893, + "tokens_seen": 1918631936 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003383590772316951, + "loss": 1.289, + "theoretical_loss": 3.470796064004496, + "tokens_seen": 1918697472 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003383430290872618, + "loss": 1.3471, + "theoretical_loss": 3.4707861806392124, + "tokens_seen": 1918763008 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003383269809428285, + "loss": 1.2051, + "theoretical_loss": 3.470776297706009, + "tokens_seen": 1918828544 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003383109327983952, + "loss": 1.2706, + "theoretical_loss": 3.4707664152048516, + "tokens_seen": 1918894080 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003382948846539619, + "loss": 1.2493, + "theoretical_loss": 3.4707565331357064, + "tokens_seen": 1918959616 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003382788365095286, + "loss": 1.1947, + "theoretical_loss": 3.47074665149854, + "tokens_seen": 1919025152 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003382627883650953, + "loss": 1.3236, + "theoretical_loss": 3.4707367702933194, + "tokens_seen": 1919090688 + }, + { + "epoch": 0.58, + "learning_rate": 0.000338246740220662, + "loss": 1.2404, + "theoretical_loss": 3.4707268895200096, + "tokens_seen": 1919156224 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003382306920762287, + "loss": 1.2783, + "theoretical_loss": 3.470717009178578, + "tokens_seen": 1919221760 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003382146439317954, + "loss": 1.3064, + "theoretical_loss": 3.4707071292689906, + "tokens_seen": 1919287296 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003381985957873621, + "loss": 1.3922, + "theoretical_loss": 3.470697249791214, + "tokens_seen": 1919352832 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003381825476429288, + "loss": 1.3519, + "theoretical_loss": 3.470687370745215, + "tokens_seen": 1919418368 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003381664994984955, + "loss": 1.3373, + "theoretical_loss": 3.4706774921309584, + "tokens_seen": 1919483904 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003381504513540622, + "loss": 1.3182, + "theoretical_loss": 3.4706676139484127, + "tokens_seen": 1919549440 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003381344032096289, + "loss": 1.2923, + "theoretical_loss": 3.4706577361975426, + "tokens_seen": 1919614976 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003381183550651956, + "loss": 1.2898, + "theoretical_loss": 3.470647858878315, + "tokens_seen": 1919680512 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003381023069207623, + "loss": 1.2055, + "theoretical_loss": 3.4706379819906967, + "tokens_seen": 1919746048 + }, + { + "epoch": 0.58, + "learning_rate": 0.000338086258776329, + "loss": 1.2727, + "theoretical_loss": 3.4706281055346544, + "tokens_seen": 1919811584 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003380702106318957, + "loss": 1.3049, + "theoretical_loss": 3.470618229510153, + "tokens_seen": 1919877120 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003380541624874624, + "loss": 1.301, + "theoretical_loss": 3.470608353917161, + "tokens_seen": 1919942656 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003380381143430291, + "loss": 1.3751, + "theoretical_loss": 3.470598478755643, + "tokens_seen": 1920008192 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003380220661985958, + "loss": 1.2357, + "theoretical_loss": 3.470588604025566, + "tokens_seen": 1920073728 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003380060180541625, + "loss": 1.3704, + "theoretical_loss": 3.4705787297268964, + "tokens_seen": 1920139264 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 643782, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1430565118789673, + "objective/train/theoretical_loss": 3.4705688558596015, + "objective/train/tokens_used": 1940664800, + "theoretical_loss": 3.4705688558596015, + "tokens_seen": 1920204800 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003379899699097292, + "loss": 1.1265, + "theoretical_loss": 3.4705688558596015, + "tokens_seen": 1920204800 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003379739217652959, + "loss": 1.2697, + "theoretical_loss": 3.470558982423647, + "tokens_seen": 1920270336 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003379578736208626, + "loss": 1.2358, + "theoretical_loss": 3.4705491094189984, + "tokens_seen": 1920335872 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003379418254764293, + "loss": 1.3417, + "theoretical_loss": 3.470539236845624, + "tokens_seen": 1920401408 + }, + { + "epoch": 0.58, + "learning_rate": 0.000337925777331996, + "loss": 1.3283, + "theoretical_loss": 3.4705293647034887, + "tokens_seen": 1920466944 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003379097291875627, + "loss": 1.2809, + "theoretical_loss": 3.47051949299256, + "tokens_seen": 1920532480 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003378936810431294, + "loss": 1.3012, + "theoretical_loss": 3.470509621712804, + "tokens_seen": 1920598016 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003378776328986961, + "loss": 1.2338, + "theoretical_loss": 3.470499750864187, + "tokens_seen": 1920663552 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003378615847542628, + "loss": 1.2661, + "theoretical_loss": 3.4704898804466753, + "tokens_seen": 1920729088 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003378455366098295, + "loss": 1.5689, + "theoretical_loss": 3.470480010460236, + "tokens_seen": 1920794624 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003378294884653962, + "loss": 1.3454, + "theoretical_loss": 3.470470140904835, + "tokens_seen": 1920860160 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003378134403209629, + "loss": 1.2351, + "theoretical_loss": 3.470460271780439, + "tokens_seen": 1920925696 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003377973921765296, + "loss": 1.3198, + "theoretical_loss": 3.470450403087014, + "tokens_seen": 1920991232 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003377813440320963, + "loss": 1.2931, + "theoretical_loss": 3.4704405348245277, + "tokens_seen": 1921056768 + }, + { + "epoch": 0.58, + "learning_rate": 0.000337765295887663, + "loss": 1.3315, + "theoretical_loss": 3.470430666992945, + "tokens_seen": 1921122304 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003377492477432297, + "loss": 1.3322, + "theoretical_loss": 3.4704207995922336, + "tokens_seen": 1921187840 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003377331995987964, + "loss": 1.2897, + "theoretical_loss": 3.470410932622359, + "tokens_seen": 1921253376 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003377171514543631, + "loss": 1.2736, + "theoretical_loss": 3.470401066083289, + "tokens_seen": 1921318912 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003377011033099298, + "loss": 1.3734, + "theoretical_loss": 3.4703911999749892, + "tokens_seen": 1921384448 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003376850551654965, + "loss": 1.184, + "theoretical_loss": 3.470381334297426, + "tokens_seen": 1921449984 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003376690070210632, + "loss": 1.31, + "theoretical_loss": 3.4703714690505665, + "tokens_seen": 1921515520 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003376529588766299, + "loss": 1.3105, + "theoretical_loss": 3.470361604234377, + "tokens_seen": 1921581056 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003376369107321966, + "loss": 1.2669, + "theoretical_loss": 3.470351739848823, + "tokens_seen": 1921646592 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003376208625877633, + "loss": 1.242, + "theoretical_loss": 3.4703418758938724, + "tokens_seen": 1921712128 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033760481444333, + "loss": 1.3229, + "theoretical_loss": 3.470332012369491, + "tokens_seen": 1921777664 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003375887662988967, + "loss": 1.2635, + "theoretical_loss": 3.470322149275646, + "tokens_seen": 1921843200 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033757271815446343, + "loss": 1.2588, + "theoretical_loss": 3.4703122866123035, + "tokens_seen": 1921908736 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033755667001003013, + "loss": 1.3038, + "theoretical_loss": 3.4703024243794296, + "tokens_seen": 1921974272 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033754062186559683, + "loss": 1.2789, + "theoretical_loss": 3.470292562576991, + "tokens_seen": 1922039808 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033752457372116353, + "loss": 1.1687, + "theoretical_loss": 3.4702827012049546, + "tokens_seen": 1922105344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033750852557673023, + "loss": 1.2674, + "theoretical_loss": 3.4702728402632874, + "tokens_seen": 1922170880 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033749247743229693, + "loss": 1.2238, + "theoretical_loss": 3.470262979751955, + "tokens_seen": 1922236416 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033747642928786363, + "loss": 1.3842, + "theoretical_loss": 3.4702531196709243, + "tokens_seen": 1922301952 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003374603811434303, + "loss": 1.2824, + "theoretical_loss": 3.4702432600201614, + "tokens_seen": 1922367488 + }, + { + "epoch": 0.58, + "learning_rate": 0.000337444332998997, + "loss": 1.2245, + "theoretical_loss": 3.470233400799634, + "tokens_seen": 1922433024 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003374282848545637, + "loss": 1.2089, + "theoretical_loss": 3.4702235420093075, + "tokens_seen": 1922498560 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003374122367101304, + "loss": 1.354, + "theoretical_loss": 3.470213683649149, + "tokens_seen": 1922564096 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003373961885656971, + "loss": 1.2286, + "theoretical_loss": 3.470203825719125, + "tokens_seen": 1922629632 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003373801404212638, + "loss": 1.3276, + "theoretical_loss": 3.4701939682192022, + "tokens_seen": 1922695168 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003373640922768305, + "loss": 1.2005, + "theoretical_loss": 3.470184111149347, + "tokens_seen": 1922760704 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003373480441323972, + "loss": 1.1655, + "theoretical_loss": 3.470174254509526, + "tokens_seen": 1922826240 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003373319959879639, + "loss": 1.3572, + "theoretical_loss": 3.4701643982997057, + "tokens_seen": 1922891776 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003373159478435306, + "loss": 1.3709, + "theoretical_loss": 3.470154542519853, + "tokens_seen": 1922957312 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003372998996990973, + "loss": 1.2019, + "theoretical_loss": 3.4701446871699346, + "tokens_seen": 1923022848 + }, + { + "epoch": 0.58, + "learning_rate": 0.000337283851554664, + "loss": 1.3661, + "theoretical_loss": 3.470134832249916, + "tokens_seen": 1923088384 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003372678034102307, + "loss": 1.2352, + "theoretical_loss": 3.470124977759765, + "tokens_seen": 1923153920 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003372517552657974, + "loss": 1.3082, + "theoretical_loss": 3.470115123699448, + "tokens_seen": 1923219456 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003372357071213641, + "loss": 1.2888, + "theoretical_loss": 3.470105270068931, + "tokens_seen": 1923284992 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003372196589769308, + "loss": 1.3399, + "theoretical_loss": 3.4700954168681815, + "tokens_seen": 1923350528 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003372036108324975, + "loss": 1.2767, + "theoretical_loss": 3.4700855640971646, + "tokens_seen": 1923416064 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 644931, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.426339864730835, + "objective/train/theoretical_loss": 3.470075711755849, + "objective/train/tokens_used": 1943941600, + "theoretical_loss": 3.470075711755849, + "tokens_seen": 1923481600 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003371875626880642, + "loss": 1.2773, + "theoretical_loss": 3.470075711755849, + "tokens_seen": 1923481600 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003371715145436309, + "loss": 1.3313, + "theoretical_loss": 3.4700658598442, + "tokens_seen": 1923547136 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003371554663991976, + "loss": 1.3063, + "theoretical_loss": 3.4700560083621843, + "tokens_seen": 1923612672 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003371394182547643, + "loss": 1.4039, + "theoretical_loss": 3.4700461573097687, + "tokens_seen": 1923678208 + }, + { + "epoch": 0.58, + "learning_rate": 0.000337123370110331, + "loss": 1.3048, + "theoretical_loss": 3.4700363066869198, + "tokens_seen": 1923743744 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003371073219658977, + "loss": 1.3075, + "theoretical_loss": 3.470026456493604, + "tokens_seen": 1923809280 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003370912738214644, + "loss": 1.277, + "theoretical_loss": 3.4700166067297884, + "tokens_seen": 1923874816 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003370752256770311, + "loss": 1.2305, + "theoretical_loss": 3.4700067573954394, + "tokens_seen": 1923940352 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003370591775325978, + "loss": 1.3303, + "theoretical_loss": 3.4699969084905242, + "tokens_seen": 1924005888 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003370431293881645, + "loss": 1.263, + "theoretical_loss": 3.4699870600150087, + "tokens_seen": 1924071424 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003370270812437312, + "loss": 1.2816, + "theoretical_loss": 3.4699772119688594, + "tokens_seen": 1924136960 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003370110330992979, + "loss": 1.2359, + "theoretical_loss": 3.4699673643520437, + "tokens_seen": 1924202496 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003369949849548646, + "loss": 1.3286, + "theoretical_loss": 3.469957517164528, + "tokens_seen": 1924268032 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003369789368104313, + "loss": 1.2718, + "theoretical_loss": 3.4699476704062784, + "tokens_seen": 1924333568 + }, + { + "epoch": 0.58, + "learning_rate": 0.000336962888665998, + "loss": 1.324, + "theoretical_loss": 3.4699378240772623, + "tokens_seen": 1924399104 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003369468405215647, + "loss": 1.2969, + "theoretical_loss": 3.469927978177446, + "tokens_seen": 1924464640 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033693079237713145, + "loss": 1.4035, + "theoretical_loss": 3.4699181327067965, + "tokens_seen": 1924530176 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033691474423269815, + "loss": 1.2772, + "theoretical_loss": 3.4699082876652803, + "tokens_seen": 1924595712 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033689869608826485, + "loss": 1.329, + "theoretical_loss": 3.4698984430528643, + "tokens_seen": 1924661248 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033688264794383155, + "loss": 1.263, + "theoretical_loss": 3.4698885988695145, + "tokens_seen": 1924726784 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033686659979939826, + "loss": 1.3344, + "theoretical_loss": 3.4698787551151984, + "tokens_seen": 1924792320 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033685055165496496, + "loss": 1.2129, + "theoretical_loss": 3.4698689117898818, + "tokens_seen": 1924857856 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003368345035105316, + "loss": 1.3115, + "theoretical_loss": 3.4698590688935322, + "tokens_seen": 1924923392 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003368184553660983, + "loss": 1.4533, + "theoretical_loss": 3.469849226426116, + "tokens_seen": 1924988928 + }, + { + "epoch": 0.58, + "learning_rate": 0.000336802407221665, + "loss": 1.2906, + "theoretical_loss": 3.4698393843876003, + "tokens_seen": 1925054464 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003367863590772317, + "loss": 1.2317, + "theoretical_loss": 3.4698295427779513, + "tokens_seen": 1925120000 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003367703109327984, + "loss": 1.3543, + "theoretical_loss": 3.4698197015971357, + "tokens_seen": 1925185536 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003367542627883651, + "loss": 1.2315, + "theoretical_loss": 3.46980986084512, + "tokens_seen": 1925251072 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003367382146439318, + "loss": 1.2207, + "theoretical_loss": 3.469800020521872, + "tokens_seen": 1925316608 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003367221664994985, + "loss": 1.3044, + "theoretical_loss": 3.4697901806273572, + "tokens_seen": 1925382144 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003367061183550652, + "loss": 1.217, + "theoretical_loss": 3.4697803411615435, + "tokens_seen": 1925447680 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003366900702106319, + "loss": 1.2545, + "theoretical_loss": 3.4697705021243963, + "tokens_seen": 1925513216 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003366740220661986, + "loss": 1.2472, + "theoretical_loss": 3.4697606635158835, + "tokens_seen": 1925578752 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003366579739217653, + "loss": 1.2103, + "theoretical_loss": 3.469750825335971, + "tokens_seen": 1925644288 + }, + { + "epoch": 0.58, + "learning_rate": 0.000336641925777332, + "loss": 1.2106, + "theoretical_loss": 3.4697409875846263, + "tokens_seen": 1925709824 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003366258776328987, + "loss": 1.3245, + "theoretical_loss": 3.4697311502618153, + "tokens_seen": 1925775360 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003366098294884654, + "loss": 1.2573, + "theoretical_loss": 3.4697213133675056, + "tokens_seen": 1925840896 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003365937813440321, + "loss": 1.2262, + "theoretical_loss": 3.4697114769016633, + "tokens_seen": 1925906432 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003365777331995988, + "loss": 1.2229, + "theoretical_loss": 3.4697016408642556, + "tokens_seen": 1925971968 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003365616850551655, + "loss": 1.1705, + "theoretical_loss": 3.469691805255249, + "tokens_seen": 1926037504 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003365456369107322, + "loss": 1.197, + "theoretical_loss": 3.46968197007461, + "tokens_seen": 1926103040 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003365295887662989, + "loss": 1.3286, + "theoretical_loss": 3.4696721353223063, + "tokens_seen": 1926168576 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003365135406218656, + "loss": 1.2913, + "theoretical_loss": 3.4696623009983036, + "tokens_seen": 1926234112 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003364974924774323, + "loss": 1.1004, + "theoretical_loss": 3.469652467102569, + "tokens_seen": 1926299648 + }, + { + "epoch": 0.58, + "learning_rate": 0.000336481444332999, + "loss": 1.3217, + "theoretical_loss": 3.46964263363507, + "tokens_seen": 1926365184 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003364653961885657, + "loss": 1.1726, + "theoretical_loss": 3.4696328005957726, + "tokens_seen": 1926430720 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003364493480441324, + "loss": 1.2257, + "theoretical_loss": 3.4696229679846438, + "tokens_seen": 1926496256 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003364332998996991, + "loss": 1.2384, + "theoretical_loss": 3.4696131358016506, + "tokens_seen": 1926561792 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003364172517552658, + "loss": 1.2646, + "theoretical_loss": 3.4696033040467595, + "tokens_seen": 1926627328 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003364012036108325, + "loss": 1.1931, + "theoretical_loss": 3.469593472719937, + "tokens_seen": 1926692864 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 645797, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.161024570465088, + "objective/train/theoretical_loss": 3.469583641821151, + "objective/train/tokens_used": 1947218400, + "theoretical_loss": 3.469583641821151, + "tokens_seen": 1926758400 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003363851554663992, + "loss": 1.2581, + "theoretical_loss": 3.469583641821151, + "tokens_seen": 1926758400 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003363691073219659, + "loss": 1.2966, + "theoretical_loss": 3.469573811350367, + "tokens_seen": 1926823936 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003363530591775326, + "loss": 1.2587, + "theoretical_loss": 3.4695639813075525, + "tokens_seen": 1926889472 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003363370110330993, + "loss": 1.2684, + "theoretical_loss": 3.469554151692674, + "tokens_seen": 1926955008 + }, + { + "epoch": 0.58, + "learning_rate": 0.000336320962888666, + "loss": 1.162, + "theoretical_loss": 3.4695443225056994, + "tokens_seen": 1927020544 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003363049147442327, + "loss": 1.3771, + "theoretical_loss": 3.469534493746594, + "tokens_seen": 1927086080 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003362888665997994, + "loss": 1.2507, + "theoretical_loss": 3.4695246654153253, + "tokens_seen": 1927151616 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003362728184553661, + "loss": 1.262, + "theoretical_loss": 3.4695148375118605, + "tokens_seen": 1927217152 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003362567703109328, + "loss": 1.1684, + "theoretical_loss": 3.4695050100361655, + "tokens_seen": 1927282688 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003362407221664995, + "loss": 1.166, + "theoretical_loss": 3.469495182988208, + "tokens_seen": 1927348224 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003362246740220662, + "loss": 1.2162, + "theoretical_loss": 3.4694853563679544, + "tokens_seen": 1927413760 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033620862587763293, + "loss": 1.1911, + "theoretical_loss": 3.469475530175372, + "tokens_seen": 1927479296 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033619257773319963, + "loss": 1.278, + "theoretical_loss": 3.4694657044104273, + "tokens_seen": 1927544832 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033617652958876633, + "loss": 1.2794, + "theoretical_loss": 3.4694558790730867, + "tokens_seen": 1927610368 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033616048144433303, + "loss": 1.1319, + "theoretical_loss": 3.469446054163318, + "tokens_seen": 1927675904 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033614443329989973, + "loss": 1.2076, + "theoretical_loss": 3.4694362296810874, + "tokens_seen": 1927741440 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033612838515546643, + "loss": 1.3072, + "theoretical_loss": 3.469426405626362, + "tokens_seen": 1927806976 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033611233701103313, + "loss": 1.1948, + "theoretical_loss": 3.4694165819991083, + "tokens_seen": 1927872512 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033609628886659983, + "loss": 1.2288, + "theoretical_loss": 3.469406758799294, + "tokens_seen": 1927938048 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033608024072216653, + "loss": 1.3003, + "theoretical_loss": 3.469396936026885, + "tokens_seen": 1928003584 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003360641925777332, + "loss": 1.272, + "theoretical_loss": 3.4693871136818486, + "tokens_seen": 1928069120 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003360481444332999, + "loss": 1.3479, + "theoretical_loss": 3.469377291764152, + "tokens_seen": 1928134656 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003360320962888666, + "loss": 1.2533, + "theoretical_loss": 3.4693674702737614, + "tokens_seen": 1928200192 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003360160481444333, + "loss": 1.3227, + "theoretical_loss": 3.4693576492106444, + "tokens_seen": 1928265728 + }, + { + "epoch": 0.58, + "learning_rate": 0.000336, + "loss": 1.4136, + "theoretical_loss": 3.4693478285747674, + "tokens_seen": 1928331264 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003359839518555667, + "loss": 1.175, + "theoretical_loss": 3.4693380083660976, + "tokens_seen": 1928396800 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003359679037111334, + "loss": 1.26, + "theoretical_loss": 3.4693281885846017, + "tokens_seen": 1928462336 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003359518555667001, + "loss": 1.2742, + "theoretical_loss": 3.4693183692302467, + "tokens_seen": 1928527872 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003359358074222668, + "loss": 1.2938, + "theoretical_loss": 3.4693085503029994, + "tokens_seen": 1928593408 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003359197592778335, + "loss": 1.2441, + "theoretical_loss": 3.4692987318028266, + "tokens_seen": 1928658944 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003359037111334002, + "loss": 1.3074, + "theoretical_loss": 3.4692889137296956, + "tokens_seen": 1928724480 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003358876629889669, + "loss": 1.215, + "theoretical_loss": 3.4692790960835733, + "tokens_seen": 1928790016 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003358716148445336, + "loss": 1.1787, + "theoretical_loss": 3.469269278864426, + "tokens_seen": 1928855552 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003358555667001003, + "loss": 1.3828, + "theoretical_loss": 3.4692594620722215, + "tokens_seen": 1928921088 + }, + { + "epoch": 0.58, + "learning_rate": 0.000335839518555667, + "loss": 1.2958, + "theoretical_loss": 3.4692496457069257, + "tokens_seen": 1928986624 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003358234704112337, + "loss": 1.2942, + "theoretical_loss": 3.4692398297685063, + "tokens_seen": 1929052160 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003358074222668004, + "loss": 1.2572, + "theoretical_loss": 3.4692300142569303, + "tokens_seen": 1929117696 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003357913741223671, + "loss": 1.2578, + "theoretical_loss": 3.4692201991721645, + "tokens_seen": 1929183232 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003357753259779338, + "loss": 1.3917, + "theoretical_loss": 3.4692103845141755, + "tokens_seen": 1929248768 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003357592778335005, + "loss": 1.2551, + "theoretical_loss": 3.46920057028293, + "tokens_seen": 1929314304 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003357432296890672, + "loss": 1.2612, + "theoretical_loss": 3.4691907564783966, + "tokens_seen": 1929379840 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033572718154463395, + "loss": 1.2458, + "theoretical_loss": 3.46918094310054, + "tokens_seen": 1929445376 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033571113340020065, + "loss": 1.3331, + "theoretical_loss": 3.469171130149329, + "tokens_seen": 1929510912 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033569508525576735, + "loss": 1.1819, + "theoretical_loss": 3.469161317624729, + "tokens_seen": 1929576448 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033567903711133405, + "loss": 1.2741, + "theoretical_loss": 3.4691515055267086, + "tokens_seen": 1929641984 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033566298896690075, + "loss": 1.2165, + "theoretical_loss": 3.4691416938552333, + "tokens_seen": 1929707520 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033564694082246745, + "loss": 1.2814, + "theoretical_loss": 3.469131882610271, + "tokens_seen": 1929773056 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033563089267803415, + "loss": 1.2307, + "theoretical_loss": 3.469122071791788, + "tokens_seen": 1929838592 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033561484453360085, + "loss": 1.196, + "theoretical_loss": 3.469112261399752, + "tokens_seen": 1929904128 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033559879638916755, + "loss": 1.253, + "theoretical_loss": 3.4691024514341295, + "tokens_seen": 1929969664 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 646757, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2885311841964722, + "objective/train/theoretical_loss": 3.4690926418948878, + "objective/train/tokens_used": 1950495200, + "theoretical_loss": 3.4690926418948878, + "tokens_seen": 1930035200 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033558274824473425, + "loss": 1.2264, + "theoretical_loss": 3.4690926418948878, + "tokens_seen": 1930035200 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033556670010030095, + "loss": 1.2388, + "theoretical_loss": 3.469082832781994, + "tokens_seen": 1930100736 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033555065195586765, + "loss": 1.3518, + "theoretical_loss": 3.4690730240954144, + "tokens_seen": 1930166272 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033553460381143435, + "loss": 1.2608, + "theoretical_loss": 3.4690632158351162, + "tokens_seen": 1930231808 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033551855566700105, + "loss": 1.2367, + "theoretical_loss": 3.469053408001067, + "tokens_seen": 1930297344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033550250752256776, + "loss": 1.329, + "theoretical_loss": 3.469043600593233, + "tokens_seen": 1930362880 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033548645937813446, + "loss": 1.2201, + "theoretical_loss": 3.4690337936115823, + "tokens_seen": 1930428416 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033547041123370116, + "loss": 1.3149, + "theoretical_loss": 3.4690239870560804, + "tokens_seen": 1930493952 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033545436308926786, + "loss": 1.3232, + "theoretical_loss": 3.4690141809266954, + "tokens_seen": 1930559488 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033543831494483456, + "loss": 1.2863, + "theoretical_loss": 3.4690043752233946, + "tokens_seen": 1930625024 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003354222668004012, + "loss": 1.3294, + "theoretical_loss": 3.468994569946144, + "tokens_seen": 1930690560 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003354062186559679, + "loss": 1.2215, + "theoretical_loss": 3.4689847650949117, + "tokens_seen": 1930756096 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003353901705115346, + "loss": 1.2123, + "theoretical_loss": 3.4689749606696636, + "tokens_seen": 1930821632 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003353741223671013, + "loss": 1.3493, + "theoretical_loss": 3.4689651566703676, + "tokens_seen": 1930887168 + }, + { + "epoch": 0.59, + "learning_rate": 0.000335358074222668, + "loss": 1.2221, + "theoretical_loss": 3.46895535309699, + "tokens_seen": 1930952704 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003353420260782347, + "loss": 1.2583, + "theoretical_loss": 3.468945549949499, + "tokens_seen": 1931018240 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003353259779338014, + "loss": 1.2836, + "theoretical_loss": 3.46893574722786, + "tokens_seen": 1931083776 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003353099297893681, + "loss": 1.2082, + "theoretical_loss": 3.4689259449320415, + "tokens_seen": 1931149312 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003352938816449348, + "loss": 1.245, + "theoretical_loss": 3.4689161430620103, + "tokens_seen": 1931214848 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003352778335005015, + "loss": 1.2653, + "theoretical_loss": 3.468906341617733, + "tokens_seen": 1931280384 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003352617853560682, + "loss": 1.2981, + "theoretical_loss": 3.4688965405991765, + "tokens_seen": 1931345920 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003352457372116349, + "loss": 1.1898, + "theoretical_loss": 3.468886740006308, + "tokens_seen": 1931411456 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003352296890672016, + "loss": 1.1997, + "theoretical_loss": 3.4688769398390953, + "tokens_seen": 1931476992 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003352136409227683, + "loss": 1.1747, + "theoretical_loss": 3.468867140097505, + "tokens_seen": 1931542528 + }, + { + "epoch": 0.59, + "learning_rate": 0.000335197592778335, + "loss": 1.2937, + "theoretical_loss": 3.468857340781504, + "tokens_seen": 1931608064 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003351815446339017, + "loss": 1.2834, + "theoretical_loss": 3.4688475418910594, + "tokens_seen": 1931673600 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003351654964894684, + "loss": 1.3073, + "theoretical_loss": 3.468837743426138, + "tokens_seen": 1931739136 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003351494483450351, + "loss": 1.3026, + "theoretical_loss": 3.4688279453867077, + "tokens_seen": 1931804672 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003351334002006018, + "loss": 1.2804, + "theoretical_loss": 3.468818147772735, + "tokens_seen": 1931870208 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003351173520561685, + "loss": 1.1748, + "theoretical_loss": 3.4688083505841876, + "tokens_seen": 1931935744 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003351013039117352, + "loss": 1.3543, + "theoretical_loss": 3.4687985538210317, + "tokens_seen": 1932001280 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003350852557673019, + "loss": 1.1887, + "theoretical_loss": 3.4687887574832352, + "tokens_seen": 1932066816 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003350692076228686, + "loss": 1.3231, + "theoretical_loss": 3.4687789615707647, + "tokens_seen": 1932132352 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003350531594784353, + "loss": 1.1978, + "theoretical_loss": 3.468769166083587, + "tokens_seen": 1932197888 + }, + { + "epoch": 0.59, + "learning_rate": 0.000335037111334002, + "loss": 1.3247, + "theoretical_loss": 3.4687593710216698, + "tokens_seen": 1932263424 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003350210631895687, + "loss": 1.4017, + "theoretical_loss": 3.4687495763849805, + "tokens_seen": 1932328960 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003350050150451354, + "loss": 1.2183, + "theoretical_loss": 3.4687397821734853, + "tokens_seen": 1932394496 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003349889669007021, + "loss": 1.2721, + "theoretical_loss": 3.4687299883871523, + "tokens_seen": 1932460032 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003349729187562688, + "loss": 1.2949, + "theoretical_loss": 3.468720195025948, + "tokens_seen": 1932525568 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003349568706118355, + "loss": 1.3218, + "theoretical_loss": 3.46871040208984, + "tokens_seen": 1932591104 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003349408224674022, + "loss": 1.3127, + "theoretical_loss": 3.4687006095787947, + "tokens_seen": 1932656640 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003349247743229689, + "loss": 1.1875, + "theoretical_loss": 3.4686908174927797, + "tokens_seen": 1932722176 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003349087261785356, + "loss": 1.3231, + "theoretical_loss": 3.4686810258317617, + "tokens_seen": 1932787712 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003348926780341023, + "loss": 1.3218, + "theoretical_loss": 3.4686712345957087, + "tokens_seen": 1932853248 + }, + { + "epoch": 0.59, + "learning_rate": 0.000334876629889669, + "loss": 1.2315, + "theoretical_loss": 3.4686614437845877, + "tokens_seen": 1932918784 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003348605817452357, + "loss": 1.1819, + "theoretical_loss": 3.468651653398365, + "tokens_seen": 1932984320 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003348445336008024, + "loss": 1.2567, + "theoretical_loss": 3.468641863437009, + "tokens_seen": 1933049856 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033482848545636913, + "loss": 1.3268, + "theoretical_loss": 3.4686320739004852, + "tokens_seen": 1933115392 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033481243731193583, + "loss": 1.409, + "theoretical_loss": 3.4686222847887622, + "tokens_seen": 1933180928 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033479638916750253, + "loss": 1.2153, + "theoretical_loss": 3.4686124961018066, + "tokens_seen": 1933246464 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 647912, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2195017337799072, + "objective/train/theoretical_loss": 3.468602707839586, + "objective/train/tokens_used": 1953772000, + "theoretical_loss": 3.468602707839586, + "tokens_seen": 1933312000 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033478034102306923, + "loss": 1.2709, + "theoretical_loss": 3.468602707839586, + "tokens_seen": 1933312000 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033476429287863593, + "loss": 1.1965, + "theoretical_loss": 3.468592920002066, + "tokens_seen": 1933377536 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033474824473420263, + "loss": 1.2993, + "theoretical_loss": 3.4685831325892162, + "tokens_seen": 1933443072 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033473219658976933, + "loss": 1.3346, + "theoretical_loss": 3.468573345601002, + "tokens_seen": 1933508608 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033471614844533603, + "loss": 1.2599, + "theoretical_loss": 3.468563559037392, + "tokens_seen": 1933574144 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033470010030090273, + "loss": 1.2444, + "theoretical_loss": 3.4685537728983515, + "tokens_seen": 1933639680 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033468405215646943, + "loss": 1.2681, + "theoretical_loss": 3.4685439871838497, + "tokens_seen": 1933705216 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033466800401203613, + "loss": 1.2487, + "theoretical_loss": 3.4685342018938523, + "tokens_seen": 1933770752 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003346519558676028, + "loss": 1.2827, + "theoretical_loss": 3.468524417028327, + "tokens_seen": 1933836288 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003346359077231695, + "loss": 1.305, + "theoretical_loss": 3.468514632587241, + "tokens_seen": 1933901824 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003346198595787362, + "loss": 1.3369, + "theoretical_loss": 3.468504848570562, + "tokens_seen": 1933967360 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003346038114343029, + "loss": 1.3156, + "theoretical_loss": 3.4684950649782564, + "tokens_seen": 1934032896 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003345877632898696, + "loss": 1.202, + "theoretical_loss": 3.468485281810292, + "tokens_seen": 1934098432 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003345717151454363, + "loss": 1.2759, + "theoretical_loss": 3.4684754990666358, + "tokens_seen": 1934163968 + }, + { + "epoch": 0.59, + "learning_rate": 0.000334555667001003, + "loss": 1.2435, + "theoretical_loss": 3.4684657167472546, + "tokens_seen": 1934229504 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003345396188565697, + "loss": 1.2967, + "theoretical_loss": 3.4684559348521162, + "tokens_seen": 1934295040 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033452357071213644, + "loss": 1.2799, + "theoretical_loss": 3.468446153381188, + "tokens_seen": 1934360576 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033450752256770314, + "loss": 1.3757, + "theoretical_loss": 3.4684363723344367, + "tokens_seen": 1934426112 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033449147442326984, + "loss": 1.2435, + "theoretical_loss": 3.4684265917118298, + "tokens_seen": 1934491648 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033447542627883654, + "loss": 1.32, + "theoretical_loss": 3.4684168115133343, + "tokens_seen": 1934557184 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033445937813440324, + "loss": 1.1397, + "theoretical_loss": 3.4684070317389177, + "tokens_seen": 1934622720 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033444332998996994, + "loss": 1.1978, + "theoretical_loss": 3.468397252388547, + "tokens_seen": 1934688256 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033442728184553664, + "loss": 1.2974, + "theoretical_loss": 3.4683874734621902, + "tokens_seen": 1934753792 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033441123370110334, + "loss": 1.2552, + "theoretical_loss": 3.4683776949598135, + "tokens_seen": 1934819328 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033439518555667004, + "loss": 1.3097, + "theoretical_loss": 3.468367916881385, + "tokens_seen": 1934884864 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033437913741223675, + "loss": 1.2482, + "theoretical_loss": 3.4683581392268708, + "tokens_seen": 1934950400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033436308926780345, + "loss": 1.2063, + "theoretical_loss": 3.46834836199624, + "tokens_seen": 1935015936 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033434704112337015, + "loss": 1.3497, + "theoretical_loss": 3.468338585189458, + "tokens_seen": 1935081472 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033433099297893685, + "loss": 1.3699, + "theoretical_loss": 3.468328808806493, + "tokens_seen": 1935147008 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033431494483450355, + "loss": 1.3364, + "theoretical_loss": 3.4683190328473126, + "tokens_seen": 1935212544 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033429889669007025, + "loss": 1.2013, + "theoretical_loss": 3.468309257311883, + "tokens_seen": 1935278080 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033428284854563695, + "loss": 1.4048, + "theoretical_loss": 3.4682994822001727, + "tokens_seen": 1935343616 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033426680040120365, + "loss": 1.2371, + "theoretical_loss": 3.468289707512148, + "tokens_seen": 1935409152 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033425075225677035, + "loss": 1.2907, + "theoretical_loss": 3.4682799332477767, + "tokens_seen": 1935474688 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033423470411233705, + "loss": 1.4143, + "theoretical_loss": 3.4682701594070267, + "tokens_seen": 1935540224 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033421865596790375, + "loss": 1.2476, + "theoretical_loss": 3.468260385989864, + "tokens_seen": 1935605760 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033420260782347045, + "loss": 1.2119, + "theoretical_loss": 3.4682506129962563, + "tokens_seen": 1935671296 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033418655967903715, + "loss": 1.3302, + "theoretical_loss": 3.468240840426171, + "tokens_seen": 1935736832 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033417051153460385, + "loss": 1.2272, + "theoretical_loss": 3.468231068279576, + "tokens_seen": 1935802368 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033415446339017055, + "loss": 1.2881, + "theoretical_loss": 3.4682212965564383, + "tokens_seen": 1935867904 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033413841524573726, + "loss": 1.4079, + "theoretical_loss": 3.4682115252567245, + "tokens_seen": 1935933440 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033412236710130396, + "loss": 1.282, + "theoretical_loss": 3.468201754380403, + "tokens_seen": 1935998976 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033410631895687066, + "loss": 1.2786, + "theoretical_loss": 3.46819198392744, + "tokens_seen": 1936064512 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033409027081243736, + "loss": 1.3104, + "theoretical_loss": 3.468182213897804, + "tokens_seen": 1936130048 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033407422266800406, + "loss": 1.2637, + "theoretical_loss": 3.4681724442914614, + "tokens_seen": 1936195584 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033405817452357076, + "loss": 1.2341, + "theoretical_loss": 3.46816267510838, + "tokens_seen": 1936261120 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033404212637913746, + "loss": 1.3201, + "theoretical_loss": 3.468152906348527, + "tokens_seen": 1936326656 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003340260782347041, + "loss": 1.2412, + "theoretical_loss": 3.46814313801187, + "tokens_seen": 1936392192 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003340100300902708, + "loss": 1.2872, + "theoretical_loss": 3.468133370098375, + "tokens_seen": 1936457728 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003339939819458375, + "loss": 1.3168, + "theoretical_loss": 3.468123602608012, + "tokens_seen": 1936523264 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 649105, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.228218913078308, + "objective/train/theoretical_loss": 3.4681138355407457, + "objective/train/tokens_used": 1957048800, + "theoretical_loss": 3.4681138355407457, + "tokens_seen": 1936588800 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003339779338014042, + "loss": 1.2015, + "theoretical_loss": 3.4681138355407457, + "tokens_seen": 1936588800 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003339618856569709, + "loss": 1.3447, + "theoretical_loss": 3.4681040688965448, + "tokens_seen": 1936654336 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003339458375125376, + "loss": 1.3257, + "theoretical_loss": 3.4680943026753766, + "tokens_seen": 1936719872 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003339297893681043, + "loss": 1.231, + "theoretical_loss": 3.4680845368772077, + "tokens_seen": 1936785408 + }, + { + "epoch": 0.59, + "learning_rate": 0.000333913741223671, + "loss": 1.3998, + "theoretical_loss": 3.468074771502007, + "tokens_seen": 1936850944 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003338976930792377, + "loss": 1.2819, + "theoretical_loss": 3.4680650065497405, + "tokens_seen": 1936916480 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003338816449348044, + "loss": 1.3175, + "theoretical_loss": 3.468055242020376, + "tokens_seen": 1936982016 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003338655967903711, + "loss": 1.3159, + "theoretical_loss": 3.468045477913881, + "tokens_seen": 1937047552 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003338495486459378, + "loss": 1.3315, + "theoretical_loss": 3.468035714230222, + "tokens_seen": 1937113088 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003338335005015045, + "loss": 1.3219, + "theoretical_loss": 3.468025950969368, + "tokens_seen": 1937178624 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003338174523570712, + "loss": 1.262, + "theoretical_loss": 3.468016188131285, + "tokens_seen": 1937244160 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003338014042126379, + "loss": 1.1846, + "theoretical_loss": 3.468006425715941, + "tokens_seen": 1937309696 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003337853560682046, + "loss": 1.1698, + "theoretical_loss": 3.4679966637233033, + "tokens_seen": 1937375232 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003337693079237713, + "loss": 1.2867, + "theoretical_loss": 3.4679869021533394, + "tokens_seen": 1937440768 + }, + { + "epoch": 0.59, + "learning_rate": 0.000333753259779338, + "loss": 1.1951, + "theoretical_loss": 3.467977141006017, + "tokens_seen": 1937506304 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003337372116349047, + "loss": 1.2754, + "theoretical_loss": 3.4679673802813022, + "tokens_seen": 1937571840 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003337211634904714, + "loss": 1.2846, + "theoretical_loss": 3.467957619979164, + "tokens_seen": 1937637376 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003337051153460381, + "loss": 1.3171, + "theoretical_loss": 3.467947860099569, + "tokens_seen": 1937702912 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003336890672016048, + "loss": 1.2839, + "theoretical_loss": 3.4679381006424848, + "tokens_seen": 1937768448 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003336730190571715, + "loss": 1.3033, + "theoretical_loss": 3.467928341607879, + "tokens_seen": 1937833984 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003336569709127382, + "loss": 1.3823, + "theoretical_loss": 3.4679185829957184, + "tokens_seen": 1937899520 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003336409227683049, + "loss": 1.1818, + "theoretical_loss": 3.4679088248059706, + "tokens_seen": 1937965056 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003336248746238716, + "loss": 1.2439, + "theoretical_loss": 3.467899067038604, + "tokens_seen": 1938030592 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003336088264794383, + "loss": 1.2207, + "theoretical_loss": 3.467889309693585, + "tokens_seen": 1938096128 + }, + { + "epoch": 0.59, + "learning_rate": 0.000333592778335005, + "loss": 1.2102, + "theoretical_loss": 3.467879552770881, + "tokens_seen": 1938161664 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003335767301905717, + "loss": 1.3421, + "theoretical_loss": 3.46786979627046, + "tokens_seen": 1938227200 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003335606820461384, + "loss": 1.1983, + "theoretical_loss": 3.4678600401922894, + "tokens_seen": 1938292736 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003335446339017051, + "loss": 1.2339, + "theoretical_loss": 3.467850284536336, + "tokens_seen": 1938358272 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003335285857572718, + "loss": 1.2118, + "theoretical_loss": 3.4678405293025683, + "tokens_seen": 1938423808 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003335125376128385, + "loss": 1.2339, + "theoretical_loss": 3.467830774490953, + "tokens_seen": 1938489344 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003334964894684052, + "loss": 1.1343, + "theoretical_loss": 3.467821020101457, + "tokens_seen": 1938554880 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003334804413239719, + "loss": 1.2675, + "theoretical_loss": 3.4678112661340497, + "tokens_seen": 1938620416 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033346439317953863, + "loss": 1.1051, + "theoretical_loss": 3.4678015125886965, + "tokens_seen": 1938685952 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033344834503510533, + "loss": 1.2451, + "theoretical_loss": 3.467791759465366, + "tokens_seen": 1938751488 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033343229689067203, + "loss": 1.3313, + "theoretical_loss": 3.467782006764026, + "tokens_seen": 1938817024 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033341624874623873, + "loss": 1.1659, + "theoretical_loss": 3.4677722544846428, + "tokens_seen": 1938882560 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033340020060180543, + "loss": 1.276, + "theoretical_loss": 3.4677625026271848, + "tokens_seen": 1938948096 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033338415245737213, + "loss": 1.232, + "theoretical_loss": 3.467752751191619, + "tokens_seen": 1939013632 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033336810431293883, + "loss": 1.4059, + "theoretical_loss": 3.4677430001779124, + "tokens_seen": 1939079168 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033335205616850553, + "loss": 1.1998, + "theoretical_loss": 3.467733249586034, + "tokens_seen": 1939144704 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033333600802407223, + "loss": 1.251, + "theoretical_loss": 3.4677234994159503, + "tokens_seen": 1939210240 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033331995987963893, + "loss": 1.2957, + "theoretical_loss": 3.467713749667629, + "tokens_seen": 1939275776 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033330391173520563, + "loss": 1.336, + "theoretical_loss": 3.467704000341037, + "tokens_seen": 1939341312 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033328786359077233, + "loss": 1.3312, + "theoretical_loss": 3.467694251436143, + "tokens_seen": 1939406848 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033327181544633904, + "loss": 1.4445, + "theoretical_loss": 3.4676845029529133, + "tokens_seen": 1939472384 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033325576730190574, + "loss": 1.3131, + "theoretical_loss": 3.4676747548913163, + "tokens_seen": 1939537920 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033323971915747244, + "loss": 1.2349, + "theoretical_loss": 3.4676650072513193, + "tokens_seen": 1939603456 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033322367101303914, + "loss": 1.3096, + "theoretical_loss": 3.46765526003289, + "tokens_seen": 1939668992 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033320762286860584, + "loss": 1.181, + "theoretical_loss": 3.467645513235995, + "tokens_seen": 1939734528 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033319157472417254, + "loss": 1.305, + "theoretical_loss": 3.4676357668606026, + "tokens_seen": 1939800064 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 649964, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0895507335662842, + "objective/train/theoretical_loss": 3.4676260209066805, + "objective/train/tokens_used": 1960325600, + "theoretical_loss": 3.4676260209066805, + "tokens_seen": 1939865600 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033317552657973924, + "loss": 1.1975, + "theoretical_loss": 3.4676260209066805, + "tokens_seen": 1939865600 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033315947843530594, + "loss": 1.2098, + "theoretical_loss": 3.467616275374196, + "tokens_seen": 1939931136 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033314343029087264, + "loss": 1.3241, + "theoretical_loss": 3.467606530263116, + "tokens_seen": 1939996672 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033312738214643934, + "loss": 1.2988, + "theoretical_loss": 3.4675967855734093, + "tokens_seen": 1940062208 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033311133400200604, + "loss": 1.2053, + "theoretical_loss": 3.467587041305042, + "tokens_seen": 1940127744 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033309528585757274, + "loss": 1.3138, + "theoretical_loss": 3.467577297457983, + "tokens_seen": 1940193280 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033307923771313944, + "loss": 1.2471, + "theoretical_loss": 3.4675675540321995, + "tokens_seen": 1940258816 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033306318956870614, + "loss": 1.3984, + "theoretical_loss": 3.4675578110276586, + "tokens_seen": 1940324352 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033304714142427284, + "loss": 1.2049, + "theoretical_loss": 3.4675480684443274, + "tokens_seen": 1940389888 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033303109327983954, + "loss": 1.2644, + "theoretical_loss": 3.467538326282175, + "tokens_seen": 1940455424 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033301504513540625, + "loss": 1.2876, + "theoretical_loss": 3.467528584541168, + "tokens_seen": 1940520960 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033299899699097295, + "loss": 1.284, + "theoretical_loss": 3.467518843221274, + "tokens_seen": 1940586496 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033298294884653965, + "loss": 1.0794, + "theoretical_loss": 3.4675091023224605, + "tokens_seen": 1940652032 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033296690070210635, + "loss": 1.1738, + "theoretical_loss": 3.4674993618446956, + "tokens_seen": 1940717568 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033295085255767305, + "loss": 1.2098, + "theoretical_loss": 3.467489621787946, + "tokens_seen": 1940783104 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033293480441323975, + "loss": 1.2535, + "theoretical_loss": 3.4674798821521806, + "tokens_seen": 1940848640 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033291875626880645, + "loss": 1.3445, + "theoretical_loss": 3.4674701429373656, + "tokens_seen": 1940914176 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033290270812437315, + "loss": 1.2581, + "theoretical_loss": 3.46746040414347, + "tokens_seen": 1940979712 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033288665997993985, + "loss": 1.3415, + "theoretical_loss": 3.4674506657704596, + "tokens_seen": 1941045248 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033287061183550655, + "loss": 1.2149, + "theoretical_loss": 3.4674409278183034, + "tokens_seen": 1941110784 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033285456369107325, + "loss": 1.3077, + "theoretical_loss": 3.467431190286969, + "tokens_seen": 1941176320 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033283851554663995, + "loss": 1.2136, + "theoretical_loss": 3.4674214531764234, + "tokens_seen": 1941241856 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033282246740220665, + "loss": 1.1988, + "theoretical_loss": 3.467411716486634, + "tokens_seen": 1941307392 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033280641925777335, + "loss": 1.2851, + "theoretical_loss": 3.467401980217569, + "tokens_seen": 1941372928 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033279037111334005, + "loss": 1.2915, + "theoretical_loss": 3.4673922443691962, + "tokens_seen": 1941438464 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033277432296890675, + "loss": 1.2757, + "theoretical_loss": 3.4673825089414825, + "tokens_seen": 1941504000 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033275827482447346, + "loss": 1.1643, + "theoretical_loss": 3.4673727739343962, + "tokens_seen": 1941569536 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033274222668004016, + "loss": 1.2729, + "theoretical_loss": 3.4673630393479047, + "tokens_seen": 1941635072 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033272617853560686, + "loss": 1.2782, + "theoretical_loss": 3.4673533051819754, + "tokens_seen": 1941700608 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033271013039117356, + "loss": 1.1719, + "theoretical_loss": 3.467343571436576, + "tokens_seen": 1941766144 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033269408224674026, + "loss": 1.194, + "theoretical_loss": 3.4673338381116743, + "tokens_seen": 1941831680 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033267803410230696, + "loss": 1.3513, + "theoretical_loss": 3.467324105207238, + "tokens_seen": 1941897216 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033266198595787366, + "loss": 1.2258, + "theoretical_loss": 3.467314372723235, + "tokens_seen": 1941962752 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033264593781344036, + "loss": 1.3266, + "theoretical_loss": 3.4673046406596324, + "tokens_seen": 1942028288 + }, + { + "epoch": 0.59, + "learning_rate": 0.000332629889669007, + "loss": 1.1875, + "theoretical_loss": 3.467294909016398, + "tokens_seen": 1942093824 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003326138415245737, + "loss": 1.195, + "theoretical_loss": 3.4672851777934994, + "tokens_seen": 1942159360 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003325977933801404, + "loss": 1.3459, + "theoretical_loss": 3.467275446990904, + "tokens_seen": 1942224896 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003325817452357071, + "loss": 1.2465, + "theoretical_loss": 3.4672657166085807, + "tokens_seen": 1942290432 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003325656970912738, + "loss": 1.254, + "theoretical_loss": 3.467255986646496, + "tokens_seen": 1942355968 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003325496489468405, + "loss": 1.2533, + "theoretical_loss": 3.467246257104617, + "tokens_seen": 1942421504 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003325336008024072, + "loss": 1.3038, + "theoretical_loss": 3.467236527982913, + "tokens_seen": 1942487040 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003325175526579739, + "loss": 1.2383, + "theoretical_loss": 3.4672267992813515, + "tokens_seen": 1942552576 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003325015045135406, + "loss": 1.3385, + "theoretical_loss": 3.4672170709998986, + "tokens_seen": 1942618112 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003324854563691073, + "loss": 1.1908, + "theoretical_loss": 3.467207343138523, + "tokens_seen": 1942683648 + }, + { + "epoch": 0.59, + "learning_rate": 0.000332469408224674, + "loss": 1.2383, + "theoretical_loss": 3.4671976156971933, + "tokens_seen": 1942749184 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003324533600802407, + "loss": 1.289, + "theoretical_loss": 3.467187888675876, + "tokens_seen": 1942814720 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003324373119358074, + "loss": 1.2757, + "theoretical_loss": 3.4671781620745383, + "tokens_seen": 1942880256 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003324212637913741, + "loss": 1.1641, + "theoretical_loss": 3.4671684358931496, + "tokens_seen": 1942945792 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003324052156469408, + "loss": 1.2558, + "theoretical_loss": 3.4671587101316765, + "tokens_seen": 1943011328 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003323891675025075, + "loss": 1.2844, + "theoretical_loss": 3.4671489847900867, + "tokens_seen": 1943076864 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 651270, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2893718481063843, + "objective/train/theoretical_loss": 3.4671392598683477, + "objective/train/tokens_used": 1963602400, + "theoretical_loss": 3.4671392598683477, + "tokens_seen": 1943142400 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003323731193580742, + "loss": 1.2388, + "theoretical_loss": 3.4671392598683477, + "tokens_seen": 1943142400 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003323570712136409, + "loss": 1.2862, + "theoretical_loss": 3.4671295353664284, + "tokens_seen": 1943207936 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003323410230692076, + "loss": 1.2576, + "theoretical_loss": 3.467119811284295, + "tokens_seen": 1943273472 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003323249749247743, + "loss": 1.2646, + "theoretical_loss": 3.4671100876219167, + "tokens_seen": 1943339008 + }, + { + "epoch": 0.59, + "learning_rate": 0.000332308926780341, + "loss": 1.3252, + "theoretical_loss": 3.4671003643792604, + "tokens_seen": 1943404544 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003322928786359077, + "loss": 1.1911, + "theoretical_loss": 3.4670906415562937, + "tokens_seen": 1943470080 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003322768304914744, + "loss": 1.2, + "theoretical_loss": 3.4670809191529846, + "tokens_seen": 1943535616 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003322607823470411, + "loss": 1.3449, + "theoretical_loss": 3.4670711971693007, + "tokens_seen": 1943601152 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003322447342026078, + "loss": 1.4192, + "theoretical_loss": 3.46706147560521, + "tokens_seen": 1943666688 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003322286860581745, + "loss": 1.2902, + "theoretical_loss": 3.46705175446068, + "tokens_seen": 1943732224 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003322126379137412, + "loss": 1.2336, + "theoretical_loss": 3.4670420337356784, + "tokens_seen": 1943797760 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003321965897693079, + "loss": 1.2207, + "theoretical_loss": 3.467032313430173, + "tokens_seen": 1943863296 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003321805416248746, + "loss": 1.2598, + "theoretical_loss": 3.467022593544132, + "tokens_seen": 1943928832 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003321644934804413, + "loss": 1.2448, + "theoretical_loss": 3.467012874077523, + "tokens_seen": 1943994368 + }, + { + "epoch": 0.59, + "learning_rate": 0.000332148445336008, + "loss": 1.2101, + "theoretical_loss": 3.4670031550303126, + "tokens_seen": 1944059904 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003321323971915747, + "loss": 1.2665, + "theoretical_loss": 3.46699343640247, + "tokens_seen": 1944125440 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003321163490471415, + "loss": 1.3449, + "theoretical_loss": 3.466983718193963, + "tokens_seen": 1944190976 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003321003009027082, + "loss": 1.2873, + "theoretical_loss": 3.4669740004047584, + "tokens_seen": 1944256512 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003320842527582749, + "loss": 1.3207, + "theoretical_loss": 3.466964283034824, + "tokens_seen": 1944322048 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003320682046138416, + "loss": 1.3042, + "theoretical_loss": 3.4669545660841283, + "tokens_seen": 1944387584 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003320521564694083, + "loss": 1.2736, + "theoretical_loss": 3.466944849552639, + "tokens_seen": 1944453120 + }, + { + "epoch": 0.59, + "learning_rate": 0.000332036108324975, + "loss": 1.24, + "theoretical_loss": 3.466935133440324, + "tokens_seen": 1944518656 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003320200601805417, + "loss": 1.302, + "theoretical_loss": 3.4669254177471505, + "tokens_seen": 1944584192 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003320040120361084, + "loss": 1.2728, + "theoretical_loss": 3.466915702473086, + "tokens_seen": 1944649728 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033198796389167503, + "loss": 1.3289, + "theoretical_loss": 3.4669059876181, + "tokens_seen": 1944715264 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033197191574724173, + "loss": 1.2667, + "theoretical_loss": 3.466896273182158, + "tokens_seen": 1944780800 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033195586760280843, + "loss": 1.2668, + "theoretical_loss": 3.4668865591652294, + "tokens_seen": 1944846336 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033193981945837513, + "loss": 1.3277, + "theoretical_loss": 3.466876845567282, + "tokens_seen": 1944911872 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033192377131394183, + "loss": 1.2192, + "theoretical_loss": 3.4668671323882823, + "tokens_seen": 1944977408 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033190772316950853, + "loss": 1.3804, + "theoretical_loss": 3.4668574196282, + "tokens_seen": 1945042944 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033189167502507524, + "loss": 1.353, + "theoretical_loss": 3.466847707287001, + "tokens_seen": 1945108480 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033187562688064194, + "loss": 1.2273, + "theoretical_loss": 3.4668379953646546, + "tokens_seen": 1945174016 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033185957873620864, + "loss": 1.1862, + "theoretical_loss": 3.466828283861128, + "tokens_seen": 1945239552 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033184353059177534, + "loss": 1.2988, + "theoretical_loss": 3.466818572776389, + "tokens_seen": 1945305088 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033182748244734204, + "loss": 1.2282, + "theoretical_loss": 3.4668088621104056, + "tokens_seen": 1945370624 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033181143430290874, + "loss": 1.2951, + "theoretical_loss": 3.4667991518631456, + "tokens_seen": 1945436160 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033179538615847544, + "loss": 1.306, + "theoretical_loss": 3.4667894420345764, + "tokens_seen": 1945501696 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033177933801404214, + "loss": 1.2956, + "theoretical_loss": 3.466779732624667, + "tokens_seen": 1945567232 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033176328986960884, + "loss": 1.3359, + "theoretical_loss": 3.466770023633384, + "tokens_seen": 1945632768 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033174724172517554, + "loss": 1.2447, + "theoretical_loss": 3.4667603150606956, + "tokens_seen": 1945698304 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033173119358074224, + "loss": 1.3038, + "theoretical_loss": 3.4667506069065706, + "tokens_seen": 1945763840 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033171514543630894, + "loss": 1.278, + "theoretical_loss": 3.4667408991709756, + "tokens_seen": 1945829376 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033169909729187564, + "loss": 1.3088, + "theoretical_loss": 3.466731191853879, + "tokens_seen": 1945894912 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033168304914744234, + "loss": 1.288, + "theoretical_loss": 3.466721484955248, + "tokens_seen": 1945960448 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033166700100300904, + "loss": 1.2908, + "theoretical_loss": 3.4667117784750516, + "tokens_seen": 1946025984 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033165095285857574, + "loss": 1.3329, + "theoretical_loss": 3.4667020724132573, + "tokens_seen": 1946091520 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033163490471414245, + "loss": 1.2158, + "theoretical_loss": 3.4666923667698324, + "tokens_seen": 1946157056 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033161885656970915, + "loss": 1.3356, + "theoretical_loss": 3.4666826615447452, + "tokens_seen": 1946222592 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033160280842527585, + "loss": 1.191, + "theoretical_loss": 3.4666729567379635, + "tokens_seen": 1946288128 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033158676028084255, + "loss": 1.2726, + "theoretical_loss": 3.4666632523494556, + "tokens_seen": 1946353664 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 652242, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2615714073181152, + "objective/train/theoretical_loss": 3.466653548379189, + "objective/train/tokens_used": 1966879200, + "theoretical_loss": 3.466653548379189, + "tokens_seen": 1946419200 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033157071213640925, + "loss": 1.3033, + "theoretical_loss": 3.466653548379189, + "tokens_seen": 1946419200 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033155466399197595, + "loss": 1.1922, + "theoretical_loss": 3.4666438448271313, + "tokens_seen": 1946484736 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033153861584754265, + "loss": 1.3145, + "theoretical_loss": 3.466634141693251, + "tokens_seen": 1946550272 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033152256770310935, + "loss": 1.314, + "theoretical_loss": 3.4666244389775156, + "tokens_seen": 1946615808 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033150651955867605, + "loss": 1.1525, + "theoretical_loss": 3.4666147366798934, + "tokens_seen": 1946681344 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033149047141424275, + "loss": 1.2836, + "theoretical_loss": 3.466605034800352, + "tokens_seen": 1946746880 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033147442326980945, + "loss": 1.2922, + "theoretical_loss": 3.466595333338859, + "tokens_seen": 1946812416 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033145837512537615, + "loss": 1.3372, + "theoretical_loss": 3.466585632295383, + "tokens_seen": 1946877952 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033144232698094285, + "loss": 1.2432, + "theoretical_loss": 3.466575931669891, + "tokens_seen": 1946943488 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033142627883650955, + "loss": 1.3169, + "theoretical_loss": 3.4665662314623527, + "tokens_seen": 1947009024 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033141023069207625, + "loss": 1.3301, + "theoretical_loss": 3.466556531672734, + "tokens_seen": 1947074560 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033139418254764296, + "loss": 1.2713, + "theoretical_loss": 3.4665468323010034, + "tokens_seen": 1947140096 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033137813440320966, + "loss": 1.4026, + "theoretical_loss": 3.4665371333471295, + "tokens_seen": 1947205632 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033136208625877636, + "loss": 1.2622, + "theoretical_loss": 3.4665274348110797, + "tokens_seen": 1947271168 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033134603811434306, + "loss": 1.4576, + "theoretical_loss": 3.4665177366928224, + "tokens_seen": 1947336704 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033132998996990976, + "loss": 1.1986, + "theoretical_loss": 3.466508038992325, + "tokens_seen": 1947402240 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033131394182547646, + "loss": 1.3405, + "theoretical_loss": 3.4664983417095554, + "tokens_seen": 1947467776 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033129789368104316, + "loss": 1.2203, + "theoretical_loss": 3.4664886448444823, + "tokens_seen": 1947533312 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033128184553660986, + "loss": 1.222, + "theoretical_loss": 3.4664789483970733, + "tokens_seen": 1947598848 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033126579739217656, + "loss": 1.196, + "theoretical_loss": 3.4664692523672955, + "tokens_seen": 1947664384 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033124974924774326, + "loss": 1.3111, + "theoretical_loss": 3.466459556755118, + "tokens_seen": 1947729920 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033123370110330996, + "loss": 1.2485, + "theoretical_loss": 3.466449861560508, + "tokens_seen": 1947795456 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003312176529588766, + "loss": 1.2839, + "theoretical_loss": 3.466440166783434, + "tokens_seen": 1947860992 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003312016048144433, + "loss": 1.3024, + "theoretical_loss": 3.4664304724238644, + "tokens_seen": 1947926528 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033118555667001, + "loss": 1.1599, + "theoretical_loss": 3.4664207784817656, + "tokens_seen": 1947992064 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003311695085255767, + "loss": 1.3215, + "theoretical_loss": 3.466411084957107, + "tokens_seen": 1948057600 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003311534603811434, + "loss": 1.275, + "theoretical_loss": 3.466401391849856, + "tokens_seen": 1948123136 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003311374122367101, + "loss": 1.2152, + "theoretical_loss": 3.4663916991599812, + "tokens_seen": 1948188672 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003311213640922768, + "loss": 1.2924, + "theoretical_loss": 3.466382006887449, + "tokens_seen": 1948254208 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003311053159478435, + "loss": 1.2992, + "theoretical_loss": 3.4663723150322294, + "tokens_seen": 1948319744 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003310892678034102, + "loss": 1.2681, + "theoretical_loss": 3.4663626235942893, + "tokens_seen": 1948385280 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003310732196589769, + "loss": 1.3791, + "theoretical_loss": 3.4663529325735967, + "tokens_seen": 1948450816 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003310571715145436, + "loss": 1.2621, + "theoretical_loss": 3.46634324197012, + "tokens_seen": 1948516352 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003310411233701103, + "loss": 1.3842, + "theoretical_loss": 3.466333551783827, + "tokens_seen": 1948581888 + }, + { + "epoch": 0.59, + "learning_rate": 0.000331025075225677, + "loss": 1.139, + "theoretical_loss": 3.4663238620146855, + "tokens_seen": 1948647424 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003310090270812437, + "loss": 1.396, + "theoretical_loss": 3.466314172662664, + "tokens_seen": 1948712960 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003309929789368104, + "loss": 1.2506, + "theoretical_loss": 3.46630448372773, + "tokens_seen": 1948778496 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003309769307923771, + "loss": 1.1412, + "theoretical_loss": 3.4662947952098517, + "tokens_seen": 1948844032 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003309608826479438, + "loss": 1.2234, + "theoretical_loss": 3.4662851071089973, + "tokens_seen": 1948909568 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003309448345035105, + "loss": 1.2088, + "theoretical_loss": 3.4662754194251346, + "tokens_seen": 1948975104 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003309287863590773, + "loss": 1.2713, + "theoretical_loss": 3.4662657321582317, + "tokens_seen": 1949040640 + }, + { + "epoch": 0.59, + "learning_rate": 0.000330912738214644, + "loss": 1.3192, + "theoretical_loss": 3.466256045308257, + "tokens_seen": 1949106176 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003308966900702107, + "loss": 1.1845, + "theoretical_loss": 3.466246358875178, + "tokens_seen": 1949171712 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003308806419257774, + "loss": 1.2129, + "theoretical_loss": 3.4662366728589626, + "tokens_seen": 1949237248 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003308645937813441, + "loss": 1.3485, + "theoretical_loss": 3.4662269872595797, + "tokens_seen": 1949302784 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003308485456369108, + "loss": 1.1281, + "theoretical_loss": 3.4662173020769966, + "tokens_seen": 1949368320 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003308324974924775, + "loss": 1.171, + "theoretical_loss": 3.4662076173111815, + "tokens_seen": 1949433856 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003308164493480442, + "loss": 1.3374, + "theoretical_loss": 3.466197932962103, + "tokens_seen": 1949499392 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003308004012036109, + "loss": 1.244, + "theoretical_loss": 3.466188249029728, + "tokens_seen": 1949564928 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003307843530591776, + "loss": 1.2408, + "theoretical_loss": 3.466178565514026, + "tokens_seen": 1949630464 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 653381, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3046982288360596, + "objective/train/theoretical_loss": 3.466168882414964, + "objective/train/tokens_used": 1970156000, + "theoretical_loss": 3.466168882414964, + "tokens_seen": 1949696000 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003307683049147443, + "loss": 1.3013, + "theoretical_loss": 3.466168882414964, + "tokens_seen": 1949696000 + }, + { + "epoch": 0.59, + "learning_rate": 0.000330752256770311, + "loss": 1.2409, + "theoretical_loss": 3.4661591997325107, + "tokens_seen": 1949761536 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003307362086258777, + "loss": 1.2333, + "theoretical_loss": 3.4661495174666332, + "tokens_seen": 1949827072 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003307201604814444, + "loss": 1.246, + "theoretical_loss": 3.4661398356173008, + "tokens_seen": 1949892608 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003307041123370111, + "loss": 1.1652, + "theoretical_loss": 3.466130154184481, + "tokens_seen": 1949958144 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003306880641925778, + "loss": 1.2934, + "theoretical_loss": 3.4661204731681416, + "tokens_seen": 1950023680 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003306720160481445, + "loss": 1.2639, + "theoretical_loss": 3.4661107925682515, + "tokens_seen": 1950089216 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003306559679037112, + "loss": 1.269, + "theoretical_loss": 3.466101112384778, + "tokens_seen": 1950154752 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003306399197592779, + "loss": 1.2779, + "theoretical_loss": 3.4660914326176897, + "tokens_seen": 1950220288 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003306238716148446, + "loss": 1.2395, + "theoretical_loss": 3.466081753266954, + "tokens_seen": 1950285824 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003306078234704113, + "loss": 1.3344, + "theoretical_loss": 3.4660720743325397, + "tokens_seen": 1950351360 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033059177532597793, + "loss": 1.2097, + "theoretical_loss": 3.466062395814415, + "tokens_seen": 1950416896 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033057572718154463, + "loss": 1.1702, + "theoretical_loss": 3.4660527177125475, + "tokens_seen": 1950482432 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033055967903711133, + "loss": 1.3363, + "theoretical_loss": 3.4660430400269058, + "tokens_seen": 1950547968 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033054363089267803, + "loss": 1.218, + "theoretical_loss": 3.4660333627574573, + "tokens_seen": 1950613504 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033052758274824474, + "loss": 1.3456, + "theoretical_loss": 3.4660236859041706, + "tokens_seen": 1950679040 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033051153460381144, + "loss": 1.2671, + "theoretical_loss": 3.466014009467014, + "tokens_seen": 1950744576 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033049548645937814, + "loss": 1.2164, + "theoretical_loss": 3.4660043334459556, + "tokens_seen": 1950810112 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033047943831494484, + "loss": 1.2605, + "theoretical_loss": 3.4659946578409633, + "tokens_seen": 1950875648 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033046339017051154, + "loss": 1.2373, + "theoretical_loss": 3.465984982652005, + "tokens_seen": 1950941184 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033044734202607824, + "loss": 1.3095, + "theoretical_loss": 3.4659753078790487, + "tokens_seen": 1951006720 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033043129388164494, + "loss": 1.2732, + "theoretical_loss": 3.465965633522064, + "tokens_seen": 1951072256 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033041524573721164, + "loss": 1.3532, + "theoretical_loss": 3.4659559595810174, + "tokens_seen": 1951137792 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033039919759277834, + "loss": 1.2832, + "theoretical_loss": 3.4659462860558774, + "tokens_seen": 1951203328 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033038314944834504, + "loss": 1.2847, + "theoretical_loss": 3.4659366129466127, + "tokens_seen": 1951268864 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033036710130391174, + "loss": 1.3175, + "theoretical_loss": 3.465926940253191, + "tokens_seen": 1951334400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033035105315947844, + "loss": 1.2282, + "theoretical_loss": 3.4659172679755805, + "tokens_seen": 1951399936 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033033500501504514, + "loss": 1.353, + "theoretical_loss": 3.46590759611375, + "tokens_seen": 1951465472 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033031895687061184, + "loss": 1.2742, + "theoretical_loss": 3.4658979246676664, + "tokens_seen": 1951531008 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033030290872617854, + "loss": 1.2683, + "theoretical_loss": 3.4658882536372992, + "tokens_seen": 1951596544 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033028686058174524, + "loss": 1.2848, + "theoretical_loss": 3.465878583022616, + "tokens_seen": 1951662080 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033027081243731195, + "loss": 1.4071, + "theoretical_loss": 3.4658689128235842, + "tokens_seen": 1951727616 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033025476429287865, + "loss": 1.2527, + "theoretical_loss": 3.4658592430401733, + "tokens_seen": 1951793152 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033023871614844535, + "loss": 1.1321, + "theoretical_loss": 3.465849573672351, + "tokens_seen": 1951858688 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033022266800401205, + "loss": 1.2115, + "theoretical_loss": 3.465839904720085, + "tokens_seen": 1951924224 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033020661985957875, + "loss": 1.229, + "theoretical_loss": 3.465830236183344, + "tokens_seen": 1951989760 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033019057171514545, + "loss": 1.27, + "theoretical_loss": 3.465820568062096, + "tokens_seen": 1952055296 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033017452357071215, + "loss": 1.2049, + "theoretical_loss": 3.4658109003563093, + "tokens_seen": 1952120832 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033015847542627885, + "loss": 1.3145, + "theoretical_loss": 3.4658012330659522, + "tokens_seen": 1952186368 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033014242728184555, + "loss": 1.2956, + "theoretical_loss": 3.465791566190993, + "tokens_seen": 1952251904 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033012637913741225, + "loss": 1.233, + "theoretical_loss": 3.4657818997313994, + "tokens_seen": 1952317440 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033011033099297895, + "loss": 1.2568, + "theoretical_loss": 3.4657722336871397, + "tokens_seen": 1952382976 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033009428284854565, + "loss": 1.2994, + "theoretical_loss": 3.4657625680581825, + "tokens_seen": 1952448512 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033007823470411235, + "loss": 1.2272, + "theoretical_loss": 3.4657529028444958, + "tokens_seen": 1952514048 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033006218655967905, + "loss": 1.2572, + "theoretical_loss": 3.465743238046048, + "tokens_seen": 1952579584 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033004613841524575, + "loss": 1.2266, + "theoretical_loss": 3.465733573662807, + "tokens_seen": 1952645120 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033003009027081245, + "loss": 1.2673, + "theoretical_loss": 3.4657239096947414, + "tokens_seen": 1952710656 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033001404212637916, + "loss": 1.2322, + "theoretical_loss": 3.465714246141819, + "tokens_seen": 1952776192 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032999799398194586, + "loss": 1.2502, + "theoretical_loss": 3.4657045830040083, + "tokens_seen": 1952841728 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032998194583751256, + "loss": 1.2254, + "theoretical_loss": 3.465694920281277, + "tokens_seen": 1952907264 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 654198, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1338186264038086, + "objective/train/theoretical_loss": 3.4656852579735946, + "objective/train/tokens_used": 1973432800, + "theoretical_loss": 3.4656852579735946, + "tokens_seen": 1952972800 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032996589769307926, + "loss": 1.1054, + "theoretical_loss": 3.4656852579735946, + "tokens_seen": 1952972800 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032994984954864596, + "loss": 1.3011, + "theoretical_loss": 3.4656755960809282, + "tokens_seen": 1953038336 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032993380140421266, + "loss": 1.2378, + "theoretical_loss": 3.465665934603247, + "tokens_seen": 1953103872 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032991775325977936, + "loss": 1.2829, + "theoretical_loss": 3.465656273540518, + "tokens_seen": 1953169408 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032990170511534606, + "loss": 1.2974, + "theoretical_loss": 3.4656466128927104, + "tokens_seen": 1953234944 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032988565697091276, + "loss": 1.3214, + "theoretical_loss": 3.465636952659792, + "tokens_seen": 1953300480 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032986960882647946, + "loss": 1.2414, + "theoretical_loss": 3.4656272928417313, + "tokens_seen": 1953366016 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032985356068204616, + "loss": 1.2052, + "theoretical_loss": 3.4656176334384967, + "tokens_seen": 1953431552 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032983751253761286, + "loss": 1.2891, + "theoretical_loss": 3.465607974450056, + "tokens_seen": 1953497088 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003298214643931795, + "loss": 1.2851, + "theoretical_loss": 3.4655983158763783, + "tokens_seen": 1953562624 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003298054162487462, + "loss": 1.1997, + "theoretical_loss": 3.465588657717431, + "tokens_seen": 1953628160 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003297893681043129, + "loss": 1.2775, + "theoretical_loss": 3.465578999973183, + "tokens_seen": 1953693696 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003297733199598796, + "loss": 1.2187, + "theoretical_loss": 3.465569342643602, + "tokens_seen": 1953759232 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003297572718154463, + "loss": 1.2553, + "theoretical_loss": 3.465559685728657, + "tokens_seen": 1953824768 + }, + { + "epoch": 0.59, + "learning_rate": 0.000329741223671013, + "loss": 1.1714, + "theoretical_loss": 3.4655500292283152, + "tokens_seen": 1953890304 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032972517552657977, + "loss": 1.2089, + "theoretical_loss": 3.465540373142546, + "tokens_seen": 1953955840 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032970912738214647, + "loss": 1.2255, + "theoretical_loss": 3.4655307174713172, + "tokens_seen": 1954021376 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032969307923771317, + "loss": 1.2529, + "theoretical_loss": 3.4655210622145978, + "tokens_seen": 1954086912 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032967703109327987, + "loss": 1.2381, + "theoretical_loss": 3.4655114073723547, + "tokens_seen": 1954152448 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032966098294884657, + "loss": 1.1665, + "theoretical_loss": 3.465501752944557, + "tokens_seen": 1954217984 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032964493480441327, + "loss": 1.267, + "theoretical_loss": 3.4654920989311737, + "tokens_seen": 1954283520 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032962888665997997, + "loss": 1.2521, + "theoretical_loss": 3.4654824453321718, + "tokens_seen": 1954349056 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032961283851554667, + "loss": 1.2265, + "theoretical_loss": 3.46547279214752, + "tokens_seen": 1954414592 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032959679037111337, + "loss": 1.1826, + "theoretical_loss": 3.4654631393771878, + "tokens_seen": 1954480128 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032958074222668007, + "loss": 1.342, + "theoretical_loss": 3.4654534870211418, + "tokens_seen": 1954545664 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003295646940822468, + "loss": 1.1846, + "theoretical_loss": 3.465443835079351, + "tokens_seen": 1954611200 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003295486459378135, + "loss": 1.1963, + "theoretical_loss": 3.4654341835517846, + "tokens_seen": 1954676736 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003295325977933802, + "loss": 1.2572, + "theoretical_loss": 3.46542453243841, + "tokens_seen": 1954742272 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003295165496489469, + "loss": 1.2404, + "theoretical_loss": 3.4654148817391954, + "tokens_seen": 1954807808 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003295005015045136, + "loss": 1.2157, + "theoretical_loss": 3.4654052314541097, + "tokens_seen": 1954873344 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003294844533600803, + "loss": 1.2038, + "theoretical_loss": 3.4653955815831208, + "tokens_seen": 1954938880 + }, + { + "epoch": 0.59, + "learning_rate": 0.000329468405215647, + "loss": 1.4662, + "theoretical_loss": 3.4653859321261975, + "tokens_seen": 1955004416 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003294523570712137, + "loss": 1.25, + "theoretical_loss": 3.465376283083308, + "tokens_seen": 1955069952 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003294363089267804, + "loss": 1.2503, + "theoretical_loss": 3.46536663445442, + "tokens_seen": 1955135488 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003294202607823471, + "loss": 1.2677, + "theoretical_loss": 3.4653569862395033, + "tokens_seen": 1955201024 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003294042126379138, + "loss": 1.3817, + "theoretical_loss": 3.4653473384385247, + "tokens_seen": 1955266560 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003293881644934805, + "loss": 1.3955, + "theoretical_loss": 3.4653376910514533, + "tokens_seen": 1955332096 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003293721163490472, + "loss": 1.3591, + "theoretical_loss": 3.465328044078258, + "tokens_seen": 1955397632 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003293560682046139, + "loss": 1.1738, + "theoretical_loss": 3.465318397518906, + "tokens_seen": 1955463168 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003293400200601806, + "loss": 1.2135, + "theoretical_loss": 3.4653087513733665, + "tokens_seen": 1955528704 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003293239719157473, + "loss": 1.2499, + "theoretical_loss": 3.465299105641608, + "tokens_seen": 1955594240 + }, + { + "epoch": 0.59, + "learning_rate": 0.000329307923771314, + "loss": 1.3249, + "theoretical_loss": 3.465289460323598, + "tokens_seen": 1955659776 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003292918756268807, + "loss": 1.3, + "theoretical_loss": 3.465279815419306, + "tokens_seen": 1955725312 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003292758274824474, + "loss": 1.2575, + "theoretical_loss": 3.465270170928699, + "tokens_seen": 1955790848 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003292597793380141, + "loss": 1.2391, + "theoretical_loss": 3.465260526851747, + "tokens_seen": 1955856384 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003292437311935808, + "loss": 1.2982, + "theoretical_loss": 3.4652508831884177, + "tokens_seen": 1955921920 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003292276830491475, + "loss": 1.3291, + "theoretical_loss": 3.4652412399386794, + "tokens_seen": 1955987456 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003292116349047142, + "loss": 1.3256, + "theoretical_loss": 3.4652315971025, + "tokens_seen": 1956052992 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003291955867602809, + "loss": 1.365, + "theoretical_loss": 3.4652219546798486, + "tokens_seen": 1956118528 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032917953861584753, + "loss": 1.2115, + "theoretical_loss": 3.4652123126706935, + "tokens_seen": 1956184064 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 655460, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.333320140838623, + "objective/train/theoretical_loss": 3.4652026710750032, + "objective/train/tokens_used": 1976709600, + "theoretical_loss": 3.4652026710750032, + "tokens_seen": 1956249600 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032916349047141423, + "loss": 1.4231, + "theoretical_loss": 3.4652026710750032, + "tokens_seen": 1956249600 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032914744232698094, + "loss": 1.2317, + "theoretical_loss": 3.4651930298927462, + "tokens_seen": 1956315136 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032913139418254764, + "loss": 1.1809, + "theoretical_loss": 3.46518338912389, + "tokens_seen": 1956380672 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032911534603811434, + "loss": 1.2871, + "theoretical_loss": 3.4651737487684047, + "tokens_seen": 1956446208 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032909929789368104, + "loss": 1.3129, + "theoretical_loss": 3.465164108826257, + "tokens_seen": 1956511744 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032908324974924774, + "loss": 1.1499, + "theoretical_loss": 3.4651544692974166, + "tokens_seen": 1956577280 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032906720160481444, + "loss": 1.2438, + "theoretical_loss": 3.4651448301818513, + "tokens_seen": 1956642816 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032905115346038114, + "loss": 1.3475, + "theoretical_loss": 3.4651351914795296, + "tokens_seen": 1956708352 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032903510531594784, + "loss": 1.3061, + "theoretical_loss": 3.46512555319042, + "tokens_seen": 1956773888 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032901905717151454, + "loss": 1.2135, + "theoretical_loss": 3.465115915314491, + "tokens_seen": 1956839424 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032900300902708124, + "loss": 1.1992, + "theoretical_loss": 3.465106277851711, + "tokens_seen": 1956904960 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032898696088264794, + "loss": 1.2647, + "theoretical_loss": 3.4650966408020487, + "tokens_seen": 1956970496 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032897091273821464, + "loss": 1.3494, + "theoretical_loss": 3.465087004165472, + "tokens_seen": 1957036032 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032895486459378134, + "loss": 1.3465, + "theoretical_loss": 3.46507736794195, + "tokens_seen": 1957101568 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032893881644934804, + "loss": 1.2072, + "theoretical_loss": 3.465067732131451, + "tokens_seen": 1957167104 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032892276830491474, + "loss": 1.3217, + "theoretical_loss": 3.4650580967339426, + "tokens_seen": 1957232640 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032890672016048145, + "loss": 1.3029, + "theoretical_loss": 3.4650484617493946, + "tokens_seen": 1957298176 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032889067201604815, + "loss": 1.2609, + "theoretical_loss": 3.465038827177775, + "tokens_seen": 1957363712 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032887462387161485, + "loss": 1.3106, + "theoretical_loss": 3.4650291930190518, + "tokens_seen": 1957429248 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032885857572718155, + "loss": 1.2851, + "theoretical_loss": 3.465019559273194, + "tokens_seen": 1957494784 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032884252758274825, + "loss": 1.2054, + "theoretical_loss": 3.46500992594017, + "tokens_seen": 1957560320 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032882647943831495, + "loss": 1.3576, + "theoretical_loss": 3.4650002930199477, + "tokens_seen": 1957625856 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032881043129388165, + "loss": 1.2345, + "theoretical_loss": 3.4649906605124965, + "tokens_seen": 1957691392 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032879438314944835, + "loss": 1.2881, + "theoretical_loss": 3.4649810284177844, + "tokens_seen": 1957756928 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032877833500501505, + "loss": 1.2482, + "theoretical_loss": 3.4649713967357805, + "tokens_seen": 1957822464 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032876228686058175, + "loss": 1.3024, + "theoretical_loss": 3.4649617654664526, + "tokens_seen": 1957888000 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032874623871614845, + "loss": 1.2561, + "theoretical_loss": 3.464952134609769, + "tokens_seen": 1957953536 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032873019057171515, + "loss": 1.242, + "theoretical_loss": 3.4649425041656987, + "tokens_seen": 1958019072 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032871414242728185, + "loss": 1.3327, + "theoretical_loss": 3.4649328741342105, + "tokens_seen": 1958084608 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032869809428284855, + "loss": 1.1571, + "theoretical_loss": 3.464923244515272, + "tokens_seen": 1958150144 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032868204613841525, + "loss": 1.1995, + "theoretical_loss": 3.464913615308853, + "tokens_seen": 1958215680 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032866599799398195, + "loss": 1.2167, + "theoretical_loss": 3.4649039865149205, + "tokens_seen": 1958281216 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032864994984954866, + "loss": 1.3154, + "theoretical_loss": 3.464894358133444, + "tokens_seen": 1958346752 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032863390170511536, + "loss": 1.3238, + "theoretical_loss": 3.4648847301643926, + "tokens_seen": 1958412288 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032861785356068206, + "loss": 1.2233, + "theoretical_loss": 3.464875102607733, + "tokens_seen": 1958477824 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032860180541624876, + "loss": 1.2041, + "theoretical_loss": 3.4648654754634354, + "tokens_seen": 1958543360 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032858575727181546, + "loss": 1.3761, + "theoretical_loss": 3.464855848731468, + "tokens_seen": 1958608896 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032856970912738216, + "loss": 1.2536, + "theoretical_loss": 3.464846222411799, + "tokens_seen": 1958674432 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032855366098294886, + "loss": 1.3121, + "theoretical_loss": 3.464836596504396, + "tokens_seen": 1958739968 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032853761283851556, + "loss": 1.3333, + "theoretical_loss": 3.46482697100923, + "tokens_seen": 1958805504 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032852156469408226, + "loss": 1.2164, + "theoretical_loss": 3.4648173459262672, + "tokens_seen": 1958871040 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032850551654964896, + "loss": 1.3231, + "theoretical_loss": 3.464807721255478, + "tokens_seen": 1958936576 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032848946840521566, + "loss": 1.3195, + "theoretical_loss": 3.4647980969968293, + "tokens_seen": 1959002112 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032847342026078236, + "loss": 1.2719, + "theoretical_loss": 3.464788473150291, + "tokens_seen": 1959067648 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032845737211634906, + "loss": 1.1816, + "theoretical_loss": 3.4647788497158305, + "tokens_seen": 1959133184 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032844132397191576, + "loss": 1.1692, + "theoretical_loss": 3.4647692266934174, + "tokens_seen": 1959198720 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032842527582748246, + "loss": 1.2307, + "theoretical_loss": 3.46475960408302, + "tokens_seen": 1959264256 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032840922768304916, + "loss": 1.2162, + "theoretical_loss": 3.464749981884606, + "tokens_seen": 1959329792 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032839317953861587, + "loss": 1.2723, + "theoretical_loss": 3.4647403600981455, + "tokens_seen": 1959395328 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032837713139418257, + "loss": 1.2184, + "theoretical_loss": 3.464730738723606, + "tokens_seen": 1959460864 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 656810, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5059330463409424, + "objective/train/theoretical_loss": 3.4647211177609565, + "objective/train/tokens_used": 1979986400, + "theoretical_loss": 3.4647211177609565, + "tokens_seen": 1959526400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032836108324974927, + "loss": 1.4133, + "theoretical_loss": 3.4647211177609565, + "tokens_seen": 1959526400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032834503510531597, + "loss": 1.3079, + "theoretical_loss": 3.464711497210165, + "tokens_seen": 1959591936 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032832898696088267, + "loss": 1.2449, + "theoretical_loss": 3.4647018770712013, + "tokens_seen": 1959657472 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032831293881644937, + "loss": 1.291, + "theoretical_loss": 3.4646922573440326, + "tokens_seen": 1959723008 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032829689067201607, + "loss": 1.2494, + "theoretical_loss": 3.464682638028629, + "tokens_seen": 1959788544 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032828084252758277, + "loss": 1.2652, + "theoretical_loss": 3.4646730191249575, + "tokens_seen": 1959854080 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032826479438314947, + "loss": 1.1878, + "theoretical_loss": 3.464663400632988, + "tokens_seen": 1959919616 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032824874623871617, + "loss": 1.3116, + "theoretical_loss": 3.4646537825526877, + "tokens_seen": 1959985152 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032823269809428287, + "loss": 1.1721, + "theoretical_loss": 3.464644164884027, + "tokens_seen": 1960050688 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032821664994984957, + "loss": 1.2462, + "theoretical_loss": 3.4646345476269733, + "tokens_seen": 1960116224 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003282006018054163, + "loss": 1.2995, + "theoretical_loss": 3.464624930781496, + "tokens_seen": 1960181760 + }, + { + "epoch": 0.59, + "learning_rate": 0.000328184553660983, + "loss": 1.3637, + "theoretical_loss": 3.4646153143475633, + "tokens_seen": 1960247296 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003281685055165497, + "loss": 1.2276, + "theoretical_loss": 3.464605698325143, + "tokens_seen": 1960312832 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003281524573721164, + "loss": 1.2547, + "theoretical_loss": 3.4645960827142055, + "tokens_seen": 1960378368 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003281364092276831, + "loss": 1.3351, + "theoretical_loss": 3.464586467514718, + "tokens_seen": 1960443904 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003281203610832498, + "loss": 1.3077, + "theoretical_loss": 3.46457685272665, + "tokens_seen": 1960509440 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003281043129388165, + "loss": 1.156, + "theoretical_loss": 3.4645672383499693, + "tokens_seen": 1960574976 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003280882647943832, + "loss": 1.2096, + "theoretical_loss": 3.4645576243846454, + "tokens_seen": 1960640512 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003280722166499499, + "loss": 1.351, + "theoretical_loss": 3.4645480108306463, + "tokens_seen": 1960706048 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003280561685055166, + "loss": 1.3297, + "theoretical_loss": 3.4645383976879414, + "tokens_seen": 1960771584 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003280401203610833, + "loss": 1.2935, + "theoretical_loss": 3.4645287849564985, + "tokens_seen": 1960837120 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032802407221665, + "loss": 1.3245, + "theoretical_loss": 3.4645191726362867, + "tokens_seen": 1960902656 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003280080240722167, + "loss": 1.3396, + "theoretical_loss": 3.4645095607272745, + "tokens_seen": 1960968192 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003279919759277834, + "loss": 1.2586, + "theoretical_loss": 3.464499949229431, + "tokens_seen": 1961033728 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003279759277833501, + "loss": 1.261, + "theoretical_loss": 3.464490338142724, + "tokens_seen": 1961099264 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003279598796389168, + "loss": 1.3892, + "theoretical_loss": 3.464480727467123, + "tokens_seen": 1961164800 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003279438314944835, + "loss": 1.2508, + "theoretical_loss": 3.4644711172025966, + "tokens_seen": 1961230336 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003279277833500502, + "loss": 1.3271, + "theoretical_loss": 3.464461507349113, + "tokens_seen": 1961295872 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003279117352056169, + "loss": 1.211, + "theoretical_loss": 3.464451897906642, + "tokens_seen": 1961361408 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003278956870611836, + "loss": 1.3761, + "theoretical_loss": 3.4644422888751505, + "tokens_seen": 1961426944 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003278796389167503, + "loss": 1.2552, + "theoretical_loss": 3.4644326802546086, + "tokens_seen": 1961492480 + }, + { + "epoch": 0.59, + "learning_rate": 0.000327863590772317, + "loss": 1.3391, + "theoretical_loss": 3.4644230720449842, + "tokens_seen": 1961558016 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003278475426278837, + "loss": 1.2184, + "theoretical_loss": 3.4644134642462467, + "tokens_seen": 1961623552 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003278314944834504, + "loss": 1.2387, + "theoretical_loss": 3.4644038568583646, + "tokens_seen": 1961689088 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003278154463390171, + "loss": 1.2228, + "theoretical_loss": 3.464394249881306, + "tokens_seen": 1961754624 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003277993981945838, + "loss": 1.2147, + "theoretical_loss": 3.4643846433150403, + "tokens_seen": 1961820160 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032778335005015044, + "loss": 1.2534, + "theoretical_loss": 3.464375037159536, + "tokens_seen": 1961885696 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032776730190571714, + "loss": 1.2739, + "theoretical_loss": 3.4643654314147616, + "tokens_seen": 1961951232 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032775125376128384, + "loss": 1.3218, + "theoretical_loss": 3.464355826080686, + "tokens_seen": 1962016768 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032773520561685054, + "loss": 1.2753, + "theoretical_loss": 3.464346221157278, + "tokens_seen": 1962082304 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032771915747241724, + "loss": 1.2796, + "theoretical_loss": 3.4643366166445064, + "tokens_seen": 1962147840 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032770310932798394, + "loss": 1.2492, + "theoretical_loss": 3.46432701254234, + "tokens_seen": 1962213376 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032768706118355064, + "loss": 1.1715, + "theoretical_loss": 3.4643174088507473, + "tokens_seen": 1962278912 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032767101303911734, + "loss": 1.379, + "theoretical_loss": 3.464307805569697, + "tokens_seen": 1962344448 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032765496489468404, + "loss": 1.1766, + "theoretical_loss": 3.4642982026991573, + "tokens_seen": 1962409984 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032763891675025074, + "loss": 1.2037, + "theoretical_loss": 3.4642886002390982, + "tokens_seen": 1962475520 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032762286860581744, + "loss": 1.2102, + "theoretical_loss": 3.4642789981894877, + "tokens_seen": 1962541056 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032760682046138414, + "loss": 1.3144, + "theoretical_loss": 3.4642693965502946, + "tokens_seen": 1962606592 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032759077231695084, + "loss": 1.3125, + "theoretical_loss": 3.464259795321487, + "tokens_seen": 1962672128 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032757472417251754, + "loss": 1.2605, + "theoretical_loss": 3.4642501945030357, + "tokens_seen": 1962737664 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 657689, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2876102924346924, + "objective/train/theoretical_loss": 3.4642405940949073, + "objective/train/tokens_used": 1983263200, + "theoretical_loss": 3.4642405940949073, + "tokens_seen": 1962803200 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032755867602808424, + "loss": 1.2158, + "theoretical_loss": 3.4642405940949073, + "tokens_seen": 1962803200 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032754262788365094, + "loss": 1.2748, + "theoretical_loss": 3.4642309940970715, + "tokens_seen": 1962868736 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032752657973921765, + "loss": 1.2317, + "theoretical_loss": 3.464221394509497, + "tokens_seen": 1962934272 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032751053159478435, + "loss": 1.1704, + "theoretical_loss": 3.4642117953321523, + "tokens_seen": 1962999808 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032749448345035105, + "loss": 1.3569, + "theoretical_loss": 3.4642021965650067, + "tokens_seen": 1963065344 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032747843530591775, + "loss": 1.2822, + "theoretical_loss": 3.4641925982080286, + "tokens_seen": 1963130880 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032746238716148445, + "loss": 1.2775, + "theoretical_loss": 3.464183000261187, + "tokens_seen": 1963196416 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032744633901705115, + "loss": 1.1282, + "theoretical_loss": 3.46417340272445, + "tokens_seen": 1963261952 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032743029087261785, + "loss": 1.2584, + "theoretical_loss": 3.4641638055977872, + "tokens_seen": 1963327488 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032741424272818455, + "loss": 1.2974, + "theoretical_loss": 3.464154208881167, + "tokens_seen": 1963393024 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032739819458375125, + "loss": 1.2633, + "theoretical_loss": 3.4641446125745587, + "tokens_seen": 1963458560 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032738214643931795, + "loss": 1.1956, + "theoretical_loss": 3.464135016677931, + "tokens_seen": 1963524096 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032736609829488465, + "loss": 1.2721, + "theoretical_loss": 3.4641254211912518, + "tokens_seen": 1963589632 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032735005015045135, + "loss": 1.1859, + "theoretical_loss": 3.46411582611449, + "tokens_seen": 1963655168 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032733400200601805, + "loss": 1.3437, + "theoretical_loss": 3.464106231447616, + "tokens_seen": 1963720704 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003273179538615848, + "loss": 1.2472, + "theoretical_loss": 3.464096637190597, + "tokens_seen": 1963786240 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003273019057171515, + "loss": 1.3096, + "theoretical_loss": 3.4640870433434023, + "tokens_seen": 1963851776 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003272858575727182, + "loss": 1.1336, + "theoretical_loss": 3.464077449906001, + "tokens_seen": 1963917312 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003272698094282849, + "loss": 1.3053, + "theoretical_loss": 3.4640678568783616, + "tokens_seen": 1963982848 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003272537612838516, + "loss": 1.2316, + "theoretical_loss": 3.464058264260453, + "tokens_seen": 1964048384 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003272377131394183, + "loss": 1.0924, + "theoretical_loss": 3.464048672052244, + "tokens_seen": 1964113920 + }, + { + "epoch": 0.6, + "learning_rate": 0.000327221664994985, + "loss": 1.302, + "theoretical_loss": 3.4640390802537038, + "tokens_seen": 1964179456 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003272056168505517, + "loss": 1.3307, + "theoretical_loss": 3.4640294888648, + "tokens_seen": 1964244992 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003271895687061184, + "loss": 1.3096, + "theoretical_loss": 3.4640198978855032, + "tokens_seen": 1964310528 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003271735205616851, + "loss": 1.2609, + "theoretical_loss": 3.4640103073157817, + "tokens_seen": 1964376064 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032715747241725176, + "loss": 1.1945, + "theoretical_loss": 3.4640007171556033, + "tokens_seen": 1964441600 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032714142427281846, + "loss": 1.3418, + "theoretical_loss": 3.4639911274049378, + "tokens_seen": 1964507136 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032712537612838516, + "loss": 1.1271, + "theoretical_loss": 3.463981538063754, + "tokens_seen": 1964572672 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032710932798395186, + "loss": 1.264, + "theoretical_loss": 3.4639719491320204, + "tokens_seen": 1964638208 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032709327983951856, + "loss": 1.3211, + "theoretical_loss": 3.463962360609706, + "tokens_seen": 1964703744 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032707723169508526, + "loss": 1.2012, + "theoretical_loss": 3.4639527724967802, + "tokens_seen": 1964769280 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032706118355065196, + "loss": 1.3207, + "theoretical_loss": 3.463943184793211, + "tokens_seen": 1964834816 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032704513540621866, + "loss": 1.2717, + "theoretical_loss": 3.4639335974989676, + "tokens_seen": 1964900352 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032702908726178537, + "loss": 1.3023, + "theoretical_loss": 3.463924010614019, + "tokens_seen": 1964965888 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032701303911735207, + "loss": 1.1863, + "theoretical_loss": 3.463914424138334, + "tokens_seen": 1965031424 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032699699097291877, + "loss": 1.2445, + "theoretical_loss": 3.4639048380718815, + "tokens_seen": 1965096960 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032698094282848547, + "loss": 1.2871, + "theoretical_loss": 3.4638952524146305, + "tokens_seen": 1965162496 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032696489468405217, + "loss": 1.2847, + "theoretical_loss": 3.46388566716655, + "tokens_seen": 1965228032 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032694884653961887, + "loss": 1.2845, + "theoretical_loss": 3.4638760823276082, + "tokens_seen": 1965293568 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032693279839518557, + "loss": 1.2086, + "theoretical_loss": 3.4638664978977745, + "tokens_seen": 1965359104 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032691675025075227, + "loss": 1.1956, + "theoretical_loss": 3.463856913877018, + "tokens_seen": 1965424640 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032690070210631897, + "loss": 1.2191, + "theoretical_loss": 3.463847330265307, + "tokens_seen": 1965490176 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032688465396188567, + "loss": 1.2504, + "theoretical_loss": 3.4638377470626107, + "tokens_seen": 1965555712 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032686860581745237, + "loss": 1.2687, + "theoretical_loss": 3.4638281642688984, + "tokens_seen": 1965621248 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032685255767301907, + "loss": 1.1754, + "theoretical_loss": 3.463818581884139, + "tokens_seen": 1965686784 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032683650952858577, + "loss": 1.3173, + "theoretical_loss": 3.4638089999083004, + "tokens_seen": 1965752320 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003268204613841525, + "loss": 1.1933, + "theoretical_loss": 3.4637994183413525, + "tokens_seen": 1965817856 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003268044132397192, + "loss": 1.1741, + "theoretical_loss": 3.4637898371832634, + "tokens_seen": 1965883392 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003267883650952859, + "loss": 1.2382, + "theoretical_loss": 3.463780256434003, + "tokens_seen": 1965948928 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003267723169508526, + "loss": 1.309, + "theoretical_loss": 3.4637706760935396, + "tokens_seen": 1966014464 + }, + { + "debugging/Compilability": 0.0, + "debugging/distinct-1-grams": 0.7325822315813355, + "debugging/entropy-1-grams": 5.057375499583508, + "debugging/length": 560.6, + "debugging/num_segments": 10, + "debugging/score": 0.065810269192784, + "debugging/score_std": 0.01045031538172114, + "epoch": 0.6, + "objective/train/docs_used": 658485, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.171479344367981, + "objective/train/theoretical_loss": 3.463761096161843, + "objective/train/tokens_used": 1986540000, + "theoretical_loss": 3.463761096161843, + "tokens_seen": 1966080000 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003267562688064193, + "loss": 1.2288, + "theoretical_loss": 3.463761096161843, + "tokens_seen": 1966080000 + }, + { + "epoch": 0.6, + "learning_rate": 0.000326740220661986, + "loss": 1.2614, + "theoretical_loss": 3.4637515166388804, + "tokens_seen": 1966145536 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003267241725175527, + "loss": 1.1963, + "theoretical_loss": 3.463741937524622, + "tokens_seen": 1966211072 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003267081243731194, + "loss": 1.3209, + "theoretical_loss": 3.463732358819037, + "tokens_seen": 1966276608 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003266920762286861, + "loss": 1.3658, + "theoretical_loss": 3.463722780522094, + "tokens_seen": 1966342144 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003266760280842528, + "loss": 1.2307, + "theoretical_loss": 3.4637132026337607, + "tokens_seen": 1966407680 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003266599799398195, + "loss": 1.2746, + "theoretical_loss": 3.4637036251540083, + "tokens_seen": 1966473216 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003266439317953862, + "loss": 1.2292, + "theoretical_loss": 3.463694048082804, + "tokens_seen": 1966538752 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003266278836509529, + "loss": 1.2924, + "theoretical_loss": 3.463684471420118, + "tokens_seen": 1966604288 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003266118355065196, + "loss": 1.2146, + "theoretical_loss": 3.4636748951659175, + "tokens_seen": 1966669824 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003265957873620863, + "loss": 1.3442, + "theoretical_loss": 3.4636653193201736, + "tokens_seen": 1966735360 + }, + { + "epoch": 0.6, + "learning_rate": 0.000326579739217653, + "loss": 1.2257, + "theoretical_loss": 3.463655743882854, + "tokens_seen": 1966800896 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003265636910732197, + "loss": 1.2411, + "theoretical_loss": 3.463646168853928, + "tokens_seen": 1966866432 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003265476429287864, + "loss": 1.3025, + "theoretical_loss": 3.463636594233364, + "tokens_seen": 1966931968 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003265315947843531, + "loss": 1.197, + "theoretical_loss": 3.463627020021132, + "tokens_seen": 1966997504 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003265155466399198, + "loss": 1.2567, + "theoretical_loss": 3.4636174462172002, + "tokens_seen": 1967063040 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003264994984954865, + "loss": 1.2023, + "theoretical_loss": 3.463607872821538, + "tokens_seen": 1967128576 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003264834503510532, + "loss": 1.3127, + "theoretical_loss": 3.463598299834114, + "tokens_seen": 1967194112 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003264674022066199, + "loss": 1.2774, + "theoretical_loss": 3.4635887272548977, + "tokens_seen": 1967259648 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003264513540621866, + "loss": 1.3666, + "theoretical_loss": 3.463579155083858, + "tokens_seen": 1967325184 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003264353059177533, + "loss": 1.2816, + "theoretical_loss": 3.4635695833209637, + "tokens_seen": 1967390720 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032641925777332, + "loss": 1.3354, + "theoretical_loss": 3.4635600119661833, + "tokens_seen": 1967456256 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003264032096288867, + "loss": 1.3263, + "theoretical_loss": 3.463550441019487, + "tokens_seen": 1967521792 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032638716148445334, + "loss": 1.2991, + "theoretical_loss": 3.463540870480843, + "tokens_seen": 1967587328 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032637111334002004, + "loss": 1.206, + "theoretical_loss": 3.46353130035022, + "tokens_seen": 1967652864 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032635506519558674, + "loss": 1.2202, + "theoretical_loss": 3.463521730627588, + "tokens_seen": 1967718400 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032633901705115344, + "loss": 1.2919, + "theoretical_loss": 3.463512161312915, + "tokens_seen": 1967783936 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032632296890672014, + "loss": 1.1321, + "theoretical_loss": 3.463502592406171, + "tokens_seen": 1967849472 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032630692076228684, + "loss": 1.229, + "theoretical_loss": 3.4634930239073243, + "tokens_seen": 1967915008 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032629087261785354, + "loss": 1.266, + "theoretical_loss": 3.4634834558163443, + "tokens_seen": 1967980544 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032627482447342024, + "loss": 1.2568, + "theoretical_loss": 3.4634738881331995, + "tokens_seen": 1968046080 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032625877632898694, + "loss": 1.1917, + "theoretical_loss": 3.46346432085786, + "tokens_seen": 1968111616 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032624272818455364, + "loss": 1.2718, + "theoretical_loss": 3.463454753990294, + "tokens_seen": 1968177152 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032622668004012034, + "loss": 1.3484, + "theoretical_loss": 3.4634451875304704, + "tokens_seen": 1968242688 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032621063189568704, + "loss": 1.4432, + "theoretical_loss": 3.4634356214783586, + "tokens_seen": 1968308224 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032619458375125374, + "loss": 1.252, + "theoretical_loss": 3.463426055833928, + "tokens_seen": 1968373760 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032617853560682044, + "loss": 1.3883, + "theoretical_loss": 3.463416490597147, + "tokens_seen": 1968439296 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032616248746238715, + "loss": 1.2708, + "theoretical_loss": 3.463406925767985, + "tokens_seen": 1968504832 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032614643931795385, + "loss": 1.214, + "theoretical_loss": 3.463397361346411, + "tokens_seen": 1968570368 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032613039117352055, + "loss": 1.2125, + "theoretical_loss": 3.463387797332394, + "tokens_seen": 1968635904 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003261143430290873, + "loss": 1.2935, + "theoretical_loss": 3.463378233725903, + "tokens_seen": 1968701440 + }, + { + "epoch": 0.6, + "learning_rate": 0.000326098294884654, + "loss": 1.3059, + "theoretical_loss": 3.4633686705269073, + "tokens_seen": 1968766976 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003260822467402207, + "loss": 1.2776, + "theoretical_loss": 3.463359107735376, + "tokens_seen": 1968832512 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003260661985957874, + "loss": 1.3121, + "theoretical_loss": 3.463349545351278, + "tokens_seen": 1968898048 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003260501504513541, + "loss": 1.3264, + "theoretical_loss": 3.463339983374582, + "tokens_seen": 1968963584 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003260341023069208, + "loss": 1.2132, + "theoretical_loss": 3.4633304218052583, + "tokens_seen": 1969029120 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003260180541624875, + "loss": 1.2347, + "theoretical_loss": 3.4633208606432744, + "tokens_seen": 1969094656 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003260020060180542, + "loss": 1.2675, + "theoretical_loss": 3.463311299888601, + "tokens_seen": 1969160192 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003259859578736209, + "loss": 1.248, + "theoretical_loss": 3.4633017395412056, + "tokens_seen": 1969225728 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003259699097291876, + "loss": 1.2427, + "theoretical_loss": 3.4632921796010585, + "tokens_seen": 1969291264 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 659633, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3816819190979004, + "objective/train/theoretical_loss": 3.463282620068128, + "objective/train/tokens_used": 1989816800, + "theoretical_loss": 3.463282620068128, + "tokens_seen": 1969356800 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003259538615847543, + "loss": 1.3355, + "theoretical_loss": 3.463282620068128, + "tokens_seen": 1969356800 + }, + { + "epoch": 0.6, + "learning_rate": 0.000325937813440321, + "loss": 1.24, + "theoretical_loss": 3.463273060942384, + "tokens_seen": 1969422336 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003259217652958877, + "loss": 1.2217, + "theoretical_loss": 3.4632635022237954, + "tokens_seen": 1969487872 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003259057171514544, + "loss": 1.231, + "theoretical_loss": 3.4632539439123304, + "tokens_seen": 1969553408 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003258896690070211, + "loss": 1.3296, + "theoretical_loss": 3.4632443860079594, + "tokens_seen": 1969618944 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003258736208625878, + "loss": 1.2672, + "theoretical_loss": 3.4632348285106502, + "tokens_seen": 1969684480 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003258575727181545, + "loss": 1.2484, + "theoretical_loss": 3.4632252714203733, + "tokens_seen": 1969750016 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003258415245737212, + "loss": 1.3167, + "theoretical_loss": 3.463215714737097, + "tokens_seen": 1969815552 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003258254764292879, + "loss": 1.2029, + "theoretical_loss": 3.4632061584607907, + "tokens_seen": 1969881088 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003258094282848546, + "loss": 1.2591, + "theoretical_loss": 3.463196602591423, + "tokens_seen": 1969946624 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003257933801404213, + "loss": 1.3009, + "theoretical_loss": 3.4631870471289634, + "tokens_seen": 1970012160 + }, + { + "epoch": 0.6, + "learning_rate": 0.000325777331995988, + "loss": 1.3173, + "theoretical_loss": 3.4631774920733815, + "tokens_seen": 1970077696 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003257612838515547, + "loss": 1.1998, + "theoretical_loss": 3.463167937424646, + "tokens_seen": 1970143232 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032574523570712136, + "loss": 1.2499, + "theoretical_loss": 3.4631583831827264, + "tokens_seen": 1970208768 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032572918756268806, + "loss": 1.2767, + "theoretical_loss": 3.463148829347591, + "tokens_seen": 1970274304 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032571313941825476, + "loss": 1.2591, + "theoretical_loss": 3.4631392759192092, + "tokens_seen": 1970339840 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032569709127382146, + "loss": 1.4643, + "theoretical_loss": 3.463129722897551, + "tokens_seen": 1970405376 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032568104312938816, + "loss": 1.1429, + "theoretical_loss": 3.4631201702825853, + "tokens_seen": 1970470912 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032566499498495486, + "loss": 1.3224, + "theoretical_loss": 3.46311061807428, + "tokens_seen": 1970536448 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032564894684052157, + "loss": 1.2521, + "theoretical_loss": 3.463101066272606, + "tokens_seen": 1970601984 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032563289869608827, + "loss": 1.3094, + "theoretical_loss": 3.4630915148775316, + "tokens_seen": 1970667520 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032561685055165497, + "loss": 1.2385, + "theoretical_loss": 3.463081963889026, + "tokens_seen": 1970733056 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032560080240722167, + "loss": 1.3842, + "theoretical_loss": 3.4630724133070583, + "tokens_seen": 1970798592 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032558475426278837, + "loss": 1.2771, + "theoretical_loss": 3.4630628631315976, + "tokens_seen": 1970864128 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032556870611835507, + "loss": 1.3529, + "theoretical_loss": 3.4630533133626136, + "tokens_seen": 1970929664 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032555265797392177, + "loss": 1.2603, + "theoretical_loss": 3.463043764000075, + "tokens_seen": 1970995200 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032553660982948847, + "loss": 1.173, + "theoretical_loss": 3.4630342150439515, + "tokens_seen": 1971060736 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032552056168505517, + "loss": 1.363, + "theoretical_loss": 3.4630246664942117, + "tokens_seen": 1971126272 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032550451354062187, + "loss": 1.2181, + "theoretical_loss": 3.4630151183508247, + "tokens_seen": 1971191808 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032548846539618857, + "loss": 1.2296, + "theoretical_loss": 3.4630055706137606, + "tokens_seen": 1971257344 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032547241725175527, + "loss": 1.1683, + "theoretical_loss": 3.462996023282988, + "tokens_seen": 1971322880 + }, + { + "epoch": 0.6, + "learning_rate": 0.000325456369107322, + "loss": 1.2918, + "theoretical_loss": 3.462986476358476, + "tokens_seen": 1971388416 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003254403209628887, + "loss": 1.2018, + "theoretical_loss": 3.4629769298401936, + "tokens_seen": 1971453952 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003254242728184554, + "loss": 1.2581, + "theoretical_loss": 3.462967383728111, + "tokens_seen": 1971519488 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003254082246740221, + "loss": 1.3155, + "theoretical_loss": 3.4629578380221964, + "tokens_seen": 1971585024 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003253921765295888, + "loss": 1.2152, + "theoretical_loss": 3.4629482927224196, + "tokens_seen": 1971650560 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003253761283851555, + "loss": 1.3117, + "theoretical_loss": 3.4629387478287494, + "tokens_seen": 1971716096 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003253600802407222, + "loss": 1.2997, + "theoretical_loss": 3.4629292033411554, + "tokens_seen": 1971781632 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003253440320962889, + "loss": 1.2661, + "theoretical_loss": 3.4629196592596068, + "tokens_seen": 1971847168 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003253279839518556, + "loss": 1.2099, + "theoretical_loss": 3.4629101155840725, + "tokens_seen": 1971912704 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003253119358074223, + "loss": 1.2358, + "theoretical_loss": 3.462900572314522, + "tokens_seen": 1971978240 + }, + { + "epoch": 0.6, + "learning_rate": 0.000325295887662989, + "loss": 1.2764, + "theoretical_loss": 3.4628910294509243, + "tokens_seen": 1972043776 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003252798395185557, + "loss": 1.3344, + "theoretical_loss": 3.462881486993249, + "tokens_seen": 1972109312 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003252637913741224, + "loss": 1.2796, + "theoretical_loss": 3.4628719449414653, + "tokens_seen": 1972174848 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003252477432296891, + "loss": 1.2834, + "theoretical_loss": 3.462862403295542, + "tokens_seen": 1972240384 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003252316950852558, + "loss": 1.245, + "theoretical_loss": 3.462852862055449, + "tokens_seen": 1972305920 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003252156469408225, + "loss": 1.2679, + "theoretical_loss": 3.462843321221155, + "tokens_seen": 1972371456 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003251995987963892, + "loss": 1.1321, + "theoretical_loss": 3.4628337807926295, + "tokens_seen": 1972436992 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003251835506519559, + "loss": 1.2254, + "theoretical_loss": 3.4628242407698417, + "tokens_seen": 1972502528 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003251675025075226, + "loss": 1.3533, + "theoretical_loss": 3.4628147011527615, + "tokens_seen": 1972568064 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 660847, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.228856086730957, + "objective/train/theoretical_loss": 3.4628051619413567, + "objective/train/tokens_used": 1993093600, + "theoretical_loss": 3.4628051619413567, + "tokens_seen": 1972633600 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003251514543630893, + "loss": 1.2568, + "theoretical_loss": 3.4628051619413567, + "tokens_seen": 1972633600 + }, + { + "epoch": 0.6, + "learning_rate": 0.000325135406218656, + "loss": 1.3084, + "theoretical_loss": 3.462795623135598, + "tokens_seen": 1972699136 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003251193580742227, + "loss": 1.2835, + "theoretical_loss": 3.462786084735454, + "tokens_seen": 1972764672 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003251033099297894, + "loss": 1.223, + "theoretical_loss": 3.462776546740894, + "tokens_seen": 1972830208 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003250872617853561, + "loss": 1.3382, + "theoretical_loss": 3.4627670091518876, + "tokens_seen": 1972895744 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003250712136409228, + "loss": 1.2468, + "theoretical_loss": 3.4627574719684038, + "tokens_seen": 1972961280 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003250551654964895, + "loss": 1.2697, + "theoretical_loss": 3.4627479351904116, + "tokens_seen": 1973026816 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003250391173520562, + "loss": 1.2133, + "theoretical_loss": 3.462738398817881, + "tokens_seen": 1973092352 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003250230692076229, + "loss": 1.2424, + "theoretical_loss": 3.4627288628507813, + "tokens_seen": 1973157888 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003250070210631896, + "loss": 1.2329, + "theoretical_loss": 3.4627193272890806, + "tokens_seen": 1973223424 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003249909729187563, + "loss": 1.2905, + "theoretical_loss": 3.4627097921327494, + "tokens_seen": 1973288960 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032497492477432294, + "loss": 1.221, + "theoretical_loss": 3.4627002573817567, + "tokens_seen": 1973354496 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032495887662988964, + "loss": 1.3335, + "theoretical_loss": 3.462690723036072, + "tokens_seen": 1973420032 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032494282848545634, + "loss": 1.2219, + "theoretical_loss": 3.4626811890956644, + "tokens_seen": 1973485568 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032492678034102304, + "loss": 1.2665, + "theoretical_loss": 3.4626716555605026, + "tokens_seen": 1973551104 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003249107321965898, + "loss": 1.203, + "theoretical_loss": 3.4626621224305567, + "tokens_seen": 1973616640 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003248946840521565, + "loss": 1.3343, + "theoretical_loss": 3.462652589705796, + "tokens_seen": 1973682176 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003248786359077232, + "loss": 1.3397, + "theoretical_loss": 3.4626430573861895, + "tokens_seen": 1973747712 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003248625877632899, + "loss": 1.3358, + "theoretical_loss": 3.462633525471707, + "tokens_seen": 1973813248 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003248465396188566, + "loss": 1.2848, + "theoretical_loss": 3.462623993962317, + "tokens_seen": 1973878784 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003248304914744233, + "loss": 1.3387, + "theoretical_loss": 3.4626144628579896, + "tokens_seen": 1973944320 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032481444332999, + "loss": 1.3364, + "theoretical_loss": 3.462604932158694, + "tokens_seen": 1974009856 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003247983951855567, + "loss": 1.3473, + "theoretical_loss": 3.4625954018643994, + "tokens_seen": 1974075392 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003247823470411234, + "loss": 1.2092, + "theoretical_loss": 3.462585871975075, + "tokens_seen": 1974140928 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003247662988966901, + "loss": 1.323, + "theoretical_loss": 3.4625763424906904, + "tokens_seen": 1974206464 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003247502507522568, + "loss": 1.2732, + "theoretical_loss": 3.4625668134112146, + "tokens_seen": 1974272000 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003247342026078235, + "loss": 1.1487, + "theoretical_loss": 3.4625572847366177, + "tokens_seen": 1974337536 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003247181544633902, + "loss": 1.2068, + "theoretical_loss": 3.4625477564668685, + "tokens_seen": 1974403072 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003247021063189569, + "loss": 1.2197, + "theoretical_loss": 3.462538228601937, + "tokens_seen": 1974468608 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003246860581745236, + "loss": 1.2228, + "theoretical_loss": 3.4625287011417907, + "tokens_seen": 1974534144 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003246700100300903, + "loss": 1.3324, + "theoretical_loss": 3.462519174086401, + "tokens_seen": 1974599680 + }, + { + "epoch": 0.6, + "learning_rate": 0.000324653961885657, + "loss": 1.299, + "theoretical_loss": 3.462509647435737, + "tokens_seen": 1974665216 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003246379137412237, + "loss": 1.2588, + "theoretical_loss": 3.462500121189767, + "tokens_seen": 1974730752 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003246218655967904, + "loss": 1.2224, + "theoretical_loss": 3.462490595348461, + "tokens_seen": 1974796288 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003246058174523571, + "loss": 1.2631, + "theoretical_loss": 3.4624810699117887, + "tokens_seen": 1974861824 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003245897693079238, + "loss": 1.3196, + "theoretical_loss": 3.4624715448797185, + "tokens_seen": 1974927360 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003245737211634905, + "loss": 1.2767, + "theoretical_loss": 3.462462020252221, + "tokens_seen": 1974992896 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003245576730190572, + "loss": 1.2753, + "theoretical_loss": 3.4624524960292655, + "tokens_seen": 1975058432 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003245416248746239, + "loss": 1.424, + "theoretical_loss": 3.46244297221082, + "tokens_seen": 1975123968 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003245255767301906, + "loss": 1.2469, + "theoretical_loss": 3.4624334487968556, + "tokens_seen": 1975189504 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003245095285857573, + "loss": 1.2875, + "theoretical_loss": 3.4624239257873404, + "tokens_seen": 1975255040 + }, + { + "epoch": 0.6, + "learning_rate": 0.000324493480441324, + "loss": 1.291, + "theoretical_loss": 3.462414403182245, + "tokens_seen": 1975320576 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003244774322968907, + "loss": 1.3391, + "theoretical_loss": 3.4624048809815378, + "tokens_seen": 1975386112 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003244613841524574, + "loss": 1.2151, + "theoretical_loss": 3.462395359185188, + "tokens_seen": 1975451648 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003244453360080241, + "loss": 1.3123, + "theoretical_loss": 3.4623858377931667, + "tokens_seen": 1975517184 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003244292878635908, + "loss": 1.2665, + "theoretical_loss": 3.4623763168054413, + "tokens_seen": 1975582720 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003244132397191575, + "loss": 1.2887, + "theoretical_loss": 3.462366796221983, + "tokens_seen": 1975648256 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003243971915747242, + "loss": 1.2273, + "theoretical_loss": 3.4623572760427592, + "tokens_seen": 1975713792 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003243811434302909, + "loss": 1.1689, + "theoretical_loss": 3.462347756267741, + "tokens_seen": 1975779328 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003243650952858576, + "loss": 1.193, + "theoretical_loss": 3.4623382368968976, + "tokens_seen": 1975844864 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 661744, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3551071882247925, + "objective/train/theoretical_loss": 3.4623287179301983, + "objective/train/tokens_used": 1996370400, + "theoretical_loss": 3.4623287179301983, + "tokens_seen": 1975910400 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032434904714142426, + "loss": 1.3324, + "theoretical_loss": 3.4623287179301983, + "tokens_seen": 1975910400 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032433299899699096, + "loss": 1.2214, + "theoretical_loss": 3.462319199367612, + "tokens_seen": 1975975936 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032431695085255766, + "loss": 1.227, + "theoretical_loss": 3.462309681209109, + "tokens_seen": 1976041472 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032430090270812436, + "loss": 1.2567, + "theoretical_loss": 3.4623001634546577, + "tokens_seen": 1976107008 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032428485456369107, + "loss": 1.2492, + "theoretical_loss": 3.462290646104228, + "tokens_seen": 1976172544 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032426880641925777, + "loss": 1.3693, + "theoretical_loss": 3.46228112915779, + "tokens_seen": 1976238080 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032425275827482447, + "loss": 1.268, + "theoretical_loss": 3.4622716126153126, + "tokens_seen": 1976303616 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032423671013039117, + "loss": 1.2839, + "theoretical_loss": 3.462262096476765, + "tokens_seen": 1976369152 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032422066198595787, + "loss": 1.1754, + "theoretical_loss": 3.462252580742117, + "tokens_seen": 1976434688 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032420461384152457, + "loss": 1.2756, + "theoretical_loss": 3.462243065411338, + "tokens_seen": 1976500224 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032418856569709127, + "loss": 1.3355, + "theoretical_loss": 3.462233550484398, + "tokens_seen": 1976565760 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032417251755265797, + "loss": 1.3453, + "theoretical_loss": 3.462224035961265, + "tokens_seen": 1976631296 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032415646940822467, + "loss": 1.2701, + "theoretical_loss": 3.4622145218419105, + "tokens_seen": 1976696832 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032414042126379137, + "loss": 1.3588, + "theoretical_loss": 3.4622050081263023, + "tokens_seen": 1976762368 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032412437311935807, + "loss": 1.3484, + "theoretical_loss": 3.4621954948144107, + "tokens_seen": 1976827904 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032410832497492477, + "loss": 1.1935, + "theoretical_loss": 3.462185981906205, + "tokens_seen": 1976893440 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032409227683049147, + "loss": 1.2972, + "theoretical_loss": 3.4621764694016544, + "tokens_seen": 1976958976 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003240762286860582, + "loss": 1.2619, + "theoretical_loss": 3.4621669573007288, + "tokens_seen": 1977024512 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003240601805416249, + "loss": 1.2888, + "theoretical_loss": 3.462157445603397, + "tokens_seen": 1977090048 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003240441323971916, + "loss": 1.31, + "theoretical_loss": 3.4621479343096304, + "tokens_seen": 1977155584 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003240280842527583, + "loss": 1.3589, + "theoretical_loss": 3.462138423419396, + "tokens_seen": 1977221120 + }, + { + "epoch": 0.6, + "learning_rate": 0.000324012036108325, + "loss": 1.2796, + "theoretical_loss": 3.462128912932665, + "tokens_seen": 1977286656 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003239959879638917, + "loss": 1.252, + "theoretical_loss": 3.4621194028494062, + "tokens_seen": 1977352192 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003239799398194584, + "loss": 1.3325, + "theoretical_loss": 3.462109893169589, + "tokens_seen": 1977417728 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003239638916750251, + "loss": 1.2834, + "theoretical_loss": 3.4621003838931834, + "tokens_seen": 1977483264 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003239478435305918, + "loss": 1.1203, + "theoretical_loss": 3.462090875020159, + "tokens_seen": 1977548800 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003239317953861585, + "loss": 1.2622, + "theoretical_loss": 3.4620813665504846, + "tokens_seen": 1977614336 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003239157472417252, + "loss": 1.2962, + "theoretical_loss": 3.462071858484131, + "tokens_seen": 1977679872 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003238996990972919, + "loss": 1.2436, + "theoretical_loss": 3.462062350821066, + "tokens_seen": 1977745408 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003238836509528586, + "loss": 1.2434, + "theoretical_loss": 3.46205284356126, + "tokens_seen": 1977810944 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003238676028084253, + "loss": 1.1758, + "theoretical_loss": 3.4620433367046832, + "tokens_seen": 1977876480 + }, + { + "epoch": 0.6, + "learning_rate": 0.000323851554663992, + "loss": 1.349, + "theoretical_loss": 3.462033830251304, + "tokens_seen": 1977942016 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003238355065195587, + "loss": 1.2886, + "theoretical_loss": 3.462024324201092, + "tokens_seen": 1978007552 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003238194583751254, + "loss": 1.2048, + "theoretical_loss": 3.4620148185540183, + "tokens_seen": 1978073088 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003238034102306921, + "loss": 1.3347, + "theoretical_loss": 3.4620053133100503, + "tokens_seen": 1978138624 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003237873620862588, + "loss": 1.3814, + "theoretical_loss": 3.461995808469159, + "tokens_seen": 1978204160 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003237713139418255, + "loss": 1.3033, + "theoretical_loss": 3.461986304031314, + "tokens_seen": 1978269696 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003237552657973922, + "loss": 1.2435, + "theoretical_loss": 3.461976799996484, + "tokens_seen": 1978335232 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003237392176529589, + "loss": 1.2223, + "theoretical_loss": 3.461967296364639, + "tokens_seen": 1978400768 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003237231695085256, + "loss": 1.3353, + "theoretical_loss": 3.461957793135748, + "tokens_seen": 1978466304 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003237071213640923, + "loss": 1.2502, + "theoretical_loss": 3.4619482903097816, + "tokens_seen": 1978531840 + }, + { + "epoch": 0.6, + "learning_rate": 0.000323691073219659, + "loss": 1.2918, + "theoretical_loss": 3.4619387878867087, + "tokens_seen": 1978597376 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003236750250752257, + "loss": 1.3095, + "theoretical_loss": 3.461929285866499, + "tokens_seen": 1978662912 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003236589769307924, + "loss": 1.2574, + "theoretical_loss": 3.4619197842491225, + "tokens_seen": 1978728448 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003236429287863591, + "loss": 1.2511, + "theoretical_loss": 3.461910283034548, + "tokens_seen": 1978793984 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003236268806419258, + "loss": 1.2603, + "theoretical_loss": 3.461900782222745, + "tokens_seen": 1978859520 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003236108324974925, + "loss": 1.2968, + "theoretical_loss": 3.461891281813685, + "tokens_seen": 1978925056 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003235947843530592, + "loss": 1.3442, + "theoretical_loss": 3.4618817818073344, + "tokens_seen": 1978990592 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003235787362086259, + "loss": 1.3922, + "theoretical_loss": 3.461872282203666, + "tokens_seen": 1979056128 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003235626880641926, + "loss": 1.1998, + "theoretical_loss": 3.461862783002647, + "tokens_seen": 1979121664 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 662851, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.253308892250061, + "objective/train/theoretical_loss": 3.461853284204248, + "objective/train/tokens_used": 1999647200, + "theoretical_loss": 3.461853284204248, + "tokens_seen": 1979187200 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003235466399197593, + "loss": 1.2605, + "theoretical_loss": 3.461853284204248, + "tokens_seen": 1979187200 + }, + { + "epoch": 0.6, + "learning_rate": 0.000323530591775326, + "loss": 1.3037, + "theoretical_loss": 3.461843785808439, + "tokens_seen": 1979252736 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003235145436308927, + "loss": 1.3261, + "theoretical_loss": 3.461834287815189, + "tokens_seen": 1979318272 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003234984954864594, + "loss": 1.3114, + "theoretical_loss": 3.461824790224468, + "tokens_seen": 1979383808 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003234824473420261, + "loss": 1.3257, + "theoretical_loss": 3.4618152930362447, + "tokens_seen": 1979449344 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003234663991975928, + "loss": 1.2895, + "theoretical_loss": 3.46180579625049, + "tokens_seen": 1979514880 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003234503510531595, + "loss": 1.3307, + "theoretical_loss": 3.4617962998671725, + "tokens_seen": 1979580416 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003234343029087262, + "loss": 1.3148, + "theoretical_loss": 3.461786803886263, + "tokens_seen": 1979645952 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003234182547642929, + "loss": 1.2103, + "theoretical_loss": 3.4617773083077297, + "tokens_seen": 1979711488 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003234022066198596, + "loss": 1.1496, + "theoretical_loss": 3.461767813131543, + "tokens_seen": 1979777024 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003233861584754263, + "loss": 1.3028, + "theoretical_loss": 3.4617583183576723, + "tokens_seen": 1979842560 + }, + { + "epoch": 0.6, + "learning_rate": 0.000323370110330993, + "loss": 1.2057, + "theoretical_loss": 3.4617488239860874, + "tokens_seen": 1979908096 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003233540621865597, + "loss": 1.2974, + "theoretical_loss": 3.4617393300167585, + "tokens_seen": 1979973632 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003233380140421264, + "loss": 1.1871, + "theoretical_loss": 3.4617298364496536, + "tokens_seen": 1980039168 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003233219658976931, + "loss": 1.1707, + "theoretical_loss": 3.4617203432847443, + "tokens_seen": 1980104704 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003233059177532598, + "loss": 1.2787, + "theoretical_loss": 3.461710850521999, + "tokens_seen": 1980170240 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003232898696088265, + "loss": 1.3066, + "theoretical_loss": 3.4617013581613882, + "tokens_seen": 1980235776 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003232738214643932, + "loss": 1.2491, + "theoretical_loss": 3.46169186620288, + "tokens_seen": 1980301312 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003232577733199599, + "loss": 1.2275, + "theoretical_loss": 3.461682374646446, + "tokens_seen": 1980366848 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003232417251755266, + "loss": 1.3797, + "theoretical_loss": 3.4616728834920547, + "tokens_seen": 1980432384 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003232256770310933, + "loss": 1.2025, + "theoretical_loss": 3.461663392739676, + "tokens_seen": 1980497920 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032320962888666, + "loss": 1.2285, + "theoretical_loss": 3.46165390238928, + "tokens_seen": 1980563456 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003231935807422267, + "loss": 1.2729, + "theoretical_loss": 3.461644412440836, + "tokens_seen": 1980628992 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003231775325977934, + "loss": 1.2762, + "theoretical_loss": 3.461634922894313, + "tokens_seen": 1980694528 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003231614844533601, + "loss": 1.1996, + "theoretical_loss": 3.4616254337496812, + "tokens_seen": 1980760064 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003231454363089268, + "loss": 1.1993, + "theoretical_loss": 3.461615945006911, + "tokens_seen": 1980825600 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003231293881644935, + "loss": 1.2744, + "theoretical_loss": 3.4616064566659714, + "tokens_seen": 1980891136 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003231133400200602, + "loss": 1.2419, + "theoretical_loss": 3.4615969687268326, + "tokens_seen": 1980956672 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003230972918756269, + "loss": 1.194, + "theoretical_loss": 3.461587481189464, + "tokens_seen": 1981022208 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003230812437311936, + "loss": 1.2668, + "theoretical_loss": 3.4615779940538345, + "tokens_seen": 1981087744 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003230651955867603, + "loss": 1.2812, + "theoretical_loss": 3.4615685073199147, + "tokens_seen": 1981153280 + }, + { + "epoch": 0.6, + "learning_rate": 0.000323049147442327, + "loss": 1.2192, + "theoretical_loss": 3.461559020987674, + "tokens_seen": 1981218816 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003230330992978937, + "loss": 1.346, + "theoretical_loss": 3.461549535057083, + "tokens_seen": 1981284352 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003230170511534604, + "loss": 1.297, + "theoretical_loss": 3.46154004952811, + "tokens_seen": 1981349888 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003230010030090271, + "loss": 1.282, + "theoretical_loss": 3.461530564400725, + "tokens_seen": 1981415424 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003229849548645938, + "loss": 1.3516, + "theoretical_loss": 3.4615210796748985, + "tokens_seen": 1981480960 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003229689067201605, + "loss": 1.2885, + "theoretical_loss": 3.4615115953505997, + "tokens_seen": 1981546496 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032295285857572716, + "loss": 1.3675, + "theoretical_loss": 3.4615021114277984, + "tokens_seen": 1981612032 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032293681043129386, + "loss": 1.203, + "theoretical_loss": 3.4614926279064644, + "tokens_seen": 1981677568 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032292076228686056, + "loss": 1.4016, + "theoretical_loss": 3.461483144786567, + "tokens_seen": 1981743104 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032290471414242727, + "loss": 1.2464, + "theoretical_loss": 3.4614736620680766, + "tokens_seen": 1981808640 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032288866599799397, + "loss": 1.3215, + "theoretical_loss": 3.4614641797509624, + "tokens_seen": 1981874176 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032287261785356067, + "loss": 1.2891, + "theoretical_loss": 3.4614546978351948, + "tokens_seen": 1981939712 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032285656970912737, + "loss": 1.2287, + "theoretical_loss": 3.4614452163207425, + "tokens_seen": 1982005248 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032284052156469407, + "loss": 1.3051, + "theoretical_loss": 3.4614357352075764, + "tokens_seen": 1982070784 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032282447342026077, + "loss": 1.2015, + "theoretical_loss": 3.4614262544956653, + "tokens_seen": 1982136320 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032280842527582747, + "loss": 1.2178, + "theoretical_loss": 3.461416774184979, + "tokens_seen": 1982201856 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032279237713139417, + "loss": 1.2741, + "theoretical_loss": 3.4614072942754883, + "tokens_seen": 1982267392 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032277632898696087, + "loss": 1.2974, + "theoretical_loss": 3.461397814767162, + "tokens_seen": 1982332928 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032276028084252757, + "loss": 1.3706, + "theoretical_loss": 3.4613883356599704, + "tokens_seen": 1982398464 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 663739, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3018696308135986, + "objective/train/theoretical_loss": 3.461378856953883, + "objective/train/tokens_used": 2002924000, + "theoretical_loss": 3.461378856953883, + "tokens_seen": 1982464000 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032274423269809427, + "loss": 1.3068, + "theoretical_loss": 3.461378856953883, + "tokens_seen": 1982464000 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032272818455366097, + "loss": 1.1333, + "theoretical_loss": 3.461369378648869, + "tokens_seen": 1982529536 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003227121364092277, + "loss": 1.2512, + "theoretical_loss": 3.461359900744899, + "tokens_seen": 1982595072 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003226960882647944, + "loss": 1.3132, + "theoretical_loss": 3.4613504232419423, + "tokens_seen": 1982660608 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003226800401203611, + "loss": 1.2537, + "theoretical_loss": 3.461340946139969, + "tokens_seen": 1982726144 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003226639919759278, + "loss": 1.3256, + "theoretical_loss": 3.461331469438949, + "tokens_seen": 1982791680 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003226479438314945, + "loss": 1.298, + "theoretical_loss": 3.461321993138852, + "tokens_seen": 1982857216 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003226318956870612, + "loss": 1.2358, + "theoretical_loss": 3.461312517239647, + "tokens_seen": 1982922752 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003226158475426279, + "loss": 1.273, + "theoretical_loss": 3.4613030417413047, + "tokens_seen": 1982988288 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003225997993981946, + "loss": 1.2748, + "theoretical_loss": 3.4612935666437945, + "tokens_seen": 1983053824 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003225837512537613, + "loss": 1.259, + "theoretical_loss": 3.4612840919470864, + "tokens_seen": 1983119360 + }, + { + "epoch": 0.6, + "learning_rate": 0.000322567703109328, + "loss": 1.2519, + "theoretical_loss": 3.46127461765115, + "tokens_seen": 1983184896 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003225516549648947, + "loss": 1.2249, + "theoretical_loss": 3.461265143755956, + "tokens_seen": 1983250432 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003225356068204614, + "loss": 1.2658, + "theoretical_loss": 3.4612556702614725, + "tokens_seen": 1983315968 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032251955867602813, + "loss": 1.2707, + "theoretical_loss": 3.4612461971676707, + "tokens_seen": 1983381504 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032250351053159484, + "loss": 1.3275, + "theoretical_loss": 3.46123672447452, + "tokens_seen": 1983447040 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032248746238716154, + "loss": 1.3517, + "theoretical_loss": 3.46122725218199, + "tokens_seen": 1983512576 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032247141424272824, + "loss": 1.201, + "theoretical_loss": 3.461217780290051, + "tokens_seen": 1983578112 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032245536609829494, + "loss": 1.3523, + "theoretical_loss": 3.461208308798672, + "tokens_seen": 1983643648 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032243931795386164, + "loss": 1.2982, + "theoretical_loss": 3.4611988377078236, + "tokens_seen": 1983709184 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032242326980942834, + "loss": 1.2372, + "theoretical_loss": 3.4611893670174756, + "tokens_seen": 1983774720 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032240722166499504, + "loss": 1.1743, + "theoretical_loss": 3.4611798967275975, + "tokens_seen": 1983840256 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032239117352056174, + "loss": 1.1264, + "theoretical_loss": 3.4611704268381587, + "tokens_seen": 1983905792 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032237512537612844, + "loss": 1.2904, + "theoretical_loss": 3.46116095734913, + "tokens_seen": 1983971328 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032235907723169514, + "loss": 1.3452, + "theoretical_loss": 3.461151488260481, + "tokens_seen": 1984036864 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032234302908726184, + "loss": 1.2573, + "theoretical_loss": 3.461142019572182, + "tokens_seen": 1984102400 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032232698094282854, + "loss": 1.2881, + "theoretical_loss": 3.461132551284201, + "tokens_seen": 1984167936 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003223109327983952, + "loss": 1.2507, + "theoretical_loss": 3.46112308339651, + "tokens_seen": 1984233472 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003222948846539619, + "loss": 1.3358, + "theoretical_loss": 3.461113615909077, + "tokens_seen": 1984299008 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003222788365095286, + "loss": 1.2354, + "theoretical_loss": 3.4611041488218737, + "tokens_seen": 1984364544 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003222627883650953, + "loss": 1.3219, + "theoretical_loss": 3.4610946821348687, + "tokens_seen": 1984430080 + }, + { + "epoch": 0.6, + "learning_rate": 0.000322246740220662, + "loss": 1.2491, + "theoretical_loss": 3.461085215848032, + "tokens_seen": 1984495616 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003222306920762287, + "loss": 1.3243, + "theoretical_loss": 3.461075749961334, + "tokens_seen": 1984561152 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003222146439317954, + "loss": 1.3347, + "theoretical_loss": 3.4610662844747444, + "tokens_seen": 1984626688 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003221985957873621, + "loss": 1.3372, + "theoretical_loss": 3.461056819388233, + "tokens_seen": 1984692224 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003221825476429288, + "loss": 1.2272, + "theoretical_loss": 3.461047354701769, + "tokens_seen": 1984757760 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003221664994984955, + "loss": 1.2722, + "theoretical_loss": 3.461037890415324, + "tokens_seen": 1984823296 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003221504513540622, + "loss": 1.2487, + "theoretical_loss": 3.4610284265288653, + "tokens_seen": 1984888832 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003221344032096289, + "loss": 1.2598, + "theoretical_loss": 3.4610189630423656, + "tokens_seen": 1984954368 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003221183550651956, + "loss": 1.3557, + "theoretical_loss": 3.461009499955793, + "tokens_seen": 1985019904 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003221023069207623, + "loss": 1.32, + "theoretical_loss": 3.461000037269118, + "tokens_seen": 1985085440 + }, + { + "epoch": 0.6, + "learning_rate": 0.000322086258776329, + "loss": 1.3916, + "theoretical_loss": 3.4609905749823104, + "tokens_seen": 1985150976 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003220702106318957, + "loss": 1.3763, + "theoretical_loss": 3.4609811130953396, + "tokens_seen": 1985216512 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003220541624874624, + "loss": 1.1434, + "theoretical_loss": 3.4609716516081765, + "tokens_seen": 1985282048 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003220381143430291, + "loss": 1.2956, + "theoretical_loss": 3.4609621905207906, + "tokens_seen": 1985347584 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003220220661985958, + "loss": 1.1931, + "theoretical_loss": 3.460952729833151, + "tokens_seen": 1985413120 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003220060180541625, + "loss": 1.3251, + "theoretical_loss": 3.460943269545229, + "tokens_seen": 1985478656 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003219899699097292, + "loss": 1.3677, + "theoretical_loss": 3.4609338096569937, + "tokens_seen": 1985544192 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003219739217652959, + "loss": 1.3312, + "theoretical_loss": 3.4609243501684155, + "tokens_seen": 1985609728 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003219578736208626, + "loss": 1.2691, + "theoretical_loss": 3.4609148910794634, + "tokens_seen": 1985675264 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 664892, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1819332838058472, + "objective/train/theoretical_loss": 3.460905432390108, + "objective/train/tokens_used": 2006200800, + "theoretical_loss": 3.460905432390108, + "tokens_seen": 1985740800 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003219418254764293, + "loss": 1.2115, + "theoretical_loss": 3.460905432390108, + "tokens_seen": 1985740800 + }, + { + "epoch": 0.6, + "learning_rate": 0.000321925777331996, + "loss": 1.2688, + "theoretical_loss": 3.4608959741003193, + "tokens_seen": 1985806336 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003219097291875627, + "loss": 1.282, + "theoretical_loss": 3.460886516210067, + "tokens_seen": 1985871872 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003218936810431294, + "loss": 1.171, + "theoretical_loss": 3.460877058719322, + "tokens_seen": 1985937408 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003218776328986961, + "loss": 1.3415, + "theoretical_loss": 3.460867601628052, + "tokens_seen": 1986002944 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003218615847542628, + "loss": 1.333, + "theoretical_loss": 3.4608581449362292, + "tokens_seen": 1986068480 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003218455366098295, + "loss": 1.2052, + "theoretical_loss": 3.4608486886438223, + "tokens_seen": 1986134016 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003218294884653962, + "loss": 1.1502, + "theoretical_loss": 3.4608392327508017, + "tokens_seen": 1986199552 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003218134403209629, + "loss": 1.1916, + "theoretical_loss": 3.4608297772571373, + "tokens_seen": 1986265088 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003217973921765296, + "loss": 1.1894, + "theoretical_loss": 3.4608203221627987, + "tokens_seen": 1986330624 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003217813440320963, + "loss": 1.2802, + "theoretical_loss": 3.4608108674677567, + "tokens_seen": 1986396160 + }, + { + "epoch": 0.6, + "learning_rate": 0.000321765295887663, + "loss": 1.3686, + "theoretical_loss": 3.4608014131719806, + "tokens_seen": 1986461696 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003217492477432297, + "loss": 1.4064, + "theoretical_loss": 3.460791959275441, + "tokens_seen": 1986527232 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003217331995987964, + "loss": 1.3098, + "theoretical_loss": 3.4607825057781065, + "tokens_seen": 1986592768 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003217171514543631, + "loss": 1.2966, + "theoretical_loss": 3.460773052679948, + "tokens_seen": 1986658304 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003217011033099298, + "loss": 1.1314, + "theoretical_loss": 3.460763599980936, + "tokens_seen": 1986723840 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003216850551654965, + "loss": 1.1984, + "theoretical_loss": 3.4607541476810395, + "tokens_seen": 1986789376 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003216690070210632, + "loss": 1.2855, + "theoretical_loss": 3.460744695780229, + "tokens_seen": 1986854912 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003216529588766299, + "loss": 1.2458, + "theoretical_loss": 3.460735244278475, + "tokens_seen": 1986920448 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003216369107321966, + "loss": 1.3415, + "theoretical_loss": 3.4607257931757465, + "tokens_seen": 1986985984 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003216208625877633, + "loss": 1.1949, + "theoretical_loss": 3.4607163424720135, + "tokens_seen": 1987051520 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032160481444333, + "loss": 1.2702, + "theoretical_loss": 3.460706892167247, + "tokens_seen": 1987117056 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003215887662988967, + "loss": 1.2309, + "theoretical_loss": 3.4606974422614156, + "tokens_seen": 1987182592 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003215727181544634, + "loss": 1.3673, + "theoretical_loss": 3.4606879927544907, + "tokens_seen": 1987248128 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003215566700100301, + "loss": 1.3188, + "theoretical_loss": 3.4606785436464413, + "tokens_seen": 1987313664 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032154062186559677, + "loss": 1.2936, + "theoretical_loss": 3.460669094937238, + "tokens_seen": 1987379200 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032152457372116347, + "loss": 1.29, + "theoretical_loss": 3.460659646626851, + "tokens_seen": 1987444736 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032150852557673017, + "loss": 1.2297, + "theoretical_loss": 3.4606501987152494, + "tokens_seen": 1987510272 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032149247743229687, + "loss": 1.2765, + "theoretical_loss": 3.460640751202404, + "tokens_seen": 1987575808 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032147642928786357, + "loss": 1.2303, + "theoretical_loss": 3.4606313040882846, + "tokens_seen": 1987641344 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032146038114343027, + "loss": 1.2595, + "theoretical_loss": 3.460621857372861, + "tokens_seen": 1987706880 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032144433299899697, + "loss": 1.3184, + "theoretical_loss": 3.460612411056103, + "tokens_seen": 1987772416 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032142828485456367, + "loss": 1.2134, + "theoretical_loss": 3.4606029651379817, + "tokens_seen": 1987837952 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032141223671013037, + "loss": 1.3586, + "theoretical_loss": 3.4605935196184667, + "tokens_seen": 1987903488 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032139618856569707, + "loss": 1.217, + "theoretical_loss": 3.460584074497527, + "tokens_seen": 1987969024 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032138014042126377, + "loss": 1.1311, + "theoretical_loss": 3.4605746297751345, + "tokens_seen": 1988034560 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032136409227683047, + "loss": 1.1807, + "theoretical_loss": 3.460565185451258, + "tokens_seen": 1988100096 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032134804413239717, + "loss": 1.1941, + "theoretical_loss": 3.4605557415258668, + "tokens_seen": 1988165632 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003213319959879639, + "loss": 1.2327, + "theoretical_loss": 3.460546297998933, + "tokens_seen": 1988231168 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032131594784353063, + "loss": 1.2142, + "theoretical_loss": 3.460536854870425, + "tokens_seen": 1988296704 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032129989969909733, + "loss": 1.367, + "theoretical_loss": 3.460527412140314, + "tokens_seen": 1988362240 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032128385155466403, + "loss": 1.2278, + "theoretical_loss": 3.460517969808569, + "tokens_seen": 1988427776 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032126780341023073, + "loss": 1.2829, + "theoretical_loss": 3.4605085278751604, + "tokens_seen": 1988493312 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032125175526579743, + "loss": 1.2688, + "theoretical_loss": 3.4604990863400586, + "tokens_seen": 1988558848 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032123570712136413, + "loss": 1.3267, + "theoretical_loss": 3.460489645203234, + "tokens_seen": 1988624384 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032121965897693083, + "loss": 1.2963, + "theoretical_loss": 3.4604802044646554, + "tokens_seen": 1988689920 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032120361083249753, + "loss": 1.2214, + "theoretical_loss": 3.460470764124294, + "tokens_seen": 1988755456 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032118756268806423, + "loss": 1.2053, + "theoretical_loss": 3.46046132418212, + "tokens_seen": 1988820992 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032117151454363093, + "loss": 1.1426, + "theoretical_loss": 3.4604518846381023, + "tokens_seen": 1988886528 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032115546639919763, + "loss": 1.2487, + "theoretical_loss": 3.460442445492212, + "tokens_seen": 1988952064 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 665696, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3255927562713623, + "objective/train/theoretical_loss": 3.460433006744419, + "objective/train/tokens_used": 2009477600, + "theoretical_loss": 3.460433006744419, + "tokens_seen": 1989017600 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032113941825476434, + "loss": 1.2721, + "theoretical_loss": 3.460433006744419, + "tokens_seen": 1989017600 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032112337011033104, + "loss": 1.2777, + "theoretical_loss": 3.460423568394693, + "tokens_seen": 1989083136 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032110732196589774, + "loss": 1.3445, + "theoretical_loss": 3.4604141304430045, + "tokens_seen": 1989148672 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032109127382146444, + "loss": 1.3093, + "theoretical_loss": 3.4604046928893237, + "tokens_seen": 1989214208 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032107522567703114, + "loss": 1.2275, + "theoretical_loss": 3.46039525573362, + "tokens_seen": 1989279744 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032105917753259784, + "loss": 1.2871, + "theoretical_loss": 3.4603858189758645, + "tokens_seen": 1989345280 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032104312938816454, + "loss": 1.1227, + "theoretical_loss": 3.460376382616027, + "tokens_seen": 1989410816 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032102708124373124, + "loss": 1.2227, + "theoretical_loss": 3.4603669466540774, + "tokens_seen": 1989476352 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032101103309929794, + "loss": 1.3095, + "theoretical_loss": 3.4603575110899856, + "tokens_seen": 1989541888 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032099498495486464, + "loss": 1.269, + "theoretical_loss": 3.4603480759237217, + "tokens_seen": 1989607424 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032097893681043134, + "loss": 1.1718, + "theoretical_loss": 3.460338641155256, + "tokens_seen": 1989672960 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032096288866599804, + "loss": 1.2449, + "theoretical_loss": 3.4603292067845595, + "tokens_seen": 1989738496 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032094684052156474, + "loss": 1.2095, + "theoretical_loss": 3.460319772811601, + "tokens_seen": 1989804032 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032093079237713144, + "loss": 1.2088, + "theoretical_loss": 3.4603103392363512, + "tokens_seen": 1989869568 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003209147442326981, + "loss": 1.2959, + "theoretical_loss": 3.46030090605878, + "tokens_seen": 1989935104 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003208986960882648, + "loss": 1.1243, + "theoretical_loss": 3.4602914732788586, + "tokens_seen": 1990000640 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003208826479438315, + "loss": 1.2151, + "theoretical_loss": 3.4602820408965558, + "tokens_seen": 1990066176 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003208665997993982, + "loss": 1.2857, + "theoretical_loss": 3.4602726089118416, + "tokens_seen": 1990131712 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003208505516549649, + "loss": 1.2606, + "theoretical_loss": 3.4602631773246877, + "tokens_seen": 1990197248 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003208345035105316, + "loss": 1.2455, + "theoretical_loss": 3.460253746135063, + "tokens_seen": 1990262784 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003208184553660983, + "loss": 1.2167, + "theoretical_loss": 3.460244315342938, + "tokens_seen": 1990328320 + }, + { + "epoch": 0.6, + "learning_rate": 0.000320802407221665, + "loss": 1.178, + "theoretical_loss": 3.4602348849482825, + "tokens_seen": 1990393856 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003207863590772317, + "loss": 1.3953, + "theoretical_loss": 3.4602254549510674, + "tokens_seen": 1990459392 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003207703109327984, + "loss": 1.2184, + "theoretical_loss": 3.460216025351263, + "tokens_seen": 1990524928 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003207542627883651, + "loss": 1.1341, + "theoretical_loss": 3.460206596148838, + "tokens_seen": 1990590464 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003207382146439318, + "loss": 1.1911, + "theoretical_loss": 3.4601971673437637, + "tokens_seen": 1990656000 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003207221664994985, + "loss": 1.3415, + "theoretical_loss": 3.4601877389360105, + "tokens_seen": 1990721536 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003207061183550652, + "loss": 1.3122, + "theoretical_loss": 3.4601783109255484, + "tokens_seen": 1990787072 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003206900702106319, + "loss": 1.3456, + "theoretical_loss": 3.4601688833123467, + "tokens_seen": 1990852608 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003206740220661986, + "loss": 1.2308, + "theoretical_loss": 3.4601594560963767, + "tokens_seen": 1990918144 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003206579739217653, + "loss": 1.1944, + "theoretical_loss": 3.460150029277608, + "tokens_seen": 1990983680 + }, + { + "epoch": 0.6, + "learning_rate": 0.000320641925777332, + "loss": 1.3037, + "theoretical_loss": 3.4601406028560104, + "tokens_seen": 1991049216 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003206258776328987, + "loss": 1.2505, + "theoretical_loss": 3.4601311768315552, + "tokens_seen": 1991114752 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003206098294884654, + "loss": 1.2569, + "theoretical_loss": 3.4601217512042117, + "tokens_seen": 1991180288 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003205937813440321, + "loss": 1.2272, + "theoretical_loss": 3.460112325973951, + "tokens_seen": 1991245824 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003205777331995988, + "loss": 1.4319, + "theoretical_loss": 3.4601029011407425, + "tokens_seen": 1991311360 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003205616850551655, + "loss": 1.3005, + "theoretical_loss": 3.4600934767045564, + "tokens_seen": 1991376896 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003205456369107322, + "loss": 1.2495, + "theoretical_loss": 3.460084052665363, + "tokens_seen": 1991442432 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003205295887662989, + "loss": 1.3024, + "theoretical_loss": 3.460074629023133, + "tokens_seen": 1991507968 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003205135406218656, + "loss": 1.2622, + "theoretical_loss": 3.460065205777836, + "tokens_seen": 1991573504 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003204974924774323, + "loss": 1.2789, + "theoretical_loss": 3.4600557829294427, + "tokens_seen": 1991639040 + }, + { + "epoch": 0.6, + "learning_rate": 0.000320481444332999, + "loss": 1.1847, + "theoretical_loss": 3.460046360477923, + "tokens_seen": 1991704576 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003204653961885657, + "loss": 1.1588, + "theoretical_loss": 3.460036938423247, + "tokens_seen": 1991770112 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003204493480441324, + "loss": 1.3105, + "theoretical_loss": 3.4600275167653853, + "tokens_seen": 1991835648 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003204332998996991, + "loss": 1.3075, + "theoretical_loss": 3.4600180955043083, + "tokens_seen": 1991901184 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003204172517552658, + "loss": 1.3275, + "theoretical_loss": 3.4600086746399854, + "tokens_seen": 1991966720 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003204012036108325, + "loss": 1.1806, + "theoretical_loss": 3.459999254172388, + "tokens_seen": 1992032256 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003203851554663992, + "loss": 1.204, + "theoretical_loss": 3.459989834101485, + "tokens_seen": 1992097792 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003203691073219659, + "loss": 1.2993, + "theoretical_loss": 3.459980414427248, + "tokens_seen": 1992163328 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003203530591775326, + "loss": 1.2179, + "theoretical_loss": 3.4599709951496465, + "tokens_seen": 1992228864 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 666903, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2762541770935059, + "objective/train/theoretical_loss": 3.4599615762686504, + "objective/train/tokens_used": 2012754400, + "theoretical_loss": 3.4599615762686504, + "tokens_seen": 1992294400 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003203370110330993, + "loss": 1.2829, + "theoretical_loss": 3.4599615762686504, + "tokens_seen": 1992294400 + }, + { + "epoch": 0.6, + "learning_rate": 0.000320320962888666, + "loss": 1.2643, + "theoretical_loss": 3.4599521577842305, + "tokens_seen": 1992359936 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003203049147442327, + "loss": 1.2236, + "theoretical_loss": 3.4599427396963573, + "tokens_seen": 1992425472 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003202888665997994, + "loss": 1.3269, + "theoretical_loss": 3.459933322005001, + "tokens_seen": 1992491008 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003202728184553661, + "loss": 1.2183, + "theoretical_loss": 3.4599239047101307, + "tokens_seen": 1992556544 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003202567703109328, + "loss": 1.2264, + "theoretical_loss": 3.4599144878117185, + "tokens_seen": 1992622080 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003202407221664995, + "loss": 1.2591, + "theoretical_loss": 3.4599050713097332, + "tokens_seen": 1992687616 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003202246740220662, + "loss": 1.2588, + "theoretical_loss": 3.4598956552041455, + "tokens_seen": 1992753152 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003202086258776329, + "loss": 1.2807, + "theoretical_loss": 3.459886239494926, + "tokens_seen": 1992818688 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003201925777331996, + "loss": 1.1525, + "theoretical_loss": 3.4598768241820452, + "tokens_seen": 1992884224 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003201765295887663, + "loss": 1.2788, + "theoretical_loss": 3.459867409265472, + "tokens_seen": 1992949760 + }, + { + "epoch": 0.6, + "learning_rate": 0.000320160481444333, + "loss": 1.2724, + "theoretical_loss": 3.4598579947451786, + "tokens_seen": 1993015296 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032014443329989967, + "loss": 1.3001, + "theoretical_loss": 3.459848580621134, + "tokens_seen": 1993080832 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032012838515546637, + "loss": 1.1552, + "theoretical_loss": 3.4598391668933086, + "tokens_seen": 1993146368 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003201123370110331, + "loss": 1.2294, + "theoretical_loss": 3.4598297535616735, + "tokens_seen": 1993211904 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003200962888665998, + "loss": 1.3142, + "theoretical_loss": 3.459820340626198, + "tokens_seen": 1993277440 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003200802407221665, + "loss": 1.266, + "theoretical_loss": 3.4598109280868528, + "tokens_seen": 1993342976 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003200641925777332, + "loss": 1.2689, + "theoretical_loss": 3.459801515943608, + "tokens_seen": 1993408512 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003200481444332999, + "loss": 1.1802, + "theoretical_loss": 3.4597921041964343, + "tokens_seen": 1993474048 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003200320962888666, + "loss": 1.2232, + "theoretical_loss": 3.4597826928453017, + "tokens_seen": 1993539584 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003200160481444333, + "loss": 1.3042, + "theoretical_loss": 3.4597732818901816, + "tokens_seen": 1993605120 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032, + "loss": 1.2034, + "theoretical_loss": 3.4597638713310426, + "tokens_seen": 1993670656 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003199839518555667, + "loss": 1.213, + "theoretical_loss": 3.459754461167856, + "tokens_seen": 1993736192 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031996790371113343, + "loss": 1.2727, + "theoretical_loss": 3.4597450514005916, + "tokens_seen": 1993801728 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031995185556670013, + "loss": 1.3125, + "theoretical_loss": 3.4597356420292202, + "tokens_seen": 1993867264 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031993580742226683, + "loss": 1.2843, + "theoretical_loss": 3.459726233053712, + "tokens_seen": 1993932800 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031991975927783353, + "loss": 1.1092, + "theoretical_loss": 3.4597168244740377, + "tokens_seen": 1993998336 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031990371113340023, + "loss": 1.2958, + "theoretical_loss": 3.459707416290167, + "tokens_seen": 1994063872 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031988766298896693, + "loss": 1.3187, + "theoretical_loss": 3.4596980085020705, + "tokens_seen": 1994129408 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031987161484453363, + "loss": 1.3156, + "theoretical_loss": 3.459688601109719, + "tokens_seen": 1994194944 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031985556670010033, + "loss": 1.2592, + "theoretical_loss": 3.459679194113082, + "tokens_seen": 1994260480 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031983951855566703, + "loss": 1.2869, + "theoretical_loss": 3.45966978751213, + "tokens_seen": 1994326016 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031982347041123373, + "loss": 1.1774, + "theoretical_loss": 3.4596603813068345, + "tokens_seen": 1994391552 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031980742226680043, + "loss": 1.2784, + "theoretical_loss": 3.459650975497164, + "tokens_seen": 1994457088 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031979137412236713, + "loss": 1.1861, + "theoretical_loss": 3.4596415700830905, + "tokens_seen": 1994522624 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031977532597793384, + "loss": 1.2795, + "theoretical_loss": 3.4596321650645834, + "tokens_seen": 1994588160 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031975927783350054, + "loss": 1.2154, + "theoretical_loss": 3.459622760441613, + "tokens_seen": 1994653696 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031974322968906724, + "loss": 1.2445, + "theoretical_loss": 3.4596133562141507, + "tokens_seen": 1994719232 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031972718154463394, + "loss": 1.2929, + "theoretical_loss": 3.459603952382166, + "tokens_seen": 1994784768 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031971113340020064, + "loss": 1.2318, + "theoretical_loss": 3.4595945489456295, + "tokens_seen": 1994850304 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031969508525576734, + "loss": 1.3232, + "theoretical_loss": 3.4595851459045113, + "tokens_seen": 1994915840 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031967903711133404, + "loss": 1.2139, + "theoretical_loss": 3.459575743258782, + "tokens_seen": 1994981376 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031966298896690074, + "loss": 1.1856, + "theoretical_loss": 3.4595663410084123, + "tokens_seen": 1995046912 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031964694082246744, + "loss": 1.2733, + "theoretical_loss": 3.4595569391533725, + "tokens_seen": 1995112448 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031963089267803414, + "loss": 1.2606, + "theoretical_loss": 3.4595475376936324, + "tokens_seen": 1995177984 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031961484453360084, + "loss": 1.2216, + "theoretical_loss": 3.4595381366291633, + "tokens_seen": 1995243520 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031959879638916754, + "loss": 1.2656, + "theoretical_loss": 3.4595287359599345, + "tokens_seen": 1995309056 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031958274824473424, + "loss": 1.2624, + "theoretical_loss": 3.4595193356859175, + "tokens_seen": 1995374592 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031956670010030094, + "loss": 1.362, + "theoretical_loss": 3.459509935807082, + "tokens_seen": 1995440128 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031955065195586764, + "loss": 1.1668, + "theoretical_loss": 3.4595005363233984, + "tokens_seen": 1995505664 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 667918, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2358568906784058, + "objective/train/theoretical_loss": 3.4594911372348376, + "objective/train/tokens_used": 2016031200, + "theoretical_loss": 3.4594911372348376, + "tokens_seen": 1995571200 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031953460381143434, + "loss": 1.2884, + "theoretical_loss": 3.4594911372348376, + "tokens_seen": 1995571200 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031951855566700105, + "loss": 1.211, + "theoretical_loss": 3.45948173854137, + "tokens_seen": 1995636736 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003195025075225677, + "loss": 1.248, + "theoretical_loss": 3.4594723402429657, + "tokens_seen": 1995702272 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003194864593781344, + "loss": 1.3623, + "theoretical_loss": 3.459462942339595, + "tokens_seen": 1995767808 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003194704112337011, + "loss": 1.1604, + "theoretical_loss": 3.459453544831229, + "tokens_seen": 1995833344 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003194543630892678, + "loss": 1.2344, + "theoretical_loss": 3.4594441477178366, + "tokens_seen": 1995898880 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003194383149448345, + "loss": 1.167, + "theoretical_loss": 3.45943475099939, + "tokens_seen": 1995964416 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003194222668004012, + "loss": 1.3237, + "theoretical_loss": 3.459425354675859, + "tokens_seen": 1996029952 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003194062186559679, + "loss": 1.2519, + "theoretical_loss": 3.459415958747214, + "tokens_seen": 1996095488 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003193901705115346, + "loss": 1.3209, + "theoretical_loss": 3.4594065632134248, + "tokens_seen": 1996161024 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003193741223671013, + "loss": 1.3826, + "theoretical_loss": 3.459397168074463, + "tokens_seen": 1996226560 + }, + { + "epoch": 0.6, + "learning_rate": 0.000319358074222668, + "loss": 1.2225, + "theoretical_loss": 3.459387773330298, + "tokens_seen": 1996292096 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003193420260782347, + "loss": 1.41, + "theoretical_loss": 3.459378378980901, + "tokens_seen": 1996357632 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003193259779338014, + "loss": 1.3175, + "theoretical_loss": 3.4593689850262423, + "tokens_seen": 1996423168 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003193099297893681, + "loss": 1.2849, + "theoretical_loss": 3.459359591466292, + "tokens_seen": 1996488704 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003192938816449348, + "loss": 1.2669, + "theoretical_loss": 3.459350198301021, + "tokens_seen": 1996554240 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003192778335005015, + "loss": 1.2577, + "theoretical_loss": 3.4593408055304, + "tokens_seen": 1996619776 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003192617853560682, + "loss": 1.3572, + "theoretical_loss": 3.4593314131543984, + "tokens_seen": 1996685312 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003192457372116349, + "loss": 1.2545, + "theoretical_loss": 3.459322021172987, + "tokens_seen": 1996750848 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003192296890672016, + "loss": 1.3256, + "theoretical_loss": 3.459312629586137, + "tokens_seen": 1996816384 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003192136409227683, + "loss": 1.2982, + "theoretical_loss": 3.4593032383938187, + "tokens_seen": 1996881920 + }, + { + "epoch": 0.61, + "learning_rate": 0.000319197592778335, + "loss": 1.2946, + "theoretical_loss": 3.459293847596002, + "tokens_seen": 1996947456 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003191815446339017, + "loss": 1.2394, + "theoretical_loss": 3.4592844571926578, + "tokens_seen": 1997012992 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003191654964894684, + "loss": 1.259, + "theoretical_loss": 3.459275067183756, + "tokens_seen": 1997078528 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003191494483450351, + "loss": 1.3783, + "theoretical_loss": 3.4592656775692685, + "tokens_seen": 1997144064 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003191334002006018, + "loss": 1.3465, + "theoretical_loss": 3.4592562883491644, + "tokens_seen": 1997209600 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003191173520561685, + "loss": 1.2976, + "theoretical_loss": 3.4592468995234147, + "tokens_seen": 1997275136 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003191013039117352, + "loss": 1.3208, + "theoretical_loss": 3.45923751109199, + "tokens_seen": 1997340672 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003190852557673019, + "loss": 1.1888, + "theoretical_loss": 3.4592281230548605, + "tokens_seen": 1997406208 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003190692076228686, + "loss": 1.2235, + "theoretical_loss": 3.459218735411997, + "tokens_seen": 1997471744 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003190531594784353, + "loss": 1.3613, + "theoretical_loss": 3.4592093481633697, + "tokens_seen": 1997537280 + }, + { + "epoch": 0.61, + "learning_rate": 0.000319037111334002, + "loss": 1.232, + "theoretical_loss": 3.4591999613089492, + "tokens_seen": 1997602816 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003190210631895687, + "loss": 1.358, + "theoretical_loss": 3.459190574848706, + "tokens_seen": 1997668352 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003190050150451354, + "loss": 1.2946, + "theoretical_loss": 3.4591811887826114, + "tokens_seen": 1997733888 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003189889669007021, + "loss": 1.2784, + "theoretical_loss": 3.4591718031106344, + "tokens_seen": 1997799424 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003189729187562688, + "loss": 1.2532, + "theoretical_loss": 3.459162417832747, + "tokens_seen": 1997864960 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003189568706118355, + "loss": 1.186, + "theoretical_loss": 3.459153032948919, + "tokens_seen": 1997930496 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003189408224674022, + "loss": 1.3609, + "theoretical_loss": 3.4591436484591207, + "tokens_seen": 1997996032 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003189247743229689, + "loss": 1.2931, + "theoretical_loss": 3.4591342643633234, + "tokens_seen": 1998061568 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031890872617853567, + "loss": 1.2729, + "theoretical_loss": 3.4591248806614967, + "tokens_seen": 1998127104 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031889267803410237, + "loss": 1.2624, + "theoretical_loss": 3.459115497353612, + "tokens_seen": 1998192640 + }, + { + "epoch": 0.61, + "learning_rate": 0.000318876629889669, + "loss": 1.2548, + "theoretical_loss": 3.4591061144396393, + "tokens_seen": 1998258176 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003188605817452357, + "loss": 1.3345, + "theoretical_loss": 3.4590967319195496, + "tokens_seen": 1998323712 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003188445336008024, + "loss": 1.1979, + "theoretical_loss": 3.459087349793313, + "tokens_seen": 1998389248 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003188284854563691, + "loss": 1.243, + "theoretical_loss": 3.4590779680608996, + "tokens_seen": 1998454784 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003188124373119358, + "loss": 1.2696, + "theoretical_loss": 3.4590685867222812, + "tokens_seen": 1998520320 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003187963891675025, + "loss": 1.3569, + "theoretical_loss": 3.4590592057774274, + "tokens_seen": 1998585856 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003187803410230692, + "loss": 1.2444, + "theoretical_loss": 3.4590498252263098, + "tokens_seen": 1998651392 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003187642928786359, + "loss": 1.2312, + "theoretical_loss": 3.459040445068897, + "tokens_seen": 1998716928 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003187482447342026, + "loss": 1.1778, + "theoretical_loss": 3.4590310653051617, + "tokens_seen": 1998782464 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 669104, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2669732570648193, + "objective/train/theoretical_loss": 3.459021685935074, + "objective/train/tokens_used": 2019308000, + "theoretical_loss": 3.459021685935074, + "tokens_seen": 1998848000 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003187321965897693, + "loss": 1.2328, + "theoretical_loss": 3.459021685935074, + "tokens_seen": 1998848000 + }, + { + "epoch": 0.61, + "learning_rate": 0.000318716148445336, + "loss": 1.155, + "theoretical_loss": 3.459012306958603, + "tokens_seen": 1998913536 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003187001003009027, + "loss": 1.243, + "theoretical_loss": 3.459002928375721, + "tokens_seen": 1998979072 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003186840521564694, + "loss": 1.3237, + "theoretical_loss": 3.4589935501863978, + "tokens_seen": 1999044608 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003186680040120361, + "loss": 1.2906, + "theoretical_loss": 3.458984172390604, + "tokens_seen": 1999110144 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003186519558676028, + "loss": 1.3192, + "theoretical_loss": 3.4589747949883103, + "tokens_seen": 1999175680 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003186359077231695, + "loss": 1.254, + "theoretical_loss": 3.4589654179794875, + "tokens_seen": 1999241216 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003186198595787362, + "loss": 1.2594, + "theoretical_loss": 3.4589560413641056, + "tokens_seen": 1999306752 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031860381143430293, + "loss": 1.1965, + "theoretical_loss": 3.458946665142136, + "tokens_seen": 1999372288 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031858776328986963, + "loss": 1.265, + "theoretical_loss": 3.458937289313549, + "tokens_seen": 1999437824 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031857171514543633, + "loss": 1.3567, + "theoretical_loss": 3.4589279138783144, + "tokens_seen": 1999503360 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031855566700100303, + "loss": 1.3493, + "theoretical_loss": 3.4589185388364037, + "tokens_seen": 1999568896 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031853961885656973, + "loss": 1.2755, + "theoretical_loss": 3.4589091641877876, + "tokens_seen": 1999634432 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031852357071213643, + "loss": 1.2332, + "theoretical_loss": 3.458899789932436, + "tokens_seen": 1999699968 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031850752256770313, + "loss": 1.1523, + "theoretical_loss": 3.4588904160703207, + "tokens_seen": 1999765504 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031849147442326983, + "loss": 1.3182, + "theoretical_loss": 3.4588810426014103, + "tokens_seen": 1999831040 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031847542627883653, + "loss": 1.3163, + "theoretical_loss": 3.458871669525678, + "tokens_seen": 1999896576 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031845937813440323, + "loss": 1.2679, + "theoretical_loss": 3.4588622968430918, + "tokens_seen": 1999962112 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031844332998996993, + "loss": 1.2808, + "theoretical_loss": 3.458852924553624, + "tokens_seen": 2000027648 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031842728184553663, + "loss": 1.3126, + "theoretical_loss": 3.458843552657245, + "tokens_seen": 2000093184 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031841123370110333, + "loss": 1.19, + "theoretical_loss": 3.4588341811539256, + "tokens_seen": 2000158720 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031839518555667004, + "loss": 1.1766, + "theoretical_loss": 3.4588248100436356, + "tokens_seen": 2000224256 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031837913741223674, + "loss": 1.2651, + "theoretical_loss": 3.4588154393263464, + "tokens_seen": 2000289792 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031836308926780344, + "loss": 1.1677, + "theoretical_loss": 3.458806069002028, + "tokens_seen": 2000355328 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031834704112337014, + "loss": 1.3727, + "theoretical_loss": 3.458796699070652, + "tokens_seen": 2000420864 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031833099297893684, + "loss": 1.249, + "theoretical_loss": 3.4587873295321883, + "tokens_seen": 2000486400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031831494483450354, + "loss": 1.3561, + "theoretical_loss": 3.458777960386608, + "tokens_seen": 2000551936 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031829889669007024, + "loss": 1.2476, + "theoretical_loss": 3.4587685916338806, + "tokens_seen": 2000617472 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031828284854563694, + "loss": 1.2328, + "theoretical_loss": 3.4587592232739786, + "tokens_seen": 2000683008 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031826680040120364, + "loss": 1.2158, + "theoretical_loss": 3.458749855306871, + "tokens_seen": 2000748544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031825075225677034, + "loss": 1.2674, + "theoretical_loss": 3.4587404877325296, + "tokens_seen": 2000814080 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031823470411233704, + "loss": 1.2494, + "theoretical_loss": 3.4587311205509246, + "tokens_seen": 2000879616 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031821865596790374, + "loss": 1.3109, + "theoretical_loss": 3.4587217537620267, + "tokens_seen": 2000945152 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031820260782347044, + "loss": 1.2851, + "theoretical_loss": 3.4587123873658063, + "tokens_seen": 2001010688 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031818655967903714, + "loss": 1.222, + "theoretical_loss": 3.458703021362235, + "tokens_seen": 2001076224 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031817051153460384, + "loss": 1.1502, + "theoretical_loss": 3.458693655751283, + "tokens_seen": 2001141760 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031815446339017054, + "loss": 1.2481, + "theoretical_loss": 3.45868429053292, + "tokens_seen": 2001207296 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031813841524573725, + "loss": 1.2375, + "theoretical_loss": 3.4586749257071174, + "tokens_seen": 2001272832 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031812236710130395, + "loss": 1.2768, + "theoretical_loss": 3.458665561273847, + "tokens_seen": 2001338368 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003181063189568706, + "loss": 1.3149, + "theoretical_loss": 3.458656197233078, + "tokens_seen": 2001403904 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003180902708124373, + "loss": 1.2879, + "theoretical_loss": 3.4586468335847815, + "tokens_seen": 2001469440 + }, + { + "epoch": 0.61, + "learning_rate": 0.000318074222668004, + "loss": 1.3222, + "theoretical_loss": 3.458637470328928, + "tokens_seen": 2001534976 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003180581745235707, + "loss": 1.3136, + "theoretical_loss": 3.4586281074654894, + "tokens_seen": 2001600512 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003180421263791374, + "loss": 1.2326, + "theoretical_loss": 3.458618744994435, + "tokens_seen": 2001666048 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003180260782347041, + "loss": 1.2437, + "theoretical_loss": 3.458609382915736, + "tokens_seen": 2001731584 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003180100300902708, + "loss": 1.2383, + "theoretical_loss": 3.4586000212293633, + "tokens_seen": 2001797120 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003179939819458375, + "loss": 1.2834, + "theoretical_loss": 3.4585906599352874, + "tokens_seen": 2001862656 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003179779338014042, + "loss": 1.2408, + "theoretical_loss": 3.458581299033479, + "tokens_seen": 2001928192 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003179618856569709, + "loss": 1.2478, + "theoretical_loss": 3.458571938523909, + "tokens_seen": 2001993728 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003179458375125376, + "loss": 1.1506, + "theoretical_loss": 3.458562578406548, + "tokens_seen": 2002059264 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 670405, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1057586669921875, + "objective/train/theoretical_loss": 3.4585532186813666, + "objective/train/tokens_used": 2022584800, + "theoretical_loss": 3.4585532186813666, + "tokens_seen": 2002124800 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003179297893681043, + "loss": 1.1716, + "theoretical_loss": 3.4585532186813666, + "tokens_seen": 2002124800 + }, + { + "epoch": 0.61, + "learning_rate": 0.000317913741223671, + "loss": 1.2949, + "theoretical_loss": 3.458543859348336, + "tokens_seen": 2002190336 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003178976930792377, + "loss": 1.3765, + "theoretical_loss": 3.4585345004074264, + "tokens_seen": 2002255872 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003178816449348044, + "loss": 1.2908, + "theoretical_loss": 3.4585251418586087, + "tokens_seen": 2002321408 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003178655967903711, + "loss": 1.3964, + "theoretical_loss": 3.458515783701854, + "tokens_seen": 2002386944 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003178495486459378, + "loss": 1.1777, + "theoretical_loss": 3.4585064259371325, + "tokens_seen": 2002452480 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003178335005015045, + "loss": 1.2345, + "theoretical_loss": 3.458497068564415, + "tokens_seen": 2002518016 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003178174523570712, + "loss": 1.2328, + "theoretical_loss": 3.4584877115836727, + "tokens_seen": 2002583552 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003178014042126379, + "loss": 1.2715, + "theoretical_loss": 3.458478354994876, + "tokens_seen": 2002649088 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003177853560682046, + "loss": 1.2451, + "theoretical_loss": 3.4584689987979957, + "tokens_seen": 2002714624 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003177693079237713, + "loss": 1.311, + "theoretical_loss": 3.458459642993003, + "tokens_seen": 2002780160 + }, + { + "epoch": 0.61, + "learning_rate": 0.000317753259779338, + "loss": 1.3298, + "theoretical_loss": 3.4584502875798675, + "tokens_seen": 2002845696 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003177372116349047, + "loss": 1.2584, + "theoretical_loss": 3.4584409325585614, + "tokens_seen": 2002911232 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003177211634904714, + "loss": 1.27, + "theoretical_loss": 3.458431577929055, + "tokens_seen": 2002976768 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031770511534603816, + "loss": 1.1746, + "theoretical_loss": 3.4584222236913185, + "tokens_seen": 2003042304 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031768906720160486, + "loss": 1.3136, + "theoretical_loss": 3.458412869845323, + "tokens_seen": 2003107840 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031767301905717156, + "loss": 1.2681, + "theoretical_loss": 3.458403516391039, + "tokens_seen": 2003173376 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031765697091273826, + "loss": 1.2695, + "theoretical_loss": 3.458394163328438, + "tokens_seen": 2003238912 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031764092276830497, + "loss": 1.2699, + "theoretical_loss": 3.4583848106574906, + "tokens_seen": 2003304448 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031762487462387167, + "loss": 1.2239, + "theoretical_loss": 3.458375458378167, + "tokens_seen": 2003369984 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031760882647943837, + "loss": 1.2757, + "theoretical_loss": 3.4583661064904385, + "tokens_seen": 2003435520 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031759277833500507, + "loss": 1.2416, + "theoretical_loss": 3.458356754994276, + "tokens_seen": 2003501056 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031757673019057177, + "loss": 1.3232, + "theoretical_loss": 3.4583474038896496, + "tokens_seen": 2003566592 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031756068204613847, + "loss": 1.3648, + "theoretical_loss": 3.458338053176531, + "tokens_seen": 2003632128 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031754463390170517, + "loss": 1.3, + "theoretical_loss": 3.4583287028548906, + "tokens_seen": 2003697664 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031752858575727187, + "loss": 1.2187, + "theoretical_loss": 3.458319352924699, + "tokens_seen": 2003763200 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031751253761283857, + "loss": 1.2122, + "theoretical_loss": 3.458310003385927, + "tokens_seen": 2003828736 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031749648946840527, + "loss": 1.3151, + "theoretical_loss": 3.4583006542385464, + "tokens_seen": 2003894272 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003174804413239719, + "loss": 1.2365, + "theoretical_loss": 3.4582913054825264, + "tokens_seen": 2003959808 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003174643931795386, + "loss": 1.2237, + "theoretical_loss": 3.4582819571178387, + "tokens_seen": 2004025344 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003174483450351053, + "loss": 1.178, + "theoretical_loss": 3.4582726091444544, + "tokens_seen": 2004090880 + }, + { + "epoch": 0.61, + "learning_rate": 0.000317432296890672, + "loss": 1.1719, + "theoretical_loss": 3.458263261562344, + "tokens_seen": 2004156416 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003174162487462387, + "loss": 1.3065, + "theoretical_loss": 3.458253914371478, + "tokens_seen": 2004221952 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003174002006018054, + "loss": 1.3441, + "theoretical_loss": 3.458244567571828, + "tokens_seen": 2004287488 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003173841524573721, + "loss": 1.268, + "theoretical_loss": 3.4582352211633642, + "tokens_seen": 2004353024 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003173681043129388, + "loss": 1.3368, + "theoretical_loss": 3.4582258751460575, + "tokens_seen": 2004418560 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003173520561685055, + "loss": 1.1921, + "theoretical_loss": 3.458216529519879, + "tokens_seen": 2004484096 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003173360080240722, + "loss": 1.2697, + "theoretical_loss": 3.458207184284799, + "tokens_seen": 2004549632 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003173199598796389, + "loss": 1.185, + "theoretical_loss": 3.458197839440789, + "tokens_seen": 2004615168 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003173039117352056, + "loss": 1.3091, + "theoretical_loss": 3.4581884949878194, + "tokens_seen": 2004680704 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003172878635907723, + "loss": 1.1381, + "theoretical_loss": 3.4581791509258615, + "tokens_seen": 2004746240 + }, + { + "epoch": 0.61, + "learning_rate": 0.000317271815446339, + "loss": 1.4164, + "theoretical_loss": 3.458169807254886, + "tokens_seen": 2004811776 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003172557673019057, + "loss": 1.1845, + "theoretical_loss": 3.4581604639748638, + "tokens_seen": 2004877312 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003172397191574724, + "loss": 1.2115, + "theoretical_loss": 3.4581511210857654, + "tokens_seen": 2004942848 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031722367101303913, + "loss": 1.3386, + "theoretical_loss": 3.458141778587562, + "tokens_seen": 2005008384 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031720762286860583, + "loss": 1.2416, + "theoretical_loss": 3.4581324364802244, + "tokens_seen": 2005073920 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031719157472417253, + "loss": 1.2667, + "theoretical_loss": 3.4581230947637236, + "tokens_seen": 2005139456 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031717552657973923, + "loss": 1.2578, + "theoretical_loss": 3.45811375343803, + "tokens_seen": 2005204992 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031715947843530593, + "loss": 1.2442, + "theoretical_loss": 3.458104412503115, + "tokens_seen": 2005270528 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031714343029087263, + "loss": 1.3346, + "theoretical_loss": 3.458095071958949, + "tokens_seen": 2005336064 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 671345, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2735745906829834, + "objective/train/theoretical_loss": 3.4580857318055034, + "objective/train/tokens_used": 2025861600, + "theoretical_loss": 3.4580857318055034, + "tokens_seen": 2005401600 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031712738214643933, + "loss": 1.2609, + "theoretical_loss": 3.4580857318055034, + "tokens_seen": 2005401600 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031711133400200603, + "loss": 1.2532, + "theoretical_loss": 3.4580763920427486, + "tokens_seen": 2005467136 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031709528585757273, + "loss": 1.2701, + "theoretical_loss": 3.458067052670656, + "tokens_seen": 2005532672 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031707923771313943, + "loss": 1.1594, + "theoretical_loss": 3.458057713689196, + "tokens_seen": 2005598208 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031706318956870613, + "loss": 1.259, + "theoretical_loss": 3.45804837509834, + "tokens_seen": 2005663744 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031704714142427283, + "loss": 1.1806, + "theoretical_loss": 3.458039036898059, + "tokens_seen": 2005729280 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031703109327983954, + "loss": 1.2136, + "theoretical_loss": 3.458029699088323, + "tokens_seen": 2005794816 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031701504513540624, + "loss": 1.2813, + "theoretical_loss": 3.4580203616691034, + "tokens_seen": 2005860352 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031699899699097294, + "loss": 1.2302, + "theoretical_loss": 3.4580110246403715, + "tokens_seen": 2005925888 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031698294884653964, + "loss": 1.2539, + "theoretical_loss": 3.4580016880020974, + "tokens_seen": 2005991424 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031696690070210634, + "loss": 1.2526, + "theoretical_loss": 3.4579923517542532, + "tokens_seen": 2006056960 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031695085255767304, + "loss": 1.1677, + "theoretical_loss": 3.457983015896809, + "tokens_seen": 2006122496 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031693480441323974, + "loss": 1.2033, + "theoretical_loss": 3.457973680429735, + "tokens_seen": 2006188032 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031691875626880644, + "loss": 1.1112, + "theoretical_loss": 3.4579643453530036, + "tokens_seen": 2006253568 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031690270812437314, + "loss": 1.1891, + "theoretical_loss": 3.457955010666585, + "tokens_seen": 2006319104 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031688665997993984, + "loss": 1.2892, + "theoretical_loss": 3.4579456763704504, + "tokens_seen": 2006384640 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031687061183550654, + "loss": 1.3585, + "theoretical_loss": 3.4579363424645706, + "tokens_seen": 2006450176 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031685456369107324, + "loss": 1.2335, + "theoretical_loss": 3.457927008948916, + "tokens_seen": 2006515712 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031683851554663994, + "loss": 1.2356, + "theoretical_loss": 3.457917675823458, + "tokens_seen": 2006581248 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031682246740220664, + "loss": 1.135, + "theoretical_loss": 3.4579083430881683, + "tokens_seen": 2006646784 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031680641925777334, + "loss": 1.2078, + "theoretical_loss": 3.4578990107430165, + "tokens_seen": 2006712320 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031679037111334004, + "loss": 1.3739, + "theoretical_loss": 3.457889678787975, + "tokens_seen": 2006777856 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031677432296890675, + "loss": 1.1849, + "theoretical_loss": 3.457880347223013, + "tokens_seen": 2006843392 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031675827482447345, + "loss": 1.267, + "theoretical_loss": 3.4578710160481023, + "tokens_seen": 2006908928 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031674222668004015, + "loss": 1.1858, + "theoretical_loss": 3.4578616852632145, + "tokens_seen": 2006974464 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031672617853560685, + "loss": 1.2572, + "theoretical_loss": 3.45785235486832, + "tokens_seen": 2007040000 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003167101303911735, + "loss": 1.1521, + "theoretical_loss": 3.4578430248633896, + "tokens_seen": 2007105536 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003166940822467402, + "loss": 1.1478, + "theoretical_loss": 3.457833695248395, + "tokens_seen": 2007171072 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003166780341023069, + "loss": 1.2805, + "theoretical_loss": 3.4578243660233055, + "tokens_seen": 2007236608 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003166619859578736, + "loss": 1.3188, + "theoretical_loss": 3.4578150371880936, + "tokens_seen": 2007302144 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003166459378134403, + "loss": 1.361, + "theoretical_loss": 3.4578057087427303, + "tokens_seen": 2007367680 + }, + { + "epoch": 0.61, + "learning_rate": 0.000316629889669007, + "loss": 1.2297, + "theoretical_loss": 3.457796380687186, + "tokens_seen": 2007433216 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003166138415245737, + "loss": 1.2545, + "theoretical_loss": 3.4577870530214314, + "tokens_seen": 2007498752 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003165977933801404, + "loss": 1.3016, + "theoretical_loss": 3.4577777257454385, + "tokens_seen": 2007564288 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003165817452357071, + "loss": 1.3507, + "theoretical_loss": 3.4577683988591774, + "tokens_seen": 2007629824 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003165656970912738, + "loss": 1.2373, + "theoretical_loss": 3.4577590723626193, + "tokens_seen": 2007695360 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003165496489468405, + "loss": 1.2689, + "theoretical_loss": 3.4577497462557356, + "tokens_seen": 2007760896 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003165336008024072, + "loss": 1.3171, + "theoretical_loss": 3.4577404205384967, + "tokens_seen": 2007826432 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003165175526579739, + "loss": 1.1521, + "theoretical_loss": 3.4577310952108746, + "tokens_seen": 2007891968 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031650150451354066, + "loss": 1.178, + "theoretical_loss": 3.4577217702728387, + "tokens_seen": 2007957504 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031648545636910736, + "loss": 1.3053, + "theoretical_loss": 3.4577124457243613, + "tokens_seen": 2008023040 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031646940822467406, + "loss": 1.2603, + "theoretical_loss": 3.4577031215654133, + "tokens_seen": 2008088576 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031645336008024076, + "loss": 1.1754, + "theoretical_loss": 3.457693797795965, + "tokens_seen": 2008154112 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031643731193580746, + "loss": 1.298, + "theoretical_loss": 3.4576844744159883, + "tokens_seen": 2008219648 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031642126379137416, + "loss": 1.2631, + "theoretical_loss": 3.4576751514254536, + "tokens_seen": 2008285184 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031640521564694086, + "loss": 1.272, + "theoretical_loss": 3.4576658288243323, + "tokens_seen": 2008350720 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031638916750250756, + "loss": 1.3267, + "theoretical_loss": 3.4576565066125946, + "tokens_seen": 2008416256 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031637311935807426, + "loss": 1.2737, + "theoretical_loss": 3.457647184790213, + "tokens_seen": 2008481792 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031635707121364096, + "loss": 1.2197, + "theoretical_loss": 3.4576378633571574, + "tokens_seen": 2008547328 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031634102306920766, + "loss": 1.2409, + "theoretical_loss": 3.4576285423133992, + "tokens_seen": 2008612864 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 672663, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.369638442993164, + "objective/train/theoretical_loss": 3.4576192216589092, + "objective/train/tokens_used": 2029138400, + "theoretical_loss": 3.4576192216589092, + "tokens_seen": 2008678400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031632497492477436, + "loss": 1.3517, + "theoretical_loss": 3.4576192216589092, + "tokens_seen": 2008678400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031630892678034106, + "loss": 1.2304, + "theoretical_loss": 3.4576099013936585, + "tokens_seen": 2008743936 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031629287863590776, + "loss": 1.1586, + "theoretical_loss": 3.4576005815176183, + "tokens_seen": 2008809472 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031627683049147446, + "loss": 1.3099, + "theoretical_loss": 3.45759126203076, + "tokens_seen": 2008875008 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031626078234704117, + "loss": 1.3467, + "theoretical_loss": 3.457581942933054, + "tokens_seen": 2008940544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031624473420260787, + "loss": 1.237, + "theoretical_loss": 3.457572624224472, + "tokens_seen": 2009006080 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031622868605817457, + "loss": 1.2176, + "theoretical_loss": 3.4575633059049844, + "tokens_seen": 2009071616 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031621263791374127, + "loss": 1.2523, + "theoretical_loss": 3.4575539879745625, + "tokens_seen": 2009137152 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031619658976930797, + "loss": 1.2515, + "theoretical_loss": 3.4575446704331774, + "tokens_seen": 2009202688 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031618054162487467, + "loss": 1.265, + "theoretical_loss": 3.4575353532808, + "tokens_seen": 2009268224 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031616449348044137, + "loss": 1.2193, + "theoretical_loss": 3.457526036517402, + "tokens_seen": 2009333760 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031614844533600807, + "loss": 1.2702, + "theoretical_loss": 3.457516720142954, + "tokens_seen": 2009399296 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031613239719157477, + "loss": 1.3199, + "theoretical_loss": 3.457507404157427, + "tokens_seen": 2009464832 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031611634904714147, + "loss": 1.3002, + "theoretical_loss": 3.4574980885607927, + "tokens_seen": 2009530368 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031610030090270817, + "loss": 1.3355, + "theoretical_loss": 3.4574887733530213, + "tokens_seen": 2009595904 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031608425275827487, + "loss": 1.3138, + "theoretical_loss": 3.457479458534084, + "tokens_seen": 2009661440 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003160682046138415, + "loss": 1.3121, + "theoretical_loss": 3.4574701441039526, + "tokens_seen": 2009726976 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003160521564694082, + "loss": 1.2193, + "theoretical_loss": 3.457460830062597, + "tokens_seen": 2009792512 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003160361083249749, + "loss": 1.2748, + "theoretical_loss": 3.4574515164099897, + "tokens_seen": 2009858048 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003160200601805416, + "loss": 1.2837, + "theoretical_loss": 3.457442203146101, + "tokens_seen": 2009923584 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003160040120361083, + "loss": 1.33, + "theoretical_loss": 3.457432890270902, + "tokens_seen": 2009989120 + }, + { + "epoch": 0.61, + "learning_rate": 0.000315987963891675, + "loss": 1.2559, + "theoretical_loss": 3.4574235777843643, + "tokens_seen": 2010054656 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003159719157472417, + "loss": 1.2591, + "theoretical_loss": 3.4574142656864586, + "tokens_seen": 2010120192 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003159558676028084, + "loss": 1.3054, + "theoretical_loss": 3.457404953977156, + "tokens_seen": 2010185728 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003159398194583751, + "loss": 1.2851, + "theoretical_loss": 3.4573956426564276, + "tokens_seen": 2010251264 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003159237713139418, + "loss": 1.258, + "theoretical_loss": 3.4573863317242446, + "tokens_seen": 2010316800 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003159077231695085, + "loss": 1.2512, + "theoretical_loss": 3.4573770211805783, + "tokens_seen": 2010382336 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003158916750250752, + "loss": 1.2494, + "theoretical_loss": 3.457367711025399, + "tokens_seen": 2010447872 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003158756268806419, + "loss": 1.2882, + "theoretical_loss": 3.4573584012586793, + "tokens_seen": 2010513408 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031585957873620863, + "loss": 1.2828, + "theoretical_loss": 3.4573490918803893, + "tokens_seen": 2010578944 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031584353059177533, + "loss": 1.2842, + "theoretical_loss": 3.4573397828905, + "tokens_seen": 2010644480 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031582748244734203, + "loss": 1.23, + "theoretical_loss": 3.4573304742889834, + "tokens_seen": 2010710016 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031581143430290873, + "loss": 1.3398, + "theoretical_loss": 3.45732116607581, + "tokens_seen": 2010775552 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031579538615847543, + "loss": 1.2349, + "theoretical_loss": 3.457311858250951, + "tokens_seen": 2010841088 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031577933801404213, + "loss": 1.1745, + "theoretical_loss": 3.4573025508143775, + "tokens_seen": 2010906624 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031576328986960883, + "loss": 1.3773, + "theoretical_loss": 3.4572932437660606, + "tokens_seen": 2010972160 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031574724172517553, + "loss": 1.2861, + "theoretical_loss": 3.457283937105972, + "tokens_seen": 2011037696 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031573119358074223, + "loss": 1.2441, + "theoretical_loss": 3.457274630834082, + "tokens_seen": 2011103232 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031571514543630893, + "loss": 1.2162, + "theoretical_loss": 3.4572653249503627, + "tokens_seen": 2011168768 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031569909729187563, + "loss": 1.2249, + "theoretical_loss": 3.4572560194547846, + "tokens_seen": 2011234304 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031568304914744233, + "loss": 1.4085, + "theoretical_loss": 3.4572467143473187, + "tokens_seen": 2011299840 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031566700100300903, + "loss": 1.2425, + "theoretical_loss": 3.457237409627937, + "tokens_seen": 2011365376 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031565095285857574, + "loss": 1.2921, + "theoretical_loss": 3.4572281052966103, + "tokens_seen": 2011430912 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031563490471414244, + "loss": 1.2687, + "theoretical_loss": 3.457218801353309, + "tokens_seen": 2011496448 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031561885656970914, + "loss": 1.3747, + "theoretical_loss": 3.4572094977980052, + "tokens_seen": 2011561984 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031560280842527584, + "loss": 1.2738, + "theoretical_loss": 3.45720019463067, + "tokens_seen": 2011627520 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031558676028084254, + "loss": 1.332, + "theoretical_loss": 3.4571908918512744, + "tokens_seen": 2011693056 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031557071213640924, + "loss": 1.328, + "theoretical_loss": 3.4571815894597897, + "tokens_seen": 2011758592 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031555466399197594, + "loss": 1.1391, + "theoretical_loss": 3.4571722874561868, + "tokens_seen": 2011824128 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031553861584754264, + "loss": 1.1817, + "theoretical_loss": 3.457162985840437, + "tokens_seen": 2011889664 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 673428, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3442015647888184, + "objective/train/theoretical_loss": 3.4571536846125115, + "objective/train/tokens_used": 2032415200, + "theoretical_loss": 3.4571536846125115, + "tokens_seen": 2011955200 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031552256770310934, + "loss": 1.3111, + "theoretical_loss": 3.4571536846125115, + "tokens_seen": 2011955200 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031550651955867604, + "loss": 1.361, + "theoretical_loss": 3.4571443837723814, + "tokens_seen": 2012020736 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031549047141424274, + "loss": 1.3822, + "theoretical_loss": 3.4571350833200185, + "tokens_seen": 2012086272 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031547442326980944, + "loss": 1.2538, + "theoretical_loss": 3.4571257832553934, + "tokens_seen": 2012151808 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031545837512537614, + "loss": 1.2017, + "theoretical_loss": 3.4571164835784773, + "tokens_seen": 2012217344 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031544232698094284, + "loss": 1.2188, + "theoretical_loss": 3.457107184289242, + "tokens_seen": 2012282880 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031542627883650954, + "loss": 1.2986, + "theoretical_loss": 3.457097885387658, + "tokens_seen": 2012348416 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031541023069207625, + "loss": 1.2914, + "theoretical_loss": 3.457088586873697, + "tokens_seen": 2012413952 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031539418254764295, + "loss": 1.2898, + "theoretical_loss": 3.4570792887473294, + "tokens_seen": 2012479488 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031537813440320965, + "loss": 1.2779, + "theoretical_loss": 3.4570699910085274, + "tokens_seen": 2012545024 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031536208625877635, + "loss": 1.2538, + "theoretical_loss": 3.457060693657262, + "tokens_seen": 2012610560 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031534603811434305, + "loss": 1.2666, + "theoretical_loss": 3.4570513966935046, + "tokens_seen": 2012676096 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031532998996990975, + "loss": 1.2307, + "theoretical_loss": 3.457042100117226, + "tokens_seen": 2012741632 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031531394182547645, + "loss": 1.1729, + "theoretical_loss": 3.457032803928397, + "tokens_seen": 2012807168 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031529789368104315, + "loss": 1.2087, + "theoretical_loss": 3.4570235081269898, + "tokens_seen": 2012872704 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031528184553660985, + "loss": 1.1964, + "theoretical_loss": 3.4570142127129753, + "tokens_seen": 2012938240 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031526579739217655, + "loss": 1.2167, + "theoretical_loss": 3.4570049176863247, + "tokens_seen": 2013003776 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031524974924774325, + "loss": 1.2488, + "theoretical_loss": 3.4569956230470087, + "tokens_seen": 2013069312 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031523370110330995, + "loss": 1.2628, + "theoretical_loss": 3.456986328795, + "tokens_seen": 2013134848 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031521765295887665, + "loss": 1.3149, + "theoretical_loss": 3.4569770349302686, + "tokens_seen": 2013200384 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031520160481444335, + "loss": 1.3399, + "theoretical_loss": 3.456967741452786, + "tokens_seen": 2013265920 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031518555667001005, + "loss": 1.2906, + "theoretical_loss": 3.4569584483625233, + "tokens_seen": 2013331456 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031516950852557675, + "loss": 1.2277, + "theoretical_loss": 3.4569491556594523, + "tokens_seen": 2013396992 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031515346038114346, + "loss": 1.2712, + "theoretical_loss": 3.4569398633435444, + "tokens_seen": 2013462528 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031513741223671016, + "loss": 1.2211, + "theoretical_loss": 3.4569305714147696, + "tokens_seen": 2013528064 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031512136409227686, + "loss": 1.1884, + "theoretical_loss": 3.4569212798731006, + "tokens_seen": 2013593600 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031510531594784356, + "loss": 1.2723, + "theoretical_loss": 3.456911988718508, + "tokens_seen": 2013659136 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031508926780341026, + "loss": 1.2175, + "theoretical_loss": 3.456902697950963, + "tokens_seen": 2013724672 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031507321965897696, + "loss": 1.3602, + "theoretical_loss": 3.456893407570438, + "tokens_seen": 2013790208 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031505717151454366, + "loss": 1.1215, + "theoretical_loss": 3.456884117576902, + "tokens_seen": 2013855744 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031504112337011036, + "loss": 1.2069, + "theoretical_loss": 3.4568748279703283, + "tokens_seen": 2013921280 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031502507522567706, + "loss": 1.2375, + "theoretical_loss": 3.4568655387506873, + "tokens_seen": 2013986816 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031500902708124376, + "loss": 1.2578, + "theoretical_loss": 3.4568562499179505, + "tokens_seen": 2014052352 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031499297893681046, + "loss": 1.3153, + "theoretical_loss": 3.4568469614720896, + "tokens_seen": 2014117888 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031497693079237716, + "loss": 1.2058, + "theoretical_loss": 3.4568376734130752, + "tokens_seen": 2014183424 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031496088264794386, + "loss": 1.2386, + "theoretical_loss": 3.456828385740879, + "tokens_seen": 2014248960 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031494483450351056, + "loss": 1.1387, + "theoretical_loss": 3.4568190984554725, + "tokens_seen": 2014314496 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031492878635907726, + "loss": 1.2466, + "theoretical_loss": 3.4568098115568264, + "tokens_seen": 2014380032 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031491273821464396, + "loss": 1.3329, + "theoretical_loss": 3.4568005250449123, + "tokens_seen": 2014445568 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031489669007021067, + "loss": 1.2391, + "theoretical_loss": 3.4567912389197017, + "tokens_seen": 2014511104 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031488064192577737, + "loss": 1.2908, + "theoretical_loss": 3.4567819531811654, + "tokens_seen": 2014576640 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031486459378134407, + "loss": 1.1989, + "theoretical_loss": 3.4567726678292754, + "tokens_seen": 2014642176 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031484854563691077, + "loss": 1.2283, + "theoretical_loss": 3.456763382864003, + "tokens_seen": 2014707712 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031483249749247747, + "loss": 1.3018, + "theoretical_loss": 3.456754098285319, + "tokens_seen": 2014773248 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031481644934804417, + "loss": 1.2549, + "theoretical_loss": 3.456744814093195, + "tokens_seen": 2014838784 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031480040120361087, + "loss": 1.34, + "theoretical_loss": 3.4567355302876024, + "tokens_seen": 2014904320 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031478435305917757, + "loss": 1.2595, + "theoretical_loss": 3.456726246868512, + "tokens_seen": 2014969856 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031476830491474427, + "loss": 1.2001, + "theoretical_loss": 3.456716963835896, + "tokens_seen": 2015035392 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031475225677031097, + "loss": 1.2267, + "theoretical_loss": 3.4567076811897253, + "tokens_seen": 2015100928 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031473620862587767, + "loss": 1.2498, + "theoretical_loss": 3.456698398929971, + "tokens_seen": 2015166464 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 674624, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2796549797058105, + "objective/train/theoretical_loss": 3.456689117056605, + "objective/train/tokens_used": 2035692000, + "theoretical_loss": 3.456689117056605, + "tokens_seen": 2015232000 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031472016048144437, + "loss": 1.1848, + "theoretical_loss": 3.456689117056605, + "tokens_seen": 2015232000 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003147041123370111, + "loss": 1.2098, + "theoretical_loss": 3.4566798355695987, + "tokens_seen": 2015297536 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003146880641925778, + "loss": 1.185, + "theoretical_loss": 3.4566705544689222, + "tokens_seen": 2015363072 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003146720160481444, + "loss": 1.3028, + "theoretical_loss": 3.4566612737545483, + "tokens_seen": 2015428608 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003146559679037111, + "loss": 1.2858, + "theoretical_loss": 3.4566519934264477, + "tokens_seen": 2015494144 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003146399197592778, + "loss": 1.2488, + "theoretical_loss": 3.4566427134845923, + "tokens_seen": 2015559680 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003146238716148445, + "loss": 1.2474, + "theoretical_loss": 3.456633433928953, + "tokens_seen": 2015625216 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003146078234704112, + "loss": 1.3715, + "theoretical_loss": 3.456624154759501, + "tokens_seen": 2015690752 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003145917753259779, + "loss": 1.2395, + "theoretical_loss": 3.456614875976208, + "tokens_seen": 2015756288 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003145757271815446, + "loss": 1.2891, + "theoretical_loss": 3.456605597579045, + "tokens_seen": 2015821824 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003145596790371113, + "loss": 1.2606, + "theoretical_loss": 3.456596319567984, + "tokens_seen": 2015887360 + }, + { + "epoch": 0.61, + "learning_rate": 0.000314543630892678, + "loss": 1.3007, + "theoretical_loss": 3.456587041942996, + "tokens_seen": 2015952896 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003145275827482447, + "loss": 1.2029, + "theoretical_loss": 3.4565777647040523, + "tokens_seen": 2016018432 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003145115346038114, + "loss": 1.1563, + "theoretical_loss": 3.4565684878511247, + "tokens_seen": 2016083968 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003144954864593781, + "loss": 1.2271, + "theoretical_loss": 3.4565592113841843, + "tokens_seen": 2016149504 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031447943831494483, + "loss": 1.2535, + "theoretical_loss": 3.456549935303202, + "tokens_seen": 2016215040 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031446339017051153, + "loss": 1.1762, + "theoretical_loss": 3.45654065960815, + "tokens_seen": 2016280576 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031444734202607823, + "loss": 1.2038, + "theoretical_loss": 3.4565313842989998, + "tokens_seen": 2016346112 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031443129388164493, + "loss": 1.1687, + "theoretical_loss": 3.4565221093757224, + "tokens_seen": 2016411648 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031441524573721163, + "loss": 1.3511, + "theoretical_loss": 3.4565128348382883, + "tokens_seen": 2016477184 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031439919759277833, + "loss": 1.296, + "theoretical_loss": 3.4565035606866705, + "tokens_seen": 2016542720 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031438314944834503, + "loss": 1.244, + "theoretical_loss": 3.4564942869208397, + "tokens_seen": 2016608256 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031436710130391173, + "loss": 1.2937, + "theoretical_loss": 3.4564850135407674, + "tokens_seen": 2016673792 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031435105315947843, + "loss": 1.25, + "theoretical_loss": 3.456475740546425, + "tokens_seen": 2016739328 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031433500501504513, + "loss": 1.3736, + "theoretical_loss": 3.4564664679377834, + "tokens_seen": 2016804864 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031431895687061183, + "loss": 1.391, + "theoretical_loss": 3.4564571957148154, + "tokens_seen": 2016870400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031430290872617853, + "loss": 1.2982, + "theoretical_loss": 3.456447923877491, + "tokens_seen": 2016935936 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031428686058174524, + "loss": 1.2943, + "theoretical_loss": 3.456438652425782, + "tokens_seen": 2017001472 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031427081243731194, + "loss": 1.315, + "theoretical_loss": 3.4564293813596603, + "tokens_seen": 2017067008 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031425476429287864, + "loss": 1.3477, + "theoretical_loss": 3.456420110679097, + "tokens_seen": 2017132544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031423871614844534, + "loss": 1.2892, + "theoretical_loss": 3.4564108403840637, + "tokens_seen": 2017198080 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031422266800401204, + "loss": 1.2892, + "theoretical_loss": 3.4564015704745312, + "tokens_seen": 2017263616 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031420661985957874, + "loss": 1.2969, + "theoretical_loss": 3.4563923009504722, + "tokens_seen": 2017329152 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031419057171514544, + "loss": 1.2524, + "theoretical_loss": 3.456383031811857, + "tokens_seen": 2017394688 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031417452357071214, + "loss": 1.2666, + "theoretical_loss": 3.4563737630586577, + "tokens_seen": 2017460224 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031415847542627884, + "loss": 1.2107, + "theoretical_loss": 3.4563644946908454, + "tokens_seen": 2017525760 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031414242728184554, + "loss": 1.1601, + "theoretical_loss": 3.456355226708392, + "tokens_seen": 2017591296 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031412637913741224, + "loss": 1.3043, + "theoretical_loss": 3.456345959111268, + "tokens_seen": 2017656832 + }, + { + "epoch": 0.61, + "learning_rate": 0.000314110330992979, + "loss": 1.3037, + "theoretical_loss": 3.4563366918994456, + "tokens_seen": 2017722368 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003140942828485457, + "loss": 1.3059, + "theoretical_loss": 3.456327425072897, + "tokens_seen": 2017787904 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003140782347041124, + "loss": 1.3482, + "theoretical_loss": 3.456318158631592, + "tokens_seen": 2017853440 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003140621865596791, + "loss": 1.3435, + "theoretical_loss": 3.4563088925755032, + "tokens_seen": 2017918976 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003140461384152458, + "loss": 1.3079, + "theoretical_loss": 3.456299626904602, + "tokens_seen": 2017984512 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031403009027081245, + "loss": 1.289, + "theoretical_loss": 3.4562903616188594, + "tokens_seen": 2018050048 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031401404212637915, + "loss": 1.2096, + "theoretical_loss": 3.4562810967182473, + "tokens_seen": 2018115584 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031399799398194585, + "loss": 1.2272, + "theoretical_loss": 3.456271832202737, + "tokens_seen": 2018181120 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031398194583751255, + "loss": 1.1956, + "theoretical_loss": 3.4562625680723, + "tokens_seen": 2018246656 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031396589769307925, + "loss": 1.2056, + "theoretical_loss": 3.456253304326908, + "tokens_seen": 2018312192 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031394984954864595, + "loss": 1.3314, + "theoretical_loss": 3.4562440409665323, + "tokens_seen": 2018377728 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031393380140421265, + "loss": 1.2872, + "theoretical_loss": 3.456234777991144, + "tokens_seen": 2018443264 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 675599, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.253434419631958, + "objective/train/theoretical_loss": 3.4562255154007158, + "objective/train/tokens_used": 2038968800, + "theoretical_loss": 3.4562255154007158, + "tokens_seen": 2018508800 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031391775325977935, + "loss": 1.3645, + "theoretical_loss": 3.4562255154007158, + "tokens_seen": 2018508800 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031390170511534605, + "loss": 1.2885, + "theoretical_loss": 3.4562162531952176, + "tokens_seen": 2018574336 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031388565697091275, + "loss": 1.199, + "theoretical_loss": 3.4562069913746223, + "tokens_seen": 2018639872 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031386960882647945, + "loss": 1.3032, + "theoretical_loss": 3.456197729938901, + "tokens_seen": 2018705408 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031385356068204615, + "loss": 1.4003, + "theoretical_loss": 3.4561884688880244, + "tokens_seen": 2018770944 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031383751253761285, + "loss": 1.2505, + "theoretical_loss": 3.4561792082219647, + "tokens_seen": 2018836480 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031382146439317955, + "loss": 1.1888, + "theoretical_loss": 3.456169947940694, + "tokens_seen": 2018902016 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031380541624874625, + "loss": 1.2354, + "theoretical_loss": 3.4561606880441826, + "tokens_seen": 2018967552 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031378936810431295, + "loss": 1.2211, + "theoretical_loss": 3.4561514285324026, + "tokens_seen": 2019033088 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031377331995987966, + "loss": 1.3486, + "theoretical_loss": 3.456142169405326, + "tokens_seen": 2019098624 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031375727181544636, + "loss": 1.2352, + "theoretical_loss": 3.456132910662924, + "tokens_seen": 2019164160 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031374122367101306, + "loss": 1.145, + "theoretical_loss": 3.456123652305168, + "tokens_seen": 2019229696 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031372517552657976, + "loss": 1.3951, + "theoretical_loss": 3.4561143943320287, + "tokens_seen": 2019295232 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031370912738214646, + "loss": 1.3111, + "theoretical_loss": 3.4561051367434796, + "tokens_seen": 2019360768 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031369307923771316, + "loss": 1.1926, + "theoretical_loss": 3.4560958795394905, + "tokens_seen": 2019426304 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031367703109327986, + "loss": 1.3605, + "theoretical_loss": 3.4560866227200338, + "tokens_seen": 2019491840 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031366098294884656, + "loss": 1.2575, + "theoretical_loss": 3.456077366285081, + "tokens_seen": 2019557376 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031364493480441326, + "loss": 1.2603, + "theoretical_loss": 3.4560681102346034, + "tokens_seen": 2019622912 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031362888665997996, + "loss": 1.2136, + "theoretical_loss": 3.4560588545685724, + "tokens_seen": 2019688448 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031361283851554666, + "loss": 1.1928, + "theoretical_loss": 3.4560495992869598, + "tokens_seen": 2019753984 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031359679037111336, + "loss": 1.2462, + "theoretical_loss": 3.4560403443897374, + "tokens_seen": 2019819520 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031358074222668006, + "loss": 1.3372, + "theoretical_loss": 3.456031089876877, + "tokens_seen": 2019885056 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031356469408224676, + "loss": 1.3215, + "theoretical_loss": 3.4560218357483485, + "tokens_seen": 2019950592 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031354864593781346, + "loss": 1.3099, + "theoretical_loss": 3.4560125820041256, + "tokens_seen": 2020016128 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031353259779338017, + "loss": 1.3239, + "theoretical_loss": 3.4560033286441785, + "tokens_seen": 2020081664 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031351654964894687, + "loss": 1.2715, + "theoretical_loss": 3.45599407566848, + "tokens_seen": 2020147200 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031350050150451357, + "loss": 1.1761, + "theoretical_loss": 3.4559848230770003, + "tokens_seen": 2020212736 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031348445336008027, + "loss": 1.3067, + "theoretical_loss": 3.455975570869711, + "tokens_seen": 2020278272 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031346840521564697, + "loss": 1.3007, + "theoretical_loss": 3.4559663190465852, + "tokens_seen": 2020343808 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031345235707121367, + "loss": 1.3201, + "theoretical_loss": 3.455957067607593, + "tokens_seen": 2020409344 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031343630892678037, + "loss": 1.1481, + "theoretical_loss": 3.455947816552707, + "tokens_seen": 2020474880 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031342026078234707, + "loss": 1.3199, + "theoretical_loss": 3.4559385658818984, + "tokens_seen": 2020540416 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031340421263791377, + "loss": 1.2353, + "theoretical_loss": 3.4559293155951383, + "tokens_seen": 2020605952 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031338816449348047, + "loss": 1.2676, + "theoretical_loss": 3.455920065692399, + "tokens_seen": 2020671488 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031337211634904717, + "loss": 1.258, + "theoretical_loss": 3.4559108161736516, + "tokens_seen": 2020737024 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031335606820461387, + "loss": 1.2013, + "theoretical_loss": 3.455901567038868, + "tokens_seen": 2020802560 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031334002006018057, + "loss": 1.2853, + "theoretical_loss": 3.45589231828802, + "tokens_seen": 2020868096 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003133239719157473, + "loss": 1.2827, + "theoretical_loss": 3.4558830699210787, + "tokens_seen": 2020933632 + }, + { + "epoch": 0.61, + "learning_rate": 0.000313307923771314, + "loss": 1.3942, + "theoretical_loss": 3.455873821938016, + "tokens_seen": 2020999168 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003132918756268807, + "loss": 1.1987, + "theoretical_loss": 3.4558645743388037, + "tokens_seen": 2021064704 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003132758274824474, + "loss": 1.2831, + "theoretical_loss": 3.4558553271234134, + "tokens_seen": 2021130240 + }, + { + "epoch": 0.61, + "learning_rate": 0.000313259779338014, + "loss": 1.2775, + "theoretical_loss": 3.455846080291816, + "tokens_seen": 2021195776 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003132437311935807, + "loss": 1.199, + "theoretical_loss": 3.455836833843984, + "tokens_seen": 2021261312 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003132276830491474, + "loss": 1.2561, + "theoretical_loss": 3.455827587779889, + "tokens_seen": 2021326848 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003132116349047141, + "loss": 1.1722, + "theoretical_loss": 3.4558183420995015, + "tokens_seen": 2021392384 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003131955867602808, + "loss": 1.3859, + "theoretical_loss": 3.4558090968027946, + "tokens_seen": 2021457920 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003131795386158475, + "loss": 1.2778, + "theoretical_loss": 3.4557998518897395, + "tokens_seen": 2021523456 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003131634904714142, + "loss": 1.2392, + "theoretical_loss": 3.4557906073603073, + "tokens_seen": 2021588992 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003131474423269809, + "loss": 1.2568, + "theoretical_loss": 3.45578136321447, + "tokens_seen": 2021654528 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003131313941825476, + "loss": 1.3003, + "theoretical_loss": 3.4557721194521998, + "tokens_seen": 2021720064 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 676834, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.277672290802002, + "objective/train/theoretical_loss": 3.455762876073467, + "objective/train/tokens_used": 2042245600, + "theoretical_loss": 3.455762876073467, + "tokens_seen": 2021785600 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031311534603811433, + "loss": 1.2835, + "theoretical_loss": 3.455762876073467, + "tokens_seen": 2021785600 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031309929789368103, + "loss": 1.2825, + "theoretical_loss": 3.4557536330782446, + "tokens_seen": 2021851136 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031308324974924773, + "loss": 1.2647, + "theoretical_loss": 3.4557443904665037, + "tokens_seen": 2021916672 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031306720160481443, + "loss": 1.2218, + "theoretical_loss": 3.4557351482382153, + "tokens_seen": 2021982208 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031305115346038113, + "loss": 1.3394, + "theoretical_loss": 3.4557259063933525, + "tokens_seen": 2022047744 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031303510531594783, + "loss": 1.2657, + "theoretical_loss": 3.4557166649318862, + "tokens_seen": 2022113280 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031301905717151453, + "loss": 1.1825, + "theoretical_loss": 3.455707423853788, + "tokens_seen": 2022178816 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031300300902708123, + "loss": 1.2714, + "theoretical_loss": 3.455698183159029, + "tokens_seen": 2022244352 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031298696088264793, + "loss": 1.2884, + "theoretical_loss": 3.4556889428475825, + "tokens_seen": 2022309888 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031297091273821463, + "loss": 1.2069, + "theoretical_loss": 3.4556797029194186, + "tokens_seen": 2022375424 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031295486459378133, + "loss": 1.2378, + "theoretical_loss": 3.45567046337451, + "tokens_seen": 2022440960 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031293881644934803, + "loss": 1.3573, + "theoretical_loss": 3.455661224212828, + "tokens_seen": 2022506496 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031292276830491473, + "loss": 1.2299, + "theoretical_loss": 3.4556519854343435, + "tokens_seen": 2022572032 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003129067201604815, + "loss": 1.3058, + "theoretical_loss": 3.45564274703903, + "tokens_seen": 2022637568 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003128906720160482, + "loss": 1.2403, + "theoretical_loss": 3.4556335090268573, + "tokens_seen": 2022703104 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003128746238716149, + "loss": 1.2602, + "theoretical_loss": 3.4556242713977983, + "tokens_seen": 2022768640 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003128585757271816, + "loss": 1.2551, + "theoretical_loss": 3.4556150341518244, + "tokens_seen": 2022834176 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003128425275827483, + "loss": 1.3021, + "theoretical_loss": 3.455605797288907, + "tokens_seen": 2022899712 + }, + { + "epoch": 0.61, + "learning_rate": 0.000312826479438315, + "loss": 1.2694, + "theoretical_loss": 3.4555965608090187, + "tokens_seen": 2022965248 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003128104312938817, + "loss": 1.3417, + "theoretical_loss": 3.4555873247121296, + "tokens_seen": 2023030784 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003127943831494484, + "loss": 1.3842, + "theoretical_loss": 3.4555780889982133, + "tokens_seen": 2023096320 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003127783350050151, + "loss": 1.2525, + "theoretical_loss": 3.45556885366724, + "tokens_seen": 2023161856 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003127622868605818, + "loss": 1.2779, + "theoretical_loss": 3.455559618719182, + "tokens_seen": 2023227392 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003127462387161485, + "loss": 1.2569, + "theoretical_loss": 3.4555503841540114, + "tokens_seen": 2023292928 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003127301905717152, + "loss": 1.2426, + "theoretical_loss": 3.4555411499716993, + "tokens_seen": 2023358464 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003127141424272819, + "loss": 1.3509, + "theoretical_loss": 3.4555319161722178, + "tokens_seen": 2023424000 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003126980942828486, + "loss": 1.1984, + "theoretical_loss": 3.4555226827555385, + "tokens_seen": 2023489536 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003126820461384153, + "loss": 1.2994, + "theoretical_loss": 3.455513449721633, + "tokens_seen": 2023555072 + }, + { + "epoch": 0.61, + "learning_rate": 0.000312665997993982, + "loss": 1.1901, + "theoretical_loss": 3.4555042170704735, + "tokens_seen": 2023620608 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003126499498495487, + "loss": 1.2047, + "theoretical_loss": 3.4554949848020313, + "tokens_seen": 2023686144 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031263390170511535, + "loss": 1.3177, + "theoretical_loss": 3.455485752916278, + "tokens_seen": 2023751680 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031261785356068205, + "loss": 1.2972, + "theoretical_loss": 3.455476521413186, + "tokens_seen": 2023817216 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031260180541624875, + "loss": 1.1906, + "theoretical_loss": 3.4554672902927264, + "tokens_seen": 2023882752 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031258575727181545, + "loss": 1.1792, + "theoretical_loss": 3.4554580595548714, + "tokens_seen": 2023948288 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031256970912738215, + "loss": 1.1914, + "theoretical_loss": 3.455448829199592, + "tokens_seen": 2024013824 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031255366098294885, + "loss": 1.2878, + "theoretical_loss": 3.455439599226861, + "tokens_seen": 2024079360 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031253761283851555, + "loss": 1.2059, + "theoretical_loss": 3.45543036963665, + "tokens_seen": 2024144896 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031252156469408225, + "loss": 1.29, + "theoretical_loss": 3.45542114042893, + "tokens_seen": 2024210432 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031250551654964895, + "loss": 1.2594, + "theoretical_loss": 3.4554119116036732, + "tokens_seen": 2024275968 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031248946840521565, + "loss": 1.2785, + "theoretical_loss": 3.4554026831608518, + "tokens_seen": 2024341504 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031247342026078235, + "loss": 1.2444, + "theoretical_loss": 3.4553934551004364, + "tokens_seen": 2024407040 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031245737211634905, + "loss": 1.3177, + "theoretical_loss": 3.4553842274224005, + "tokens_seen": 2024472576 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031244132397191575, + "loss": 1.2194, + "theoretical_loss": 3.455375000126714, + "tokens_seen": 2024538112 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031242527582748245, + "loss": 1.2977, + "theoretical_loss": 3.45536577321335, + "tokens_seen": 2024603648 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031240922768304916, + "loss": 1.2189, + "theoretical_loss": 3.45535654668228, + "tokens_seen": 2024669184 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031239317953861586, + "loss": 1.2709, + "theoretical_loss": 3.455347320533475, + "tokens_seen": 2024734720 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031237713139418256, + "loss": 1.2646, + "theoretical_loss": 3.455338094766908, + "tokens_seen": 2024800256 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031236108324974926, + "loss": 1.3225, + "theoretical_loss": 3.45532886938255, + "tokens_seen": 2024865792 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031234503510531596, + "loss": 1.3044, + "theoretical_loss": 3.4553196443803733, + "tokens_seen": 2024931328 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031232898696088266, + "loss": 1.201, + "theoretical_loss": 3.455310419760349, + "tokens_seen": 2024996864 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 678168, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2667977809906006, + "objective/train/theoretical_loss": 3.4553011955224497, + "objective/train/tokens_used": 2045522400, + "theoretical_loss": 3.4553011955224497, + "tokens_seen": 2025062400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031231293881644936, + "loss": 1.3154, + "theoretical_loss": 3.4553011955224497, + "tokens_seen": 2025062400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031229689067201606, + "loss": 1.1993, + "theoretical_loss": 3.455291971666647, + "tokens_seen": 2025127936 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031228084252758276, + "loss": 1.3423, + "theoretical_loss": 3.4552827481929125, + "tokens_seen": 2025193472 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031226479438314946, + "loss": 1.3148, + "theoretical_loss": 3.4552735251012177, + "tokens_seen": 2025259008 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031224874623871616, + "loss": 1.3603, + "theoretical_loss": 3.4552643023915346, + "tokens_seen": 2025324544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031223269809428286, + "loss": 1.3334, + "theoretical_loss": 3.4552550800638357, + "tokens_seen": 2025390080 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031221664994984956, + "loss": 1.2204, + "theoretical_loss": 3.4552458581180923, + "tokens_seen": 2025455616 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031220060180541626, + "loss": 1.2288, + "theoretical_loss": 3.455236636554276, + "tokens_seen": 2025521152 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031218455366098296, + "loss": 1.3527, + "theoretical_loss": 3.455227415372359, + "tokens_seen": 2025586688 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031216850551654966, + "loss": 1.1527, + "theoretical_loss": 3.455218194572313, + "tokens_seen": 2025652224 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031215245737211637, + "loss": 1.2404, + "theoretical_loss": 3.4552089741541097, + "tokens_seen": 2025717760 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031213640922768307, + "loss": 1.2347, + "theoretical_loss": 3.4551997541177206, + "tokens_seen": 2025783296 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031212036108324977, + "loss": 1.2124, + "theoretical_loss": 3.455190534463119, + "tokens_seen": 2025848832 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031210431293881647, + "loss": 1.1825, + "theoretical_loss": 3.4551813151902753, + "tokens_seen": 2025914368 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031208826479438317, + "loss": 1.1782, + "theoretical_loss": 3.4551720962991617, + "tokens_seen": 2025979904 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031207221664994987, + "loss": 1.205, + "theoretical_loss": 3.45516287778975, + "tokens_seen": 2026045440 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031205616850551657, + "loss": 1.3805, + "theoretical_loss": 3.4551536596620123, + "tokens_seen": 2026110976 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031204012036108327, + "loss": 1.3957, + "theoretical_loss": 3.45514444191592, + "tokens_seen": 2026176512 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031202407221664997, + "loss": 1.1958, + "theoretical_loss": 3.4551352245514453, + "tokens_seen": 2026242048 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031200802407221667, + "loss": 1.3274, + "theoretical_loss": 3.455126007568561, + "tokens_seen": 2026307584 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031199197592778337, + "loss": 1.2468, + "theoretical_loss": 3.455116790967237, + "tokens_seen": 2026373120 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031197592778335007, + "loss": 1.1995, + "theoretical_loss": 3.4551075747474465, + "tokens_seen": 2026438656 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003119598796389168, + "loss": 1.2539, + "theoretical_loss": 3.455098358909161, + "tokens_seen": 2026504192 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003119438314944835, + "loss": 1.2121, + "theoretical_loss": 3.4550891434523523, + "tokens_seen": 2026569728 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003119277833500502, + "loss": 1.202, + "theoretical_loss": 3.4550799283769926, + "tokens_seen": 2026635264 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003119117352056169, + "loss": 1.2713, + "theoretical_loss": 3.4550707136830536, + "tokens_seen": 2026700800 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003118956870611836, + "loss": 1.2331, + "theoretical_loss": 3.4550614993705073, + "tokens_seen": 2026766336 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003118796389167503, + "loss": 1.2344, + "theoretical_loss": 3.455052285439325, + "tokens_seen": 2026831872 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003118635907723169, + "loss": 1.2727, + "theoretical_loss": 3.455043071889479, + "tokens_seen": 2026897408 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003118475426278836, + "loss": 1.2442, + "theoretical_loss": 3.4550338587209413, + "tokens_seen": 2026962944 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003118314944834503, + "loss": 1.2247, + "theoretical_loss": 3.4550246459336833, + "tokens_seen": 2027028480 + }, + { + "epoch": 0.61, + "learning_rate": 0.000311815446339017, + "loss": 1.3285, + "theoretical_loss": 3.455015433527678, + "tokens_seen": 2027094016 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003117993981945837, + "loss": 1.3078, + "theoretical_loss": 3.455006221502896, + "tokens_seen": 2027159552 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003117833500501504, + "loss": 1.1593, + "theoretical_loss": 3.4549970098593104, + "tokens_seen": 2027225088 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003117673019057171, + "loss": 1.2019, + "theoretical_loss": 3.4549877985968918, + "tokens_seen": 2027290624 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003117512537612838, + "loss": 1.2882, + "theoretical_loss": 3.4549785877156136, + "tokens_seen": 2027356160 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031173520561685053, + "loss": 1.3323, + "theoretical_loss": 3.454969377215446, + "tokens_seen": 2027421696 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031171915747241723, + "loss": 1.25, + "theoretical_loss": 3.4549601670963623, + "tokens_seen": 2027487232 + }, + { + "epoch": 0.61, + "learning_rate": 0.000311703109327984, + "loss": 1.2143, + "theoretical_loss": 3.4549509573583337, + "tokens_seen": 2027552768 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003116870611835507, + "loss": 1.2289, + "theoretical_loss": 3.4549417480013327, + "tokens_seen": 2027618304 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003116710130391174, + "loss": 1.2829, + "theoretical_loss": 3.4549325390253305, + "tokens_seen": 2027683840 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003116549648946841, + "loss": 1.1596, + "theoretical_loss": 3.4549233304302995, + "tokens_seen": 2027749376 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003116389167502508, + "loss": 1.3987, + "theoretical_loss": 3.454914122216212, + "tokens_seen": 2027814912 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003116228686058175, + "loss": 1.2311, + "theoretical_loss": 3.4549049143830386, + "tokens_seen": 2027880448 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003116068204613842, + "loss": 1.3269, + "theoretical_loss": 3.4548957069307527, + "tokens_seen": 2027945984 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003115907723169509, + "loss": 1.3215, + "theoretical_loss": 3.454886499859325, + "tokens_seen": 2028011520 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003115747241725176, + "loss": 1.2291, + "theoretical_loss": 3.4548772931687286, + "tokens_seen": 2028077056 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003115586760280843, + "loss": 1.3623, + "theoretical_loss": 3.4548680868589345, + "tokens_seen": 2028142592 + }, + { + "epoch": 0.61, + "learning_rate": 0.000311542627883651, + "loss": 1.3173, + "theoretical_loss": 3.4548588809299154, + "tokens_seen": 2028208128 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003115265797392177, + "loss": 1.2773, + "theoretical_loss": 3.454849675381643, + "tokens_seen": 2028273664 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 679103, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3795264959335327, + "objective/train/theoretical_loss": 3.454840470214089, + "objective/train/tokens_used": 2048799200, + "theoretical_loss": 3.454840470214089, + "tokens_seen": 2028339200 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003115105315947844, + "loss": 1.2951, + "theoretical_loss": 3.454840470214089, + "tokens_seen": 2028339200 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003114944834503511, + "loss": 1.2206, + "theoretical_loss": 3.454831265427225, + "tokens_seen": 2028404736 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003114784353059178, + "loss": 1.2835, + "theoretical_loss": 3.454822061021024, + "tokens_seen": 2028470272 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003114623871614845, + "loss": 1.2287, + "theoretical_loss": 3.4548128569954573, + "tokens_seen": 2028535808 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003114463390170512, + "loss": 1.2621, + "theoretical_loss": 3.454803653350497, + "tokens_seen": 2028601344 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003114302908726179, + "loss": 1.1892, + "theoretical_loss": 3.4547944500861147, + "tokens_seen": 2028666880 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003114142427281846, + "loss": 1.2138, + "theoretical_loss": 3.454785247202283, + "tokens_seen": 2028732416 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003113981945837513, + "loss": 1.2948, + "theoretical_loss": 3.4547760446989737, + "tokens_seen": 2028797952 + }, + { + "epoch": 0.61, + "learning_rate": 0.000311382146439318, + "loss": 1.2765, + "theoretical_loss": 3.4547668425761584, + "tokens_seen": 2028863488 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003113660982948847, + "loss": 1.3274, + "theoretical_loss": 3.45475764083381, + "tokens_seen": 2028929024 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003113500501504514, + "loss": 1.3042, + "theoretical_loss": 3.454748439471899, + "tokens_seen": 2028994560 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003113340020060181, + "loss": 1.1862, + "theoretical_loss": 3.454739238490398, + "tokens_seen": 2029060096 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003113179538615848, + "loss": 1.3196, + "theoretical_loss": 3.45473003788928, + "tokens_seen": 2029125632 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003113019057171515, + "loss": 1.1663, + "theoretical_loss": 3.454720837668516, + "tokens_seen": 2029191168 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003112858575727182, + "loss": 1.2578, + "theoretical_loss": 3.454711637828078, + "tokens_seen": 2029256704 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003112698094282849, + "loss": 1.2119, + "theoretical_loss": 3.454702438367938, + "tokens_seen": 2029322240 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003112537612838516, + "loss": 1.2571, + "theoretical_loss": 3.4546932392880683, + "tokens_seen": 2029387776 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031123771313941825, + "loss": 1.1998, + "theoretical_loss": 3.454684040588441, + "tokens_seen": 2029453312 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031122166499498495, + "loss": 1.2048, + "theoretical_loss": 3.4546748422690277, + "tokens_seen": 2029518848 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031120561685055165, + "loss": 1.2493, + "theoretical_loss": 3.4546656443298005, + "tokens_seen": 2029584384 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031118956870611835, + "loss": 1.187, + "theoretical_loss": 3.4546564467707315, + "tokens_seen": 2029649920 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031117352056168505, + "loss": 1.3663, + "theoretical_loss": 3.454647249591793, + "tokens_seen": 2029715456 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031115747241725175, + "loss": 1.3059, + "theoretical_loss": 3.454638052792956, + "tokens_seen": 2029780992 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031114142427281845, + "loss": 1.3897, + "theoretical_loss": 3.454628856374194, + "tokens_seen": 2029846528 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031112537612838515, + "loss": 1.3425, + "theoretical_loss": 3.454619660335478, + "tokens_seen": 2029912064 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031110932798395185, + "loss": 1.261, + "theoretical_loss": 3.45461046467678, + "tokens_seen": 2029977600 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031109327983951855, + "loss": 1.2895, + "theoretical_loss": 3.4546012693980725, + "tokens_seen": 2030043136 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031107723169508525, + "loss": 1.2439, + "theoretical_loss": 3.4545920744993275, + "tokens_seen": 2030108672 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031106118355065195, + "loss": 1.2184, + "theoretical_loss": 3.4545828799805167, + "tokens_seen": 2030174208 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031104513540621866, + "loss": 1.2177, + "theoretical_loss": 3.4545736858416127, + "tokens_seen": 2030239744 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031102908726178536, + "loss": 1.2373, + "theoretical_loss": 3.4545644920825866, + "tokens_seen": 2030305280 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031101303911735206, + "loss": 1.2976, + "theoretical_loss": 3.4545552987034114, + "tokens_seen": 2030370816 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031099699097291876, + "loss": 1.2443, + "theoretical_loss": 3.4545461057040585, + "tokens_seen": 2030436352 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031098094282848546, + "loss": 1.2184, + "theoretical_loss": 3.4545369130845, + "tokens_seen": 2030501888 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031096489468405216, + "loss": 1.2728, + "theoretical_loss": 3.454527720844709, + "tokens_seen": 2030567424 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031094884653961886, + "loss": 1.2414, + "theoretical_loss": 3.454518528984656, + "tokens_seen": 2030632960 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031093279839518556, + "loss": 1.3182, + "theoretical_loss": 3.4545093375043137, + "tokens_seen": 2030698496 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031091675025075226, + "loss": 1.264, + "theoretical_loss": 3.4545001464036544, + "tokens_seen": 2030764032 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031090070210631896, + "loss": 1.2105, + "theoretical_loss": 3.45449095568265, + "tokens_seen": 2030829568 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031090070210631896, + "loss": 1.145, + "theoretical_loss": 3.4544817653412725, + "tokens_seen": 2030895104 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031088465396188566, + "loss": 1.2857, + "theoretical_loss": 3.4544725753794943, + "tokens_seen": 2030960640 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031086860581745236, + "loss": 1.2781, + "theoretical_loss": 3.4544633857972866, + "tokens_seen": 2031026176 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031085255767301906, + "loss": 1.2642, + "theoretical_loss": 3.4544541965946225, + "tokens_seen": 2031091712 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031083650952858576, + "loss": 1.1627, + "theoretical_loss": 3.454445007771474, + "tokens_seen": 2031157248 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031082046138415246, + "loss": 1.2297, + "theoretical_loss": 3.454435819327812, + "tokens_seen": 2031222784 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031080441323971916, + "loss": 1.3006, + "theoretical_loss": 3.45442663126361, + "tokens_seen": 2031288320 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031078836509528587, + "loss": 1.2529, + "theoretical_loss": 3.454417443578839, + "tokens_seen": 2031353856 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031077231695085257, + "loss": 1.1862, + "theoretical_loss": 3.4544082562734717, + "tokens_seen": 2031419392 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031075626880641927, + "loss": 1.2043, + "theoretical_loss": 3.4543990693474806, + "tokens_seen": 2031484928 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031074022066198597, + "loss": 1.2927, + "theoretical_loss": 3.454389882800837, + "tokens_seen": 2031550464 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 680422, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2129366397857666, + "objective/train/theoretical_loss": 3.454380696633513, + "objective/train/tokens_used": 2052076000, + "theoretical_loss": 3.454380696633513, + "tokens_seen": 2031616000 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031072417251755267, + "loss": 1.2718, + "theoretical_loss": 3.454380696633513, + "tokens_seen": 2031616000 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031070812437311937, + "loss": 1.3017, + "theoretical_loss": 3.4543715108454816, + "tokens_seen": 2031681536 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031069207622868607, + "loss": 1.2935, + "theoretical_loss": 3.4543623254367137, + "tokens_seen": 2031747072 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031067602808425277, + "loss": 1.3038, + "theoretical_loss": 3.454353140407182, + "tokens_seen": 2031812608 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031065997993981947, + "loss": 1.2507, + "theoretical_loss": 3.454343955756859, + "tokens_seen": 2031878144 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031064393179538617, + "loss": 1.3444, + "theoretical_loss": 3.4543347714857164, + "tokens_seen": 2031943680 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031062788365095287, + "loss": 1.2736, + "theoretical_loss": 3.454325587593726, + "tokens_seen": 2032009216 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031061183550651957, + "loss": 1.2639, + "theoretical_loss": 3.4543164040808603, + "tokens_seen": 2032074752 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031059578736208627, + "loss": 1.4047, + "theoretical_loss": 3.4543072209470917, + "tokens_seen": 2032140288 + }, + { + "epoch": 0.62, + "learning_rate": 0.000310579739217653, + "loss": 1.2556, + "theoretical_loss": 3.4542980381923916, + "tokens_seen": 2032205824 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003105636910732197, + "loss": 1.2748, + "theoretical_loss": 3.454288855816733, + "tokens_seen": 2032271360 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003105476429287864, + "loss": 1.237, + "theoretical_loss": 3.4542796738200874, + "tokens_seen": 2032336896 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003105315947843531, + "loss": 1.2959, + "theoretical_loss": 3.4542704922024265, + "tokens_seen": 2032402432 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003105155466399198, + "loss": 1.1759, + "theoretical_loss": 3.454261310963724, + "tokens_seen": 2032467968 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003104994984954865, + "loss": 1.2335, + "theoretical_loss": 3.45425213010395, + "tokens_seen": 2032533504 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003104834503510532, + "loss": 1.1578, + "theoretical_loss": 3.4542429496230787, + "tokens_seen": 2032599040 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003104674022066199, + "loss": 1.2512, + "theoretical_loss": 3.454233769521081, + "tokens_seen": 2032664576 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003104513540621866, + "loss": 1.2635, + "theoretical_loss": 3.454224589797929, + "tokens_seen": 2032730112 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003104353059177533, + "loss": 1.2513, + "theoretical_loss": 3.4542154104535956, + "tokens_seen": 2032795648 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031041925777332, + "loss": 1.2434, + "theoretical_loss": 3.4542062314880524, + "tokens_seen": 2032861184 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003104032096288867, + "loss": 1.2282, + "theoretical_loss": 3.4541970529012715, + "tokens_seen": 2032926720 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003103871614844534, + "loss": 1.3071, + "theoretical_loss": 3.4541878746932255, + "tokens_seen": 2032992256 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003103711133400201, + "loss": 1.327, + "theoretical_loss": 3.4541786968638863, + "tokens_seen": 2033057792 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003103550651955868, + "loss": 1.334, + "theoretical_loss": 3.4541695194132256, + "tokens_seen": 2033123328 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003103390170511535, + "loss": 1.2841, + "theoretical_loss": 3.4541603423412166, + "tokens_seen": 2033188864 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003103229689067202, + "loss": 1.2503, + "theoretical_loss": 3.454151165647831, + "tokens_seen": 2033254400 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003103069207622869, + "loss": 1.1638, + "theoretical_loss": 3.4541419893330403, + "tokens_seen": 2033319936 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003102908726178536, + "loss": 1.2743, + "theoretical_loss": 3.454132813396818, + "tokens_seen": 2033385472 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003102748244734203, + "loss": 1.3382, + "theoretical_loss": 3.4541236378391345, + "tokens_seen": 2033451008 + }, + { + "epoch": 0.62, + "learning_rate": 0.000310258776328987, + "loss": 1.3748, + "theoretical_loss": 3.454114462659964, + "tokens_seen": 2033516544 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003102427281845537, + "loss": 1.1973, + "theoretical_loss": 3.454105287859277, + "tokens_seen": 2033582080 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003102266800401204, + "loss": 1.2616, + "theoretical_loss": 3.454096113437047, + "tokens_seen": 2033647616 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003102106318956871, + "loss": 1.27, + "theoretical_loss": 3.454086939393245, + "tokens_seen": 2033713152 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003101945837512538, + "loss": 1.214, + "theoretical_loss": 3.4540777657278445, + "tokens_seen": 2033778688 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003101785356068205, + "loss": 1.2816, + "theoretical_loss": 3.4540685924408168, + "tokens_seen": 2033844224 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003101624874623872, + "loss": 1.3315, + "theoretical_loss": 3.4540594195321344, + "tokens_seen": 2033909760 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003101464393179539, + "loss": 1.2739, + "theoretical_loss": 3.4540502470017693, + "tokens_seen": 2033975296 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003101303911735206, + "loss": 1.3093, + "theoretical_loss": 3.4540410748496937, + "tokens_seen": 2034040832 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003101143430290873, + "loss": 1.2014, + "theoretical_loss": 3.4540319030758804, + "tokens_seen": 2034106368 + }, + { + "epoch": 0.62, + "learning_rate": 0.000310098294884654, + "loss": 1.3909, + "theoretical_loss": 3.4540227316803005, + "tokens_seen": 2034171904 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003100822467402207, + "loss": 1.2074, + "theoretical_loss": 3.4540135606629274, + "tokens_seen": 2034237440 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003100661985957874, + "loss": 1.2799, + "theoretical_loss": 3.4540043900237327, + "tokens_seen": 2034302976 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003100501504513541, + "loss": 1.3112, + "theoretical_loss": 3.453995219762688, + "tokens_seen": 2034368512 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003100341023069208, + "loss": 1.1292, + "theoretical_loss": 3.453986049879767, + "tokens_seen": 2034434048 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003100180541624875, + "loss": 1.3172, + "theoretical_loss": 3.453976880374941, + "tokens_seen": 2034499584 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003100020060180542, + "loss": 1.2884, + "theoretical_loss": 3.4539677112481826, + "tokens_seen": 2034565120 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003099859578736209, + "loss": 1.2518, + "theoretical_loss": 3.453958542499463, + "tokens_seen": 2034630656 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003099699097291876, + "loss": 1.2859, + "theoretical_loss": 3.453949374128756, + "tokens_seen": 2034696192 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003099538615847543, + "loss": 1.2139, + "theoretical_loss": 3.453940206136033, + "tokens_seen": 2034761728 + }, + { + "epoch": 0.62, + "learning_rate": 0.000309937813440321, + "loss": 1.4163, + "theoretical_loss": 3.453931038521266, + "tokens_seen": 2034827264 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 681328, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.229004144668579, + "objective/train/theoretical_loss": 3.453921871284428, + "objective/train/tokens_used": 2055352800, + "theoretical_loss": 3.453921871284428, + "tokens_seen": 2034892800 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003099217652958877, + "loss": 1.1501, + "theoretical_loss": 3.453921871284428, + "tokens_seen": 2034892800 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003099057171514544, + "loss": 1.1942, + "theoretical_loss": 3.453912704425491, + "tokens_seen": 2034958336 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003098896690070211, + "loss": 1.346, + "theoretical_loss": 3.4539035379444267, + "tokens_seen": 2035023872 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003098736208625878, + "loss": 1.2499, + "theoretical_loss": 3.4538943718412076, + "tokens_seen": 2035089408 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003098575727181545, + "loss": 1.2287, + "theoretical_loss": 3.4538852061158067, + "tokens_seen": 2035154944 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003098415245737212, + "loss": 1.2177, + "theoretical_loss": 3.453876040768195, + "tokens_seen": 2035220480 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030982547642928785, + "loss": 1.2186, + "theoretical_loss": 3.4538668757983464, + "tokens_seen": 2035286016 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030980942828485455, + "loss": 1.2747, + "theoretical_loss": 3.4538577112062314, + "tokens_seen": 2035351552 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030979338014042125, + "loss": 1.3151, + "theoretical_loss": 3.453848546991823, + "tokens_seen": 2035417088 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030977733199598795, + "loss": 1.291, + "theoretical_loss": 3.453839383155094, + "tokens_seen": 2035482624 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030976128385155465, + "loss": 1.2808, + "theoretical_loss": 3.453830219696016, + "tokens_seen": 2035548160 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030974523570712135, + "loss": 1.2123, + "theoretical_loss": 3.453821056614562, + "tokens_seen": 2035613696 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030972918756268805, + "loss": 1.3032, + "theoretical_loss": 3.453811893910703, + "tokens_seen": 2035679232 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030971313941825475, + "loss": 1.3071, + "theoretical_loss": 3.4538027315844126, + "tokens_seen": 2035744768 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030969709127382145, + "loss": 1.181, + "theoretical_loss": 3.4537935696356623, + "tokens_seen": 2035810304 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030968104312938815, + "loss": 1.272, + "theoretical_loss": 3.4537844080644247, + "tokens_seen": 2035875840 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030966499498495486, + "loss": 1.2686, + "theoretical_loss": 3.453775246870672, + "tokens_seen": 2035941376 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030964894684052156, + "loss": 1.3049, + "theoretical_loss": 3.453766086054377, + "tokens_seen": 2036006912 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030963289869608826, + "loss": 1.3939, + "theoretical_loss": 3.4537569256155107, + "tokens_seen": 2036072448 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030961685055165496, + "loss": 1.171, + "theoretical_loss": 3.4537477655540467, + "tokens_seen": 2036137984 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030960080240722166, + "loss": 1.2487, + "theoretical_loss": 3.453738605869957, + "tokens_seen": 2036203520 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030958475426278836, + "loss": 1.3922, + "theoretical_loss": 3.4537294465632136, + "tokens_seen": 2036269056 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030956870611835506, + "loss": 1.2367, + "theoretical_loss": 3.4537202876337885, + "tokens_seen": 2036334592 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030955265797392176, + "loss": 1.2443, + "theoretical_loss": 3.453711129081655, + "tokens_seen": 2036400128 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030953660982948846, + "loss": 1.1168, + "theoretical_loss": 3.453701970906785, + "tokens_seen": 2036465664 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030952056168505516, + "loss": 1.3242, + "theoretical_loss": 3.4536928131091504, + "tokens_seen": 2036531200 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030950451354062186, + "loss": 1.1845, + "theoretical_loss": 3.453683655688724, + "tokens_seen": 2036596736 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030948846539618856, + "loss": 1.2701, + "theoretical_loss": 3.4536744986454777, + "tokens_seen": 2036662272 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030947241725175526, + "loss": 1.2069, + "theoretical_loss": 3.4536653419793844, + "tokens_seen": 2036727808 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030945636910732196, + "loss": 1.2663, + "theoretical_loss": 3.4536561856904155, + "tokens_seen": 2036793344 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030944032096288866, + "loss": 1.3526, + "theoretical_loss": 3.4536470297785447, + "tokens_seen": 2036858880 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030942427281845536, + "loss": 1.217, + "theoretical_loss": 3.4536378742437432, + "tokens_seen": 2036924416 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030940822467402207, + "loss": 1.3023, + "theoretical_loss": 3.453628719085984, + "tokens_seen": 2036989952 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030939217652958877, + "loss": 1.2773, + "theoretical_loss": 3.4536195643052388, + "tokens_seen": 2037055488 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030937612838515547, + "loss": 1.1703, + "theoretical_loss": 3.4536104099014806, + "tokens_seen": 2037121024 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030936008024072217, + "loss": 1.3722, + "theoretical_loss": 3.4536012558746814, + "tokens_seen": 2037186560 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030934403209628887, + "loss": 1.2888, + "theoretical_loss": 3.4535921022248135, + "tokens_seen": 2037252096 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030932798395185557, + "loss": 1.2585, + "theoretical_loss": 3.4535829489518495, + "tokens_seen": 2037317632 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030931193580742227, + "loss": 1.3362, + "theoretical_loss": 3.4535737960557613, + "tokens_seen": 2037383168 + }, + { + "epoch": 0.62, + "learning_rate": 0.000309295887662989, + "loss": 1.2632, + "theoretical_loss": 3.453564643536522, + "tokens_seen": 2037448704 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003092798395185557, + "loss": 1.2392, + "theoretical_loss": 3.453555491394104, + "tokens_seen": 2037514240 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003092637913741224, + "loss": 1.4172, + "theoretical_loss": 3.4535463396284785, + "tokens_seen": 2037579776 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003092477432296891, + "loss": 1.3359, + "theoretical_loss": 3.4535371882396184, + "tokens_seen": 2037645312 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003092316950852558, + "loss": 1.2016, + "theoretical_loss": 3.453528037227497, + "tokens_seen": 2037710848 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030921564694082253, + "loss": 1.3472, + "theoretical_loss": 3.453518886592085, + "tokens_seen": 2037776384 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003091995987963892, + "loss": 1.2386, + "theoretical_loss": 3.4535097363333565, + "tokens_seen": 2037841920 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003091835506519559, + "loss": 1.2378, + "theoretical_loss": 3.4535005864512827, + "tokens_seen": 2037907456 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003091675025075226, + "loss": 1.3157, + "theoretical_loss": 3.4534914369458365, + "tokens_seen": 2037972992 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003091514543630893, + "loss": 1.3474, + "theoretical_loss": 3.4534822878169904, + "tokens_seen": 2038038528 + }, + { + "epoch": 0.62, + "learning_rate": 0.000309135406218656, + "loss": 1.3072, + "theoretical_loss": 3.453473139064716, + "tokens_seen": 2038104064 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 682560, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.257143497467041, + "objective/train/theoretical_loss": 3.453463990688987, + "objective/train/tokens_used": 2058629600, + "theoretical_loss": 3.453463990688987, + "tokens_seen": 2038169600 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003091193580742227, + "loss": 1.2463, + "theoretical_loss": 3.453463990688987, + "tokens_seen": 2038169600 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003091033099297894, + "loss": 1.3082, + "theoretical_loss": 3.4534548426897747, + "tokens_seen": 2038235136 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003090872617853561, + "loss": 1.2113, + "theoretical_loss": 3.4534456950670513, + "tokens_seen": 2038300672 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003090712136409228, + "loss": 1.3351, + "theoretical_loss": 3.4534365478207905, + "tokens_seen": 2038366208 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003090551654964895, + "loss": 1.2689, + "theoretical_loss": 3.4534274009509636, + "tokens_seen": 2038431744 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003090391173520562, + "loss": 1.21, + "theoretical_loss": 3.453418254457543, + "tokens_seen": 2038497280 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003090230692076229, + "loss": 1.2668, + "theoretical_loss": 3.453409108340502, + "tokens_seen": 2038562816 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003090070210631896, + "loss": 1.3273, + "theoretical_loss": 3.4533999625998124, + "tokens_seen": 2038628352 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003089909729187563, + "loss": 1.3636, + "theoretical_loss": 3.4533908172354466, + "tokens_seen": 2038693888 + }, + { + "epoch": 0.62, + "learning_rate": 0.000308974924774323, + "loss": 1.2644, + "theoretical_loss": 3.453381672247377, + "tokens_seen": 2038759424 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003089588766298897, + "loss": 1.2004, + "theoretical_loss": 3.4533725276355764, + "tokens_seen": 2038824960 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003089428284854564, + "loss": 1.2006, + "theoretical_loss": 3.453363383400017, + "tokens_seen": 2038890496 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003089267803410231, + "loss": 1.2727, + "theoretical_loss": 3.453354239540671, + "tokens_seen": 2038956032 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003089107321965898, + "loss": 1.36, + "theoretical_loss": 3.453345096057511, + "tokens_seen": 2039021568 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003088946840521565, + "loss": 1.3566, + "theoretical_loss": 3.453335952950509, + "tokens_seen": 2039087104 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003088786359077232, + "loss": 1.2552, + "theoretical_loss": 3.4533268102196386, + "tokens_seen": 2039152640 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003088625877632899, + "loss": 1.1587, + "theoretical_loss": 3.453317667864871, + "tokens_seen": 2039218176 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003088465396188566, + "loss": 1.3309, + "theoretical_loss": 3.4533085258861798, + "tokens_seen": 2039283712 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003088304914744233, + "loss": 1.3319, + "theoretical_loss": 3.453299384283536, + "tokens_seen": 2039349248 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030881444332999, + "loss": 1.3868, + "theoretical_loss": 3.4532902430569132, + "tokens_seen": 2039414784 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003087983951855567, + "loss": 1.344, + "theoretical_loss": 3.453281102206284, + "tokens_seen": 2039480320 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003087823470411234, + "loss": 1.2656, + "theoretical_loss": 3.4532719617316197, + "tokens_seen": 2039545856 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003087662988966901, + "loss": 1.34, + "theoretical_loss": 3.4532628216328938, + "tokens_seen": 2039611392 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003087502507522568, + "loss": 1.3847, + "theoretical_loss": 3.453253681910078, + "tokens_seen": 2039676928 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003087342026078235, + "loss": 1.328, + "theoretical_loss": 3.4532445425631453, + "tokens_seen": 2039742464 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003087181544633902, + "loss": 1.3318, + "theoretical_loss": 3.4532354035920685, + "tokens_seen": 2039808000 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003087021063189569, + "loss": 1.2651, + "theoretical_loss": 3.453226264996819, + "tokens_seen": 2039873536 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003086860581745236, + "loss": 1.2475, + "theoretical_loss": 3.45321712677737, + "tokens_seen": 2039939072 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003086700100300903, + "loss": 1.3265, + "theoretical_loss": 3.4532079889336935, + "tokens_seen": 2040004608 + }, + { + "epoch": 0.62, + "learning_rate": 0.000308653961885657, + "loss": 1.4194, + "theoretical_loss": 3.4531988514657628, + "tokens_seen": 2040070144 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003086379137412237, + "loss": 1.2217, + "theoretical_loss": 3.4531897143735497, + "tokens_seen": 2040135680 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003086218655967904, + "loss": 1.2882, + "theoretical_loss": 3.453180577657027, + "tokens_seen": 2040201216 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003086058174523571, + "loss": 1.2884, + "theoretical_loss": 3.4531714413161665, + "tokens_seen": 2040266752 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003085897693079238, + "loss": 1.2864, + "theoretical_loss": 3.4531623053509417, + "tokens_seen": 2040332288 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003085737211634905, + "loss": 1.2622, + "theoretical_loss": 3.4531531697613245, + "tokens_seen": 2040397824 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003085576730190572, + "loss": 1.3145, + "theoretical_loss": 3.4531440345472877, + "tokens_seen": 2040463360 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003085416248746239, + "loss": 1.2018, + "theoretical_loss": 3.453134899708803, + "tokens_seen": 2040528896 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003085255767301906, + "loss": 1.2823, + "theoretical_loss": 3.453125765245844, + "tokens_seen": 2040594432 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003085095285857573, + "loss": 1.2244, + "theoretical_loss": 3.4531166311583825, + "tokens_seen": 2040659968 + }, + { + "epoch": 0.62, + "learning_rate": 0.000308493480441324, + "loss": 1.273, + "theoretical_loss": 3.453107497446392, + "tokens_seen": 2040725504 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003084774322968907, + "loss": 1.1891, + "theoretical_loss": 3.453098364109843, + "tokens_seen": 2040791040 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003084613841524574, + "loss": 1.3062, + "theoretical_loss": 3.45308923114871, + "tokens_seen": 2040856576 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003084453360080241, + "loss": 1.2875, + "theoretical_loss": 3.4530800985629644, + "tokens_seen": 2040922112 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030842928786359075, + "loss": 1.2361, + "theoretical_loss": 3.453070966352579, + "tokens_seen": 2040987648 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030841323971915745, + "loss": 1.2962, + "theoretical_loss": 3.453061834517527, + "tokens_seen": 2041053184 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030839719157472415, + "loss": 1.3262, + "theoretical_loss": 3.4530527030577796, + "tokens_seen": 2041118720 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030838114343029085, + "loss": 1.214, + "theoretical_loss": 3.4530435719733106, + "tokens_seen": 2041184256 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030836509528585755, + "loss": 1.1722, + "theoretical_loss": 3.4530344412640916, + "tokens_seen": 2041249792 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030834904714142425, + "loss": 1.3134, + "theoretical_loss": 3.4530253109300957, + "tokens_seen": 2041315328 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030833299899699095, + "loss": 1.3019, + "theoretical_loss": 3.453016180971295, + "tokens_seen": 2041380864 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 683764, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5399019718170166, + "objective/train/theoretical_loss": 3.4530070513876625, + "objective/train/tokens_used": 2061906400, + "theoretical_loss": 3.4530070513876625, + "tokens_seen": 2041446400 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030831695085255765, + "loss": 1.3933, + "theoretical_loss": 3.4530070513876625, + "tokens_seen": 2041446400 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030830090270812436, + "loss": 1.1885, + "theoretical_loss": 3.4529979221791707, + "tokens_seen": 2041511936 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030828485456369106, + "loss": 1.2352, + "theoretical_loss": 3.4529887933457917, + "tokens_seen": 2041577472 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030826880641925776, + "loss": 1.2362, + "theoretical_loss": 3.452979664887498, + "tokens_seen": 2041643008 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030825275827482446, + "loss": 1.3164, + "theoretical_loss": 3.452970536804263, + "tokens_seen": 2041708544 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030823671013039116, + "loss": 1.2778, + "theoretical_loss": 3.4529614090960585, + "tokens_seen": 2041774080 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030822066198595786, + "loss": 1.331, + "theoretical_loss": 3.4529522817628573, + "tokens_seen": 2041839616 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030820461384152456, + "loss": 1.3027, + "theoretical_loss": 3.4529431548046317, + "tokens_seen": 2041905152 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030818856569709126, + "loss": 1.3107, + "theoretical_loss": 3.452934028221355, + "tokens_seen": 2041970688 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030817251755265796, + "loss": 1.2855, + "theoretical_loss": 3.452924902012999, + "tokens_seen": 2042036224 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030815646940822466, + "loss": 1.1502, + "theoretical_loss": 3.4529157761795357, + "tokens_seen": 2042101760 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030814042126379136, + "loss": 1.2762, + "theoretical_loss": 3.4529066507209394, + "tokens_seen": 2042167296 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030812437311935806, + "loss": 1.273, + "theoretical_loss": 3.4528975256371814, + "tokens_seen": 2042232832 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030810832497492476, + "loss": 1.2352, + "theoretical_loss": 3.452888400928235, + "tokens_seen": 2042298368 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003080922768304915, + "loss": 1.2075, + "theoretical_loss": 3.452879276594072, + "tokens_seen": 2042363904 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003080762286860582, + "loss": 1.3181, + "theoretical_loss": 3.4528701526346657, + "tokens_seen": 2042429440 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003080601805416249, + "loss": 1.2302, + "theoretical_loss": 3.4528610290499877, + "tokens_seen": 2042494976 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003080441323971916, + "loss": 1.2892, + "theoretical_loss": 3.452851905840012, + "tokens_seen": 2042560512 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003080280842527583, + "loss": 1.2011, + "theoretical_loss": 3.45284278300471, + "tokens_seen": 2042626048 + }, + { + "epoch": 0.62, + "learning_rate": 0.000308012036108325, + "loss": 1.3596, + "theoretical_loss": 3.452833660544055, + "tokens_seen": 2042691584 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003079959879638917, + "loss": 1.2907, + "theoretical_loss": 3.452824538458019, + "tokens_seen": 2042757120 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003079799398194584, + "loss": 1.2684, + "theoretical_loss": 3.452815416746575, + "tokens_seen": 2042822656 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003079638916750251, + "loss": 1.3933, + "theoretical_loss": 3.452806295409696, + "tokens_seen": 2042888192 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003079478435305918, + "loss": 1.3091, + "theoretical_loss": 3.452797174447354, + "tokens_seen": 2042953728 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003079317953861585, + "loss": 1.28, + "theoretical_loss": 3.4527880538595213, + "tokens_seen": 2043019264 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003079157472417252, + "loss": 1.2713, + "theoretical_loss": 3.4527789336461714, + "tokens_seen": 2043084800 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003078996990972919, + "loss": 1.2647, + "theoretical_loss": 3.452769813807276, + "tokens_seen": 2043150336 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003078836509528586, + "loss": 1.2771, + "theoretical_loss": 3.4527606943428086, + "tokens_seen": 2043215872 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003078676028084253, + "loss": 1.2049, + "theoretical_loss": 3.452751575252741, + "tokens_seen": 2043281408 + }, + { + "epoch": 0.62, + "learning_rate": 0.000307851554663992, + "loss": 1.3456, + "theoretical_loss": 3.452742456537047, + "tokens_seen": 2043346944 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030783550651955873, + "loss": 1.3662, + "theoretical_loss": 3.4527333381956975, + "tokens_seen": 2043412480 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030781945837512543, + "loss": 1.3178, + "theoretical_loss": 3.4527242202286663, + "tokens_seen": 2043478016 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030780341023069213, + "loss": 1.3491, + "theoretical_loss": 3.452715102635926, + "tokens_seen": 2043543552 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003077873620862588, + "loss": 1.3203, + "theoretical_loss": 3.452705985417449, + "tokens_seen": 2043609088 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003077713139418255, + "loss": 1.2479, + "theoretical_loss": 3.452696868573208, + "tokens_seen": 2043674624 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003077552657973922, + "loss": 1.1678, + "theoretical_loss": 3.452687752103176, + "tokens_seen": 2043740160 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003077392176529589, + "loss": 1.2707, + "theoretical_loss": 3.4526786360073247, + "tokens_seen": 2043805696 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003077231695085256, + "loss": 1.2166, + "theoretical_loss": 3.4526695202856272, + "tokens_seen": 2043871232 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003077071213640923, + "loss": 1.32, + "theoretical_loss": 3.4526604049380563, + "tokens_seen": 2043936768 + }, + { + "epoch": 0.62, + "learning_rate": 0.000307691073219659, + "loss": 1.2236, + "theoretical_loss": 3.452651289964585, + "tokens_seen": 2044002304 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003076750250752257, + "loss": 1.3187, + "theoretical_loss": 3.4526421753651855, + "tokens_seen": 2044067840 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003076589769307924, + "loss": 1.2508, + "theoretical_loss": 3.4526330611398297, + "tokens_seen": 2044133376 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003076429287863591, + "loss": 1.2905, + "theoretical_loss": 3.452623947288492, + "tokens_seen": 2044198912 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003076268806419258, + "loss": 1.1559, + "theoretical_loss": 3.4526148338111433, + "tokens_seen": 2044264448 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003076108324974925, + "loss": 1.2295, + "theoretical_loss": 3.4526057207077576, + "tokens_seen": 2044329984 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003075947843530592, + "loss": 1.2918, + "theoretical_loss": 3.4525966079783075, + "tokens_seen": 2044395520 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003075787362086259, + "loss": 1.2014, + "theoretical_loss": 3.4525874956227645, + "tokens_seen": 2044461056 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003075626880641926, + "loss": 1.2849, + "theoretical_loss": 3.452578383641102, + "tokens_seen": 2044526592 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003075466399197593, + "loss": 1.3276, + "theoretical_loss": 3.4525692720332932, + "tokens_seen": 2044592128 + }, + { + "epoch": 0.62, + "learning_rate": 0.000307530591775326, + "loss": 1.2813, + "theoretical_loss": 3.45256016079931, + "tokens_seen": 2044657664 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 684631, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2299048900604248, + "objective/train/theoretical_loss": 3.4525510499391245, + "objective/train/tokens_used": 2065183200, + "theoretical_loss": 3.4525510499391245, + "tokens_seen": 2044723200 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003075145436308927, + "loss": 1.2609, + "theoretical_loss": 3.4525510499391245, + "tokens_seen": 2044723200 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003074984954864594, + "loss": 1.3623, + "theoretical_loss": 3.4525419394527113, + "tokens_seen": 2044788736 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003074824473420261, + "loss": 1.2515, + "theoretical_loss": 3.4525328293400417, + "tokens_seen": 2044854272 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003074663991975928, + "loss": 1.1655, + "theoretical_loss": 3.4525237196010883, + "tokens_seen": 2044919808 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003074503510531595, + "loss": 1.3252, + "theoretical_loss": 3.4525146102358244, + "tokens_seen": 2044985344 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003074343029087262, + "loss": 1.3235, + "theoretical_loss": 3.452505501244223, + "tokens_seen": 2045050880 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003074182547642929, + "loss": 1.292, + "theoretical_loss": 3.452496392626256, + "tokens_seen": 2045116416 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003074022066198596, + "loss": 1.2388, + "theoretical_loss": 3.452487284381896, + "tokens_seen": 2045181952 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003073861584754263, + "loss": 1.2679, + "theoretical_loss": 3.452478176511116, + "tokens_seen": 2045247488 + }, + { + "epoch": 0.62, + "learning_rate": 0.000307370110330993, + "loss": 1.2823, + "theoretical_loss": 3.4524690690138895, + "tokens_seen": 2045313024 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003073540621865597, + "loss": 1.2521, + "theoretical_loss": 3.452459961890188, + "tokens_seen": 2045378560 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003073380140421264, + "loss": 1.2804, + "theoretical_loss": 3.452450855139985, + "tokens_seen": 2045444096 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003073219658976931, + "loss": 1.253, + "theoretical_loss": 3.4524417487632526, + "tokens_seen": 2045509632 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003073059177532598, + "loss": 1.3117, + "theoretical_loss": 3.4524326427599643, + "tokens_seen": 2045575168 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003072898696088265, + "loss": 1.2996, + "theoretical_loss": 3.452423537130092, + "tokens_seen": 2045640704 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003072738214643932, + "loss": 1.2578, + "theoretical_loss": 3.452414431873609, + "tokens_seen": 2045706240 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003072577733199599, + "loss": 1.1737, + "theoretical_loss": 3.4524053269904873, + "tokens_seen": 2045771776 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003072417251755266, + "loss": 1.2432, + "theoretical_loss": 3.4523962224807008, + "tokens_seen": 2045837312 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003072256770310933, + "loss": 1.2415, + "theoretical_loss": 3.4523871183442214, + "tokens_seen": 2045902848 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030720962888666, + "loss": 1.3258, + "theoretical_loss": 3.4523780145810217, + "tokens_seen": 2045968384 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003071935807422267, + "loss": 1.2345, + "theoretical_loss": 3.4523689111910754, + "tokens_seen": 2046033920 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003071775325977934, + "loss": 1.237, + "theoretical_loss": 3.452359808174354, + "tokens_seen": 2046099456 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003071614844533601, + "loss": 1.2516, + "theoretical_loss": 3.452350705530831, + "tokens_seen": 2046164992 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003071454363089268, + "loss": 1.2471, + "theoretical_loss": 3.452341603260479, + "tokens_seen": 2046230528 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003071293881644935, + "loss": 1.3384, + "theoretical_loss": 3.452332501363271, + "tokens_seen": 2046296064 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003071133400200602, + "loss": 1.3427, + "theoretical_loss": 3.4523233998391794, + "tokens_seen": 2046361600 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003070972918756269, + "loss": 1.2222, + "theoretical_loss": 3.4523142986881767, + "tokens_seen": 2046427136 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003070812437311936, + "loss": 1.3624, + "theoretical_loss": 3.4523051979102366, + "tokens_seen": 2046492672 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003070651955867603, + "loss": 1.2361, + "theoretical_loss": 3.452296097505331, + "tokens_seen": 2046558208 + }, + { + "epoch": 0.62, + "learning_rate": 0.000307049147442327, + "loss": 1.2431, + "theoretical_loss": 3.4522869974734327, + "tokens_seen": 2046623744 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003070330992978937, + "loss": 1.2026, + "theoretical_loss": 3.4522778978145148, + "tokens_seen": 2046689280 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030701705115346035, + "loss": 1.2529, + "theoretical_loss": 3.4522687985285505, + "tokens_seen": 2046754816 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030700100300902705, + "loss": 1.2662, + "theoretical_loss": 3.4522596996155115, + "tokens_seen": 2046820352 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030698495486459375, + "loss": 1.3028, + "theoretical_loss": 3.4522506010753715, + "tokens_seen": 2046885888 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030696890672016045, + "loss": 1.3296, + "theoretical_loss": 3.4522415029081026, + "tokens_seen": 2046951424 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030695285857572715, + "loss": 1.3201, + "theoretical_loss": 3.452232405113678, + "tokens_seen": 2047016960 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030693681043129385, + "loss": 1.2371, + "theoretical_loss": 3.45222330769207, + "tokens_seen": 2047082496 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030692076228686056, + "loss": 1.319, + "theoretical_loss": 3.4522142106432527, + "tokens_seen": 2047148032 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003069047141424273, + "loss": 1.2867, + "theoretical_loss": 3.452205113967197, + "tokens_seen": 2047213568 + }, + { + "epoch": 0.62, + "learning_rate": 0.000306888665997994, + "loss": 1.2849, + "theoretical_loss": 3.4521960176638773, + "tokens_seen": 2047279104 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003068726178535607, + "loss": 1.3054, + "theoretical_loss": 3.4521869217332655, + "tokens_seen": 2047344640 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003068565697091274, + "loss": 1.3061, + "theoretical_loss": 3.4521778261753346, + "tokens_seen": 2047410176 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003068405215646941, + "loss": 1.3928, + "theoretical_loss": 3.4521687309900577, + "tokens_seen": 2047475712 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003068244734202608, + "loss": 1.3086, + "theoretical_loss": 3.4521596361774067, + "tokens_seen": 2047541248 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003068084252758275, + "loss": 1.3244, + "theoretical_loss": 3.452150541737356, + "tokens_seen": 2047606784 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003067923771313942, + "loss": 1.2439, + "theoretical_loss": 3.4521414476698764, + "tokens_seen": 2047672320 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003067763289869609, + "loss": 1.3025, + "theoretical_loss": 3.452132353974943, + "tokens_seen": 2047737856 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003067602808425276, + "loss": 1.2693, + "theoretical_loss": 3.4521232606525265, + "tokens_seen": 2047803392 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003067442326980943, + "loss": 1.3337, + "theoretical_loss": 3.452114167702601, + "tokens_seen": 2047868928 + }, + { + "epoch": 0.62, + "learning_rate": 0.000306728184553661, + "loss": 1.2676, + "theoretical_loss": 3.4521050751251385, + "tokens_seen": 2047934464 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 685800, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1857306957244873, + "objective/train/theoretical_loss": 3.4520959829201128, + "objective/train/tokens_used": 2068460000, + "theoretical_loss": 3.4520959829201128, + "tokens_seen": 2048000000 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003067121364092277, + "loss": 1.2659, + "theoretical_loss": 3.4520959829201128, + "tokens_seen": 2048000000 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003066960882647944, + "loss": 1.3608, + "theoretical_loss": 3.452086891087496, + "tokens_seen": 2048065536 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003066800401203611, + "loss": 1.2833, + "theoretical_loss": 3.452077799627261, + "tokens_seen": 2048131072 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003066639919759278, + "loss": 1.2188, + "theoretical_loss": 3.4520687085393806, + "tokens_seen": 2048196608 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003066479438314945, + "loss": 1.2748, + "theoretical_loss": 3.452059617823828, + "tokens_seen": 2048262144 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003066318956870612, + "loss": 1.3237, + "theoretical_loss": 3.4520505274805764, + "tokens_seen": 2048327680 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003066158475426279, + "loss": 1.2631, + "theoretical_loss": 3.452041437509597, + "tokens_seen": 2048393216 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003065997993981946, + "loss": 1.1814, + "theoretical_loss": 3.4520323479108646, + "tokens_seen": 2048458752 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003065837512537613, + "loss": 1.3237, + "theoretical_loss": 3.452023258684351, + "tokens_seen": 2048524288 + }, + { + "epoch": 0.62, + "learning_rate": 0.000306567703109328, + "loss": 1.3989, + "theoretical_loss": 3.4520141698300284, + "tokens_seen": 2048589824 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003065516549648947, + "loss": 1.2769, + "theoretical_loss": 3.4520050813478713, + "tokens_seen": 2048655360 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003065356068204614, + "loss": 1.2536, + "theoretical_loss": 3.4519959932378512, + "tokens_seen": 2048720896 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003065195586760281, + "loss": 1.2378, + "theoretical_loss": 3.4519869054999424, + "tokens_seen": 2048786432 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003065035105315948, + "loss": 1.2761, + "theoretical_loss": 3.451977818134116, + "tokens_seen": 2048851968 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003064874623871615, + "loss": 1.2268, + "theoretical_loss": 3.451968731140346, + "tokens_seen": 2048917504 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030647141424272823, + "loss": 1.3496, + "theoretical_loss": 3.4519596445186047, + "tokens_seen": 2048983040 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030645536609829493, + "loss": 1.3143, + "theoretical_loss": 3.451950558268866, + "tokens_seen": 2049048576 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030643931795386163, + "loss": 1.2292, + "theoretical_loss": 3.451941472391101, + "tokens_seen": 2049114112 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030642326980942833, + "loss": 1.356, + "theoretical_loss": 3.451932386885284, + "tokens_seen": 2049179648 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030640722166499503, + "loss": 1.1521, + "theoretical_loss": 3.4519233017513877, + "tokens_seen": 2049245184 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003063911735205617, + "loss": 1.2458, + "theoretical_loss": 3.4519142169893846, + "tokens_seen": 2049310720 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003063751253761284, + "loss": 1.2745, + "theoretical_loss": 3.451905132599248, + "tokens_seen": 2049376256 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003063590772316951, + "loss": 1.2585, + "theoretical_loss": 3.45189604858095, + "tokens_seen": 2049441792 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003063430290872618, + "loss": 1.3821, + "theoretical_loss": 3.4518869649344643, + "tokens_seen": 2049507328 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003063269809428285, + "loss": 1.2214, + "theoretical_loss": 3.4518778816597635, + "tokens_seen": 2049572864 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003063109327983952, + "loss": 1.3102, + "theoretical_loss": 3.451868798756821, + "tokens_seen": 2049638400 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003062948846539619, + "loss": 1.2913, + "theoretical_loss": 3.4518597162256084, + "tokens_seen": 2049703936 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003062788365095286, + "loss": 1.284, + "theoretical_loss": 3.4518506340660995, + "tokens_seen": 2049769472 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003062627883650953, + "loss": 1.2228, + "theoretical_loss": 3.4518415522782675, + "tokens_seen": 2049835008 + }, + { + "epoch": 0.62, + "learning_rate": 0.000306246740220662, + "loss": 1.3518, + "theoretical_loss": 3.451832470862085, + "tokens_seen": 2049900544 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003062306920762287, + "loss": 1.3234, + "theoretical_loss": 3.4518233898175246, + "tokens_seen": 2049966080 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003062146439317954, + "loss": 1.1897, + "theoretical_loss": 3.4518143091445594, + "tokens_seen": 2050031616 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003061985957873621, + "loss": 1.3907, + "theoretical_loss": 3.4518052288431624, + "tokens_seen": 2050097152 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003061825476429288, + "loss": 1.2745, + "theoretical_loss": 3.4517961489133064, + "tokens_seen": 2050162688 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003061664994984955, + "loss": 1.3842, + "theoretical_loss": 3.451787069354965, + "tokens_seen": 2050228224 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003061504513540622, + "loss": 1.2386, + "theoretical_loss": 3.4517779901681096, + "tokens_seen": 2050293760 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003061344032096289, + "loss": 1.2721, + "theoretical_loss": 3.4517689113527146, + "tokens_seen": 2050359296 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003061183550651956, + "loss": 1.3492, + "theoretical_loss": 3.4517598329087518, + "tokens_seen": 2050424832 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003061023069207623, + "loss": 1.1884, + "theoretical_loss": 3.4517507548361954, + "tokens_seen": 2050490368 + }, + { + "epoch": 0.62, + "learning_rate": 0.000306086258776329, + "loss": 1.3203, + "theoretical_loss": 3.4517416771350176, + "tokens_seen": 2050555904 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003060702106318957, + "loss": 1.2926, + "theoretical_loss": 3.451732599805191, + "tokens_seen": 2050621440 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003060541624874624, + "loss": 1.3281, + "theoretical_loss": 3.451723522846689, + "tokens_seen": 2050686976 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003060381143430291, + "loss": 1.1767, + "theoretical_loss": 3.4517144462594844, + "tokens_seen": 2050752512 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003060220661985958, + "loss": 1.1938, + "theoretical_loss": 3.451705370043551, + "tokens_seen": 2050818048 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003060060180541625, + "loss": 1.2403, + "theoretical_loss": 3.45169629419886, + "tokens_seen": 2050883584 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003059899699097292, + "loss": 1.364, + "theoretical_loss": 3.451687218725386, + "tokens_seen": 2050949120 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003059739217652959, + "loss": 1.2582, + "theoretical_loss": 3.4516781436231008, + "tokens_seen": 2051014656 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003059578736208626, + "loss": 1.2003, + "theoretical_loss": 3.451669068891978, + "tokens_seen": 2051080192 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003059418254764293, + "loss": 1.3335, + "theoretical_loss": 3.4516599945319903, + "tokens_seen": 2051145728 + }, + { + "epoch": 0.62, + "learning_rate": 0.000305925777331996, + "loss": 1.2301, + "theoretical_loss": 3.451650920543111, + "tokens_seen": 2051211264 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 686726, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3169500827789307, + "objective/train/theoretical_loss": 3.451641846925313, + "objective/train/tokens_used": 2071736800, + "theoretical_loss": 3.451641846925313, + "tokens_seen": 2051276800 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003059097291875627, + "loss": 1.2802, + "theoretical_loss": 3.451641846925313, + "tokens_seen": 2051276800 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003058936810431294, + "loss": 1.3173, + "theoretical_loss": 3.4516327736785684, + "tokens_seen": 2051342336 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003058776328986961, + "loss": 1.2207, + "theoretical_loss": 3.451623700802852, + "tokens_seen": 2051407872 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003058615847542628, + "loss": 1.3639, + "theoretical_loss": 3.4516146282981346, + "tokens_seen": 2051473408 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003058455366098295, + "loss": 1.3164, + "theoretical_loss": 3.451605556164391, + "tokens_seen": 2051538944 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003058294884653962, + "loss": 1.4238, + "theoretical_loss": 3.451596484401592, + "tokens_seen": 2051604480 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003058134403209629, + "loss": 1.1302, + "theoretical_loss": 3.4515874130097135, + "tokens_seen": 2051670016 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003057973921765296, + "loss": 1.2408, + "theoretical_loss": 3.4515783419887263, + "tokens_seen": 2051735552 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003057813440320963, + "loss": 1.2066, + "theoretical_loss": 3.451569271338604, + "tokens_seen": 2051801088 + }, + { + "epoch": 0.62, + "learning_rate": 0.000305765295887663, + "loss": 1.3544, + "theoretical_loss": 3.45156020105932, + "tokens_seen": 2051866624 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003057492477432297, + "loss": 1.3271, + "theoretical_loss": 3.451551131150847, + "tokens_seen": 2051932160 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003057331995987964, + "loss": 1.3013, + "theoretical_loss": 3.4515420616131576, + "tokens_seen": 2051997696 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003057171514543631, + "loss": 1.4297, + "theoretical_loss": 3.4515329924462255, + "tokens_seen": 2052063232 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003057011033099298, + "loss": 1.292, + "theoretical_loss": 3.451523923650023, + "tokens_seen": 2052128768 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003056850551654965, + "loss": 1.274, + "theoretical_loss": 3.4515148552245236, + "tokens_seen": 2052194304 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003056690070210632, + "loss": 1.2692, + "theoretical_loss": 3.4515057871697, + "tokens_seen": 2052259840 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003056529588766299, + "loss": 1.2282, + "theoretical_loss": 3.451496719485526, + "tokens_seen": 2052325376 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003056369107321966, + "loss": 1.2513, + "theoretical_loss": 3.4514876521719735, + "tokens_seen": 2052390912 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003056208625877633, + "loss": 1.1837, + "theoretical_loss": 3.451478585229016, + "tokens_seen": 2052456448 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030560481444333, + "loss": 1.2053, + "theoretical_loss": 3.451469518656627, + "tokens_seen": 2052521984 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003055887662988967, + "loss": 1.307, + "theoretical_loss": 3.4514604524547785, + "tokens_seen": 2052587520 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003055727181544634, + "loss": 1.3516, + "theoretical_loss": 3.4514513866234444, + "tokens_seen": 2052653056 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003055566700100301, + "loss": 1.2605, + "theoretical_loss": 3.451442321162597, + "tokens_seen": 2052718592 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003055406218655968, + "loss": 1.2582, + "theoretical_loss": 3.4514332560722103, + "tokens_seen": 2052784128 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003055245737211635, + "loss": 1.3299, + "theoretical_loss": 3.451424191352257, + "tokens_seen": 2052849664 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003055085255767302, + "loss": 1.1915, + "theoretical_loss": 3.451415127002709, + "tokens_seen": 2052915200 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003054924774322969, + "loss": 1.2641, + "theoretical_loss": 3.4514060630235406, + "tokens_seen": 2052980736 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003054764292878636, + "loss": 1.1938, + "theoretical_loss": 3.451396999414725, + "tokens_seen": 2053046272 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003054603811434303, + "loss": 1.2671, + "theoretical_loss": 3.4513879361762343, + "tokens_seen": 2053111808 + }, + { + "epoch": 0.62, + "learning_rate": 0.000305444332998997, + "loss": 1.182, + "theoretical_loss": 3.451378873308042, + "tokens_seen": 2053177344 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003054282848545637, + "loss": 1.3227, + "theoretical_loss": 3.4513698108101214, + "tokens_seen": 2053242880 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003054122367101304, + "loss": 1.3401, + "theoretical_loss": 3.451360748682445, + "tokens_seen": 2053308416 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003053961885656971, + "loss": 1.2524, + "theoretical_loss": 3.451351686924986, + "tokens_seen": 2053373952 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003053801404212638, + "loss": 1.3038, + "theoretical_loss": 3.4513426255377184, + "tokens_seen": 2053439488 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003053640922768305, + "loss": 1.1896, + "theoretical_loss": 3.4513335645206142, + "tokens_seen": 2053505024 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003053480441323972, + "loss": 1.3894, + "theoretical_loss": 3.4513245038736464, + "tokens_seen": 2053570560 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003053319959879639, + "loss": 1.2162, + "theoretical_loss": 3.4513154435967888, + "tokens_seen": 2053636096 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003053159478435306, + "loss": 1.2657, + "theoretical_loss": 3.4513063836900137, + "tokens_seen": 2053701632 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003052998996990973, + "loss": 1.2464, + "theoretical_loss": 3.4512973241532947, + "tokens_seen": 2053767168 + }, + { + "epoch": 0.62, + "learning_rate": 0.000305283851554664, + "loss": 1.2792, + "theoretical_loss": 3.451288264986605, + "tokens_seen": 2053832704 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003052678034102307, + "loss": 1.2409, + "theoretical_loss": 3.451279206189917, + "tokens_seen": 2053898240 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003052517552657974, + "loss": 1.349, + "theoretical_loss": 3.4512701477632044, + "tokens_seen": 2053963776 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003052357071213641, + "loss": 1.3268, + "theoretical_loss": 3.45126108970644, + "tokens_seen": 2054029312 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003052196589769308, + "loss": 1.412, + "theoretical_loss": 3.4512520320195974, + "tokens_seen": 2054094848 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003052036108324975, + "loss": 1.2705, + "theoretical_loss": 3.451242974702649, + "tokens_seen": 2054160384 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003051875626880642, + "loss": 1.1753, + "theoretical_loss": 3.451233917755568, + "tokens_seen": 2054225920 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003051715145436309, + "loss": 1.2702, + "theoretical_loss": 3.451224861178328, + "tokens_seen": 2054291456 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003051554663991976, + "loss": 1.4642, + "theoretical_loss": 3.451215804970902, + "tokens_seen": 2054356992 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003051394182547643, + "loss": 1.2314, + "theoretical_loss": 3.451206749133262, + "tokens_seen": 2054422528 + }, + { + "epoch": 0.62, + "learning_rate": 0.000305123370110331, + "loss": 1.2388, + "theoretical_loss": 3.4511976936653825, + "tokens_seen": 2054488064 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 687773, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3332979679107666, + "objective/train/theoretical_loss": 3.4511886385672357, + "objective/train/tokens_used": 2075013600, + "theoretical_loss": 3.4511886385672357, + "tokens_seen": 2054553600 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030510732196589773, + "loss": 1.2636, + "theoretical_loss": 3.4511886385672357, + "tokens_seen": 2054553600 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030509127382146443, + "loss": 1.2771, + "theoretical_loss": 3.4511795838387958, + "tokens_seen": 2054619136 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030507522567703113, + "loss": 1.2273, + "theoretical_loss": 3.4511705294800343, + "tokens_seen": 2054684672 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030505917753259783, + "loss": 1.3907, + "theoretical_loss": 3.4511614754909257, + "tokens_seen": 2054750208 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030504312938816453, + "loss": 1.4565, + "theoretical_loss": 3.451152421871443, + "tokens_seen": 2054815744 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030502708124373123, + "loss": 1.2172, + "theoretical_loss": 3.4511433686215582, + "tokens_seen": 2054881280 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030501103309929793, + "loss": 1.2495, + "theoretical_loss": 3.451134315741246, + "tokens_seen": 2054946816 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003049949849548646, + "loss": 1.2772, + "theoretical_loss": 3.451125263230478, + "tokens_seen": 2055012352 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003049789368104313, + "loss": 1.3352, + "theoretical_loss": 3.4511162110892286, + "tokens_seen": 2055077888 + }, + { + "epoch": 0.62, + "learning_rate": 0.000304962888665998, + "loss": 1.3278, + "theoretical_loss": 3.45110715931747, + "tokens_seen": 2055143424 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003049468405215647, + "loss": 1.3033, + "theoretical_loss": 3.451098107915176, + "tokens_seen": 2055208960 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003049307923771314, + "loss": 1.3228, + "theoretical_loss": 3.451089056882319, + "tokens_seen": 2055274496 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003049147442326981, + "loss": 1.2982, + "theoretical_loss": 3.4510800062188727, + "tokens_seen": 2055340032 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003048986960882648, + "loss": 1.2146, + "theoretical_loss": 3.451070955924811, + "tokens_seen": 2055405568 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003048826479438315, + "loss": 1.3313, + "theoretical_loss": 3.4510619060001053, + "tokens_seen": 2055471104 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003048665997993982, + "loss": 1.2213, + "theoretical_loss": 3.4510528564447296, + "tokens_seen": 2055536640 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003048505516549649, + "loss": 1.1385, + "theoretical_loss": 3.4510438072586576, + "tokens_seen": 2055602176 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003048345035105316, + "loss": 1.2984, + "theoretical_loss": 3.4510347584418612, + "tokens_seen": 2055667712 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003048184553660983, + "loss": 1.2993, + "theoretical_loss": 3.451025709994315, + "tokens_seen": 2055733248 + }, + { + "epoch": 0.62, + "learning_rate": 0.000304802407221665, + "loss": 1.2631, + "theoretical_loss": 3.4510166619159914, + "tokens_seen": 2055798784 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003047863590772317, + "loss": 1.2308, + "theoretical_loss": 3.4510076142068633, + "tokens_seen": 2055864320 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003047703109327984, + "loss": 1.2614, + "theoretical_loss": 3.4509985668669043, + "tokens_seen": 2055929856 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003047542627883651, + "loss": 1.2306, + "theoretical_loss": 3.450989519896088, + "tokens_seen": 2055995392 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003047382146439318, + "loss": 1.2293, + "theoretical_loss": 3.4509804732943863, + "tokens_seen": 2056060928 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003047221664994985, + "loss": 1.3185, + "theoretical_loss": 3.4509714270617735, + "tokens_seen": 2056126464 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003047061183550652, + "loss": 1.3, + "theoretical_loss": 3.450962381198222, + "tokens_seen": 2056192000 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003046900702106319, + "loss": 1.2521, + "theoretical_loss": 3.450953335703706, + "tokens_seen": 2056257536 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003046740220661986, + "loss": 1.2354, + "theoretical_loss": 3.4509442905781973, + "tokens_seen": 2056323072 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003046579739217653, + "loss": 1.2061, + "theoretical_loss": 3.4509352458216704, + "tokens_seen": 2056388608 + }, + { + "epoch": 0.62, + "learning_rate": 0.000304641925777332, + "loss": 1.3107, + "theoretical_loss": 3.450926201434098, + "tokens_seen": 2056454144 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003046258776328987, + "loss": 1.248, + "theoretical_loss": 3.4509171574154527, + "tokens_seen": 2056519680 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003046098294884654, + "loss": 1.2084, + "theoretical_loss": 3.4509081137657085, + "tokens_seen": 2056585216 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003045937813440321, + "loss": 1.2186, + "theoretical_loss": 3.450899070484838, + "tokens_seen": 2056650752 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003045777331995988, + "loss": 1.3615, + "theoretical_loss": 3.4508900275728154, + "tokens_seen": 2056716288 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003045616850551655, + "loss": 1.29, + "theoretical_loss": 3.450880985029613, + "tokens_seen": 2056781824 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003045456369107322, + "loss": 1.1823, + "theoretical_loss": 3.450871942855204, + "tokens_seen": 2056847360 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003045295887662989, + "loss": 1.2327, + "theoretical_loss": 3.450862901049562, + "tokens_seen": 2056912896 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003045135406218656, + "loss": 1.2362, + "theoretical_loss": 3.45085385961266, + "tokens_seen": 2056978432 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030449749247743235, + "loss": 1.2931, + "theoretical_loss": 3.4508448185444713, + "tokens_seen": 2057043968 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030448144433299905, + "loss": 1.2555, + "theoretical_loss": 3.4508357778449694, + "tokens_seen": 2057109504 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030446539618856575, + "loss": 1.2788, + "theoretical_loss": 3.4508267375141264, + "tokens_seen": 2057175040 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030444934804413245, + "loss": 1.2358, + "theoretical_loss": 3.450817697551917, + "tokens_seen": 2057240576 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030443329989969915, + "loss": 1.2054, + "theoretical_loss": 3.4508086579583135, + "tokens_seen": 2057306112 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030441725175526585, + "loss": 1.2856, + "theoretical_loss": 3.4507996187332894, + "tokens_seen": 2057371648 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030440120361083256, + "loss": 1.2238, + "theoretical_loss": 3.450790579876818, + "tokens_seen": 2057437184 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030438515546639926, + "loss": 1.2619, + "theoretical_loss": 3.4507815413888725, + "tokens_seen": 2057502720 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030436910732196596, + "loss": 1.2797, + "theoretical_loss": 3.450772503269426, + "tokens_seen": 2057568256 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003043530591775326, + "loss": 1.3414, + "theoretical_loss": 3.4507634655184516, + "tokens_seen": 2057633792 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003043370110330993, + "loss": 1.2458, + "theoretical_loss": 3.450754428135923, + "tokens_seen": 2057699328 + }, + { + "epoch": 0.62, + "learning_rate": 0.000304320962888666, + "loss": 1.3248, + "theoretical_loss": 3.450745391121813, + "tokens_seen": 2057764864 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 688746, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3244448900222778, + "objective/train/theoretical_loss": 3.4507363544760956, + "objective/train/tokens_used": 2078290400, + "theoretical_loss": 3.4507363544760956, + "tokens_seen": 2057830400 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003043049147442327, + "loss": 1.2471, + "theoretical_loss": 3.4507363544760956, + "tokens_seen": 2057830400 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003042888665997994, + "loss": 1.2969, + "theoretical_loss": 3.4507273181987426, + "tokens_seen": 2057895936 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003042728184553661, + "loss": 1.2506, + "theoretical_loss": 3.4507182822897287, + "tokens_seen": 2057961472 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003042567703109328, + "loss": 1.2169, + "theoretical_loss": 3.4507092467490272, + "tokens_seen": 2058027008 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003042407221664995, + "loss": 1.3024, + "theoretical_loss": 3.45070021157661, + "tokens_seen": 2058092544 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003042246740220662, + "loss": 1.1788, + "theoretical_loss": 3.4506911767724513, + "tokens_seen": 2058158080 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003042086258776329, + "loss": 1.1999, + "theoretical_loss": 3.4506821423365244, + "tokens_seen": 2058223616 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003041925777331996, + "loss": 1.2391, + "theoretical_loss": 3.450673108268802, + "tokens_seen": 2058289152 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003041765295887663, + "loss": 1.3035, + "theoretical_loss": 3.450664074569258, + "tokens_seen": 2058354688 + }, + { + "epoch": 0.62, + "learning_rate": 0.000304160481444333, + "loss": 1.2367, + "theoretical_loss": 3.4506550412378654, + "tokens_seen": 2058420224 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003041444332998997, + "loss": 1.2123, + "theoretical_loss": 3.4506460082745978, + "tokens_seen": 2058485760 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003041283851554664, + "loss": 1.3235, + "theoretical_loss": 3.4506369756794277, + "tokens_seen": 2058551296 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003041123370110331, + "loss": 1.2073, + "theoretical_loss": 3.450627943452329, + "tokens_seen": 2058616832 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003040962888665998, + "loss": 1.2638, + "theoretical_loss": 3.450618911593275, + "tokens_seen": 2058682368 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003040802407221665, + "loss": 1.2701, + "theoretical_loss": 3.4506098801022387, + "tokens_seen": 2058747904 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003040641925777332, + "loss": 1.2289, + "theoretical_loss": 3.4506008489791933, + "tokens_seen": 2058813440 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003040481444332999, + "loss": 1.2574, + "theoretical_loss": 3.4505918182241127, + "tokens_seen": 2058878976 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003040320962888666, + "loss": 1.3613, + "theoretical_loss": 3.4505827878369697, + "tokens_seen": 2058944512 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003040160481444333, + "loss": 1.2264, + "theoretical_loss": 3.450573757817738, + "tokens_seen": 2059010048 + }, + { + "epoch": 0.62, + "learning_rate": 0.000304, + "loss": 1.2085, + "theoretical_loss": 3.45056472816639, + "tokens_seen": 2059075584 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003039839518555667, + "loss": 1.2419, + "theoretical_loss": 3.4505556988828996, + "tokens_seen": 2059141120 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003039679037111334, + "loss": 1.2468, + "theoretical_loss": 3.4505466699672405, + "tokens_seen": 2059206656 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003039518555667001, + "loss": 1.3193, + "theoretical_loss": 3.4505376414193853, + "tokens_seen": 2059272192 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003039358074222668, + "loss": 1.4149, + "theoretical_loss": 3.450528613239308, + "tokens_seen": 2059337728 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003039197592778335, + "loss": 1.2797, + "theoretical_loss": 3.4505195854269815, + "tokens_seen": 2059403264 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003039037111334002, + "loss": 1.2463, + "theoretical_loss": 3.450510557982379, + "tokens_seen": 2059468800 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003038876629889669, + "loss": 1.2455, + "theoretical_loss": 3.4505015309054743, + "tokens_seen": 2059534336 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003038716148445336, + "loss": 1.3298, + "theoretical_loss": 3.45049250419624, + "tokens_seen": 2059599872 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003038555667001003, + "loss": 1.4246, + "theoretical_loss": 3.4504834778546503, + "tokens_seen": 2059665408 + }, + { + "epoch": 0.62, + "learning_rate": 0.000303839518555667, + "loss": 1.236, + "theoretical_loss": 3.4504744518806776, + "tokens_seen": 2059730944 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003038234704112337, + "loss": 1.2963, + "theoretical_loss": 3.4504654262742958, + "tokens_seen": 2059796480 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003038074222668004, + "loss": 1.3202, + "theoretical_loss": 3.4504564010354786, + "tokens_seen": 2059862016 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003037913741223671, + "loss": 1.2154, + "theoretical_loss": 3.450447376164198, + "tokens_seen": 2059927552 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003037753259779338, + "loss": 1.2535, + "theoretical_loss": 3.450438351660429, + "tokens_seen": 2059993088 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003037592778335005, + "loss": 1.2597, + "theoretical_loss": 3.4504293275241436, + "tokens_seen": 2060058624 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003037432296890672, + "loss": 1.2386, + "theoretical_loss": 3.4504203037553163, + "tokens_seen": 2060124160 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030372718154463393, + "loss": 1.4104, + "theoretical_loss": 3.4504112803539195, + "tokens_seen": 2060189696 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030371113340020063, + "loss": 1.349, + "theoretical_loss": 3.450402257319927, + "tokens_seen": 2060255232 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030369508525576733, + "loss": 1.2928, + "theoretical_loss": 3.450393234653312, + "tokens_seen": 2060320768 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030367903711133403, + "loss": 1.3125, + "theoretical_loss": 3.450384212354048, + "tokens_seen": 2060386304 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030366298896690073, + "loss": 1.3297, + "theoretical_loss": 3.4503751904221085, + "tokens_seen": 2060451840 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030364694082246743, + "loss": 1.3422, + "theoretical_loss": 3.450366168857466, + "tokens_seen": 2060517376 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030363089267803413, + "loss": 1.3286, + "theoretical_loss": 3.450357147660095, + "tokens_seen": 2060582912 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030361484453360083, + "loss": 1.2436, + "theoretical_loss": 3.4503481268299683, + "tokens_seen": 2060648448 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030359879638916753, + "loss": 1.3507, + "theoretical_loss": 3.4503391063670596, + "tokens_seen": 2060713984 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003035827482447342, + "loss": 1.2979, + "theoretical_loss": 3.4503300862713413, + "tokens_seen": 2060779520 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003035667001003009, + "loss": 1.2859, + "theoretical_loss": 3.450321066542788, + "tokens_seen": 2060845056 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003035506519558676, + "loss": 1.2971, + "theoretical_loss": 3.450312047181373, + "tokens_seen": 2060910592 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003035346038114343, + "loss": 1.3308, + "theoretical_loss": 3.4503030281870686, + "tokens_seen": 2060976128 + }, + { + "epoch": 0.62, + "learning_rate": 0.000303518555667001, + "loss": 1.3051, + "theoretical_loss": 3.4502940095598484, + "tokens_seen": 2061041664 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 689953, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2605319023132324, + "objective/train/theoretical_loss": 3.450284991299687, + "objective/train/tokens_used": 2081567200, + "theoretical_loss": 3.450284991299687, + "tokens_seen": 2061107200 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003035025075225677, + "loss": 1.2442, + "theoretical_loss": 3.450284991299687, + "tokens_seen": 2061107200 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003034864593781344, + "loss": 1.2675, + "theoretical_loss": 3.4502759734065567, + "tokens_seen": 2061172736 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003034704112337011, + "loss": 1.3474, + "theoretical_loss": 3.4502669558804318, + "tokens_seen": 2061238272 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003034543630892678, + "loss": 1.3098, + "theoretical_loss": 3.4502579387212844, + "tokens_seen": 2061303808 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003034383149448345, + "loss": 1.2708, + "theoretical_loss": 3.450248921929089, + "tokens_seen": 2061369344 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003034222668004012, + "loss": 1.2995, + "theoretical_loss": 3.4502399055038184, + "tokens_seen": 2061434880 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003034062186559679, + "loss": 1.3767, + "theoretical_loss": 3.4502308894454465, + "tokens_seen": 2061500416 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003033901705115346, + "loss": 1.1735, + "theoretical_loss": 3.4502218737539456, + "tokens_seen": 2061565952 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003033741223671013, + "loss": 1.3899, + "theoretical_loss": 3.450212858429291, + "tokens_seen": 2061631488 + }, + { + "epoch": 0.62, + "learning_rate": 0.000303358074222668, + "loss": 1.2221, + "theoretical_loss": 3.4502038434714546, + "tokens_seen": 2061697024 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003033420260782347, + "loss": 1.3501, + "theoretical_loss": 3.4501948288804103, + "tokens_seen": 2061762560 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003033259779338014, + "loss": 1.2522, + "theoretical_loss": 3.4501858146561313, + "tokens_seen": 2061828096 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003033099297893681, + "loss": 1.2616, + "theoretical_loss": 3.450176800798591, + "tokens_seen": 2061893632 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030329388164493484, + "loss": 1.2815, + "theoretical_loss": 3.4501677873077634, + "tokens_seen": 2061959168 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030327783350050155, + "loss": 1.121, + "theoretical_loss": 3.4501587741836213, + "tokens_seen": 2062024704 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030326178535606825, + "loss": 1.3171, + "theoretical_loss": 3.4501497614261387, + "tokens_seen": 2062090240 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030324573721163495, + "loss": 1.2913, + "theoretical_loss": 3.4501407490352887, + "tokens_seen": 2062155776 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030322968906720165, + "loss": 1.2652, + "theoretical_loss": 3.4501317370110445, + "tokens_seen": 2062221312 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030321364092276835, + "loss": 1.1489, + "theoretical_loss": 3.45012272535338, + "tokens_seen": 2062286848 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030319759277833505, + "loss": 1.2835, + "theoretical_loss": 3.450113714062268, + "tokens_seen": 2062352384 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030318154463390175, + "loss": 1.2961, + "theoretical_loss": 3.450104703137683, + "tokens_seen": 2062417920 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030316549648946845, + "loss": 1.1276, + "theoretical_loss": 3.4500956925795974, + "tokens_seen": 2062483456 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030314944834503515, + "loss": 1.2574, + "theoretical_loss": 3.450086682387985, + "tokens_seen": 2062548992 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030313340020060185, + "loss": 1.3237, + "theoretical_loss": 3.450077672562819, + "tokens_seen": 2062614528 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030311735205616855, + "loss": 1.2853, + "theoretical_loss": 3.450068663104074, + "tokens_seen": 2062680064 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030310130391173525, + "loss": 1.248, + "theoretical_loss": 3.4500596540117217, + "tokens_seen": 2062745600 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030308525576730195, + "loss": 1.344, + "theoretical_loss": 3.4500506452857373, + "tokens_seen": 2062811136 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030306920762286865, + "loss": 1.3345, + "theoretical_loss": 3.450041636926093, + "tokens_seen": 2062876672 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030305315947843535, + "loss": 1.1895, + "theoretical_loss": 3.450032628932763, + "tokens_seen": 2062942208 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030303711133400205, + "loss": 1.2672, + "theoretical_loss": 3.45002362130572, + "tokens_seen": 2063007744 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030302106318956876, + "loss": 1.2416, + "theoretical_loss": 3.4500146140449384, + "tokens_seen": 2063073280 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030300501504513546, + "loss": 1.3193, + "theoretical_loss": 3.450005607150391, + "tokens_seen": 2063138816 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030298896690070216, + "loss": 1.3452, + "theoretical_loss": 3.4499966006220513, + "tokens_seen": 2063204352 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030297291875626886, + "loss": 1.1495, + "theoretical_loss": 3.449987594459893, + "tokens_seen": 2063269888 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003029568706118355, + "loss": 1.3296, + "theoretical_loss": 3.44997858866389, + "tokens_seen": 2063335424 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003029408224674022, + "loss": 1.2986, + "theoretical_loss": 3.449969583234015, + "tokens_seen": 2063400960 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003029247743229689, + "loss": 1.2938, + "theoretical_loss": 3.4499605781702423, + "tokens_seen": 2063466496 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003029087261785356, + "loss": 1.2456, + "theoretical_loss": 3.4499515734725446, + "tokens_seen": 2063532032 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003028926780341023, + "loss": 1.2635, + "theoretical_loss": 3.4499425691408954, + "tokens_seen": 2063597568 + }, + { + "epoch": 0.63, + "learning_rate": 0.000302876629889669, + "loss": 1.3664, + "theoretical_loss": 3.449933565175269, + "tokens_seen": 2063663104 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003028605817452357, + "loss": 1.2576, + "theoretical_loss": 3.4499245615756378, + "tokens_seen": 2063728640 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003028445336008024, + "loss": 1.3084, + "theoretical_loss": 3.449915558341976, + "tokens_seen": 2063794176 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003028284854563691, + "loss": 1.2893, + "theoretical_loss": 3.449906555474257, + "tokens_seen": 2063859712 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003028124373119358, + "loss": 1.2733, + "theoretical_loss": 3.449897552972455, + "tokens_seen": 2063925248 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003027963891675025, + "loss": 1.2683, + "theoretical_loss": 3.4498885508365422, + "tokens_seen": 2063990784 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003027803410230692, + "loss": 1.1852, + "theoretical_loss": 3.449879549066493, + "tokens_seen": 2064056320 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003027642928786359, + "loss": 1.1761, + "theoretical_loss": 3.4498705476622806, + "tokens_seen": 2064121856 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003027482447342026, + "loss": 1.2735, + "theoretical_loss": 3.4498615466238784, + "tokens_seen": 2064187392 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003027321965897693, + "loss": 1.3544, + "theoretical_loss": 3.44985254595126, + "tokens_seen": 2064252928 + }, + { + "epoch": 0.63, + "learning_rate": 0.000302716148445336, + "loss": 1.3144, + "theoretical_loss": 3.4498435456443994, + "tokens_seen": 2064318464 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 690841, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.329054832458496, + "objective/train/theoretical_loss": 3.4498345457032698, + "objective/train/tokens_used": 2084844000, + "theoretical_loss": 3.4498345457032698, + "tokens_seen": 2064384000 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003027001003009027, + "loss": 1.2285, + "theoretical_loss": 3.4498345457032698, + "tokens_seen": 2064384000 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003026840521564694, + "loss": 1.267, + "theoretical_loss": 3.449825546127844, + "tokens_seen": 2064449536 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003026680040120361, + "loss": 1.2597, + "theoretical_loss": 3.449816546918097, + "tokens_seen": 2064515072 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003026519558676028, + "loss": 1.249, + "theoretical_loss": 3.449807548074001, + "tokens_seen": 2064580608 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003026359077231695, + "loss": 1.1722, + "theoretical_loss": 3.4497985495955303, + "tokens_seen": 2064646144 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003026198595787362, + "loss": 1.2829, + "theoretical_loss": 3.4497895514826578, + "tokens_seen": 2064711680 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003026038114343029, + "loss": 1.3092, + "theoretical_loss": 3.4497805537353576, + "tokens_seen": 2064777216 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003025877632898696, + "loss": 1.2339, + "theoretical_loss": 3.4497715563536033, + "tokens_seen": 2064842752 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003025717151454363, + "loss": 1.2411, + "theoretical_loss": 3.449762559337368, + "tokens_seen": 2064908288 + }, + { + "epoch": 0.63, + "learning_rate": 0.000302555667001003, + "loss": 1.273, + "theoretical_loss": 3.449753562686626, + "tokens_seen": 2064973824 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003025396188565697, + "loss": 1.2267, + "theoretical_loss": 3.4497445664013497, + "tokens_seen": 2065039360 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003025235707121364, + "loss": 1.3444, + "theoretical_loss": 3.449735570481514, + "tokens_seen": 2065104896 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003025075225677031, + "loss": 1.271, + "theoretical_loss": 3.4497265749270913, + "tokens_seen": 2065170432 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003024914744232698, + "loss": 1.3509, + "theoretical_loss": 3.4497175797380555, + "tokens_seen": 2065235968 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003024754262788365, + "loss": 1.3245, + "theoretical_loss": 3.4497085849143807, + "tokens_seen": 2065301504 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003024593781344032, + "loss": 1.1995, + "theoretical_loss": 3.44969959045604, + "tokens_seen": 2065367040 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003024433299899699, + "loss": 1.2226, + "theoretical_loss": 3.4496905963630065, + "tokens_seen": 2065432576 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003024272818455366, + "loss": 1.2995, + "theoretical_loss": 3.449681602635255, + "tokens_seen": 2065498112 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003024112337011033, + "loss": 1.2104, + "theoretical_loss": 3.449672609272758, + "tokens_seen": 2065563648 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030239518555667, + "loss": 1.2581, + "theoretical_loss": 3.4496636162754895, + "tokens_seen": 2065629184 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003023791374122367, + "loss": 1.3995, + "theoretical_loss": 3.449654623643423, + "tokens_seen": 2065694720 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030236308926780343, + "loss": 1.2915, + "theoretical_loss": 3.449645631376532, + "tokens_seen": 2065760256 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030234704112337013, + "loss": 1.2394, + "theoretical_loss": 3.4496366394747904, + "tokens_seen": 2065825792 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030233099297893683, + "loss": 1.226, + "theoretical_loss": 3.449627647938171, + "tokens_seen": 2065891328 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030231494483450353, + "loss": 1.2297, + "theoretical_loss": 3.449618656766649, + "tokens_seen": 2065956864 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030229889669007023, + "loss": 1.3147, + "theoretical_loss": 3.4496096659601965, + "tokens_seen": 2066022400 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030228284854563693, + "loss": 1.297, + "theoretical_loss": 3.4496006755187874, + "tokens_seen": 2066087936 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030226680040120363, + "loss": 1.2629, + "theoretical_loss": 3.4495916854423956, + "tokens_seen": 2066153472 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030225075225677033, + "loss": 1.2883, + "theoretical_loss": 3.449582695730995, + "tokens_seen": 2066219008 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030223470411233703, + "loss": 1.2547, + "theoretical_loss": 3.449573706384558, + "tokens_seen": 2066284544 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030221865596790373, + "loss": 1.2043, + "theoretical_loss": 3.4495647174030593, + "tokens_seen": 2066350080 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030220260782347043, + "loss": 1.2756, + "theoretical_loss": 3.449555728786472, + "tokens_seen": 2066415616 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003021865596790371, + "loss": 1.2408, + "theoretical_loss": 3.44954674053477, + "tokens_seen": 2066481152 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003021705115346038, + "loss": 1.2913, + "theoretical_loss": 3.4495377526479274, + "tokens_seen": 2066546688 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003021544633901705, + "loss": 1.2239, + "theoretical_loss": 3.449528765125917, + "tokens_seen": 2066612224 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003021384152457372, + "loss": 1.3645, + "theoretical_loss": 3.449519777968712, + "tokens_seen": 2066677760 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003021223671013039, + "loss": 1.256, + "theoretical_loss": 3.4495107911762877, + "tokens_seen": 2066743296 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003021063189568706, + "loss": 1.1682, + "theoretical_loss": 3.4495018047486163, + "tokens_seen": 2066808832 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030209027081243734, + "loss": 1.2273, + "theoretical_loss": 3.449492818685672, + "tokens_seen": 2066874368 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030207422266800404, + "loss": 1.3652, + "theoretical_loss": 3.4494838329874282, + "tokens_seen": 2066939904 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030205817452357074, + "loss": 1.2044, + "theoretical_loss": 3.449474847653858, + "tokens_seen": 2067005440 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030204212637913744, + "loss": 1.283, + "theoretical_loss": 3.4494658626849364, + "tokens_seen": 2067070976 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030202607823470414, + "loss": 1.1578, + "theoretical_loss": 3.4494568780806363, + "tokens_seen": 2067136512 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030201003009027084, + "loss": 1.2333, + "theoretical_loss": 3.449447893840931, + "tokens_seen": 2067202048 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030199398194583754, + "loss": 1.2334, + "theoretical_loss": 3.4494389099657945, + "tokens_seen": 2067267584 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030197793380140424, + "loss": 1.2486, + "theoretical_loss": 3.449429926455201, + "tokens_seen": 2067333120 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030196188565697094, + "loss": 1.2371, + "theoretical_loss": 3.4494209433091227, + "tokens_seen": 2067398656 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030194583751253764, + "loss": 1.2584, + "theoretical_loss": 3.4494119605275344, + "tokens_seen": 2067464192 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030192978936810434, + "loss": 1.2016, + "theoretical_loss": 3.44940297811041, + "tokens_seen": 2067529728 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030191374122367105, + "loss": 1.0863, + "theoretical_loss": 3.4493939960577222, + "tokens_seen": 2067595264 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 692057, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3954368829727173, + "objective/train/theoretical_loss": 3.4493850143694456, + "objective/train/tokens_used": 2088120800, + "theoretical_loss": 3.4493850143694456, + "tokens_seen": 2067660800 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030189769307923775, + "loss": 1.2912, + "theoretical_loss": 3.4493850143694456, + "tokens_seen": 2067660800 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030188164493480445, + "loss": 1.3383, + "theoretical_loss": 3.449376033045553, + "tokens_seen": 2067726336 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030186559679037115, + "loss": 1.2597, + "theoretical_loss": 3.4493670520860187, + "tokens_seen": 2067791872 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030184954864593785, + "loss": 1.3128, + "theoretical_loss": 3.449358071490816, + "tokens_seen": 2067857408 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030183350050150455, + "loss": 1.2128, + "theoretical_loss": 3.4493490912599185, + "tokens_seen": 2067922944 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030181745235707125, + "loss": 1.3361, + "theoretical_loss": 3.4493401113933007, + "tokens_seen": 2067988480 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030180140421263795, + "loss": 1.1504, + "theoretical_loss": 3.4493311318909354, + "tokens_seen": 2068054016 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030178535606820465, + "loss": 1.3057, + "theoretical_loss": 3.4493221527527966, + "tokens_seen": 2068119552 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030176930792377135, + "loss": 1.2004, + "theoretical_loss": 3.449313173978857, + "tokens_seen": 2068185088 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030175325977933805, + "loss": 1.2571, + "theoretical_loss": 3.4493041955690926, + "tokens_seen": 2068250624 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030173721163490475, + "loss": 1.3868, + "theoretical_loss": 3.449295217523475, + "tokens_seen": 2068316160 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030172116349047145, + "loss": 1.2875, + "theoretical_loss": 3.449286239841979, + "tokens_seen": 2068381696 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030170511534603815, + "loss": 1.2768, + "theoretical_loss": 3.4492772625245776, + "tokens_seen": 2068447232 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030168906720160485, + "loss": 1.3147, + "theoretical_loss": 3.449268285571245, + "tokens_seen": 2068512768 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030167301905717155, + "loss": 1.2807, + "theoretical_loss": 3.4492593089819543, + "tokens_seen": 2068578304 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030165697091273826, + "loss": 1.3243, + "theoretical_loss": 3.44925033275668, + "tokens_seen": 2068643840 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030164092276830496, + "loss": 1.356, + "theoretical_loss": 3.449241356895395, + "tokens_seen": 2068709376 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030162487462387166, + "loss": 1.2533, + "theoretical_loss": 3.449232381398074, + "tokens_seen": 2068774912 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030160882647943836, + "loss": 1.293, + "theoretical_loss": 3.44922340626469, + "tokens_seen": 2068840448 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030159277833500506, + "loss": 1.1874, + "theoretical_loss": 3.4492144314952164, + "tokens_seen": 2068905984 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030157673019057176, + "loss": 1.2967, + "theoretical_loss": 3.4492054570896276, + "tokens_seen": 2068971520 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030156068204613846, + "loss": 1.2675, + "theoretical_loss": 3.4491964830478974, + "tokens_seen": 2069037056 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003015446339017051, + "loss": 1.2156, + "theoretical_loss": 3.449187509369999, + "tokens_seen": 2069102592 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003015285857572718, + "loss": 1.2127, + "theoretical_loss": 3.449178536055906, + "tokens_seen": 2069168128 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003015125376128385, + "loss": 1.3288, + "theoretical_loss": 3.4491695631055928, + "tokens_seen": 2069233664 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003014964894684052, + "loss": 1.2187, + "theoretical_loss": 3.449160590519033, + "tokens_seen": 2069299200 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003014804413239719, + "loss": 1.2864, + "theoretical_loss": 3.4491516182962, + "tokens_seen": 2069364736 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003014643931795386, + "loss": 1.2844, + "theoretical_loss": 3.4491426464370676, + "tokens_seen": 2069430272 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003014483450351053, + "loss": 1.2983, + "theoretical_loss": 3.44913367494161, + "tokens_seen": 2069495808 + }, + { + "epoch": 0.63, + "learning_rate": 0.000301432296890672, + "loss": 1.1868, + "theoretical_loss": 3.4491247038098, + "tokens_seen": 2069561344 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003014162487462387, + "loss": 1.2534, + "theoretical_loss": 3.4491157330416122, + "tokens_seen": 2069626880 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003014002006018054, + "loss": 1.2948, + "theoretical_loss": 3.4491067626370198, + "tokens_seen": 2069692416 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003013841524573721, + "loss": 1.1721, + "theoretical_loss": 3.449097792595997, + "tokens_seen": 2069757952 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003013681043129388, + "loss": 1.2437, + "theoretical_loss": 3.449088822918517, + "tokens_seen": 2069823488 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003013520561685055, + "loss": 1.1332, + "theoretical_loss": 3.4490798536045544, + "tokens_seen": 2069889024 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003013360080240722, + "loss": 1.2327, + "theoretical_loss": 3.4490708846540823, + "tokens_seen": 2069954560 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003013199598796389, + "loss": 1.2069, + "theoretical_loss": 3.449061916067074, + "tokens_seen": 2070020096 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003013039117352056, + "loss": 1.1512, + "theoretical_loss": 3.449052947843505, + "tokens_seen": 2070085632 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003012878635907723, + "loss": 1.303, + "theoretical_loss": 3.4490439799833474, + "tokens_seen": 2070151168 + }, + { + "epoch": 0.63, + "learning_rate": 0.000301271815446339, + "loss": 1.2966, + "theoretical_loss": 3.4490350124865756, + "tokens_seen": 2070216704 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003012557673019057, + "loss": 1.2817, + "theoretical_loss": 3.449026045353163, + "tokens_seen": 2070282240 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003012397191574724, + "loss": 1.2739, + "theoretical_loss": 3.449017078583084, + "tokens_seen": 2070347776 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003012236710130391, + "loss": 1.3001, + "theoretical_loss": 3.449008112176312, + "tokens_seen": 2070413312 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003012076228686058, + "loss": 1.2974, + "theoretical_loss": 3.448999146132821, + "tokens_seen": 2070478848 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003011915747241725, + "loss": 1.2203, + "theoretical_loss": 3.4489901804525847, + "tokens_seen": 2070544384 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003011755265797392, + "loss": 1.2141, + "theoretical_loss": 3.4489812151355763, + "tokens_seen": 2070609920 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003011594784353059, + "loss": 1.2504, + "theoretical_loss": 3.4489722501817703, + "tokens_seen": 2070675456 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003011434302908726, + "loss": 1.2738, + "theoretical_loss": 3.44896328559114, + "tokens_seen": 2070740992 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003011273821464393, + "loss": 1.1986, + "theoretical_loss": 3.44895432136366, + "tokens_seen": 2070806528 + }, + { + "epoch": 0.63, + "learning_rate": 0.000301111334002006, + "loss": 1.2715, + "theoretical_loss": 3.448945357499303, + "tokens_seen": 2070872064 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 693021, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3823573589324951, + "objective/train/theoretical_loss": 3.4489363939980446, + "objective/train/tokens_used": 2091397600, + "theoretical_loss": 3.4489363939980446, + "tokens_seen": 2070937600 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003010952858575727, + "loss": 1.356, + "theoretical_loss": 3.4489363939980446, + "tokens_seen": 2070937600 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003010792377131394, + "loss": 1.2463, + "theoretical_loss": 3.4489274308598565, + "tokens_seen": 2071003136 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003010631895687061, + "loss": 1.1544, + "theoretical_loss": 3.448918468084713, + "tokens_seen": 2071068672 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003010471414242728, + "loss": 1.2459, + "theoretical_loss": 3.4489095056725887, + "tokens_seen": 2071134208 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003010310932798395, + "loss": 1.1969, + "theoretical_loss": 3.4489005436234574, + "tokens_seen": 2071199744 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003010150451354062, + "loss": 1.181, + "theoretical_loss": 3.448891581937292, + "tokens_seen": 2071265280 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003009989969909729, + "loss": 1.2674, + "theoretical_loss": 3.448882620614067, + "tokens_seen": 2071330816 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030098294884653963, + "loss": 1.251, + "theoretical_loss": 3.4488736596537564, + "tokens_seen": 2071396352 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030096690070210633, + "loss": 1.3765, + "theoretical_loss": 3.448864699056333, + "tokens_seen": 2071461888 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030095085255767303, + "loss": 1.2401, + "theoretical_loss": 3.4488557388217718, + "tokens_seen": 2071527424 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030093480441323973, + "loss": 1.2398, + "theoretical_loss": 3.4488467789500454, + "tokens_seen": 2071592960 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030091875626880643, + "loss": 1.2332, + "theoretical_loss": 3.4488378194411293, + "tokens_seen": 2071658496 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030090270812437313, + "loss": 1.2895, + "theoretical_loss": 3.448828860294996, + "tokens_seen": 2071724032 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030088665997993983, + "loss": 1.1977, + "theoretical_loss": 3.4488199015116194, + "tokens_seen": 2071789568 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030087061183550653, + "loss": 1.2498, + "theoretical_loss": 3.448810943090974, + "tokens_seen": 2071855104 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030085456369107323, + "loss": 1.2028, + "theoretical_loss": 3.4488019850330334, + "tokens_seen": 2071920640 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030083851554663993, + "loss": 1.2523, + "theoretical_loss": 3.4487930273377714, + "tokens_seen": 2071986176 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030082246740220663, + "loss": 1.3033, + "theoretical_loss": 3.4487840700051615, + "tokens_seen": 2072051712 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030080641925777333, + "loss": 1.2357, + "theoretical_loss": 3.4487751130351776, + "tokens_seen": 2072117248 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030079037111334004, + "loss": 1.2496, + "theoretical_loss": 3.4487661564277943, + "tokens_seen": 2072182784 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030077432296890674, + "loss": 1.3466, + "theoretical_loss": 3.4487572001829845, + "tokens_seen": 2072248320 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030075827482447344, + "loss": 1.3033, + "theoretical_loss": 3.448748244300723, + "tokens_seen": 2072313856 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030074222668004014, + "loss": 1.1555, + "theoretical_loss": 3.4487392887809825, + "tokens_seen": 2072379392 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030072617853560684, + "loss": 1.1323, + "theoretical_loss": 3.448730333623738, + "tokens_seen": 2072444928 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030071013039117354, + "loss": 1.2852, + "theoretical_loss": 3.448721378828963, + "tokens_seen": 2072510464 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030069408224674024, + "loss": 1.2393, + "theoretical_loss": 3.4487124243966307, + "tokens_seen": 2072576000 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030067803410230694, + "loss": 1.3238, + "theoretical_loss": 3.448703470326716, + "tokens_seen": 2072641536 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030066198595787364, + "loss": 1.3059, + "theoretical_loss": 3.448694516619192, + "tokens_seen": 2072707072 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030064593781344034, + "loss": 1.1547, + "theoretical_loss": 3.448685563274033, + "tokens_seen": 2072772608 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030062988966900704, + "loss": 1.2905, + "theoretical_loss": 3.448676610291213, + "tokens_seen": 2072838144 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030061384152457374, + "loss": 1.2688, + "theoretical_loss": 3.448667657670705, + "tokens_seen": 2072903680 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030059779338014044, + "loss": 1.2954, + "theoretical_loss": 3.448658705412484, + "tokens_seen": 2072969216 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030058174523570714, + "loss": 1.3422, + "theoretical_loss": 3.4486497535165235, + "tokens_seen": 2073034752 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030056569709127384, + "loss": 1.1013, + "theoretical_loss": 3.4486408019827968, + "tokens_seen": 2073100288 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030054964894684054, + "loss": 1.193, + "theoretical_loss": 3.4486318508112785, + "tokens_seen": 2073165824 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030053360080240725, + "loss": 1.3257, + "theoretical_loss": 3.4486229000019426, + "tokens_seen": 2073231360 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030051755265797395, + "loss": 1.2226, + "theoretical_loss": 3.4486139495547623, + "tokens_seen": 2073296896 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030050150451354065, + "loss": 1.3084, + "theoretical_loss": 3.448604999469712, + "tokens_seen": 2073362432 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030048545636910735, + "loss": 1.3016, + "theoretical_loss": 3.4485960497467656, + "tokens_seen": 2073427968 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030046940822467405, + "loss": 1.2284, + "theoretical_loss": 3.4485871003858968, + "tokens_seen": 2073493504 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030045336008024075, + "loss": 1.3217, + "theoretical_loss": 3.4485781513870792, + "tokens_seen": 2073559040 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030043731193580745, + "loss": 1.2323, + "theoretical_loss": 3.4485692027502877, + "tokens_seen": 2073624576 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030042126379137415, + "loss": 1.2912, + "theoretical_loss": 3.4485602544754954, + "tokens_seen": 2073690112 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030040521564694085, + "loss": 1.2085, + "theoretical_loss": 3.448551306562676, + "tokens_seen": 2073755648 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030038916750250755, + "loss": 1.2638, + "theoretical_loss": 3.448542359011804, + "tokens_seen": 2073821184 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030037311935807425, + "loss": 1.2204, + "theoretical_loss": 3.4485334118228534, + "tokens_seen": 2073886720 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030035707121364095, + "loss": 1.2571, + "theoretical_loss": 3.4485244649957982, + "tokens_seen": 2073952256 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030034102306920765, + "loss": 1.2493, + "theoretical_loss": 3.448515518530612, + "tokens_seen": 2074017792 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030032497492477435, + "loss": 1.2684, + "theoretical_loss": 3.4485065724272683, + "tokens_seen": 2074083328 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030030892678034105, + "loss": 1.2903, + "theoretical_loss": 3.4484976266857412, + "tokens_seen": 2074148864 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 694242, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2040023803710938, + "objective/train/theoretical_loss": 3.4484886813060056, + "objective/train/tokens_used": 2094674400, + "theoretical_loss": 3.4484886813060056, + "tokens_seen": 2074214400 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030029287863590775, + "loss": 1.2175, + "theoretical_loss": 3.4484886813060056, + "tokens_seen": 2074214400 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030027683049147446, + "loss": 1.3644, + "theoretical_loss": 3.448479736288035, + "tokens_seen": 2074279936 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030026078234704116, + "loss": 1.2882, + "theoretical_loss": 3.4484707916318023, + "tokens_seen": 2074345472 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030024473420260786, + "loss": 1.2555, + "theoretical_loss": 3.448461847337282, + "tokens_seen": 2074411008 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030022868605817456, + "loss": 1.205, + "theoretical_loss": 3.4484529034044495, + "tokens_seen": 2074476544 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030021263791374126, + "loss": 1.2684, + "theoretical_loss": 3.4484439598332766, + "tokens_seen": 2074542080 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030019658976930796, + "loss": 1.2296, + "theoretical_loss": 3.4484350166237387, + "tokens_seen": 2074607616 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030018054162487466, + "loss": 1.2432, + "theoretical_loss": 3.4484260737758086, + "tokens_seen": 2074673152 + }, + { + "epoch": 0.63, + "learning_rate": 0.00030016449348044136, + "loss": 1.1925, + "theoretical_loss": 3.448417131289461, + "tokens_seen": 2074738688 + }, + { + "epoch": 0.63, + "learning_rate": 0.000300148445336008, + "loss": 1.2899, + "theoretical_loss": 3.4484081891646703, + "tokens_seen": 2074804224 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003001323971915747, + "loss": 1.3365, + "theoretical_loss": 3.44839924740141, + "tokens_seen": 2074869760 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003001163490471414, + "loss": 1.251, + "theoretical_loss": 3.4483903059996535, + "tokens_seen": 2074935296 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003001003009027081, + "loss": 1.2492, + "theoretical_loss": 3.448381364959375, + "tokens_seen": 2075000832 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003000842527582748, + "loss": 1.2864, + "theoretical_loss": 3.4483724242805494, + "tokens_seen": 2075066368 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003000682046138415, + "loss": 1.2724, + "theoretical_loss": 3.44836348396315, + "tokens_seen": 2075131904 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003000521564694082, + "loss": 1.1596, + "theoretical_loss": 3.44835454400715, + "tokens_seen": 2075197440 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003000361083249749, + "loss": 1.274, + "theoretical_loss": 3.448345604412525, + "tokens_seen": 2075262976 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003000200601805416, + "loss": 1.2431, + "theoretical_loss": 3.4483366651792475, + "tokens_seen": 2075328512 + }, + { + "epoch": 0.63, + "learning_rate": 0.0003000040120361083, + "loss": 1.2547, + "theoretical_loss": 3.4483277263072925, + "tokens_seen": 2075394048 + }, + { + "epoch": 0.63, + "learning_rate": 0.000299987963891675, + "loss": 1.2258, + "theoretical_loss": 3.4483187877966337, + "tokens_seen": 2075459584 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002999719157472417, + "loss": 1.1796, + "theoretical_loss": 3.4483098496472446, + "tokens_seen": 2075525120 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002999558676028084, + "loss": 1.1559, + "theoretical_loss": 3.4483009118591, + "tokens_seen": 2075590656 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002999398194583751, + "loss": 1.3152, + "theoretical_loss": 3.448291974432173, + "tokens_seen": 2075656192 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002999237713139418, + "loss": 1.1918, + "theoretical_loss": 3.448283037366438, + "tokens_seen": 2075721728 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002999077231695085, + "loss": 1.2346, + "theoretical_loss": 3.4482741006618696, + "tokens_seen": 2075787264 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002998916750250752, + "loss": 1.2611, + "theoretical_loss": 3.4482651643184408, + "tokens_seen": 2075852800 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002998756268806419, + "loss": 1.2301, + "theoretical_loss": 3.4482562283361267, + "tokens_seen": 2075918336 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002998595787362086, + "loss": 1.1728, + "theoretical_loss": 3.4482472927148997, + "tokens_seen": 2075983872 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002998435305917753, + "loss": 1.3189, + "theoretical_loss": 3.448238357454736, + "tokens_seen": 2076049408 + }, + { + "epoch": 0.63, + "learning_rate": 0.000299827482447342, + "loss": 1.2279, + "theoretical_loss": 3.448229422555608, + "tokens_seen": 2076114944 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002998114343029087, + "loss": 1.4536, + "theoretical_loss": 3.4482204880174896, + "tokens_seen": 2076180480 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002997953861584754, + "loss": 1.1946, + "theoretical_loss": 3.4482115538403555, + "tokens_seen": 2076246016 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002997793380140421, + "loss": 1.2129, + "theoretical_loss": 3.4482026200241798, + "tokens_seen": 2076311552 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002997632898696088, + "loss": 1.232, + "theoretical_loss": 3.448193686568936, + "tokens_seen": 2076377088 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002997472417251755, + "loss": 1.371, + "theoretical_loss": 3.4481847534745986, + "tokens_seen": 2076442624 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002997311935807422, + "loss": 1.2729, + "theoretical_loss": 3.4481758207411417, + "tokens_seen": 2076508160 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002997151454363089, + "loss": 1.2036, + "theoretical_loss": 3.4481668883685392, + "tokens_seen": 2076573696 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002996990972918757, + "loss": 1.2965, + "theoretical_loss": 3.4481579563567646, + "tokens_seen": 2076639232 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002996830491474424, + "loss": 1.2059, + "theoretical_loss": 3.448149024705792, + "tokens_seen": 2076704768 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002996670010030091, + "loss": 1.2239, + "theoretical_loss": 3.4481400934155966, + "tokens_seen": 2076770304 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002996509528585758, + "loss": 1.1539, + "theoretical_loss": 3.4481311624861513, + "tokens_seen": 2076835840 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002996349047141425, + "loss": 1.231, + "theoretical_loss": 3.4481222319174307, + "tokens_seen": 2076901376 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002996188565697092, + "loss": 1.2545, + "theoretical_loss": 3.448113301709408, + "tokens_seen": 2076966912 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002996028084252759, + "loss": 1.243, + "theoretical_loss": 3.4481043718620583, + "tokens_seen": 2077032448 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002995867602808426, + "loss": 1.2651, + "theoretical_loss": 3.4480954423753554, + "tokens_seen": 2077097984 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002995707121364093, + "loss": 1.3025, + "theoretical_loss": 3.448086513249273, + "tokens_seen": 2077163520 + }, + { + "epoch": 0.63, + "learning_rate": 0.000299554663991976, + "loss": 1.3754, + "theoretical_loss": 3.4480775844837854, + "tokens_seen": 2077229056 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002995386158475427, + "loss": 1.2639, + "theoretical_loss": 3.4480686560788665, + "tokens_seen": 2077294592 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029952256770310933, + "loss": 1.1699, + "theoretical_loss": 3.4480597280344902, + "tokens_seen": 2077360128 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029950651955867603, + "loss": 1.281, + "theoretical_loss": 3.4480508003506314, + "tokens_seen": 2077425664 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 695255, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.125720739364624, + "objective/train/theoretical_loss": 3.4480418730272637, + "objective/train/tokens_used": 2097951200, + "theoretical_loss": 3.4480418730272637, + "tokens_seen": 2077491200 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029949047141424273, + "loss": 1.237, + "theoretical_loss": 3.4480418730272637, + "tokens_seen": 2077491200 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029947442326980943, + "loss": 1.22, + "theoretical_loss": 3.4480329460643606, + "tokens_seen": 2077556736 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029945837512537613, + "loss": 1.254, + "theoretical_loss": 3.448024019461897, + "tokens_seen": 2077622272 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029944232698094283, + "loss": 1.2214, + "theoretical_loss": 3.4480150932198463, + "tokens_seen": 2077687808 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029942627883650953, + "loss": 1.1979, + "theoretical_loss": 3.448006167338183, + "tokens_seen": 2077753344 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029941023069207624, + "loss": 1.2815, + "theoretical_loss": 3.447997241816881, + "tokens_seen": 2077818880 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029939418254764294, + "loss": 1.1726, + "theoretical_loss": 3.447988316655915, + "tokens_seen": 2077884416 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029937813440320964, + "loss": 1.2554, + "theoretical_loss": 3.447979391855258, + "tokens_seen": 2077949952 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029936208625877634, + "loss": 1.3344, + "theoretical_loss": 3.447970467414885, + "tokens_seen": 2078015488 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029934603811434304, + "loss": 1.1866, + "theoretical_loss": 3.44796154333477, + "tokens_seen": 2078081024 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029932998996990974, + "loss": 1.1973, + "theoretical_loss": 3.4479526196148864, + "tokens_seen": 2078146560 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029931394182547644, + "loss": 1.2373, + "theoretical_loss": 3.447943696255209, + "tokens_seen": 2078212096 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029929789368104314, + "loss": 1.1871, + "theoretical_loss": 3.447934773255712, + "tokens_seen": 2078277632 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029928184553660984, + "loss": 1.276, + "theoretical_loss": 3.447925850616368, + "tokens_seen": 2078343168 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029926579739217654, + "loss": 1.2682, + "theoretical_loss": 3.447916928337153, + "tokens_seen": 2078408704 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029924974924774324, + "loss": 1.1884, + "theoretical_loss": 3.4479080064180407, + "tokens_seen": 2078474240 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029923370110330994, + "loss": 1.3241, + "theoretical_loss": 3.4478990848590048, + "tokens_seen": 2078539776 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029921765295887664, + "loss": 1.3157, + "theoretical_loss": 3.447890163660019, + "tokens_seen": 2078605312 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029920160481444334, + "loss": 1.3705, + "theoretical_loss": 3.447881242821058, + "tokens_seen": 2078670848 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029918555667001004, + "loss": 1.2891, + "theoretical_loss": 3.4478723223420964, + "tokens_seen": 2078736384 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029916950852557675, + "loss": 1.2647, + "theoretical_loss": 3.447863402223107, + "tokens_seen": 2078801920 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029915346038114345, + "loss": 1.3147, + "theoretical_loss": 3.4478544824640656, + "tokens_seen": 2078867456 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029913741223671015, + "loss": 1.24, + "theoretical_loss": 3.447845563064945, + "tokens_seen": 2078932992 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029912136409227685, + "loss": 1.3141, + "theoretical_loss": 3.4478366440257195, + "tokens_seen": 2078998528 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029910531594784355, + "loss": 1.2183, + "theoretical_loss": 3.447827725346364, + "tokens_seen": 2079064064 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029908926780341025, + "loss": 1.2379, + "theoretical_loss": 3.447818807026852, + "tokens_seen": 2079129600 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029907321965897695, + "loss": 1.2783, + "theoretical_loss": 3.447809889067157, + "tokens_seen": 2079195136 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029905717151454365, + "loss": 1.2371, + "theoretical_loss": 3.447800971467255, + "tokens_seen": 2079260672 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029904112337011035, + "loss": 1.2518, + "theoretical_loss": 3.4477920542271185, + "tokens_seen": 2079326208 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029902507522567705, + "loss": 1.3157, + "theoretical_loss": 3.4477831373467223, + "tokens_seen": 2079391744 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029900902708124375, + "loss": 1.2536, + "theoretical_loss": 3.4477742208260405, + "tokens_seen": 2079457280 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029899297893681045, + "loss": 1.2268, + "theoretical_loss": 3.4477653046650474, + "tokens_seen": 2079522816 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029897693079237715, + "loss": 1.322, + "theoretical_loss": 3.4477563888637164, + "tokens_seen": 2079588352 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029896088264794385, + "loss": 1.2246, + "theoretical_loss": 3.4477474734220226, + "tokens_seen": 2079653888 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029894483450351055, + "loss": 1.2457, + "theoretical_loss": 3.44773855833994, + "tokens_seen": 2079719424 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029892878635907725, + "loss": 1.2413, + "theoretical_loss": 3.447729643617442, + "tokens_seen": 2079784960 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029891273821464396, + "loss": 1.2093, + "theoretical_loss": 3.4477207292545033, + "tokens_seen": 2079850496 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029889669007021066, + "loss": 1.3034, + "theoretical_loss": 3.4477118152510986, + "tokens_seen": 2079916032 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029888064192577736, + "loss": 1.273, + "theoretical_loss": 3.447702901607201, + "tokens_seen": 2079981568 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029886459378134406, + "loss": 1.2364, + "theoretical_loss": 3.4476939883227855, + "tokens_seen": 2080047104 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029884854563691076, + "loss": 1.2857, + "theoretical_loss": 3.4476850753978265, + "tokens_seen": 2080112640 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029883249749247746, + "loss": 1.2876, + "theoretical_loss": 3.447676162832297, + "tokens_seen": 2080178176 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029881644934804416, + "loss": 1.2884, + "theoretical_loss": 3.447667250626172, + "tokens_seen": 2080243712 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029880040120361086, + "loss": 1.3132, + "theoretical_loss": 3.4476583387794255, + "tokens_seen": 2080309248 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029878435305917756, + "loss": 1.3254, + "theoretical_loss": 3.447649427292032, + "tokens_seen": 2080374784 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029876830491474426, + "loss": 1.4085, + "theoretical_loss": 3.447640516163965, + "tokens_seen": 2080440320 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002987522567703109, + "loss": 1.2418, + "theoretical_loss": 3.4476316053951996, + "tokens_seen": 2080505856 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002987362086258776, + "loss": 1.1698, + "theoretical_loss": 3.447622694985709, + "tokens_seen": 2080571392 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002987201604814443, + "loss": 1.2114, + "theoretical_loss": 3.447613784935468, + "tokens_seen": 2080636928 + }, + { + "epoch": 0.63, + "learning_rate": 0.000298704112337011, + "loss": 1.198, + "theoretical_loss": 3.4476048752444513, + "tokens_seen": 2080702464 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 696439, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2649786472320557, + "objective/train/theoretical_loss": 3.4475959659126323, + "objective/train/tokens_used": 2101228000, + "theoretical_loss": 3.4475959659126323, + "tokens_seen": 2080768000 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002986880641925777, + "loss": 1.2952, + "theoretical_loss": 3.4475959659126323, + "tokens_seen": 2080768000 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002986720160481444, + "loss": 1.3234, + "theoretical_loss": 3.447587056939985, + "tokens_seen": 2080833536 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002986559679037111, + "loss": 1.1959, + "theoretical_loss": 3.4475781483264845, + "tokens_seen": 2080899072 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002986399197592778, + "loss": 1.1853, + "theoretical_loss": 3.447569240072104, + "tokens_seen": 2080964608 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002986238716148445, + "loss": 1.22, + "theoretical_loss": 3.4475603321768187, + "tokens_seen": 2081030144 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002986078234704112, + "loss": 1.2734, + "theoretical_loss": 3.4475514246406025, + "tokens_seen": 2081095680 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002985917753259779, + "loss": 1.2263, + "theoretical_loss": 3.447542517463429, + "tokens_seen": 2081161216 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002985757271815446, + "loss": 1.2903, + "theoretical_loss": 3.447533610645273, + "tokens_seen": 2081226752 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002985596790371113, + "loss": 1.3758, + "theoretical_loss": 3.447524704186109, + "tokens_seen": 2081292288 + }, + { + "epoch": 0.63, + "learning_rate": 0.000298543630892678, + "loss": 1.2131, + "theoretical_loss": 3.4475157980859104, + "tokens_seen": 2081357824 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002985275827482447, + "loss": 1.2638, + "theoretical_loss": 3.447506892344652, + "tokens_seen": 2081423360 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002985115346038114, + "loss": 1.2597, + "theoretical_loss": 3.4474979869623086, + "tokens_seen": 2081488896 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029849548645937817, + "loss": 1.3461, + "theoretical_loss": 3.447489081938853, + "tokens_seen": 2081554432 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029847943831494487, + "loss": 1.3332, + "theoretical_loss": 3.4474801772742607, + "tokens_seen": 2081619968 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002984633901705116, + "loss": 1.2435, + "theoretical_loss": 3.447471272968505, + "tokens_seen": 2081685504 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002984473420260783, + "loss": 1.192, + "theoretical_loss": 3.4474623690215607, + "tokens_seen": 2081751040 + }, + { + "epoch": 0.63, + "learning_rate": 0.000298431293881645, + "loss": 1.2103, + "theoretical_loss": 3.4474534654334024, + "tokens_seen": 2081816576 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002984152457372117, + "loss": 1.1925, + "theoretical_loss": 3.4474445622040033, + "tokens_seen": 2081882112 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002983991975927784, + "loss": 1.2782, + "theoretical_loss": 3.4474356593333386, + "tokens_seen": 2081947648 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002983831494483451, + "loss": 1.2284, + "theoretical_loss": 3.4474267568213817, + "tokens_seen": 2082013184 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002983671013039118, + "loss": 1.2486, + "theoretical_loss": 3.4474178546681076, + "tokens_seen": 2082078720 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002983510531594785, + "loss": 1.333, + "theoretical_loss": 3.4474089528734906, + "tokens_seen": 2082144256 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002983350050150452, + "loss": 1.2327, + "theoretical_loss": 3.4474000514375045, + "tokens_seen": 2082209792 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002983189568706119, + "loss": 1.2449, + "theoretical_loss": 3.4473911503601236, + "tokens_seen": 2082275328 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002983029087261786, + "loss": 1.1468, + "theoretical_loss": 3.4473822496413224, + "tokens_seen": 2082340864 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002982868605817453, + "loss": 1.2998, + "theoretical_loss": 3.447373349281075, + "tokens_seen": 2082406400 + }, + { + "epoch": 0.63, + "learning_rate": 0.000298270812437312, + "loss": 1.2871, + "theoretical_loss": 3.447364449279356, + "tokens_seen": 2082471936 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002982547642928787, + "loss": 1.2146, + "theoretical_loss": 3.447355549636139, + "tokens_seen": 2082537472 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002982387161484454, + "loss": 1.2798, + "theoretical_loss": 3.447346650351399, + "tokens_seen": 2082603008 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002982226680040121, + "loss": 1.1781, + "theoretical_loss": 3.44733775142511, + "tokens_seen": 2082668544 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002982066198595788, + "loss": 1.2442, + "theoretical_loss": 3.447328852857246, + "tokens_seen": 2082734080 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002981905717151455, + "loss": 1.291, + "theoretical_loss": 3.4473199546477815, + "tokens_seen": 2082799616 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002981745235707122, + "loss": 1.2626, + "theoretical_loss": 3.447311056796691, + "tokens_seen": 2082865152 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002981584754262789, + "loss": 1.206, + "theoretical_loss": 3.4473021593039483, + "tokens_seen": 2082930688 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002981424272818456, + "loss": 1.3288, + "theoretical_loss": 3.4472932621695285, + "tokens_seen": 2082996224 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002981263791374123, + "loss": 1.2887, + "theoretical_loss": 3.447284365393405, + "tokens_seen": 2083061760 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029811033099297893, + "loss": 1.3171, + "theoretical_loss": 3.447275468975553, + "tokens_seen": 2083127296 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029809428284854563, + "loss": 1.2533, + "theoretical_loss": 3.447266572915946, + "tokens_seen": 2083192832 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029807823470411233, + "loss": 1.2579, + "theoretical_loss": 3.4472576772145587, + "tokens_seen": 2083258368 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029806218655967903, + "loss": 1.3818, + "theoretical_loss": 3.4472487818713655, + "tokens_seen": 2083323904 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029804613841524574, + "loss": 1.265, + "theoretical_loss": 3.44723988688634, + "tokens_seen": 2083389440 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029803009027081244, + "loss": 1.1827, + "theoretical_loss": 3.447230992259458, + "tokens_seen": 2083454976 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029801404212637914, + "loss": 1.2305, + "theoretical_loss": 3.447222097990692, + "tokens_seen": 2083520512 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029799799398194584, + "loss": 1.2945, + "theoretical_loss": 3.4472132040800174, + "tokens_seen": 2083586048 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029798194583751254, + "loss": 1.2065, + "theoretical_loss": 3.4472043105274084, + "tokens_seen": 2083651584 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029796589769307924, + "loss": 1.2726, + "theoretical_loss": 3.447195417332839, + "tokens_seen": 2083717120 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029794984954864594, + "loss": 1.2113, + "theoretical_loss": 3.447186524496284, + "tokens_seen": 2083782656 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029793380140421264, + "loss": 1.2808, + "theoretical_loss": 3.447177632017717, + "tokens_seen": 2083848192 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029791775325977934, + "loss": 1.1871, + "theoretical_loss": 3.447168739897114, + "tokens_seen": 2083913728 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029790170511534604, + "loss": 1.249, + "theoretical_loss": 3.447159848134447, + "tokens_seen": 2083979264 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 697706, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2984927892684937, + "objective/train/theoretical_loss": 3.4471509567296916, + "objective/train/tokens_used": 2104504800, + "theoretical_loss": 3.4471509567296916, + "tokens_seen": 2084044800 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029788565697091274, + "loss": 1.3036, + "theoretical_loss": 3.4471509567296916, + "tokens_seen": 2084044800 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029786960882647944, + "loss": 1.2431, + "theoretical_loss": 3.4471420656828227, + "tokens_seen": 2084110336 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029785356068204614, + "loss": 1.1548, + "theoretical_loss": 3.447133174993813, + "tokens_seen": 2084175872 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029783751253761284, + "loss": 1.2451, + "theoretical_loss": 3.447124284662639, + "tokens_seen": 2084241408 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029782146439317954, + "loss": 1.2719, + "theoretical_loss": 3.447115394689273, + "tokens_seen": 2084306944 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029780541624874624, + "loss": 1.3192, + "theoretical_loss": 3.44710650507369, + "tokens_seen": 2084372480 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029778936810431295, + "loss": 1.3094, + "theoretical_loss": 3.4470976158158653, + "tokens_seen": 2084438016 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029777331995987965, + "loss": 1.2804, + "theoretical_loss": 3.447088726915772, + "tokens_seen": 2084503552 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029775727181544635, + "loss": 1.2738, + "theoretical_loss": 3.4470798383733854, + "tokens_seen": 2084569088 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029774122367101305, + "loss": 1.2337, + "theoretical_loss": 3.447070950188679, + "tokens_seen": 2084634624 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029772517552657975, + "loss": 1.2368, + "theoretical_loss": 3.4470620623616277, + "tokens_seen": 2084700160 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029770912738214645, + "loss": 1.3068, + "theoretical_loss": 3.447053174892206, + "tokens_seen": 2084765696 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029769307923771315, + "loss": 1.3187, + "theoretical_loss": 3.447044287780387, + "tokens_seen": 2084831232 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029767703109327985, + "loss": 1.3644, + "theoretical_loss": 3.447035401026147, + "tokens_seen": 2084896768 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029766098294884655, + "loss": 1.2888, + "theoretical_loss": 3.4470265146294596, + "tokens_seen": 2084962304 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029764493480441325, + "loss": 1.2274, + "theoretical_loss": 3.4470176285902987, + "tokens_seen": 2085027840 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029762888665997995, + "loss": 1.26, + "theoretical_loss": 3.4470087429086393, + "tokens_seen": 2085093376 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029761283851554665, + "loss": 1.1482, + "theoretical_loss": 3.446999857584455, + "tokens_seen": 2085158912 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029759679037111335, + "loss": 1.29, + "theoretical_loss": 3.446990972617721, + "tokens_seen": 2085224448 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029758074222668005, + "loss": 1.1949, + "theoretical_loss": 3.4469820880084114, + "tokens_seen": 2085289984 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029756469408224675, + "loss": 1.2758, + "theoretical_loss": 3.4469732037565004, + "tokens_seen": 2085355520 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029754864593781345, + "loss": 1.212, + "theoretical_loss": 3.446964319861963, + "tokens_seen": 2085421056 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029753259779338016, + "loss": 1.1884, + "theoretical_loss": 3.4469554363247727, + "tokens_seen": 2085486592 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029751654964894686, + "loss": 1.2054, + "theoretical_loss": 3.4469465531449046, + "tokens_seen": 2085552128 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029750050150451356, + "loss": 1.2183, + "theoretical_loss": 3.446937670322333, + "tokens_seen": 2085617664 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029748445336008026, + "loss": 1.1213, + "theoretical_loss": 3.446928787857032, + "tokens_seen": 2085683200 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029746840521564696, + "loss": 1.1907, + "theoretical_loss": 3.4469199057489757, + "tokens_seen": 2085748736 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029745235707121366, + "loss": 1.2878, + "theoretical_loss": 3.4469110239981395, + "tokens_seen": 2085814272 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029743630892678036, + "loss": 1.1109, + "theoretical_loss": 3.446902142604497, + "tokens_seen": 2085879808 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029742026078234706, + "loss": 1.2331, + "theoretical_loss": 3.4468932615680226, + "tokens_seen": 2085945344 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029740421263791376, + "loss": 1.3025, + "theoretical_loss": 3.4468843808886915, + "tokens_seen": 2086010880 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029738816449348046, + "loss": 1.2117, + "theoretical_loss": 3.4468755005664775, + "tokens_seen": 2086076416 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029737211634904716, + "loss": 1.3032, + "theoretical_loss": 3.446866620601355, + "tokens_seen": 2086141952 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029735606820461386, + "loss": 1.2764, + "theoretical_loss": 3.446857740993299, + "tokens_seen": 2086207488 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002973400200601805, + "loss": 1.268, + "theoretical_loss": 3.446848861742283, + "tokens_seen": 2086273024 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002973239719157472, + "loss": 1.303, + "theoretical_loss": 3.4468399828482816, + "tokens_seen": 2086338560 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002973079237713139, + "loss": 1.2532, + "theoretical_loss": 3.4468311043112703, + "tokens_seen": 2086404096 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029729187562688067, + "loss": 1.3145, + "theoretical_loss": 3.4468222261312222, + "tokens_seen": 2086469632 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029727582748244737, + "loss": 1.2108, + "theoretical_loss": 3.4468133483081127, + "tokens_seen": 2086535168 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029725977933801407, + "loss": 1.3114, + "theoretical_loss": 3.446804470841916, + "tokens_seen": 2086600704 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029724373119358077, + "loss": 1.3037, + "theoretical_loss": 3.446795593732606, + "tokens_seen": 2086666240 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029722768304914747, + "loss": 1.3834, + "theoretical_loss": 3.4467867169801574, + "tokens_seen": 2086731776 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029721163490471417, + "loss": 1.2186, + "theoretical_loss": 3.446777840584545, + "tokens_seen": 2086797312 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029719558676028087, + "loss": 1.2076, + "theoretical_loss": 3.4467689645457433, + "tokens_seen": 2086862848 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029717953861584757, + "loss": 1.2087, + "theoretical_loss": 3.4467600888637264, + "tokens_seen": 2086928384 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029716349047141427, + "loss": 1.1611, + "theoretical_loss": 3.4467512135384686, + "tokens_seen": 2086993920 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029714744232698097, + "loss": 1.2285, + "theoretical_loss": 3.446742338569945, + "tokens_seen": 2087059456 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029713139418254767, + "loss": 1.1244, + "theoretical_loss": 3.4467334639581293, + "tokens_seen": 2087124992 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029711534603811437, + "loss": 1.3254, + "theoretical_loss": 3.4467245897029963, + "tokens_seen": 2087190528 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029709929789368107, + "loss": 1.287, + "theoretical_loss": 3.4467157158045207, + "tokens_seen": 2087256064 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 698576, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2330121994018555, + "objective/train/theoretical_loss": 3.446706842262677, + "objective/train/tokens_used": 2107781600, + "theoretical_loss": 3.446706842262677, + "tokens_seen": 2087321600 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002970832497492478, + "loss": 1.237, + "theoretical_loss": 3.446706842262677, + "tokens_seen": 2087321600 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002970672016048145, + "loss": 1.21, + "theoretical_loss": 3.446697969077439, + "tokens_seen": 2087387136 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002970511534603812, + "loss": 1.1829, + "theoretical_loss": 3.4466890962487815, + "tokens_seen": 2087452672 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002970351053159479, + "loss": 1.2723, + "theoretical_loss": 3.4466802237766796, + "tokens_seen": 2087518208 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002970190571715146, + "loss": 1.2567, + "theoretical_loss": 3.4466713516611067, + "tokens_seen": 2087583744 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002970030090270813, + "loss": 1.2757, + "theoretical_loss": 3.4466624799020384, + "tokens_seen": 2087649280 + }, + { + "epoch": 0.63, + "learning_rate": 0.000296986960882648, + "loss": 1.2271, + "theoretical_loss": 3.4466536084994486, + "tokens_seen": 2087714816 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002969709127382147, + "loss": 1.2229, + "theoretical_loss": 3.4466447374533113, + "tokens_seen": 2087780352 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002969548645937814, + "loss": 1.2971, + "theoretical_loss": 3.446635866763602, + "tokens_seen": 2087845888 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002969388164493481, + "loss": 1.2552, + "theoretical_loss": 3.4466269964302945, + "tokens_seen": 2087911424 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002969227683049148, + "loss": 1.2638, + "theoretical_loss": 3.4466181264533633, + "tokens_seen": 2087976960 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002969067201604815, + "loss": 1.2966, + "theoretical_loss": 3.446609256832783, + "tokens_seen": 2088042496 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002968906720160482, + "loss": 1.2428, + "theoretical_loss": 3.4466003875685285, + "tokens_seen": 2088108032 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002968746238716149, + "loss": 1.2396, + "theoretical_loss": 3.446591518660574, + "tokens_seen": 2088173568 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002968585757271816, + "loss": 1.2942, + "theoretical_loss": 3.446582650108894, + "tokens_seen": 2088239104 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002968425275827483, + "loss": 1.3371, + "theoretical_loss": 3.4465737819134628, + "tokens_seen": 2088304640 + }, + { + "epoch": 0.63, + "learning_rate": 0.000296826479438315, + "loss": 1.2624, + "theoretical_loss": 3.4465649140742554, + "tokens_seen": 2088370176 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002968104312938817, + "loss": 1.3233, + "theoretical_loss": 3.4465560465912457, + "tokens_seen": 2088435712 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002967943831494484, + "loss": 1.2756, + "theoretical_loss": 3.446547179464409, + "tokens_seen": 2088501248 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002967783350050151, + "loss": 1.3114, + "theoretical_loss": 3.446538312693719, + "tokens_seen": 2088566784 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002967622868605818, + "loss": 1.2946, + "theoretical_loss": 3.4465294462791505, + "tokens_seen": 2088632320 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002967462387161485, + "loss": 1.1559, + "theoretical_loss": 3.446520580220678, + "tokens_seen": 2088697856 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002967301905717152, + "loss": 1.288, + "theoretical_loss": 3.4465117145182766, + "tokens_seen": 2088763392 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029671414242728183, + "loss": 1.4131, + "theoretical_loss": 3.44650284917192, + "tokens_seen": 2088828928 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029669809428284853, + "loss": 1.2796, + "theoretical_loss": 3.446493984181583, + "tokens_seen": 2088894464 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029668204613841524, + "loss": 1.3392, + "theoretical_loss": 3.4464851195472406, + "tokens_seen": 2088960000 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029666599799398194, + "loss": 1.2528, + "theoretical_loss": 3.446476255268867, + "tokens_seen": 2089025536 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029664994984954864, + "loss": 1.2454, + "theoretical_loss": 3.4464673913464363, + "tokens_seen": 2089091072 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029663390170511534, + "loss": 1.2182, + "theoretical_loss": 3.4464585277799236, + "tokens_seen": 2089156608 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029661785356068204, + "loss": 1.2263, + "theoretical_loss": 3.446449664569303, + "tokens_seen": 2089222144 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029660180541624874, + "loss": 1.2434, + "theoretical_loss": 3.44644080171455, + "tokens_seen": 2089287680 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029658575727181544, + "loss": 1.0842, + "theoretical_loss": 3.446431939215638, + "tokens_seen": 2089353216 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029656970912738214, + "loss": 1.2815, + "theoretical_loss": 3.4464230770725424, + "tokens_seen": 2089418752 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029655366098294884, + "loss": 1.2757, + "theoretical_loss": 3.4464142152852366, + "tokens_seen": 2089484288 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029653761283851554, + "loss": 1.2542, + "theoretical_loss": 3.446405353853697, + "tokens_seen": 2089549824 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029652156469408224, + "loss": 1.2442, + "theoretical_loss": 3.4463964927778963, + "tokens_seen": 2089615360 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029650551654964894, + "loss": 1.1499, + "theoretical_loss": 3.44638763205781, + "tokens_seen": 2089680896 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029648946840521564, + "loss": 1.2379, + "theoretical_loss": 3.446378771693413, + "tokens_seen": 2089746432 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029647342026078234, + "loss": 1.2965, + "theoretical_loss": 3.4463699116846787, + "tokens_seen": 2089811968 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029645737211634904, + "loss": 1.2244, + "theoretical_loss": 3.446361052031583, + "tokens_seen": 2089877504 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029644132397191574, + "loss": 1.386, + "theoretical_loss": 3.4463521927340994, + "tokens_seen": 2089943040 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029642527582748245, + "loss": 1.3183, + "theoretical_loss": 3.4463433337922034, + "tokens_seen": 2090008576 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029640922768304915, + "loss": 1.2349, + "theoretical_loss": 3.4463344752058687, + "tokens_seen": 2090074112 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029639317953861585, + "loss": 1.0941, + "theoretical_loss": 3.44632561697507, + "tokens_seen": 2090139648 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029637713139418255, + "loss": 1.2311, + "theoretical_loss": 3.4463167590997825, + "tokens_seen": 2090205184 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029636108324974925, + "loss": 1.2813, + "theoretical_loss": 3.4463079015799805, + "tokens_seen": 2090270720 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029634503510531595, + "loss": 1.2843, + "theoretical_loss": 3.446299044415639, + "tokens_seen": 2090336256 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029632898696088265, + "loss": 1.1622, + "theoretical_loss": 3.4462901876067313, + "tokens_seen": 2090401792 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029631293881644935, + "loss": 1.2731, + "theoretical_loss": 3.446281331153233, + "tokens_seen": 2090467328 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029629689067201605, + "loss": 1.3292, + "theoretical_loss": 3.446272475055119, + "tokens_seen": 2090532864 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 699756, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1224358081817627, + "objective/train/theoretical_loss": 3.446263619312363, + "objective/train/tokens_used": 2111058400, + "theoretical_loss": 3.446263619312363, + "tokens_seen": 2090598400 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029628084252758275, + "loss": 1.2192, + "theoretical_loss": 3.446263619312363, + "tokens_seen": 2090598400 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029626479438314945, + "loss": 1.191, + "theoretical_loss": 3.44625476392494, + "tokens_seen": 2090663936 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029624874623871615, + "loss": 1.2315, + "theoretical_loss": 3.446245908892825, + "tokens_seen": 2090729472 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029623269809428285, + "loss": 1.2851, + "theoretical_loss": 3.4462370542159917, + "tokens_seen": 2090795008 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029621664994984955, + "loss": 1.2403, + "theoretical_loss": 3.4462281998944153, + "tokens_seen": 2090860544 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029620060180541625, + "loss": 1.1767, + "theoretical_loss": 3.4462193459280708, + "tokens_seen": 2090926080 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029618455366098295, + "loss": 1.2788, + "theoretical_loss": 3.446210492316932, + "tokens_seen": 2090991616 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029616850551654966, + "loss": 1.2252, + "theoretical_loss": 3.446201639060974, + "tokens_seen": 2091057152 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029615245737211636, + "loss": 1.1338, + "theoretical_loss": 3.4461927861601716, + "tokens_seen": 2091122688 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029613640922768306, + "loss": 1.2614, + "theoretical_loss": 3.4461839336144986, + "tokens_seen": 2091188224 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029612036108324976, + "loss": 1.3678, + "theoretical_loss": 3.4461750814239305, + "tokens_seen": 2091253760 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029610431293881646, + "loss": 1.2843, + "theoretical_loss": 3.4461662295884414, + "tokens_seen": 2091319296 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029608826479438316, + "loss": 1.1918, + "theoretical_loss": 3.4461573781080066, + "tokens_seen": 2091384832 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029607221664994986, + "loss": 1.2304, + "theoretical_loss": 3.4461485269825993, + "tokens_seen": 2091450368 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029605616850551656, + "loss": 1.2525, + "theoretical_loss": 3.446139676212196, + "tokens_seen": 2091515904 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029604012036108326, + "loss": 1.1929, + "theoretical_loss": 3.4461308257967698, + "tokens_seen": 2091581440 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029602407221664996, + "loss": 1.3232, + "theoretical_loss": 3.446121975736296, + "tokens_seen": 2091646976 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029600802407221666, + "loss": 1.3202, + "theoretical_loss": 3.44611312603075, + "tokens_seen": 2091712512 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029599197592778336, + "loss": 1.2232, + "theoretical_loss": 3.4461042766801047, + "tokens_seen": 2091778048 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029597592778335006, + "loss": 1.2833, + "theoretical_loss": 3.446095427684336, + "tokens_seen": 2091843584 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029595987963891676, + "loss": 1.2782, + "theoretical_loss": 3.4460865790434183, + "tokens_seen": 2091909120 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029594383149448346, + "loss": 1.1673, + "theoretical_loss": 3.4460777307573265, + "tokens_seen": 2091974656 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029592778335005016, + "loss": 1.3066, + "theoretical_loss": 3.4460688828260344, + "tokens_seen": 2092040192 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029591173520561687, + "loss": 1.2642, + "theoretical_loss": 3.446060035249517, + "tokens_seen": 2092105728 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029589568706118357, + "loss": 1.286, + "theoretical_loss": 3.44605118802775, + "tokens_seen": 2092171264 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029587963891675027, + "loss": 1.1543, + "theoretical_loss": 3.446042341160707, + "tokens_seen": 2092236800 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029586359077231697, + "loss": 1.2497, + "theoretical_loss": 3.446033494648363, + "tokens_seen": 2092302336 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029584754262788367, + "loss": 1.3391, + "theoretical_loss": 3.4460246484906922, + "tokens_seen": 2092367872 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029583149448345037, + "loss": 1.1767, + "theoretical_loss": 3.44601580268767, + "tokens_seen": 2092433408 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029581544633901707, + "loss": 1.2437, + "theoretical_loss": 3.4460069572392706, + "tokens_seen": 2092498944 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029579939819458377, + "loss": 1.1901, + "theoretical_loss": 3.445998112145469, + "tokens_seen": 2092564480 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029578335005015047, + "loss": 1.2284, + "theoretical_loss": 3.4459892674062393, + "tokens_seen": 2092630016 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029576730190571717, + "loss": 1.2451, + "theoretical_loss": 3.445980423021557, + "tokens_seen": 2092695552 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029575125376128387, + "loss": 1.2625, + "theoretical_loss": 3.445971578991396, + "tokens_seen": 2092761088 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029573520561685057, + "loss": 1.2259, + "theoretical_loss": 3.4459627353157316, + "tokens_seen": 2092826624 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002957191574724173, + "loss": 1.2279, + "theoretical_loss": 3.445953891994538, + "tokens_seen": 2092892160 + }, + { + "epoch": 0.63, + "learning_rate": 0.000295703109327984, + "loss": 1.2454, + "theoretical_loss": 3.44594504902779, + "tokens_seen": 2092957696 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002956870611835507, + "loss": 1.2577, + "theoretical_loss": 3.4459362064154626, + "tokens_seen": 2093023232 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002956710130391174, + "loss": 1.2545, + "theoretical_loss": 3.44592736415753, + "tokens_seen": 2093088768 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002956549648946841, + "loss": 1.213, + "theoretical_loss": 3.445918522253968, + "tokens_seen": 2093154304 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002956389167502508, + "loss": 1.2772, + "theoretical_loss": 3.4459096807047502, + "tokens_seen": 2093219840 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002956228686058175, + "loss": 1.3243, + "theoretical_loss": 3.4459008395098514, + "tokens_seen": 2093285376 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002956068204613842, + "loss": 1.2862, + "theoretical_loss": 3.4458919986692464, + "tokens_seen": 2093350912 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002955907723169509, + "loss": 1.2633, + "theoretical_loss": 3.4458831581829106, + "tokens_seen": 2093416448 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002955747241725176, + "loss": 1.1911, + "theoretical_loss": 3.445874318050818, + "tokens_seen": 2093481984 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002955586760280843, + "loss": 1.3106, + "theoretical_loss": 3.4458654782729434, + "tokens_seen": 2093547520 + }, + { + "epoch": 0.63, + "learning_rate": 0.000295542627883651, + "loss": 1.1755, + "theoretical_loss": 3.4458566388492615, + "tokens_seen": 2093613056 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002955265797392177, + "loss": 1.2403, + "theoretical_loss": 3.445847799779747, + "tokens_seen": 2093678592 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002955105315947844, + "loss": 1.1971, + "theoretical_loss": 3.445838961064375, + "tokens_seen": 2093744128 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002954944834503511, + "loss": 1.315, + "theoretical_loss": 3.44583012270312, + "tokens_seen": 2093809664 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 700782, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0766267776489258, + "objective/train/theoretical_loss": 3.4458212846959566, + "objective/train/tokens_used": 2114335200, + "theoretical_loss": 3.4458212846959566, + "tokens_seen": 2093875200 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002954784353059178, + "loss": 1.1144, + "theoretical_loss": 3.4458212846959566, + "tokens_seen": 2093875200 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002954623871614845, + "loss": 1.1034, + "theoretical_loss": 3.4458124470428597, + "tokens_seen": 2093940736 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002954463390170512, + "loss": 1.2526, + "theoretical_loss": 3.445803609743804, + "tokens_seen": 2094006272 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002954302908726179, + "loss": 1.1599, + "theoretical_loss": 3.445794772798764, + "tokens_seen": 2094071808 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002954142427281846, + "loss": 1.1921, + "theoretical_loss": 3.445785936207715, + "tokens_seen": 2094137344 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002953981945837513, + "loss": 1.1896, + "theoretical_loss": 3.445777099970631, + "tokens_seen": 2094202880 + }, + { + "epoch": 0.63, + "learning_rate": 0.000295382146439318, + "loss": 1.342, + "theoretical_loss": 3.445768264087487, + "tokens_seen": 2094268416 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002953660982948847, + "loss": 1.2053, + "theoretical_loss": 3.4457594285582585, + "tokens_seen": 2094333952 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002953500501504514, + "loss": 1.1623, + "theoretical_loss": 3.445750593382919, + "tokens_seen": 2094399488 + }, + { + "epoch": 0.63, + "learning_rate": 0.0002953340020060181, + "loss": 1.339, + "theoretical_loss": 3.4457417585614447, + "tokens_seen": 2094465024 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029531795386158473, + "loss": 1.2398, + "theoretical_loss": 3.445732924093809, + "tokens_seen": 2094530560 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029530190571715144, + "loss": 1.2713, + "theoretical_loss": 3.4457240899799872, + "tokens_seen": 2094596096 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029528585757271814, + "loss": 1.2589, + "theoretical_loss": 3.445715256219954, + "tokens_seen": 2094661632 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029526980942828484, + "loss": 1.3002, + "theoretical_loss": 3.445706422813684, + "tokens_seen": 2094727168 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029525376128385154, + "loss": 1.2714, + "theoretical_loss": 3.445697589761153, + "tokens_seen": 2094792704 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029523771313941824, + "loss": 1.2613, + "theoretical_loss": 3.4456887570623342, + "tokens_seen": 2094858240 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029522166499498494, + "loss": 1.3529, + "theoretical_loss": 3.4456799247172034, + "tokens_seen": 2094923776 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029520561685055164, + "loss": 1.2225, + "theoretical_loss": 3.445671092725735, + "tokens_seen": 2094989312 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029518956870611834, + "loss": 1.2477, + "theoretical_loss": 3.4456622610879037, + "tokens_seen": 2095054848 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029517352056168504, + "loss": 1.2897, + "theoretical_loss": 3.445653429803685, + "tokens_seen": 2095120384 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029515747241725174, + "loss": 1.205, + "theoretical_loss": 3.4456445988730526, + "tokens_seen": 2095185920 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029514142427281844, + "loss": 1.2753, + "theoretical_loss": 3.445635768295982, + "tokens_seen": 2095251456 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029512537612838514, + "loss": 1.252, + "theoretical_loss": 3.445626938072448, + "tokens_seen": 2095316992 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029510932798395184, + "loss": 1.1538, + "theoretical_loss": 3.4456181082024253, + "tokens_seen": 2095382528 + }, + { + "epoch": 0.63, + "learning_rate": 0.00029509327983951854, + "loss": 1.2913, + "theoretical_loss": 3.445609278685888, + "tokens_seen": 2095448064 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029507723169508524, + "loss": 1.2468, + "theoretical_loss": 3.4456004495228116, + "tokens_seen": 2095513600 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029506118355065194, + "loss": 1.2389, + "theoretical_loss": 3.445591620713171, + "tokens_seen": 2095579136 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029504513540621865, + "loss": 1.2258, + "theoretical_loss": 3.445582792256941, + "tokens_seen": 2095644672 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029502908726178535, + "loss": 1.2837, + "theoretical_loss": 3.4455739641540957, + "tokens_seen": 2095710208 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029501303911735205, + "loss": 1.2303, + "theoretical_loss": 3.445565136404611, + "tokens_seen": 2095775744 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029499699097291875, + "loss": 1.2866, + "theoretical_loss": 3.4455563090084604, + "tokens_seen": 2095841280 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029498094282848545, + "loss": 1.3051, + "theoretical_loss": 3.4455474819656198, + "tokens_seen": 2095906816 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029496489468405215, + "loss": 1.2438, + "theoretical_loss": 3.4455386552760636, + "tokens_seen": 2095972352 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029494884653961885, + "loss": 1.2218, + "theoretical_loss": 3.4455298289397662, + "tokens_seen": 2096037888 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029493279839518555, + "loss": 1.1865, + "theoretical_loss": 3.4455210029567036, + "tokens_seen": 2096103424 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029491675025075225, + "loss": 1.3202, + "theoretical_loss": 3.445512177326849, + "tokens_seen": 2096168960 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029490070210631895, + "loss": 1.2313, + "theoretical_loss": 3.4455033520501788, + "tokens_seen": 2096234496 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002948846539618857, + "loss": 1.1672, + "theoretical_loss": 3.445494527126667, + "tokens_seen": 2096300032 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002948686058174524, + "loss": 1.1828, + "theoretical_loss": 3.4454857025562884, + "tokens_seen": 2096365568 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002948525576730191, + "loss": 1.3516, + "theoretical_loss": 3.4454768783390177, + "tokens_seen": 2096431104 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002948365095285858, + "loss": 1.2632, + "theoretical_loss": 3.4454680544748304, + "tokens_seen": 2096496640 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002948204613841525, + "loss": 1.2106, + "theoretical_loss": 3.445459230963701, + "tokens_seen": 2096562176 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002948044132397192, + "loss": 1.2206, + "theoretical_loss": 3.4454504078056036, + "tokens_seen": 2096627712 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002947883650952859, + "loss": 1.2807, + "theoretical_loss": 3.4454415850005145, + "tokens_seen": 2096693248 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002947723169508526, + "loss": 1.1625, + "theoretical_loss": 3.4454327625484074, + "tokens_seen": 2096758784 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002947562688064193, + "loss": 1.3225, + "theoretical_loss": 3.4454239404492575, + "tokens_seen": 2096824320 + }, + { + "epoch": 0.64, + "learning_rate": 0.000294740220661986, + "loss": 1.2591, + "theoretical_loss": 3.44541511870304, + "tokens_seen": 2096889856 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002947241725175527, + "loss": 1.3978, + "theoretical_loss": 3.445406297309729, + "tokens_seen": 2096955392 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002947081243731194, + "loss": 1.1998, + "theoretical_loss": 3.4453974762692994, + "tokens_seen": 2097020928 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002946920762286861, + "loss": 1.3152, + "theoretical_loss": 3.445388655581727, + "tokens_seen": 2097086464 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 702025, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.190444827079773, + "objective/train/theoretical_loss": 3.445379835246986, + "objective/train/tokens_used": 2117612000, + "theoretical_loss": 3.445379835246986, + "tokens_seen": 2097152000 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029467602808425276, + "loss": 1.2753, + "theoretical_loss": 3.445379835246986, + "tokens_seen": 2097152000 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029465997993981946, + "loss": 1.3138, + "theoretical_loss": 3.445371015265051, + "tokens_seen": 2097217536 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029464393179538616, + "loss": 1.249, + "theoretical_loss": 3.445362195635898, + "tokens_seen": 2097283072 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029462788365095286, + "loss": 1.2635, + "theoretical_loss": 3.4453533763595003, + "tokens_seen": 2097348608 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029461183550651956, + "loss": 1.2913, + "theoretical_loss": 3.445344557435834, + "tokens_seen": 2097414144 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029459578736208626, + "loss": 1.2187, + "theoretical_loss": 3.4453357388648733, + "tokens_seen": 2097479680 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029457973921765296, + "loss": 1.266, + "theoretical_loss": 3.4453269206465933, + "tokens_seen": 2097545216 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029456369107321966, + "loss": 1.2703, + "theoretical_loss": 3.4453181027809685, + "tokens_seen": 2097610752 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029454764292878637, + "loss": 1.245, + "theoretical_loss": 3.4453092852679745, + "tokens_seen": 2097676288 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029453159478435307, + "loss": 1.2037, + "theoretical_loss": 3.445300468107586, + "tokens_seen": 2097741824 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029451554663991977, + "loss": 1.315, + "theoretical_loss": 3.4452916512997773, + "tokens_seen": 2097807360 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029449949849548647, + "loss": 1.2049, + "theoretical_loss": 3.445282834844524, + "tokens_seen": 2097872896 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029448345035105317, + "loss": 1.1896, + "theoretical_loss": 3.4452740187418005, + "tokens_seen": 2097938432 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029446740220661987, + "loss": 1.2491, + "theoretical_loss": 3.445265202991582, + "tokens_seen": 2098003968 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029445135406218657, + "loss": 1.2117, + "theoretical_loss": 3.445256387593843, + "tokens_seen": 2098069504 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029443530591775327, + "loss": 1.2746, + "theoretical_loss": 3.445247572548559, + "tokens_seen": 2098135040 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029441925777331997, + "loss": 1.2843, + "theoretical_loss": 3.445238757855704, + "tokens_seen": 2098200576 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029440320962888667, + "loss": 1.2633, + "theoretical_loss": 3.445229943515254, + "tokens_seen": 2098266112 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029438716148445337, + "loss": 1.2815, + "theoretical_loss": 3.445221129527183, + "tokens_seen": 2098331648 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029437111334002007, + "loss": 1.2502, + "theoretical_loss": 3.4452123158914665, + "tokens_seen": 2098397184 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029435506519558677, + "loss": 1.3069, + "theoretical_loss": 3.4452035026080794, + "tokens_seen": 2098462720 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002943390170511535, + "loss": 1.2349, + "theoretical_loss": 3.4451946896769963, + "tokens_seen": 2098528256 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002943229689067202, + "loss": 1.2281, + "theoretical_loss": 3.445185877098192, + "tokens_seen": 2098593792 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002943069207622869, + "loss": 1.2486, + "theoretical_loss": 3.4451770648716415, + "tokens_seen": 2098659328 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002942908726178536, + "loss": 1.2226, + "theoretical_loss": 3.4451682529973198, + "tokens_seen": 2098724864 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002942748244734203, + "loss": 1.2831, + "theoretical_loss": 3.4451594414752025, + "tokens_seen": 2098790400 + }, + { + "epoch": 0.64, + "learning_rate": 0.000294258776328987, + "loss": 1.1652, + "theoretical_loss": 3.4451506303052635, + "tokens_seen": 2098855936 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002942427281845537, + "loss": 1.2444, + "theoretical_loss": 3.4451418194874783, + "tokens_seen": 2098921472 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002942266800401204, + "loss": 1.4216, + "theoretical_loss": 3.4451330090218217, + "tokens_seen": 2098987008 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002942106318956871, + "loss": 1.3034, + "theoretical_loss": 3.4451241989082684, + "tokens_seen": 2099052544 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002941945837512538, + "loss": 1.2486, + "theoretical_loss": 3.4451153891467934, + "tokens_seen": 2099118080 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002941785356068205, + "loss": 1.2809, + "theoretical_loss": 3.445106579737372, + "tokens_seen": 2099183616 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002941624874623872, + "loss": 1.2644, + "theoretical_loss": 3.4450977706799786, + "tokens_seen": 2099249152 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002941464393179539, + "loss": 1.2968, + "theoretical_loss": 3.445088961974589, + "tokens_seen": 2099314688 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002941303911735206, + "loss": 1.3076, + "theoretical_loss": 3.445080153621177, + "tokens_seen": 2099380224 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002941143430290873, + "loss": 1.2414, + "theoretical_loss": 3.4450713456197186, + "tokens_seen": 2099445760 + }, + { + "epoch": 0.64, + "learning_rate": 0.000294098294884654, + "loss": 1.2998, + "theoretical_loss": 3.445062537970188, + "tokens_seen": 2099511296 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002940822467402207, + "loss": 1.2556, + "theoretical_loss": 3.4450537306725604, + "tokens_seen": 2099576832 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002940661985957874, + "loss": 1.3093, + "theoretical_loss": 3.4450449237268113, + "tokens_seen": 2099642368 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002940501504513541, + "loss": 1.3336, + "theoretical_loss": 3.445036117132915, + "tokens_seen": 2099707904 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002940341023069208, + "loss": 1.1623, + "theoretical_loss": 3.4450273108908465, + "tokens_seen": 2099773440 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002940180541624875, + "loss": 1.3386, + "theoretical_loss": 3.4450185050005806, + "tokens_seen": 2099838976 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002940020060180542, + "loss": 1.2946, + "theoretical_loss": 3.445009699462093, + "tokens_seen": 2099904512 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002939859578736209, + "loss": 1.2569, + "theoretical_loss": 3.445000894275358, + "tokens_seen": 2099970048 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002939699097291876, + "loss": 1.3297, + "theoretical_loss": 3.444992089440351, + "tokens_seen": 2100035584 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002939538615847543, + "loss": 1.2595, + "theoretical_loss": 3.4449832849570465, + "tokens_seen": 2100101120 + }, + { + "epoch": 0.64, + "learning_rate": 0.000293937813440321, + "loss": 1.2265, + "theoretical_loss": 3.44497448082542, + "tokens_seen": 2100166656 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002939217652958877, + "loss": 1.3455, + "theoretical_loss": 3.444965677045446, + "tokens_seen": 2100232192 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029390571715145434, + "loss": 1.3225, + "theoretical_loss": 3.4449568736171, + "tokens_seen": 2100297728 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029388966900702104, + "loss": 1.201, + "theoretical_loss": 3.444948070540357, + "tokens_seen": 2100363264 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 702928, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3683278560638428, + "objective/train/theoretical_loss": 3.4449392678151907, + "objective/train/tokens_used": 2120888800, + "theoretical_loss": 3.4449392678151907, + "tokens_seen": 2100428800 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029387362086258774, + "loss": 1.3309, + "theoretical_loss": 3.4449392678151907, + "tokens_seen": 2100428800 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029385757271815444, + "loss": 1.3092, + "theoretical_loss": 3.4449304654415775, + "tokens_seen": 2100494336 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029384152457372114, + "loss": 1.2743, + "theoretical_loss": 3.4449216634194926, + "tokens_seen": 2100559872 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029382547642928784, + "loss": 1.3148, + "theoretical_loss": 3.4449128617489095, + "tokens_seen": 2100625408 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029380942828485454, + "loss": 1.2783, + "theoretical_loss": 3.4449040604298045, + "tokens_seen": 2100690944 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029379338014042124, + "loss": 1.3382, + "theoretical_loss": 3.444895259462152, + "tokens_seen": 2100756480 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029377733199598794, + "loss": 1.1861, + "theoretical_loss": 3.444886458845927, + "tokens_seen": 2100822016 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029376128385155464, + "loss": 1.2768, + "theoretical_loss": 3.444877658581105, + "tokens_seen": 2100887552 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029374523570712134, + "loss": 1.2145, + "theoretical_loss": 3.4448688586676606, + "tokens_seen": 2100953088 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029372918756268804, + "loss": 1.3157, + "theoretical_loss": 3.4448600591055687, + "tokens_seen": 2101018624 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029371313941825474, + "loss": 1.308, + "theoretical_loss": 3.444851259894805, + "tokens_seen": 2101084160 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029369709127382144, + "loss": 1.2375, + "theoretical_loss": 3.4448424610353436, + "tokens_seen": 2101149696 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002936810431293882, + "loss": 1.2907, + "theoretical_loss": 3.44483366252716, + "tokens_seen": 2101215232 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002936649949849549, + "loss": 1.2855, + "theoretical_loss": 3.444824864370229, + "tokens_seen": 2101280768 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002936489468405216, + "loss": 1.2564, + "theoretical_loss": 3.4448160665645258, + "tokens_seen": 2101346304 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002936328986960883, + "loss": 1.3233, + "theoretical_loss": 3.4448072691100258, + "tokens_seen": 2101411840 + }, + { + "epoch": 0.64, + "learning_rate": 0.000293616850551655, + "loss": 1.2523, + "theoretical_loss": 3.444798472006703, + "tokens_seen": 2101477376 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002936008024072217, + "loss": 1.3142, + "theoretical_loss": 3.4447896752545333, + "tokens_seen": 2101542912 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002935847542627884, + "loss": 1.2352, + "theoretical_loss": 3.444780878853492, + "tokens_seen": 2101608448 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002935687061183551, + "loss": 1.3131, + "theoretical_loss": 3.444772082803553, + "tokens_seen": 2101673984 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002935526579739218, + "loss": 1.2058, + "theoretical_loss": 3.444763287104692, + "tokens_seen": 2101739520 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002935366098294885, + "loss": 1.3306, + "theoretical_loss": 3.4447544917568846, + "tokens_seen": 2101805056 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002935205616850552, + "loss": 1.3626, + "theoretical_loss": 3.4447456967601044, + "tokens_seen": 2101870592 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002935045135406219, + "loss": 1.2997, + "theoretical_loss": 3.444736902114328, + "tokens_seen": 2101936128 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002934884653961886, + "loss": 1.1453, + "theoretical_loss": 3.4447281078195293, + "tokens_seen": 2102001664 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002934724172517553, + "loss": 1.3046, + "theoretical_loss": 3.4447193138756838, + "tokens_seen": 2102067200 + }, + { + "epoch": 0.64, + "learning_rate": 0.000293456369107322, + "loss": 1.2993, + "theoretical_loss": 3.444710520282767, + "tokens_seen": 2102132736 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002934403209628887, + "loss": 1.3058, + "theoretical_loss": 3.444701727040753, + "tokens_seen": 2102198272 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002934242728184554, + "loss": 1.3147, + "theoretical_loss": 3.4446929341496175, + "tokens_seen": 2102263808 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002934082246740221, + "loss": 1.207, + "theoretical_loss": 3.4446841416093354, + "tokens_seen": 2102329344 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002933921765295888, + "loss": 1.2134, + "theoretical_loss": 3.444675349419882, + "tokens_seen": 2102394880 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002933761283851555, + "loss": 1.3701, + "theoretical_loss": 3.444666557581232, + "tokens_seen": 2102460416 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002933600802407222, + "loss": 1.1788, + "theoretical_loss": 3.4446577660933606, + "tokens_seen": 2102525952 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002933440320962889, + "loss": 1.265, + "theoretical_loss": 3.444648974956243, + "tokens_seen": 2102591488 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002933279839518556, + "loss": 1.168, + "theoretical_loss": 3.444640184169854, + "tokens_seen": 2102657024 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002933119358074223, + "loss": 1.2341, + "theoretical_loss": 3.444631393734169, + "tokens_seen": 2102722560 + }, + { + "epoch": 0.64, + "learning_rate": 0.000293295887662989, + "loss": 1.2968, + "theoretical_loss": 3.444622603649163, + "tokens_seen": 2102788096 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029327983951855566, + "loss": 1.3049, + "theoretical_loss": 3.4446138139148106, + "tokens_seen": 2102853632 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029326379137412236, + "loss": 1.2331, + "theoretical_loss": 3.4446050245310875, + "tokens_seen": 2102919168 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029324774322968906, + "loss": 1.2285, + "theoretical_loss": 3.444596235497969, + "tokens_seen": 2102984704 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029323169508525576, + "loss": 1.3213, + "theoretical_loss": 3.4445874468154294, + "tokens_seen": 2103050240 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029321564694082246, + "loss": 1.2382, + "theoretical_loss": 3.444578658483444, + "tokens_seen": 2103115776 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029319959879638916, + "loss": 1.3081, + "theoretical_loss": 3.4445698705019883, + "tokens_seen": 2103181312 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029318355065195586, + "loss": 1.1937, + "theoretical_loss": 3.4445610828710373, + "tokens_seen": 2103246848 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029316750250752257, + "loss": 1.3231, + "theoretical_loss": 3.4445522955905656, + "tokens_seen": 2103312384 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029315145436308927, + "loss": 1.2709, + "theoretical_loss": 3.4445435086605487, + "tokens_seen": 2103377920 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029313540621865597, + "loss": 1.1468, + "theoretical_loss": 3.4445347220809617, + "tokens_seen": 2103443456 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029311935807422267, + "loss": 1.2762, + "theoretical_loss": 3.4445259358517797, + "tokens_seen": 2103508992 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029310330992978937, + "loss": 1.2432, + "theoretical_loss": 3.444517149972978, + "tokens_seen": 2103574528 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029308726178535607, + "loss": 1.2176, + "theoretical_loss": 3.444508364444531, + "tokens_seen": 2103640064 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 704157, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2861976623535156, + "objective/train/theoretical_loss": 3.4444995792664144, + "objective/train/tokens_used": 2124165600, + "theoretical_loss": 3.4444995792664144, + "tokens_seen": 2103705600 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029307121364092277, + "loss": 1.19, + "theoretical_loss": 3.4444995792664144, + "tokens_seen": 2103705600 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029305516549648947, + "loss": 1.2816, + "theoretical_loss": 3.4444907944386034, + "tokens_seen": 2103771136 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029303911735205617, + "loss": 1.2838, + "theoretical_loss": 3.4444820099610727, + "tokens_seen": 2103836672 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029302306920762287, + "loss": 1.2223, + "theoretical_loss": 3.4444732258337982, + "tokens_seen": 2103902208 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029300702106318957, + "loss": 1.3129, + "theoretical_loss": 3.444464442056754, + "tokens_seen": 2103967744 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029299097291875627, + "loss": 1.2967, + "theoretical_loss": 3.4444556586299155, + "tokens_seen": 2104033280 + }, + { + "epoch": 0.64, + "learning_rate": 0.000292974924774323, + "loss": 1.2356, + "theoretical_loss": 3.4444468755532585, + "tokens_seen": 2104098816 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002929588766298897, + "loss": 1.2547, + "theoretical_loss": 3.4444380928267577, + "tokens_seen": 2104164352 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002929428284854564, + "loss": 1.2811, + "theoretical_loss": 3.4444293104503876, + "tokens_seen": 2104229888 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002929267803410231, + "loss": 1.3758, + "theoretical_loss": 3.444420528424125, + "tokens_seen": 2104295424 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002929107321965898, + "loss": 1.3592, + "theoretical_loss": 3.444411746747943, + "tokens_seen": 2104360960 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002928946840521565, + "loss": 1.2085, + "theoretical_loss": 3.4444029654218182, + "tokens_seen": 2104426496 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002928786359077232, + "loss": 1.2908, + "theoretical_loss": 3.4443941844457253, + "tokens_seen": 2104492032 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002928625877632899, + "loss": 1.2905, + "theoretical_loss": 3.444385403819639, + "tokens_seen": 2104557568 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002928465396188566, + "loss": 1.2848, + "theoretical_loss": 3.444376623543535, + "tokens_seen": 2104623104 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002928304914744233, + "loss": 1.2827, + "theoretical_loss": 3.4443678436173886, + "tokens_seen": 2104688640 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029281444332999, + "loss": 1.2556, + "theoretical_loss": 3.4443590640411745, + "tokens_seen": 2104754176 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002927983951855567, + "loss": 1.2977, + "theoretical_loss": 3.444350284814868, + "tokens_seen": 2104819712 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002927823470411234, + "loss": 1.2106, + "theoretical_loss": 3.4443415059384446, + "tokens_seen": 2104885248 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002927662988966901, + "loss": 1.2026, + "theoretical_loss": 3.444332727411879, + "tokens_seen": 2104950784 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002927502507522568, + "loss": 1.228, + "theoretical_loss": 3.4443239492351463, + "tokens_seen": 2105016320 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002927342026078235, + "loss": 1.2969, + "theoretical_loss": 3.4443151714082223, + "tokens_seen": 2105081856 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002927181544633902, + "loss": 1.1926, + "theoretical_loss": 3.4443063939310816, + "tokens_seen": 2105147392 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002927021063189569, + "loss": 1.2152, + "theoretical_loss": 3.4442976168036994, + "tokens_seen": 2105212928 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002926860581745236, + "loss": 1.165, + "theoretical_loss": 3.4442888400260507, + "tokens_seen": 2105278464 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002926700100300903, + "loss": 1.2043, + "theoretical_loss": 3.4442800635981117, + "tokens_seen": 2105344000 + }, + { + "epoch": 0.64, + "learning_rate": 0.000292653961885657, + "loss": 1.2205, + "theoretical_loss": 3.4442712875198565, + "tokens_seen": 2105409536 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002926379137412237, + "loss": 1.2781, + "theoretical_loss": 3.44426251179126, + "tokens_seen": 2105475072 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002926218655967904, + "loss": 1.2227, + "theoretical_loss": 3.444253736412299, + "tokens_seen": 2105540608 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002926058174523571, + "loss": 1.2487, + "theoretical_loss": 3.444244961382948, + "tokens_seen": 2105606144 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002925897693079238, + "loss": 1.303, + "theoretical_loss": 3.444236186703181, + "tokens_seen": 2105671680 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002925737211634905, + "loss": 1.287, + "theoretical_loss": 3.4442274123729746, + "tokens_seen": 2105737216 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002925576730190572, + "loss": 1.2965, + "theoretical_loss": 3.4442186383923032, + "tokens_seen": 2105802752 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002925416248746239, + "loss": 1.1241, + "theoretical_loss": 3.4442098647611425, + "tokens_seen": 2105868288 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002925255767301906, + "loss": 1.3987, + "theoretical_loss": 3.444201091479467, + "tokens_seen": 2105933824 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029250952858575724, + "loss": 1.2754, + "theoretical_loss": 3.4441923185472536, + "tokens_seen": 2105999360 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029249348044132394, + "loss": 1.2785, + "theoretical_loss": 3.444183545964475, + "tokens_seen": 2106064896 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002924774322968907, + "loss": 1.2474, + "theoretical_loss": 3.444174773731109, + "tokens_seen": 2106130432 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002924613841524574, + "loss": 1.2127, + "theoretical_loss": 3.4441660018471287, + "tokens_seen": 2106195968 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002924453360080241, + "loss": 1.2591, + "theoretical_loss": 3.44415723031251, + "tokens_seen": 2106261504 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002924292878635908, + "loss": 1.3267, + "theoretical_loss": 3.444148459127229, + "tokens_seen": 2106327040 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002924132397191575, + "loss": 1.2089, + "theoretical_loss": 3.4441396882912594, + "tokens_seen": 2106392576 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002923971915747242, + "loss": 1.2744, + "theoretical_loss": 3.4441309178045776, + "tokens_seen": 2106458112 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002923811434302909, + "loss": 1.2721, + "theoretical_loss": 3.4441221476671586, + "tokens_seen": 2106523648 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002923650952858576, + "loss": 1.2363, + "theoretical_loss": 3.444113377878977, + "tokens_seen": 2106589184 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002923490471414243, + "loss": 1.248, + "theoretical_loss": 3.444104608440009, + "tokens_seen": 2106654720 + }, + { + "epoch": 0.64, + "learning_rate": 0.000292332998996991, + "loss": 1.1822, + "theoretical_loss": 3.4440958393502292, + "tokens_seen": 2106720256 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002923169508525577, + "loss": 1.2854, + "theoretical_loss": 3.444087070609613, + "tokens_seen": 2106785792 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002923009027081244, + "loss": 1.265, + "theoretical_loss": 3.444078302218135, + "tokens_seen": 2106851328 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002922848545636911, + "loss": 1.2895, + "theoretical_loss": 3.444069534175772, + "tokens_seen": 2106916864 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 705118, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3771543502807617, + "objective/train/theoretical_loss": 3.4440607664824974, + "objective/train/tokens_used": 2127442400, + "theoretical_loss": 3.4440607664824974, + "tokens_seen": 2106982400 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002922688064192578, + "loss": 1.3712, + "theoretical_loss": 3.4440607664824974, + "tokens_seen": 2106982400 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002922527582748245, + "loss": 1.2341, + "theoretical_loss": 3.4440519991382876, + "tokens_seen": 2107047936 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002922367101303912, + "loss": 1.2584, + "theoretical_loss": 3.4440432321431174, + "tokens_seen": 2107113472 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002922206619859579, + "loss": 1.2007, + "theoretical_loss": 3.444034465496962, + "tokens_seen": 2107179008 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002922046138415246, + "loss": 1.2798, + "theoretical_loss": 3.4440256991997975, + "tokens_seen": 2107244544 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002921885656970913, + "loss": 1.227, + "theoretical_loss": 3.444016933251598, + "tokens_seen": 2107310080 + }, + { + "epoch": 0.64, + "learning_rate": 0.000292172517552658, + "loss": 1.3767, + "theoretical_loss": 3.4440081676523397, + "tokens_seen": 2107375616 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002921564694082247, + "loss": 1.3215, + "theoretical_loss": 3.4439994024019973, + "tokens_seen": 2107441152 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002921404212637914, + "loss": 1.2753, + "theoretical_loss": 3.443990637500546, + "tokens_seen": 2107506688 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002921243731193581, + "loss": 1.2253, + "theoretical_loss": 3.4439818729479614, + "tokens_seen": 2107572224 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002921083249749248, + "loss": 1.2442, + "theoretical_loss": 3.4439731087442187, + "tokens_seen": 2107637760 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002920922768304915, + "loss": 1.2582, + "theoretical_loss": 3.443964344889293, + "tokens_seen": 2107703296 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002920762286860582, + "loss": 1.1671, + "theoretical_loss": 3.4439555813831593, + "tokens_seen": 2107768832 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002920601805416249, + "loss": 1.2539, + "theoretical_loss": 3.4439468182257937, + "tokens_seen": 2107834368 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002920441323971916, + "loss": 1.2867, + "theoretical_loss": 3.443938055417171, + "tokens_seen": 2107899904 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002920280842527583, + "loss": 1.2575, + "theoretical_loss": 3.4439292929572662, + "tokens_seen": 2107965440 + }, + { + "epoch": 0.64, + "learning_rate": 0.000292012036108325, + "loss": 1.1907, + "theoretical_loss": 3.4439205308460554, + "tokens_seen": 2108030976 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002919959879638917, + "loss": 1.29, + "theoretical_loss": 3.443911769083513, + "tokens_seen": 2108096512 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002919799398194584, + "loss": 1.3228, + "theoretical_loss": 3.4439030076696144, + "tokens_seen": 2108162048 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002919638916750251, + "loss": 1.2098, + "theoretical_loss": 3.4438942466043354, + "tokens_seen": 2108227584 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002919478435305918, + "loss": 1.2715, + "theoretical_loss": 3.443885485887651, + "tokens_seen": 2108293120 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002919317953861585, + "loss": 1.2784, + "theoretical_loss": 3.4438767255195364, + "tokens_seen": 2108358656 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002919157472417252, + "loss": 1.2612, + "theoretical_loss": 3.443867965499967, + "tokens_seen": 2108424192 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002918996990972919, + "loss": 1.2924, + "theoretical_loss": 3.4438592058289186, + "tokens_seen": 2108489728 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002918836509528586, + "loss": 1.1931, + "theoretical_loss": 3.4438504465063655, + "tokens_seen": 2108555264 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029186760280842526, + "loss": 1.3789, + "theoretical_loss": 3.4438416875322835, + "tokens_seen": 2108620800 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029185155466399196, + "loss": 1.2194, + "theoretical_loss": 3.443832928906648, + "tokens_seen": 2108686336 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029183550651955866, + "loss": 1.2794, + "theoretical_loss": 3.4438241706294344, + "tokens_seen": 2108751872 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029181945837512536, + "loss": 1.1751, + "theoretical_loss": 3.443815412700618, + "tokens_seen": 2108817408 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029180341023069207, + "loss": 1.2164, + "theoretical_loss": 3.4438066551201736, + "tokens_seen": 2108882944 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029178736208625877, + "loss": 1.2454, + "theoretical_loss": 3.443797897888077, + "tokens_seen": 2108948480 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029177131394182547, + "loss": 1.3378, + "theoretical_loss": 3.4437891410043036, + "tokens_seen": 2109014016 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029175526579739217, + "loss": 1.3135, + "theoretical_loss": 3.4437803844688286, + "tokens_seen": 2109079552 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029173921765295887, + "loss": 1.2013, + "theoretical_loss": 3.4437716282816266, + "tokens_seen": 2109145088 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029172316950852557, + "loss": 1.2716, + "theoretical_loss": 3.443762872442674, + "tokens_seen": 2109210624 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029170712136409227, + "loss": 1.3249, + "theoretical_loss": 3.443754116951946, + "tokens_seen": 2109276160 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029169107321965897, + "loss": 1.2818, + "theoretical_loss": 3.443745361809417, + "tokens_seen": 2109341696 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029167502507522567, + "loss": 1.2243, + "theoretical_loss": 3.4437366070150635, + "tokens_seen": 2109407232 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029165897693079237, + "loss": 1.282, + "theoretical_loss": 3.44372785256886, + "tokens_seen": 2109472768 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029164292878635907, + "loss": 1.2645, + "theoretical_loss": 3.4437190984707824, + "tokens_seen": 2109538304 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029162688064192577, + "loss": 1.2352, + "theoretical_loss": 3.4437103447208055, + "tokens_seen": 2109603840 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002916108324974925, + "loss": 1.2879, + "theoretical_loss": 3.443701591318905, + "tokens_seen": 2109669376 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002915947843530592, + "loss": 1.2639, + "theoretical_loss": 3.4436928382650565, + "tokens_seen": 2109734912 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002915787362086259, + "loss": 1.1082, + "theoretical_loss": 3.4436840855592346, + "tokens_seen": 2109800448 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002915626880641926, + "loss": 1.2633, + "theoretical_loss": 3.4436753332014156, + "tokens_seen": 2109865984 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002915466399197593, + "loss": 1.3113, + "theoretical_loss": 3.4436665811915734, + "tokens_seen": 2109931520 + }, + { + "epoch": 0.64, + "learning_rate": 0.000291530591775326, + "loss": 1.2668, + "theoretical_loss": 3.443657829529685, + "tokens_seen": 2109997056 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002915145436308927, + "loss": 1.3158, + "theoretical_loss": 3.443649078215725, + "tokens_seen": 2110062592 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002914984954864594, + "loss": 1.1845, + "theoretical_loss": 3.443640327249669, + "tokens_seen": 2110128128 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002914824473420261, + "loss": 1.2768, + "theoretical_loss": 3.4436315766314918, + "tokens_seen": 2110193664 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 706481, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1775219440460205, + "objective/train/theoretical_loss": 3.4436228263611692, + "objective/train/tokens_used": 2130719200, + "theoretical_loss": 3.4436228263611692, + "tokens_seen": 2110259200 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002914663991975928, + "loss": 1.2052, + "theoretical_loss": 3.4436228263611692, + "tokens_seen": 2110259200 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002914503510531595, + "loss": 1.2148, + "theoretical_loss": 3.443614076438677, + "tokens_seen": 2110324736 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002914343029087262, + "loss": 1.2983, + "theoretical_loss": 3.4436053268639895, + "tokens_seen": 2110390272 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002914182547642929, + "loss": 1.1328, + "theoretical_loss": 3.443596577637083, + "tokens_seen": 2110455808 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002914022066198596, + "loss": 1.2008, + "theoretical_loss": 3.4435878287579325, + "tokens_seen": 2110521344 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002913861584754263, + "loss": 1.3581, + "theoretical_loss": 3.443579080226513, + "tokens_seen": 2110586880 + }, + { + "epoch": 0.64, + "learning_rate": 0.000291370110330993, + "loss": 1.2269, + "theoretical_loss": 3.4435703320428006, + "tokens_seen": 2110652416 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002913540621865597, + "loss": 1.3272, + "theoretical_loss": 3.443561584206771, + "tokens_seen": 2110717952 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002913380140421264, + "loss": 1.2745, + "theoretical_loss": 3.443552836718398, + "tokens_seen": 2110783488 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002913219658976931, + "loss": 1.3242, + "theoretical_loss": 3.4435440895776583, + "tokens_seen": 2110849024 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002913059177532598, + "loss": 1.284, + "theoretical_loss": 3.443535342784527, + "tokens_seen": 2110914560 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029128986960882654, + "loss": 1.2567, + "theoretical_loss": 3.44352659633898, + "tokens_seen": 2110980096 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029127382146439324, + "loss": 1.2878, + "theoretical_loss": 3.4435178502409913, + "tokens_seen": 2111045632 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029125777331995994, + "loss": 1.2205, + "theoretical_loss": 3.4435091044905377, + "tokens_seen": 2111111168 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002912417251755266, + "loss": 1.2643, + "theoretical_loss": 3.4435003590875937, + "tokens_seen": 2111176704 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002912256770310933, + "loss": 1.2432, + "theoretical_loss": 3.443491614032135, + "tokens_seen": 2111242240 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029120962888666, + "loss": 1.2587, + "theoretical_loss": 3.4434828693241375, + "tokens_seen": 2111307776 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002911935807422267, + "loss": 1.2997, + "theoretical_loss": 3.443474124963576, + "tokens_seen": 2111373312 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002911775325977934, + "loss": 1.2284, + "theoretical_loss": 3.4434653809504265, + "tokens_seen": 2111438848 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002911614844533601, + "loss": 1.2491, + "theoretical_loss": 3.4434566372846636, + "tokens_seen": 2111504384 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002911454363089268, + "loss": 1.2458, + "theoretical_loss": 3.4434478939662627, + "tokens_seen": 2111569920 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002911293881644935, + "loss": 1.3273, + "theoretical_loss": 3.4434391509952, + "tokens_seen": 2111635456 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002911133400200602, + "loss": 1.2392, + "theoretical_loss": 3.4434304083714506, + "tokens_seen": 2111700992 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002910972918756269, + "loss": 1.366, + "theoretical_loss": 3.4434216660949897, + "tokens_seen": 2111766528 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002910812437311936, + "loss": 1.2782, + "theoretical_loss": 3.443412924165793, + "tokens_seen": 2111832064 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002910651955867603, + "loss": 1.1609, + "theoretical_loss": 3.443404182583836, + "tokens_seen": 2111897600 + }, + { + "epoch": 0.64, + "learning_rate": 0.000291049147442327, + "loss": 1.31, + "theoretical_loss": 3.4433954413490944, + "tokens_seen": 2111963136 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002910330992978937, + "loss": 1.342, + "theoretical_loss": 3.4433867004615424, + "tokens_seen": 2112028672 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002910170511534604, + "loss": 1.1764, + "theoretical_loss": 3.4433779599211567, + "tokens_seen": 2112094208 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002910010030090271, + "loss": 1.387, + "theoretical_loss": 3.443369219727912, + "tokens_seen": 2112159744 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002909849548645938, + "loss": 1.2732, + "theoretical_loss": 3.4433604798817843, + "tokens_seen": 2112225280 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002909689067201605, + "loss": 1.3549, + "theoretical_loss": 3.4433517403827487, + "tokens_seen": 2112290816 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002909528585757272, + "loss": 1.2359, + "theoretical_loss": 3.443343001230781, + "tokens_seen": 2112356352 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002909368104312939, + "loss": 1.2515, + "theoretical_loss": 3.4433342624258563, + "tokens_seen": 2112421888 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002909207622868606, + "loss": 1.3391, + "theoretical_loss": 3.4433255239679497, + "tokens_seen": 2112487424 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002909047141424273, + "loss": 1.239, + "theoretical_loss": 3.4433167858570375, + "tokens_seen": 2112552960 + }, + { + "epoch": 0.64, + "learning_rate": 0.000290888665997994, + "loss": 1.3362, + "theoretical_loss": 3.443308048093094, + "tokens_seen": 2112618496 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002908726178535607, + "loss": 1.3537, + "theoretical_loss": 3.443299310676096, + "tokens_seen": 2112684032 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002908565697091274, + "loss": 1.2473, + "theoretical_loss": 3.4432905736060184, + "tokens_seen": 2112749568 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002908405215646941, + "loss": 1.2539, + "theoretical_loss": 3.4432818368828366, + "tokens_seen": 2112815104 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002908244734202608, + "loss": 1.2871, + "theoretical_loss": 3.4432731005065262, + "tokens_seen": 2112880640 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002908084252758275, + "loss": 1.3238, + "theoretical_loss": 3.443264364477062, + "tokens_seen": 2112946176 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002907923771313942, + "loss": 1.3158, + "theoretical_loss": 3.4432556287944207, + "tokens_seen": 2113011712 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002907763289869609, + "loss": 1.3023, + "theoretical_loss": 3.4432468934585767, + "tokens_seen": 2113077248 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002907602808425276, + "loss": 1.3082, + "theoretical_loss": 3.443238158469506, + "tokens_seen": 2113142784 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002907442326980943, + "loss": 1.1697, + "theoretical_loss": 3.443229423827184, + "tokens_seen": 2113208320 + }, + { + "epoch": 0.64, + "learning_rate": 0.000290728184553661, + "loss": 1.2475, + "theoretical_loss": 3.443220689531586, + "tokens_seen": 2113273856 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002907121364092277, + "loss": 1.2679, + "theoretical_loss": 3.4432119555826883, + "tokens_seen": 2113339392 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002906960882647944, + "loss": 1.2493, + "theoretical_loss": 3.443203221980465, + "tokens_seen": 2113404928 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002906800401203611, + "loss": 1.3285, + "theoretical_loss": 3.4431944887248926, + "tokens_seen": 2113470464 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 707433, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1259653568267822, + "objective/train/theoretical_loss": 3.4431857558159464, + "objective/train/tokens_used": 2133996000, + "theoretical_loss": 3.4431857558159464, + "tokens_seen": 2113536000 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002906639919759278, + "loss": 1.1662, + "theoretical_loss": 3.4431857558159464, + "tokens_seen": 2113536000 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002906479438314945, + "loss": 1.2663, + "theoretical_loss": 3.443177023253601, + "tokens_seen": 2113601536 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002906318956870612, + "loss": 1.3668, + "theoretical_loss": 3.443168291037834, + "tokens_seen": 2113667072 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002906158475426279, + "loss": 1.1815, + "theoretical_loss": 3.4431595591686186, + "tokens_seen": 2113732608 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002905997993981946, + "loss": 1.4156, + "theoretical_loss": 3.4431508276459315, + "tokens_seen": 2113798144 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002905837512537613, + "loss": 1.2532, + "theoretical_loss": 3.4431420964697486, + "tokens_seen": 2113863680 + }, + { + "epoch": 0.64, + "learning_rate": 0.000290567703109328, + "loss": 1.1691, + "theoretical_loss": 3.443133365640044, + "tokens_seen": 2113929216 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002905516549648947, + "loss": 1.2687, + "theoretical_loss": 3.4431246351567943, + "tokens_seen": 2113994752 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002905356068204614, + "loss": 1.2157, + "theoretical_loss": 3.443115905019975, + "tokens_seen": 2114060288 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002905195586760281, + "loss": 1.3268, + "theoretical_loss": 3.443107175229561, + "tokens_seen": 2114125824 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002905035105315948, + "loss": 1.2281, + "theoretical_loss": 3.4430984457855285, + "tokens_seen": 2114191360 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002904874623871615, + "loss": 1.2267, + "theoretical_loss": 3.4430897166878527, + "tokens_seen": 2114256896 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029047141424272816, + "loss": 1.283, + "theoretical_loss": 3.4430809879365087, + "tokens_seen": 2114322432 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029045536609829486, + "loss": 1.2251, + "theoretical_loss": 3.4430722595314727, + "tokens_seen": 2114387968 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029043931795386157, + "loss": 1.21, + "theoretical_loss": 3.4430635314727196, + "tokens_seen": 2114453504 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029042326980942827, + "loss": 1.241, + "theoretical_loss": 3.4430548037602255, + "tokens_seen": 2114519040 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029040722166499497, + "loss": 1.1837, + "theoretical_loss": 3.443046076393966, + "tokens_seen": 2114584576 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029039117352056167, + "loss": 1.2931, + "theoretical_loss": 3.4430373493739164, + "tokens_seen": 2114650112 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029037512537612837, + "loss": 1.2205, + "theoretical_loss": 3.443028622700052, + "tokens_seen": 2114715648 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029035907723169507, + "loss": 1.2717, + "theoretical_loss": 3.4430198963723484, + "tokens_seen": 2114781184 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029034302908726177, + "loss": 1.2229, + "theoretical_loss": 3.4430111703907813, + "tokens_seen": 2114846720 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029032698094282847, + "loss": 1.3069, + "theoretical_loss": 3.4430024447553267, + "tokens_seen": 2114912256 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029031093279839517, + "loss": 1.25, + "theoretical_loss": 3.4429937194659592, + "tokens_seen": 2114977792 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029029488465396187, + "loss": 1.1909, + "theoretical_loss": 3.4429849945226545, + "tokens_seen": 2115043328 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029027883650952857, + "loss": 1.1593, + "theoretical_loss": 3.442976269925389, + "tokens_seen": 2115108864 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029026278836509527, + "loss": 1.3597, + "theoretical_loss": 3.4429675456741373, + "tokens_seen": 2115174400 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029024674022066197, + "loss": 1.2895, + "theoretical_loss": 3.442958821768876, + "tokens_seen": 2115239936 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002902306920762287, + "loss": 1.2164, + "theoretical_loss": 3.4429500982095798, + "tokens_seen": 2115305472 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002902146439317954, + "loss": 1.1805, + "theoretical_loss": 3.442941374996224, + "tokens_seen": 2115371008 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002901985957873621, + "loss": 1.248, + "theoretical_loss": 3.4429326521287855, + "tokens_seen": 2115436544 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002901825476429288, + "loss": 1.2604, + "theoretical_loss": 3.4429239296072383, + "tokens_seen": 2115502080 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002901664994984955, + "loss": 1.3209, + "theoretical_loss": 3.442915207431559, + "tokens_seen": 2115567616 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002901504513540622, + "loss": 1.1991, + "theoretical_loss": 3.442906485601723, + "tokens_seen": 2115633152 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002901344032096289, + "loss": 1.2545, + "theoretical_loss": 3.4428977641177054, + "tokens_seen": 2115698688 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002901183550651956, + "loss": 1.2406, + "theoretical_loss": 3.4428890429794823, + "tokens_seen": 2115764224 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002901023069207623, + "loss": 1.2556, + "theoretical_loss": 3.4428803221870297, + "tokens_seen": 2115829760 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029008625877632903, + "loss": 1.2678, + "theoretical_loss": 3.442871601740322, + "tokens_seen": 2115895296 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029007021063189573, + "loss": 1.2768, + "theoretical_loss": 3.4428628816393347, + "tokens_seen": 2115960832 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029005416248746243, + "loss": 1.1676, + "theoretical_loss": 3.4428541618840454, + "tokens_seen": 2116026368 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029003811434302914, + "loss": 1.184, + "theoretical_loss": 3.4428454424744275, + "tokens_seen": 2116091904 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029002206619859584, + "loss": 1.3178, + "theoretical_loss": 3.4428367234104575, + "tokens_seen": 2116157440 + }, + { + "epoch": 0.64, + "learning_rate": 0.00029000601805416254, + "loss": 1.2162, + "theoretical_loss": 3.442828004692111, + "tokens_seen": 2116222976 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028998996990972924, + "loss": 1.2909, + "theoretical_loss": 3.442819286319364, + "tokens_seen": 2116288512 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028997392176529594, + "loss": 1.1848, + "theoretical_loss": 3.4428105682921912, + "tokens_seen": 2116354048 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028995787362086264, + "loss": 1.2543, + "theoretical_loss": 3.442801850610569, + "tokens_seen": 2116419584 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028994182547642934, + "loss": 1.1889, + "theoretical_loss": 3.442793133274472, + "tokens_seen": 2116485120 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028992577733199604, + "loss": 1.3355, + "theoretical_loss": 3.4427844162838768, + "tokens_seen": 2116550656 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028990972918756274, + "loss": 1.2685, + "theoretical_loss": 3.4427756996387586, + "tokens_seen": 2116616192 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028989368104312944, + "loss": 1.1742, + "theoretical_loss": 3.442766983339093, + "tokens_seen": 2116681728 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028987763289869614, + "loss": 1.2429, + "theoretical_loss": 3.442758267384856, + "tokens_seen": 2116747264 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 708587, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1421294212341309, + "objective/train/theoretical_loss": 3.4427495517760223, + "objective/train/tokens_used": 2137272800, + "theoretical_loss": 3.4427495517760223, + "tokens_seen": 2116812800 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028986158475426284, + "loss": 1.1653, + "theoretical_loss": 3.4427495517760223, + "tokens_seen": 2116812800 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002898455366098295, + "loss": 1.2893, + "theoretical_loss": 3.4427408365125687, + "tokens_seen": 2116878336 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002898294884653962, + "loss": 1.2339, + "theoretical_loss": 3.44273212159447, + "tokens_seen": 2116943872 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002898134403209629, + "loss": 1.2212, + "theoretical_loss": 3.442723407021702, + "tokens_seen": 2117009408 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002897973921765296, + "loss": 1.3382, + "theoretical_loss": 3.4427146927942402, + "tokens_seen": 2117074944 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002897813440320963, + "loss": 1.1565, + "theoretical_loss": 3.442705978912061, + "tokens_seen": 2117140480 + }, + { + "epoch": 0.64, + "learning_rate": 0.000289765295887663, + "loss": 1.2541, + "theoretical_loss": 3.442697265375139, + "tokens_seen": 2117206016 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002897492477432297, + "loss": 1.2564, + "theoretical_loss": 3.4426885521834505, + "tokens_seen": 2117271552 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002897331995987964, + "loss": 1.2667, + "theoretical_loss": 3.4426798393369706, + "tokens_seen": 2117337088 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002897171514543631, + "loss": 1.2022, + "theoretical_loss": 3.4426711268356756, + "tokens_seen": 2117402624 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002897011033099298, + "loss": 1.1869, + "theoretical_loss": 3.442662414679541, + "tokens_seen": 2117468160 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002896850551654965, + "loss": 1.2494, + "theoretical_loss": 3.4426537028685416, + "tokens_seen": 2117533696 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002896690070210632, + "loss": 1.2502, + "theoretical_loss": 3.442644991402654, + "tokens_seen": 2117599232 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002896529588766299, + "loss": 1.3387, + "theoretical_loss": 3.442636280281853, + "tokens_seen": 2117664768 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002896369107321966, + "loss": 1.2224, + "theoretical_loss": 3.442627569506116, + "tokens_seen": 2117730304 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002896208625877633, + "loss": 1.3095, + "theoretical_loss": 3.4426188590754165, + "tokens_seen": 2117795840 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028960481444333, + "loss": 1.3399, + "theoretical_loss": 3.442610148989731, + "tokens_seen": 2117861376 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002895887662988967, + "loss": 1.25, + "theoretical_loss": 3.442601439249036, + "tokens_seen": 2117926912 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002895727181544634, + "loss": 1.3456, + "theoretical_loss": 3.442592729853306, + "tokens_seen": 2117992448 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002895566700100301, + "loss": 1.2459, + "theoretical_loss": 3.442584020802517, + "tokens_seen": 2118057984 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002895406218655968, + "loss": 1.2221, + "theoretical_loss": 3.4425753120966447, + "tokens_seen": 2118123520 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002895245737211635, + "loss": 1.1989, + "theoretical_loss": 3.4425666037356653, + "tokens_seen": 2118189056 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002895085255767302, + "loss": 1.3126, + "theoretical_loss": 3.4425578957195535, + "tokens_seen": 2118254592 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002894924774322969, + "loss": 1.1686, + "theoretical_loss": 3.442549188048285, + "tokens_seen": 2118320128 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002894764292878636, + "loss": 1.3358, + "theoretical_loss": 3.4425404807218367, + "tokens_seen": 2118385664 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002894603811434303, + "loss": 1.327, + "theoretical_loss": 3.4425317737401837, + "tokens_seen": 2118451200 + }, + { + "epoch": 0.64, + "learning_rate": 0.000289444332998997, + "loss": 1.274, + "theoretical_loss": 3.442523067103301, + "tokens_seen": 2118516736 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002894282848545637, + "loss": 1.1927, + "theoretical_loss": 3.442514360811165, + "tokens_seen": 2118582272 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002894122367101304, + "loss": 1.2467, + "theoretical_loss": 3.442505654863751, + "tokens_seen": 2118647808 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002893961885656971, + "loss": 1.247, + "theoretical_loss": 3.4424969492610344, + "tokens_seen": 2118713344 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002893801404212638, + "loss": 1.3265, + "theoretical_loss": 3.442488244002992, + "tokens_seen": 2118778880 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002893640922768305, + "loss": 1.1696, + "theoretical_loss": 3.442479539089599, + "tokens_seen": 2118844416 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002893480441323972, + "loss": 1.3036, + "theoretical_loss": 3.4424708345208304, + "tokens_seen": 2118909952 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002893319959879639, + "loss": 1.2407, + "theoretical_loss": 3.4424621302966627, + "tokens_seen": 2118975488 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002893159478435306, + "loss": 1.2972, + "theoretical_loss": 3.442453426417071, + "tokens_seen": 2119041024 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002892998996990973, + "loss": 1.1693, + "theoretical_loss": 3.4424447228820316, + "tokens_seen": 2119106560 + }, + { + "epoch": 0.64, + "learning_rate": 0.000289283851554664, + "loss": 1.2878, + "theoretical_loss": 3.4424360196915202, + "tokens_seen": 2119172096 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002892678034102307, + "loss": 1.1474, + "theoretical_loss": 3.442427316845512, + "tokens_seen": 2119237632 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002892517552657974, + "loss": 1.2193, + "theoretical_loss": 3.4424186143439828, + "tokens_seen": 2119303168 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002892357071213641, + "loss": 1.2735, + "theoretical_loss": 3.4424099121869087, + "tokens_seen": 2119368704 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002892196589769308, + "loss": 1.2168, + "theoretical_loss": 3.442401210374265, + "tokens_seen": 2119434240 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002892036108324975, + "loss": 1.2205, + "theoretical_loss": 3.4423925089060274, + "tokens_seen": 2119499776 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002891875626880642, + "loss": 1.2172, + "theoretical_loss": 3.4423838077821722, + "tokens_seen": 2119565312 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002891715145436309, + "loss": 1.239, + "theoretical_loss": 3.442375107002675, + "tokens_seen": 2119630848 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002891554663991976, + "loss": 1.348, + "theoretical_loss": 3.442366406567511, + "tokens_seen": 2119696384 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002891394182547643, + "loss": 1.2413, + "theoretical_loss": 3.442357706476656, + "tokens_seen": 2119761920 + }, + { + "epoch": 0.64, + "learning_rate": 0.000289123370110331, + "loss": 1.4197, + "theoretical_loss": 3.442349006730086, + "tokens_seen": 2119827456 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002891073219658977, + "loss": 1.2307, + "theoretical_loss": 3.4423403073277763, + "tokens_seen": 2119892992 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002890912738214644, + "loss": 1.3436, + "theoretical_loss": 3.4423316082697037, + "tokens_seen": 2119958528 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028907522567703106, + "loss": 1.2063, + "theoretical_loss": 3.442322909555843, + "tokens_seen": 2120024064 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 709563, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1953669786453247, + "objective/train/theoretical_loss": 3.44231421118617, + "objective/train/tokens_used": 2140549600, + "theoretical_loss": 3.44231421118617, + "tokens_seen": 2120089600 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028905917753259777, + "loss": 1.21, + "theoretical_loss": 3.44231421118617, + "tokens_seen": 2120089600 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028904312938816447, + "loss": 1.2073, + "theoretical_loss": 3.442305513160661, + "tokens_seen": 2120155136 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028902708124373117, + "loss": 1.2469, + "theoretical_loss": 3.442296815479291, + "tokens_seen": 2120220672 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028901103309929787, + "loss": 1.2533, + "theoretical_loss": 3.442288118142036, + "tokens_seen": 2120286208 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028899498495486457, + "loss": 1.2671, + "theoretical_loss": 3.442279421148872, + "tokens_seen": 2120351744 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028897893681043127, + "loss": 1.271, + "theoretical_loss": 3.4422707244997746, + "tokens_seen": 2120417280 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028896288866599797, + "loss": 1.2723, + "theoretical_loss": 3.44226202819472, + "tokens_seen": 2120482816 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028894684052156467, + "loss": 1.2294, + "theoretical_loss": 3.4422533322336824, + "tokens_seen": 2120548352 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028893079237713137, + "loss": 1.2041, + "theoretical_loss": 3.44224463661664, + "tokens_seen": 2120613888 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028891474423269807, + "loss": 1.2519, + "theoretical_loss": 3.442235941343566, + "tokens_seen": 2120679424 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028889869608826477, + "loss": 1.2088, + "theoretical_loss": 3.442227246414438, + "tokens_seen": 2120744960 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002888826479438315, + "loss": 1.3304, + "theoretical_loss": 3.4422185518292316, + "tokens_seen": 2120810496 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028886659979939823, + "loss": 1.3289, + "theoretical_loss": 3.4422098575879216, + "tokens_seen": 2120876032 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028885055165496493, + "loss": 1.3174, + "theoretical_loss": 3.4422011636904846, + "tokens_seen": 2120941568 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028883450351053163, + "loss": 1.1963, + "theoretical_loss": 3.442192470136896, + "tokens_seen": 2121007104 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028881845536609833, + "loss": 1.3458, + "theoretical_loss": 3.4421837769271315, + "tokens_seen": 2121072640 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028880240722166503, + "loss": 1.3453, + "theoretical_loss": 3.442175084061167, + "tokens_seen": 2121138176 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028878635907723173, + "loss": 1.2966, + "theoretical_loss": 3.4421663915389784, + "tokens_seen": 2121203712 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028877031093279843, + "loss": 1.3202, + "theoretical_loss": 3.442157699360542, + "tokens_seen": 2121269248 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028875426278836513, + "loss": 1.3194, + "theoretical_loss": 3.442149007525832, + "tokens_seen": 2121334784 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028873821464393183, + "loss": 1.3137, + "theoretical_loss": 3.442140316034826, + "tokens_seen": 2121400320 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028872216649949853, + "loss": 1.2858, + "theoretical_loss": 3.442131624887499, + "tokens_seen": 2121465856 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028870611835506523, + "loss": 1.2993, + "theoretical_loss": 3.442122934083826, + "tokens_seen": 2121531392 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028869007021063193, + "loss": 1.3396, + "theoretical_loss": 3.4421142436237844, + "tokens_seen": 2121596928 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028867402206619863, + "loss": 1.1164, + "theoretical_loss": 3.4421055535073486, + "tokens_seen": 2121662464 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028865797392176534, + "loss": 1.2378, + "theoretical_loss": 3.442096863734495, + "tokens_seen": 2121728000 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028864192577733204, + "loss": 1.2166, + "theoretical_loss": 3.4420881743051996, + "tokens_seen": 2121793536 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028862587763289874, + "loss": 1.404, + "theoretical_loss": 3.442079485219438, + "tokens_seen": 2121859072 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028860982948846544, + "loss": 1.3609, + "theoretical_loss": 3.4420707964771857, + "tokens_seen": 2121924608 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028859378134403214, + "loss": 1.2224, + "theoretical_loss": 3.442062108078419, + "tokens_seen": 2121990144 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028857773319959884, + "loss": 1.2183, + "theoretical_loss": 3.4420534200231137, + "tokens_seen": 2122055680 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028856168505516554, + "loss": 1.2877, + "theoretical_loss": 3.4420447323112455, + "tokens_seen": 2122121216 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028854563691073224, + "loss": 1.233, + "theoretical_loss": 3.44203604494279, + "tokens_seen": 2122186752 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028852958876629894, + "loss": 1.3715, + "theoretical_loss": 3.4420273579177225, + "tokens_seen": 2122252288 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028851354062186564, + "loss": 1.1467, + "theoretical_loss": 3.4420186712360206, + "tokens_seen": 2122317824 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028849749247743234, + "loss": 1.293, + "theoretical_loss": 3.442009984897658, + "tokens_seen": 2122383360 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028848144433299904, + "loss": 1.3675, + "theoretical_loss": 3.4420012989026123, + "tokens_seen": 2122448896 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028846539618856574, + "loss": 1.1896, + "theoretical_loss": 3.441992613250858, + "tokens_seen": 2122514432 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028844934804413244, + "loss": 1.3655, + "theoretical_loss": 3.441983927942372, + "tokens_seen": 2122579968 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002884332998996991, + "loss": 1.209, + "theoretical_loss": 3.4419752429771293, + "tokens_seen": 2122645504 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002884172517552658, + "loss": 1.2895, + "theoretical_loss": 3.441966558355106, + "tokens_seen": 2122711040 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002884012036108325, + "loss": 1.2985, + "theoretical_loss": 3.4419578740762784, + "tokens_seen": 2122776576 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002883851554663992, + "loss": 1.2226, + "theoretical_loss": 3.4419491901406216, + "tokens_seen": 2122842112 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002883691073219659, + "loss": 1.2628, + "theoretical_loss": 3.4419405065481126, + "tokens_seen": 2122907648 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002883530591775326, + "loss": 1.2672, + "theoretical_loss": 3.4419318232987255, + "tokens_seen": 2122973184 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002883370110330993, + "loss": 1.2661, + "theoretical_loss": 3.4419231403924373, + "tokens_seen": 2123038720 + }, + { + "epoch": 0.64, + "learning_rate": 0.000288320962888666, + "loss": 1.3343, + "theoretical_loss": 3.441914457829224, + "tokens_seen": 2123104256 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002883049147442327, + "loss": 1.3214, + "theoretical_loss": 3.441905775609061, + "tokens_seen": 2123169792 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002882888665997994, + "loss": 1.2215, + "theoretical_loss": 3.4418970937319244, + "tokens_seen": 2123235328 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002882728184553661, + "loss": 1.3256, + "theoretical_loss": 3.44188841219779, + "tokens_seen": 2123300864 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 710783, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2148313522338867, + "objective/train/theoretical_loss": 3.441879731006633, + "objective/train/tokens_used": 2143826400, + "theoretical_loss": 3.441879731006633, + "tokens_seen": 2123366400 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002882567703109328, + "loss": 1.2569, + "theoretical_loss": 3.441879731006633, + "tokens_seen": 2123366400 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002882407221664995, + "loss": 1.2502, + "theoretical_loss": 3.44187105015843, + "tokens_seen": 2123431936 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002882246740220662, + "loss": 1.2611, + "theoretical_loss": 3.4418623696531574, + "tokens_seen": 2123497472 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002882086258776329, + "loss": 1.2391, + "theoretical_loss": 3.44185368949079, + "tokens_seen": 2123563008 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002881925777331996, + "loss": 1.2672, + "theoretical_loss": 3.441845009671304, + "tokens_seen": 2123628544 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002881765295887663, + "loss": 1.3013, + "theoretical_loss": 3.4418363301946755, + "tokens_seen": 2123694080 + }, + { + "epoch": 0.64, + "learning_rate": 0.000288160481444333, + "loss": 1.3159, + "theoretical_loss": 3.44182765106088, + "tokens_seen": 2123759616 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002881444332998997, + "loss": 1.2709, + "theoretical_loss": 3.441818972269894, + "tokens_seen": 2123825152 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002881283851554664, + "loss": 1.2604, + "theoretical_loss": 3.4418102938216926, + "tokens_seen": 2123890688 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002881123370110331, + "loss": 1.3081, + "theoretical_loss": 3.441801615716252, + "tokens_seen": 2123956224 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002880962888665998, + "loss": 1.3059, + "theoretical_loss": 3.4417929379535486, + "tokens_seen": 2124021760 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002880802407221665, + "loss": 1.3721, + "theoretical_loss": 3.4417842605335576, + "tokens_seen": 2124087296 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002880641925777332, + "loss": 1.2431, + "theoretical_loss": 3.441775583456255, + "tokens_seen": 2124152832 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002880481444332999, + "loss": 1.3118, + "theoretical_loss": 3.4417669067216172, + "tokens_seen": 2124218368 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002880320962888666, + "loss": 1.2263, + "theoretical_loss": 3.4417582303296195, + "tokens_seen": 2124283904 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002880160481444333, + "loss": 1.2527, + "theoretical_loss": 3.441749554280238, + "tokens_seen": 2124349440 + }, + { + "epoch": 0.64, + "learning_rate": 0.000288, + "loss": 1.3048, + "theoretical_loss": 3.441740878573449, + "tokens_seen": 2124414976 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002879839518555667, + "loss": 1.2588, + "theoretical_loss": 3.4417322032092272, + "tokens_seen": 2124480512 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002879679037111334, + "loss": 1.2845, + "theoretical_loss": 3.4417235281875502, + "tokens_seen": 2124546048 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002879518555667001, + "loss": 1.3406, + "theoretical_loss": 3.441714853508392, + "tokens_seen": 2124611584 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002879358074222668, + "loss": 1.2738, + "theoretical_loss": 3.4417061791717307, + "tokens_seen": 2124677120 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002879197592778335, + "loss": 1.3272, + "theoretical_loss": 3.4416975051775407, + "tokens_seen": 2124742656 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002879037111334002, + "loss": 1.3574, + "theoretical_loss": 3.4416888315257985, + "tokens_seen": 2124808192 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002878876629889669, + "loss": 1.3336, + "theoretical_loss": 3.4416801582164793, + "tokens_seen": 2124873728 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002878716148445336, + "loss": 1.2521, + "theoretical_loss": 3.44167148524956, + "tokens_seen": 2124939264 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002878555667001003, + "loss": 1.3123, + "theoretical_loss": 3.4416628126250153, + "tokens_seen": 2125004800 + }, + { + "epoch": 0.64, + "learning_rate": 0.000287839518555667, + "loss": 1.2726, + "theoretical_loss": 3.4416541403428225, + "tokens_seen": 2125070336 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002878234704112337, + "loss": 1.2862, + "theoretical_loss": 3.4416454684029567, + "tokens_seen": 2125135872 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002878074222668004, + "loss": 1.3182, + "theoretical_loss": 3.4416367968053945, + "tokens_seen": 2125201408 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002877913741223671, + "loss": 1.337, + "theoretical_loss": 3.4416281255501113, + "tokens_seen": 2125266944 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002877753259779338, + "loss": 1.1889, + "theoretical_loss": 3.441619454637083, + "tokens_seen": 2125332480 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002877592778335005, + "loss": 1.2525, + "theoretical_loss": 3.441610784066285, + "tokens_seen": 2125398016 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002877432296890672, + "loss": 1.3288, + "theoretical_loss": 3.4416021138376944, + "tokens_seen": 2125463552 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002877271815446339, + "loss": 1.2596, + "theoretical_loss": 3.4415934439512865, + "tokens_seen": 2125529088 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002877111334002006, + "loss": 1.316, + "theoretical_loss": 3.441584774407038, + "tokens_seen": 2125594624 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002876950852557673, + "loss": 1.317, + "theoretical_loss": 3.4415761052049234, + "tokens_seen": 2125660160 + }, + { + "epoch": 0.64, + "learning_rate": 0.000287679037111334, + "loss": 1.2413, + "theoretical_loss": 3.44156743634492, + "tokens_seen": 2125725696 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002876629889669007, + "loss": 1.3342, + "theoretical_loss": 3.441558767827003, + "tokens_seen": 2125791232 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002876469408224674, + "loss": 1.3839, + "theoretical_loss": 3.4415500996511486, + "tokens_seen": 2125856768 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002876308926780341, + "loss": 1.1593, + "theoretical_loss": 3.4415414318173325, + "tokens_seen": 2125922304 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002876148445336008, + "loss": 1.2782, + "theoretical_loss": 3.4415327643255313, + "tokens_seen": 2125987840 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002875987963891675, + "loss": 1.3625, + "theoretical_loss": 3.44152409717572, + "tokens_seen": 2126053376 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002875827482447342, + "loss": 1.2358, + "theoretical_loss": 3.441515430367876, + "tokens_seen": 2126118912 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002875667001003009, + "loss": 1.3056, + "theoretical_loss": 3.441506763901974, + "tokens_seen": 2126184448 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002875506519558676, + "loss": 1.2511, + "theoretical_loss": 3.4414980977779903, + "tokens_seen": 2126249984 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002875346038114343, + "loss": 1.266, + "theoretical_loss": 3.441489431995901, + "tokens_seen": 2126315520 + }, + { + "epoch": 0.64, + "learning_rate": 0.000287518555667001, + "loss": 1.308, + "theoretical_loss": 3.4414807665556815, + "tokens_seen": 2126381056 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002875025075225677, + "loss": 1.3626, + "theoretical_loss": 3.441472101457309, + "tokens_seen": 2126446592 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028748645937813443, + "loss": 1.3481, + "theoretical_loss": 3.4414634367007584, + "tokens_seen": 2126512128 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028747041123370113, + "loss": 1.2207, + "theoretical_loss": 3.441454772286006, + "tokens_seen": 2126577664 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 711700, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1988120079040527, + "objective/train/theoretical_loss": 3.4414461082130283, + "objective/train/tokens_used": 2147103200, + "theoretical_loss": 3.4414461082130283, + "tokens_seen": 2126643200 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028745436308926783, + "loss": 1.1536, + "theoretical_loss": 3.4414461082130283, + "tokens_seen": 2126643200 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028743831494483453, + "loss": 1.2569, + "theoretical_loss": 3.4414374444818003, + "tokens_seen": 2126708736 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028742226680040123, + "loss": 1.1813, + "theoretical_loss": 3.4414287810922985, + "tokens_seen": 2126774272 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028740621865596793, + "loss": 1.3024, + "theoretical_loss": 3.4414201180444994, + "tokens_seen": 2126839808 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028739017051153463, + "loss": 1.2308, + "theoretical_loss": 3.441411455338378, + "tokens_seen": 2126905344 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028737412236710133, + "loss": 1.2661, + "theoretical_loss": 3.4414027929739115, + "tokens_seen": 2126970880 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028735807422266803, + "loss": 1.2961, + "theoretical_loss": 3.4413941309510747, + "tokens_seen": 2127036416 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028734202607823473, + "loss": 1.2575, + "theoretical_loss": 3.441385469269844, + "tokens_seen": 2127101952 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028732597793380143, + "loss": 1.3445, + "theoretical_loss": 3.4413768079301956, + "tokens_seen": 2127167488 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028730992978936813, + "loss": 1.2895, + "theoretical_loss": 3.4413681469321054, + "tokens_seen": 2127233024 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028729388164493484, + "loss": 1.4079, + "theoretical_loss": 3.4413594862755494, + "tokens_seen": 2127298560 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028727783350050154, + "loss": 1.2045, + "theoretical_loss": 3.4413508259605035, + "tokens_seen": 2127364096 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028726178535606824, + "loss": 1.3002, + "theoretical_loss": 3.441342165986944, + "tokens_seen": 2127429632 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028724573721163494, + "loss": 1.3561, + "theoretical_loss": 3.441333506354847, + "tokens_seen": 2127495168 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028722968906720164, + "loss": 1.301, + "theoretical_loss": 3.441324847064188, + "tokens_seen": 2127560704 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028721364092276834, + "loss": 1.3778, + "theoretical_loss": 3.441316188114943, + "tokens_seen": 2127626240 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028719759277833504, + "loss": 1.2908, + "theoretical_loss": 3.441307529507089, + "tokens_seen": 2127691776 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028718154463390174, + "loss": 1.3052, + "theoretical_loss": 3.4412988712406007, + "tokens_seen": 2127757312 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028716549648946844, + "loss": 1.2593, + "theoretical_loss": 3.441290213315455, + "tokens_seen": 2127822848 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028714944834503514, + "loss": 1.1829, + "theoretical_loss": 3.441281555731628, + "tokens_seen": 2127888384 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028713340020060184, + "loss": 1.2046, + "theoretical_loss": 3.441272898489095, + "tokens_seen": 2127953920 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028711735205616854, + "loss": 1.2674, + "theoretical_loss": 3.4412642415878327, + "tokens_seen": 2128019456 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028710130391173524, + "loss": 1.2172, + "theoretical_loss": 3.4412555850278164, + "tokens_seen": 2128084992 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028708525576730194, + "loss": 1.2943, + "theoretical_loss": 3.441246928809023, + "tokens_seen": 2128150528 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028706920762286864, + "loss": 1.2691, + "theoretical_loss": 3.4412382729314284, + "tokens_seen": 2128216064 + }, + { + "epoch": 0.64, + "learning_rate": 0.00028705315947843534, + "loss": 1.384, + "theoretical_loss": 3.4412296173950083, + "tokens_seen": 2128281600 + }, + { + "epoch": 0.64, + "learning_rate": 0.000287037111334002, + "loss": 1.3061, + "theoretical_loss": 3.441220962199739, + "tokens_seen": 2128347136 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002870210631895687, + "loss": 1.2688, + "theoretical_loss": 3.4412123073455962, + "tokens_seen": 2128412672 + }, + { + "epoch": 0.64, + "learning_rate": 0.0002870050150451354, + "loss": 1.4018, + "theoretical_loss": 3.4412036528325562, + "tokens_seen": 2128478208 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002869889669007021, + "loss": 1.2682, + "theoretical_loss": 3.4411949986605954, + "tokens_seen": 2128543744 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002869729187562688, + "loss": 1.243, + "theoretical_loss": 3.441186344829689, + "tokens_seen": 2128609280 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002869568706118355, + "loss": 1.2756, + "theoretical_loss": 3.4411776913398135, + "tokens_seen": 2128674816 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002869408224674022, + "loss": 1.3485, + "theoretical_loss": 3.4411690381909454, + "tokens_seen": 2128740352 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002869247743229689, + "loss": 1.2449, + "theoretical_loss": 3.44116038538306, + "tokens_seen": 2128805888 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002869087261785356, + "loss": 1.2273, + "theoretical_loss": 3.4411517329161345, + "tokens_seen": 2128871424 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002868926780341023, + "loss": 1.3337, + "theoretical_loss": 3.4411430807901433, + "tokens_seen": 2128936960 + }, + { + "epoch": 0.65, + "learning_rate": 0.000286876629889669, + "loss": 1.2606, + "theoretical_loss": 3.441134429005064, + "tokens_seen": 2129002496 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002868605817452357, + "loss": 1.285, + "theoretical_loss": 3.4411257775608717, + "tokens_seen": 2129068032 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002868445336008024, + "loss": 1.3337, + "theoretical_loss": 3.441117126457543, + "tokens_seen": 2129133568 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002868284854563691, + "loss": 1.2708, + "theoretical_loss": 3.441108475695054, + "tokens_seen": 2129199104 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002868124373119358, + "loss": 1.2781, + "theoretical_loss": 3.4410998252733798, + "tokens_seen": 2129264640 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002867963891675025, + "loss": 1.3124, + "theoretical_loss": 3.441091175192498, + "tokens_seen": 2129330176 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002867803410230692, + "loss": 1.3762, + "theoretical_loss": 3.441082525452384, + "tokens_seen": 2129395712 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002867642928786359, + "loss": 1.2174, + "theoretical_loss": 3.4410738760530135, + "tokens_seen": 2129461248 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002867482447342026, + "loss": 1.2556, + "theoretical_loss": 3.4410652269943633, + "tokens_seen": 2129526784 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002867321965897693, + "loss": 1.3083, + "theoretical_loss": 3.4410565782764087, + "tokens_seen": 2129592320 + }, + { + "epoch": 0.65, + "learning_rate": 0.000286716148445336, + "loss": 1.2282, + "theoretical_loss": 3.441047929899127, + "tokens_seen": 2129657856 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002867001003009027, + "loss": 1.1837, + "theoretical_loss": 3.4410392818624924, + "tokens_seen": 2129723392 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002866840521564694, + "loss": 1.218, + "theoretical_loss": 3.441030634166483, + "tokens_seen": 2129788928 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002866680040120361, + "loss": 1.2565, + "theoretical_loss": 3.441021986811074, + "tokens_seen": 2129854464 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 712976, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.272249460220337, + "objective/train/theoretical_loss": 3.4410133397962412, + "objective/train/tokens_used": 2150380000, + "theoretical_loss": 3.4410133397962412, + "tokens_seen": 2129920000 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002866519558676028, + "loss": 1.319, + "theoretical_loss": 3.4410133397962412, + "tokens_seen": 2129920000 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002866359077231695, + "loss": 1.2676, + "theoretical_loss": 3.441004693121961, + "tokens_seen": 2129985536 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002866198595787362, + "loss": 1.3269, + "theoretical_loss": 3.44099604678821, + "tokens_seen": 2130051072 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002866038114343029, + "loss": 1.298, + "theoretical_loss": 3.4409874007949632, + "tokens_seen": 2130116608 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002865877632898696, + "loss": 1.1871, + "theoretical_loss": 3.440978755142198, + "tokens_seen": 2130182144 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002865717151454363, + "loss": 1.145, + "theoretical_loss": 3.4409701098298893, + "tokens_seen": 2130247680 + }, + { + "epoch": 0.65, + "learning_rate": 0.000286555667001003, + "loss": 1.2723, + "theoretical_loss": 3.4409614648580145, + "tokens_seen": 2130313216 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002865396188565697, + "loss": 1.3892, + "theoretical_loss": 3.440952820226549, + "tokens_seen": 2130378752 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002865235707121364, + "loss": 1.2214, + "theoretical_loss": 3.440944175935469, + "tokens_seen": 2130444288 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002865075225677031, + "loss": 1.2733, + "theoretical_loss": 3.44093553198475, + "tokens_seen": 2130509824 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002864914744232698, + "loss": 1.3492, + "theoretical_loss": 3.440926888374369, + "tokens_seen": 2130575360 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028647542627883657, + "loss": 1.3449, + "theoretical_loss": 3.4409182451043017, + "tokens_seen": 2130640896 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028645937813440327, + "loss": 1.2009, + "theoretical_loss": 3.440909602174525, + "tokens_seen": 2130706432 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028644332998996997, + "loss": 1.2939, + "theoretical_loss": 3.4409009595850137, + "tokens_seen": 2130771968 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028642728184553667, + "loss": 1.2052, + "theoretical_loss": 3.440892317335745, + "tokens_seen": 2130837504 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028641123370110337, + "loss": 1.296, + "theoretical_loss": 3.4408836754266945, + "tokens_seen": 2130903040 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028639518555667, + "loss": 1.3046, + "theoretical_loss": 3.440875033857839, + "tokens_seen": 2130968576 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002863791374122367, + "loss": 1.3161, + "theoretical_loss": 3.440866392629154, + "tokens_seen": 2131034112 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002863630892678034, + "loss": 1.3044, + "theoretical_loss": 3.4408577517406154, + "tokens_seen": 2131099648 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002863470411233701, + "loss": 1.3257, + "theoretical_loss": 3.4408491111922004, + "tokens_seen": 2131165184 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002863309929789368, + "loss": 1.2327, + "theoretical_loss": 3.4408404709838845, + "tokens_seen": 2131230720 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002863149448345035, + "loss": 1.1804, + "theoretical_loss": 3.440831831115643, + "tokens_seen": 2131296256 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002862988966900702, + "loss": 1.288, + "theoretical_loss": 3.4408231915874543, + "tokens_seen": 2131361792 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002862828485456369, + "loss": 1.352, + "theoretical_loss": 3.4408145523992926, + "tokens_seen": 2131427328 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002862668004012036, + "loss": 1.2042, + "theoretical_loss": 3.4408059135511344, + "tokens_seen": 2131492864 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002862507522567703, + "loss": 1.2325, + "theoretical_loss": 3.4407972750429567, + "tokens_seen": 2131558400 + }, + { + "epoch": 0.65, + "learning_rate": 0.000286234704112337, + "loss": 1.2514, + "theoretical_loss": 3.4407886368747347, + "tokens_seen": 2131623936 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002862186559679037, + "loss": 1.2866, + "theoretical_loss": 3.4407799990464456, + "tokens_seen": 2131689472 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002862026078234704, + "loss": 1.2288, + "theoretical_loss": 3.440771361558064, + "tokens_seen": 2131755008 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002861865596790371, + "loss": 1.3365, + "theoretical_loss": 3.440762724409568, + "tokens_seen": 2131820544 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002861705115346038, + "loss": 1.1642, + "theoretical_loss": 3.4407540876009324, + "tokens_seen": 2131886080 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002861544633901705, + "loss": 1.3112, + "theoretical_loss": 3.440745451132134, + "tokens_seen": 2131951616 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002861384152457372, + "loss": 1.3974, + "theoretical_loss": 3.4407368150031483, + "tokens_seen": 2132017152 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028612236710130393, + "loss": 1.2782, + "theoretical_loss": 3.440728179213952, + "tokens_seen": 2132082688 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028610631895687063, + "loss": 1.2066, + "theoretical_loss": 3.440719543764522, + "tokens_seen": 2132148224 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028609027081243733, + "loss": 1.3006, + "theoretical_loss": 3.4407109086548333, + "tokens_seen": 2132213760 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028607422266800403, + "loss": 1.28, + "theoretical_loss": 3.4407022738848623, + "tokens_seen": 2132279296 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028605817452357073, + "loss": 1.1735, + "theoretical_loss": 3.4406936394545857, + "tokens_seen": 2132344832 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028604212637913743, + "loss": 1.3347, + "theoretical_loss": 3.4406850053639793, + "tokens_seen": 2132410368 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028602607823470413, + "loss": 1.3143, + "theoretical_loss": 3.4406763716130193, + "tokens_seen": 2132475904 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028601003009027083, + "loss": 1.2306, + "theoretical_loss": 3.4406677382016824, + "tokens_seen": 2132541440 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028599398194583753, + "loss": 1.1986, + "theoretical_loss": 3.440659105129944, + "tokens_seen": 2132606976 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028597793380140423, + "loss": 1.3056, + "theoretical_loss": 3.440650472397781, + "tokens_seen": 2132672512 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028596188565697093, + "loss": 1.2364, + "theoretical_loss": 3.4406418400051693, + "tokens_seen": 2132738048 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028594583751253763, + "loss": 1.2781, + "theoretical_loss": 3.440633207952085, + "tokens_seen": 2132803584 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028592978936810433, + "loss": 1.3399, + "theoretical_loss": 3.440624576238505, + "tokens_seen": 2132869120 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028591374122367104, + "loss": 1.2428, + "theoretical_loss": 3.4406159448644047, + "tokens_seen": 2132934656 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028589769307923774, + "loss": 1.2658, + "theoretical_loss": 3.4406073138297604, + "tokens_seen": 2133000192 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028588164493480444, + "loss": 1.3416, + "theoretical_loss": 3.4405986831345485, + "tokens_seen": 2133065728 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028586559679037114, + "loss": 1.1882, + "theoretical_loss": 3.440590052778745, + "tokens_seen": 2133131264 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 714181, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3658976554870605, + "objective/train/theoretical_loss": 3.4405814227623273, + "objective/train/tokens_used": 2153656800, + "theoretical_loss": 3.4405814227623273, + "tokens_seen": 2133196800 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028584954864593784, + "loss": 1.3199, + "theoretical_loss": 3.4405814227623273, + "tokens_seen": 2133196800 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028583350050150454, + "loss": 1.2737, + "theoretical_loss": 3.44057279308527, + "tokens_seen": 2133262336 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028581745235707124, + "loss": 1.302, + "theoretical_loss": 3.44056416374755, + "tokens_seen": 2133327872 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028580140421263794, + "loss": 1.2609, + "theoretical_loss": 3.4405555347491443, + "tokens_seen": 2133393408 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028578535606820464, + "loss": 1.217, + "theoretical_loss": 3.4405469060900273, + "tokens_seen": 2133458944 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028576930792377134, + "loss": 1.3069, + "theoretical_loss": 3.440538277770177, + "tokens_seen": 2133524480 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028575325977933804, + "loss": 1.3104, + "theoretical_loss": 3.440529649789569, + "tokens_seen": 2133590016 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028573721163490474, + "loss": 1.3214, + "theoretical_loss": 3.4405210221481797, + "tokens_seen": 2133655552 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028572116349047144, + "loss": 1.2791, + "theoretical_loss": 3.4405123948459844, + "tokens_seen": 2133721088 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028570511534603814, + "loss": 1.3434, + "theoretical_loss": 3.4405037678829604, + "tokens_seen": 2133786624 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028568906720160484, + "loss": 1.2786, + "theoretical_loss": 3.440495141259084, + "tokens_seen": 2133852160 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028567301905717155, + "loss": 1.4118, + "theoretical_loss": 3.440486514974331, + "tokens_seen": 2133917696 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028565697091273825, + "loss": 1.1879, + "theoretical_loss": 3.440477889028678, + "tokens_seen": 2133983232 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028564092276830495, + "loss": 1.2928, + "theoretical_loss": 3.4404692634221004, + "tokens_seen": 2134048768 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002856248746238716, + "loss": 1.3331, + "theoretical_loss": 3.4404606381545753, + "tokens_seen": 2134114304 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002856088264794383, + "loss": 1.3225, + "theoretical_loss": 3.440452013226079, + "tokens_seen": 2134179840 + }, + { + "epoch": 0.65, + "learning_rate": 0.000285592778335005, + "loss": 1.2651, + "theoretical_loss": 3.440443388636587, + "tokens_seen": 2134245376 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002855767301905717, + "loss": 1.2758, + "theoretical_loss": 3.440434764386077, + "tokens_seen": 2134310912 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002855606820461384, + "loss": 1.3103, + "theoretical_loss": 3.4404261404745236, + "tokens_seen": 2134376448 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002855446339017051, + "loss": 1.3924, + "theoretical_loss": 3.440417516901904, + "tokens_seen": 2134441984 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002855285857572718, + "loss": 1.2982, + "theoretical_loss": 3.4404088936681942, + "tokens_seen": 2134507520 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002855125376128385, + "loss": 1.1782, + "theoretical_loss": 3.4404002707733703, + "tokens_seen": 2134573056 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002854964894684052, + "loss": 1.2809, + "theoretical_loss": 3.4403916482174095, + "tokens_seen": 2134638592 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002854804413239719, + "loss": 1.2355, + "theoretical_loss": 3.440383026000287, + "tokens_seen": 2134704128 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002854643931795386, + "loss": 1.2447, + "theoretical_loss": 3.4403744041219797, + "tokens_seen": 2134769664 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002854483450351053, + "loss": 1.3601, + "theoretical_loss": 3.4403657825824636, + "tokens_seen": 2134835200 + }, + { + "epoch": 0.65, + "learning_rate": 0.000285432296890672, + "loss": 1.2166, + "theoretical_loss": 3.4403571613817148, + "tokens_seen": 2134900736 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002854162487462387, + "loss": 1.1016, + "theoretical_loss": 3.44034854051971, + "tokens_seen": 2134966272 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002854002006018054, + "loss": 1.2066, + "theoretical_loss": 3.4403399199964255, + "tokens_seen": 2135031808 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002853841524573721, + "loss": 1.2679, + "theoretical_loss": 3.4403312998118376, + "tokens_seen": 2135097344 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002853681043129388, + "loss": 1.2458, + "theoretical_loss": 3.4403226799659223, + "tokens_seen": 2135162880 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002853520561685055, + "loss": 1.3211, + "theoretical_loss": 3.4403140604586557, + "tokens_seen": 2135228416 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002853360080240722, + "loss": 1.2906, + "theoretical_loss": 3.4403054412900147, + "tokens_seen": 2135293952 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002853199598796389, + "loss": 1.3148, + "theoretical_loss": 3.4402968224599757, + "tokens_seen": 2135359488 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002853039117352056, + "loss": 1.222, + "theoretical_loss": 3.4402882039685148, + "tokens_seen": 2135425024 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002852878635907723, + "loss": 1.327, + "theoretical_loss": 3.4402795858156074, + "tokens_seen": 2135490560 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028527181544633906, + "loss": 1.2713, + "theoretical_loss": 3.440270968001231, + "tokens_seen": 2135556096 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028525576730190576, + "loss": 1.1929, + "theoretical_loss": 3.4402623505253613, + "tokens_seen": 2135621632 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028523971915747246, + "loss": 1.2891, + "theoretical_loss": 3.440253733387975, + "tokens_seen": 2135687168 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028522367101303916, + "loss": 1.2808, + "theoretical_loss": 3.440245116589048, + "tokens_seen": 2135752704 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028520762286860586, + "loss": 1.2178, + "theoretical_loss": 3.4402365001285573, + "tokens_seen": 2135818240 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028519157472417256, + "loss": 1.2552, + "theoretical_loss": 3.440227884006479, + "tokens_seen": 2135883776 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028517552657973926, + "loss": 1.2308, + "theoretical_loss": 3.4402192682227883, + "tokens_seen": 2135949312 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028515947843530597, + "loss": 1.2823, + "theoretical_loss": 3.4402106527774627, + "tokens_seen": 2136014848 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028514343029087267, + "loss": 1.2314, + "theoretical_loss": 3.4402020376704785, + "tokens_seen": 2136080384 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028512738214643937, + "loss": 1.3729, + "theoretical_loss": 3.4401934229018116, + "tokens_seen": 2136145920 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028511133400200607, + "loss": 1.2539, + "theoretical_loss": 3.4401848084714386, + "tokens_seen": 2136211456 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028509528585757277, + "loss": 1.2693, + "theoretical_loss": 3.440176194379336, + "tokens_seen": 2136276992 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028507923771313947, + "loss": 1.2829, + "theoretical_loss": 3.44016758062548, + "tokens_seen": 2136342528 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028506318956870617, + "loss": 1.1675, + "theoretical_loss": 3.4401589672098463, + "tokens_seen": 2136408064 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 715089, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2422316074371338, + "objective/train/theoretical_loss": 3.4401503541324123, + "objective/train/tokens_used": 2156933600, + "theoretical_loss": 3.4401503541324123, + "tokens_seen": 2136473600 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028504714142427287, + "loss": 1.2682, + "theoretical_loss": 3.4401503541324123, + "tokens_seen": 2136473600 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028503109327983957, + "loss": 1.2492, + "theoretical_loss": 3.4401417413931537, + "tokens_seen": 2136539136 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028501504513540627, + "loss": 1.2835, + "theoretical_loss": 3.4401331289920467, + "tokens_seen": 2136604672 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002849989969909729, + "loss": 1.2665, + "theoretical_loss": 3.440124516929068, + "tokens_seen": 2136670208 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002849829488465396, + "loss": 1.1957, + "theoretical_loss": 3.440115905204194, + "tokens_seen": 2136735744 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002849669007021063, + "loss": 1.3285, + "theoretical_loss": 3.440107293817401, + "tokens_seen": 2136801280 + }, + { + "epoch": 0.65, + "learning_rate": 0.000284950852557673, + "loss": 1.3692, + "theoretical_loss": 3.4400986827686655, + "tokens_seen": 2136866816 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002849348044132397, + "loss": 1.2463, + "theoretical_loss": 3.4400900720579632, + "tokens_seen": 2136932352 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002849187562688064, + "loss": 1.2299, + "theoretical_loss": 3.4400814616852715, + "tokens_seen": 2136997888 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002849027081243731, + "loss": 1.1824, + "theoretical_loss": 3.440072851650566, + "tokens_seen": 2137063424 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002848866599799398, + "loss": 1.2974, + "theoretical_loss": 3.4400642419538228, + "tokens_seen": 2137128960 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002848706118355065, + "loss": 1.2442, + "theoretical_loss": 3.440055632595019, + "tokens_seen": 2137194496 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002848545636910732, + "loss": 1.3446, + "theoretical_loss": 3.440047023574131, + "tokens_seen": 2137260032 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002848385155466399, + "loss": 1.2072, + "theoretical_loss": 3.440038414891135, + "tokens_seen": 2137325568 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002848224674022066, + "loss": 1.3568, + "theoretical_loss": 3.440029806546007, + "tokens_seen": 2137391104 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002848064192577733, + "loss": 1.2776, + "theoretical_loss": 3.440021198538724, + "tokens_seen": 2137456640 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028479037111334, + "loss": 1.2603, + "theoretical_loss": 3.4400125908692614, + "tokens_seen": 2137522176 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002847743229689067, + "loss": 1.2445, + "theoretical_loss": 3.440003983537596, + "tokens_seen": 2137587712 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002847582748244734, + "loss": 1.284, + "theoretical_loss": 3.4399953765437052, + "tokens_seen": 2137653248 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028474222668004013, + "loss": 1.2469, + "theoretical_loss": 3.4399867698875646, + "tokens_seen": 2137718784 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028472617853560683, + "loss": 1.2293, + "theoretical_loss": 3.43997816356915, + "tokens_seen": 2137784320 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028471013039117353, + "loss": 1.4287, + "theoretical_loss": 3.439969557588439, + "tokens_seen": 2137849856 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028469408224674023, + "loss": 1.2873, + "theoretical_loss": 3.439960951945407, + "tokens_seen": 2137915392 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028467803410230693, + "loss": 1.3406, + "theoretical_loss": 3.4399523466400312, + "tokens_seen": 2137980928 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028466198595787363, + "loss": 1.2347, + "theoretical_loss": 3.4399437416722876, + "tokens_seen": 2138046464 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028464593781344033, + "loss": 1.3204, + "theoretical_loss": 3.439935137042152, + "tokens_seen": 2138112000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028462988966900703, + "loss": 1.3039, + "theoretical_loss": 3.4399265327496016, + "tokens_seen": 2138177536 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028461384152457373, + "loss": 1.2692, + "theoretical_loss": 3.4399179287946127, + "tokens_seen": 2138243072 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028459779338014043, + "loss": 1.3484, + "theoretical_loss": 3.4399093251771617, + "tokens_seen": 2138308608 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028458174523570713, + "loss": 1.2679, + "theoretical_loss": 3.439900721897225, + "tokens_seen": 2138374144 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028456569709127383, + "loss": 1.3801, + "theoretical_loss": 3.439892118954779, + "tokens_seen": 2138439680 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028454964894684054, + "loss": 1.2999, + "theoretical_loss": 3.4398835163497994, + "tokens_seen": 2138505216 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028453360080240724, + "loss": 1.3563, + "theoretical_loss": 3.4398749140822638, + "tokens_seen": 2138570752 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028451755265797394, + "loss": 1.2508, + "theoretical_loss": 3.439866312152148, + "tokens_seen": 2138636288 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028450150451354064, + "loss": 1.2227, + "theoretical_loss": 3.4398577105594286, + "tokens_seen": 2138701824 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028448545636910734, + "loss": 1.2314, + "theoretical_loss": 3.439849109304082, + "tokens_seen": 2138767360 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028446940822467404, + "loss": 1.1626, + "theoretical_loss": 3.4398405083860846, + "tokens_seen": 2138832896 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028445336008024074, + "loss": 1.317, + "theoretical_loss": 3.439831907805413, + "tokens_seen": 2138898432 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028443731193580744, + "loss": 1.3239, + "theoretical_loss": 3.439823307562043, + "tokens_seen": 2138963968 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028442126379137414, + "loss": 1.2671, + "theoretical_loss": 3.4398147076559518, + "tokens_seen": 2139029504 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028440521564694084, + "loss": 1.2787, + "theoretical_loss": 3.439806108087115, + "tokens_seen": 2139095040 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028438916750250754, + "loss": 1.3516, + "theoretical_loss": 3.43979750885551, + "tokens_seen": 2139160576 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028437311935807424, + "loss": 1.2763, + "theoretical_loss": 3.439788909961113, + "tokens_seen": 2139226112 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028435707121364094, + "loss": 1.3482, + "theoretical_loss": 3.4397803114039003, + "tokens_seen": 2139291648 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028434102306920764, + "loss": 1.2589, + "theoretical_loss": 3.4397717131838483, + "tokens_seen": 2139357184 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028432497492477434, + "loss": 1.2303, + "theoretical_loss": 3.4397631153009334, + "tokens_seen": 2139422720 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028430892678034104, + "loss": 1.2349, + "theoretical_loss": 3.439754517755132, + "tokens_seen": 2139488256 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028429287863590775, + "loss": 1.3468, + "theoretical_loss": 3.4397459205464207, + "tokens_seen": 2139553792 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028427683049147445, + "loss": 1.3202, + "theoretical_loss": 3.439737323674776, + "tokens_seen": 2139619328 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028426078234704115, + "loss": 1.3578, + "theoretical_loss": 3.4397287271401744, + "tokens_seen": 2139684864 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 716299, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2635315656661987, + "objective/train/theoretical_loss": 3.439720130942592, + "objective/train/tokens_used": 2160210400, + "theoretical_loss": 3.439720130942592, + "tokens_seen": 2139750400 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028424473420260785, + "loss": 1.3092, + "theoretical_loss": 3.439720130942592, + "tokens_seen": 2139750400 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002842286860581745, + "loss": 1.2773, + "theoretical_loss": 3.4397115350820053, + "tokens_seen": 2139815936 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002842126379137412, + "loss": 1.3772, + "theoretical_loss": 3.4397029395583916, + "tokens_seen": 2139881472 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002841965897693079, + "loss": 1.33, + "theoretical_loss": 3.439694344371727, + "tokens_seen": 2139947008 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002841805416248746, + "loss": 1.2345, + "theoretical_loss": 3.439685749521987, + "tokens_seen": 2140012544 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002841644934804413, + "loss": 1.2941, + "theoretical_loss": 3.439677155009149, + "tokens_seen": 2140078080 + }, + { + "epoch": 0.65, + "learning_rate": 0.000284148445336008, + "loss": 1.291, + "theoretical_loss": 3.43966856083319, + "tokens_seen": 2140143616 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002841323971915747, + "loss": 1.2447, + "theoretical_loss": 3.4396599669940846, + "tokens_seen": 2140209152 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002841163490471414, + "loss": 1.2373, + "theoretical_loss": 3.4396513734918113, + "tokens_seen": 2140274688 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002841003009027081, + "loss": 1.3992, + "theoretical_loss": 3.439642780326346, + "tokens_seen": 2140340224 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002840842527582748, + "loss": 1.2595, + "theoretical_loss": 3.439634187497664, + "tokens_seen": 2140405760 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028406820461384155, + "loss": 1.2743, + "theoretical_loss": 3.439625595005743, + "tokens_seen": 2140471296 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028405215646940825, + "loss": 1.27, + "theoretical_loss": 3.4396170028505595, + "tokens_seen": 2140536832 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028403610832497496, + "loss": 1.1822, + "theoretical_loss": 3.4396084110320895, + "tokens_seen": 2140602368 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028402006018054166, + "loss": 1.2351, + "theoretical_loss": 3.4395998195503097, + "tokens_seen": 2140667904 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028400401203610836, + "loss": 1.2489, + "theoretical_loss": 3.439591228405197, + "tokens_seen": 2140733440 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028398796389167506, + "loss": 1.1456, + "theoretical_loss": 3.439582637596727, + "tokens_seen": 2140798976 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028397191574724176, + "loss": 1.2808, + "theoretical_loss": 3.439574047124877, + "tokens_seen": 2140864512 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028395586760280846, + "loss": 1.298, + "theoretical_loss": 3.439565456989623, + "tokens_seen": 2140930048 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028393981945837516, + "loss": 1.2826, + "theoretical_loss": 3.4395568671909422, + "tokens_seen": 2140995584 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028392377131394186, + "loss": 1.2551, + "theoretical_loss": 3.43954827772881, + "tokens_seen": 2141061120 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028390772316950856, + "loss": 1.315, + "theoretical_loss": 3.439539688603204, + "tokens_seen": 2141126656 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028389167502507526, + "loss": 1.2538, + "theoretical_loss": 3.4395310998141, + "tokens_seen": 2141192192 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028387562688064196, + "loss": 1.2835, + "theoretical_loss": 3.439522511361475, + "tokens_seen": 2141257728 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028385957873620866, + "loss": 1.1975, + "theoretical_loss": 3.439513923245305, + "tokens_seen": 2141323264 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028384353059177536, + "loss": 1.2396, + "theoretical_loss": 3.4395053354655674, + "tokens_seen": 2141388800 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028382748244734206, + "loss": 1.2159, + "theoretical_loss": 3.4394967480222376, + "tokens_seen": 2141454336 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028381143430290876, + "loss": 1.3735, + "theoretical_loss": 3.4394881609152925, + "tokens_seen": 2141519872 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028379538615847547, + "loss": 1.4056, + "theoretical_loss": 3.4394795741447095, + "tokens_seen": 2141585408 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028377933801404217, + "loss": 1.3849, + "theoretical_loss": 3.4394709877104637, + "tokens_seen": 2141650944 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028376328986960887, + "loss": 1.2988, + "theoretical_loss": 3.4394624016125332, + "tokens_seen": 2141716480 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028374724172517557, + "loss": 1.3509, + "theoretical_loss": 3.439453815850893, + "tokens_seen": 2141782016 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028373119358074227, + "loss": 1.2107, + "theoretical_loss": 3.4394452304255205, + "tokens_seen": 2141847552 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028371514543630897, + "loss": 1.2127, + "theoretical_loss": 3.4394366453363925, + "tokens_seen": 2141913088 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028369909729187567, + "loss": 1.3388, + "theoretical_loss": 3.439428060583485, + "tokens_seen": 2141978624 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028368304914744237, + "loss": 1.2768, + "theoretical_loss": 3.4394194761667745, + "tokens_seen": 2142044160 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028366700100300907, + "loss": 1.2409, + "theoretical_loss": 3.4394108920862374, + "tokens_seen": 2142109696 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028365095285857577, + "loss": 1.3456, + "theoretical_loss": 3.439402308341851, + "tokens_seen": 2142175232 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028363490471414247, + "loss": 1.3051, + "theoretical_loss": 3.4393937249335913, + "tokens_seen": 2142240768 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028361885656970917, + "loss": 1.253, + "theoretical_loss": 3.439385141861435, + "tokens_seen": 2142306304 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002836028084252758, + "loss": 1.2485, + "theoretical_loss": 3.4393765591253587, + "tokens_seen": 2142371840 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002835867602808425, + "loss": 1.2345, + "theoretical_loss": 3.4393679767253387, + "tokens_seen": 2142437376 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002835707121364092, + "loss": 1.3034, + "theoretical_loss": 3.439359394661352, + "tokens_seen": 2142502912 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002835546639919759, + "loss": 1.2671, + "theoretical_loss": 3.439350812933375, + "tokens_seen": 2142568448 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002835386158475426, + "loss": 1.3994, + "theoretical_loss": 3.439342231541384, + "tokens_seen": 2142633984 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002835225677031093, + "loss": 1.2614, + "theoretical_loss": 3.4393336504853558, + "tokens_seen": 2142699520 + }, + { + "epoch": 0.65, + "learning_rate": 0.000283506519558676, + "loss": 1.2504, + "theoretical_loss": 3.4393250697652666, + "tokens_seen": 2142765056 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002834904714142427, + "loss": 1.3181, + "theoretical_loss": 3.4393164893810937, + "tokens_seen": 2142830592 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002834744232698094, + "loss": 1.3779, + "theoretical_loss": 3.439307909332814, + "tokens_seen": 2142896128 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002834583751253761, + "loss": 1.2088, + "theoretical_loss": 3.439299329620402, + "tokens_seen": 2142961664 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 717319, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1742408275604248, + "objective/train/theoretical_loss": 3.4392907502438366, + "objective/train/tokens_used": 2163487200, + "theoretical_loss": 3.4392907502438366, + "tokens_seen": 2143027200 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002834423269809428, + "loss": 1.1179, + "theoretical_loss": 3.4392907502438366, + "tokens_seen": 2143027200 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002834262788365095, + "loss": 1.3458, + "theoretical_loss": 3.439282171203093, + "tokens_seen": 2143092736 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002834102306920762, + "loss": 1.2117, + "theoretical_loss": 3.4392735924981483, + "tokens_seen": 2143158272 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002833941825476429, + "loss": 1.2574, + "theoretical_loss": 3.439265014128979, + "tokens_seen": 2143223808 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028337813440320963, + "loss": 1.2617, + "theoretical_loss": 3.4392564360955618, + "tokens_seen": 2143289344 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028336208625877633, + "loss": 1.3169, + "theoretical_loss": 3.439247858397873, + "tokens_seen": 2143354880 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028334603811434303, + "loss": 1.1802, + "theoretical_loss": 3.43923928103589, + "tokens_seen": 2143420416 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028332998996990973, + "loss": 1.2077, + "theoretical_loss": 3.4392307040095877, + "tokens_seen": 2143485952 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028331394182547643, + "loss": 1.236, + "theoretical_loss": 3.4392221273189447, + "tokens_seen": 2143551488 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028329789368104313, + "loss": 1.2567, + "theoretical_loss": 3.4392135509639363, + "tokens_seen": 2143617024 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028328184553660983, + "loss": 1.2374, + "theoretical_loss": 3.4392049749445395, + "tokens_seen": 2143682560 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028326579739217653, + "loss": 1.2954, + "theoretical_loss": 3.439196399260731, + "tokens_seen": 2143748096 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028324974924774323, + "loss": 1.2155, + "theoretical_loss": 3.4391878239124876, + "tokens_seen": 2143813632 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028323370110330993, + "loss": 1.2455, + "theoretical_loss": 3.4391792488997854, + "tokens_seen": 2143879168 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028321765295887663, + "loss": 1.2733, + "theoretical_loss": 3.439170674222601, + "tokens_seen": 2143944704 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028320160481444333, + "loss": 1.2501, + "theoretical_loss": 3.439162099880911, + "tokens_seen": 2144010240 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028318555667001004, + "loss": 1.2821, + "theoretical_loss": 3.4391535258746933, + "tokens_seen": 2144075776 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028316950852557674, + "loss": 1.1951, + "theoretical_loss": 3.4391449522039226, + "tokens_seen": 2144141312 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028315346038114344, + "loss": 1.2892, + "theoretical_loss": 3.439136378868577, + "tokens_seen": 2144206848 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028313741223671014, + "loss": 1.2367, + "theoretical_loss": 3.4391278058686323, + "tokens_seen": 2144272384 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028312136409227684, + "loss": 1.2295, + "theoretical_loss": 3.4391192332040648, + "tokens_seen": 2144337920 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028310531594784354, + "loss": 1.2583, + "theoretical_loss": 3.4391106608748525, + "tokens_seen": 2144403456 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028308926780341024, + "loss": 1.1736, + "theoretical_loss": 3.439102088880971, + "tokens_seen": 2144468992 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028307321965897694, + "loss": 1.2622, + "theoretical_loss": 3.439093517222397, + "tokens_seen": 2144534528 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028305717151454364, + "loss": 1.2108, + "theoretical_loss": 3.4390849458991077, + "tokens_seen": 2144600064 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028304112337011034, + "loss": 1.3321, + "theoretical_loss": 3.4390763749110786, + "tokens_seen": 2144665600 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028302507522567704, + "loss": 1.2596, + "theoretical_loss": 3.439067804258288, + "tokens_seen": 2144731136 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028300902708124374, + "loss": 1.2641, + "theoretical_loss": 3.4390592339407107, + "tokens_seen": 2144796672 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028299297893681044, + "loss": 1.1929, + "theoretical_loss": 3.439050663958325, + "tokens_seen": 2144862208 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028297693079237714, + "loss": 1.2431, + "theoretical_loss": 3.4390420943111066, + "tokens_seen": 2144927744 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028296088264794384, + "loss": 1.2844, + "theoretical_loss": 3.4390335249990325, + "tokens_seen": 2144993280 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028294483450351054, + "loss": 1.2738, + "theoretical_loss": 3.439024956022079, + "tokens_seen": 2145058816 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028292878635907725, + "loss": 1.1918, + "theoretical_loss": 3.439016387380223, + "tokens_seen": 2145124352 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028291273821464395, + "loss": 1.3033, + "theoretical_loss": 3.439007819073441, + "tokens_seen": 2145189888 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028289669007021065, + "loss": 1.1335, + "theoretical_loss": 3.4389992511017105, + "tokens_seen": 2145255424 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028288064192577735, + "loss": 1.3457, + "theoretical_loss": 3.438990683465007, + "tokens_seen": 2145320960 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028286459378134405, + "loss": 1.2682, + "theoretical_loss": 3.4389821161633076, + "tokens_seen": 2145386496 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028284854563691075, + "loss": 1.1651, + "theoretical_loss": 3.4389735491965894, + "tokens_seen": 2145452032 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028283249749247745, + "loss": 1.2404, + "theoretical_loss": 3.438964982564828, + "tokens_seen": 2145517568 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028281644934804415, + "loss": 1.2693, + "theoretical_loss": 3.4389564162680015, + "tokens_seen": 2145583104 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028280040120361085, + "loss": 1.3149, + "theoretical_loss": 3.4389478503060853, + "tokens_seen": 2145648640 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028278435305917755, + "loss": 1.2982, + "theoretical_loss": 3.4389392846790567, + "tokens_seen": 2145714176 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028276830491474425, + "loss": 1.2044, + "theoretical_loss": 3.4389307193868923, + "tokens_seen": 2145779712 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028275225677031095, + "loss": 1.2759, + "theoretical_loss": 3.4389221544295694, + "tokens_seen": 2145845248 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028273620862587765, + "loss": 1.1876, + "theoretical_loss": 3.438913589807063, + "tokens_seen": 2145910784 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028272016048144435, + "loss": 1.2871, + "theoretical_loss": 3.438905025519351, + "tokens_seen": 2145976320 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028270411233701105, + "loss": 1.303, + "theoretical_loss": 3.4388964615664106, + "tokens_seen": 2146041856 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028268806419257775, + "loss": 1.2778, + "theoretical_loss": 3.4388878979482174, + "tokens_seen": 2146107392 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028267201604814446, + "loss": 1.1871, + "theoretical_loss": 3.438879334664749, + "tokens_seen": 2146172928 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028265596790371116, + "loss": 1.2415, + "theoretical_loss": 3.438870771715981, + "tokens_seen": 2146238464 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 718519, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2924401760101318, + "objective/train/theoretical_loss": 3.4388622091018908, + "objective/train/tokens_used": 2166764000, + "theoretical_loss": 3.4388622091018908, + "tokens_seen": 2146304000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028263991975927786, + "loss": 1.282, + "theoretical_loss": 3.4388622091018908, + "tokens_seen": 2146304000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028262387161484456, + "loss": 1.3103, + "theoretical_loss": 3.438853646822455, + "tokens_seen": 2146369536 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028260782347041126, + "loss": 1.304, + "theoretical_loss": 3.4388450848776504, + "tokens_seen": 2146435072 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028259177532597796, + "loss": 1.2821, + "theoretical_loss": 3.4388365232674536, + "tokens_seen": 2146500608 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028257572718154466, + "loss": 1.2281, + "theoretical_loss": 3.4388279619918416, + "tokens_seen": 2146566144 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028255967903711136, + "loss": 1.3116, + "theoretical_loss": 3.4388194010507904, + "tokens_seen": 2146631680 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028254363089267806, + "loss": 1.181, + "theoretical_loss": 3.4388108404442774, + "tokens_seen": 2146697216 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028252758274824476, + "loss": 1.2712, + "theoretical_loss": 3.438802280172279, + "tokens_seen": 2146762752 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028251153460381146, + "loss": 1.1706, + "theoretical_loss": 3.438793720234772, + "tokens_seen": 2146828288 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028249548645937816, + "loss": 1.2739, + "theoretical_loss": 3.438785160631733, + "tokens_seen": 2146893824 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028247943831494486, + "loss": 1.3052, + "theoretical_loss": 3.4387766013631387, + "tokens_seen": 2146959360 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028246339017051156, + "loss": 1.1615, + "theoretical_loss": 3.4387680424289666, + "tokens_seen": 2147024896 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028244734202607826, + "loss": 1.1769, + "theoretical_loss": 3.438759483829192, + "tokens_seen": 2147090432 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028243129388164496, + "loss": 1.2976, + "theoretical_loss": 3.438750925563793, + "tokens_seen": 2147155968 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028241524573721167, + "loss": 1.2085, + "theoretical_loss": 3.438742367632745, + "tokens_seen": 2147221504 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028239919759277837, + "loss": 1.389, + "theoretical_loss": 3.438733810036026, + "tokens_seen": 2147287040 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028238314944834507, + "loss": 1.317, + "theoretical_loss": 3.4387252527736125, + "tokens_seen": 2147352576 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028236710130391177, + "loss": 1.2762, + "theoretical_loss": 3.4387166958454802, + "tokens_seen": 2147418112 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028235105315947847, + "loss": 1.2478, + "theoretical_loss": 3.438708139251607, + "tokens_seen": 2147483648 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028233500501504517, + "loss": 1.2929, + "theoretical_loss": 3.438699582991969, + "tokens_seen": 2147549184 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028231895687061187, + "loss": 1.2777, + "theoretical_loss": 3.4386910270665436, + "tokens_seen": 2147614720 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028230290872617857, + "loss": 1.2908, + "theoretical_loss": 3.438682471475307, + "tokens_seen": 2147680256 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028228686058174527, + "loss": 1.2017, + "theoretical_loss": 3.4386739162182356, + "tokens_seen": 2147745792 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028227081243731197, + "loss": 1.1989, + "theoretical_loss": 3.438665361295307, + "tokens_seen": 2147811328 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028225476429287867, + "loss": 1.3864, + "theoretical_loss": 3.438656806706497, + "tokens_seen": 2147876864 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028223871614844537, + "loss": 1.2446, + "theoretical_loss": 3.4386482524517836, + "tokens_seen": 2147942400 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002822226680040121, + "loss": 1.333, + "theoretical_loss": 3.4386396985311425, + "tokens_seen": 2148007936 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002822066198595788, + "loss": 1.2523, + "theoretical_loss": 3.4386311449445515, + "tokens_seen": 2148073472 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002821905717151454, + "loss": 1.2525, + "theoretical_loss": 3.4386225916919857, + "tokens_seen": 2148139008 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002821745235707121, + "loss": 1.2341, + "theoretical_loss": 3.438614038773424, + "tokens_seen": 2148204544 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002821584754262788, + "loss": 1.2057, + "theoretical_loss": 3.438605486188841, + "tokens_seen": 2148270080 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002821424272818455, + "loss": 1.4116, + "theoretical_loss": 3.438596933938215, + "tokens_seen": 2148335616 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002821263791374122, + "loss": 1.2248, + "theoretical_loss": 3.438588382021522, + "tokens_seen": 2148401152 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002821103309929789, + "loss": 1.2971, + "theoretical_loss": 3.43857983043874, + "tokens_seen": 2148466688 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002820942828485456, + "loss": 1.2306, + "theoretical_loss": 3.4385712791898437, + "tokens_seen": 2148532224 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002820782347041123, + "loss": 1.2755, + "theoretical_loss": 3.4385627282748112, + "tokens_seen": 2148597760 + }, + { + "epoch": 0.65, + "learning_rate": 0.000282062186559679, + "loss": 1.1841, + "theoretical_loss": 3.4385541776936197, + "tokens_seen": 2148663296 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002820461384152457, + "loss": 1.304, + "theoretical_loss": 3.4385456274462447, + "tokens_seen": 2148728832 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002820300902708124, + "loss": 1.2747, + "theoretical_loss": 3.438537077532664, + "tokens_seen": 2148794368 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028201404212637913, + "loss": 1.1649, + "theoretical_loss": 3.4385285279528546, + "tokens_seen": 2148859904 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028199799398194583, + "loss": 1.2288, + "theoretical_loss": 3.438519978706792, + "tokens_seen": 2148925440 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028198194583751253, + "loss": 1.3736, + "theoretical_loss": 3.4385114297944543, + "tokens_seen": 2148990976 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028196589769307923, + "loss": 1.2906, + "theoretical_loss": 3.4385028812158174, + "tokens_seen": 2149056512 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028194984954864593, + "loss": 1.2582, + "theoretical_loss": 3.4384943329708584, + "tokens_seen": 2149122048 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028193380140421263, + "loss": 1.2492, + "theoretical_loss": 3.4384857850595543, + "tokens_seen": 2149187584 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028191775325977933, + "loss": 1.2658, + "theoretical_loss": 3.4384772374818815, + "tokens_seen": 2149253120 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028190170511534603, + "loss": 1.2749, + "theoretical_loss": 3.4384686902378174, + "tokens_seen": 2149318656 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028188565697091273, + "loss": 1.2117, + "theoretical_loss": 3.438460143327338, + "tokens_seen": 2149384192 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028186960882647943, + "loss": 1.2582, + "theoretical_loss": 3.438451596750421, + "tokens_seen": 2149449728 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028185356068204613, + "loss": 1.2535, + "theoretical_loss": 3.4384430505070425, + "tokens_seen": 2149515264 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 719453, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.367647409439087, + "objective/train/theoretical_loss": 3.4384345045971796, + "objective/train/tokens_used": 2170040800, + "theoretical_loss": 3.4384345045971796, + "tokens_seen": 2149580800 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028183751253761283, + "loss": 1.255, + "theoretical_loss": 3.4384345045971796, + "tokens_seen": 2149580800 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028182146439317953, + "loss": 1.1786, + "theoretical_loss": 3.4384259590208095, + "tokens_seen": 2149646336 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028180541624874624, + "loss": 1.2142, + "theoretical_loss": 3.438417413777908, + "tokens_seen": 2149711872 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028178936810431294, + "loss": 1.2667, + "theoretical_loss": 3.4384088688684535, + "tokens_seen": 2149777408 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028177331995987964, + "loss": 1.2848, + "theoretical_loss": 3.438400324292421, + "tokens_seen": 2149842944 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028175727181544634, + "loss": 1.3754, + "theoretical_loss": 3.4383917800497885, + "tokens_seen": 2149908480 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028174122367101304, + "loss": 1.2372, + "theoretical_loss": 3.438383236140533, + "tokens_seen": 2149974016 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028172517552657974, + "loss": 1.2769, + "theoretical_loss": 3.43837469256463, + "tokens_seen": 2150039552 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028170912738214644, + "loss": 1.2896, + "theoretical_loss": 3.4383661493220576, + "tokens_seen": 2150105088 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028169307923771314, + "loss": 1.2648, + "theoretical_loss": 3.4383576064127923, + "tokens_seen": 2150170624 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002816770310932799, + "loss": 1.2132, + "theoretical_loss": 3.438349063836811, + "tokens_seen": 2150236160 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002816609829488466, + "loss": 1.2764, + "theoretical_loss": 3.43834052159409, + "tokens_seen": 2150301696 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002816449348044133, + "loss": 1.2283, + "theoretical_loss": 3.438331979684607, + "tokens_seen": 2150367232 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028162888665998, + "loss": 1.2449, + "theoretical_loss": 3.4383234381083376, + "tokens_seen": 2150432768 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002816128385155467, + "loss": 1.2656, + "theoretical_loss": 3.4383148968652604, + "tokens_seen": 2150498304 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002815967903711134, + "loss": 1.2463, + "theoretical_loss": 3.438306355955351, + "tokens_seen": 2150563840 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002815807422266801, + "loss": 1.2432, + "theoretical_loss": 3.4382978153785864, + "tokens_seen": 2150629376 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028156469408224674, + "loss": 1.2699, + "theoretical_loss": 3.4382892751349434, + "tokens_seen": 2150694912 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028154864593781345, + "loss": 1.3914, + "theoretical_loss": 3.4382807352243994, + "tokens_seen": 2150760448 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028153259779338015, + "loss": 1.211, + "theoretical_loss": 3.438272195646931, + "tokens_seen": 2150825984 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028151654964894685, + "loss": 1.2243, + "theoretical_loss": 3.4382636564025146, + "tokens_seen": 2150891520 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028150050150451355, + "loss": 1.3161, + "theoretical_loss": 3.4382551174911278, + "tokens_seen": 2150957056 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028148445336008025, + "loss": 1.2734, + "theoretical_loss": 3.4382465789127474, + "tokens_seen": 2151022592 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028146840521564695, + "loss": 1.3453, + "theoretical_loss": 3.4382380406673496, + "tokens_seen": 2151088128 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028145235707121365, + "loss": 1.238, + "theoretical_loss": 3.4382295027549112, + "tokens_seen": 2151153664 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028143630892678035, + "loss": 1.3722, + "theoretical_loss": 3.43822096517541, + "tokens_seen": 2151219200 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028142026078234705, + "loss": 1.2632, + "theoretical_loss": 3.438212427928822, + "tokens_seen": 2151284736 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028140421263791375, + "loss": 1.1628, + "theoretical_loss": 3.4382038910151254, + "tokens_seen": 2151350272 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028138816449348045, + "loss": 1.3094, + "theoretical_loss": 3.4381953544342956, + "tokens_seen": 2151415808 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028137211634904715, + "loss": 1.3061, + "theoretical_loss": 3.43818681818631, + "tokens_seen": 2151481344 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028135606820461385, + "loss": 1.2126, + "theoretical_loss": 3.438178282271145, + "tokens_seen": 2151546880 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028134002006018055, + "loss": 1.2818, + "theoretical_loss": 3.4381697466887786, + "tokens_seen": 2151612416 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028132397191574725, + "loss": 1.2908, + "theoretical_loss": 3.438161211439187, + "tokens_seen": 2151677952 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028130792377131396, + "loss": 1.2637, + "theoretical_loss": 3.438152676522347, + "tokens_seen": 2151743488 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028129187562688066, + "loss": 1.197, + "theoretical_loss": 3.438144141938236, + "tokens_seen": 2151809024 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028127582748244736, + "loss": 1.2516, + "theoretical_loss": 3.4381356076868306, + "tokens_seen": 2151874560 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028125977933801406, + "loss": 1.1802, + "theoretical_loss": 3.4381270737681073, + "tokens_seen": 2151940096 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028124373119358076, + "loss": 1.1881, + "theoretical_loss": 3.4381185401820438, + "tokens_seen": 2152005632 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028122768304914746, + "loss": 1.3302, + "theoretical_loss": 3.438110006928616, + "tokens_seen": 2152071168 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028121163490471416, + "loss": 1.3128, + "theoretical_loss": 3.438101474007802, + "tokens_seen": 2152136704 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028119558676028086, + "loss": 1.2822, + "theoretical_loss": 3.4380929414195776, + "tokens_seen": 2152202240 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028117953861584756, + "loss": 1.2157, + "theoretical_loss": 3.4380844091639204, + "tokens_seen": 2152267776 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028116349047141426, + "loss": 1.2927, + "theoretical_loss": 3.438075877240807, + "tokens_seen": 2152333312 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028114744232698096, + "loss": 1.2401, + "theoretical_loss": 3.4380673456502144, + "tokens_seen": 2152398848 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028113139418254766, + "loss": 1.1975, + "theoretical_loss": 3.43805881439212, + "tokens_seen": 2152464384 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028111534603811436, + "loss": 1.2704, + "theoretical_loss": 3.4380502834664997, + "tokens_seen": 2152529920 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028109929789368106, + "loss": 1.2765, + "theoretical_loss": 3.438041752873331, + "tokens_seen": 2152595456 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028108324974924776, + "loss": 1.2454, + "theoretical_loss": 3.4380332226125914, + "tokens_seen": 2152660992 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028106720160481446, + "loss": 1.2165, + "theoretical_loss": 3.4380246926842566, + "tokens_seen": 2152726528 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028105115346038117, + "loss": 1.2602, + "theoretical_loss": 3.4380161630883044, + "tokens_seen": 2152792064 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 720672, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3936753273010254, + "objective/train/theoretical_loss": 3.4380076338247116, + "objective/train/tokens_used": 2173317600, + "theoretical_loss": 3.4380076338247116, + "tokens_seen": 2152857600 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028103510531594787, + "loss": 1.3479, + "theoretical_loss": 3.4380076338247116, + "tokens_seen": 2152857600 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028101905717151457, + "loss": 1.3889, + "theoretical_loss": 3.4379991048934544, + "tokens_seen": 2152923136 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028100300902708127, + "loss": 1.3055, + "theoretical_loss": 3.437990576294511, + "tokens_seen": 2152988672 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028098696088264797, + "loss": 1.294, + "theoretical_loss": 3.4379820480278576, + "tokens_seen": 2153054208 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028097091273821467, + "loss": 1.2262, + "theoretical_loss": 3.437973520093471, + "tokens_seen": 2153119744 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028095486459378137, + "loss": 1.2827, + "theoretical_loss": 3.4379649924913283, + "tokens_seen": 2153185280 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028093881644934807, + "loss": 1.2358, + "theoretical_loss": 3.437956465221407, + "tokens_seen": 2153250816 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028092276830491477, + "loss": 1.4183, + "theoretical_loss": 3.437947938283683, + "tokens_seen": 2153316352 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028090672016048147, + "loss": 1.3137, + "theoretical_loss": 3.4379394116781343, + "tokens_seen": 2153381888 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028089067201604817, + "loss": 1.2573, + "theoretical_loss": 3.437930885404737, + "tokens_seen": 2153447424 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028087462387161487, + "loss": 1.2296, + "theoretical_loss": 3.4379223594634687, + "tokens_seen": 2153512960 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028085857572718157, + "loss": 1.173, + "theoretical_loss": 3.4379138338543056, + "tokens_seen": 2153578496 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002808425275827483, + "loss": 1.2997, + "theoretical_loss": 3.4379053085772258, + "tokens_seen": 2153644032 + }, + { + "epoch": 0.65, + "learning_rate": 0.000280826479438315, + "loss": 1.3307, + "theoretical_loss": 3.437896783632205, + "tokens_seen": 2153709568 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002808104312938817, + "loss": 1.4054, + "theoretical_loss": 3.4378882590192212, + "tokens_seen": 2153775104 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002807943831494483, + "loss": 1.2966, + "theoretical_loss": 3.4378797347382504, + "tokens_seen": 2153840640 + }, + { + "epoch": 0.65, + "learning_rate": 0.000280778335005015, + "loss": 1.215, + "theoretical_loss": 3.437871210789271, + "tokens_seen": 2153906176 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002807622868605817, + "loss": 1.2402, + "theoretical_loss": 3.4378626871722577, + "tokens_seen": 2153971712 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002807462387161484, + "loss": 1.2053, + "theoretical_loss": 3.43785416388719, + "tokens_seen": 2154037248 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002807301905717151, + "loss": 1.2436, + "theoretical_loss": 3.437845640934043, + "tokens_seen": 2154102784 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002807141424272818, + "loss": 1.2136, + "theoretical_loss": 3.4378371183127947, + "tokens_seen": 2154168320 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002806980942828485, + "loss": 1.2321, + "theoretical_loss": 3.4378285960234214, + "tokens_seen": 2154233856 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002806820461384152, + "loss": 1.1644, + "theoretical_loss": 3.437820074065901, + "tokens_seen": 2154299392 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002806659979939819, + "loss": 1.2569, + "theoretical_loss": 3.4378115524402095, + "tokens_seen": 2154364928 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002806499498495486, + "loss": 1.3015, + "theoretical_loss": 3.4378030311463244, + "tokens_seen": 2154430464 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028063390170511533, + "loss": 1.3441, + "theoretical_loss": 3.4377945101842227, + "tokens_seen": 2154496000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028061785356068203, + "loss": 1.1304, + "theoretical_loss": 3.4377859895538814, + "tokens_seen": 2154561536 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028060180541624873, + "loss": 1.3071, + "theoretical_loss": 3.437777469255277, + "tokens_seen": 2154627072 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028058575727181543, + "loss": 1.2349, + "theoretical_loss": 3.4377689492883867, + "tokens_seen": 2154692608 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028056970912738213, + "loss": 1.2034, + "theoretical_loss": 3.4377604296531885, + "tokens_seen": 2154758144 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028055366098294883, + "loss": 1.2403, + "theoretical_loss": 3.437751910349658, + "tokens_seen": 2154823680 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028053761283851553, + "loss": 1.1638, + "theoretical_loss": 3.4377433913777726, + "tokens_seen": 2154889216 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028052156469408223, + "loss": 1.2091, + "theoretical_loss": 3.4377348727375097, + "tokens_seen": 2154954752 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028050551654964893, + "loss": 1.343, + "theoretical_loss": 3.437726354428846, + "tokens_seen": 2155020288 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028048946840521563, + "loss": 1.2072, + "theoretical_loss": 3.4377178364517587, + "tokens_seen": 2155085824 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002804734202607824, + "loss": 1.3629, + "theoretical_loss": 3.437709318806225, + "tokens_seen": 2155151360 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002804573721163491, + "loss": 1.2791, + "theoretical_loss": 3.4377008014922206, + "tokens_seen": 2155216896 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002804413239719158, + "loss": 1.2546, + "theoretical_loss": 3.4376922845097244, + "tokens_seen": 2155282432 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002804252758274825, + "loss": 1.271, + "theoretical_loss": 3.4376837678587124, + "tokens_seen": 2155347968 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002804092276830492, + "loss": 1.2886, + "theoretical_loss": 3.4376752515391615, + "tokens_seen": 2155413504 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002803931795386159, + "loss": 1.1938, + "theoretical_loss": 3.437666735551049, + "tokens_seen": 2155479040 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002803771313941826, + "loss": 1.2558, + "theoretical_loss": 3.4376582198943524, + "tokens_seen": 2155544576 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002803610832497493, + "loss": 1.2699, + "theoretical_loss": 3.4376497045690475, + "tokens_seen": 2155610112 + }, + { + "epoch": 0.65, + "learning_rate": 0.000280345035105316, + "loss": 1.2987, + "theoretical_loss": 3.437641189575112, + "tokens_seen": 2155675648 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002803289869608827, + "loss": 1.2198, + "theoretical_loss": 3.437632674912524, + "tokens_seen": 2155741184 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002803129388164494, + "loss": 1.2366, + "theoretical_loss": 3.437624160581259, + "tokens_seen": 2155806720 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002802968906720161, + "loss": 1.2317, + "theoretical_loss": 3.4376156465812944, + "tokens_seen": 2155872256 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002802808425275828, + "loss": 1.3122, + "theoretical_loss": 3.437607132912608, + "tokens_seen": 2155937792 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002802647943831495, + "loss": 1.2743, + "theoretical_loss": 3.4375986195751755, + "tokens_seen": 2156003328 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002802487462387162, + "loss": 1.2334, + "theoretical_loss": 3.4375901065689747, + "tokens_seen": 2156068864 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 721882, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2278430461883545, + "objective/train/theoretical_loss": 3.4375815938939827, + "objective/train/tokens_used": 2176594400, + "theoretical_loss": 3.4375815938939827, + "tokens_seen": 2156134400 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002802326980942829, + "loss": 1.2663, + "theoretical_loss": 3.4375815938939827, + "tokens_seen": 2156134400 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002802166499498496, + "loss": 1.1849, + "theoretical_loss": 3.437573081550177, + "tokens_seen": 2156199936 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002802006018054163, + "loss": 1.3121, + "theoretical_loss": 3.4375645695375336, + "tokens_seen": 2156265472 + }, + { + "epoch": 0.65, + "learning_rate": 0.000280184553660983, + "loss": 1.2944, + "theoretical_loss": 3.43755605785603, + "tokens_seen": 2156331008 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002801685055165497, + "loss": 1.2327, + "theoretical_loss": 3.4375475465056438, + "tokens_seen": 2156396544 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028015245737211635, + "loss": 1.3913, + "theoretical_loss": 3.4375390354863513, + "tokens_seen": 2156462080 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028013640922768305, + "loss": 1.1688, + "theoretical_loss": 3.43753052479813, + "tokens_seen": 2156527616 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028012036108324975, + "loss": 1.2609, + "theoretical_loss": 3.4375220144409564, + "tokens_seen": 2156593152 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028010431293881645, + "loss": 1.1509, + "theoretical_loss": 3.4375135044148086, + "tokens_seen": 2156658688 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028008826479438315, + "loss": 1.3136, + "theoretical_loss": 3.4375049947196628, + "tokens_seen": 2156724224 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028007221664994985, + "loss": 1.289, + "theoretical_loss": 3.437496485355496, + "tokens_seen": 2156789760 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028005616850551655, + "loss": 1.2396, + "theoretical_loss": 3.437487976322286, + "tokens_seen": 2156855296 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028004012036108325, + "loss": 1.1943, + "theoretical_loss": 3.437479467620009, + "tokens_seen": 2156920832 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028002407221664995, + "loss": 1.3249, + "theoretical_loss": 3.437470959248643, + "tokens_seen": 2156986368 + }, + { + "epoch": 0.65, + "learning_rate": 0.00028000802407221665, + "loss": 1.1917, + "theoretical_loss": 3.437462451208164, + "tokens_seen": 2157051904 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027999197592778335, + "loss": 1.3302, + "theoretical_loss": 3.4374539434985505, + "tokens_seen": 2157117440 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027997592778335005, + "loss": 1.1716, + "theoretical_loss": 3.437445436119778, + "tokens_seen": 2157182976 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027995987963891675, + "loss": 1.1639, + "theoretical_loss": 3.4374369290718247, + "tokens_seen": 2157248512 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027994383149448345, + "loss": 1.327, + "theoretical_loss": 3.4374284223546674, + "tokens_seen": 2157314048 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027992778335005016, + "loss": 1.3163, + "theoretical_loss": 3.437419915968283, + "tokens_seen": 2157379584 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027991173520561686, + "loss": 1.309, + "theoretical_loss": 3.4374114099126487, + "tokens_seen": 2157445120 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027989568706118356, + "loss": 1.3796, + "theoretical_loss": 3.437402904187742, + "tokens_seen": 2157510656 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027987963891675026, + "loss": 1.287, + "theoretical_loss": 3.437394398793539, + "tokens_seen": 2157576192 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027986359077231696, + "loss": 1.239, + "theoretical_loss": 3.4373858937300175, + "tokens_seen": 2157641728 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027984754262788366, + "loss": 1.2294, + "theoretical_loss": 3.4373773889971546, + "tokens_seen": 2157707264 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027983149448345036, + "loss": 1.2523, + "theoretical_loss": 3.4373688845949273, + "tokens_seen": 2157772800 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027981544633901706, + "loss": 1.2528, + "theoretical_loss": 3.437360380523313, + "tokens_seen": 2157838336 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027979939819458376, + "loss": 1.2704, + "theoretical_loss": 3.437351876782288, + "tokens_seen": 2157903872 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027978335005015046, + "loss": 1.1192, + "theoretical_loss": 3.43734337337183, + "tokens_seen": 2157969408 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027976730190571716, + "loss": 1.3242, + "theoretical_loss": 3.4373348702919166, + "tokens_seen": 2158034944 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027975125376128386, + "loss": 1.3475, + "theoretical_loss": 3.4373263675425236, + "tokens_seen": 2158100480 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027973520561685056, + "loss": 1.2152, + "theoretical_loss": 3.4373178651236294, + "tokens_seen": 2158166016 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027971915747241726, + "loss": 1.2826, + "theoretical_loss": 3.4373093630352103, + "tokens_seen": 2158231552 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027970310932798396, + "loss": 1.2845, + "theoretical_loss": 3.4373008612772438, + "tokens_seen": 2158297088 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027968706118355066, + "loss": 1.3971, + "theoretical_loss": 3.4372923598497067, + "tokens_seen": 2158362624 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027967101303911737, + "loss": 1.3146, + "theoretical_loss": 3.4372838587525765, + "tokens_seen": 2158428160 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027965496489468407, + "loss": 1.3108, + "theoretical_loss": 3.4372753579858304, + "tokens_seen": 2158493696 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027963891675025077, + "loss": 1.1798, + "theoretical_loss": 3.437266857549445, + "tokens_seen": 2158559232 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027962286860581747, + "loss": 1.1867, + "theoretical_loss": 3.437258357443398, + "tokens_seen": 2158624768 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027960682046138417, + "loss": 1.3677, + "theoretical_loss": 3.437249857667666, + "tokens_seen": 2158690304 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027959077231695087, + "loss": 1.2645, + "theoretical_loss": 3.4372413582222263, + "tokens_seen": 2158755840 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027957472417251757, + "loss": 1.2931, + "theoretical_loss": 3.4372328591070564, + "tokens_seen": 2158821376 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027955867602808427, + "loss": 1.333, + "theoretical_loss": 3.4372243603221335, + "tokens_seen": 2158886912 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027954262788365097, + "loss": 1.2125, + "theoretical_loss": 3.437215861867434, + "tokens_seen": 2158952448 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027952657973921767, + "loss": 1.1659, + "theoretical_loss": 3.4372073637429352, + "tokens_seen": 2159017984 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027951053159478437, + "loss": 1.3248, + "theoretical_loss": 3.437198865948615, + "tokens_seen": 2159083520 + }, + { + "epoch": 0.65, + "learning_rate": 0.00027949448345035107, + "loss": 1.2228, + "theoretical_loss": 3.4371903684844503, + "tokens_seen": 2159149056 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002794784353059178, + "loss": 1.2415, + "theoretical_loss": 3.4371818713504174, + "tokens_seen": 2159214592 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002794623871614845, + "loss": 1.2723, + "theoretical_loss": 3.4371733745464947, + "tokens_seen": 2159280128 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002794463390170512, + "loss": 1.2951, + "theoretical_loss": 3.4371648780726582, + "tokens_seen": 2159345664 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 722801, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2955522537231445, + "objective/train/theoretical_loss": 3.4371563819288857, + "objective/train/tokens_used": 2179871200, + "theoretical_loss": 3.4371563819288857, + "tokens_seen": 2159411200 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002794302908726179, + "loss": 1.324, + "theoretical_loss": 3.4371563819288857, + "tokens_seen": 2159411200 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002794142427281846, + "loss": 1.2527, + "theoretical_loss": 3.4371478861151545, + "tokens_seen": 2159476736 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002793981945837513, + "loss": 1.2453, + "theoretical_loss": 3.437139390631441, + "tokens_seen": 2159542272 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002793821464393179, + "loss": 1.2313, + "theoretical_loss": 3.4371308954777233, + "tokens_seen": 2159607808 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002793660982948846, + "loss": 1.3156, + "theoretical_loss": 3.437122400653978, + "tokens_seen": 2159673344 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002793500501504513, + "loss": 1.268, + "theoretical_loss": 3.4371139061601825, + "tokens_seen": 2159738880 + }, + { + "epoch": 0.65, + "learning_rate": 0.000279334002006018, + "loss": 1.2093, + "theoretical_loss": 3.4371054119963143, + "tokens_seen": 2159804416 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002793179538615847, + "loss": 1.2553, + "theoretical_loss": 3.4370969181623496, + "tokens_seen": 2159869952 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002793019057171514, + "loss": 1.1994, + "theoretical_loss": 3.4370884246582665, + "tokens_seen": 2159935488 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002792858575727181, + "loss": 1.2408, + "theoretical_loss": 3.4370799314840417, + "tokens_seen": 2160001024 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002792698094282849, + "loss": 1.3032, + "theoretical_loss": 3.437071438639652, + "tokens_seen": 2160066560 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002792537612838516, + "loss": 1.3169, + "theoretical_loss": 3.437062946125076, + "tokens_seen": 2160132096 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002792377131394183, + "loss": 1.3403, + "theoretical_loss": 3.4370544539402896, + "tokens_seen": 2160197632 + }, + { + "epoch": 0.65, + "learning_rate": 0.000279221664994985, + "loss": 1.2272, + "theoretical_loss": 3.4370459620852705, + "tokens_seen": 2160263168 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002792056168505517, + "loss": 1.2813, + "theoretical_loss": 3.4370374705599955, + "tokens_seen": 2160328704 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002791895687061184, + "loss": 1.2493, + "theoretical_loss": 3.437028979364442, + "tokens_seen": 2160394240 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002791735205616851, + "loss": 1.1897, + "theoretical_loss": 3.4370204884985873, + "tokens_seen": 2160459776 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002791574724172518, + "loss": 1.3055, + "theoretical_loss": 3.4370119979624087, + "tokens_seen": 2160525312 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002791414242728185, + "loss": 1.3155, + "theoretical_loss": 3.4370035077558834, + "tokens_seen": 2160590848 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002791253761283852, + "loss": 1.2721, + "theoretical_loss": 3.436995017878988, + "tokens_seen": 2160656384 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002791093279839519, + "loss": 1.2184, + "theoretical_loss": 3.4369865283317003, + "tokens_seen": 2160721920 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002790932798395186, + "loss": 1.3446, + "theoretical_loss": 3.4369780391139977, + "tokens_seen": 2160787456 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002790772316950853, + "loss": 1.3699, + "theoretical_loss": 3.436969550225857, + "tokens_seen": 2160852992 + }, + { + "epoch": 0.65, + "learning_rate": 0.000279061183550652, + "loss": 1.2372, + "theoretical_loss": 3.436961061667255, + "tokens_seen": 2160918528 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002790451354062187, + "loss": 1.3306, + "theoretical_loss": 3.4369525734381696, + "tokens_seen": 2160984064 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002790290872617854, + "loss": 1.2613, + "theoretical_loss": 3.436944085538578, + "tokens_seen": 2161049600 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002790130391173521, + "loss": 1.2156, + "theoretical_loss": 3.436935597968457, + "tokens_seen": 2161115136 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002789969909729188, + "loss": 1.2649, + "theoretical_loss": 3.436927110727784, + "tokens_seen": 2161180672 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002789809428284855, + "loss": 1.2021, + "theoretical_loss": 3.436918623816537, + "tokens_seen": 2161246208 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002789648946840522, + "loss": 1.2299, + "theoretical_loss": 3.436910137234692, + "tokens_seen": 2161311744 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002789488465396189, + "loss": 1.3193, + "theoretical_loss": 3.4369016509822266, + "tokens_seen": 2161377280 + }, + { + "epoch": 0.65, + "learning_rate": 0.0002789327983951856, + "loss": 1.3036, + "theoretical_loss": 3.4368931650591183, + "tokens_seen": 2161442816 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002789167502507523, + "loss": 1.2368, + "theoretical_loss": 3.436884679465344, + "tokens_seen": 2161508352 + }, + { + "epoch": 0.66, + "learning_rate": 0.000278900702106319, + "loss": 1.2621, + "theoretical_loss": 3.4368761942008814, + "tokens_seen": 2161573888 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002788846539618857, + "loss": 1.1959, + "theoretical_loss": 3.436867709265707, + "tokens_seen": 2161639424 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002788686058174524, + "loss": 1.3286, + "theoretical_loss": 3.436859224659799, + "tokens_seen": 2161704960 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002788525576730191, + "loss": 1.1778, + "theoretical_loss": 3.436850740383134, + "tokens_seen": 2161770496 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002788365095285858, + "loss": 1.3236, + "theoretical_loss": 3.436842256435689, + "tokens_seen": 2161836032 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002788204613841525, + "loss": 1.2618, + "theoretical_loss": 3.4368337728174425, + "tokens_seen": 2161901568 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002788044132397192, + "loss": 1.2125, + "theoretical_loss": 3.43682528952837, + "tokens_seen": 2161967104 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002787883650952859, + "loss": 1.3596, + "theoretical_loss": 3.43681680656845, + "tokens_seen": 2162032640 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002787723169508526, + "loss": 1.1955, + "theoretical_loss": 3.4368083239376594, + "tokens_seen": 2162098176 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027875626880641925, + "loss": 1.1608, + "theoretical_loss": 3.4367998416359753, + "tokens_seen": 2162163712 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027874022066198595, + "loss": 1.2924, + "theoretical_loss": 3.436791359663375, + "tokens_seen": 2162229248 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027872417251755265, + "loss": 1.0921, + "theoretical_loss": 3.4367828780198364, + "tokens_seen": 2162294784 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027870812437311935, + "loss": 1.2422, + "theoretical_loss": 3.4367743967053355, + "tokens_seen": 2162360320 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027869207622868605, + "loss": 1.2413, + "theoretical_loss": 3.4367659157198505, + "tokens_seen": 2162425856 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027867602808425275, + "loss": 1.3272, + "theoretical_loss": 3.4367574350633587, + "tokens_seen": 2162491392 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027865997993981945, + "loss": 1.3192, + "theoretical_loss": 3.4367489547358367, + "tokens_seen": 2162556928 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027864393179538615, + "loss": 1.4066, + "theoretical_loss": 3.4367404747372623, + "tokens_seen": 2162622464 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 724069, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.196091651916504, + "objective/train/theoretical_loss": 3.4367319950676127, + "objective/train/tokens_used": 2183148000, + "theoretical_loss": 3.4367319950676127, + "tokens_seen": 2162688000 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027862788365095285, + "loss": 1.2863, + "theoretical_loss": 3.4367319950676127, + "tokens_seen": 2162688000 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027861183550651955, + "loss": 1.249, + "theoretical_loss": 3.4367235157268654, + "tokens_seen": 2162753536 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027859578736208625, + "loss": 1.2445, + "theoretical_loss": 3.436715036714997, + "tokens_seen": 2162819072 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027857973921765295, + "loss": 1.2465, + "theoretical_loss": 3.436706558031985, + "tokens_seen": 2162884608 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027856369107321966, + "loss": 1.2393, + "theoretical_loss": 3.4366980796778073, + "tokens_seen": 2162950144 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027854764292878636, + "loss": 1.3412, + "theoretical_loss": 3.4366896016524406, + "tokens_seen": 2163015680 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027853159478435306, + "loss": 1.1941, + "theoretical_loss": 3.4366811239558626, + "tokens_seen": 2163081216 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027851554663991976, + "loss": 1.3069, + "theoretical_loss": 3.43667264658805, + "tokens_seen": 2163146752 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027849949849548646, + "loss": 1.3146, + "theoretical_loss": 3.4366641695489806, + "tokens_seen": 2163212288 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027848345035105316, + "loss": 1.2928, + "theoretical_loss": 3.436655692838631, + "tokens_seen": 2163277824 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027846740220661986, + "loss": 1.2436, + "theoretical_loss": 3.4366472164569792, + "tokens_seen": 2163343360 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027845135406218656, + "loss": 1.3359, + "theoretical_loss": 3.436638740404003, + "tokens_seen": 2163408896 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027843530591775326, + "loss": 1.2725, + "theoretical_loss": 3.4366302646796782, + "tokens_seen": 2163474432 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027841925777331996, + "loss": 1.2524, + "theoretical_loss": 3.436621789283983, + "tokens_seen": 2163539968 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027840320962888666, + "loss": 1.2871, + "theoretical_loss": 3.4366133142168946, + "tokens_seen": 2163605504 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027838716148445336, + "loss": 1.2345, + "theoretical_loss": 3.436604839478391, + "tokens_seen": 2163671040 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027837111334002006, + "loss": 1.1999, + "theoretical_loss": 3.4365963650684477, + "tokens_seen": 2163736576 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027835506519558676, + "loss": 1.369, + "theoretical_loss": 3.4365878909870435, + "tokens_seen": 2163802112 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027833901705115346, + "loss": 1.3435, + "theoretical_loss": 3.4365794172341557, + "tokens_seen": 2163867648 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027832296890672016, + "loss": 1.2682, + "theoretical_loss": 3.436570943809761, + "tokens_seen": 2163933184 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027830692076228687, + "loss": 1.1939, + "theoretical_loss": 3.436562470713837, + "tokens_seen": 2163998720 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027829087261785357, + "loss": 1.2439, + "theoretical_loss": 3.436553997946361, + "tokens_seen": 2164064256 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027827482447342027, + "loss": 1.2257, + "theoretical_loss": 3.4365455255073103, + "tokens_seen": 2164129792 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027825877632898697, + "loss": 1.353, + "theoretical_loss": 3.436537053396662, + "tokens_seen": 2164195328 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027824272818455367, + "loss": 1.2564, + "theoretical_loss": 3.436528581614394, + "tokens_seen": 2164260864 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027822668004012037, + "loss": 1.28, + "theoretical_loss": 3.436520110160483, + "tokens_seen": 2164326400 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027821063189568707, + "loss": 1.3541, + "theoretical_loss": 3.436511639034907, + "tokens_seen": 2164391936 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027819458375125377, + "loss": 1.2283, + "theoretical_loss": 3.436503168237643, + "tokens_seen": 2164457472 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027817853560682047, + "loss": 1.2818, + "theoretical_loss": 3.436494697768668, + "tokens_seen": 2164523008 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027816248746238717, + "loss": 1.3455, + "theoretical_loss": 3.4364862276279595, + "tokens_seen": 2164588544 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027814643931795387, + "loss": 1.3924, + "theoretical_loss": 3.4364777578154952, + "tokens_seen": 2164654080 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027813039117352057, + "loss": 1.2284, + "theoretical_loss": 3.436469288331252, + "tokens_seen": 2164719616 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002781143430290873, + "loss": 1.3133, + "theoretical_loss": 3.436460819175208, + "tokens_seen": 2164785152 + }, + { + "epoch": 0.66, + "learning_rate": 0.000278098294884654, + "loss": 1.2281, + "theoretical_loss": 3.4364523503473396, + "tokens_seen": 2164850688 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002780822467402207, + "loss": 1.2501, + "theoretical_loss": 3.4364438818476244, + "tokens_seen": 2164916224 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002780661985957874, + "loss": 1.2831, + "theoretical_loss": 3.4364354136760404, + "tokens_seen": 2164981760 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002780501504513541, + "loss": 1.2576, + "theoretical_loss": 3.436426945832564, + "tokens_seen": 2165047296 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002780341023069208, + "loss": 1.1744, + "theoretical_loss": 3.436418478317173, + "tokens_seen": 2165112832 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002780180541624875, + "loss": 1.2468, + "theoretical_loss": 3.436410011129845, + "tokens_seen": 2165178368 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002780020060180542, + "loss": 1.2308, + "theoretical_loss": 3.436401544270557, + "tokens_seen": 2165243904 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002779859578736209, + "loss": 1.3324, + "theoretical_loss": 3.436393077739287, + "tokens_seen": 2165309440 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002779699097291876, + "loss": 1.1759, + "theoretical_loss": 3.436384611536011, + "tokens_seen": 2165374976 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002779538615847543, + "loss": 1.3107, + "theoretical_loss": 3.4363761456607076, + "tokens_seen": 2165440512 + }, + { + "epoch": 0.66, + "learning_rate": 0.000277937813440321, + "loss": 1.192, + "theoretical_loss": 3.436367680113354, + "tokens_seen": 2165506048 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002779217652958877, + "loss": 1.3118, + "theoretical_loss": 3.436359214893927, + "tokens_seen": 2165571584 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002779057171514544, + "loss": 1.187, + "theoretical_loss": 3.4363507500024046, + "tokens_seen": 2165637120 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002778896690070211, + "loss": 1.3252, + "theoretical_loss": 3.436342285438764, + "tokens_seen": 2165702656 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002778736208625878, + "loss": 1.2995, + "theoretical_loss": 3.436333821202982, + "tokens_seen": 2165768192 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002778575727181545, + "loss": 1.3257, + "theoretical_loss": 3.4363253572950367, + "tokens_seen": 2165833728 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002778415245737212, + "loss": 1.2101, + "theoretical_loss": 3.4363168937149053, + "tokens_seen": 2165899264 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 725023, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2539417743682861, + "objective/train/theoretical_loss": 3.4363084304625655, + "objective/train/tokens_used": 2186424800, + "theoretical_loss": 3.4363084304625655, + "tokens_seen": 2165964800 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002778254764292879, + "loss": 1.2215, + "theoretical_loss": 3.4363084304625655, + "tokens_seen": 2165964800 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002778094282848546, + "loss": 1.3366, + "theoretical_loss": 3.4362999675379937, + "tokens_seen": 2166030336 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002777933801404213, + "loss": 1.2662, + "theoretical_loss": 3.4362915049411686, + "tokens_seen": 2166095872 + }, + { + "epoch": 0.66, + "learning_rate": 0.000277777331995988, + "loss": 1.3058, + "theoretical_loss": 3.4362830426720663, + "tokens_seen": 2166161408 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002777612838515547, + "loss": 1.3021, + "theoretical_loss": 3.4362745807306654, + "tokens_seen": 2166226944 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002777452357071214, + "loss": 1.3195, + "theoretical_loss": 3.4362661191169424, + "tokens_seen": 2166292480 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002777291875626881, + "loss": 1.2236, + "theoretical_loss": 3.436257657830875, + "tokens_seen": 2166358016 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002777131394182548, + "loss": 1.2025, + "theoretical_loss": 3.4362491968724402, + "tokens_seen": 2166423552 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002776970912738215, + "loss": 1.3413, + "theoretical_loss": 3.4362407362416163, + "tokens_seen": 2166489088 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002776810431293882, + "loss": 1.3679, + "theoretical_loss": 3.43623227593838, + "tokens_seen": 2166554624 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002776649949849549, + "loss": 1.3038, + "theoretical_loss": 3.436223815962709, + "tokens_seen": 2166620160 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002776489468405216, + "loss": 1.2452, + "theoretical_loss": 3.4362153563145807, + "tokens_seen": 2166685696 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002776328986960883, + "loss": 1.2922, + "theoretical_loss": 3.4362068969939727, + "tokens_seen": 2166751232 + }, + { + "epoch": 0.66, + "learning_rate": 0.000277616850551655, + "loss": 1.2617, + "theoretical_loss": 3.4361984380008614, + "tokens_seen": 2166816768 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002776008024072217, + "loss": 1.2217, + "theoretical_loss": 3.436189979335226, + "tokens_seen": 2166882304 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002775847542627884, + "loss": 1.2308, + "theoretical_loss": 3.436181520997042, + "tokens_seen": 2166947840 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002775687061183551, + "loss": 1.2311, + "theoretical_loss": 3.436173062986288, + "tokens_seen": 2167013376 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002775526579739218, + "loss": 1.2988, + "theoretical_loss": 3.436164605302941, + "tokens_seen": 2167078912 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002775366098294885, + "loss": 1.2135, + "theoretical_loss": 3.4361561479469787, + "tokens_seen": 2167144448 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002775205616850552, + "loss": 1.2829, + "theoretical_loss": 3.4361476909183786, + "tokens_seen": 2167209984 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002775045135406219, + "loss": 1.2641, + "theoretical_loss": 3.4361392342171175, + "tokens_seen": 2167275520 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002774884653961886, + "loss": 1.1933, + "theoretical_loss": 3.4361307778431733, + "tokens_seen": 2167341056 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002774724172517553, + "loss": 1.1872, + "theoretical_loss": 3.4361223217965238, + "tokens_seen": 2167406592 + }, + { + "epoch": 0.66, + "learning_rate": 0.000277456369107322, + "loss": 1.1886, + "theoretical_loss": 3.4361138660771458, + "tokens_seen": 2167472128 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002774403209628887, + "loss": 1.3763, + "theoretical_loss": 3.4361054106850166, + "tokens_seen": 2167537664 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002774242728184554, + "loss": 1.2282, + "theoretical_loss": 3.4360969556201146, + "tokens_seen": 2167603200 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002774082246740221, + "loss": 1.2399, + "theoretical_loss": 3.4360885008824162, + "tokens_seen": 2167668736 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002773921765295888, + "loss": 1.2881, + "theoretical_loss": 3.4360800464718992, + "tokens_seen": 2167734272 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002773761283851555, + "loss": 1.2013, + "theoretical_loss": 3.4360715923885414, + "tokens_seen": 2167799808 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027736008024072215, + "loss": 1.2175, + "theoretical_loss": 3.4360631386323197, + "tokens_seen": 2167865344 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027734403209628885, + "loss": 1.2507, + "theoretical_loss": 3.4360546852032123, + "tokens_seen": 2167930880 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027732798395185555, + "loss": 1.2905, + "theoretical_loss": 3.4360462321011958, + "tokens_seen": 2167996416 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027731193580742225, + "loss": 1.2662, + "theoretical_loss": 3.4360377793262487, + "tokens_seen": 2168061952 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027729588766298895, + "loss": 1.2878, + "theoretical_loss": 3.436029326878347, + "tokens_seen": 2168127488 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027727983951855565, + "loss": 1.2268, + "theoretical_loss": 3.436020874757469, + "tokens_seen": 2168193024 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027726379137412235, + "loss": 1.2632, + "theoretical_loss": 3.436012422963593, + "tokens_seen": 2168258560 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027724774322968905, + "loss": 1.2978, + "theoretical_loss": 3.4360039714966946, + "tokens_seen": 2168324096 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027723169508525575, + "loss": 1.2359, + "theoretical_loss": 3.4359955203567525, + "tokens_seen": 2168389632 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027721564694082245, + "loss": 1.2701, + "theoretical_loss": 3.4359870695437444, + "tokens_seen": 2168455168 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027719959879638915, + "loss": 1.2189, + "theoretical_loss": 3.435978619057647, + "tokens_seen": 2168520704 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027718355065195586, + "loss": 1.2232, + "theoretical_loss": 3.435970168898438, + "tokens_seen": 2168586240 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027716750250752256, + "loss": 1.3604, + "theoretical_loss": 3.4359617190660954, + "tokens_seen": 2168651776 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027715145436308926, + "loss": 1.2792, + "theoretical_loss": 3.4359532695605957, + "tokens_seen": 2168717312 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027713540621865596, + "loss": 1.2758, + "theoretical_loss": 3.435944820381917, + "tokens_seen": 2168782848 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027711935807422266, + "loss": 1.246, + "theoretical_loss": 3.435936371530037, + "tokens_seen": 2168848384 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027710330992978936, + "loss": 1.3601, + "theoretical_loss": 3.4359279230049324, + "tokens_seen": 2168913920 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027708726178535606, + "loss": 1.355, + "theoretical_loss": 3.4359194748065818, + "tokens_seen": 2168979456 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027707121364092276, + "loss": 1.308, + "theoretical_loss": 3.4359110269349618, + "tokens_seen": 2169044992 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027705516549648946, + "loss": 1.3152, + "theoretical_loss": 3.43590257939005, + "tokens_seen": 2169110528 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027703911735205616, + "loss": 1.259, + "theoretical_loss": 3.435894132171824, + "tokens_seen": 2169176064 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 726246, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2564411163330078, + "objective/train/theoretical_loss": 3.4358856852802617, + "objective/train/tokens_used": 2189701600, + "theoretical_loss": 3.4358856852802617, + "tokens_seen": 2169241600 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027702306920762286, + "loss": 1.2804, + "theoretical_loss": 3.4358856852802617, + "tokens_seen": 2169241600 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027700702106318956, + "loss": 1.2098, + "theoretical_loss": 3.43587723871534, + "tokens_seen": 2169307136 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027699097291875626, + "loss": 1.2285, + "theoretical_loss": 3.435868792477037, + "tokens_seen": 2169372672 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027697492477432296, + "loss": 1.1938, + "theoretical_loss": 3.4358603465653292, + "tokens_seen": 2169438208 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027695887662988966, + "loss": 1.2302, + "theoretical_loss": 3.4358519009801953, + "tokens_seen": 2169503744 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027694282848545637, + "loss": 1.25, + "theoretical_loss": 3.435843455721612, + "tokens_seen": 2169569280 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027692678034102307, + "loss": 1.2779, + "theoretical_loss": 3.435835010789557, + "tokens_seen": 2169634816 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027691073219658977, + "loss": 1.2488, + "theoretical_loss": 3.4358265661840086, + "tokens_seen": 2169700352 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027689468405215647, + "loss": 1.2754, + "theoretical_loss": 3.435818121904943, + "tokens_seen": 2169765888 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027687863590772317, + "loss": 1.329, + "theoretical_loss": 3.435809677952338, + "tokens_seen": 2169831424 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002768625877632899, + "loss": 1.237, + "theoretical_loss": 3.4358012343261723, + "tokens_seen": 2169896960 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002768465396188566, + "loss": 1.2092, + "theoretical_loss": 3.435792791026422, + "tokens_seen": 2169962496 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002768304914744233, + "loss": 1.3439, + "theoretical_loss": 3.4357843480530654, + "tokens_seen": 2170028032 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027681444332999, + "loss": 1.2607, + "theoretical_loss": 3.4357759054060795, + "tokens_seen": 2170093568 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002767983951855567, + "loss": 1.2746, + "theoretical_loss": 3.4357674630854422, + "tokens_seen": 2170159104 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002767823470411234, + "loss": 1.191, + "theoretical_loss": 3.4357590210911315, + "tokens_seen": 2170224640 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002767662988966901, + "loss": 1.2514, + "theoretical_loss": 3.435750579423124, + "tokens_seen": 2170290176 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002767502507522568, + "loss": 1.1695, + "theoretical_loss": 3.4357421380813977, + "tokens_seen": 2170355712 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027673420260782353, + "loss": 1.3135, + "theoretical_loss": 3.4357336970659302, + "tokens_seen": 2170421248 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002767181544633902, + "loss": 1.2709, + "theoretical_loss": 3.4357252563766987, + "tokens_seen": 2170486784 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002767021063189569, + "loss": 1.2385, + "theoretical_loss": 3.435716816013681, + "tokens_seen": 2170552320 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002766860581745236, + "loss": 1.2866, + "theoretical_loss": 3.4357083759768545, + "tokens_seen": 2170617856 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002766700100300903, + "loss": 1.3206, + "theoretical_loss": 3.435699936266197, + "tokens_seen": 2170683392 + }, + { + "epoch": 0.66, + "learning_rate": 0.000276653961885657, + "loss": 1.1749, + "theoretical_loss": 3.435691496881686, + "tokens_seen": 2170748928 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002766379137412237, + "loss": 1.2275, + "theoretical_loss": 3.4356830578232986, + "tokens_seen": 2170814464 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002766218655967904, + "loss": 1.3343, + "theoretical_loss": 3.4356746190910132, + "tokens_seen": 2170880000 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002766058174523571, + "loss": 1.251, + "theoretical_loss": 3.4356661806848066, + "tokens_seen": 2170945536 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002765897693079238, + "loss": 1.301, + "theoretical_loss": 3.4356577426046564, + "tokens_seen": 2171011072 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002765737211634905, + "loss": 1.2532, + "theoretical_loss": 3.4356493048505405, + "tokens_seen": 2171076608 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002765576730190572, + "loss": 1.3233, + "theoretical_loss": 3.4356408674224364, + "tokens_seen": 2171142144 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002765416248746239, + "loss": 1.2797, + "theoretical_loss": 3.4356324303203216, + "tokens_seen": 2171207680 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002765255767301906, + "loss": 1.2676, + "theoretical_loss": 3.4356239935441737, + "tokens_seen": 2171273216 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002765095285857573, + "loss": 1.3007, + "theoretical_loss": 3.4356155570939704, + "tokens_seen": 2171338752 + }, + { + "epoch": 0.66, + "learning_rate": 0.000276493480441324, + "loss": 1.1645, + "theoretical_loss": 3.4356071209696886, + "tokens_seen": 2171404288 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002764774322968907, + "loss": 1.3095, + "theoretical_loss": 3.435598685171307, + "tokens_seen": 2171469824 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002764613841524574, + "loss": 1.2493, + "theoretical_loss": 3.4355902496988016, + "tokens_seen": 2171535360 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002764453360080241, + "loss": 1.1354, + "theoretical_loss": 3.435581814552152, + "tokens_seen": 2171600896 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002764292878635908, + "loss": 1.3093, + "theoretical_loss": 3.435573379731334, + "tokens_seen": 2171666432 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002764132397191575, + "loss": 1.2135, + "theoretical_loss": 3.435564945236326, + "tokens_seen": 2171731968 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002763971915747242, + "loss": 1.2994, + "theoretical_loss": 3.4355565110671056, + "tokens_seen": 2171797504 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002763811434302909, + "loss": 1.3061, + "theoretical_loss": 3.4355480772236504, + "tokens_seen": 2171863040 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002763650952858576, + "loss": 1.3247, + "theoretical_loss": 3.435539643705938, + "tokens_seen": 2171928576 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002763490471414243, + "loss": 1.2691, + "theoretical_loss": 3.435531210513945, + "tokens_seen": 2171994112 + }, + { + "epoch": 0.66, + "learning_rate": 0.000276332998996991, + "loss": 1.3157, + "theoretical_loss": 3.4355227776476505, + "tokens_seen": 2172059648 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002763169508525577, + "loss": 1.1872, + "theoretical_loss": 3.4355143451070314, + "tokens_seen": 2172125184 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002763009027081244, + "loss": 1.2944, + "theoretical_loss": 3.435505912892065, + "tokens_seen": 2172190720 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002762848545636911, + "loss": 1.3045, + "theoretical_loss": 3.435497481002729, + "tokens_seen": 2172256256 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002762688064192578, + "loss": 1.3385, + "theoretical_loss": 3.435489049439002, + "tokens_seen": 2172321792 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002762527582748245, + "loss": 1.2899, + "theoretical_loss": 3.4354806182008604, + "tokens_seen": 2172387328 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002762367101303912, + "loss": 1.1809, + "theoretical_loss": 3.435472187288282, + "tokens_seen": 2172452864 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 727225, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3450744152069092, + "objective/train/theoretical_loss": 3.4354637567012447, + "objective/train/tokens_used": 2192978400, + "theoretical_loss": 3.4354637567012447, + "tokens_seen": 2172518400 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002762206619859579, + "loss": 1.3029, + "theoretical_loss": 3.4354637567012447, + "tokens_seen": 2172518400 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002762046138415246, + "loss": 1.2384, + "theoretical_loss": 3.4354553264397265, + "tokens_seen": 2172583936 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002761885656970913, + "loss": 1.2204, + "theoretical_loss": 3.4354468965037044, + "tokens_seen": 2172649472 + }, + { + "epoch": 0.66, + "learning_rate": 0.000276172517552658, + "loss": 1.222, + "theoretical_loss": 3.4354384668931557, + "tokens_seen": 2172715008 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002761564694082247, + "loss": 1.2255, + "theoretical_loss": 3.435430037608059, + "tokens_seen": 2172780544 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002761404212637914, + "loss": 1.287, + "theoretical_loss": 3.435421608648391, + "tokens_seen": 2172846080 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002761243731193581, + "loss": 1.2286, + "theoretical_loss": 3.4354131800141303, + "tokens_seen": 2172911616 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002761083249749248, + "loss": 1.1989, + "theoretical_loss": 3.4354047517052537, + "tokens_seen": 2172977152 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002760922768304915, + "loss": 1.2868, + "theoretical_loss": 3.4353963237217386, + "tokens_seen": 2173042688 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002760762286860582, + "loss": 1.2537, + "theoretical_loss": 3.4353878960635638, + "tokens_seen": 2173108224 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002760601805416249, + "loss": 1.3856, + "theoretical_loss": 3.435379468730706, + "tokens_seen": 2173173760 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002760441323971916, + "loss": 1.3105, + "theoretical_loss": 3.4353710417231427, + "tokens_seen": 2173239296 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002760280842527583, + "loss": 1.3522, + "theoretical_loss": 3.4353626150408525, + "tokens_seen": 2173304832 + }, + { + "epoch": 0.66, + "learning_rate": 0.000276012036108325, + "loss": 1.2926, + "theoretical_loss": 3.435354188683812, + "tokens_seen": 2173370368 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002759959879638917, + "loss": 1.2617, + "theoretical_loss": 3.435345762651999, + "tokens_seen": 2173435904 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002759799398194584, + "loss": 1.2295, + "theoretical_loss": 3.435337336945392, + "tokens_seen": 2173501440 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002759638916750251, + "loss": 1.364, + "theoretical_loss": 3.4353289115639685, + "tokens_seen": 2173566976 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027594784353059175, + "loss": 1.2711, + "theoretical_loss": 3.435320486507705, + "tokens_seen": 2173632512 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027593179538615845, + "loss": 1.2468, + "theoretical_loss": 3.43531206177658, + "tokens_seen": 2173698048 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027591574724172515, + "loss": 1.1475, + "theoretical_loss": 3.4353036373705708, + "tokens_seen": 2173763584 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027589969909729185, + "loss": 1.2761, + "theoretical_loss": 3.4352952132896553, + "tokens_seen": 2173829120 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027588365095285855, + "loss": 1.2536, + "theoretical_loss": 3.4352867895338113, + "tokens_seen": 2173894656 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027586760280842525, + "loss": 1.3076, + "theoretical_loss": 3.4352783661030166, + "tokens_seen": 2173960192 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027585155466399195, + "loss": 1.233, + "theoretical_loss": 3.435269942997248, + "tokens_seen": 2174025728 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027583550651955865, + "loss": 1.2202, + "theoretical_loss": 3.435261520216484, + "tokens_seen": 2174091264 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027581945837512536, + "loss": 1.3136, + "theoretical_loss": 3.4352530977607016, + "tokens_seen": 2174156800 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027580341023069206, + "loss": 1.2573, + "theoretical_loss": 3.435244675629879, + "tokens_seen": 2174222336 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027578736208625876, + "loss": 1.1988, + "theoretical_loss": 3.435236253823994, + "tokens_seen": 2174287872 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027577131394182546, + "loss": 1.2127, + "theoretical_loss": 3.4352278323430236, + "tokens_seen": 2174353408 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027575526579739216, + "loss": 1.2602, + "theoretical_loss": 3.435219411186946, + "tokens_seen": 2174418944 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027573921765295886, + "loss": 1.2297, + "theoretical_loss": 3.4352109903557384, + "tokens_seen": 2174484480 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027572316950852556, + "loss": 1.3152, + "theoretical_loss": 3.435202569849379, + "tokens_seen": 2174550016 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027570712136409226, + "loss": 1.2631, + "theoretical_loss": 3.4351941496678453, + "tokens_seen": 2174615552 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027569107321965896, + "loss": 1.2269, + "theoretical_loss": 3.4351857298111153, + "tokens_seen": 2174681088 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002756750250752257, + "loss": 1.3053, + "theoretical_loss": 3.435177310279166, + "tokens_seen": 2174746624 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002756589769307924, + "loss": 1.2675, + "theoretical_loss": 3.4351688910719753, + "tokens_seen": 2174812160 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002756429287863591, + "loss": 1.227, + "theoretical_loss": 3.435160472189521, + "tokens_seen": 2174877696 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002756268806419258, + "loss": 1.2539, + "theoretical_loss": 3.4351520536317808, + "tokens_seen": 2174943232 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002756108324974925, + "loss": 1.3607, + "theoretical_loss": 3.4351436353987324, + "tokens_seen": 2175008768 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002755947843530592, + "loss": 1.2301, + "theoretical_loss": 3.4351352174903536, + "tokens_seen": 2175074304 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002755787362086259, + "loss": 1.2755, + "theoretical_loss": 3.4351267999066217, + "tokens_seen": 2175139840 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002755626880641926, + "loss": 1.2091, + "theoretical_loss": 3.4351183826475147, + "tokens_seen": 2175205376 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002755466399197593, + "loss": 1.2878, + "theoretical_loss": 3.4351099657130106, + "tokens_seen": 2175270912 + }, + { + "epoch": 0.66, + "learning_rate": 0.000275530591775326, + "loss": 1.2171, + "theoretical_loss": 3.4351015491030865, + "tokens_seen": 2175336448 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002755145436308927, + "loss": 1.3066, + "theoretical_loss": 3.4350931328177206, + "tokens_seen": 2175401984 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002754984954864594, + "loss": 1.2454, + "theoretical_loss": 3.43508471685689, + "tokens_seen": 2175467520 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002754824473420261, + "loss": 1.2123, + "theoretical_loss": 3.4350763012205734, + "tokens_seen": 2175533056 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002754663991975928, + "loss": 1.29, + "theoretical_loss": 3.4350678859087473, + "tokens_seen": 2175598592 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002754503510531595, + "loss": 1.186, + "theoretical_loss": 3.43505947092139, + "tokens_seen": 2175664128 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002754343029087262, + "loss": 1.3757, + "theoretical_loss": 3.4350510562584797, + "tokens_seen": 2175729664 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 728507, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2073462009429932, + "objective/train/theoretical_loss": 3.4350426419199938, + "objective/train/tokens_used": 2196255200, + "theoretical_loss": 3.4350426419199938, + "tokens_seen": 2175795200 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002754182547642929, + "loss": 1.2139, + "theoretical_loss": 3.4350426419199938, + "tokens_seen": 2175795200 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002754022066198596, + "loss": 1.2933, + "theoretical_loss": 3.435034227905909, + "tokens_seen": 2175860736 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002753861584754263, + "loss": 1.3251, + "theoretical_loss": 3.4350258142162047, + "tokens_seen": 2175926272 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027537011033099303, + "loss": 1.3179, + "theoretical_loss": 3.4350174008508576, + "tokens_seen": 2175991808 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027535406218655973, + "loss": 1.3092, + "theoretical_loss": 3.4350089878098453, + "tokens_seen": 2176057344 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027533801404212643, + "loss": 1.2077, + "theoretical_loss": 3.4350005750931465, + "tokens_seen": 2176122880 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002753219658976931, + "loss": 1.3125, + "theoretical_loss": 3.434992162700738, + "tokens_seen": 2176188416 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002753059177532598, + "loss": 1.2809, + "theoretical_loss": 3.4349837506325978, + "tokens_seen": 2176253952 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002752898696088265, + "loss": 1.3384, + "theoretical_loss": 3.434975338888704, + "tokens_seen": 2176319488 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002752738214643932, + "loss": 1.2628, + "theoretical_loss": 3.4349669274690338, + "tokens_seen": 2176385024 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002752577733199599, + "loss": 1.1707, + "theoretical_loss": 3.434958516373565, + "tokens_seen": 2176450560 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002752417251755266, + "loss": 1.1902, + "theoretical_loss": 3.4349501056022755, + "tokens_seen": 2176516096 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002752256770310933, + "loss": 1.2094, + "theoretical_loss": 3.4349416951551435, + "tokens_seen": 2176581632 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027520962888666, + "loss": 1.2816, + "theoretical_loss": 3.434933285032146, + "tokens_seen": 2176647168 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002751935807422267, + "loss": 1.3582, + "theoretical_loss": 3.4349248752332606, + "tokens_seen": 2176712704 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002751775325977934, + "loss": 1.3919, + "theoretical_loss": 3.434916465758466, + "tokens_seen": 2176778240 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002751614844533601, + "loss": 1.235, + "theoretical_loss": 3.4349080566077395, + "tokens_seen": 2176843776 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002751454363089268, + "loss": 1.2004, + "theoretical_loss": 3.4348996477810587, + "tokens_seen": 2176909312 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002751293881644935, + "loss": 1.2158, + "theoretical_loss": 3.4348912392784015, + "tokens_seen": 2176974848 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002751133400200602, + "loss": 1.1515, + "theoretical_loss": 3.4348828310997455, + "tokens_seen": 2177040384 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002750972918756269, + "loss": 1.1633, + "theoretical_loss": 3.434874423245069, + "tokens_seen": 2177105920 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002750812437311936, + "loss": 1.3084, + "theoretical_loss": 3.4348660157143485, + "tokens_seen": 2177171456 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002750651955867603, + "loss": 1.3353, + "theoretical_loss": 3.4348576085075635, + "tokens_seen": 2177236992 + }, + { + "epoch": 0.66, + "learning_rate": 0.000275049147442327, + "loss": 1.4075, + "theoretical_loss": 3.4348492016246905, + "tokens_seen": 2177302528 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002750330992978937, + "loss": 1.2179, + "theoretical_loss": 3.434840795065708, + "tokens_seen": 2177368064 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002750170511534604, + "loss": 1.2448, + "theoretical_loss": 3.4348323888305927, + "tokens_seen": 2177433600 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002750010030090271, + "loss": 1.299, + "theoretical_loss": 3.4348239829193234, + "tokens_seen": 2177499136 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002749849548645938, + "loss": 1.3144, + "theoretical_loss": 3.434815577331878, + "tokens_seen": 2177564672 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002749689067201605, + "loss": 1.3272, + "theoretical_loss": 3.4348071720682336, + "tokens_seen": 2177630208 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002749528585757272, + "loss": 1.1974, + "theoretical_loss": 3.434798767128368, + "tokens_seen": 2177695744 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002749368104312939, + "loss": 1.2834, + "theoretical_loss": 3.4347903625122598, + "tokens_seen": 2177761280 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002749207622868606, + "loss": 1.1783, + "theoretical_loss": 3.4347819582198857, + "tokens_seen": 2177826816 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002749047141424273, + "loss": 1.2499, + "theoretical_loss": 3.4347735542512243, + "tokens_seen": 2177892352 + }, + { + "epoch": 0.66, + "learning_rate": 0.000274888665997994, + "loss": 1.2854, + "theoretical_loss": 3.434765150606253, + "tokens_seen": 2177957888 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002748726178535607, + "loss": 1.2936, + "theoretical_loss": 3.4347567472849496, + "tokens_seen": 2178023424 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002748565697091274, + "loss": 1.264, + "theoretical_loss": 3.434748344287292, + "tokens_seen": 2178088960 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002748405215646941, + "loss": 1.3377, + "theoretical_loss": 3.434739941613258, + "tokens_seen": 2178154496 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002748244734202608, + "loss": 1.2703, + "theoretical_loss": 3.4347315392628257, + "tokens_seen": 2178220032 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002748084252758275, + "loss": 1.2726, + "theoretical_loss": 3.4347231372359723, + "tokens_seen": 2178285568 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002747923771313942, + "loss": 1.3109, + "theoretical_loss": 3.4347147355326757, + "tokens_seen": 2178351104 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002747763289869609, + "loss": 1.3036, + "theoretical_loss": 3.434706334152914, + "tokens_seen": 2178416640 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002747602808425276, + "loss": 1.2241, + "theoretical_loss": 3.434697933096665, + "tokens_seen": 2178482176 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002747442326980943, + "loss": 1.2488, + "theoretical_loss": 3.4346895323639064, + "tokens_seen": 2178547712 + }, + { + "epoch": 0.66, + "learning_rate": 0.000274728184553661, + "loss": 1.286, + "theoretical_loss": 3.4346811319546164, + "tokens_seen": 2178613248 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002747121364092277, + "loss": 1.3469, + "theoretical_loss": 3.434672731868772, + "tokens_seen": 2178678784 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002746960882647944, + "loss": 1.2434, + "theoretical_loss": 3.434664332106352, + "tokens_seen": 2178744320 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002746800401203611, + "loss": 1.2891, + "theoretical_loss": 3.434655932667333, + "tokens_seen": 2178809856 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002746639919759278, + "loss": 1.1401, + "theoretical_loss": 3.4346475335516935, + "tokens_seen": 2178875392 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002746479438314945, + "loss": 1.3492, + "theoretical_loss": 3.434639134759412, + "tokens_seen": 2178940928 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002746318956870612, + "loss": 1.2154, + "theoretical_loss": 3.434630736290465, + "tokens_seen": 2179006464 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 729371, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2518582344055176, + "objective/train/theoretical_loss": 3.4346223381448313, + "objective/train/tokens_used": 2199532000, + "theoretical_loss": 3.4346223381448313, + "tokens_seen": 2179072000 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002746158475426279, + "loss": 1.2806, + "theoretical_loss": 3.4346223381448313, + "tokens_seen": 2179072000 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002745997993981946, + "loss": 1.2542, + "theoretical_loss": 3.4346139403224885, + "tokens_seen": 2179137536 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002745837512537613, + "loss": 1.2155, + "theoretical_loss": 3.434605542823414, + "tokens_seen": 2179203072 + }, + { + "epoch": 0.66, + "learning_rate": 0.000274567703109328, + "loss": 1.2492, + "theoretical_loss": 3.434597145647586, + "tokens_seen": 2179268608 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027455165496489465, + "loss": 1.3191, + "theoretical_loss": 3.4345887487949827, + "tokens_seen": 2179334144 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027453560682046135, + "loss": 1.2907, + "theoretical_loss": 3.4345803522655816, + "tokens_seen": 2179399680 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027451955867602805, + "loss": 1.2163, + "theoretical_loss": 3.43457195605936, + "tokens_seen": 2179465216 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027450351053159475, + "loss": 1.2425, + "theoretical_loss": 3.4345635601762967, + "tokens_seen": 2179530752 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027448746238716145, + "loss": 1.2769, + "theoretical_loss": 3.434555164616369, + "tokens_seen": 2179596288 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002744714142427282, + "loss": 1.178, + "theoretical_loss": 3.4345467693795544, + "tokens_seen": 2179661824 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002744553660982949, + "loss": 1.2632, + "theoretical_loss": 3.4345383744658315, + "tokens_seen": 2179727360 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002744393179538616, + "loss": 1.2443, + "theoretical_loss": 3.434529979875178, + "tokens_seen": 2179792896 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002744232698094283, + "loss": 1.2643, + "theoretical_loss": 3.4345215856075715, + "tokens_seen": 2179858432 + }, + { + "epoch": 0.66, + "learning_rate": 0.000274407221664995, + "loss": 1.2245, + "theoretical_loss": 3.4345131916629894, + "tokens_seen": 2179923968 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002743911735205617, + "loss": 1.252, + "theoretical_loss": 3.4345047980414107, + "tokens_seen": 2179989504 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002743751253761284, + "loss": 1.2517, + "theoretical_loss": 3.434496404742813, + "tokens_seen": 2180055040 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002743590772316951, + "loss": 1.267, + "theoretical_loss": 3.434488011767173, + "tokens_seen": 2180120576 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002743430290872618, + "loss": 1.2699, + "theoretical_loss": 3.4344796191144695, + "tokens_seen": 2180186112 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002743269809428285, + "loss": 1.2405, + "theoretical_loss": 3.434471226784681, + "tokens_seen": 2180251648 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002743109327983952, + "loss": 1.2004, + "theoretical_loss": 3.4344628347777837, + "tokens_seen": 2180317184 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002742948846539619, + "loss": 1.2749, + "theoretical_loss": 3.434454443093757, + "tokens_seen": 2180382720 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002742788365095286, + "loss": 1.3867, + "theoretical_loss": 3.434446051732578, + "tokens_seen": 2180448256 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002742627883650953, + "loss": 1.2356, + "theoretical_loss": 3.434437660694225, + "tokens_seen": 2180513792 + }, + { + "epoch": 0.66, + "learning_rate": 0.000274246740220662, + "loss": 1.2854, + "theoretical_loss": 3.4344292699786756, + "tokens_seen": 2180579328 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002742306920762287, + "loss": 1.2404, + "theoretical_loss": 3.4344208795859075, + "tokens_seen": 2180644864 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002742146439317954, + "loss": 1.3423, + "theoretical_loss": 3.4344124895158985, + "tokens_seen": 2180710400 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002741985957873621, + "loss": 1.2093, + "theoretical_loss": 3.4344040997686274, + "tokens_seen": 2180775936 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002741825476429288, + "loss": 1.2617, + "theoretical_loss": 3.434395710344071, + "tokens_seen": 2180841472 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002741664994984955, + "loss": 1.2901, + "theoretical_loss": 3.434387321242208, + "tokens_seen": 2180907008 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002741504513540622, + "loss": 1.2759, + "theoretical_loss": 3.434378932463016, + "tokens_seen": 2180972544 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002741344032096289, + "loss": 1.1744, + "theoretical_loss": 3.4343705440064722, + "tokens_seen": 2181038080 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002741183550651956, + "loss": 1.3334, + "theoretical_loss": 3.434362155872556, + "tokens_seen": 2181103616 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002741023069207623, + "loss": 1.2214, + "theoretical_loss": 3.4343537680612437, + "tokens_seen": 2181169152 + }, + { + "epoch": 0.66, + "learning_rate": 0.000274086258776329, + "loss": 1.2653, + "theoretical_loss": 3.4343453805725144, + "tokens_seen": 2181234688 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002740702106318957, + "loss": 1.2438, + "theoretical_loss": 3.4343369934063457, + "tokens_seen": 2181300224 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002740541624874624, + "loss": 1.1942, + "theoretical_loss": 3.4343286065627145, + "tokens_seen": 2181365760 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002740381143430291, + "loss": 1.2304, + "theoretical_loss": 3.4343202200416005, + "tokens_seen": 2181431296 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002740220661985958, + "loss": 1.234, + "theoretical_loss": 3.43431183384298, + "tokens_seen": 2181496832 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002740060180541625, + "loss": 1.2002, + "theoretical_loss": 3.434303447966832, + "tokens_seen": 2181562368 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027398996990972923, + "loss": 1.1793, + "theoretical_loss": 3.434295062413134, + "tokens_seen": 2181627904 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027397392176529593, + "loss": 1.2862, + "theoretical_loss": 3.4342866771818636, + "tokens_seen": 2181693440 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027395787362086263, + "loss": 1.208, + "theoretical_loss": 3.434278292272999, + "tokens_seen": 2181758976 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027394182547642933, + "loss": 1.2871, + "theoretical_loss": 3.4342699076865184, + "tokens_seen": 2181824512 + }, + { + "epoch": 0.66, + "learning_rate": 0.000273925777331996, + "loss": 1.2709, + "theoretical_loss": 3.434261523422399, + "tokens_seen": 2181890048 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002739097291875627, + "loss": 1.3568, + "theoretical_loss": 3.43425313948062, + "tokens_seen": 2181955584 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002738936810431294, + "loss": 1.2199, + "theoretical_loss": 3.4342447558611577, + "tokens_seen": 2182021120 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002738776328986961, + "loss": 1.239, + "theoretical_loss": 3.4342363725639915, + "tokens_seen": 2182086656 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002738615847542628, + "loss": 1.1954, + "theoretical_loss": 3.434227989589098, + "tokens_seen": 2182152192 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002738455366098295, + "loss": 1.4862, + "theoretical_loss": 3.4342196069364563, + "tokens_seen": 2182217728 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002738294884653962, + "loss": 1.2685, + "theoretical_loss": 3.4342112246060434, + "tokens_seen": 2182283264 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 730643, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3123812675476074, + "objective/train/theoretical_loss": 3.434202842597838, + "objective/train/tokens_used": 2202808800, + "theoretical_loss": 3.434202842597838, + "tokens_seen": 2182348800 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002738134403209629, + "loss": 1.2078, + "theoretical_loss": 3.434202842597838, + "tokens_seen": 2182348800 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002737973921765296, + "loss": 1.2615, + "theoretical_loss": 3.434194460911818, + "tokens_seen": 2182414336 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002737813440320963, + "loss": 1.2898, + "theoretical_loss": 3.4341860795479606, + "tokens_seen": 2182479872 + }, + { + "epoch": 0.66, + "learning_rate": 0.000273765295887663, + "loss": 1.1619, + "theoretical_loss": 3.434177698506244, + "tokens_seen": 2182545408 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002737492477432297, + "loss": 1.3331, + "theoretical_loss": 3.434169317786647, + "tokens_seen": 2182610944 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002737331995987964, + "loss": 1.1833, + "theoretical_loss": 3.4341609373891466, + "tokens_seen": 2182676480 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002737171514543631, + "loss": 1.3169, + "theoretical_loss": 3.434152557313721, + "tokens_seen": 2182742016 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002737011033099298, + "loss": 1.2581, + "theoretical_loss": 3.434144177560348, + "tokens_seen": 2182807552 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002736850551654965, + "loss": 1.1956, + "theoretical_loss": 3.434135798129006, + "tokens_seen": 2182873088 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002736690070210632, + "loss": 1.1916, + "theoretical_loss": 3.434127419019673, + "tokens_seen": 2182938624 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002736529588766299, + "loss": 1.3026, + "theoretical_loss": 3.4341190402323263, + "tokens_seen": 2183004160 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002736369107321966, + "loss": 1.3305, + "theoretical_loss": 3.434110661766944, + "tokens_seen": 2183069696 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002736208625877633, + "loss": 1.2512, + "theoretical_loss": 3.4341022836235044, + "tokens_seen": 2183135232 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027360481444333, + "loss": 1.3034, + "theoretical_loss": 3.4340939058019857, + "tokens_seen": 2183200768 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002735887662988967, + "loss": 1.2633, + "theoretical_loss": 3.4340855283023655, + "tokens_seen": 2183266304 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002735727181544634, + "loss": 1.2445, + "theoretical_loss": 3.4340771511246215, + "tokens_seen": 2183331840 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002735566700100301, + "loss": 1.2815, + "theoretical_loss": 3.434068774268732, + "tokens_seen": 2183397376 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002735406218655968, + "loss": 1.2949, + "theoretical_loss": 3.434060397734675, + "tokens_seen": 2183462912 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002735245737211635, + "loss": 1.2032, + "theoretical_loss": 3.4340520215224286, + "tokens_seen": 2183528448 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002735085255767302, + "loss": 1.257, + "theoretical_loss": 3.4340436456319705, + "tokens_seen": 2183593984 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002734924774322969, + "loss": 1.2265, + "theoretical_loss": 3.4340352700632786, + "tokens_seen": 2183659520 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002734764292878636, + "loss": 1.2131, + "theoretical_loss": 3.434026894816331, + "tokens_seen": 2183725056 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002734603811434303, + "loss": 1.2897, + "theoretical_loss": 3.434018519891106, + "tokens_seen": 2183790592 + }, + { + "epoch": 0.66, + "learning_rate": 0.000273444332998997, + "loss": 1.2334, + "theoretical_loss": 3.4340101452875813, + "tokens_seen": 2183856128 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002734282848545637, + "loss": 1.1846, + "theoretical_loss": 3.434001771005735, + "tokens_seen": 2183921664 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002734122367101304, + "loss": 1.2991, + "theoretical_loss": 3.433993397045545, + "tokens_seen": 2183987200 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002733961885656971, + "loss": 1.2285, + "theoretical_loss": 3.433985023406989, + "tokens_seen": 2184052736 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002733801404212638, + "loss": 1.2308, + "theoretical_loss": 3.4339766500900453, + "tokens_seen": 2184118272 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002733640922768305, + "loss": 1.2202, + "theoretical_loss": 3.433968277094692, + "tokens_seen": 2184183808 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002733480441323972, + "loss": 1.2758, + "theoretical_loss": 3.4339599044209073, + "tokens_seen": 2184249344 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002733319959879639, + "loss": 1.3189, + "theoretical_loss": 3.4339515320686687, + "tokens_seen": 2184314880 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002733159478435306, + "loss": 1.232, + "theoretical_loss": 3.433943160037954, + "tokens_seen": 2184380416 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002732998996990973, + "loss": 1.3561, + "theoretical_loss": 3.4339347883287425, + "tokens_seen": 2184445952 + }, + { + "epoch": 0.66, + "learning_rate": 0.000273283851554664, + "loss": 1.2056, + "theoretical_loss": 3.4339264169410106, + "tokens_seen": 2184511488 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002732678034102307, + "loss": 1.2955, + "theoretical_loss": 3.433918045874737, + "tokens_seen": 2184577024 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002732517552657974, + "loss": 1.2344, + "theoretical_loss": 3.4339096751299003, + "tokens_seen": 2184642560 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002732357071213641, + "loss": 1.2858, + "theoretical_loss": 3.4339013047064775, + "tokens_seen": 2184708096 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002732196589769308, + "loss": 1.2592, + "theoretical_loss": 3.4338929346044473, + "tokens_seen": 2184773632 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002732036108324975, + "loss": 1.2947, + "theoretical_loss": 3.4338845648237877, + "tokens_seen": 2184839168 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002731875626880642, + "loss": 1.2275, + "theoretical_loss": 3.433876195364476, + "tokens_seen": 2184904704 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002731715145436309, + "loss": 1.302, + "theoretical_loss": 3.433867826226491, + "tokens_seen": 2184970240 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002731554663991976, + "loss": 1.223, + "theoretical_loss": 3.4338594574098105, + "tokens_seen": 2185035776 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002731394182547643, + "loss": 1.2574, + "theoretical_loss": 3.4338510889144125, + "tokens_seen": 2185101312 + }, + { + "epoch": 0.66, + "learning_rate": 0.000273123370110331, + "loss": 1.2727, + "theoretical_loss": 3.433842720740275, + "tokens_seen": 2185166848 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002731073219658977, + "loss": 1.2995, + "theoretical_loss": 3.4338343528873763, + "tokens_seen": 2185232384 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002730912738214644, + "loss": 1.2758, + "theoretical_loss": 3.4338259853556936, + "tokens_seen": 2185297920 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002730752256770311, + "loss": 1.3197, + "theoretical_loss": 3.433817618145206, + "tokens_seen": 2185363456 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002730591775325978, + "loss": 1.305, + "theoretical_loss": 3.433809251255891, + "tokens_seen": 2185428992 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002730431293881645, + "loss": 1.3012, + "theoretical_loss": 3.4338008846877264, + "tokens_seen": 2185494528 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002730270812437312, + "loss": 1.2831, + "theoretical_loss": 3.4337925184406908, + "tokens_seen": 2185560064 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 731623, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2505277395248413, + "objective/train/theoretical_loss": 3.433784152514762, + "objective/train/tokens_used": 2206085600, + "theoretical_loss": 3.433784152514762, + "tokens_seen": 2185625600 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002730110330992979, + "loss": 1.2646, + "theoretical_loss": 3.433784152514762, + "tokens_seen": 2185625600 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002729949849548646, + "loss": 1.2002, + "theoretical_loss": 3.433775786909918, + "tokens_seen": 2185691136 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002729789368104313, + "loss": 1.2039, + "theoretical_loss": 3.433767421626137, + "tokens_seen": 2185756672 + }, + { + "epoch": 0.66, + "learning_rate": 0.000272962888665998, + "loss": 1.3329, + "theoretical_loss": 3.433759056663397, + "tokens_seen": 2185822208 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002729468405215647, + "loss": 1.2831, + "theoretical_loss": 3.433750692021676, + "tokens_seen": 2185887744 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002729307923771314, + "loss": 1.2542, + "theoretical_loss": 3.433742327700952, + "tokens_seen": 2185953280 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002729147442326981, + "loss": 1.2321, + "theoretical_loss": 3.433733963701203, + "tokens_seen": 2186018816 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002728986960882648, + "loss": 1.265, + "theoretical_loss": 3.4337256000224072, + "tokens_seen": 2186084352 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002728826479438315, + "loss": 1.3418, + "theoretical_loss": 3.4337172366645428, + "tokens_seen": 2186149888 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002728665997993982, + "loss": 1.2669, + "theoretical_loss": 3.4337088736275874, + "tokens_seen": 2186215424 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002728505516549649, + "loss": 1.2032, + "theoretical_loss": 3.4337005109115197, + "tokens_seen": 2186280960 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002728345035105316, + "loss": 1.2839, + "theoretical_loss": 3.433692148516317, + "tokens_seen": 2186346496 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002728184553660983, + "loss": 1.3225, + "theoretical_loss": 3.4336837864419585, + "tokens_seen": 2186412032 + }, + { + "epoch": 0.66, + "learning_rate": 0.000272802407221665, + "loss": 1.3451, + "theoretical_loss": 3.4336754246884214, + "tokens_seen": 2186477568 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002727863590772317, + "loss": 1.3112, + "theoretical_loss": 3.4336670632556836, + "tokens_seen": 2186543104 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002727703109327984, + "loss": 1.3068, + "theoretical_loss": 3.433658702143724, + "tokens_seen": 2186608640 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002727542627883651, + "loss": 1.3228, + "theoretical_loss": 3.4336503413525197, + "tokens_seen": 2186674176 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002727382146439318, + "loss": 1.251, + "theoretical_loss": 3.4336419808820495, + "tokens_seen": 2186739712 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002727221664994985, + "loss": 1.3134, + "theoretical_loss": 3.4336336207322917, + "tokens_seen": 2186805248 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002727061183550652, + "loss": 1.2953, + "theoretical_loss": 3.4336252609032236, + "tokens_seen": 2186870784 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002726900702106319, + "loss": 1.2165, + "theoretical_loss": 3.4336169013948235, + "tokens_seen": 2186936320 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002726740220661986, + "loss": 1.3344, + "theoretical_loss": 3.43360854220707, + "tokens_seen": 2187001856 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002726579739217653, + "loss": 1.2352, + "theoretical_loss": 3.4336001833399408, + "tokens_seen": 2187067392 + }, + { + "epoch": 0.66, + "learning_rate": 0.000272641925777332, + "loss": 1.2273, + "theoretical_loss": 3.433591824793414, + "tokens_seen": 2187132928 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027262587763289873, + "loss": 1.2975, + "theoretical_loss": 3.4335834665674674, + "tokens_seen": 2187198464 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027260982948846543, + "loss": 1.2102, + "theoretical_loss": 3.4335751086620796, + "tokens_seen": 2187264000 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027259378134403213, + "loss": 1.2986, + "theoretical_loss": 3.4335667510772288, + "tokens_seen": 2187329536 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027257773319959883, + "loss": 1.3181, + "theoretical_loss": 3.4335583938128926, + "tokens_seen": 2187395072 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027256168505516553, + "loss": 1.2769, + "theoretical_loss": 3.4335500368690495, + "tokens_seen": 2187460608 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027254563691073223, + "loss": 1.2327, + "theoretical_loss": 3.4335416802456775, + "tokens_seen": 2187526144 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027252958876629893, + "loss": 1.2265, + "theoretical_loss": 3.4335333239427546, + "tokens_seen": 2187591680 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002725135406218656, + "loss": 1.2468, + "theoretical_loss": 3.433524967960259, + "tokens_seen": 2187657216 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002724974924774323, + "loss": 1.3969, + "theoretical_loss": 3.4335166122981686, + "tokens_seen": 2187722752 + }, + { + "epoch": 0.66, + "learning_rate": 0.000272481444332999, + "loss": 1.3756, + "theoretical_loss": 3.433508256956462, + "tokens_seen": 2187788288 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002724653961885657, + "loss": 1.3174, + "theoretical_loss": 3.433499901935117, + "tokens_seen": 2187853824 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002724493480441324, + "loss": 1.3112, + "theoretical_loss": 3.4334915472341114, + "tokens_seen": 2187919360 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002724332998996991, + "loss": 1.1969, + "theoretical_loss": 3.433483192853424, + "tokens_seen": 2187984896 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002724172517552658, + "loss": 1.2444, + "theoretical_loss": 3.4334748387930327, + "tokens_seen": 2188050432 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002724012036108325, + "loss": 1.3377, + "theoretical_loss": 3.4334664850529153, + "tokens_seen": 2188115968 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002723851554663992, + "loss": 1.2265, + "theoretical_loss": 3.4334581316330506, + "tokens_seen": 2188181504 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002723691073219659, + "loss": 1.2653, + "theoretical_loss": 3.433449778533416, + "tokens_seen": 2188247040 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002723530591775326, + "loss": 1.3012, + "theoretical_loss": 3.4334414257539896, + "tokens_seen": 2188312576 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002723370110330993, + "loss": 1.2674, + "theoretical_loss": 3.4334330732947502, + "tokens_seen": 2188378112 + }, + { + "epoch": 0.66, + "learning_rate": 0.000272320962888666, + "loss": 1.2738, + "theoretical_loss": 3.433424721155675, + "tokens_seen": 2188443648 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002723049147442327, + "loss": 1.1709, + "theoretical_loss": 3.4334163693367437, + "tokens_seen": 2188509184 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002722888665997994, + "loss": 1.2487, + "theoretical_loss": 3.433408017837933, + "tokens_seen": 2188574720 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002722728184553661, + "loss": 1.1693, + "theoretical_loss": 3.4333996666592217, + "tokens_seen": 2188640256 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002722567703109328, + "loss": 1.2151, + "theoretical_loss": 3.4333913158005873, + "tokens_seen": 2188705792 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002722407221664995, + "loss": 1.2476, + "theoretical_loss": 3.433382965262009, + "tokens_seen": 2188771328 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002722246740220662, + "loss": 1.2152, + "theoretical_loss": 3.433374615043464, + "tokens_seen": 2188836864 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 732904, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.46730375289917, + "objective/train/theoretical_loss": 3.433366265144931, + "objective/train/tokens_used": 2209362400, + "theoretical_loss": 3.433366265144931, + "tokens_seen": 2188902400 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002722086258776329, + "loss": 1.4159, + "theoretical_loss": 3.433366265144931, + "tokens_seen": 2188902400 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002721925777331996, + "loss": 1.2832, + "theoretical_loss": 3.433357915566388, + "tokens_seen": 2188967936 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002721765295887663, + "loss": 1.2477, + "theoretical_loss": 3.433349566307813, + "tokens_seen": 2189033472 + }, + { + "epoch": 0.66, + "learning_rate": 0.000272160481444333, + "loss": 1.3136, + "theoretical_loss": 3.4333412173691844, + "tokens_seen": 2189099008 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002721444332998997, + "loss": 1.2506, + "theoretical_loss": 3.43333286875048, + "tokens_seen": 2189164544 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002721283851554664, + "loss": 1.1821, + "theoretical_loss": 3.433324520451679, + "tokens_seen": 2189230080 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002721123370110331, + "loss": 1.3606, + "theoretical_loss": 3.433316172472758, + "tokens_seen": 2189295616 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002720962888665998, + "loss": 1.2758, + "theoretical_loss": 3.433307824813696, + "tokens_seen": 2189361152 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002720802407221665, + "loss": 1.274, + "theoretical_loss": 3.4332994774744714, + "tokens_seen": 2189426688 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027206419257773325, + "loss": 1.3246, + "theoretical_loss": 3.433291130455062, + "tokens_seen": 2189492224 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027204814443329995, + "loss": 1.2384, + "theoretical_loss": 3.433282783755446, + "tokens_seen": 2189557760 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027203209628886665, + "loss": 1.473, + "theoretical_loss": 3.4332744373756015, + "tokens_seen": 2189623296 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027201604814443335, + "loss": 1.2529, + "theoretical_loss": 3.4332660913155073, + "tokens_seen": 2189688832 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027200000000000005, + "loss": 1.2508, + "theoretical_loss": 3.433257745575141, + "tokens_seen": 2189754368 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027198395185556675, + "loss": 1.2115, + "theoretical_loss": 3.4332494001544807, + "tokens_seen": 2189819904 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027196790371113345, + "loss": 1.3055, + "theoretical_loss": 3.4332410550535046, + "tokens_seen": 2189885440 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027195185556670015, + "loss": 1.1699, + "theoretical_loss": 3.4332327102721916, + "tokens_seen": 2189950976 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027193580742226685, + "loss": 1.2941, + "theoretical_loss": 3.433224365810519, + "tokens_seen": 2190016512 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027191975927783356, + "loss": 1.283, + "theoretical_loss": 3.433216021668465, + "tokens_seen": 2190082048 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027190371113340026, + "loss": 1.2748, + "theoretical_loss": 3.433207677846009, + "tokens_seen": 2190147584 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002718876629889669, + "loss": 1.302, + "theoretical_loss": 3.4331993343431275, + "tokens_seen": 2190213120 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002718716148445336, + "loss": 1.2686, + "theoretical_loss": 3.4331909911598, + "tokens_seen": 2190278656 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002718555667001003, + "loss": 1.2531, + "theoretical_loss": 3.4331826482960044, + "tokens_seen": 2190344192 + }, + { + "epoch": 0.66, + "learning_rate": 0.000271839518555667, + "loss": 1.2229, + "theoretical_loss": 3.4331743057517183, + "tokens_seen": 2190409728 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002718234704112337, + "loss": 1.2915, + "theoretical_loss": 3.4331659635269207, + "tokens_seen": 2190475264 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002718074222668004, + "loss": 1.2542, + "theoretical_loss": 3.4331576216215893, + "tokens_seen": 2190540800 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002717913741223671, + "loss": 1.3559, + "theoretical_loss": 3.4331492800357024, + "tokens_seen": 2190606336 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002717753259779338, + "loss": 1.28, + "theoretical_loss": 3.4331409387692386, + "tokens_seen": 2190671872 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002717592778335005, + "loss": 1.1738, + "theoretical_loss": 3.4331325978221754, + "tokens_seen": 2190737408 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002717432296890672, + "loss": 1.1663, + "theoretical_loss": 3.4331242571944913, + "tokens_seen": 2190802944 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002717271815446339, + "loss": 1.2851, + "theoretical_loss": 3.433115916886165, + "tokens_seen": 2190868480 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002717111334002006, + "loss": 1.2457, + "theoretical_loss": 3.433107576897174, + "tokens_seen": 2190934016 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002716950852557673, + "loss": 1.2872, + "theoretical_loss": 3.433099237227497, + "tokens_seen": 2190999552 + }, + { + "epoch": 0.66, + "learning_rate": 0.000271679037111334, + "loss": 1.1734, + "theoretical_loss": 3.433090897877112, + "tokens_seen": 2191065088 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002716629889669007, + "loss": 1.2922, + "theoretical_loss": 3.4330825588459977, + "tokens_seen": 2191130624 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002716469408224674, + "loss": 1.2493, + "theoretical_loss": 3.4330742201341318, + "tokens_seen": 2191196160 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002716308926780341, + "loss": 1.2942, + "theoretical_loss": 3.4330658817414923, + "tokens_seen": 2191261696 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002716148445336008, + "loss": 1.2626, + "theoretical_loss": 3.433057543668058, + "tokens_seen": 2191327232 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002715987963891675, + "loss": 1.399, + "theoretical_loss": 3.4330492059138074, + "tokens_seen": 2191392768 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002715827482447342, + "loss": 1.353, + "theoretical_loss": 3.4330408684787175, + "tokens_seen": 2191458304 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002715667001003009, + "loss": 1.3003, + "theoretical_loss": 3.4330325313627674, + "tokens_seen": 2191523840 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002715506519558676, + "loss": 1.1713, + "theoretical_loss": 3.4330241945659354, + "tokens_seen": 2191589376 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002715346038114343, + "loss": 1.2605, + "theoretical_loss": 3.4330158580881998, + "tokens_seen": 2191654912 + }, + { + "epoch": 0.66, + "learning_rate": 0.000271518555667001, + "loss": 1.2558, + "theoretical_loss": 3.4330075219295386, + "tokens_seen": 2191720448 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002715025075225677, + "loss": 1.2202, + "theoretical_loss": 3.43299918608993, + "tokens_seen": 2191785984 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002714864593781344, + "loss": 1.2727, + "theoretical_loss": 3.432990850569352, + "tokens_seen": 2191851520 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002714704112337011, + "loss": 1.1913, + "theoretical_loss": 3.432982515367784, + "tokens_seen": 2191917056 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002714543630892678, + "loss": 1.2485, + "theoretical_loss": 3.4329741804852025, + "tokens_seen": 2191982592 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002714383149448345, + "loss": 1.1969, + "theoretical_loss": 3.4329658459215873, + "tokens_seen": 2192048128 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002714222668004012, + "loss": 1.163, + "theoretical_loss": 3.432957511676916, + "tokens_seen": 2192113664 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 733863, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2483806610107422, + "objective/train/theoretical_loss": 3.4329491777511665, + "objective/train/tokens_used": 2212639200, + "theoretical_loss": 3.4329491777511665, + "tokens_seen": 2192179200 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002714062186559679, + "loss": 1.2159, + "theoretical_loss": 3.4329491777511665, + "tokens_seen": 2192179200 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002713901705115346, + "loss": 1.341, + "theoretical_loss": 3.432940844144318, + "tokens_seen": 2192244736 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002713741223671013, + "loss": 1.2892, + "theoretical_loss": 3.432932510856348, + "tokens_seen": 2192310272 + }, + { + "epoch": 0.66, + "learning_rate": 0.000271358074222668, + "loss": 1.2212, + "theoretical_loss": 3.432924177887235, + "tokens_seen": 2192375808 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002713420260782347, + "loss": 1.3107, + "theoretical_loss": 3.4329158452369573, + "tokens_seen": 2192441344 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002713259779338014, + "loss": 1.2408, + "theoretical_loss": 3.432907512905493, + "tokens_seen": 2192506880 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002713099297893681, + "loss": 1.2682, + "theoretical_loss": 3.4328991808928206, + "tokens_seen": 2192572416 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002712938816449348, + "loss": 1.2602, + "theoretical_loss": 3.4328908491989183, + "tokens_seen": 2192637952 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002712778335005015, + "loss": 1.2583, + "theoretical_loss": 3.4328825178237645, + "tokens_seen": 2192703488 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002712617853560682, + "loss": 1.3101, + "theoretical_loss": 3.432874186767337, + "tokens_seen": 2192769024 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027124573721163493, + "loss": 1.2182, + "theoretical_loss": 3.432865856029615, + "tokens_seen": 2192834560 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027122968906720163, + "loss": 1.2433, + "theoretical_loss": 3.4328575256105758, + "tokens_seen": 2192900096 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027121364092276833, + "loss": 1.2139, + "theoretical_loss": 3.4328491955101983, + "tokens_seen": 2192965632 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027119759277833503, + "loss": 1.2454, + "theoretical_loss": 3.43284086572846, + "tokens_seen": 2193031168 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027118154463390173, + "loss": 1.2587, + "theoretical_loss": 3.4328325362653405, + "tokens_seen": 2193096704 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027116549648946843, + "loss": 1.2008, + "theoretical_loss": 3.4328242071208175, + "tokens_seen": 2193162240 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027114944834503513, + "loss": 1.305, + "theoretical_loss": 3.4328158782948686, + "tokens_seen": 2193227776 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027113340020060183, + "loss": 1.2119, + "theoretical_loss": 3.432807549787473, + "tokens_seen": 2193293312 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002711173520561685, + "loss": 1.1968, + "theoretical_loss": 3.432799221598608, + "tokens_seen": 2193358848 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002711013039117352, + "loss": 1.2479, + "theoretical_loss": 3.4327908937282534, + "tokens_seen": 2193424384 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002710852557673019, + "loss": 1.2537, + "theoretical_loss": 3.432782566176386, + "tokens_seen": 2193489920 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002710692076228686, + "loss": 1.2576, + "theoretical_loss": 3.432774238942985, + "tokens_seen": 2193555456 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002710531594784353, + "loss": 1.242, + "theoretical_loss": 3.432765912028029, + "tokens_seen": 2193620992 + }, + { + "epoch": 0.66, + "learning_rate": 0.000271037111334002, + "loss": 1.2886, + "theoretical_loss": 3.4327575854314953, + "tokens_seen": 2193686528 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002710210631895687, + "loss": 1.1737, + "theoretical_loss": 3.4327492591533626, + "tokens_seen": 2193752064 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002710050150451354, + "loss": 1.2621, + "theoretical_loss": 3.4327409331936094, + "tokens_seen": 2193817600 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002709889669007021, + "loss": 1.337, + "theoretical_loss": 3.432732607552214, + "tokens_seen": 2193883136 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002709729187562688, + "loss": 1.3282, + "theoretical_loss": 3.4327242822291546, + "tokens_seen": 2193948672 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002709568706118355, + "loss": 1.197, + "theoretical_loss": 3.43271595722441, + "tokens_seen": 2194014208 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002709408224674022, + "loss": 1.2183, + "theoretical_loss": 3.4327076325379573, + "tokens_seen": 2194079744 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002709247743229689, + "loss": 1.3134, + "theoretical_loss": 3.432699308169776, + "tokens_seen": 2194145280 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002709087261785356, + "loss": 1.1669, + "theoretical_loss": 3.432690984119844, + "tokens_seen": 2194210816 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002708926780341023, + "loss": 1.3069, + "theoretical_loss": 3.4326826603881395, + "tokens_seen": 2194276352 + }, + { + "epoch": 0.66, + "learning_rate": 0.000270876629889669, + "loss": 1.2845, + "theoretical_loss": 3.432674336974641, + "tokens_seen": 2194341888 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027086058174523574, + "loss": 1.2932, + "theoretical_loss": 3.4326660138793272, + "tokens_seen": 2194407424 + }, + { + "epoch": 0.66, + "learning_rate": 0.00027084453360080244, + "loss": 1.2751, + "theoretical_loss": 3.432657691102176, + "tokens_seen": 2194472960 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027082848545636914, + "loss": 1.2929, + "theoretical_loss": 3.4326493686431654, + "tokens_seen": 2194538496 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027081243731193584, + "loss": 1.135, + "theoretical_loss": 3.4326410465022743, + "tokens_seen": 2194604032 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027079638916750255, + "loss": 1.1969, + "theoretical_loss": 3.4326327246794808, + "tokens_seen": 2194669568 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027078034102306925, + "loss": 1.2494, + "theoretical_loss": 3.4326244031747635, + "tokens_seen": 2194735104 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027076429287863595, + "loss": 1.2099, + "theoretical_loss": 3.4326160819881006, + "tokens_seen": 2194800640 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027074824473420265, + "loss": 1.1847, + "theoretical_loss": 3.43260776111947, + "tokens_seen": 2194866176 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027073219658976935, + "loss": 1.212, + "theoretical_loss": 3.432599440568851, + "tokens_seen": 2194931712 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027071614844533605, + "loss": 1.3114, + "theoretical_loss": 3.432591120336221, + "tokens_seen": 2194997248 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027070010030090275, + "loss": 1.2093, + "theoretical_loss": 3.432582800421559, + "tokens_seen": 2195062784 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027068405215646945, + "loss": 1.2646, + "theoretical_loss": 3.432574480824843, + "tokens_seen": 2195128320 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027066800401203615, + "loss": 1.2579, + "theoretical_loss": 3.4325661615460517, + "tokens_seen": 2195193856 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027065195586760285, + "loss": 1.2372, + "theoretical_loss": 3.4325578425851626, + "tokens_seen": 2195259392 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027063590772316955, + "loss": 1.2807, + "theoretical_loss": 3.4325495239421553, + "tokens_seen": 2195324928 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027061985957873625, + "loss": 1.1864, + "theoretical_loss": 3.4325412056170075, + "tokens_seen": 2195390464 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 735101, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.258005976676941, + "objective/train/theoretical_loss": 3.4325328876096974, + "objective/train/tokens_used": 2215916000, + "theoretical_loss": 3.4325328876096974, + "tokens_seen": 2195456000 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027060381143430295, + "loss": 1.2914, + "theoretical_loss": 3.4325328876096974, + "tokens_seen": 2195456000 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027058776328986965, + "loss": 1.145, + "theoretical_loss": 3.432524569920204, + "tokens_seen": 2195521536 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027057171514543635, + "loss": 1.3145, + "theoretical_loss": 3.432516252548505, + "tokens_seen": 2195587072 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027055566700100305, + "loss": 1.3557, + "theoretical_loss": 3.432507935494579, + "tokens_seen": 2195652608 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027053961885656976, + "loss": 1.2459, + "theoretical_loss": 3.4324996187584045, + "tokens_seen": 2195718144 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027052357071213646, + "loss": 1.2123, + "theoretical_loss": 3.4324913023399595, + "tokens_seen": 2195783680 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027050752256770316, + "loss": 1.2614, + "theoretical_loss": 3.432482986239223, + "tokens_seen": 2195849216 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027049147442326986, + "loss": 1.2463, + "theoretical_loss": 3.432474670456173, + "tokens_seen": 2195914752 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002704754262788365, + "loss": 1.2255, + "theoretical_loss": 3.4324663549907877, + "tokens_seen": 2195980288 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002704593781344032, + "loss": 1.1144, + "theoretical_loss": 3.432458039843046, + "tokens_seen": 2196045824 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002704433299899699, + "loss": 1.1742, + "theoretical_loss": 3.4324497250129262, + "tokens_seen": 2196111360 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002704272818455366, + "loss": 1.2132, + "theoretical_loss": 3.432441410500406, + "tokens_seen": 2196176896 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002704112337011033, + "loss": 1.2292, + "theoretical_loss": 3.4324330963054646, + "tokens_seen": 2196242432 + }, + { + "epoch": 0.67, + "learning_rate": 0.00027039518555667, + "loss": 1.3344, + "theoretical_loss": 3.43242478242808, + "tokens_seen": 2196307968 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002703791374122367, + "loss": 1.2638, + "theoretical_loss": 3.432416468868231, + "tokens_seen": 2196373504 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002703630892678034, + "loss": 1.3223, + "theoretical_loss": 3.432408155625895, + "tokens_seen": 2196439040 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002703470411233701, + "loss": 1.178, + "theoretical_loss": 3.4323998427010514, + "tokens_seen": 2196504576 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002703309929789368, + "loss": 1.2271, + "theoretical_loss": 3.4323915300936783, + "tokens_seen": 2196570112 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002703149448345035, + "loss": 1.2382, + "theoretical_loss": 3.4323832178037543, + "tokens_seen": 2196635648 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002702988966900702, + "loss": 1.2946, + "theoretical_loss": 3.4323749058312574, + "tokens_seen": 2196701184 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002702828485456369, + "loss": 1.1953, + "theoretical_loss": 3.432366594176166, + "tokens_seen": 2196766720 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002702668004012036, + "loss": 1.2693, + "theoretical_loss": 3.432358282838459, + "tokens_seen": 2196832256 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002702507522567703, + "loss": 1.2531, + "theoretical_loss": 3.4323499718181147, + "tokens_seen": 2196897792 + }, + { + "epoch": 0.67, + "learning_rate": 0.000270234704112337, + "loss": 1.2974, + "theoretical_loss": 3.4323416611151107, + "tokens_seen": 2196963328 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002702186559679037, + "loss": 1.2197, + "theoretical_loss": 3.4323333507294267, + "tokens_seen": 2197028864 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002702026078234704, + "loss": 1.306, + "theoretical_loss": 3.4323250406610404, + "tokens_seen": 2197094400 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002701865596790371, + "loss": 1.3408, + "theoretical_loss": 3.43231673090993, + "tokens_seen": 2197159936 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002701705115346038, + "loss": 1.3094, + "theoretical_loss": 3.4323084214760744, + "tokens_seen": 2197225472 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002701544633901705, + "loss": 1.2987, + "theoretical_loss": 3.4323001123594516, + "tokens_seen": 2197291008 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002701384152457372, + "loss": 1.2499, + "theoretical_loss": 3.4322918035600405, + "tokens_seen": 2197356544 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002701223671013039, + "loss": 1.2786, + "theoretical_loss": 3.432283495077819, + "tokens_seen": 2197422080 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002701063189568706, + "loss": 1.326, + "theoretical_loss": 3.4322751869127663, + "tokens_seen": 2197487616 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002700902708124373, + "loss": 1.2045, + "theoretical_loss": 3.4322668790648603, + "tokens_seen": 2197553152 + }, + { + "epoch": 0.67, + "learning_rate": 0.000270074222668004, + "loss": 1.2295, + "theoretical_loss": 3.432258571534079, + "tokens_seen": 2197618688 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002700581745235707, + "loss": 1.2475, + "theoretical_loss": 3.432250264320402, + "tokens_seen": 2197684224 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002700421263791374, + "loss": 1.239, + "theoretical_loss": 3.4322419574238068, + "tokens_seen": 2197749760 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002700260782347041, + "loss": 1.2133, + "theoretical_loss": 3.432233650844272, + "tokens_seen": 2197815296 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002700100300902708, + "loss": 1.1795, + "theoretical_loss": 3.4322253445817763, + "tokens_seen": 2197880832 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002699939819458375, + "loss": 1.2336, + "theoretical_loss": 3.432217038636298, + "tokens_seen": 2197946368 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002699779338014042, + "loss": 1.2313, + "theoretical_loss": 3.4322087330078155, + "tokens_seen": 2198011904 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002699618856569709, + "loss": 1.2663, + "theoretical_loss": 3.432200427696307, + "tokens_seen": 2198077440 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002699458375125376, + "loss": 1.1997, + "theoretical_loss": 3.432192122701752, + "tokens_seen": 2198142976 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002699297893681043, + "loss": 1.2451, + "theoretical_loss": 3.4321838180241278, + "tokens_seen": 2198208512 + }, + { + "epoch": 0.67, + "learning_rate": 0.000269913741223671, + "loss": 1.1874, + "theoretical_loss": 3.432175513663413, + "tokens_seen": 2198274048 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002698976930792377, + "loss": 1.2854, + "theoretical_loss": 3.432167209619587, + "tokens_seen": 2198339584 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026988164493480443, + "loss": 1.2318, + "theoretical_loss": 3.4321589058926274, + "tokens_seen": 2198405120 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026986559679037113, + "loss": 1.2368, + "theoretical_loss": 3.432150602482513, + "tokens_seen": 2198470656 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026984954864593783, + "loss": 1.2765, + "theoretical_loss": 3.4321422993892217, + "tokens_seen": 2198536192 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026983350050150453, + "loss": 1.0955, + "theoretical_loss": 3.432133996612732, + "tokens_seen": 2198601728 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026981745235707123, + "loss": 1.2752, + "theoretical_loss": 3.432125694153024, + "tokens_seen": 2198667264 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 736290, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.264587163925171, + "objective/train/theoretical_loss": 3.432117392010074, + "objective/train/tokens_used": 2219192800, + "theoretical_loss": 3.432117392010074, + "tokens_seen": 2198732800 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026980140421263793, + "loss": 1.1972, + "theoretical_loss": 3.432117392010074, + "tokens_seen": 2198732800 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026978535606820463, + "loss": 1.2159, + "theoretical_loss": 3.432109090183862, + "tokens_seen": 2198798336 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026976930792377133, + "loss": 1.3173, + "theoretical_loss": 3.432100788674365, + "tokens_seen": 2198863872 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026975325977933803, + "loss": 1.2344, + "theoretical_loss": 3.4320924874815635, + "tokens_seen": 2198929408 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026973721163490473, + "loss": 1.2741, + "theoretical_loss": 3.432084186605434, + "tokens_seen": 2198994944 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026972116349047143, + "loss": 1.2819, + "theoretical_loss": 3.4320758860459564, + "tokens_seen": 2199060480 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002697051153460381, + "loss": 1.3089, + "theoretical_loss": 3.4320675858031082, + "tokens_seen": 2199126016 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002696890672016048, + "loss": 1.2844, + "theoretical_loss": 3.4320592858768686, + "tokens_seen": 2199191552 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002696730190571715, + "loss": 1.2121, + "theoretical_loss": 3.4320509862672157, + "tokens_seen": 2199257088 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026965697091273824, + "loss": 1.2555, + "theoretical_loss": 3.432042686974128, + "tokens_seen": 2199322624 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026964092276830494, + "loss": 1.2942, + "theoretical_loss": 3.432034387997584, + "tokens_seen": 2199388160 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026962487462387164, + "loss": 1.312, + "theoretical_loss": 3.4320260893375623, + "tokens_seen": 2199453696 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026960882647943834, + "loss": 1.3241, + "theoretical_loss": 3.4320177909940415, + "tokens_seen": 2199519232 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026959277833500504, + "loss": 1.249, + "theoretical_loss": 3.432009492967, + "tokens_seen": 2199584768 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026957673019057174, + "loss": 1.3359, + "theoretical_loss": 3.432001195256416, + "tokens_seen": 2199650304 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026956068204613844, + "loss": 1.2252, + "theoretical_loss": 3.4319928978622687, + "tokens_seen": 2199715840 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026954463390170514, + "loss": 1.2686, + "theoretical_loss": 3.431984600784536, + "tokens_seen": 2199781376 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026952858575727184, + "loss": 1.2417, + "theoretical_loss": 3.4319763040231965, + "tokens_seen": 2199846912 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026951253761283854, + "loss": 1.3448, + "theoretical_loss": 3.431968007578229, + "tokens_seen": 2199912448 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026949648946840524, + "loss": 1.2031, + "theoretical_loss": 3.431959711449611, + "tokens_seen": 2199977984 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026948044132397194, + "loss": 1.1484, + "theoretical_loss": 3.431951415637323, + "tokens_seen": 2200043520 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026946439317953864, + "loss": 1.3712, + "theoretical_loss": 3.4319431201413417, + "tokens_seen": 2200109056 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026944834503510534, + "loss": 1.2543, + "theoretical_loss": 3.431934824961646, + "tokens_seen": 2200174592 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026943229689067205, + "loss": 1.3211, + "theoretical_loss": 3.4319265300982154, + "tokens_seen": 2200240128 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026941624874623875, + "loss": 1.2639, + "theoretical_loss": 3.431918235551027, + "tokens_seen": 2200305664 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026940020060180545, + "loss": 1.3417, + "theoretical_loss": 3.4319099413200607, + "tokens_seen": 2200371200 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026938415245737215, + "loss": 1.3156, + "theoretical_loss": 3.4319016474052937, + "tokens_seen": 2200436736 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026936810431293885, + "loss": 1.3307, + "theoretical_loss": 3.431893353806706, + "tokens_seen": 2200502272 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026935205616850555, + "loss": 1.0877, + "theoretical_loss": 3.4318850605242743, + "tokens_seen": 2200567808 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026933600802407225, + "loss": 1.2511, + "theoretical_loss": 3.431876767557979, + "tokens_seen": 2200633344 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026931995987963895, + "loss": 1.2661, + "theoretical_loss": 3.4318684749077972, + "tokens_seen": 2200698880 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026930391173520565, + "loss": 1.2161, + "theoretical_loss": 3.431860182573708, + "tokens_seen": 2200764416 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026928786359077235, + "loss": 1.1842, + "theoretical_loss": 3.4318518905556905, + "tokens_seen": 2200829952 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026927181544633905, + "loss": 1.2475, + "theoretical_loss": 3.431843598853722, + "tokens_seen": 2200895488 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026925576730190575, + "loss": 1.1703, + "theoretical_loss": 3.4318353074677823, + "tokens_seen": 2200961024 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026923971915747245, + "loss": 1.1451, + "theoretical_loss": 3.431827016397849, + "tokens_seen": 2201026560 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026922367101303915, + "loss": 1.2075, + "theoretical_loss": 3.4318187256439012, + "tokens_seen": 2201092096 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026920762286860585, + "loss": 1.2883, + "theoretical_loss": 3.431810435205917, + "tokens_seen": 2201157632 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026919157472417255, + "loss": 1.2326, + "theoretical_loss": 3.431802145083876, + "tokens_seen": 2201223168 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026917552657973926, + "loss": 1.2004, + "theoretical_loss": 3.431793855277755, + "tokens_seen": 2201288704 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026915947843530596, + "loss": 1.2013, + "theoretical_loss": 3.4317855657875342, + "tokens_seen": 2201354240 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026914343029087266, + "loss": 1.3181, + "theoretical_loss": 3.4317772766131913, + "tokens_seen": 2201419776 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026912738214643936, + "loss": 1.2636, + "theoretical_loss": 3.431768987754705, + "tokens_seen": 2201485312 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026911133400200606, + "loss": 1.2769, + "theoretical_loss": 3.431760699212054, + "tokens_seen": 2201550848 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026909528585757276, + "loss": 1.3557, + "theoretical_loss": 3.431752410985217, + "tokens_seen": 2201616384 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002690792377131394, + "loss": 1.2522, + "theoretical_loss": 3.4317441230741723, + "tokens_seen": 2201681920 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002690631895687061, + "loss": 1.2836, + "theoretical_loss": 3.4317358354788983, + "tokens_seen": 2201747456 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002690471414242728, + "loss": 1.2729, + "theoretical_loss": 3.431727548199374, + "tokens_seen": 2201812992 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002690310932798395, + "loss": 1.199, + "theoretical_loss": 3.4317192612355774, + "tokens_seen": 2201878528 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002690150451354062, + "loss": 1.3288, + "theoretical_loss": 3.431710974587488, + "tokens_seen": 2201944064 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 737257, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1944656372070312, + "objective/train/theoretical_loss": 3.4317026882550827, + "objective/train/tokens_used": 2222469600, + "theoretical_loss": 3.4317026882550827, + "tokens_seen": 2202009600 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002689989969909729, + "loss": 1.2246, + "theoretical_loss": 3.4317026882550827, + "tokens_seen": 2202009600 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002689829488465396, + "loss": 1.2792, + "theoretical_loss": 3.4316944022383424, + "tokens_seen": 2202075136 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002689669007021063, + "loss": 1.1584, + "theoretical_loss": 3.431686116537244, + "tokens_seen": 2202140672 + }, + { + "epoch": 0.67, + "learning_rate": 0.000268950852557673, + "loss": 1.274, + "theoretical_loss": 3.4316778311517666, + "tokens_seen": 2202206208 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002689348044132397, + "loss": 1.2699, + "theoretical_loss": 3.4316695460818885, + "tokens_seen": 2202271744 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002689187562688064, + "loss": 1.2094, + "theoretical_loss": 3.431661261327589, + "tokens_seen": 2202337280 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002689027081243731, + "loss": 1.219, + "theoretical_loss": 3.431652976888846, + "tokens_seen": 2202402816 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002688866599799398, + "loss": 1.2498, + "theoretical_loss": 3.431644692765638, + "tokens_seen": 2202468352 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002688706118355065, + "loss": 1.2601, + "theoretical_loss": 3.4316364089579445, + "tokens_seen": 2202533888 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002688545636910732, + "loss": 1.1866, + "theoretical_loss": 3.431628125465743, + "tokens_seen": 2202599424 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002688385155466399, + "loss": 1.2359, + "theoretical_loss": 3.4316198422890127, + "tokens_seen": 2202664960 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002688224674022066, + "loss": 1.2937, + "theoretical_loss": 3.4316115594277323, + "tokens_seen": 2202730496 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002688064192577733, + "loss": 1.2418, + "theoretical_loss": 3.43160327688188, + "tokens_seen": 2202796032 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026879037111334, + "loss": 1.1501, + "theoretical_loss": 3.4315949946514346, + "tokens_seen": 2202861568 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002687743229689067, + "loss": 1.2672, + "theoretical_loss": 3.431586712736375, + "tokens_seen": 2202927104 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002687582748244734, + "loss": 1.1957, + "theoretical_loss": 3.4315784311366793, + "tokens_seen": 2202992640 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002687422266800401, + "loss": 1.251, + "theoretical_loss": 3.4315701498523263, + "tokens_seen": 2203058176 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002687261785356068, + "loss": 1.2124, + "theoretical_loss": 3.431561868883295, + "tokens_seen": 2203123712 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002687101303911735, + "loss": 1.2345, + "theoretical_loss": 3.431553588229563, + "tokens_seen": 2203189248 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002686940822467402, + "loss": 1.2779, + "theoretical_loss": 3.4315453078911102, + "tokens_seen": 2203254784 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002686780341023069, + "loss": 1.2664, + "theoretical_loss": 3.4315370278679143, + "tokens_seen": 2203320320 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002686619859578736, + "loss": 1.2021, + "theoretical_loss": 3.431528748159954, + "tokens_seen": 2203385856 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002686459378134403, + "loss": 1.235, + "theoretical_loss": 3.4315204687672085, + "tokens_seen": 2203451392 + }, + { + "epoch": 0.67, + "learning_rate": 0.000268629889669007, + "loss": 1.3057, + "theoretical_loss": 3.4315121896896557, + "tokens_seen": 2203516928 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002686138415245737, + "loss": 1.0619, + "theoretical_loss": 3.4315039109272747, + "tokens_seen": 2203582464 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002685977933801404, + "loss": 1.1764, + "theoretical_loss": 3.4314956324800443, + "tokens_seen": 2203648000 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002685817452357071, + "loss": 1.2816, + "theoretical_loss": 3.4314873543479427, + "tokens_seen": 2203713536 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002685656970912738, + "loss": 1.1819, + "theoretical_loss": 3.4314790765309486, + "tokens_seen": 2203779072 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002685496489468405, + "loss": 1.4157, + "theoretical_loss": 3.4314707990290407, + "tokens_seen": 2203844608 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002685336008024072, + "loss": 1.2172, + "theoretical_loss": 3.4314625218421977, + "tokens_seen": 2203910144 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026851755265797393, + "loss": 1.2681, + "theoretical_loss": 3.431454244970398, + "tokens_seen": 2203975680 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026850150451354063, + "loss": 1.2259, + "theoretical_loss": 3.431445968413621, + "tokens_seen": 2204041216 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026848545636910733, + "loss": 1.1786, + "theoretical_loss": 3.431437692171844, + "tokens_seen": 2204106752 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026846940822467403, + "loss": 1.2506, + "theoretical_loss": 3.431429416245047, + "tokens_seen": 2204172288 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026845336008024073, + "loss": 1.3745, + "theoretical_loss": 3.4314211406332076, + "tokens_seen": 2204237824 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026843731193580743, + "loss": 1.1679, + "theoretical_loss": 3.4314128653363056, + "tokens_seen": 2204303360 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026842126379137413, + "loss": 1.2446, + "theoretical_loss": 3.4314045903543184, + "tokens_seen": 2204368896 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026840521564694083, + "loss": 1.3217, + "theoretical_loss": 3.4313963156872256, + "tokens_seen": 2204434432 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026838916750250753, + "loss": 1.2546, + "theoretical_loss": 3.431388041335005, + "tokens_seen": 2204499968 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026837311935807423, + "loss": 1.2129, + "theoretical_loss": 3.4313797672976363, + "tokens_seen": 2204565504 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026835707121364093, + "loss": 1.267, + "theoretical_loss": 3.431371493575097, + "tokens_seen": 2204631040 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026834102306920763, + "loss": 1.318, + "theoretical_loss": 3.4313632201673667, + "tokens_seen": 2204696576 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026832497492477433, + "loss": 1.293, + "theoretical_loss": 3.4313549470744236, + "tokens_seen": 2204762112 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026830892678034104, + "loss": 1.2878, + "theoretical_loss": 3.431346674296247, + "tokens_seen": 2204827648 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026829287863590774, + "loss": 1.3415, + "theoretical_loss": 3.4313384018328144, + "tokens_seen": 2204893184 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026827683049147444, + "loss": 1.1541, + "theoretical_loss": 3.4313301296841052, + "tokens_seen": 2204958720 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026826078234704114, + "loss": 1.2497, + "theoretical_loss": 3.431321857850098, + "tokens_seen": 2205024256 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026824473420260784, + "loss": 1.1723, + "theoretical_loss": 3.4313135863307718, + "tokens_seen": 2205089792 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026822868605817454, + "loss": 1.2063, + "theoretical_loss": 3.431305315126105, + "tokens_seen": 2205155328 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026821263791374124, + "loss": 1.2542, + "theoretical_loss": 3.431297044236076, + "tokens_seen": 2205220864 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 738571, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2584826946258545, + "objective/train/theoretical_loss": 3.431288773660664, + "objective/train/tokens_used": 2225746400, + "theoretical_loss": 3.431288773660664, + "tokens_seen": 2205286400 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026819658976930794, + "loss": 1.2553, + "theoretical_loss": 3.431288773660664, + "tokens_seen": 2205286400 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026818054162487464, + "loss": 1.2945, + "theoretical_loss": 3.431280503399847, + "tokens_seen": 2205351936 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026816449348044134, + "loss": 1.298, + "theoretical_loss": 3.4312722334536043, + "tokens_seen": 2205417472 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026814844533600804, + "loss": 1.2904, + "theoretical_loss": 3.4312639638219142, + "tokens_seen": 2205483008 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026813239719157474, + "loss": 1.3272, + "theoretical_loss": 3.4312556945047556, + "tokens_seen": 2205548544 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026811634904714144, + "loss": 1.2175, + "theoretical_loss": 3.4312474255021073, + "tokens_seen": 2205614080 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026810030090270814, + "loss": 1.2232, + "theoretical_loss": 3.431239156813948, + "tokens_seen": 2205679616 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026808425275827484, + "loss": 1.251, + "theoretical_loss": 3.431230888440256, + "tokens_seen": 2205745152 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026806820461384154, + "loss": 1.2148, + "theoretical_loss": 3.43122262038101, + "tokens_seen": 2205810688 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026805215646940825, + "loss": 1.1803, + "theoretical_loss": 3.4312143526361893, + "tokens_seen": 2205876224 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026803610832497495, + "loss": 1.282, + "theoretical_loss": 3.431206085205772, + "tokens_seen": 2205941760 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026802006018054165, + "loss": 1.1219, + "theoretical_loss": 3.4311978180897373, + "tokens_seen": 2206007296 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026800401203610835, + "loss": 1.1861, + "theoretical_loss": 3.431189551288064, + "tokens_seen": 2206072832 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026798796389167505, + "loss": 1.293, + "theoretical_loss": 3.4311812848007297, + "tokens_seen": 2206138368 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026797191574724175, + "loss": 1.2752, + "theoretical_loss": 3.4311730186277143, + "tokens_seen": 2206203904 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026795586760280845, + "loss": 1.2216, + "theoretical_loss": 3.431164752768996, + "tokens_seen": 2206269440 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026793981945837515, + "loss": 1.3175, + "theoretical_loss": 3.4311564872245532, + "tokens_seen": 2206334976 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026792377131394185, + "loss": 1.3222, + "theoretical_loss": 3.4311482219943654, + "tokens_seen": 2206400512 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026790772316950855, + "loss": 1.2963, + "theoretical_loss": 3.431139957078411, + "tokens_seen": 2206466048 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026789167502507525, + "loss": 1.2396, + "theoretical_loss": 3.4311316924766686, + "tokens_seen": 2206531584 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026787562688064195, + "loss": 1.2999, + "theoretical_loss": 3.431123428189117, + "tokens_seen": 2206597120 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026785957873620865, + "loss": 1.3069, + "theoretical_loss": 3.431115164215735, + "tokens_seen": 2206662656 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026784353059177535, + "loss": 1.3743, + "theoretical_loss": 3.4311069005565007, + "tokens_seen": 2206728192 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026782748244734205, + "loss": 1.2237, + "theoretical_loss": 3.431098637211394, + "tokens_seen": 2206793728 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026781143430290876, + "loss": 1.2152, + "theoretical_loss": 3.4310903741803926, + "tokens_seen": 2206859264 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026779538615847546, + "loss": 1.1995, + "theoretical_loss": 3.4310821114634757, + "tokens_seen": 2206924800 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026777933801404216, + "loss": 1.2101, + "theoretical_loss": 3.431073849060622, + "tokens_seen": 2206990336 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026776328986960886, + "loss": 1.2514, + "theoretical_loss": 3.43106558697181, + "tokens_seen": 2207055872 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026774724172517556, + "loss": 1.2366, + "theoretical_loss": 3.4310573251970187, + "tokens_seen": 2207121408 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026773119358074226, + "loss": 1.3865, + "theoretical_loss": 3.4310490637362268, + "tokens_seen": 2207186944 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026771514543630896, + "loss": 1.1972, + "theoretical_loss": 3.431040802589413, + "tokens_seen": 2207252480 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026769909729187566, + "loss": 1.3258, + "theoretical_loss": 3.431032541756556, + "tokens_seen": 2207318016 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002676830491474423, + "loss": 1.2836, + "theoretical_loss": 3.431024281237635, + "tokens_seen": 2207383552 + }, + { + "epoch": 0.67, + "learning_rate": 0.000267667001003009, + "loss": 1.2401, + "theoretical_loss": 3.4310160210326277, + "tokens_seen": 2207449088 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002676509528585757, + "loss": 1.3105, + "theoretical_loss": 3.431007761141514, + "tokens_seen": 2207514624 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002676349047141424, + "loss": 1.2134, + "theoretical_loss": 3.4309995015642722, + "tokens_seen": 2207580160 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002676349047141424, + "loss": 1.3403, + "theoretical_loss": 3.430991242300881, + "tokens_seen": 2207645696 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002676188565697091, + "loss": 1.3762, + "theoretical_loss": 3.430982983351319, + "tokens_seen": 2207711232 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002676028084252758, + "loss": 1.316, + "theoretical_loss": 3.430974724715565, + "tokens_seen": 2207776768 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002675867602808425, + "loss": 1.2614, + "theoretical_loss": 3.430966466393598, + "tokens_seen": 2207842304 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002675707121364092, + "loss": 1.2531, + "theoretical_loss": 3.430958208385397, + "tokens_seen": 2207907840 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002675546639919759, + "loss": 1.1853, + "theoretical_loss": 3.4309499506909398, + "tokens_seen": 2207973376 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002675386158475426, + "loss": 1.3063, + "theoretical_loss": 3.4309416933102064, + "tokens_seen": 2208038912 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002675225677031093, + "loss": 1.3596, + "theoretical_loss": 3.4309334362431745, + "tokens_seen": 2208104448 + }, + { + "epoch": 0.67, + "learning_rate": 0.000267506519558676, + "loss": 1.2408, + "theoretical_loss": 3.4309251794898237, + "tokens_seen": 2208169984 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002674904714142427, + "loss": 1.2789, + "theoretical_loss": 3.430916923050132, + "tokens_seen": 2208235520 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002674744232698094, + "loss": 1.4038, + "theoretical_loss": 3.430908666924079, + "tokens_seen": 2208301056 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002674583751253761, + "loss": 1.2123, + "theoretical_loss": 3.430900411111643, + "tokens_seen": 2208366592 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002674423269809428, + "loss": 1.1715, + "theoretical_loss": 3.4308921556128027, + "tokens_seen": 2208432128 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002674262788365095, + "loss": 1.2378, + "theoretical_loss": 3.430883900427537, + "tokens_seen": 2208497664 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 739468, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2530725002288818, + "objective/train/theoretical_loss": 3.4308756455558247, + "objective/train/tokens_used": 2229023200, + "theoretical_loss": 3.4308756455558247, + "tokens_seen": 2208563200 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002674102306920762, + "loss": 1.1894, + "theoretical_loss": 3.4308756455558247, + "tokens_seen": 2208563200 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002673941825476429, + "loss": 1.2515, + "theoretical_loss": 3.430867390997645, + "tokens_seen": 2208628736 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002673781344032096, + "loss": 1.2938, + "theoretical_loss": 3.430859136752976, + "tokens_seen": 2208694272 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002673620862587763, + "loss": 1.2841, + "theoretical_loss": 3.4308508828217965, + "tokens_seen": 2208759808 + }, + { + "epoch": 0.67, + "learning_rate": 0.000267346038114343, + "loss": 1.2789, + "theoretical_loss": 3.430842629204086, + "tokens_seen": 2208825344 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002673299899699097, + "loss": 1.1935, + "theoretical_loss": 3.4308343758998223, + "tokens_seen": 2208890880 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002673139418254764, + "loss": 1.2613, + "theoretical_loss": 3.4308261229089854, + "tokens_seen": 2208956416 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002672978936810431, + "loss": 1.3262, + "theoretical_loss": 3.4308178702315533, + "tokens_seen": 2209021952 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002672818455366098, + "loss": 1.3713, + "theoretical_loss": 3.4308096178675047, + "tokens_seen": 2209087488 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002672657973921766, + "loss": 1.2207, + "theoretical_loss": 3.430801365816819, + "tokens_seen": 2209153024 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002672497492477433, + "loss": 1.3124, + "theoretical_loss": 3.4307931140794743, + "tokens_seen": 2209218560 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026723370110331, + "loss": 1.2379, + "theoretical_loss": 3.4307848626554502, + "tokens_seen": 2209284096 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002672176529588767, + "loss": 1.2952, + "theoretical_loss": 3.430776611544725, + "tokens_seen": 2209349632 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002672016048144434, + "loss": 1.2849, + "theoretical_loss": 3.4307683607472774, + "tokens_seen": 2209415168 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002671855566700101, + "loss": 1.2368, + "theoretical_loss": 3.4307601102630865, + "tokens_seen": 2209480704 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002671695085255768, + "loss": 1.2818, + "theoretical_loss": 3.4307518600921307, + "tokens_seen": 2209546240 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002671534603811435, + "loss": 1.2495, + "theoretical_loss": 3.43074361023439, + "tokens_seen": 2209611776 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002671374122367102, + "loss": 1.2528, + "theoretical_loss": 3.430735360689842, + "tokens_seen": 2209677312 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002671213640922769, + "loss": 1.2654, + "theoretical_loss": 3.430727111458465, + "tokens_seen": 2209742848 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002671053159478436, + "loss": 1.3642, + "theoretical_loss": 3.43071886254024, + "tokens_seen": 2209808384 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002670892678034103, + "loss": 1.1572, + "theoretical_loss": 3.430710613935144, + "tokens_seen": 2209873920 + }, + { + "epoch": 0.67, + "learning_rate": 0.000267073219658977, + "loss": 1.2414, + "theoretical_loss": 3.4307023656431563, + "tokens_seen": 2209939456 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002670571715145437, + "loss": 1.3203, + "theoretical_loss": 3.4306941176642556, + "tokens_seen": 2210004992 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026704112337011033, + "loss": 1.1671, + "theoretical_loss": 3.4306858699984213, + "tokens_seen": 2210070528 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026702507522567703, + "loss": 1.3222, + "theoretical_loss": 3.4306776226456317, + "tokens_seen": 2210136064 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026700902708124373, + "loss": 1.3078, + "theoretical_loss": 3.430669375605866, + "tokens_seen": 2210201600 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026699297893681043, + "loss": 1.245, + "theoretical_loss": 3.4306611288791027, + "tokens_seen": 2210267136 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026697693079237713, + "loss": 1.3091, + "theoretical_loss": 3.4306528824653206, + "tokens_seen": 2210332672 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026696088264794383, + "loss": 1.2031, + "theoretical_loss": 3.430644636364499, + "tokens_seen": 2210398208 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026694483450351054, + "loss": 1.2827, + "theoretical_loss": 3.430636390576616, + "tokens_seen": 2210463744 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026692878635907724, + "loss": 1.1262, + "theoretical_loss": 3.4306281451016516, + "tokens_seen": 2210529280 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026691273821464394, + "loss": 1.2952, + "theoretical_loss": 3.4306198999395834, + "tokens_seen": 2210594816 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026689669007021064, + "loss": 1.2693, + "theoretical_loss": 3.430611655090391, + "tokens_seen": 2210660352 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026688064192577734, + "loss": 1.2666, + "theoretical_loss": 3.4306034105540535, + "tokens_seen": 2210725888 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026686459378134404, + "loss": 1.2148, + "theoretical_loss": 3.4305951663305487, + "tokens_seen": 2210791424 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026684854563691074, + "loss": 1.3327, + "theoretical_loss": 3.4305869224198564, + "tokens_seen": 2210856960 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026683249749247744, + "loss": 1.2478, + "theoretical_loss": 3.4305786788219548, + "tokens_seen": 2210922496 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026681644934804414, + "loss": 1.2099, + "theoretical_loss": 3.4305704355368234, + "tokens_seen": 2210988032 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026680040120361084, + "loss": 1.282, + "theoretical_loss": 3.430562192564441, + "tokens_seen": 2211053568 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026678435305917754, + "loss": 1.2132, + "theoretical_loss": 3.4305539499047857, + "tokens_seen": 2211119104 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026676830491474424, + "loss": 1.1741, + "theoretical_loss": 3.4305457075578367, + "tokens_seen": 2211184640 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026675225677031094, + "loss": 1.2348, + "theoretical_loss": 3.4305374655235736, + "tokens_seen": 2211250176 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026673620862587764, + "loss": 1.1937, + "theoretical_loss": 3.4305292238019747, + "tokens_seen": 2211315712 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026672016048144434, + "loss": 1.2857, + "theoretical_loss": 3.430520982393019, + "tokens_seen": 2211381248 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026670411233701104, + "loss": 1.2855, + "theoretical_loss": 3.4305127412966847, + "tokens_seen": 2211446784 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026668806419257775, + "loss": 1.3192, + "theoretical_loss": 3.430504500512952, + "tokens_seen": 2211512320 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026667201604814445, + "loss": 1.2752, + "theoretical_loss": 3.4304962600417985, + "tokens_seen": 2211577856 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026665596790371115, + "loss": 1.3304, + "theoretical_loss": 3.430488019883204, + "tokens_seen": 2211643392 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026663991975927785, + "loss": 1.2509, + "theoretical_loss": 3.4304797800371465, + "tokens_seen": 2211708928 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026662387161484455, + "loss": 1.2734, + "theoretical_loss": 3.4304715405036057, + "tokens_seen": 2211774464 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 740733, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.34280264377594, + "objective/train/theoretical_loss": 3.43046330128256, + "objective/train/tokens_used": 2232300000, + "theoretical_loss": 3.43046330128256, + "tokens_seen": 2211840000 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026660782347041125, + "loss": 1.3035, + "theoretical_loss": 3.43046330128256, + "tokens_seen": 2211840000 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026659177532597795, + "loss": 1.2203, + "theoretical_loss": 3.4304550623739885, + "tokens_seen": 2211905536 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026657572718154465, + "loss": 1.2365, + "theoretical_loss": 3.4304468237778702, + "tokens_seen": 2211971072 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026655967903711135, + "loss": 1.3074, + "theoretical_loss": 3.4304385854941835, + "tokens_seen": 2212036608 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026654363089267805, + "loss": 1.3366, + "theoretical_loss": 3.430430347522908, + "tokens_seen": 2212102144 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026652758274824475, + "loss": 1.2185, + "theoretical_loss": 3.430422109864022, + "tokens_seen": 2212167680 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026651153460381145, + "loss": 1.2484, + "theoretical_loss": 3.430413872517505, + "tokens_seen": 2212233216 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026649548645937815, + "loss": 1.1955, + "theoretical_loss": 3.4304056354833357, + "tokens_seen": 2212298752 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026647943831494485, + "loss": 1.1223, + "theoretical_loss": 3.4303973987614924, + "tokens_seen": 2212364288 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026646339017051155, + "loss": 1.2724, + "theoretical_loss": 3.430389162351954, + "tokens_seen": 2212429824 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026644734202607825, + "loss": 1.2329, + "theoretical_loss": 3.430380926254701, + "tokens_seen": 2212495360 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026643129388164496, + "loss": 1.3075, + "theoretical_loss": 3.43037269046971, + "tokens_seen": 2212560896 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026641524573721166, + "loss": 1.2121, + "theoretical_loss": 3.430364454996962, + "tokens_seen": 2212626432 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026639919759277836, + "loss": 1.223, + "theoretical_loss": 3.4303562198364346, + "tokens_seen": 2212691968 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026638314944834506, + "loss": 1.2591, + "theoretical_loss": 3.4303479849881073, + "tokens_seen": 2212757504 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026636710130391176, + "loss": 1.2226, + "theoretical_loss": 3.4303397504519584, + "tokens_seen": 2212823040 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026635105315947846, + "loss": 1.2788, + "theoretical_loss": 3.4303315162279677, + "tokens_seen": 2212888576 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026633500501504516, + "loss": 1.3317, + "theoretical_loss": 3.4303232823161136, + "tokens_seen": 2212954112 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026631895687061186, + "loss": 1.2292, + "theoretical_loss": 3.4303150487163747, + "tokens_seen": 2213019648 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026630290872617856, + "loss": 1.301, + "theoretical_loss": 3.4303068154287306, + "tokens_seen": 2213085184 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026628686058174526, + "loss": 1.2443, + "theoretical_loss": 3.43029858245316, + "tokens_seen": 2213150720 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002662708124373119, + "loss": 1.3267, + "theoretical_loss": 3.430290349789642, + "tokens_seen": 2213216256 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002662547642928786, + "loss": 1.2838, + "theoretical_loss": 3.4302821174381553, + "tokens_seen": 2213281792 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002662387161484453, + "loss": 1.2504, + "theoretical_loss": 3.4302738853986785, + "tokens_seen": 2213347328 + }, + { + "epoch": 0.67, + "learning_rate": 0.000266222668004012, + "loss": 1.1883, + "theoretical_loss": 3.4302656536711913, + "tokens_seen": 2213412864 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002662066198595787, + "loss": 1.316, + "theoretical_loss": 3.430257422255672, + "tokens_seen": 2213478400 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002661905717151454, + "loss": 1.2652, + "theoretical_loss": 3.4302491911520994, + "tokens_seen": 2213543936 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002661745235707121, + "loss": 1.3176, + "theoretical_loss": 3.4302409603604533, + "tokens_seen": 2213609472 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002661584754262788, + "loss": 1.3476, + "theoretical_loss": 3.4302327298807116, + "tokens_seen": 2213675008 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002661424272818455, + "loss": 1.1453, + "theoretical_loss": 3.4302244997128546, + "tokens_seen": 2213740544 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002661263791374122, + "loss": 1.2118, + "theoretical_loss": 3.43021626985686, + "tokens_seen": 2213806080 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002661103309929789, + "loss": 1.2844, + "theoretical_loss": 3.4302080403127073, + "tokens_seen": 2213871616 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002660942828485456, + "loss": 1.1662, + "theoretical_loss": 3.4301998110803753, + "tokens_seen": 2213937152 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002660782347041123, + "loss": 1.2062, + "theoretical_loss": 3.4301915821598428, + "tokens_seen": 2214002688 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026606218655967907, + "loss": 1.2674, + "theoretical_loss": 3.4301833535510893, + "tokens_seen": 2214068224 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026604613841524577, + "loss": 1.4501, + "theoretical_loss": 3.4301751252540935, + "tokens_seen": 2214133760 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026603009027081247, + "loss": 1.2543, + "theoretical_loss": 3.4301668972688337, + "tokens_seen": 2214199296 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026601404212637917, + "loss": 1.2645, + "theoretical_loss": 3.43015866959529, + "tokens_seen": 2214264832 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026599799398194587, + "loss": 1.3218, + "theoretical_loss": 3.430150442233441, + "tokens_seen": 2214330368 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002659819458375126, + "loss": 1.2636, + "theoretical_loss": 3.430142215183265, + "tokens_seen": 2214395904 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002659658976930793, + "loss": 1.3182, + "theoretical_loss": 3.4301339884447417, + "tokens_seen": 2214461440 + }, + { + "epoch": 0.67, + "learning_rate": 0.000265949849548646, + "loss": 1.2001, + "theoretical_loss": 3.4301257620178496, + "tokens_seen": 2214526976 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002659338014042127, + "loss": 1.2217, + "theoretical_loss": 3.430117535902568, + "tokens_seen": 2214592512 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002659177532597794, + "loss": 1.2243, + "theoretical_loss": 3.4301093100988758, + "tokens_seen": 2214658048 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002659017051153461, + "loss": 1.1896, + "theoretical_loss": 3.430101084606752, + "tokens_seen": 2214723584 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002658856569709128, + "loss": 1.2669, + "theoretical_loss": 3.4300928594261757, + "tokens_seen": 2214789120 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002658696088264795, + "loss": 1.2647, + "theoretical_loss": 3.4300846345571254, + "tokens_seen": 2214854656 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002658535606820462, + "loss": 1.3232, + "theoretical_loss": 3.4300764099995806, + "tokens_seen": 2214920192 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002658375125376129, + "loss": 1.2699, + "theoretical_loss": 3.4300681857535196, + "tokens_seen": 2214985728 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002658214643931796, + "loss": 1.283, + "theoretical_loss": 3.4300599618189223, + "tokens_seen": 2215051264 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 741599, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2668308019638062, + "objective/train/theoretical_loss": 3.4300517381957674, + "objective/train/tokens_used": 2235576800, + "theoretical_loss": 3.4300517381957674, + "tokens_seen": 2215116800 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002658054162487463, + "loss": 1.246, + "theoretical_loss": 3.4300517381957674, + "tokens_seen": 2215116800 + }, + { + "epoch": 0.67, + "learning_rate": 0.000265789368104313, + "loss": 1.2519, + "theoretical_loss": 3.4300435148840336, + "tokens_seen": 2215182336 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002657733199598797, + "loss": 1.2321, + "theoretical_loss": 3.4300352918837, + "tokens_seen": 2215247872 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002657572718154464, + "loss": 1.3017, + "theoretical_loss": 3.4300270691947454, + "tokens_seen": 2215313408 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002657412236710131, + "loss": 1.1633, + "theoretical_loss": 3.4300188468171497, + "tokens_seen": 2215378944 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002657251755265798, + "loss": 1.2899, + "theoretical_loss": 3.430010624750891, + "tokens_seen": 2215444480 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002657091273821465, + "loss": 1.1492, + "theoretical_loss": 3.430002402995948, + "tokens_seen": 2215510016 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002656930792377132, + "loss": 1.2309, + "theoretical_loss": 3.429994181552301, + "tokens_seen": 2215575552 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002656770310932799, + "loss": 1.3256, + "theoretical_loss": 3.429985960419928, + "tokens_seen": 2215641088 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002656609829488466, + "loss": 1.252, + "theoretical_loss": 3.429977739598808, + "tokens_seen": 2215706624 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026564493480441323, + "loss": 1.2215, + "theoretical_loss": 3.42996951908892, + "tokens_seen": 2215772160 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026562888665997993, + "loss": 1.2155, + "theoretical_loss": 3.4299612988902437, + "tokens_seen": 2215837696 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026561283851554663, + "loss": 1.2592, + "theoretical_loss": 3.429953079002758, + "tokens_seen": 2215903232 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026559679037111333, + "loss": 1.3305, + "theoretical_loss": 3.429944859426441, + "tokens_seen": 2215968768 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026558074222668003, + "loss": 1.2528, + "theoretical_loss": 3.429936640161273, + "tokens_seen": 2216034304 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026556469408224674, + "loss": 1.1434, + "theoretical_loss": 3.4299284212072316, + "tokens_seen": 2216099840 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026554864593781344, + "loss": 1.2164, + "theoretical_loss": 3.429920202564297, + "tokens_seen": 2216165376 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026553259779338014, + "loss": 1.2262, + "theoretical_loss": 3.429911984232448, + "tokens_seen": 2216230912 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026551654964894684, + "loss": 1.2188, + "theoretical_loss": 3.429903766211663, + "tokens_seen": 2216296448 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026550050150451354, + "loss": 1.2551, + "theoretical_loss": 3.429895548501922, + "tokens_seen": 2216361984 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026548445336008024, + "loss": 1.2345, + "theoretical_loss": 3.429887331103203, + "tokens_seen": 2216427520 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026546840521564694, + "loss": 1.3294, + "theoretical_loss": 3.4298791140154856, + "tokens_seen": 2216493056 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026545235707121364, + "loss": 1.2265, + "theoretical_loss": 3.429870897238749, + "tokens_seen": 2216558592 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026543630892678034, + "loss": 1.3159, + "theoretical_loss": 3.4298626807729713, + "tokens_seen": 2216624128 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026542026078234704, + "loss": 1.227, + "theoretical_loss": 3.429854464618133, + "tokens_seen": 2216689664 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026540421263791374, + "loss": 1.2221, + "theoretical_loss": 3.4298462487742123, + "tokens_seen": 2216755200 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026538816449348044, + "loss": 1.2271, + "theoretical_loss": 3.4298380332411877, + "tokens_seen": 2216820736 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026537211634904714, + "loss": 1.2427, + "theoretical_loss": 3.4298298180190394, + "tokens_seen": 2216886272 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026535606820461384, + "loss": 1.2649, + "theoretical_loss": 3.4298216031077455, + "tokens_seen": 2216951808 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026534002006018054, + "loss": 1.1281, + "theoretical_loss": 3.429813388507286, + "tokens_seen": 2217017344 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026532397191574724, + "loss": 1.2233, + "theoretical_loss": 3.429805174217639, + "tokens_seen": 2217082880 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026530792377131395, + "loss": 1.2497, + "theoretical_loss": 3.429796960238784, + "tokens_seen": 2217148416 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026529187562688065, + "loss": 1.2981, + "theoretical_loss": 3.4297887465707, + "tokens_seen": 2217213952 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026527582748244735, + "loss": 1.1977, + "theoretical_loss": 3.4297805332133664, + "tokens_seen": 2217279488 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026525977933801405, + "loss": 1.2931, + "theoretical_loss": 3.4297723201667614, + "tokens_seen": 2217345024 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026524373119358075, + "loss": 1.2103, + "theoretical_loss": 3.429764107430865, + "tokens_seen": 2217410560 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026522768304914745, + "loss": 1.2586, + "theoretical_loss": 3.4297558950056555, + "tokens_seen": 2217476096 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026521163490471415, + "loss": 1.2126, + "theoretical_loss": 3.429747682891113, + "tokens_seen": 2217541632 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026519558676028085, + "loss": 1.2299, + "theoretical_loss": 3.429739471087215, + "tokens_seen": 2217607168 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026517953861584755, + "loss": 1.1767, + "theoretical_loss": 3.429731259593942, + "tokens_seen": 2217672704 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026516349047141425, + "loss": 1.1833, + "theoretical_loss": 3.4297230484112724, + "tokens_seen": 2217738240 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026514744232698095, + "loss": 1.3009, + "theoretical_loss": 3.4297148375391853, + "tokens_seen": 2217803776 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026513139418254765, + "loss": 1.3177, + "theoretical_loss": 3.4297066269776604, + "tokens_seen": 2217869312 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026511534603811435, + "loss": 1.2841, + "theoretical_loss": 3.4296984167266755, + "tokens_seen": 2217934848 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026509929789368105, + "loss": 1.237, + "theoretical_loss": 3.4296902067862107, + "tokens_seen": 2218000384 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026508324974924775, + "loss": 1.1843, + "theoretical_loss": 3.4296819971562447, + "tokens_seen": 2218065920 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026506720160481446, + "loss": 1.2867, + "theoretical_loss": 3.429673787836757, + "tokens_seen": 2218131456 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026505115346038116, + "loss": 1.249, + "theoretical_loss": 3.429665578827726, + "tokens_seen": 2218196992 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026503510531594786, + "loss": 1.2335, + "theoretical_loss": 3.4296573701291315, + "tokens_seen": 2218262528 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026501905717151456, + "loss": 1.2415, + "theoretical_loss": 3.429649161740952, + "tokens_seen": 2218328064 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 742689, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3651707172393799, + "objective/train/theoretical_loss": 3.429640953663167, + "objective/train/tokens_used": 2238853600, + "theoretical_loss": 3.429640953663167, + "tokens_seen": 2218393600 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026500300902708126, + "loss": 1.3749, + "theoretical_loss": 3.429640953663167, + "tokens_seen": 2218393600 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026498696088264796, + "loss": 1.1721, + "theoretical_loss": 3.4296327458957556, + "tokens_seen": 2218459136 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026497091273821466, + "loss": 1.2124, + "theoretical_loss": 3.4296245384386963, + "tokens_seen": 2218524672 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026495486459378136, + "loss": 1.1852, + "theoretical_loss": 3.429616331291969, + "tokens_seen": 2218590208 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026493881644934806, + "loss": 1.247, + "theoretical_loss": 3.4296081244555525, + "tokens_seen": 2218655744 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026492276830491476, + "loss": 1.28, + "theoretical_loss": 3.4295999179294254, + "tokens_seen": 2218721280 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026490672016048146, + "loss": 1.2093, + "theoretical_loss": 3.4295917117135675, + "tokens_seen": 2218786816 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026489067201604816, + "loss": 1.2738, + "theoretical_loss": 3.429583505807958, + "tokens_seen": 2218852352 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002648746238716148, + "loss": 1.3016, + "theoretical_loss": 3.4295753002125746, + "tokens_seen": 2218917888 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026485857572718156, + "loss": 1.2112, + "theoretical_loss": 3.4295670949273984, + "tokens_seen": 2218983424 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026484252758274826, + "loss": 1.2438, + "theoretical_loss": 3.4295588899524074, + "tokens_seen": 2219048960 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026482647943831496, + "loss": 1.2925, + "theoretical_loss": 3.429550685287581, + "tokens_seen": 2219114496 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026481043129388167, + "loss": 1.2579, + "theoretical_loss": 3.4295424809328976, + "tokens_seen": 2219180032 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026479438314944837, + "loss": 1.1911, + "theoretical_loss": 3.4295342768883375, + "tokens_seen": 2219245568 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026477833500501507, + "loss": 1.202, + "theoretical_loss": 3.429526073153879, + "tokens_seen": 2219311104 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026476228686058177, + "loss": 1.2689, + "theoretical_loss": 3.4295178697295015, + "tokens_seen": 2219376640 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026474623871614847, + "loss": 1.2285, + "theoretical_loss": 3.429509666615184, + "tokens_seen": 2219442176 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026473019057171517, + "loss": 1.292, + "theoretical_loss": 3.429501463810906, + "tokens_seen": 2219507712 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026471414242728187, + "loss": 1.2811, + "theoretical_loss": 3.429493261316646, + "tokens_seen": 2219573248 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026469809428284857, + "loss": 1.3281, + "theoretical_loss": 3.4294850591323836, + "tokens_seen": 2219638784 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026468204613841527, + "loss": 1.2245, + "theoretical_loss": 3.4294768572580976, + "tokens_seen": 2219704320 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026466599799398197, + "loss": 1.2102, + "theoretical_loss": 3.4294686556937677, + "tokens_seen": 2219769856 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026464994984954867, + "loss": 1.2666, + "theoretical_loss": 3.4294604544393725, + "tokens_seen": 2219835392 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026463390170511537, + "loss": 1.2258, + "theoretical_loss": 3.4294522534948912, + "tokens_seen": 2219900928 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002646178535606821, + "loss": 1.2509, + "theoretical_loss": 3.4294440528603034, + "tokens_seen": 2219966464 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002646018054162488, + "loss": 1.2552, + "theoretical_loss": 3.4294358525355877, + "tokens_seen": 2220032000 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002645857572718155, + "loss": 1.1515, + "theoretical_loss": 3.4294276525207232, + "tokens_seen": 2220097536 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002645697091273822, + "loss": 1.3023, + "theoretical_loss": 3.4294194528156896, + "tokens_seen": 2220163072 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002645536609829489, + "loss": 1.233, + "theoretical_loss": 3.4294112534204655, + "tokens_seen": 2220228608 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002645376128385156, + "loss": 1.1878, + "theoretical_loss": 3.4294030543350305, + "tokens_seen": 2220294144 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002645215646940823, + "loss": 1.1959, + "theoretical_loss": 3.429394855559363, + "tokens_seen": 2220359680 + }, + { + "epoch": 0.67, + "learning_rate": 0.000264505516549649, + "loss": 1.2426, + "theoretical_loss": 3.4293866570934437, + "tokens_seen": 2220425216 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002644894684052157, + "loss": 1.2454, + "theoretical_loss": 3.42937845893725, + "tokens_seen": 2220490752 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002644734202607824, + "loss": 1.3891, + "theoretical_loss": 3.429370261090762, + "tokens_seen": 2220556288 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002644573721163491, + "loss": 1.2777, + "theoretical_loss": 3.4293620635539583, + "tokens_seen": 2220621824 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002644413239719158, + "loss": 1.2472, + "theoretical_loss": 3.429353866326819, + "tokens_seen": 2220687360 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002644252758274825, + "loss": 1.3108, + "theoretical_loss": 3.4293456694093223, + "tokens_seen": 2220752896 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002644092276830492, + "loss": 1.2307, + "theoretical_loss": 3.429337472801448, + "tokens_seen": 2220818432 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002643931795386159, + "loss": 1.1358, + "theoretical_loss": 3.4293292765031747, + "tokens_seen": 2220883968 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002643771313941826, + "loss": 1.1489, + "theoretical_loss": 3.429321080514482, + "tokens_seen": 2220949504 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002643610832497493, + "loss": 1.2172, + "theoretical_loss": 3.4293128848353494, + "tokens_seen": 2221015040 + }, + { + "epoch": 0.67, + "learning_rate": 0.000264345035105316, + "loss": 1.174, + "theoretical_loss": 3.429304689465755, + "tokens_seen": 2221080576 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002643289869608827, + "loss": 1.292, + "theoretical_loss": 3.429296494405679, + "tokens_seen": 2221146112 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002643129388164494, + "loss": 1.2696, + "theoretical_loss": 3.4292882996551, + "tokens_seen": 2221211648 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002642968906720161, + "loss": 1.2637, + "theoretical_loss": 3.4292801052139974, + "tokens_seen": 2221277184 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002642808425275828, + "loss": 1.3076, + "theoretical_loss": 3.4292719110823504, + "tokens_seen": 2221342720 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002642647943831495, + "loss": 1.304, + "theoretical_loss": 3.429263717260138, + "tokens_seen": 2221408256 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002642487462387162, + "loss": 1.2701, + "theoretical_loss": 3.4292555237473397, + "tokens_seen": 2221473792 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026423269809428283, + "loss": 1.2394, + "theoretical_loss": 3.4292473305439346, + "tokens_seen": 2221539328 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026421664994984953, + "loss": 1.2298, + "theoretical_loss": 3.4292391376499016, + "tokens_seen": 2221604864 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 743980, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3372083902359009, + "objective/train/theoretical_loss": 3.4292309450652203, + "objective/train/tokens_used": 2242130400, + "theoretical_loss": 3.4292309450652203, + "tokens_seen": 2221670400 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026420060180541624, + "loss": 1.3112, + "theoretical_loss": 3.4292309450652203, + "tokens_seen": 2221670400 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026418455366098294, + "loss": 1.1827, + "theoretical_loss": 3.4292227527898693, + "tokens_seen": 2221735936 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026416850551654964, + "loss": 1.2013, + "theoretical_loss": 3.4292145608238287, + "tokens_seen": 2221801472 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026415245737211634, + "loss": 1.3176, + "theoretical_loss": 3.429206369167077, + "tokens_seen": 2221867008 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026413640922768304, + "loss": 1.2815, + "theoretical_loss": 3.4291981778195932, + "tokens_seen": 2221932544 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026412036108324974, + "loss": 1.2655, + "theoretical_loss": 3.429189986781357, + "tokens_seen": 2221998080 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026410431293881644, + "loss": 1.2826, + "theoretical_loss": 3.429181796052348, + "tokens_seen": 2222063616 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026408826479438314, + "loss": 1.346, + "theoretical_loss": 3.4291736056325446, + "tokens_seen": 2222129152 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026407221664994984, + "loss": 1.2676, + "theoretical_loss": 3.4291654155219264, + "tokens_seen": 2222194688 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026405616850551654, + "loss": 1.2431, + "theoretical_loss": 3.4291572257204725, + "tokens_seen": 2222260224 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026404012036108324, + "loss": 1.2655, + "theoretical_loss": 3.429149036228162, + "tokens_seen": 2222325760 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026402407221664994, + "loss": 1.252, + "theoretical_loss": 3.4291408470449745, + "tokens_seen": 2222391296 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026400802407221664, + "loss": 1.2551, + "theoretical_loss": 3.4291326581708885, + "tokens_seen": 2222456832 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026399197592778334, + "loss": 1.1958, + "theoretical_loss": 3.429124469605884, + "tokens_seen": 2222522368 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026397592778335004, + "loss": 1.2822, + "theoretical_loss": 3.42911628134994, + "tokens_seen": 2222587904 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026395987963891674, + "loss": 1.2139, + "theoretical_loss": 3.4291080934030354, + "tokens_seen": 2222653440 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026394383149448345, + "loss": 1.3089, + "theoretical_loss": 3.42909990576515, + "tokens_seen": 2222718976 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026392778335005015, + "loss": 1.2895, + "theoretical_loss": 3.429091718436262, + "tokens_seen": 2222784512 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026391173520561685, + "loss": 1.2373, + "theoretical_loss": 3.4290835314163517, + "tokens_seen": 2222850048 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026389568706118355, + "loss": 1.2129, + "theoretical_loss": 3.429075344705398, + "tokens_seen": 2222915584 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026387963891675025, + "loss": 1.2081, + "theoretical_loss": 3.42906715830338, + "tokens_seen": 2222981120 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026386359077231695, + "loss": 1.1812, + "theoretical_loss": 3.429058972210277, + "tokens_seen": 2223046656 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026384754262788365, + "loss": 1.2683, + "theoretical_loss": 3.4290507864260684, + "tokens_seen": 2223112192 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026383149448345035, + "loss": 1.2361, + "theoretical_loss": 3.429042600950733, + "tokens_seen": 2223177728 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026381544633901705, + "loss": 1.2501, + "theoretical_loss": 3.4290344157842503, + "tokens_seen": 2223243264 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026379939819458375, + "loss": 1.3059, + "theoretical_loss": 3.4290262309266, + "tokens_seen": 2223308800 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026378335005015045, + "loss": 1.2287, + "theoretical_loss": 3.4290180463777604, + "tokens_seen": 2223374336 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026376730190571715, + "loss": 1.1958, + "theoretical_loss": 3.4290098621377116, + "tokens_seen": 2223439872 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026375125376128385, + "loss": 1.2548, + "theoretical_loss": 3.4290016782064323, + "tokens_seen": 2223505408 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026373520561685055, + "loss": 1.253, + "theoretical_loss": 3.428993494583902, + "tokens_seen": 2223570944 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026371915747241725, + "loss": 1.1872, + "theoretical_loss": 3.4289853112701, + "tokens_seen": 2223636480 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026370310932798395, + "loss": 1.2781, + "theoretical_loss": 3.4289771282650054, + "tokens_seen": 2223702016 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026368706118355066, + "loss": 1.2777, + "theoretical_loss": 3.4289689455685974, + "tokens_seen": 2223767552 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026367101303911736, + "loss": 1.2456, + "theoretical_loss": 3.4289607631808554, + "tokens_seen": 2223833088 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002636549648946841, + "loss": 1.266, + "theoretical_loss": 3.428952581101759, + "tokens_seen": 2223898624 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002636389167502508, + "loss": 1.3292, + "theoretical_loss": 3.4289443993312867, + "tokens_seen": 2223964160 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002636228686058175, + "loss": 1.1492, + "theoretical_loss": 3.4289362178694183, + "tokens_seen": 2224029696 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026360682046138416, + "loss": 1.1917, + "theoretical_loss": 3.4289280367161328, + "tokens_seen": 2224095232 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026359077231695086, + "loss": 1.3213, + "theoretical_loss": 3.42891985587141, + "tokens_seen": 2224160768 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026357472417251756, + "loss": 1.1221, + "theoretical_loss": 3.428911675335228, + "tokens_seen": 2224226304 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026355867602808426, + "loss": 1.1716, + "theoretical_loss": 3.4289034951075674, + "tokens_seen": 2224291840 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026354262788365096, + "loss": 1.1981, + "theoretical_loss": 3.428895315188407, + "tokens_seen": 2224357376 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026352657973921766, + "loss": 1.2274, + "theoretical_loss": 3.428887135577726, + "tokens_seen": 2224422912 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026351053159478436, + "loss": 1.291, + "theoretical_loss": 3.4288789562755033, + "tokens_seen": 2224488448 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026349448345035106, + "loss": 1.3646, + "theoretical_loss": 3.428870777281719, + "tokens_seen": 2224553984 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026347843530591776, + "loss": 1.2991, + "theoretical_loss": 3.4288625985963517, + "tokens_seen": 2224619520 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026346238716148446, + "loss": 1.2858, + "theoretical_loss": 3.4288544202193805, + "tokens_seen": 2224685056 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026344633901705117, + "loss": 1.294, + "theoretical_loss": 3.4288462421507857, + "tokens_seen": 2224750592 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026343029087261787, + "loss": 1.2628, + "theoretical_loss": 3.428838064390546, + "tokens_seen": 2224816128 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026341424272818457, + "loss": 1.3204, + "theoretical_loss": 3.42882988693864, + "tokens_seen": 2224881664 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 744927, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2962076663970947, + "objective/train/theoretical_loss": 3.4288217097950486, + "objective/train/tokens_used": 2245407200, + "theoretical_loss": 3.4288217097950486, + "tokens_seen": 2224947200 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026339819458375127, + "loss": 1.2519, + "theoretical_loss": 3.4288217097950486, + "tokens_seen": 2224947200 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026338214643931797, + "loss": 1.2846, + "theoretical_loss": 3.4288135329597496, + "tokens_seen": 2225012736 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026336609829488467, + "loss": 1.2197, + "theoretical_loss": 3.428805356432723, + "tokens_seen": 2225078272 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026335005015045137, + "loss": 1.3023, + "theoretical_loss": 3.428797180213948, + "tokens_seen": 2225143808 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026333400200601807, + "loss": 1.2095, + "theoretical_loss": 3.428789004303404, + "tokens_seen": 2225209344 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026331795386158477, + "loss": 1.1658, + "theoretical_loss": 3.42878082870107, + "tokens_seen": 2225274880 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026330190571715147, + "loss": 1.2342, + "theoretical_loss": 3.4287726534069254, + "tokens_seen": 2225340416 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026328585757271817, + "loss": 1.2725, + "theoretical_loss": 3.4287644784209497, + "tokens_seen": 2225405952 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026326980942828487, + "loss": 1.208, + "theoretical_loss": 3.428756303743122, + "tokens_seen": 2225471488 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026325376128385157, + "loss": 1.2908, + "theoretical_loss": 3.428748129373422, + "tokens_seen": 2225537024 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002632377131394183, + "loss": 1.2185, + "theoretical_loss": 3.4287399553118285, + "tokens_seen": 2225602560 + }, + { + "epoch": 0.67, + "learning_rate": 0.000263221664994985, + "loss": 1.2851, + "theoretical_loss": 3.428731781558321, + "tokens_seen": 2225668096 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002632056168505517, + "loss": 1.2984, + "theoretical_loss": 3.428723608112879, + "tokens_seen": 2225733632 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002631895687061184, + "loss": 1.2627, + "theoretical_loss": 3.428715434975482, + "tokens_seen": 2225799168 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002631735205616851, + "loss": 1.2726, + "theoretical_loss": 3.428707262146108, + "tokens_seen": 2225864704 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002631574724172518, + "loss": 1.2096, + "theoretical_loss": 3.428699089624738, + "tokens_seen": 2225930240 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002631414242728185, + "loss": 1.174, + "theoretical_loss": 3.4286909174113505, + "tokens_seen": 2225995776 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002631253761283852, + "loss": 1.2857, + "theoretical_loss": 3.428682745505925, + "tokens_seen": 2226061312 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002631093279839519, + "loss": 1.3091, + "theoretical_loss": 3.4286745739084408, + "tokens_seen": 2226126848 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002630932798395186, + "loss": 1.1975, + "theoretical_loss": 3.428666402618877, + "tokens_seen": 2226192384 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002630772316950853, + "loss": 1.2904, + "theoretical_loss": 3.4286582316372134, + "tokens_seen": 2226257920 + }, + { + "epoch": 0.67, + "learning_rate": 0.000263061183550652, + "loss": 1.2741, + "theoretical_loss": 3.428650060963429, + "tokens_seen": 2226323456 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002630451354062187, + "loss": 1.2023, + "theoretical_loss": 3.428641890597503, + "tokens_seen": 2226388992 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002630290872617854, + "loss": 1.2791, + "theoretical_loss": 3.4286337205394153, + "tokens_seen": 2226454528 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002630130391173521, + "loss": 1.2535, + "theoretical_loss": 3.428625550789145, + "tokens_seen": 2226520064 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002629969909729188, + "loss": 1.2165, + "theoretical_loss": 3.428617381346671, + "tokens_seen": 2226585600 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002629809428284855, + "loss": 1.2885, + "theoretical_loss": 3.4286092122119736, + "tokens_seen": 2226651136 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002629648946840522, + "loss": 1.3736, + "theoretical_loss": 3.428601043385031, + "tokens_seen": 2226716672 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002629488465396189, + "loss": 1.3208, + "theoretical_loss": 3.4285928748658234, + "tokens_seen": 2226782208 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002629327983951856, + "loss": 1.2053, + "theoretical_loss": 3.4285847066543296, + "tokens_seen": 2226847744 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002629167502507523, + "loss": 1.2479, + "theoretical_loss": 3.428576538750529, + "tokens_seen": 2226913280 + }, + { + "epoch": 0.67, + "learning_rate": 0.000262900702106319, + "loss": 1.2131, + "theoretical_loss": 3.4285683711544013, + "tokens_seen": 2226978816 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002628846539618857, + "loss": 1.1953, + "theoretical_loss": 3.428560203865926, + "tokens_seen": 2227044352 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002628686058174524, + "loss": 1.1443, + "theoretical_loss": 3.428552036885082, + "tokens_seen": 2227109888 + }, + { + "epoch": 0.67, + "learning_rate": 0.0002628525576730191, + "loss": 1.3213, + "theoretical_loss": 3.4285438702118487, + "tokens_seen": 2227175424 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026283650952858573, + "loss": 1.2421, + "theoretical_loss": 3.4285357038462054, + "tokens_seen": 2227240960 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026282046138415244, + "loss": 1.2776, + "theoretical_loss": 3.428527537788132, + "tokens_seen": 2227306496 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026280441323971914, + "loss": 1.2285, + "theoretical_loss": 3.4285193720376075, + "tokens_seen": 2227372032 + }, + { + "epoch": 0.67, + "learning_rate": 0.00026278836509528584, + "loss": 1.2301, + "theoretical_loss": 3.428511206594611, + "tokens_seen": 2227437568 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026277231695085254, + "loss": 1.2464, + "theoretical_loss": 3.4285030414591224, + "tokens_seen": 2227503104 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026275626880641924, + "loss": 1.2532, + "theoretical_loss": 3.428494876631121, + "tokens_seen": 2227568640 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026274022066198594, + "loss": 1.1369, + "theoretical_loss": 3.428486712110586, + "tokens_seen": 2227634176 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026272417251755264, + "loss": 1.2456, + "theoretical_loss": 3.4284785478974964, + "tokens_seen": 2227699712 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026270812437311934, + "loss": 1.3122, + "theoretical_loss": 3.428470383991832, + "tokens_seen": 2227765248 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026269207622868604, + "loss": 1.284, + "theoretical_loss": 3.4284622203935724, + "tokens_seen": 2227830784 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026267602808425274, + "loss": 1.2617, + "theoretical_loss": 3.4284540571026967, + "tokens_seen": 2227896320 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026265997993981944, + "loss": 1.2117, + "theoretical_loss": 3.428445894119184, + "tokens_seen": 2227961856 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026264393179538614, + "loss": 1.1856, + "theoretical_loss": 3.4284377314430143, + "tokens_seen": 2228027392 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026262788365095284, + "loss": 1.2032, + "theoretical_loss": 3.428429569074167, + "tokens_seen": 2228092928 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026261183550651954, + "loss": 1.2223, + "theoretical_loss": 3.42842140701262, + "tokens_seen": 2228158464 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 746243, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2754054069519043, + "objective/train/theoretical_loss": 3.428413245258355, + "objective/train/tokens_used": 2248684000, + "theoretical_loss": 3.428413245258355, + "tokens_seen": 2228224000 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026259578736208624, + "loss": 1.2565, + "theoretical_loss": 3.428413245258355, + "tokens_seen": 2228224000 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026257973921765295, + "loss": 1.2181, + "theoretical_loss": 3.42840508381135, + "tokens_seen": 2228289536 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026256369107321965, + "loss": 1.2252, + "theoretical_loss": 3.4283969226715842, + "tokens_seen": 2228355072 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026254764292878635, + "loss": 1.2196, + "theoretical_loss": 3.428388761839038, + "tokens_seen": 2228420608 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026253159478435305, + "loss": 1.1996, + "theoretical_loss": 3.42838060131369, + "tokens_seen": 2228486144 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026251554663991975, + "loss": 1.2926, + "theoretical_loss": 3.42837244109552, + "tokens_seen": 2228551680 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026249949849548645, + "loss": 1.1935, + "theoretical_loss": 3.428364281184507, + "tokens_seen": 2228617216 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026248345035105315, + "loss": 1.2346, + "theoretical_loss": 3.428356121580631, + "tokens_seen": 2228682752 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026246740220661985, + "loss": 1.1546, + "theoretical_loss": 3.428347962283871, + "tokens_seen": 2228748288 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002624513540621866, + "loss": 1.2163, + "theoretical_loss": 3.4283398032942065, + "tokens_seen": 2228813824 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002624353059177533, + "loss": 1.3449, + "theoretical_loss": 3.4283316446116165, + "tokens_seen": 2228879360 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026241925777332, + "loss": 1.1917, + "theoretical_loss": 3.4283234862360814, + "tokens_seen": 2228944896 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002624032096288867, + "loss": 1.132, + "theoretical_loss": 3.42831532816758, + "tokens_seen": 2229010432 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002623871614844534, + "loss": 1.1881, + "theoretical_loss": 3.4283071704060912, + "tokens_seen": 2229075968 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002623711133400201, + "loss": 1.2752, + "theoretical_loss": 3.428299012951595, + "tokens_seen": 2229141504 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002623550651955868, + "loss": 1.3274, + "theoretical_loss": 3.4282908558040712, + "tokens_seen": 2229207040 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002623390170511535, + "loss": 1.2011, + "theoretical_loss": 3.4282826989634985, + "tokens_seen": 2229272576 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002623229689067202, + "loss": 1.2383, + "theoretical_loss": 3.4282745424298566, + "tokens_seen": 2229338112 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002623069207622869, + "loss": 1.2052, + "theoretical_loss": 3.4282663862031253, + "tokens_seen": 2229403648 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002622908726178536, + "loss": 1.264, + "theoretical_loss": 3.4282582302832836, + "tokens_seen": 2229469184 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002622748244734203, + "loss": 1.2322, + "theoretical_loss": 3.428250074670311, + "tokens_seen": 2229534720 + }, + { + "epoch": 0.68, + "learning_rate": 0.000262258776328987, + "loss": 1.2603, + "theoretical_loss": 3.4282419193641873, + "tokens_seen": 2229600256 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002622427281845537, + "loss": 1.1535, + "theoretical_loss": 3.428233764364891, + "tokens_seen": 2229665792 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002622266800401204, + "loss": 1.2632, + "theoretical_loss": 3.4282256096724026, + "tokens_seen": 2229731328 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026221063189568706, + "loss": 1.1906, + "theoretical_loss": 3.428217455286701, + "tokens_seen": 2229796864 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026219458375125376, + "loss": 1.2608, + "theoretical_loss": 3.4282093012077652, + "tokens_seen": 2229862400 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026217853560682046, + "loss": 1.1806, + "theoretical_loss": 3.428201147435576, + "tokens_seen": 2229927936 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026216248746238716, + "loss": 1.2624, + "theoretical_loss": 3.4281929939701112, + "tokens_seen": 2229993472 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026214643931795386, + "loss": 1.2116, + "theoretical_loss": 3.4281848408113516, + "tokens_seen": 2230059008 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026213039117352056, + "loss": 1.2791, + "theoretical_loss": 3.428176687959276, + "tokens_seen": 2230124544 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026211434302908726, + "loss": 1.1573, + "theoretical_loss": 3.428168535413864, + "tokens_seen": 2230190080 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026209829488465396, + "loss": 1.2152, + "theoretical_loss": 3.428160383175095, + "tokens_seen": 2230255616 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026208224674022066, + "loss": 1.2047, + "theoretical_loss": 3.428152231242948, + "tokens_seen": 2230321152 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026206619859578737, + "loss": 1.3543, + "theoretical_loss": 3.428144079617404, + "tokens_seen": 2230386688 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026205015045135407, + "loss": 1.2442, + "theoretical_loss": 3.4281359282984405, + "tokens_seen": 2230452224 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026203410230692077, + "loss": 1.2624, + "theoretical_loss": 3.428127777286038, + "tokens_seen": 2230517760 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026201805416248747, + "loss": 1.3008, + "theoretical_loss": 3.428119626580176, + "tokens_seen": 2230583296 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026200200601805417, + "loss": 1.2781, + "theoretical_loss": 3.428111476180834, + "tokens_seen": 2230648832 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026198595787362087, + "loss": 1.33, + "theoretical_loss": 3.4281033260879905, + "tokens_seen": 2230714368 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026196990972918757, + "loss": 1.3113, + "theoretical_loss": 3.428095176301626, + "tokens_seen": 2230779904 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026195386158475427, + "loss": 1.3817, + "theoretical_loss": 3.42808702682172, + "tokens_seen": 2230845440 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026193781344032097, + "loss": 1.271, + "theoretical_loss": 3.4280788776482516, + "tokens_seen": 2230910976 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026192176529588767, + "loss": 1.2731, + "theoretical_loss": 3.4280707287812002, + "tokens_seen": 2230976512 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026190571715145437, + "loss": 1.304, + "theoretical_loss": 3.4280625802205456, + "tokens_seen": 2231042048 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026188966900702107, + "loss": 1.2808, + "theoretical_loss": 3.4280544319662667, + "tokens_seen": 2231107584 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002618736208625878, + "loss": 1.2304, + "theoretical_loss": 3.428046284018344, + "tokens_seen": 2231173120 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002618575727181545, + "loss": 1.2817, + "theoretical_loss": 3.428038136376756, + "tokens_seen": 2231238656 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002618415245737212, + "loss": 1.2842, + "theoretical_loss": 3.4280299890414825, + "tokens_seen": 2231304192 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002618254764292879, + "loss": 1.29, + "theoretical_loss": 3.428021842012503, + "tokens_seen": 2231369728 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002618094282848546, + "loss": 1.1693, + "theoretical_loss": 3.4280136952897973, + "tokens_seen": 2231435264 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 747198, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.172689437866211, + "objective/train/theoretical_loss": 3.4280055488733447, + "objective/train/tokens_used": 2251960800, + "theoretical_loss": 3.4280055488733447, + "tokens_seen": 2231500800 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002617933801404213, + "loss": 1.1688, + "theoretical_loss": 3.4280055488733447, + "tokens_seen": 2231500800 + }, + { + "epoch": 0.68, + "learning_rate": 0.000261777331995988, + "loss": 1.2212, + "theoretical_loss": 3.427997402763124, + "tokens_seen": 2231566336 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002617612838515547, + "loss": 1.2918, + "theoretical_loss": 3.427989256959116, + "tokens_seen": 2231631872 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002617452357071214, + "loss": 1.2737, + "theoretical_loss": 3.427981111461299, + "tokens_seen": 2231697408 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002617291875626881, + "loss": 1.2546, + "theoretical_loss": 3.427972966269653, + "tokens_seen": 2231762944 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002617131394182548, + "loss": 1.2056, + "theoretical_loss": 3.4279648213841583, + "tokens_seen": 2231828480 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002616970912738215, + "loss": 1.2933, + "theoretical_loss": 3.4279566768047927, + "tokens_seen": 2231894016 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002616810431293882, + "loss": 1.2207, + "theoretical_loss": 3.427948532531537, + "tokens_seen": 2231959552 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002616649949849549, + "loss": 1.1649, + "theoretical_loss": 3.4279403885643704, + "tokens_seen": 2232025088 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002616489468405216, + "loss": 1.1676, + "theoretical_loss": 3.427932244903272, + "tokens_seen": 2232090624 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002616328986960883, + "loss": 1.2509, + "theoretical_loss": 3.427924101548222, + "tokens_seen": 2232156160 + }, + { + "epoch": 0.68, + "learning_rate": 0.000261616850551655, + "loss": 1.1529, + "theoretical_loss": 3.427915958499199, + "tokens_seen": 2232221696 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002616008024072217, + "loss": 1.3053, + "theoretical_loss": 3.4279078157561838, + "tokens_seen": 2232287232 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002615847542627884, + "loss": 1.133, + "theoretical_loss": 3.4278996733191547, + "tokens_seen": 2232352768 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002615687061183551, + "loss": 1.164, + "theoretical_loss": 3.427891531188092, + "tokens_seen": 2232418304 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002615526579739218, + "loss": 1.2439, + "theoretical_loss": 3.4278833893629748, + "tokens_seen": 2232483840 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002615366098294885, + "loss": 1.2517, + "theoretical_loss": 3.4278752478437826, + "tokens_seen": 2232549376 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002615205616850552, + "loss": 1.2513, + "theoretical_loss": 3.4278671066304955, + "tokens_seen": 2232614912 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002615045135406219, + "loss": 1.1903, + "theoretical_loss": 3.4278589657230922, + "tokens_seen": 2232680448 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002614884653961886, + "loss": 1.2529, + "theoretical_loss": 3.4278508251215527, + "tokens_seen": 2232745984 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002614724172517553, + "loss": 1.2645, + "theoretical_loss": 3.4278426848258565, + "tokens_seen": 2232811520 + }, + { + "epoch": 0.68, + "learning_rate": 0.000261456369107322, + "loss": 1.3, + "theoretical_loss": 3.4278345448359833, + "tokens_seen": 2232877056 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026144032096288864, + "loss": 1.1505, + "theoretical_loss": 3.427826405151912, + "tokens_seen": 2232942592 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026142427281845534, + "loss": 1.1819, + "theoretical_loss": 3.427818265773623, + "tokens_seen": 2233008128 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026140822467402204, + "loss": 1.1705, + "theoretical_loss": 3.427810126701095, + "tokens_seen": 2233073664 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026139217652958874, + "loss": 1.246, + "theoretical_loss": 3.4278019879343087, + "tokens_seen": 2233139200 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026137612838515544, + "loss": 1.2657, + "theoretical_loss": 3.4277938494732423, + "tokens_seen": 2233204736 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026136008024072214, + "loss": 1.2396, + "theoretical_loss": 3.427785711317876, + "tokens_seen": 2233270272 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026134403209628884, + "loss": 1.3277, + "theoretical_loss": 3.4277775734681892, + "tokens_seen": 2233335808 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026132798395185554, + "loss": 1.252, + "theoretical_loss": 3.4277694359241617, + "tokens_seen": 2233401344 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026131193580742224, + "loss": 1.2186, + "theoretical_loss": 3.427761298685773, + "tokens_seen": 2233466880 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026129588766298894, + "loss": 1.3087, + "theoretical_loss": 3.427753161753002, + "tokens_seen": 2233532416 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026127983951855564, + "loss": 1.2315, + "theoretical_loss": 3.4277450251258292, + "tokens_seen": 2233597952 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026126379137412234, + "loss": 1.2606, + "theoretical_loss": 3.427736888804234, + "tokens_seen": 2233663488 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002612477432296891, + "loss": 1.3204, + "theoretical_loss": 3.4277287527881954, + "tokens_seen": 2233729024 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002612316950852558, + "loss": 1.2508, + "theoretical_loss": 3.4277206170776933, + "tokens_seen": 2233794560 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002612156469408225, + "loss": 1.2384, + "theoretical_loss": 3.427712481672707, + "tokens_seen": 2233860096 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002611995987963892, + "loss": 1.2048, + "theoretical_loss": 3.4277043465732167, + "tokens_seen": 2233925632 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002611835506519559, + "loss": 1.2482, + "theoretical_loss": 3.4276962117792014, + "tokens_seen": 2233991168 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002611675025075226, + "loss": 1.23, + "theoretical_loss": 3.427688077290641, + "tokens_seen": 2234056704 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002611514543630893, + "loss": 1.3504, + "theoretical_loss": 3.4276799431075147, + "tokens_seen": 2234122240 + }, + { + "epoch": 0.68, + "learning_rate": 0.000261135406218656, + "loss": 1.2464, + "theoretical_loss": 3.4276718092298024, + "tokens_seen": 2234187776 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002611193580742227, + "loss": 1.1954, + "theoretical_loss": 3.427663675657483, + "tokens_seen": 2234253312 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002611033099297894, + "loss": 1.278, + "theoretical_loss": 3.4276555423905375, + "tokens_seen": 2234318848 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002610872617853561, + "loss": 1.2021, + "theoretical_loss": 3.427647409428944, + "tokens_seen": 2234384384 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002610712136409228, + "loss": 1.2345, + "theoretical_loss": 3.4276392767726827, + "tokens_seen": 2234449920 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002610551654964895, + "loss": 1.2396, + "theoretical_loss": 3.427631144421733, + "tokens_seen": 2234515456 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002610391173520562, + "loss": 1.1555, + "theoretical_loss": 3.4276230123760754, + "tokens_seen": 2234580992 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002610230692076229, + "loss": 1.2077, + "theoretical_loss": 3.427614880635688, + "tokens_seen": 2234646528 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002610070210631896, + "loss": 1.2569, + "theoretical_loss": 3.4276067492005518, + "tokens_seen": 2234712064 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 748461, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2609285116195679, + "objective/train/theoretical_loss": 3.427598618070645, + "objective/train/tokens_used": 2255237600, + "theoretical_loss": 3.427598618070645, + "tokens_seen": 2234777600 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002609909729187563, + "loss": 1.2526, + "theoretical_loss": 3.427598618070645, + "tokens_seen": 2234777600 + }, + { + "epoch": 0.68, + "learning_rate": 0.000260974924774323, + "loss": 1.2286, + "theoretical_loss": 3.4275904872459484, + "tokens_seen": 2234843136 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002609588766298897, + "loss": 1.1968, + "theoretical_loss": 3.427582356726441, + "tokens_seen": 2234908672 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002609428284854564, + "loss": 1.2968, + "theoretical_loss": 3.4275742265121023, + "tokens_seen": 2234974208 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002609267803410231, + "loss": 1.2207, + "theoretical_loss": 3.4275660966029124, + "tokens_seen": 2235039744 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002609107321965898, + "loss": 1.2829, + "theoretical_loss": 3.42755796699885, + "tokens_seen": 2235105280 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002608946840521565, + "loss": 1.2631, + "theoretical_loss": 3.427549837699896, + "tokens_seen": 2235170816 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002608786359077232, + "loss": 1.2794, + "theoretical_loss": 3.427541708706029, + "tokens_seen": 2235236352 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002608625877632899, + "loss": 1.2915, + "theoretical_loss": 3.427533580017229, + "tokens_seen": 2235301888 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002608465396188566, + "loss": 1.1816, + "theoretical_loss": 3.427525451633475, + "tokens_seen": 2235367424 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002608304914744233, + "loss": 1.2738, + "theoretical_loss": 3.427517323554748, + "tokens_seen": 2235432960 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026081444332999, + "loss": 1.2457, + "theoretical_loss": 3.427509195781026, + "tokens_seen": 2235498496 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026079839518555666, + "loss": 1.1903, + "theoretical_loss": 3.42750106831229, + "tokens_seen": 2235564032 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026078234704112336, + "loss": 1.2621, + "theoretical_loss": 3.427492941148518, + "tokens_seen": 2235629568 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026076629889669006, + "loss": 1.3453, + "theoretical_loss": 3.4274848142896914, + "tokens_seen": 2235695104 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026075025075225676, + "loss": 1.2494, + "theoretical_loss": 3.4274766877357887, + "tokens_seen": 2235760640 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026073420260782346, + "loss": 1.2067, + "theoretical_loss": 3.42746856148679, + "tokens_seen": 2235826176 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026071815446339016, + "loss": 1.2309, + "theoretical_loss": 3.4274604355426748, + "tokens_seen": 2235891712 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026070210631895687, + "loss": 1.2126, + "theoretical_loss": 3.4274523099034226, + "tokens_seen": 2235957248 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026068605817452357, + "loss": 1.2818, + "theoretical_loss": 3.427444184569013, + "tokens_seen": 2236022784 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026067001003009027, + "loss": 1.2656, + "theoretical_loss": 3.4274360595394255, + "tokens_seen": 2236088320 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026065396188565697, + "loss": 1.2914, + "theoretical_loss": 3.42742793481464, + "tokens_seen": 2236153856 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026063791374122367, + "loss": 1.1957, + "theoretical_loss": 3.4274198103946367, + "tokens_seen": 2236219392 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026062186559679037, + "loss": 1.3751, + "theoretical_loss": 3.427411686279394, + "tokens_seen": 2236284928 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026060581745235707, + "loss": 1.4579, + "theoretical_loss": 3.4274035624688923, + "tokens_seen": 2236350464 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026058976930792377, + "loss": 1.3407, + "theoretical_loss": 3.4273954389631114, + "tokens_seen": 2236416000 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026057372116349047, + "loss": 1.2947, + "theoretical_loss": 3.4273873157620303, + "tokens_seen": 2236481536 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026055767301905717, + "loss": 1.2215, + "theoretical_loss": 3.4273791928656294, + "tokens_seen": 2236547072 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026054162487462387, + "loss": 1.2538, + "theoretical_loss": 3.4273710702738875, + "tokens_seen": 2236612608 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026052557673019057, + "loss": 1.201, + "theoretical_loss": 3.4273629479867846, + "tokens_seen": 2236678144 + }, + { + "epoch": 0.68, + "learning_rate": 0.00026050952858575727, + "loss": 1.2066, + "theoretical_loss": 3.4273548260043007, + "tokens_seen": 2236743680 + }, + { + "epoch": 0.68, + "learning_rate": 0.000260493480441324, + "loss": 1.2592, + "theoretical_loss": 3.427346704326415, + "tokens_seen": 2236809216 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002604774322968907, + "loss": 1.1896, + "theoretical_loss": 3.427338582953108, + "tokens_seen": 2236874752 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002604613841524574, + "loss": 1.372, + "theoretical_loss": 3.427330461884358, + "tokens_seen": 2236940288 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002604453360080241, + "loss": 1.2002, + "theoretical_loss": 3.427322341120145, + "tokens_seen": 2237005824 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002604292878635908, + "loss": 1.297, + "theoretical_loss": 3.42731422066045, + "tokens_seen": 2237071360 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002604132397191575, + "loss": 1.2618, + "theoretical_loss": 3.427306100505251, + "tokens_seen": 2237136896 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002603971915747242, + "loss": 1.2693, + "theoretical_loss": 3.4272979806545285, + "tokens_seen": 2237202432 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002603811434302909, + "loss": 1.1659, + "theoretical_loss": 3.4272898611082616, + "tokens_seen": 2237267968 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002603650952858576, + "loss": 1.3611, + "theoretical_loss": 3.4272817418664308, + "tokens_seen": 2237333504 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002603490471414243, + "loss": 1.2429, + "theoretical_loss": 3.4272736229290155, + "tokens_seen": 2237399040 + }, + { + "epoch": 0.68, + "learning_rate": 0.000260332998996991, + "loss": 1.2929, + "theoretical_loss": 3.4272655042959945, + "tokens_seen": 2237464576 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002603169508525577, + "loss": 1.2511, + "theoretical_loss": 3.4272573859673487, + "tokens_seen": 2237530112 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002603009027081244, + "loss": 1.2249, + "theoretical_loss": 3.4272492679430573, + "tokens_seen": 2237595648 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002602848545636911, + "loss": 1.2362, + "theoretical_loss": 3.4272411502230997, + "tokens_seen": 2237661184 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002602688064192578, + "loss": 1.3199, + "theoretical_loss": 3.4272330328074556, + "tokens_seen": 2237726720 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002602527582748245, + "loss": 1.1886, + "theoretical_loss": 3.4272249156961054, + "tokens_seen": 2237792256 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002602367101303912, + "loss": 1.2064, + "theoretical_loss": 3.427216798889028, + "tokens_seen": 2237857792 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002602206619859579, + "loss": 1.3207, + "theoretical_loss": 3.427208682386203, + "tokens_seen": 2237923328 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002602046138415246, + "loss": 1.1761, + "theoretical_loss": 3.427200566187611, + "tokens_seen": 2237988864 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 749372, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2783803939819336, + "objective/train/theoretical_loss": 3.427192450293231, + "objective/train/tokens_used": 2258514400, + "theoretical_loss": 3.427192450293231, + "tokens_seen": 2238054400 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002601885656970913, + "loss": 1.3294, + "theoretical_loss": 3.427192450293231, + "tokens_seen": 2238054400 + }, + { + "epoch": 0.68, + "learning_rate": 0.000260172517552658, + "loss": 1.3497, + "theoretical_loss": 3.4271843347030426, + "tokens_seen": 2238119936 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002601564694082247, + "loss": 1.2225, + "theoretical_loss": 3.4271762194170257, + "tokens_seen": 2238185472 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002601404212637914, + "loss": 1.2537, + "theoretical_loss": 3.42716810443516, + "tokens_seen": 2238251008 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002601243731193581, + "loss": 1.275, + "theoretical_loss": 3.427159989757425, + "tokens_seen": 2238316544 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002601083249749248, + "loss": 1.2975, + "theoretical_loss": 3.4271518753838013, + "tokens_seen": 2238382080 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002600922768304915, + "loss": 1.2252, + "theoretical_loss": 3.4271437613142672, + "tokens_seen": 2238447616 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002600762286860582, + "loss": 1.2945, + "theoretical_loss": 3.4271356475488033, + "tokens_seen": 2238513152 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002600601805416249, + "loss": 1.2099, + "theoretical_loss": 3.427127534087389, + "tokens_seen": 2238578688 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002600441323971916, + "loss": 1.2675, + "theoretical_loss": 3.427119420930004, + "tokens_seen": 2238644224 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002600280842527583, + "loss": 1.268, + "theoretical_loss": 3.4271113080766282, + "tokens_seen": 2238709760 + }, + { + "epoch": 0.68, + "learning_rate": 0.000260012036108325, + "loss": 1.3131, + "theoretical_loss": 3.4271031955272413, + "tokens_seen": 2238775296 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002599959879638917, + "loss": 1.2708, + "theoretical_loss": 3.4270950832818228, + "tokens_seen": 2238840832 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002599799398194584, + "loss": 1.2699, + "theoretical_loss": 3.427086971340353, + "tokens_seen": 2238906368 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002599638916750251, + "loss": 1.2659, + "theoretical_loss": 3.4270788597028106, + "tokens_seen": 2238971904 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002599478435305918, + "loss": 1.2474, + "theoretical_loss": 3.427070748369176, + "tokens_seen": 2239037440 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002599317953861585, + "loss": 1.26, + "theoretical_loss": 3.4270626373394286, + "tokens_seen": 2239102976 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002599157472417252, + "loss": 1.3254, + "theoretical_loss": 3.427054526613549, + "tokens_seen": 2239168512 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002598996990972919, + "loss": 1.2616, + "theoretical_loss": 3.4270464161915157, + "tokens_seen": 2239234048 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002598836509528586, + "loss": 1.2158, + "theoretical_loss": 3.427038306073309, + "tokens_seen": 2239299584 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002598676028084253, + "loss": 1.237, + "theoretical_loss": 3.4270301962589085, + "tokens_seen": 2239365120 + }, + { + "epoch": 0.68, + "learning_rate": 0.000259851554663992, + "loss": 1.2999, + "theoretical_loss": 3.4270220867482943, + "tokens_seen": 2239430656 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002598355065195587, + "loss": 1.2559, + "theoretical_loss": 3.4270139775414457, + "tokens_seen": 2239496192 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002598194583751254, + "loss": 1.2476, + "theoretical_loss": 3.4270058686383424, + "tokens_seen": 2239561728 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002598034102306921, + "loss": 1.2037, + "theoretical_loss": 3.4269977600389643, + "tokens_seen": 2239627264 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002597873620862588, + "loss": 1.1522, + "theoretical_loss": 3.4269896517432916, + "tokens_seen": 2239692800 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002597713139418255, + "loss": 1.2902, + "theoretical_loss": 3.426981543751303, + "tokens_seen": 2239758336 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002597552657973922, + "loss": 1.1457, + "theoretical_loss": 3.4269734360629793, + "tokens_seen": 2239823872 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002597392176529589, + "loss": 1.1659, + "theoretical_loss": 3.4269653286782997, + "tokens_seen": 2239889408 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002597231695085256, + "loss": 1.2123, + "theoretical_loss": 3.4269572215972435, + "tokens_seen": 2239954944 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002597071213640923, + "loss": 1.298, + "theoretical_loss": 3.4269491148197915, + "tokens_seen": 2240020480 + }, + { + "epoch": 0.68, + "learning_rate": 0.000259691073219659, + "loss": 1.2627, + "theoretical_loss": 3.426941008345923, + "tokens_seen": 2240086016 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002596750250752257, + "loss": 1.2419, + "theoretical_loss": 3.426932902175617, + "tokens_seen": 2240151552 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002596589769307924, + "loss": 1.35, + "theoretical_loss": 3.4269247963088545, + "tokens_seen": 2240217088 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002596429287863591, + "loss": 1.3486, + "theoretical_loss": 3.4269166907456143, + "tokens_seen": 2240282624 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002596268806419258, + "loss": 1.3065, + "theoretical_loss": 3.4269085854858767, + "tokens_seen": 2240348160 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002596108324974925, + "loss": 1.3444, + "theoretical_loss": 3.4269004805296213, + "tokens_seen": 2240413696 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002595947843530592, + "loss": 1.3182, + "theoretical_loss": 3.426892375876828, + "tokens_seen": 2240479232 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002595787362086259, + "loss": 1.307, + "theoretical_loss": 3.4268842715274763, + "tokens_seen": 2240544768 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002595626880641926, + "loss": 1.2944, + "theoretical_loss": 3.426876167481546, + "tokens_seen": 2240610304 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002595466399197593, + "loss": 1.1759, + "theoretical_loss": 3.4268680637390165, + "tokens_seen": 2240675840 + }, + { + "epoch": 0.68, + "learning_rate": 0.000259530591775326, + "loss": 1.3129, + "theoretical_loss": 3.426859960299869, + "tokens_seen": 2240741376 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002595145436308927, + "loss": 1.2801, + "theoretical_loss": 3.426851857164081, + "tokens_seen": 2240806912 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002594984954864594, + "loss": 1.194, + "theoretical_loss": 3.4268437543316344, + "tokens_seen": 2240872448 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002594824473420261, + "loss": 1.2469, + "theoretical_loss": 3.426835651802508, + "tokens_seen": 2240937984 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002594663991975928, + "loss": 1.306, + "theoretical_loss": 3.426827549576682, + "tokens_seen": 2241003520 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002594503510531595, + "loss": 1.13, + "theoretical_loss": 3.426819447654135, + "tokens_seen": 2241069056 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002594343029087262, + "loss": 1.2889, + "theoretical_loss": 3.426811346034848, + "tokens_seen": 2241134592 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002594182547642929, + "loss": 1.3935, + "theoretical_loss": 3.426803244718801, + "tokens_seen": 2241200128 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025940220661985956, + "loss": 1.268, + "theoretical_loss": 3.4267951437059727, + "tokens_seen": 2241265664 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 750607, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2871696949005127, + "objective/train/theoretical_loss": 3.4267870429963434, + "objective/train/tokens_used": 2261791200, + "theoretical_loss": 3.4267870429963434, + "tokens_seen": 2241331200 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025938615847542626, + "loss": 1.252, + "theoretical_loss": 3.4267870429963434, + "tokens_seen": 2241331200 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025937011033099296, + "loss": 1.2497, + "theoretical_loss": 3.426778942589893, + "tokens_seen": 2241396736 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025935406218655966, + "loss": 1.2834, + "theoretical_loss": 3.426770842486601, + "tokens_seen": 2241462272 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025933801404212636, + "loss": 1.3216, + "theoretical_loss": 3.4267627426864475, + "tokens_seen": 2241527808 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025932196589769307, + "loss": 1.2537, + "theoretical_loss": 3.426754643189412, + "tokens_seen": 2241593344 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025930591775325977, + "loss": 1.2831, + "theoretical_loss": 3.4267465439954745, + "tokens_seen": 2241658880 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025928986960882647, + "loss": 1.2709, + "theoretical_loss": 3.426738445104615, + "tokens_seen": 2241724416 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025927382146439317, + "loss": 1.2344, + "theoretical_loss": 3.4267303465168126, + "tokens_seen": 2241789952 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025925777331995987, + "loss": 1.3044, + "theoretical_loss": 3.426722248232048, + "tokens_seen": 2241855488 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025924172517552657, + "loss": 1.2432, + "theoretical_loss": 3.4267141502503002, + "tokens_seen": 2241921024 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025922567703109327, + "loss": 1.1919, + "theoretical_loss": 3.426706052571549, + "tokens_seen": 2241986560 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025920962888665997, + "loss": 1.1862, + "theoretical_loss": 3.4266979551957757, + "tokens_seen": 2242052096 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025919358074222667, + "loss": 1.2909, + "theoretical_loss": 3.426689858122958, + "tokens_seen": 2242117632 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025917753259779337, + "loss": 1.1483, + "theoretical_loss": 3.426681761353077, + "tokens_seen": 2242183168 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025916148445336007, + "loss": 1.2412, + "theoretical_loss": 3.4266736648861125, + "tokens_seen": 2242248704 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025914543630892677, + "loss": 1.2736, + "theoretical_loss": 3.4266655687220435, + "tokens_seen": 2242314240 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002591293881644935, + "loss": 1.248, + "theoretical_loss": 3.4266574728608505, + "tokens_seen": 2242379776 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002591133400200602, + "loss": 1.3304, + "theoretical_loss": 3.426649377302513, + "tokens_seen": 2242445312 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002590972918756269, + "loss": 1.2917, + "theoretical_loss": 3.426641282047011, + "tokens_seen": 2242510848 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002590812437311936, + "loss": 1.2937, + "theoretical_loss": 3.4266331870943247, + "tokens_seen": 2242576384 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002590651955867603, + "loss": 1.3004, + "theoretical_loss": 3.426625092444433, + "tokens_seen": 2242641920 + }, + { + "epoch": 0.68, + "learning_rate": 0.000259049147442327, + "loss": 1.3978, + "theoretical_loss": 3.4266169980973165, + "tokens_seen": 2242707456 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002590330992978937, + "loss": 1.2985, + "theoretical_loss": 3.4266089040529546, + "tokens_seen": 2242772992 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002590170511534604, + "loss": 1.2557, + "theoretical_loss": 3.4266008103113275, + "tokens_seen": 2242838528 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002590010030090271, + "loss": 1.2487, + "theoretical_loss": 3.4265927168724146, + "tokens_seen": 2242904064 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002589849548645938, + "loss": 1.2023, + "theoretical_loss": 3.426584623736196, + "tokens_seen": 2242969600 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002589689067201605, + "loss": 1.2825, + "theoretical_loss": 3.4265765309026515, + "tokens_seen": 2243035136 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002589528585757272, + "loss": 1.2971, + "theoretical_loss": 3.4265684383717607, + "tokens_seen": 2243100672 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002589368104312939, + "loss": 1.2662, + "theoretical_loss": 3.426560346143504, + "tokens_seen": 2243166208 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002589207622868606, + "loss": 1.2661, + "theoretical_loss": 3.426552254217861, + "tokens_seen": 2243231744 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002589047141424273, + "loss": 1.2408, + "theoretical_loss": 3.426544162594811, + "tokens_seen": 2243297280 + }, + { + "epoch": 0.68, + "learning_rate": 0.000258888665997994, + "loss": 1.205, + "theoretical_loss": 3.4265360712743345, + "tokens_seen": 2243362816 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002588726178535607, + "loss": 1.2912, + "theoretical_loss": 3.4265279802564113, + "tokens_seen": 2243428352 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025885656970912744, + "loss": 1.228, + "theoretical_loss": 3.426519889541021, + "tokens_seen": 2243493888 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025884052156469414, + "loss": 1.2382, + "theoretical_loss": 3.4265117991281437, + "tokens_seen": 2243559424 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025882447342026084, + "loss": 1.2655, + "theoretical_loss": 3.4265037090177586, + "tokens_seen": 2243624960 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025880842527582754, + "loss": 1.2319, + "theoretical_loss": 3.4264956192098466, + "tokens_seen": 2243690496 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025879237713139424, + "loss": 1.2381, + "theoretical_loss": 3.426487529704387, + "tokens_seen": 2243756032 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025877632898696094, + "loss": 1.1888, + "theoretical_loss": 3.4264794405013594, + "tokens_seen": 2243821568 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002587602808425276, + "loss": 1.2146, + "theoretical_loss": 3.426471351600744, + "tokens_seen": 2243887104 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002587442326980943, + "loss": 1.3117, + "theoretical_loss": 3.4264632630025202, + "tokens_seen": 2243952640 + }, + { + "epoch": 0.68, + "learning_rate": 0.000258728184553661, + "loss": 1.2189, + "theoretical_loss": 3.4264551747066685, + "tokens_seen": 2244018176 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002587121364092277, + "loss": 1.234, + "theoretical_loss": 3.4264470867131687, + "tokens_seen": 2244083712 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002586960882647944, + "loss": 1.2452, + "theoretical_loss": 3.4264389990220003, + "tokens_seen": 2244149248 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002586800401203611, + "loss": 1.2628, + "theoretical_loss": 3.4264309116331435, + "tokens_seen": 2244214784 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002586639919759278, + "loss": 1.2893, + "theoretical_loss": 3.426422824546578, + "tokens_seen": 2244280320 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002586479438314945, + "loss": 1.2525, + "theoretical_loss": 3.4264147377622836, + "tokens_seen": 2244345856 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002586318956870612, + "loss": 1.2108, + "theoretical_loss": 3.4264066512802405, + "tokens_seen": 2244411392 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002586158475426279, + "loss": 1.2643, + "theoretical_loss": 3.4263985651004285, + "tokens_seen": 2244476928 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002585997993981946, + "loss": 1.3556, + "theoretical_loss": 3.426390479222827, + "tokens_seen": 2244542464 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 751484, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1918597221374512, + "objective/train/theoretical_loss": 3.4263823936474163, + "objective/train/tokens_used": 2265068000, + "theoretical_loss": 3.4263823936474163, + "tokens_seen": 2244608000 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002585837512537613, + "loss": 1.2784, + "theoretical_loss": 3.4263823936474163, + "tokens_seen": 2244608000 + }, + { + "epoch": 0.68, + "learning_rate": 0.000258567703109328, + "loss": 1.3167, + "theoretical_loss": 3.426374308374176, + "tokens_seen": 2244673536 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002585516549648947, + "loss": 1.2461, + "theoretical_loss": 3.4263662234030865, + "tokens_seen": 2244739072 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002585356068204614, + "loss": 1.119, + "theoretical_loss": 3.4263581387341273, + "tokens_seen": 2244804608 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002585195586760281, + "loss": 1.1333, + "theoretical_loss": 3.426350054367278, + "tokens_seen": 2244870144 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002585035105315948, + "loss": 1.354, + "theoretical_loss": 3.426341970302519, + "tokens_seen": 2244935680 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002584874623871615, + "loss": 1.3536, + "theoretical_loss": 3.4263338865398305, + "tokens_seen": 2245001216 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002584714142427282, + "loss": 1.3903, + "theoretical_loss": 3.426325803079192, + "tokens_seen": 2245066752 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002584553660982949, + "loss": 1.2681, + "theoretical_loss": 3.4263177199205828, + "tokens_seen": 2245132288 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002584393179538616, + "loss": 1.2298, + "theoretical_loss": 3.4263096370639836, + "tokens_seen": 2245197824 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002584232698094283, + "loss": 1.3126, + "theoretical_loss": 3.426301554509374, + "tokens_seen": 2245263360 + }, + { + "epoch": 0.68, + "learning_rate": 0.000258407221664995, + "loss": 1.3224, + "theoretical_loss": 3.426293472256734, + "tokens_seen": 2245328896 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002583911735205617, + "loss": 1.2536, + "theoretical_loss": 3.4262853903060435, + "tokens_seen": 2245394432 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002583751253761284, + "loss": 1.2142, + "theoretical_loss": 3.4262773086572818, + "tokens_seen": 2245459968 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002583590772316951, + "loss": 1.1508, + "theoretical_loss": 3.42626922731043, + "tokens_seen": 2245525504 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002583430290872618, + "loss": 1.1756, + "theoretical_loss": 3.426261146265467, + "tokens_seen": 2245591040 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002583269809428285, + "loss": 1.1798, + "theoretical_loss": 3.4262530655223733, + "tokens_seen": 2245656576 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002583109327983952, + "loss": 1.2357, + "theoretical_loss": 3.4262449850811287, + "tokens_seen": 2245722112 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002582948846539619, + "loss": 1.2656, + "theoretical_loss": 3.426236904941713, + "tokens_seen": 2245787648 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002582788365095286, + "loss": 1.3233, + "theoretical_loss": 3.426228825104106, + "tokens_seen": 2245853184 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002582627883650953, + "loss": 1.3315, + "theoretical_loss": 3.426220745568288, + "tokens_seen": 2245918720 + }, + { + "epoch": 0.68, + "learning_rate": 0.000258246740220662, + "loss": 1.3161, + "theoretical_loss": 3.4262126663342385, + "tokens_seen": 2245984256 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002582306920762287, + "loss": 1.3041, + "theoretical_loss": 3.4262045874019376, + "tokens_seen": 2246049792 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002582146439317954, + "loss": 1.2157, + "theoretical_loss": 3.426196508771365, + "tokens_seen": 2246115328 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002581985957873621, + "loss": 1.2135, + "theoretical_loss": 3.426188430442501, + "tokens_seen": 2246180864 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002581825476429288, + "loss": 1.2739, + "theoretical_loss": 3.426180352415326, + "tokens_seen": 2246246400 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002581664994984955, + "loss": 1.2549, + "theoretical_loss": 3.4261722746898187, + "tokens_seen": 2246311936 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002581504513540622, + "loss": 1.3188, + "theoretical_loss": 3.4261641972659596, + "tokens_seen": 2246377472 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002581344032096289, + "loss": 1.3489, + "theoretical_loss": 3.4261561201437285, + "tokens_seen": 2246443008 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002581183550651956, + "loss": 1.2, + "theoretical_loss": 3.426148043323106, + "tokens_seen": 2246508544 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002581023069207623, + "loss": 1.3102, + "theoretical_loss": 3.4261399668040715, + "tokens_seen": 2246574080 + }, + { + "epoch": 0.68, + "learning_rate": 0.000258086258776329, + "loss": 1.3588, + "theoretical_loss": 3.426131890586605, + "tokens_seen": 2246639616 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002580702106318957, + "loss": 1.3681, + "theoretical_loss": 3.4261238146706865, + "tokens_seen": 2246705152 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002580541624874624, + "loss": 1.2636, + "theoretical_loss": 3.4261157390562955, + "tokens_seen": 2246770688 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002580381143430291, + "loss": 1.2944, + "theoretical_loss": 3.4261076637434127, + "tokens_seen": 2246836224 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002580220661985958, + "loss": 1.4175, + "theoretical_loss": 3.4260995887320176, + "tokens_seen": 2246901760 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002580060180541625, + "loss": 1.3159, + "theoretical_loss": 3.4260915140220902, + "tokens_seen": 2246967296 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025798996990972916, + "loss": 1.3104, + "theoretical_loss": 3.4260834396136106, + "tokens_seen": 2247032832 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025797392176529586, + "loss": 1.2345, + "theoretical_loss": 3.426075365506559, + "tokens_seen": 2247098368 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025795787362086257, + "loss": 1.2815, + "theoretical_loss": 3.4260672917009147, + "tokens_seen": 2247163904 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025794182547642927, + "loss": 1.2066, + "theoretical_loss": 3.4260592181966576, + "tokens_seen": 2247229440 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025792577733199597, + "loss": 1.2734, + "theoretical_loss": 3.4260511449937683, + "tokens_seen": 2247294976 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025790972918756267, + "loss": 1.2326, + "theoretical_loss": 3.426043072092227, + "tokens_seen": 2247360512 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025789368104312937, + "loss": 1.3732, + "theoretical_loss": 3.4260349994920123, + "tokens_seen": 2247426048 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025787763289869607, + "loss": 1.2563, + "theoretical_loss": 3.4260269271931056, + "tokens_seen": 2247491584 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025786158475426277, + "loss": 1.3138, + "theoretical_loss": 3.426018855195486, + "tokens_seen": 2247557120 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025784553660982947, + "loss": 1.0993, + "theoretical_loss": 3.426010783499134, + "tokens_seen": 2247622656 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025782948846539617, + "loss": 1.1972, + "theoretical_loss": 3.426002712104029, + "tokens_seen": 2247688192 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025781344032096287, + "loss": 1.1782, + "theoretical_loss": 3.425994641010152, + "tokens_seen": 2247753728 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025779739217652957, + "loss": 1.253, + "theoretical_loss": 3.425986570217481, + "tokens_seen": 2247819264 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 752699, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1567444801330566, + "objective/train/theoretical_loss": 3.4259784997259985, + "objective/train/tokens_used": 2268344800, + "theoretical_loss": 3.4259784997259985, + "tokens_seen": 2247884800 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025778134403209627, + "loss": 1.1792, + "theoretical_loss": 3.4259784997259985, + "tokens_seen": 2247884800 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025776529588766297, + "loss": 1.2379, + "theoretical_loss": 3.425970429535683, + "tokens_seen": 2247950336 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002577492477432297, + "loss": 1.2762, + "theoretical_loss": 3.425962359646514, + "tokens_seen": 2248015872 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002577331995987964, + "loss": 1.179, + "theoretical_loss": 3.425954290058473, + "tokens_seen": 2248081408 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002577171514543631, + "loss": 1.2794, + "theoretical_loss": 3.425946220771539, + "tokens_seen": 2248146944 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002577011033099298, + "loss": 1.2385, + "theoretical_loss": 3.425938151785692, + "tokens_seen": 2248212480 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002576850551654965, + "loss": 1.2444, + "theoretical_loss": 3.425930083100912, + "tokens_seen": 2248278016 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002576690070210632, + "loss": 1.2346, + "theoretical_loss": 3.4259220147171794, + "tokens_seen": 2248343552 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025765295887662993, + "loss": 1.2604, + "theoretical_loss": 3.4259139466344743, + "tokens_seen": 2248409088 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025763691073219663, + "loss": 1.3066, + "theoretical_loss": 3.4259058788527756, + "tokens_seen": 2248474624 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025762086258776333, + "loss": 1.2267, + "theoretical_loss": 3.4258978113720646, + "tokens_seen": 2248540160 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025760481444333003, + "loss": 1.2654, + "theoretical_loss": 3.425889744192321, + "tokens_seen": 2248605696 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025758876629889673, + "loss": 1.2398, + "theoretical_loss": 3.425881677313524, + "tokens_seen": 2248671232 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025757271815446343, + "loss": 1.2718, + "theoretical_loss": 3.425873610735654, + "tokens_seen": 2248736768 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025755667001003014, + "loss": 1.259, + "theoretical_loss": 3.4258655444586914, + "tokens_seen": 2248802304 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025754062186559684, + "loss": 1.2431, + "theoretical_loss": 3.4258574784826163, + "tokens_seen": 2248867840 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025752457372116354, + "loss": 1.2773, + "theoretical_loss": 3.425849412807408, + "tokens_seen": 2248933376 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025750852557673024, + "loss": 1.1889, + "theoretical_loss": 3.425841347433047, + "tokens_seen": 2248998912 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025749247743229694, + "loss": 1.2735, + "theoretical_loss": 3.425833282359513, + "tokens_seen": 2249064448 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025747642928786364, + "loss": 1.3222, + "theoretical_loss": 3.4258252175867865, + "tokens_seen": 2249129984 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025746038114343034, + "loss": 1.184, + "theoretical_loss": 3.425817153114847, + "tokens_seen": 2249195520 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025744433299899704, + "loss": 1.2551, + "theoretical_loss": 3.4258090889436748, + "tokens_seen": 2249261056 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025742828485456374, + "loss": 1.2662, + "theoretical_loss": 3.42580102507325, + "tokens_seen": 2249326592 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025741223671013044, + "loss": 1.2804, + "theoretical_loss": 3.4257929615035523, + "tokens_seen": 2249392128 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025739618856569714, + "loss": 1.3647, + "theoretical_loss": 3.425784898234562, + "tokens_seen": 2249457664 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025738014042126384, + "loss": 1.1655, + "theoretical_loss": 3.4257768352662588, + "tokens_seen": 2249523200 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002573640922768305, + "loss": 1.2565, + "theoretical_loss": 3.425768772598623, + "tokens_seen": 2249588736 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002573480441323972, + "loss": 1.2632, + "theoretical_loss": 3.4257607102316348, + "tokens_seen": 2249654272 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002573319959879639, + "loss": 1.2603, + "theoretical_loss": 3.425752648165274, + "tokens_seen": 2249719808 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002573159478435306, + "loss": 1.2878, + "theoretical_loss": 3.425744586399521, + "tokens_seen": 2249785344 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002572998996990973, + "loss": 1.1345, + "theoretical_loss": 3.4257365249343543, + "tokens_seen": 2249850880 + }, + { + "epoch": 0.68, + "learning_rate": 0.000257283851554664, + "loss": 1.2098, + "theoretical_loss": 3.4257284637697563, + "tokens_seen": 2249916416 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002572678034102307, + "loss": 1.2999, + "theoretical_loss": 3.4257204029057053, + "tokens_seen": 2249981952 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002572517552657974, + "loss": 1.2553, + "theoretical_loss": 3.425712342342182, + "tokens_seen": 2250047488 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002572357071213641, + "loss": 1.2098, + "theoretical_loss": 3.4257042820791663, + "tokens_seen": 2250113024 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002572196589769308, + "loss": 1.2514, + "theoretical_loss": 3.4256962221166383, + "tokens_seen": 2250178560 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002572036108324975, + "loss": 1.2668, + "theoretical_loss": 3.4256881624545783, + "tokens_seen": 2250244096 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002571875626880642, + "loss": 1.1187, + "theoretical_loss": 3.425680103092966, + "tokens_seen": 2250309632 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002571715145436309, + "loss": 1.1878, + "theoretical_loss": 3.4256720440317814, + "tokens_seen": 2250375168 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002571554663991976, + "loss": 1.3023, + "theoretical_loss": 3.4256639852710045, + "tokens_seen": 2250440704 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002571394182547643, + "loss": 1.2913, + "theoretical_loss": 3.425655926810616, + "tokens_seen": 2250506240 + }, + { + "epoch": 0.68, + "learning_rate": 0.000257123370110331, + "loss": 1.227, + "theoretical_loss": 3.425647868650595, + "tokens_seen": 2250571776 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002571073219658977, + "loss": 1.2792, + "theoretical_loss": 3.4256398107909227, + "tokens_seen": 2250637312 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002570912738214644, + "loss": 1.3699, + "theoretical_loss": 3.425631753231578, + "tokens_seen": 2250702848 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002570752256770311, + "loss": 1.2681, + "theoretical_loss": 3.4256236959725417, + "tokens_seen": 2250768384 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002570591775325978, + "loss": 1.1681, + "theoretical_loss": 3.4256156390137935, + "tokens_seen": 2250833920 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002570431293881645, + "loss": 1.2877, + "theoretical_loss": 3.4256075823553136, + "tokens_seen": 2250899456 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002570270812437312, + "loss": 1.3389, + "theoretical_loss": 3.4255995259970824, + "tokens_seen": 2250964992 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002570110330992979, + "loss": 1.2114, + "theoretical_loss": 3.4255914699390795, + "tokens_seen": 2251030528 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002569949849548646, + "loss": 1.198, + "theoretical_loss": 3.425583414181285, + "tokens_seen": 2251096064 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 753469, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.215314269065857, + "objective/train/theoretical_loss": 3.4255753587236786, + "objective/train/tokens_used": 2271621600, + "theoretical_loss": 3.4255753587236786, + "tokens_seen": 2251161600 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002569789368104313, + "loss": 1.1527, + "theoretical_loss": 3.4255753587236786, + "tokens_seen": 2251161600 + }, + { + "epoch": 0.68, + "learning_rate": 0.000256962888665998, + "loss": 1.1617, + "theoretical_loss": 3.4255673035662415, + "tokens_seen": 2251227136 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002569468405215647, + "loss": 1.2568, + "theoretical_loss": 3.425559248708953, + "tokens_seen": 2251292672 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002569307923771314, + "loss": 1.1948, + "theoretical_loss": 3.4255511941517933, + "tokens_seen": 2251358208 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002569147442326981, + "loss": 1.352, + "theoretical_loss": 3.4255431398947427, + "tokens_seen": 2251423744 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002568986960882648, + "loss": 1.2929, + "theoretical_loss": 3.425535085937781, + "tokens_seen": 2251489280 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002568826479438315, + "loss": 1.2623, + "theoretical_loss": 3.425527032280888, + "tokens_seen": 2251554816 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002568665997993982, + "loss": 1.3188, + "theoretical_loss": 3.4255189789240443, + "tokens_seen": 2251620352 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002568505516549649, + "loss": 1.3254, + "theoretical_loss": 3.42551092586723, + "tokens_seen": 2251685888 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002568345035105316, + "loss": 1.2982, + "theoretical_loss": 3.425502873110425, + "tokens_seen": 2251751424 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002568184553660983, + "loss": 1.2842, + "theoretical_loss": 3.4254948206536096, + "tokens_seen": 2251816960 + }, + { + "epoch": 0.68, + "learning_rate": 0.000256802407221665, + "loss": 1.2603, + "theoretical_loss": 3.4254867684967634, + "tokens_seen": 2251882496 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002567863590772317, + "loss": 1.2819, + "theoretical_loss": 3.425478716639867, + "tokens_seen": 2251948032 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002567703109327984, + "loss": 1.2873, + "theoretical_loss": 3.4254706650829005, + "tokens_seen": 2252013568 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002567542627883651, + "loss": 1.1428, + "theoretical_loss": 3.4254626138258435, + "tokens_seen": 2252079104 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002567382146439318, + "loss": 1.2405, + "theoretical_loss": 3.4254545628686768, + "tokens_seen": 2252144640 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002567221664994985, + "loss": 1.1668, + "theoretical_loss": 3.42544651221138, + "tokens_seen": 2252210176 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002567061183550652, + "loss": 1.3322, + "theoretical_loss": 3.4254384618539326, + "tokens_seen": 2252275712 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002566900702106319, + "loss": 1.3657, + "theoretical_loss": 3.425430411796316, + "tokens_seen": 2252341248 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002566740220661986, + "loss": 1.2937, + "theoretical_loss": 3.42542236203851, + "tokens_seen": 2252406784 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002566579739217653, + "loss": 1.2307, + "theoretical_loss": 3.425414312580494, + "tokens_seen": 2252472320 + }, + { + "epoch": 0.68, + "learning_rate": 0.000256641925777332, + "loss": 1.2622, + "theoretical_loss": 3.425406263422249, + "tokens_seen": 2252537856 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002566258776328987, + "loss": 1.2367, + "theoretical_loss": 3.4253982145637543, + "tokens_seen": 2252603392 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002566098294884654, + "loss": 1.2514, + "theoretical_loss": 3.425390166004991, + "tokens_seen": 2252668928 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025659378134403206, + "loss": 1.3413, + "theoretical_loss": 3.425382117745938, + "tokens_seen": 2252734464 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025657773319959877, + "loss": 1.3059, + "theoretical_loss": 3.425374069786576, + "tokens_seen": 2252800000 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025656168505516547, + "loss": 1.2375, + "theoretical_loss": 3.4253660221268856, + "tokens_seen": 2252865536 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025654563691073217, + "loss": 1.3065, + "theoretical_loss": 3.425357974766846, + "tokens_seen": 2252931072 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025652958876629887, + "loss": 1.2044, + "theoretical_loss": 3.425349927706438, + "tokens_seen": 2252996608 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025651354062186557, + "loss": 1.3195, + "theoretical_loss": 3.4253418809456417, + "tokens_seen": 2253062144 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025649749247743227, + "loss": 1.1761, + "theoretical_loss": 3.4253338344844373, + "tokens_seen": 2253127680 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025648144433299897, + "loss": 1.2669, + "theoretical_loss": 3.4253257883228043, + "tokens_seen": 2253193216 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025646539618856567, + "loss": 1.2669, + "theoretical_loss": 3.425317742460723, + "tokens_seen": 2253258752 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002564493480441324, + "loss": 1.2762, + "theoretical_loss": 3.425309696898174, + "tokens_seen": 2253324288 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002564332998996991, + "loss": 1.1921, + "theoretical_loss": 3.4253016516351376, + "tokens_seen": 2253389824 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002564172517552658, + "loss": 1.2248, + "theoretical_loss": 3.4252936066715933, + "tokens_seen": 2253455360 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002564012036108325, + "loss": 1.3451, + "theoretical_loss": 3.4252855620075215, + "tokens_seen": 2253520896 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025638515546639923, + "loss": 1.138, + "theoretical_loss": 3.4252775176429022, + "tokens_seen": 2253586432 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025636910732196593, + "loss": 1.1937, + "theoretical_loss": 3.425269473577716, + "tokens_seen": 2253651968 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025635305917753263, + "loss": 1.2648, + "theoretical_loss": 3.4252614298119424, + "tokens_seen": 2253717504 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025633701103309933, + "loss": 1.1999, + "theoretical_loss": 3.425253386345562, + "tokens_seen": 2253783040 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025632096288866603, + "loss": 1.3652, + "theoretical_loss": 3.425245343178555, + "tokens_seen": 2253848576 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025630491474423273, + "loss": 1.2079, + "theoretical_loss": 3.4252373003109007, + "tokens_seen": 2253914112 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025628886659979943, + "loss": 1.2153, + "theoretical_loss": 3.4252292577425805, + "tokens_seen": 2253979648 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025627281845536613, + "loss": 1.2059, + "theoretical_loss": 3.425221215473574, + "tokens_seen": 2254045184 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025625677031093283, + "loss": 1.2275, + "theoretical_loss": 3.4252131735038613, + "tokens_seen": 2254110720 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025624072216649953, + "loss": 1.2685, + "theoretical_loss": 3.4252051318334225, + "tokens_seen": 2254176256 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025622467402206623, + "loss": 1.3108, + "theoretical_loss": 3.425197090462238, + "tokens_seen": 2254241792 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025620862587763293, + "loss": 1.3067, + "theoretical_loss": 3.425189049390288, + "tokens_seen": 2254307328 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025619257773319964, + "loss": 1.2607, + "theoretical_loss": 3.4251810086175523, + "tokens_seen": 2254372864 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 754585, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1932061910629272, + "objective/train/theoretical_loss": 3.425172968144011, + "objective/train/tokens_used": 2274898400, + "theoretical_loss": 3.425172968144011, + "tokens_seen": 2254438400 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025617652958876634, + "loss": 1.2115, + "theoretical_loss": 3.425172968144011, + "tokens_seen": 2254438400 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025616048144433304, + "loss": 1.1674, + "theoretical_loss": 3.425164927969645, + "tokens_seen": 2254503936 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025614443329989974, + "loss": 1.2241, + "theoretical_loss": 3.425156888094434, + "tokens_seen": 2254569472 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025612838515546644, + "loss": 1.1817, + "theoretical_loss": 3.425148848518358, + "tokens_seen": 2254635008 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025611233701103314, + "loss": 1.2647, + "theoretical_loss": 3.4251408092413973, + "tokens_seen": 2254700544 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025609628886659984, + "loss": 1.3923, + "theoretical_loss": 3.425132770263532, + "tokens_seen": 2254766080 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025608024072216654, + "loss": 1.221, + "theoretical_loss": 3.425124731584743, + "tokens_seen": 2254831616 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025606419257773324, + "loss": 1.1826, + "theoretical_loss": 3.4251166932050094, + "tokens_seen": 2254897152 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025604814443329994, + "loss": 1.2809, + "theoretical_loss": 3.4251086551243124, + "tokens_seen": 2254962688 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025603209628886664, + "loss": 1.2202, + "theoretical_loss": 3.4251006173426313, + "tokens_seen": 2255028224 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025601604814443334, + "loss": 1.3595, + "theoretical_loss": 3.4250925798599465, + "tokens_seen": 2255093760 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025600000000000004, + "loss": 1.2818, + "theoretical_loss": 3.4250845426762386, + "tokens_seen": 2255159296 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025598395185556674, + "loss": 1.1745, + "theoretical_loss": 3.4250765057914876, + "tokens_seen": 2255224832 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002559679037111334, + "loss": 1.2497, + "theoretical_loss": 3.4250684692056734, + "tokens_seen": 2255290368 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002559518555667001, + "loss": 1.2763, + "theoretical_loss": 3.4250604329187766, + "tokens_seen": 2255355904 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002559358074222668, + "loss": 1.1942, + "theoretical_loss": 3.425052396930777, + "tokens_seen": 2255421440 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002559197592778335, + "loss": 1.2346, + "theoretical_loss": 3.4250443612416555, + "tokens_seen": 2255486976 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002559037111334002, + "loss": 1.2667, + "theoretical_loss": 3.4250363258513916, + "tokens_seen": 2255552512 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002558876629889669, + "loss": 1.2527, + "theoretical_loss": 3.4250282907599656, + "tokens_seen": 2255618048 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002558716148445336, + "loss": 1.2668, + "theoretical_loss": 3.4250202559673575, + "tokens_seen": 2255683584 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002558555667001003, + "loss": 1.3108, + "theoretical_loss": 3.4250122214735486, + "tokens_seen": 2255749120 + }, + { + "epoch": 0.68, + "learning_rate": 0.000255839518555667, + "loss": 1.3033, + "theoretical_loss": 3.4250041872785175, + "tokens_seen": 2255814656 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002558234704112337, + "loss": 1.2607, + "theoretical_loss": 3.4249961533822457, + "tokens_seen": 2255880192 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002558074222668004, + "loss": 1.2957, + "theoretical_loss": 3.424988119784713, + "tokens_seen": 2255945728 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002557913741223671, + "loss": 1.2816, + "theoretical_loss": 3.4249800864859, + "tokens_seen": 2256011264 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002557753259779338, + "loss": 1.2041, + "theoretical_loss": 3.424972053485786, + "tokens_seen": 2256076800 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002557592778335005, + "loss": 1.3353, + "theoretical_loss": 3.4249640207843517, + "tokens_seen": 2256142336 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002557432296890672, + "loss": 1.2474, + "theoretical_loss": 3.424955988381577, + "tokens_seen": 2256207872 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002557271815446339, + "loss": 1.2486, + "theoretical_loss": 3.424947956277443, + "tokens_seen": 2256273408 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002557111334002006, + "loss": 1.3202, + "theoretical_loss": 3.4249399244719294, + "tokens_seen": 2256338944 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002556950852557673, + "loss": 1.304, + "theoretical_loss": 3.424931892965016, + "tokens_seen": 2256404480 + }, + { + "epoch": 0.68, + "learning_rate": 0.000255679037111334, + "loss": 1.3286, + "theoretical_loss": 3.4249238617566835, + "tokens_seen": 2256470016 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002556629889669007, + "loss": 1.1692, + "theoretical_loss": 3.4249158308469125, + "tokens_seen": 2256535552 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002556469408224674, + "loss": 1.1953, + "theoretical_loss": 3.4249078002356823, + "tokens_seen": 2256601088 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002556308926780341, + "loss": 1.347, + "theoretical_loss": 3.4248997699229737, + "tokens_seen": 2256666624 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002556148445336008, + "loss": 1.272, + "theoretical_loss": 3.424891739908767, + "tokens_seen": 2256732160 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002555987963891675, + "loss": 1.31, + "theoretical_loss": 3.424883710193042, + "tokens_seen": 2256797696 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002555827482447342, + "loss": 1.3133, + "theoretical_loss": 3.4248756807757794, + "tokens_seen": 2256863232 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002555667001003009, + "loss": 1.3467, + "theoretical_loss": 3.4248676516569594, + "tokens_seen": 2256928768 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002555506519558676, + "loss": 1.235, + "theoretical_loss": 3.424859622836562, + "tokens_seen": 2256994304 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002555346038114343, + "loss": 1.2618, + "theoretical_loss": 3.4248515943145676, + "tokens_seen": 2257059840 + }, + { + "epoch": 0.68, + "learning_rate": 0.000255518555667001, + "loss": 1.2116, + "theoretical_loss": 3.4248435660909564, + "tokens_seen": 2257125376 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002555025075225677, + "loss": 1.2972, + "theoretical_loss": 3.4248355381657083, + "tokens_seen": 2257190912 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002554864593781344, + "loss": 1.3585, + "theoretical_loss": 3.424827510538804, + "tokens_seen": 2257256448 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002554704112337011, + "loss": 1.2303, + "theoretical_loss": 3.424819483210224, + "tokens_seen": 2257321984 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002554543630892678, + "loss": 1.2153, + "theoretical_loss": 3.424811456179948, + "tokens_seen": 2257387520 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002554383149448345, + "loss": 1.2916, + "theoretical_loss": 3.4248034294479566, + "tokens_seen": 2257453056 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002554222668004012, + "loss": 1.2517, + "theoretical_loss": 3.4247954030142296, + "tokens_seen": 2257518592 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002554062186559679, + "loss": 1.2494, + "theoretical_loss": 3.4247873768787476, + "tokens_seen": 2257584128 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002553901705115346, + "loss": 1.18, + "theoretical_loss": 3.424779351041491, + "tokens_seen": 2257649664 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 755500, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2898519039154053, + "objective/train/theoretical_loss": 3.42477132550244, + "objective/train/tokens_used": 2278175200, + "theoretical_loss": 3.42477132550244, + "tokens_seen": 2257715200 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002553741223671013, + "loss": 1.3005, + "theoretical_loss": 3.42477132550244, + "tokens_seen": 2257715200 + }, + { + "epoch": 0.68, + "learning_rate": 0.000255358074222668, + "loss": 1.2849, + "theoretical_loss": 3.4247633002615743, + "tokens_seen": 2257780736 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002553420260782347, + "loss": 1.3798, + "theoretical_loss": 3.424755275318875, + "tokens_seen": 2257846272 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002553259779338014, + "loss": 1.2874, + "theoretical_loss": 3.424747250674322, + "tokens_seen": 2257911808 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002553099297893681, + "loss": 1.1557, + "theoretical_loss": 3.4247392263278953, + "tokens_seen": 2257977344 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002552938816449348, + "loss": 1.3045, + "theoretical_loss": 3.4247312022795757, + "tokens_seen": 2258042880 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002552778335005015, + "loss": 1.2919, + "theoretical_loss": 3.424723178529343, + "tokens_seen": 2258108416 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002552617853560682, + "loss": 1.1924, + "theoretical_loss": 3.4247151550771777, + "tokens_seen": 2258173952 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002552457372116349, + "loss": 1.2231, + "theoretical_loss": 3.42470713192306, + "tokens_seen": 2258239488 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002552296890672016, + "loss": 1.1726, + "theoretical_loss": 3.4246991090669705, + "tokens_seen": 2258305024 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002552136409227683, + "loss": 1.2395, + "theoretical_loss": 3.424691086508889, + "tokens_seen": 2258370560 + }, + { + "epoch": 0.68, + "learning_rate": 0.000255197592778335, + "loss": 1.2466, + "theoretical_loss": 3.424683064248796, + "tokens_seen": 2258436096 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002551815446339017, + "loss": 1.2591, + "theoretical_loss": 3.4246750422866716, + "tokens_seen": 2258501632 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002551654964894684, + "loss": 1.1827, + "theoretical_loss": 3.424667020622497, + "tokens_seen": 2258567168 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002551494483450351, + "loss": 1.1719, + "theoretical_loss": 3.424658999256251, + "tokens_seen": 2258632704 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002551334002006018, + "loss": 1.2197, + "theoretical_loss": 3.4246509781879153, + "tokens_seen": 2258698240 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002551173520561685, + "loss": 1.3275, + "theoretical_loss": 3.424642957417469, + "tokens_seen": 2258763776 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002551013039117352, + "loss": 1.2389, + "theoretical_loss": 3.424634936944893, + "tokens_seen": 2258829312 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002550852557673019, + "loss": 1.2772, + "theoretical_loss": 3.4246269167701677, + "tokens_seen": 2258894848 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002550692076228686, + "loss": 1.2107, + "theoretical_loss": 3.424618896893273, + "tokens_seen": 2258960384 + }, + { + "epoch": 0.68, + "learning_rate": 0.0002550531594784353, + "loss": 1.3469, + "theoretical_loss": 3.4246108773141897, + "tokens_seen": 2259025920 + }, + { + "epoch": 0.68, + "learning_rate": 0.000255037111334002, + "loss": 1.2449, + "theoretical_loss": 3.4246028580328973, + "tokens_seen": 2259091456 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025502106318956873, + "loss": 1.3107, + "theoretical_loss": 3.4245948390493774, + "tokens_seen": 2259156992 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025500501504513543, + "loss": 1.228, + "theoretical_loss": 3.424586820363609, + "tokens_seen": 2259222528 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025498896690070213, + "loss": 1.2238, + "theoretical_loss": 3.4245788019755734, + "tokens_seen": 2259288064 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025497291875626883, + "loss": 1.4206, + "theoretical_loss": 3.42457078388525, + "tokens_seen": 2259353600 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025495687061183553, + "loss": 1.2871, + "theoretical_loss": 3.4245627660926194, + "tokens_seen": 2259419136 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025494082246740223, + "loss": 1.239, + "theoretical_loss": 3.4245547485976626, + "tokens_seen": 2259484672 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025492477432296893, + "loss": 1.176, + "theoretical_loss": 3.4245467314003593, + "tokens_seen": 2259550208 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025490872617853563, + "loss": 1.3052, + "theoretical_loss": 3.42453871450069, + "tokens_seen": 2259615744 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025489267803410233, + "loss": 1.2952, + "theoretical_loss": 3.4245306978986347, + "tokens_seen": 2259681280 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025487662988966903, + "loss": 1.324, + "theoretical_loss": 3.424522681594174, + "tokens_seen": 2259746816 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025486058174523573, + "loss": 1.2491, + "theoretical_loss": 3.4245146655872882, + "tokens_seen": 2259812352 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025484453360080243, + "loss": 1.2126, + "theoretical_loss": 3.4245066498779577, + "tokens_seen": 2259877888 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025482848545636913, + "loss": 1.1547, + "theoretical_loss": 3.4244986344661625, + "tokens_seen": 2259943424 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025481243731193584, + "loss": 1.2935, + "theoretical_loss": 3.424490619351883, + "tokens_seen": 2260008960 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025479638916750254, + "loss": 1.1152, + "theoretical_loss": 3.4244826045351004, + "tokens_seen": 2260074496 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025478034102306924, + "loss": 1.3116, + "theoretical_loss": 3.4244745900157936, + "tokens_seen": 2260140032 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025476429287863594, + "loss": 1.2537, + "theoretical_loss": 3.4244665757939443, + "tokens_seen": 2260205568 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025474824473420264, + "loss": 1.2539, + "theoretical_loss": 3.424458561869532, + "tokens_seen": 2260271104 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025473219658976934, + "loss": 1.2467, + "theoretical_loss": 3.4244505482425365, + "tokens_seen": 2260336640 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025471614844533604, + "loss": 1.2374, + "theoretical_loss": 3.4244425349129397, + "tokens_seen": 2260402176 + }, + { + "epoch": 0.68, + "learning_rate": 0.00025470010030090274, + "loss": 1.3068, + "theoretical_loss": 3.4244345218807206, + "tokens_seen": 2260467712 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025468405215646944, + "loss": 1.2747, + "theoretical_loss": 3.42442650914586, + "tokens_seen": 2260533248 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025466800401203614, + "loss": 1.1944, + "theoretical_loss": 3.4244184967083386, + "tokens_seen": 2260598784 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025465195586760284, + "loss": 1.0917, + "theoretical_loss": 3.4244104845681367, + "tokens_seen": 2260664320 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025463590772316954, + "loss": 1.1865, + "theoretical_loss": 3.424402472725234, + "tokens_seen": 2260729856 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025461985957873624, + "loss": 1.1873, + "theoretical_loss": 3.4243944611796113, + "tokens_seen": 2260795392 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025460381143430294, + "loss": 1.2376, + "theoretical_loss": 3.424386449931249, + "tokens_seen": 2260860928 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025458776328986964, + "loss": 1.4568, + "theoretical_loss": 3.424378438980127, + "tokens_seen": 2260926464 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 756703, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1879603862762451, + "objective/train/theoretical_loss": 3.424370428326226, + "objective/train/tokens_used": 2281452000, + "theoretical_loss": 3.424370428326226, + "tokens_seen": 2260992000 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025457171514543634, + "loss": 1.1289, + "theoretical_loss": 3.424370428326226, + "tokens_seen": 2260992000 + }, + { + "epoch": 0.69, + "learning_rate": 0.000254555667001003, + "loss": 1.2832, + "theoretical_loss": 3.424362417969527, + "tokens_seen": 2261057536 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002545396188565697, + "loss": 1.3101, + "theoretical_loss": 3.424354407910009, + "tokens_seen": 2261123072 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002545235707121364, + "loss": 1.2765, + "theoretical_loss": 3.4243463981476534, + "tokens_seen": 2261188608 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002545075225677031, + "loss": 1.3419, + "theoretical_loss": 3.4243383886824406, + "tokens_seen": 2261254144 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002544914744232698, + "loss": 1.1994, + "theoretical_loss": 3.42433037951435, + "tokens_seen": 2261319680 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002544754262788365, + "loss": 1.217, + "theoretical_loss": 3.4243223706433628, + "tokens_seen": 2261385216 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002544593781344032, + "loss": 1.3251, + "theoretical_loss": 3.4243143620694587, + "tokens_seen": 2261450752 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002544433299899699, + "loss": 1.2569, + "theoretical_loss": 3.4243063537926193, + "tokens_seen": 2261516288 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002544272818455366, + "loss": 1.2834, + "theoretical_loss": 3.424298345812824, + "tokens_seen": 2261581824 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002544112337011033, + "loss": 1.1707, + "theoretical_loss": 3.424290338130053, + "tokens_seen": 2261647360 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025439518555667, + "loss": 1.2109, + "theoretical_loss": 3.4242823307442873, + "tokens_seen": 2261712896 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002543791374122367, + "loss": 1.2241, + "theoretical_loss": 3.424274323655507, + "tokens_seen": 2261778432 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002543630892678034, + "loss": 1.0865, + "theoretical_loss": 3.4242663168636924, + "tokens_seen": 2261843968 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002543470411233701, + "loss": 1.2122, + "theoretical_loss": 3.424258310368824, + "tokens_seen": 2261909504 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002543309929789368, + "loss": 1.2553, + "theoretical_loss": 3.424250304170882, + "tokens_seen": 2261975040 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002543149448345035, + "loss": 1.2152, + "theoretical_loss": 3.4242422982698475, + "tokens_seen": 2262040576 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002542988966900702, + "loss": 1.2306, + "theoretical_loss": 3.4242342926657, + "tokens_seen": 2262106112 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002542828485456369, + "loss": 1.2504, + "theoretical_loss": 3.42422628735842, + "tokens_seen": 2262171648 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002542668004012036, + "loss": 1.2626, + "theoretical_loss": 3.4242182823479883, + "tokens_seen": 2262237184 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002542507522567703, + "loss": 1.2583, + "theoretical_loss": 3.4242102776343852, + "tokens_seen": 2262302720 + }, + { + "epoch": 0.69, + "learning_rate": 0.000254234704112337, + "loss": 1.1822, + "theoretical_loss": 3.424202273217591, + "tokens_seen": 2262368256 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002542186559679037, + "loss": 1.2763, + "theoretical_loss": 3.424194269097586, + "tokens_seen": 2262433792 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002542026078234704, + "loss": 1.2583, + "theoretical_loss": 3.4241862652743507, + "tokens_seen": 2262499328 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002541865596790371, + "loss": 1.352, + "theoretical_loss": 3.4241782617478655, + "tokens_seen": 2262564864 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002541705115346038, + "loss": 1.2293, + "theoretical_loss": 3.424170258518111, + "tokens_seen": 2262630400 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002541544633901705, + "loss": 1.1916, + "theoretical_loss": 3.424162255585067, + "tokens_seen": 2262695936 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002541384152457372, + "loss": 1.2708, + "theoretical_loss": 3.4241542529487146, + "tokens_seen": 2262761472 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002541223671013039, + "loss": 1.227, + "theoretical_loss": 3.424146250609034, + "tokens_seen": 2262827008 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002541063189568706, + "loss": 1.225, + "theoretical_loss": 3.4241382485660052, + "tokens_seen": 2262892544 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002540902708124373, + "loss": 1.2593, + "theoretical_loss": 3.424130246819609, + "tokens_seen": 2262958080 + }, + { + "epoch": 0.69, + "learning_rate": 0.000254074222668004, + "loss": 1.2914, + "theoretical_loss": 3.4241222453698263, + "tokens_seen": 2263023616 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002540581745235707, + "loss": 1.3057, + "theoretical_loss": 3.4241142442166366, + "tokens_seen": 2263089152 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025404212637913747, + "loss": 1.1903, + "theoretical_loss": 3.4241062433600207, + "tokens_seen": 2263154688 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025402607823470417, + "loss": 1.2713, + "theoretical_loss": 3.424098242799959, + "tokens_seen": 2263220224 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025401003009027087, + "loss": 1.1481, + "theoretical_loss": 3.4240902425364323, + "tokens_seen": 2263285760 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025399398194583757, + "loss": 1.2993, + "theoretical_loss": 3.42408224256942, + "tokens_seen": 2263351296 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025397793380140427, + "loss": 1.2071, + "theoretical_loss": 3.4240742428989037, + "tokens_seen": 2263416832 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025396188565697097, + "loss": 1.2496, + "theoretical_loss": 3.424066243524863, + "tokens_seen": 2263482368 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025394583751253767, + "loss": 1.2413, + "theoretical_loss": 3.424058244447279, + "tokens_seen": 2263547904 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002539297893681043, + "loss": 1.1994, + "theoretical_loss": 3.4240502456661313, + "tokens_seen": 2263613440 + }, + { + "epoch": 0.69, + "learning_rate": 0.000253913741223671, + "loss": 1.1629, + "theoretical_loss": 3.424042247181401, + "tokens_seen": 2263678976 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002538976930792377, + "loss": 1.3103, + "theoretical_loss": 3.4240342489930686, + "tokens_seen": 2263744512 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002538816449348044, + "loss": 1.2784, + "theoretical_loss": 3.4240262511011137, + "tokens_seen": 2263810048 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002538655967903711, + "loss": 1.1305, + "theoretical_loss": 3.4240182535055177, + "tokens_seen": 2263875584 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002538495486459378, + "loss": 1.2864, + "theoretical_loss": 3.4240102562062606, + "tokens_seen": 2263941120 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002538335005015045, + "loss": 1.1519, + "theoretical_loss": 3.424002259203323, + "tokens_seen": 2264006656 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002538174523570712, + "loss": 1.1906, + "theoretical_loss": 3.4239942624966853, + "tokens_seen": 2264072192 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002538014042126379, + "loss": 1.4173, + "theoretical_loss": 3.4239862660863274, + "tokens_seen": 2264137728 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002537853560682046, + "loss": 1.1836, + "theoretical_loss": 3.4239782699722308, + "tokens_seen": 2264203264 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 757625, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2602403163909912, + "objective/train/theoretical_loss": 3.423970274154375, + "objective/train/tokens_used": 2284728800, + "theoretical_loss": 3.423970274154375, + "tokens_seen": 2264268800 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002537693079237713, + "loss": 1.2912, + "theoretical_loss": 3.423970274154375, + "tokens_seen": 2264268800 + }, + { + "epoch": 0.69, + "learning_rate": 0.000253753259779338, + "loss": 1.1354, + "theoretical_loss": 3.4239622786327413, + "tokens_seen": 2264334336 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002537372116349047, + "loss": 1.157, + "theoretical_loss": 3.4239542834073093, + "tokens_seen": 2264399872 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002537211634904714, + "loss": 1.3084, + "theoretical_loss": 3.4239462884780596, + "tokens_seen": 2264465408 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002537051153460381, + "loss": 1.2993, + "theoretical_loss": 3.423938293844973, + "tokens_seen": 2264530944 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002536890672016048, + "loss": 1.2535, + "theoretical_loss": 3.4239302995080303, + "tokens_seen": 2264596480 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002536730190571715, + "loss": 1.2121, + "theoretical_loss": 3.423922305467211, + "tokens_seen": 2264662016 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002536569709127382, + "loss": 1.2473, + "theoretical_loss": 3.4239143117224966, + "tokens_seen": 2264727552 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025364092276830493, + "loss": 1.2107, + "theoretical_loss": 3.4239063182738665, + "tokens_seen": 2264793088 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025362487462387163, + "loss": 1.2191, + "theoretical_loss": 3.4238983251213018, + "tokens_seen": 2264858624 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025360882647943833, + "loss": 1.2205, + "theoretical_loss": 3.423890332264783, + "tokens_seen": 2264924160 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025359277833500503, + "loss": 1.1555, + "theoretical_loss": 3.4238823397042903, + "tokens_seen": 2264989696 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025357673019057173, + "loss": 1.2833, + "theoretical_loss": 3.4238743474398046, + "tokens_seen": 2265055232 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025356068204613843, + "loss": 1.2103, + "theoretical_loss": 3.423866355471306, + "tokens_seen": 2265120768 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025354463390170513, + "loss": 1.1955, + "theoretical_loss": 3.423858363798775, + "tokens_seen": 2265186304 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025352858575727183, + "loss": 1.2207, + "theoretical_loss": 3.4238503724221925, + "tokens_seen": 2265251840 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025351253761283853, + "loss": 1.2607, + "theoretical_loss": 3.423842381341538, + "tokens_seen": 2265317376 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025349648946840523, + "loss": 1.2788, + "theoretical_loss": 3.423834390556793, + "tokens_seen": 2265382912 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025348044132397193, + "loss": 1.2468, + "theoretical_loss": 3.423826400067937, + "tokens_seen": 2265448448 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025346439317953863, + "loss": 1.2413, + "theoretical_loss": 3.423818409874952, + "tokens_seen": 2265513984 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025344834503510534, + "loss": 1.3598, + "theoretical_loss": 3.423810419977817, + "tokens_seen": 2265579520 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025343229689067204, + "loss": 1.2453, + "theoretical_loss": 3.423802430376513, + "tokens_seen": 2265645056 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025341624874623874, + "loss": 1.2103, + "theoretical_loss": 3.4237944410710206, + "tokens_seen": 2265710592 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025340020060180544, + "loss": 1.2374, + "theoretical_loss": 3.42378645206132, + "tokens_seen": 2265776128 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025338415245737214, + "loss": 1.2984, + "theoretical_loss": 3.4237784633473924, + "tokens_seen": 2265841664 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025336810431293884, + "loss": 1.1565, + "theoretical_loss": 3.4237704749292175, + "tokens_seen": 2265907200 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025335205616850554, + "loss": 1.327, + "theoretical_loss": 3.423762486806776, + "tokens_seen": 2265972736 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025333600802407224, + "loss": 1.2541, + "theoretical_loss": 3.423754498980049, + "tokens_seen": 2266038272 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025331995987963894, + "loss": 1.1624, + "theoretical_loss": 3.423746511449016, + "tokens_seen": 2266103808 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025330391173520564, + "loss": 1.2236, + "theoretical_loss": 3.4237385242136584, + "tokens_seen": 2266169344 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025328786359077234, + "loss": 1.2687, + "theoretical_loss": 3.423730537273956, + "tokens_seen": 2266234880 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025327181544633904, + "loss": 1.1889, + "theoretical_loss": 3.42372255062989, + "tokens_seen": 2266300416 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025325576730190574, + "loss": 1.2716, + "theoretical_loss": 3.42371456428144, + "tokens_seen": 2266365952 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025323971915747244, + "loss": 1.1959, + "theoretical_loss": 3.4237065782285874, + "tokens_seen": 2266431488 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025322367101303914, + "loss": 1.1599, + "theoretical_loss": 3.4236985924713124, + "tokens_seen": 2266497024 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025320762286860584, + "loss": 1.1607, + "theoretical_loss": 3.423690607009595, + "tokens_seen": 2266562560 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025319157472417255, + "loss": 1.2472, + "theoretical_loss": 3.4236826218434167, + "tokens_seen": 2266628096 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025317552657973925, + "loss": 1.3503, + "theoretical_loss": 3.4236746369727573, + "tokens_seen": 2266693632 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002531594784353059, + "loss": 1.2719, + "theoretical_loss": 3.4236666523975976, + "tokens_seen": 2266759168 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002531434302908726, + "loss": 1.226, + "theoretical_loss": 3.423658668117918, + "tokens_seen": 2266824704 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002531273821464393, + "loss": 1.283, + "theoretical_loss": 3.423650684133699, + "tokens_seen": 2266890240 + }, + { + "epoch": 0.69, + "learning_rate": 0.000253111334002006, + "loss": 1.2581, + "theoretical_loss": 3.423642700444921, + "tokens_seen": 2266955776 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002530952858575727, + "loss": 1.1902, + "theoretical_loss": 3.423634717051565, + "tokens_seen": 2267021312 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002530792377131394, + "loss": 1.1994, + "theoretical_loss": 3.4236267339536113, + "tokens_seen": 2267086848 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002530631895687061, + "loss": 1.2128, + "theoretical_loss": 3.4236187511510403, + "tokens_seen": 2267152384 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002530471414242728, + "loss": 1.2073, + "theoretical_loss": 3.423610768643832, + "tokens_seen": 2267217920 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002530310932798395, + "loss": 1.2642, + "theoretical_loss": 3.4236027864319682, + "tokens_seen": 2267283456 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002530150451354062, + "loss": 1.1769, + "theoretical_loss": 3.423594804515429, + "tokens_seen": 2267348992 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002529989969909729, + "loss": 1.301, + "theoretical_loss": 3.4235868228941944, + "tokens_seen": 2267414528 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002529829488465396, + "loss": 1.1972, + "theoretical_loss": 3.4235788415682453, + "tokens_seen": 2267480064 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 758824, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.331571340560913, + "objective/train/theoretical_loss": 3.423570860537562, + "objective/train/tokens_used": 2288005600, + "theoretical_loss": 3.423570860537562, + "tokens_seen": 2267545600 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002529669007021063, + "loss": 1.3733, + "theoretical_loss": 3.423570860537562, + "tokens_seen": 2267545600 + }, + { + "epoch": 0.69, + "learning_rate": 0.000252950852557673, + "loss": 1.0821, + "theoretical_loss": 3.4235628798021254, + "tokens_seen": 2267611136 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002529348044132397, + "loss": 1.2548, + "theoretical_loss": 3.423554899361916, + "tokens_seen": 2267676672 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002529187562688064, + "loss": 1.2521, + "theoretical_loss": 3.423546919216914, + "tokens_seen": 2267742208 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002529027081243731, + "loss": 1.2387, + "theoretical_loss": 3.4235389393671003, + "tokens_seen": 2267807744 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002528866599799398, + "loss": 1.2203, + "theoretical_loss": 3.4235309598124553, + "tokens_seen": 2267873280 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002528706118355065, + "loss": 1.2548, + "theoretical_loss": 3.4235229805529594, + "tokens_seen": 2267938816 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002528545636910732, + "loss": 1.2381, + "theoretical_loss": 3.423515001588594, + "tokens_seen": 2268004352 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025283851554663996, + "loss": 1.2329, + "theoretical_loss": 3.4235070229193383, + "tokens_seen": 2268069888 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025282246740220666, + "loss": 1.2003, + "theoretical_loss": 3.423499044545174, + "tokens_seen": 2268135424 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025280641925777336, + "loss": 1.2083, + "theoretical_loss": 3.423491066466081, + "tokens_seen": 2268200960 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025279037111334006, + "loss": 1.1842, + "theoretical_loss": 3.42348308868204, + "tokens_seen": 2268266496 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025277432296890676, + "loss": 1.1414, + "theoretical_loss": 3.4234751111930315, + "tokens_seen": 2268332032 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025275827482447346, + "loss": 1.2506, + "theoretical_loss": 3.4234671339990363, + "tokens_seen": 2268397568 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025274222668004016, + "loss": 1.2079, + "theoretical_loss": 3.4234591571000355, + "tokens_seen": 2268463104 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025272617853560686, + "loss": 1.2527, + "theoretical_loss": 3.4234511804960084, + "tokens_seen": 2268528640 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025271013039117356, + "loss": 1.2668, + "theoretical_loss": 3.4234432041869365, + "tokens_seen": 2268594176 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025269408224674026, + "loss": 1.2663, + "theoretical_loss": 3.4234352281727998, + "tokens_seen": 2268659712 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025267803410230697, + "loss": 1.3754, + "theoretical_loss": 3.423427252453579, + "tokens_seen": 2268725248 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025266198595787367, + "loss": 1.209, + "theoretical_loss": 3.4234192770292555, + "tokens_seen": 2268790784 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025264593781344037, + "loss": 1.2915, + "theoretical_loss": 3.423411301899809, + "tokens_seen": 2268856320 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025262988966900707, + "loss": 1.2496, + "theoretical_loss": 3.4234033270652198, + "tokens_seen": 2268921856 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025261384152457377, + "loss": 1.227, + "theoretical_loss": 3.4233953525254694, + "tokens_seen": 2268987392 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025259779338014047, + "loss": 1.2188, + "theoretical_loss": 3.423387378280538, + "tokens_seen": 2269052928 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025258174523570717, + "loss": 1.2183, + "theoretical_loss": 3.423379404330406, + "tokens_seen": 2269118464 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025256569709127387, + "loss": 1.1318, + "theoretical_loss": 3.4233714306750542, + "tokens_seen": 2269184000 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025254964894684057, + "loss": 1.2962, + "theoretical_loss": 3.4233634573144633, + "tokens_seen": 2269249536 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025253360080240727, + "loss": 1.2614, + "theoretical_loss": 3.4233554842486136, + "tokens_seen": 2269315072 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002525175526579739, + "loss": 1.1076, + "theoretical_loss": 3.423347511477486, + "tokens_seen": 2269380608 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002525015045135406, + "loss": 1.2734, + "theoretical_loss": 3.4233395390010606, + "tokens_seen": 2269446144 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002524854563691073, + "loss": 1.2971, + "theoretical_loss": 3.4233315668193183, + "tokens_seen": 2269511680 + }, + { + "epoch": 0.69, + "learning_rate": 0.000252469408224674, + "loss": 1.2631, + "theoretical_loss": 3.42332359493224, + "tokens_seen": 2269577216 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002524533600802407, + "loss": 1.2931, + "theoretical_loss": 3.4233156233398057, + "tokens_seen": 2269642752 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002524373119358074, + "loss": 1.2804, + "theoretical_loss": 3.4233076520419963, + "tokens_seen": 2269708288 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002524212637913741, + "loss": 1.3802, + "theoretical_loss": 3.4232996810387926, + "tokens_seen": 2269773824 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002524052156469408, + "loss": 1.1924, + "theoretical_loss": 3.423291710330175, + "tokens_seen": 2269839360 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002523891675025075, + "loss": 1.1484, + "theoretical_loss": 3.423283739916124, + "tokens_seen": 2269904896 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002523731193580742, + "loss": 1.3077, + "theoretical_loss": 3.42327576979662, + "tokens_seen": 2269970432 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002523570712136409, + "loss": 1.349, + "theoretical_loss": 3.4232677999716445, + "tokens_seen": 2270035968 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002523410230692076, + "loss": 1.3718, + "theoretical_loss": 3.4232598304411774, + "tokens_seen": 2270101504 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002523249749247743, + "loss": 1.3387, + "theoretical_loss": 3.4232518612051996, + "tokens_seen": 2270167040 + }, + { + "epoch": 0.69, + "learning_rate": 0.000252308926780341, + "loss": 1.2438, + "theoretical_loss": 3.4232438922636916, + "tokens_seen": 2270232576 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002522928786359077, + "loss": 1.2954, + "theoretical_loss": 3.423235923616634, + "tokens_seen": 2270298112 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025227683049147443, + "loss": 1.2451, + "theoretical_loss": 3.423227955264007, + "tokens_seen": 2270363648 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025226078234704113, + "loss": 1.3469, + "theoretical_loss": 3.4232199872057922, + "tokens_seen": 2270429184 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025224473420260783, + "loss": 1.117, + "theoretical_loss": 3.4232120194419693, + "tokens_seen": 2270494720 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025222868605817453, + "loss": 1.2371, + "theoretical_loss": 3.4232040519725193, + "tokens_seen": 2270560256 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025221263791374123, + "loss": 1.2156, + "theoretical_loss": 3.423196084797423, + "tokens_seen": 2270625792 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025219658976930793, + "loss": 1.2532, + "theoretical_loss": 3.423188117916661, + "tokens_seen": 2270691328 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025218054162487463, + "loss": 1.2504, + "theoretical_loss": 3.4231801513302136, + "tokens_seen": 2270756864 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 759696, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3452606201171875, + "objective/train/theoretical_loss": 3.4231721850380614, + "objective/train/tokens_used": 2291282400, + "theoretical_loss": 3.4231721850380614, + "tokens_seen": 2270822400 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025216449348044133, + "loss": 1.3798, + "theoretical_loss": 3.4231721850380614, + "tokens_seen": 2270822400 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025214844533600803, + "loss": 1.1902, + "theoretical_loss": 3.423164219040186, + "tokens_seen": 2270887936 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025213239719157473, + "loss": 1.2556, + "theoretical_loss": 3.4231562533365665, + "tokens_seen": 2270953472 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025211634904714143, + "loss": 1.3508, + "theoretical_loss": 3.4231482879271846, + "tokens_seen": 2271019008 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025210030090270813, + "loss": 1.1938, + "theoretical_loss": 3.4231403228120207, + "tokens_seen": 2271084544 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025208425275827483, + "loss": 1.2114, + "theoretical_loss": 3.4231323579910553, + "tokens_seen": 2271150080 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025206820461384154, + "loss": 1.2922, + "theoretical_loss": 3.4231243934642697, + "tokens_seen": 2271215616 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025205215646940824, + "loss": 1.2219, + "theoretical_loss": 3.4231164292316434, + "tokens_seen": 2271281152 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025203610832497494, + "loss": 1.2933, + "theoretical_loss": 3.423108465293158, + "tokens_seen": 2271346688 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025202006018054164, + "loss": 1.3095, + "theoretical_loss": 3.423100501648794, + "tokens_seen": 2271412224 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025200401203610834, + "loss": 1.254, + "theoretical_loss": 3.4230925382985316, + "tokens_seen": 2271477760 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025198796389167504, + "loss": 1.2416, + "theoretical_loss": 3.4230845752423518, + "tokens_seen": 2271543296 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025197191574724174, + "loss": 1.2451, + "theoretical_loss": 3.4230766124802345, + "tokens_seen": 2271608832 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025195586760280844, + "loss": 1.3325, + "theoretical_loss": 3.423068650012162, + "tokens_seen": 2271674368 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025193981945837514, + "loss": 1.2924, + "theoretical_loss": 3.4230606878381136, + "tokens_seen": 2271739904 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025192377131394184, + "loss": 1.2111, + "theoretical_loss": 3.4230527259580703, + "tokens_seen": 2271805440 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025190772316950854, + "loss": 1.2447, + "theoretical_loss": 3.4230447643720128, + "tokens_seen": 2271870976 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025189167502507524, + "loss": 1.2762, + "theoretical_loss": 3.4230368030799223, + "tokens_seen": 2271936512 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025187562688064194, + "loss": 1.2232, + "theoretical_loss": 3.423028842081778, + "tokens_seen": 2272002048 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025185957873620864, + "loss": 1.305, + "theoretical_loss": 3.423020881377562, + "tokens_seen": 2272067584 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025184353059177534, + "loss": 1.2474, + "theoretical_loss": 3.423012920967255, + "tokens_seen": 2272133120 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025182748244734204, + "loss": 1.2309, + "theoretical_loss": 3.4230049608508364, + "tokens_seen": 2272198656 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025181143430290875, + "loss": 1.2731, + "theoretical_loss": 3.422997001028288, + "tokens_seen": 2272264192 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025179538615847545, + "loss": 1.2984, + "theoretical_loss": 3.42298904149959, + "tokens_seen": 2272329728 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025177933801404215, + "loss": 1.3255, + "theoretical_loss": 3.422981082264723, + "tokens_seen": 2272395264 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025176328986960885, + "loss": 1.206, + "theoretical_loss": 3.4229731233236684, + "tokens_seen": 2272460800 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002517472417251755, + "loss": 1.1753, + "theoretical_loss": 3.422965164676406, + "tokens_seen": 2272526336 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002517311935807422, + "loss": 1.3235, + "theoretical_loss": 3.422957206322917, + "tokens_seen": 2272591872 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002517151454363089, + "loss": 1.1814, + "theoretical_loss": 3.4229492482631816, + "tokens_seen": 2272657408 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002516990972918756, + "loss": 1.2417, + "theoretical_loss": 3.4229412904971808, + "tokens_seen": 2272722944 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002516830491474423, + "loss": 1.295, + "theoretical_loss": 3.422933333024895, + "tokens_seen": 2272788480 + }, + { + "epoch": 0.69, + "learning_rate": 0.000251667001003009, + "loss": 1.2484, + "theoretical_loss": 3.4229253758463063, + "tokens_seen": 2272854016 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025165095285857575, + "loss": 1.2029, + "theoretical_loss": 3.4229174189613936, + "tokens_seen": 2272919552 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025163490471414245, + "loss": 1.2615, + "theoretical_loss": 3.422909462370138, + "tokens_seen": 2272985088 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025161885656970915, + "loss": 1.1895, + "theoretical_loss": 3.4229015060725207, + "tokens_seen": 2273050624 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025160280842527585, + "loss": 1.2615, + "theoretical_loss": 3.4228935500685225, + "tokens_seen": 2273116160 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025158676028084255, + "loss": 1.288, + "theoretical_loss": 3.422885594358123, + "tokens_seen": 2273181696 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025157071213640926, + "loss": 1.2891, + "theoretical_loss": 3.422877638941304, + "tokens_seen": 2273247232 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025155466399197596, + "loss": 1.3102, + "theoretical_loss": 3.422869683818046, + "tokens_seen": 2273312768 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025153861584754266, + "loss": 1.2433, + "theoretical_loss": 3.42286172898833, + "tokens_seen": 2273378304 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025152256770310936, + "loss": 1.2624, + "theoretical_loss": 3.4228537744521357, + "tokens_seen": 2273443840 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025150651955867606, + "loss": 1.2897, + "theoretical_loss": 3.4228458202094445, + "tokens_seen": 2273509376 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025149047141424276, + "loss": 1.2954, + "theoretical_loss": 3.422837866260237, + "tokens_seen": 2273574912 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025147442326980946, + "loss": 1.2628, + "theoretical_loss": 3.422829912604494, + "tokens_seen": 2273640448 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025145837512537616, + "loss": 1.3347, + "theoretical_loss": 3.422821959242196, + "tokens_seen": 2273705984 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025144232698094286, + "loss": 1.3089, + "theoretical_loss": 3.4228140061733234, + "tokens_seen": 2273771520 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025142627883650956, + "loss": 1.2281, + "theoretical_loss": 3.422806053397858, + "tokens_seen": 2273837056 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025141023069207626, + "loss": 1.2387, + "theoretical_loss": 3.4227981009157795, + "tokens_seen": 2273902592 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025139418254764296, + "loss": 1.2452, + "theoretical_loss": 3.4227901487270693, + "tokens_seen": 2273968128 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025137813440320966, + "loss": 1.3538, + "theoretical_loss": 3.4227821968317076, + "tokens_seen": 2274033664 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 761000, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.278111219406128, + "objective/train/theoretical_loss": 3.422774245229675, + "objective/train/tokens_used": 2294559200, + "theoretical_loss": 3.422774245229675, + "tokens_seen": 2274099200 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025136208625877636, + "loss": 1.2351, + "theoretical_loss": 3.422774245229675, + "tokens_seen": 2274099200 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025134603811434306, + "loss": 1.1902, + "theoretical_loss": 3.4227662939209536, + "tokens_seen": 2274164736 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025132998996990976, + "loss": 1.2983, + "theoretical_loss": 3.422758342905522, + "tokens_seen": 2274230272 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025131394182547647, + "loss": 1.3415, + "theoretical_loss": 3.422750392183363, + "tokens_seen": 2274295808 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025129789368104317, + "loss": 1.2504, + "theoretical_loss": 3.4227424417544556, + "tokens_seen": 2274361344 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025128184553660987, + "loss": 1.2615, + "theoretical_loss": 3.422734491618782, + "tokens_seen": 2274426880 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025126579739217657, + "loss": 1.2974, + "theoretical_loss": 3.4227265417763215, + "tokens_seen": 2274492416 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025124974924774327, + "loss": 1.2285, + "theoretical_loss": 3.422718592227056, + "tokens_seen": 2274557952 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025123370110330997, + "loss": 1.2849, + "theoretical_loss": 3.4227106429709657, + "tokens_seen": 2274623488 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025121765295887667, + "loss": 1.2496, + "theoretical_loss": 3.4227026940080316, + "tokens_seen": 2274689024 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025120160481444337, + "loss": 1.1948, + "theoretical_loss": 3.4226947453382337, + "tokens_seen": 2274754560 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025118555667001007, + "loss": 1.2033, + "theoretical_loss": 3.4226867969615538, + "tokens_seen": 2274820096 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025116950852557677, + "loss": 1.1145, + "theoretical_loss": 3.4226788488779722, + "tokens_seen": 2274885632 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025115346038114347, + "loss": 1.2668, + "theoretical_loss": 3.4226709010874696, + "tokens_seen": 2274951168 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025113741223671017, + "loss": 1.2375, + "theoretical_loss": 3.422662953590027, + "tokens_seen": 2275016704 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002511213640922768, + "loss": 1.3184, + "theoretical_loss": 3.4226550063856247, + "tokens_seen": 2275082240 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002511053159478435, + "loss": 1.3255, + "theoretical_loss": 3.4226470594742437, + "tokens_seen": 2275147776 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002510892678034102, + "loss": 1.2455, + "theoretical_loss": 3.4226391128558644, + "tokens_seen": 2275213312 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002510732196589769, + "loss": 1.2343, + "theoretical_loss": 3.4226311665304685, + "tokens_seen": 2275278848 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002510571715145436, + "loss": 1.2811, + "theoretical_loss": 3.422623220498036, + "tokens_seen": 2275344384 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002510411233701103, + "loss": 1.1684, + "theoretical_loss": 3.4226152747585474, + "tokens_seen": 2275409920 + }, + { + "epoch": 0.69, + "learning_rate": 0.000251025075225677, + "loss": 1.2704, + "theoretical_loss": 3.4226073293119845, + "tokens_seen": 2275475456 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002510090270812437, + "loss": 1.264, + "theoretical_loss": 3.4225993841583273, + "tokens_seen": 2275540992 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002509929789368104, + "loss": 1.237, + "theoretical_loss": 3.4225914392975563, + "tokens_seen": 2275606528 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002509769307923771, + "loss": 1.2718, + "theoretical_loss": 3.422583494729653, + "tokens_seen": 2275672064 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002509608826479438, + "loss": 1.3219, + "theoretical_loss": 3.422575550454598, + "tokens_seen": 2275737600 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002509448345035105, + "loss": 1.2766, + "theoretical_loss": 3.422567606472372, + "tokens_seen": 2275803136 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002509287863590772, + "loss": 1.2475, + "theoretical_loss": 3.4225596627829553, + "tokens_seen": 2275868672 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002509127382146439, + "loss": 1.1821, + "theoretical_loss": 3.422551719386329, + "tokens_seen": 2275934208 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025089669007021063, + "loss": 1.2273, + "theoretical_loss": 3.4225437762824744, + "tokens_seen": 2275999744 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025088064192577733, + "loss": 1.2221, + "theoretical_loss": 3.422535833471372, + "tokens_seen": 2276065280 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025086459378134403, + "loss": 1.1338, + "theoretical_loss": 3.4225278909530017, + "tokens_seen": 2276130816 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025084854563691073, + "loss": 1.2709, + "theoretical_loss": 3.4225199487273454, + "tokens_seen": 2276196352 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025083249749247743, + "loss": 1.2279, + "theoretical_loss": 3.4225120067943835, + "tokens_seen": 2276261888 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025081644934804413, + "loss": 1.2021, + "theoretical_loss": 3.4225040651540968, + "tokens_seen": 2276327424 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025080040120361083, + "loss": 1.1562, + "theoretical_loss": 3.422496123806466, + "tokens_seen": 2276392960 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025078435305917753, + "loss": 1.2187, + "theoretical_loss": 3.422488182751472, + "tokens_seen": 2276458496 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025076830491474423, + "loss": 1.2989, + "theoretical_loss": 3.4224802419890956, + "tokens_seen": 2276524032 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025075225677031093, + "loss": 1.2342, + "theoretical_loss": 3.422472301519317, + "tokens_seen": 2276589568 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025073620862587763, + "loss": 1.1945, + "theoretical_loss": 3.422464361342118, + "tokens_seen": 2276655104 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025072016048144433, + "loss": 1.2482, + "theoretical_loss": 3.422456421457479, + "tokens_seen": 2276720640 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025070411233701104, + "loss": 1.2803, + "theoretical_loss": 3.4224484818653806, + "tokens_seen": 2276786176 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025068806419257774, + "loss": 1.2759, + "theoretical_loss": 3.422440542565804, + "tokens_seen": 2276851712 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025067201604814444, + "loss": 1.2826, + "theoretical_loss": 3.42243260355873, + "tokens_seen": 2276917248 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025065596790371114, + "loss": 1.2774, + "theoretical_loss": 3.4224246648441383, + "tokens_seen": 2276982784 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025063991975927784, + "loss": 1.2701, + "theoretical_loss": 3.422416726422011, + "tokens_seen": 2277048320 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025062387161484454, + "loss": 1.1897, + "theoretical_loss": 3.422408788292328, + "tokens_seen": 2277113856 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025060782347041124, + "loss": 1.2243, + "theoretical_loss": 3.4224008504550714, + "tokens_seen": 2277179392 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025059177532597794, + "loss": 1.184, + "theoretical_loss": 3.4223929129102206, + "tokens_seen": 2277244928 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025057572718154464, + "loss": 1.3507, + "theoretical_loss": 3.4223849756577573, + "tokens_seen": 2277310464 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 761954, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1957485675811768, + "objective/train/theoretical_loss": 3.422377038697662, + "objective/train/tokens_used": 2297836000, + "theoretical_loss": 3.422377038697662, + "tokens_seen": 2277376000 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025055967903711134, + "loss": 1.2288, + "theoretical_loss": 3.422377038697662, + "tokens_seen": 2277376000 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025054363089267804, + "loss": 1.3385, + "theoretical_loss": 3.422369102029915, + "tokens_seen": 2277441536 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025052758274824474, + "loss": 1.1808, + "theoretical_loss": 3.422361165654498, + "tokens_seen": 2277507072 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025051153460381144, + "loss": 1.1632, + "theoretical_loss": 3.422353229571392, + "tokens_seen": 2277572608 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025049548645937814, + "loss": 1.3477, + "theoretical_loss": 3.4223452937805767, + "tokens_seen": 2277638144 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025047943831494484, + "loss": 1.3277, + "theoretical_loss": 3.422337358282033, + "tokens_seen": 2277703680 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025046339017051154, + "loss": 1.2989, + "theoretical_loss": 3.4223294230757433, + "tokens_seen": 2277769216 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025044734202607825, + "loss": 1.3211, + "theoretical_loss": 3.422321488161687, + "tokens_seen": 2277834752 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025043129388164495, + "loss": 1.3001, + "theoretical_loss": 3.422313553539845, + "tokens_seen": 2277900288 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025041524573721165, + "loss": 1.259, + "theoretical_loss": 3.422305619210199, + "tokens_seen": 2277965824 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025039919759277835, + "loss": 1.2461, + "theoretical_loss": 3.4222976851727287, + "tokens_seen": 2278031360 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025038314944834505, + "loss": 1.25, + "theoretical_loss": 3.4222897514274155, + "tokens_seen": 2278096896 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025036710130391175, + "loss": 1.2492, + "theoretical_loss": 3.4222818179742407, + "tokens_seen": 2278162432 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025035105315947845, + "loss": 1.1898, + "theoretical_loss": 3.4222738848131846, + "tokens_seen": 2278227968 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025033500501504515, + "loss": 1.2585, + "theoretical_loss": 3.422265951944228, + "tokens_seen": 2278293504 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025031895687061185, + "loss": 1.1689, + "theoretical_loss": 3.4222580193673515, + "tokens_seen": 2278359040 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025030290872617855, + "loss": 1.1538, + "theoretical_loss": 3.422250087082537, + "tokens_seen": 2278424576 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025028686058174525, + "loss": 1.1711, + "theoretical_loss": 3.4222421550897644, + "tokens_seen": 2278490112 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025027081243731195, + "loss": 1.2656, + "theoretical_loss": 3.4222342233890144, + "tokens_seen": 2278555648 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025025476429287865, + "loss": 1.1542, + "theoretical_loss": 3.422226291980269, + "tokens_seen": 2278621184 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025023871614844535, + "loss": 1.2361, + "theoretical_loss": 3.422218360863508, + "tokens_seen": 2278686720 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025022266800401205, + "loss": 1.2583, + "theoretical_loss": 3.422210430038712, + "tokens_seen": 2278752256 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025020661985957875, + "loss": 1.2362, + "theoretical_loss": 3.4222024995058633, + "tokens_seen": 2278817792 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025019057171514546, + "loss": 1.2091, + "theoretical_loss": 3.422194569264941, + "tokens_seen": 2278883328 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025017452357071216, + "loss": 1.2911, + "theoretical_loss": 3.4221866393159273, + "tokens_seen": 2278948864 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025015847542627886, + "loss": 1.2593, + "theoretical_loss": 3.422178709658803, + "tokens_seen": 2279014400 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025014242728184556, + "loss": 1.2402, + "theoretical_loss": 3.422170780293548, + "tokens_seen": 2279079936 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025012637913741226, + "loss": 1.2509, + "theoretical_loss": 3.422162851220144, + "tokens_seen": 2279145472 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025011033099297896, + "loss": 1.2829, + "theoretical_loss": 3.4221549224385717, + "tokens_seen": 2279211008 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025009428284854566, + "loss": 1.3057, + "theoretical_loss": 3.422146993948811, + "tokens_seen": 2279276544 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025007823470411236, + "loss": 1.3934, + "theoretical_loss": 3.4221390657508444, + "tokens_seen": 2279342080 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025006218655967906, + "loss": 1.2668, + "theoretical_loss": 3.422131137844652, + "tokens_seen": 2279407616 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025004613841524576, + "loss": 1.243, + "theoretical_loss": 3.4221232102302146, + "tokens_seen": 2279473152 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025003009027081246, + "loss": 1.227, + "theoretical_loss": 3.422115282907513, + "tokens_seen": 2279538688 + }, + { + "epoch": 0.69, + "learning_rate": 0.00025001404212637916, + "loss": 1.2628, + "theoretical_loss": 3.4221073558765283, + "tokens_seen": 2279604224 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024999799398194586, + "loss": 1.2096, + "theoretical_loss": 3.422099429137241, + "tokens_seen": 2279669760 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024998194583751256, + "loss": 1.1972, + "theoretical_loss": 3.422091502689633, + "tokens_seen": 2279735296 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024996589769307926, + "loss": 1.2533, + "theoretical_loss": 3.4220835765336837, + "tokens_seen": 2279800832 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024994984954864597, + "loss": 1.2317, + "theoretical_loss": 3.4220756506693752, + "tokens_seen": 2279866368 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024993380140421267, + "loss": 1.1989, + "theoretical_loss": 3.4220677250966878, + "tokens_seen": 2279931904 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024991775325977937, + "loss": 1.256, + "theoretical_loss": 3.4220597998156026, + "tokens_seen": 2279997440 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024990170511534607, + "loss": 1.1136, + "theoretical_loss": 3.4220518748261, + "tokens_seen": 2280062976 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024988565697091277, + "loss": 1.2905, + "theoretical_loss": 3.4220439501281623, + "tokens_seen": 2280128512 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024986960882647947, + "loss": 1.2216, + "theoretical_loss": 3.4220360257217686, + "tokens_seen": 2280194048 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024985356068204617, + "loss": 1.2576, + "theoretical_loss": 3.4220281016069007, + "tokens_seen": 2280259584 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024983751253761287, + "loss": 1.2055, + "theoretical_loss": 3.4220201777835397, + "tokens_seen": 2280325120 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024982146439317957, + "loss": 1.1951, + "theoretical_loss": 3.422012254251666, + "tokens_seen": 2280390656 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024980541624874627, + "loss": 1.2612, + "theoretical_loss": 3.4220043310112604, + "tokens_seen": 2280456192 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024978936810431297, + "loss": 1.1299, + "theoretical_loss": 3.421996408062305, + "tokens_seen": 2280521728 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024977331995987967, + "loss": 1.4156, + "theoretical_loss": 3.421988485404779, + "tokens_seen": 2280587264 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 763124, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3014075756072998, + "objective/train/theoretical_loss": 3.421980563038664, + "objective/train/tokens_used": 2301112800, + "theoretical_loss": 3.421980563038664, + "tokens_seen": 2280652800 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024975727181544637, + "loss": 1.3021, + "theoretical_loss": 3.421980563038664, + "tokens_seen": 2280652800 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002497412236710131, + "loss": 1.2063, + "theoretical_loss": 3.4219726409639413, + "tokens_seen": 2280718336 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002497251755265797, + "loss": 1.2816, + "theoretical_loss": 3.4219647191805915, + "tokens_seen": 2280783872 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002497091273821464, + "loss": 1.243, + "theoretical_loss": 3.421956797688596, + "tokens_seen": 2280849408 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002496930792377131, + "loss": 1.207, + "theoretical_loss": 3.4219488764879347, + "tokens_seen": 2280914944 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002496770310932798, + "loss": 1.2472, + "theoretical_loss": 3.421940955578589, + "tokens_seen": 2280980480 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002496609829488465, + "loss": 1.3039, + "theoretical_loss": 3.42193303496054, + "tokens_seen": 2281046016 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002496449348044132, + "loss": 1.2362, + "theoretical_loss": 3.4219251146337686, + "tokens_seen": 2281111552 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002496288866599799, + "loss": 1.3183, + "theoretical_loss": 3.4219171945982554, + "tokens_seen": 2281177088 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002496128385155466, + "loss": 1.3315, + "theoretical_loss": 3.421909274853982, + "tokens_seen": 2281242624 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002495967903711133, + "loss": 1.2234, + "theoretical_loss": 3.4219013554009283, + "tokens_seen": 2281308160 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024958074222668, + "loss": 1.3365, + "theoretical_loss": 3.4218934362390763, + "tokens_seen": 2281373696 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002495646940822467, + "loss": 1.2075, + "theoretical_loss": 3.4218855173684064, + "tokens_seen": 2281439232 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002495486459378134, + "loss": 1.1919, + "theoretical_loss": 3.421877598788899, + "tokens_seen": 2281504768 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024953259779338013, + "loss": 1.2921, + "theoretical_loss": 3.421869680500536, + "tokens_seen": 2281570304 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024951654964894683, + "loss": 1.3631, + "theoretical_loss": 3.421861762503298, + "tokens_seen": 2281635840 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024950050150451353, + "loss": 1.2799, + "theoretical_loss": 3.4218538447971656, + "tokens_seen": 2281701376 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024948445336008023, + "loss": 1.1703, + "theoretical_loss": 3.4218459273821202, + "tokens_seen": 2281766912 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024946840521564693, + "loss": 1.1603, + "theoretical_loss": 3.4218380102581425, + "tokens_seen": 2281832448 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024945235707121363, + "loss": 1.2096, + "theoretical_loss": 3.4218300934252133, + "tokens_seen": 2281897984 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024943630892678033, + "loss": 1.3404, + "theoretical_loss": 3.421822176883314, + "tokens_seen": 2281963520 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024942026078234703, + "loss": 1.2757, + "theoretical_loss": 3.421814260632425, + "tokens_seen": 2282029056 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024940421263791373, + "loss": 1.2191, + "theoretical_loss": 3.421806344672527, + "tokens_seen": 2282094592 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024938816449348043, + "loss": 1.27, + "theoretical_loss": 3.4217984290036023, + "tokens_seen": 2282160128 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024937211634904713, + "loss": 1.3596, + "theoretical_loss": 3.4217905136256306, + "tokens_seen": 2282225664 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024935606820461383, + "loss": 1.303, + "theoretical_loss": 3.4217825985385932, + "tokens_seen": 2282291200 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024934002006018053, + "loss": 1.289, + "theoretical_loss": 3.4217746837424716, + "tokens_seen": 2282356736 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024932397191574724, + "loss": 1.3012, + "theoretical_loss": 3.4217667692372458, + "tokens_seen": 2282422272 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024930792377131394, + "loss": 1.201, + "theoretical_loss": 3.421758855022897, + "tokens_seen": 2282487808 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024929187562688064, + "loss": 1.2263, + "theoretical_loss": 3.4217509410994067, + "tokens_seen": 2282553344 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024927582748244734, + "loss": 1.1863, + "theoretical_loss": 3.4217430274667557, + "tokens_seen": 2282618880 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024925977933801404, + "loss": 1.2636, + "theoretical_loss": 3.4217351141249246, + "tokens_seen": 2282684416 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002492437311935808, + "loss": 1.1624, + "theoretical_loss": 3.421727201073894, + "tokens_seen": 2282749952 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002492276830491475, + "loss": 1.2393, + "theoretical_loss": 3.4217192883136462, + "tokens_seen": 2282815488 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002492116349047142, + "loss": 1.1816, + "theoretical_loss": 3.4217113758441613, + "tokens_seen": 2282881024 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002491955867602809, + "loss": 1.1797, + "theoretical_loss": 3.42170346366542, + "tokens_seen": 2282946560 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002491795386158476, + "loss": 1.1957, + "theoretical_loss": 3.421695551777404, + "tokens_seen": 2283012096 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002491634904714143, + "loss": 1.1948, + "theoretical_loss": 3.4216876401800933, + "tokens_seen": 2283077632 + }, + { + "epoch": 0.69, + "learning_rate": 0.000249147442326981, + "loss": 1.333, + "theoretical_loss": 3.42167972887347, + "tokens_seen": 2283143168 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002491313941825477, + "loss": 1.2469, + "theoretical_loss": 3.4216718178575145, + "tokens_seen": 2283208704 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002491153460381144, + "loss": 1.2349, + "theoretical_loss": 3.4216639071322077, + "tokens_seen": 2283274240 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002490992978936811, + "loss": 1.1901, + "theoretical_loss": 3.4216559966975306, + "tokens_seen": 2283339776 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024908324974924775, + "loss": 1.287, + "theoretical_loss": 3.4216480865534646, + "tokens_seen": 2283405312 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024906720160481445, + "loss": 1.3362, + "theoretical_loss": 3.42164017669999, + "tokens_seen": 2283470848 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024905115346038115, + "loss": 1.2318, + "theoretical_loss": 3.4216322671370882, + "tokens_seen": 2283536384 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024903510531594785, + "loss": 1.2531, + "theoretical_loss": 3.4216243578647405, + "tokens_seen": 2283601920 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024901905717151455, + "loss": 1.2933, + "theoretical_loss": 3.4216164488829275, + "tokens_seen": 2283667456 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024900300902708125, + "loss": 1.2873, + "theoretical_loss": 3.42160854019163, + "tokens_seen": 2283732992 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024898696088264795, + "loss": 1.3486, + "theoretical_loss": 3.421600631790829, + "tokens_seen": 2283798528 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024897091273821465, + "loss": 1.2556, + "theoretical_loss": 3.421592723680506, + "tokens_seen": 2283864064 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 764062, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2559454441070557, + "objective/train/theoretical_loss": 3.421584815860642, + "objective/train/tokens_used": 2304389600, + "theoretical_loss": 3.421584815860642, + "tokens_seen": 2283929600 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024895486459378135, + "loss": 1.3086, + "theoretical_loss": 3.421584815860642, + "tokens_seen": 2283929600 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024893881644934805, + "loss": 1.2857, + "theoretical_loss": 3.4215769083312173, + "tokens_seen": 2283995136 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024892276830491475, + "loss": 1.275, + "theoretical_loss": 3.4215690010922133, + "tokens_seen": 2284060672 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024890672016048145, + "loss": 1.2679, + "theoretical_loss": 3.4215610941436108, + "tokens_seen": 2284126208 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024889067201604815, + "loss": 1.1957, + "theoretical_loss": 3.4215531874853915, + "tokens_seen": 2284191744 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024887462387161485, + "loss": 1.1776, + "theoretical_loss": 3.4215452811175355, + "tokens_seen": 2284257280 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024885857572718155, + "loss": 1.2411, + "theoretical_loss": 3.4215373750400246, + "tokens_seen": 2284322816 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024884252758274825, + "loss": 1.1065, + "theoretical_loss": 3.421529469252839, + "tokens_seen": 2284388352 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024882647943831496, + "loss": 1.3828, + "theoretical_loss": 3.4215215637559604, + "tokens_seen": 2284453888 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024881043129388166, + "loss": 1.2827, + "theoretical_loss": 3.4215136585493697, + "tokens_seen": 2284519424 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024879438314944836, + "loss": 1.3173, + "theoretical_loss": 3.4215057536330473, + "tokens_seen": 2284584960 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024877833500501506, + "loss": 1.2537, + "theoretical_loss": 3.421497849006975, + "tokens_seen": 2284650496 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024876228686058176, + "loss": 1.269, + "theoretical_loss": 3.4214899446711335, + "tokens_seen": 2284716032 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024874623871614846, + "loss": 1.2905, + "theoretical_loss": 3.4214820406255035, + "tokens_seen": 2284781568 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024873019057171516, + "loss": 1.1967, + "theoretical_loss": 3.4214741368700663, + "tokens_seen": 2284847104 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024871414242728186, + "loss": 1.2141, + "theoretical_loss": 3.4214662334048036, + "tokens_seen": 2284912640 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024869809428284856, + "loss": 1.2936, + "theoretical_loss": 3.4214583302296955, + "tokens_seen": 2284978176 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024868204613841526, + "loss": 1.1737, + "theoretical_loss": 3.421450427344723, + "tokens_seen": 2285043712 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024866599799398196, + "loss": 1.2265, + "theoretical_loss": 3.4214425247498674, + "tokens_seen": 2285109248 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024864994984954866, + "loss": 1.3074, + "theoretical_loss": 3.42143462244511, + "tokens_seen": 2285174784 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024863390170511536, + "loss": 1.2719, + "theoretical_loss": 3.4214267204304316, + "tokens_seen": 2285240320 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024861785356068206, + "loss": 1.2865, + "theoretical_loss": 3.4214188187058134, + "tokens_seen": 2285305856 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024860180541624876, + "loss": 1.2344, + "theoretical_loss": 3.421410917271236, + "tokens_seen": 2285371392 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024858575727181546, + "loss": 1.3586, + "theoretical_loss": 3.4214030161266806, + "tokens_seen": 2285436928 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024856970912738217, + "loss": 1.1407, + "theoretical_loss": 3.4213951152721287, + "tokens_seen": 2285502464 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024855366098294887, + "loss": 1.2684, + "theoretical_loss": 3.421387214707561, + "tokens_seen": 2285568000 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024853761283851557, + "loss": 1.2, + "theoretical_loss": 3.421379314432958, + "tokens_seen": 2285633536 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024852156469408227, + "loss": 1.3126, + "theoretical_loss": 3.4213714144483016, + "tokens_seen": 2285699072 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024850551654964897, + "loss": 1.2138, + "theoretical_loss": 3.4213635147535726, + "tokens_seen": 2285764608 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024848946840521567, + "loss": 1.2638, + "theoretical_loss": 3.4213556153487517, + "tokens_seen": 2285830144 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024847342026078237, + "loss": 1.2949, + "theoretical_loss": 3.42134771623382, + "tokens_seen": 2285895680 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024845737211634907, + "loss": 1.2191, + "theoretical_loss": 3.4213398174087595, + "tokens_seen": 2285961216 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024844132397191577, + "loss": 1.1829, + "theoretical_loss": 3.42133191887355, + "tokens_seen": 2286026752 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024842527582748247, + "loss": 1.2199, + "theoretical_loss": 3.421324020628173, + "tokens_seen": 2286092288 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024840922768304917, + "loss": 1.1818, + "theoretical_loss": 3.42131612267261, + "tokens_seen": 2286157824 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024839317953861587, + "loss": 1.3233, + "theoretical_loss": 3.4213082250068414, + "tokens_seen": 2286223360 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002483771313941826, + "loss": 1.2637, + "theoretical_loss": 3.4213003276308487, + "tokens_seen": 2286288896 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002483610832497493, + "loss": 1.2902, + "theoretical_loss": 3.4212924305446126, + "tokens_seen": 2286354432 + }, + { + "epoch": 0.69, + "learning_rate": 0.000248345035105316, + "loss": 1.2908, + "theoretical_loss": 3.4212845337481146, + "tokens_seen": 2286419968 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002483289869608827, + "loss": 1.3549, + "theoretical_loss": 3.4212766372413355, + "tokens_seen": 2286485504 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002483129388164493, + "loss": 1.2363, + "theoretical_loss": 3.4212687410242566, + "tokens_seen": 2286551040 + }, + { + "epoch": 0.69, + "learning_rate": 0.000248296890672016, + "loss": 1.2563, + "theoretical_loss": 3.421260845096858, + "tokens_seen": 2286616576 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002482808425275827, + "loss": 1.2451, + "theoretical_loss": 3.4212529494591224, + "tokens_seen": 2286682112 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002482647943831494, + "loss": 1.2791, + "theoretical_loss": 3.4212450541110293, + "tokens_seen": 2286747648 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002482487462387161, + "loss": 1.1951, + "theoretical_loss": 3.421237159052561, + "tokens_seen": 2286813184 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002482326980942828, + "loss": 1.2896, + "theoretical_loss": 3.421229264283698, + "tokens_seen": 2286878720 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002482326980942828, + "loss": 1.1929, + "theoretical_loss": 3.421221369804421, + "tokens_seen": 2286944256 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002482166499498495, + "loss": 1.1349, + "theoretical_loss": 3.421213475614712, + "tokens_seen": 2287009792 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002482006018054162, + "loss": 1.298, + "theoretical_loss": 3.4212055817145517, + "tokens_seen": 2287075328 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002481845536609829, + "loss": 1.2029, + "theoretical_loss": 3.4211976881039208, + "tokens_seen": 2287140864 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 765245, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3369064331054688, + "objective/train/theoretical_loss": 3.4211897947828005, + "objective/train/tokens_used": 2307666400, + "theoretical_loss": 3.4211897947828005, + "tokens_seen": 2287206400 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002481685055165496, + "loss": 1.405, + "theoretical_loss": 3.4211897947828005, + "tokens_seen": 2287206400 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024815245737211633, + "loss": 1.2553, + "theoretical_loss": 3.421181901751172, + "tokens_seen": 2287271936 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024813640922768303, + "loss": 1.238, + "theoretical_loss": 3.4211740090090172, + "tokens_seen": 2287337472 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024812036108324973, + "loss": 1.252, + "theoretical_loss": 3.4211661165563156, + "tokens_seen": 2287403008 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024810431293881643, + "loss": 1.3089, + "theoretical_loss": 3.4211582243930496, + "tokens_seen": 2287468544 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024808826479438313, + "loss": 1.2312, + "theoretical_loss": 3.4211503325191996, + "tokens_seen": 2287534080 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024807221664994983, + "loss": 1.2604, + "theoretical_loss": 3.421142440934747, + "tokens_seen": 2287599616 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024805616850551653, + "loss": 1.2303, + "theoretical_loss": 3.421134549639673, + "tokens_seen": 2287665152 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002480401203610833, + "loss": 1.2721, + "theoretical_loss": 3.421126658633958, + "tokens_seen": 2287730688 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024802407221665, + "loss": 1.1785, + "theoretical_loss": 3.421118767917584, + "tokens_seen": 2287796224 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002480080240722167, + "loss": 1.3106, + "theoretical_loss": 3.421110877490532, + "tokens_seen": 2287861760 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002479919759277834, + "loss": 1.1664, + "theoretical_loss": 3.421102987352782, + "tokens_seen": 2287927296 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002479759277833501, + "loss": 1.2783, + "theoretical_loss": 3.421095097504317, + "tokens_seen": 2287992832 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002479598796389168, + "loss": 1.2455, + "theoretical_loss": 3.4210872079451162, + "tokens_seen": 2288058368 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002479438314944835, + "loss": 1.1549, + "theoretical_loss": 3.4210793186751616, + "tokens_seen": 2288123904 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002479277833500502, + "loss": 1.1562, + "theoretical_loss": 3.4210714296944347, + "tokens_seen": 2288189440 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002479117352056169, + "loss": 1.2328, + "theoretical_loss": 3.421063541002916, + "tokens_seen": 2288254976 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002478956870611836, + "loss": 1.2258, + "theoretical_loss": 3.4210556526005864, + "tokens_seen": 2288320512 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002478796389167503, + "loss": 1.3053, + "theoretical_loss": 3.4210477644874278, + "tokens_seen": 2288386048 + }, + { + "epoch": 0.69, + "learning_rate": 0.000247863590772317, + "loss": 1.1928, + "theoretical_loss": 3.421039876663421, + "tokens_seen": 2288451584 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002478475426278837, + "loss": 1.2939, + "theoretical_loss": 3.421031989128547, + "tokens_seen": 2288517120 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002478314944834504, + "loss": 1.1944, + "theoretical_loss": 3.4210241018827867, + "tokens_seen": 2288582656 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002478154463390171, + "loss": 1.2884, + "theoretical_loss": 3.421016214926121, + "tokens_seen": 2288648192 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002477993981945838, + "loss": 1.216, + "theoretical_loss": 3.4210083282585324, + "tokens_seen": 2288713728 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002477833500501505, + "loss": 1.323, + "theoretical_loss": 3.421000441880001, + "tokens_seen": 2288779264 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002477673019057172, + "loss": 1.234, + "theoretical_loss": 3.420992555790508, + "tokens_seen": 2288844800 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002477512537612839, + "loss": 1.1943, + "theoretical_loss": 3.4209846699900344, + "tokens_seen": 2288910336 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002477352056168506, + "loss": 1.2016, + "theoretical_loss": 3.4209767844785617, + "tokens_seen": 2288975872 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002477191574724173, + "loss": 1.2223, + "theoretical_loss": 3.4209688992560707, + "tokens_seen": 2289041408 + }, + { + "epoch": 0.69, + "learning_rate": 0.000247703109327984, + "loss": 1.3215, + "theoretical_loss": 3.4209610143225433, + "tokens_seen": 2289106944 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024768706118355065, + "loss": 1.1826, + "theoretical_loss": 3.4209531296779594, + "tokens_seen": 2289172480 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024767101303911735, + "loss": 1.2606, + "theoretical_loss": 3.420945245322301, + "tokens_seen": 2289238016 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024765496489468405, + "loss": 1.3239, + "theoretical_loss": 3.420937361255549, + "tokens_seen": 2289303552 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024763891675025075, + "loss": 1.335, + "theoretical_loss": 3.4209294774776846, + "tokens_seen": 2289369088 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024762286860581745, + "loss": 1.3064, + "theoretical_loss": 3.420921593988689, + "tokens_seen": 2289434624 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024760682046138415, + "loss": 1.2655, + "theoretical_loss": 3.420913710788543, + "tokens_seen": 2289500160 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024759077231695085, + "loss": 1.3181, + "theoretical_loss": 3.420905827877228, + "tokens_seen": 2289565696 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024757472417251755, + "loss": 1.2623, + "theoretical_loss": 3.4208979452547252, + "tokens_seen": 2289631232 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024755867602808425, + "loss": 1.187, + "theoretical_loss": 3.420890062921016, + "tokens_seen": 2289696768 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024754262788365095, + "loss": 1.2983, + "theoretical_loss": 3.420882180876081, + "tokens_seen": 2289762304 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024752657973921765, + "loss": 1.2948, + "theoretical_loss": 3.4208742991199017, + "tokens_seen": 2289827840 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024751053159478435, + "loss": 1.3701, + "theoretical_loss": 3.4208664176524595, + "tokens_seen": 2289893376 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024749448345035105, + "loss": 1.3513, + "theoretical_loss": 3.4208585364737347, + "tokens_seen": 2289958912 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024747843530591775, + "loss": 1.1207, + "theoretical_loss": 3.4208506555837093, + "tokens_seen": 2290024448 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024746238716148445, + "loss": 1.3078, + "theoretical_loss": 3.420842774982364, + "tokens_seen": 2290089984 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024744633901705116, + "loss": 1.264, + "theoretical_loss": 3.4208348946696803, + "tokens_seen": 2290155520 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024743029087261786, + "loss": 1.3753, + "theoretical_loss": 3.420827014645639, + "tokens_seen": 2290221056 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024741424272818456, + "loss": 1.2255, + "theoretical_loss": 3.4208191349102215, + "tokens_seen": 2290286592 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024739819458375126, + "loss": 1.296, + "theoretical_loss": 3.420811255463409, + "tokens_seen": 2290352128 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024738214643931796, + "loss": 1.2068, + "theoretical_loss": 3.420803376305183, + "tokens_seen": 2290417664 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 766171, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2719933986663818, + "objective/train/theoretical_loss": 3.420795497435524, + "objective/train/tokens_used": 2310943200, + "theoretical_loss": 3.420795497435524, + "tokens_seen": 2290483200 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024736609829488466, + "loss": 1.2678, + "theoretical_loss": 3.420795497435524, + "tokens_seen": 2290483200 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024735005015045136, + "loss": 1.2002, + "theoretical_loss": 3.420787618854413, + "tokens_seen": 2290548736 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024733400200601806, + "loss": 1.2135, + "theoretical_loss": 3.420779740561832, + "tokens_seen": 2290614272 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024731795386158476, + "loss": 1.2334, + "theoretical_loss": 3.420771862557762, + "tokens_seen": 2290679808 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024730190571715146, + "loss": 1.1837, + "theoretical_loss": 3.420763984842184, + "tokens_seen": 2290745344 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024728585757271816, + "loss": 1.2834, + "theoretical_loss": 3.4207561074150785, + "tokens_seen": 2290810880 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024726980942828486, + "loss": 1.2231, + "theoretical_loss": 3.420748230276428, + "tokens_seen": 2290876416 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024725376128385156, + "loss": 1.2383, + "theoretical_loss": 3.420740353426213, + "tokens_seen": 2290941952 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024723771313941826, + "loss": 1.2755, + "theoretical_loss": 3.4207324768644147, + "tokens_seen": 2291007488 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024722166499498496, + "loss": 1.3361, + "theoretical_loss": 3.420724600591014, + "tokens_seen": 2291073024 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024720561685055167, + "loss": 1.0879, + "theoretical_loss": 3.4207167246059926, + "tokens_seen": 2291138560 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024718956870611837, + "loss": 1.1967, + "theoretical_loss": 3.4207088489093316, + "tokens_seen": 2291204096 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024717352056168507, + "loss": 1.2709, + "theoretical_loss": 3.4207009735010123, + "tokens_seen": 2291269632 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024715747241725177, + "loss": 1.297, + "theoretical_loss": 3.4206930983810153, + "tokens_seen": 2291335168 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024714142427281847, + "loss": 1.2275, + "theoretical_loss": 3.4206852235493224, + "tokens_seen": 2291400704 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024712537612838517, + "loss": 1.1992, + "theoretical_loss": 3.4206773490059144, + "tokens_seen": 2291466240 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024710932798395187, + "loss": 1.1943, + "theoretical_loss": 3.4206694747507727, + "tokens_seen": 2291531776 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024709327983951857, + "loss": 1.193, + "theoretical_loss": 3.4206616007838786, + "tokens_seen": 2291597312 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024707723169508527, + "loss": 1.228, + "theoretical_loss": 3.420653727105213, + "tokens_seen": 2291662848 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024706118355065197, + "loss": 1.251, + "theoretical_loss": 3.420645853714758, + "tokens_seen": 2291728384 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024704513540621867, + "loss": 1.3244, + "theoretical_loss": 3.4206379806124936, + "tokens_seen": 2291793920 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024702908726178537, + "loss": 1.2594, + "theoretical_loss": 3.420630107798402, + "tokens_seen": 2291859456 + }, + { + "epoch": 0.69, + "learning_rate": 0.00024701303911735207, + "loss": 1.2738, + "theoretical_loss": 3.420622235272463, + "tokens_seen": 2291924992 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002469969909729188, + "loss": 1.211, + "theoretical_loss": 3.4206143630346597, + "tokens_seen": 2291990528 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002469809428284855, + "loss": 1.1404, + "theoretical_loss": 3.4206064910849716, + "tokens_seen": 2292056064 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002469648946840522, + "loss": 1.3157, + "theoretical_loss": 3.420598619423381, + "tokens_seen": 2292121600 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002469488465396189, + "loss": 1.2794, + "theoretical_loss": 3.420590748049869, + "tokens_seen": 2292187136 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002469327983951856, + "loss": 1.1263, + "theoretical_loss": 3.4205828769644167, + "tokens_seen": 2292252672 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002469167502507522, + "loss": 1.304, + "theoretical_loss": 3.420575006167005, + "tokens_seen": 2292318208 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002469007021063189, + "loss": 1.2076, + "theoretical_loss": 3.4205671356576155, + "tokens_seen": 2292383744 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002468846539618856, + "loss": 1.2339, + "theoretical_loss": 3.4205592654362293, + "tokens_seen": 2292449280 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002468686058174523, + "loss": 1.2402, + "theoretical_loss": 3.4205513955028275, + "tokens_seen": 2292514816 + }, + { + "epoch": 0.69, + "learning_rate": 0.000246852557673019, + "loss": 1.319, + "theoretical_loss": 3.4205435258573917, + "tokens_seen": 2292580352 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002468365095285858, + "loss": 1.228, + "theoretical_loss": 3.4205356564999025, + "tokens_seen": 2292645888 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002468204613841525, + "loss": 1.3041, + "theoretical_loss": 3.420527787430342, + "tokens_seen": 2292711424 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002468044132397192, + "loss": 1.2073, + "theoretical_loss": 3.420519918648691, + "tokens_seen": 2292776960 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002467883650952859, + "loss": 1.0901, + "theoretical_loss": 3.4205120501549304, + "tokens_seen": 2292842496 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002467723169508526, + "loss": 1.2486, + "theoretical_loss": 3.420504181949042, + "tokens_seen": 2292908032 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002467562688064193, + "loss": 1.1366, + "theoretical_loss": 3.420496314031006, + "tokens_seen": 2292973568 + }, + { + "epoch": 0.69, + "learning_rate": 0.000246740220661986, + "loss": 1.2153, + "theoretical_loss": 3.420488446400805, + "tokens_seen": 2293039104 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002467241725175527, + "loss": 1.249, + "theoretical_loss": 3.42048057905842, + "tokens_seen": 2293104640 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002467081243731194, + "loss": 1.2397, + "theoretical_loss": 3.420472712003831, + "tokens_seen": 2293170176 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002466920762286861, + "loss": 1.1215, + "theoretical_loss": 3.420464845237021, + "tokens_seen": 2293235712 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002466760280842528, + "loss": 1.2605, + "theoretical_loss": 3.4204569787579704, + "tokens_seen": 2293301248 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002466599799398195, + "loss": 1.3926, + "theoretical_loss": 3.4204491125666605, + "tokens_seen": 2293366784 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002466439317953862, + "loss": 1.2673, + "theoretical_loss": 3.420441246663072, + "tokens_seen": 2293432320 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002466278836509529, + "loss": 1.2498, + "theoretical_loss": 3.4204333810471867, + "tokens_seen": 2293497856 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002466118355065196, + "loss": 1.1075, + "theoretical_loss": 3.4204255157189865, + "tokens_seen": 2293563392 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002465957873620863, + "loss": 1.3213, + "theoretical_loss": 3.420417650678451, + "tokens_seen": 2293628928 + }, + { + "epoch": 0.7, + "learning_rate": 0.000246579739217653, + "loss": 1.3007, + "theoretical_loss": 3.420409785925563, + "tokens_seen": 2293694464 + }, + { + "debugging/Compilability": 0.0, + "debugging/distinct-1-grams": 0.7368157089654065, + "debugging/entropy-1-grams": 5.260448350002519, + "debugging/length": 548.8333333333334, + "debugging/num_segments": 12, + "debugging/score": 0.05299388900931896, + "debugging/score_std": 0.02491763752851157, + "epoch": 0.7, + "objective/train/docs_used": 767417, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2914656400680542, + "objective/train/theoretical_loss": 3.4204019214603036, + "objective/train/tokens_used": 2314220000, + "theoretical_loss": 3.4204019214603036, + "tokens_seen": 2293760000 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002465636910732197, + "loss": 1.2804, + "theoretical_loss": 3.4204019214603036, + "tokens_seen": 2293760000 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002465476429287864, + "loss": 1.0869, + "theoretical_loss": 3.420394057282653, + "tokens_seen": 2293825536 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002465315947843531, + "loss": 1.1691, + "theoretical_loss": 3.4203861933925936, + "tokens_seen": 2293891072 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002465155466399198, + "loss": 1.2505, + "theoretical_loss": 3.420378329790106, + "tokens_seen": 2293956608 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002464994984954865, + "loss": 1.2482, + "theoretical_loss": 3.4203704664751715, + "tokens_seen": 2294022144 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002464834503510532, + "loss": 1.2548, + "theoretical_loss": 3.4203626034477717, + "tokens_seen": 2294087680 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002464674022066199, + "loss": 1.1889, + "theoretical_loss": 3.420354740707888, + "tokens_seen": 2294153216 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002464513540621866, + "loss": 1.1673, + "theoretical_loss": 3.420346878255501, + "tokens_seen": 2294218752 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002464353059177533, + "loss": 1.175, + "theoretical_loss": 3.4203390160905927, + "tokens_seen": 2294284288 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024641925777332, + "loss": 1.1598, + "theoretical_loss": 3.4203311542131436, + "tokens_seen": 2294349824 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002464032096288867, + "loss": 1.3002, + "theoretical_loss": 3.4203232926231357, + "tokens_seen": 2294415360 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002463871614844534, + "loss": 1.2029, + "theoretical_loss": 3.42031543132055, + "tokens_seen": 2294480896 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002463711133400201, + "loss": 1.1759, + "theoretical_loss": 3.4203075703053676, + "tokens_seen": 2294546432 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002463550651955868, + "loss": 1.2217, + "theoretical_loss": 3.42029970957757, + "tokens_seen": 2294611968 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002463390170511535, + "loss": 1.2513, + "theoretical_loss": 3.4202918491371386, + "tokens_seen": 2294677504 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002463229689067202, + "loss": 1.2122, + "theoretical_loss": 3.4202839889840546, + "tokens_seen": 2294743040 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002463069207622869, + "loss": 1.2356, + "theoretical_loss": 3.420276129118299, + "tokens_seen": 2294808576 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024629087261785355, + "loss": 1.2523, + "theoretical_loss": 3.4202682695398536, + "tokens_seen": 2294874112 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024627482447342025, + "loss": 1.1966, + "theoretical_loss": 3.4202604102486993, + "tokens_seen": 2294939648 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024625877632898695, + "loss": 1.2442, + "theoretical_loss": 3.4202525512448174, + "tokens_seen": 2295005184 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024624272818455365, + "loss": 1.299, + "theoretical_loss": 3.4202446925281893, + "tokens_seen": 2295070720 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024622668004012035, + "loss": 1.2816, + "theoretical_loss": 3.4202368340987963, + "tokens_seen": 2295136256 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024621063189568705, + "loss": 1.1848, + "theoretical_loss": 3.42022897595662, + "tokens_seen": 2295201792 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024619458375125375, + "loss": 1.2692, + "theoretical_loss": 3.420221118101641, + "tokens_seen": 2295267328 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024617853560682045, + "loss": 1.2605, + "theoretical_loss": 3.420213260533841, + "tokens_seen": 2295332864 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024616248746238715, + "loss": 1.2786, + "theoretical_loss": 3.420205403253202, + "tokens_seen": 2295398400 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024614643931795385, + "loss": 1.1801, + "theoretical_loss": 3.420197546259704, + "tokens_seen": 2295463936 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024613039117352055, + "loss": 1.3359, + "theoretical_loss": 3.4201896895533292, + "tokens_seen": 2295529472 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024611434302908725, + "loss": 1.2123, + "theoretical_loss": 3.4201818331340585, + "tokens_seen": 2295595008 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024609829488465395, + "loss": 1.2212, + "theoretical_loss": 3.4201739770018733, + "tokens_seen": 2295660544 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024608224674022066, + "loss": 1.2301, + "theoretical_loss": 3.4201661211567553, + "tokens_seen": 2295726080 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024606619859578736, + "loss": 1.188, + "theoretical_loss": 3.420158265598685, + "tokens_seen": 2295791616 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024605015045135406, + "loss": 1.1659, + "theoretical_loss": 3.4201504103276448, + "tokens_seen": 2295857152 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024603410230692076, + "loss": 1.1569, + "theoretical_loss": 3.420142555343615, + "tokens_seen": 2295922688 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024601805416248746, + "loss": 1.1837, + "theoretical_loss": 3.420134700646577, + "tokens_seen": 2295988224 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024600200601805416, + "loss": 1.2489, + "theoretical_loss": 3.4201268462365135, + "tokens_seen": 2296053760 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024598595787362086, + "loss": 1.2533, + "theoretical_loss": 3.4201189921134043, + "tokens_seen": 2296119296 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024596990972918756, + "loss": 1.2695, + "theoretical_loss": 3.420111138277231, + "tokens_seen": 2296184832 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024595386158475426, + "loss": 1.4606, + "theoretical_loss": 3.420103284727975, + "tokens_seen": 2296250368 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024593781344032096, + "loss": 1.1763, + "theoretical_loss": 3.4200954314656182, + "tokens_seen": 2296315904 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024592176529588766, + "loss": 1.2866, + "theoretical_loss": 3.4200875784901417, + "tokens_seen": 2296381440 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024590571715145436, + "loss": 1.1622, + "theoretical_loss": 3.420079725801526, + "tokens_seen": 2296446976 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024588966900702106, + "loss": 1.231, + "theoretical_loss": 3.4200718733997535, + "tokens_seen": 2296512512 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024587362086258776, + "loss": 1.1985, + "theoretical_loss": 3.420064021284805, + "tokens_seen": 2296578048 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024585757271815446, + "loss": 1.3075, + "theoretical_loss": 3.420056169456662, + "tokens_seen": 2296643584 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024584152457372116, + "loss": 1.2166, + "theoretical_loss": 3.4200483179153056, + "tokens_seen": 2296709120 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024582547642928787, + "loss": 1.246, + "theoretical_loss": 3.4200404666607174, + "tokens_seen": 2296774656 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024580942828485457, + "loss": 1.2715, + "theoretical_loss": 3.420032615692879, + "tokens_seen": 2296840192 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024579338014042127, + "loss": 1.2971, + "theoretical_loss": 3.420024765011771, + "tokens_seen": 2296905728 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024577733199598797, + "loss": 1.3177, + "theoretical_loss": 3.4200169146173756, + "tokens_seen": 2296971264 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 768307, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.105657935142517, + "objective/train/theoretical_loss": 3.4200090645096735, + "objective/train/tokens_used": 2317496800, + "theoretical_loss": 3.4200090645096735, + "tokens_seen": 2297036800 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024576128385155467, + "loss": 1.1518, + "theoretical_loss": 3.4200090645096735, + "tokens_seen": 2297036800 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024574523570712137, + "loss": 1.2841, + "theoretical_loss": 3.420001214688646, + "tokens_seen": 2297102336 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024572918756268807, + "loss": 1.3217, + "theoretical_loss": 3.4199933651542755, + "tokens_seen": 2297167872 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024571313941825477, + "loss": 1.296, + "theoretical_loss": 3.419985515906542, + "tokens_seen": 2297233408 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024569709127382147, + "loss": 1.1431, + "theoretical_loss": 3.419977666945427, + "tokens_seen": 2297298944 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024568104312938817, + "loss": 1.2402, + "theoretical_loss": 3.4199698182709133, + "tokens_seen": 2297364480 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024566499498495487, + "loss": 1.2115, + "theoretical_loss": 3.419961969882981, + "tokens_seen": 2297430016 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024564894684052157, + "loss": 1.2586, + "theoretical_loss": 3.419954121781611, + "tokens_seen": 2297495552 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002456328986960883, + "loss": 1.2367, + "theoretical_loss": 3.419946273966786, + "tokens_seen": 2297561088 + }, + { + "epoch": 0.7, + "learning_rate": 0.000245616850551655, + "loss": 1.2727, + "theoretical_loss": 3.419938426438487, + "tokens_seen": 2297626624 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002456008024072217, + "loss": 1.2503, + "theoretical_loss": 3.4199305791966945, + "tokens_seen": 2297692160 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002455847542627884, + "loss": 1.2832, + "theoretical_loss": 3.4199227322413908, + "tokens_seen": 2297757696 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002455687061183551, + "loss": 1.1593, + "theoretical_loss": 3.419914885572557, + "tokens_seen": 2297823232 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002455526579739218, + "loss": 1.3186, + "theoretical_loss": 3.419907039190174, + "tokens_seen": 2297888768 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002455366098294885, + "loss": 1.2784, + "theoretical_loss": 3.419899193094224, + "tokens_seen": 2297954304 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002455205616850552, + "loss": 1.2032, + "theoretical_loss": 3.419891347284688, + "tokens_seen": 2298019840 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002455045135406219, + "loss": 1.1354, + "theoretical_loss": 3.419883501761547, + "tokens_seen": 2298085376 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002454884653961886, + "loss": 1.2374, + "theoretical_loss": 3.419875656524783, + "tokens_seen": 2298150912 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002454724172517553, + "loss": 1.2615, + "theoretical_loss": 3.419867811574377, + "tokens_seen": 2298216448 + }, + { + "epoch": 0.7, + "learning_rate": 0.000245456369107322, + "loss": 1.2857, + "theoretical_loss": 3.4198599669103107, + "tokens_seen": 2298281984 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002454403209628887, + "loss": 1.2169, + "theoretical_loss": 3.419852122532565, + "tokens_seen": 2298347520 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002454242728184554, + "loss": 1.2835, + "theoretical_loss": 3.4198442784411216, + "tokens_seen": 2298413056 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002454082246740221, + "loss": 1.2993, + "theoretical_loss": 3.419836434635962, + "tokens_seen": 2298478592 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002454082246740221, + "loss": 1.352, + "theoretical_loss": 3.419828591117067, + "tokens_seen": 2298544128 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002453921765295888, + "loss": 1.273, + "theoretical_loss": 3.4198207478844194, + "tokens_seen": 2298609664 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002453761283851555, + "loss": 1.3216, + "theoretical_loss": 3.4198129049379986, + "tokens_seen": 2298675200 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002453600802407222, + "loss": 1.3042, + "theoretical_loss": 3.4198050622777876, + "tokens_seen": 2298740736 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002453440320962889, + "loss": 1.3101, + "theoretical_loss": 3.419797219903767, + "tokens_seen": 2298806272 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002453279839518556, + "loss": 1.327, + "theoretical_loss": 3.4197893778159187, + "tokens_seen": 2298871808 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002453119358074223, + "loss": 1.2516, + "theoretical_loss": 3.4197815360142236, + "tokens_seen": 2298937344 + }, + { + "epoch": 0.7, + "learning_rate": 0.000245295887662989, + "loss": 1.3472, + "theoretical_loss": 3.419773694498663, + "tokens_seen": 2299002880 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002452798395185557, + "loss": 1.2326, + "theoretical_loss": 3.419765853269219, + "tokens_seen": 2299068416 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002452637913741224, + "loss": 1.2125, + "theoretical_loss": 3.4197580123258726, + "tokens_seen": 2299133952 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002452477432296891, + "loss": 1.3399, + "theoretical_loss": 3.419750171668605, + "tokens_seen": 2299199488 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002452316950852558, + "loss": 1.1849, + "theoretical_loss": 3.419742331297398, + "tokens_seen": 2299265024 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002452156469408225, + "loss": 1.2078, + "theoretical_loss": 3.4197344912122327, + "tokens_seen": 2299330560 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002451995987963892, + "loss": 1.2277, + "theoretical_loss": 3.419726651413091, + "tokens_seen": 2299396096 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002451835506519559, + "loss": 1.2681, + "theoretical_loss": 3.419718811899954, + "tokens_seen": 2299461632 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002451675025075226, + "loss": 1.1143, + "theoretical_loss": 3.4197109726728026, + "tokens_seen": 2299527168 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002451514543630893, + "loss": 1.2242, + "theoretical_loss": 3.419703133731619, + "tokens_seen": 2299592704 + }, + { + "epoch": 0.7, + "learning_rate": 0.000245135406218656, + "loss": 1.1756, + "theoretical_loss": 3.419695295076384, + "tokens_seen": 2299658240 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002451193580742227, + "loss": 1.2637, + "theoretical_loss": 3.41968745670708, + "tokens_seen": 2299723776 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002451033099297894, + "loss": 1.2711, + "theoretical_loss": 3.419679618623687, + "tokens_seen": 2299789312 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002450872617853561, + "loss": 1.1857, + "theoretical_loss": 3.419671780826188, + "tokens_seen": 2299854848 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002450712136409228, + "loss": 1.3093, + "theoretical_loss": 3.419663943314563, + "tokens_seen": 2299920384 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002450551654964895, + "loss": 1.2069, + "theoretical_loss": 3.419656106088794, + "tokens_seen": 2299985920 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002450391173520562, + "loss": 1.3225, + "theoretical_loss": 3.4196482691488628, + "tokens_seen": 2300051456 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002450230692076229, + "loss": 1.3024, + "theoretical_loss": 3.4196404324947505, + "tokens_seen": 2300116992 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002450070210631896, + "loss": 1.195, + "theoretical_loss": 3.419632596126438, + "tokens_seen": 2300182528 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002449909729187563, + "loss": 1.1644, + "theoretical_loss": 3.4196247600439076, + "tokens_seen": 2300248064 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 769491, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4527536630630493, + "objective/train/theoretical_loss": 3.4196169242471406, + "objective/train/tokens_used": 2320773600, + "theoretical_loss": 3.4196169242471406, + "tokens_seen": 2300313600 + }, + { + "epoch": 0.7, + "learning_rate": 0.000244974924774323, + "loss": 1.3522, + "theoretical_loss": 3.4196169242471406, + "tokens_seen": 2300313600 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002449588766298897, + "loss": 1.2813, + "theoretical_loss": 3.4196090887361184, + "tokens_seen": 2300379136 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002449428284854564, + "loss": 1.4101, + "theoretical_loss": 3.419601253510822, + "tokens_seen": 2300444672 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002449267803410231, + "loss": 1.175, + "theoretical_loss": 3.419593418571233, + "tokens_seen": 2300510208 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002449107321965898, + "loss": 1.2367, + "theoretical_loss": 3.4195855839173332, + "tokens_seen": 2300575744 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002448946840521565, + "loss": 1.1824, + "theoretical_loss": 3.4195777495491035, + "tokens_seen": 2300641280 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024487863590772315, + "loss": 1.3752, + "theoretical_loss": 3.419569915466526, + "tokens_seen": 2300706816 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024486258776328985, + "loss": 1.2257, + "theoretical_loss": 3.4195620816695818, + "tokens_seen": 2300772352 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024484653961885655, + "loss": 1.3311, + "theoretical_loss": 3.419554248158252, + "tokens_seen": 2300837888 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024483049147442325, + "loss": 1.2443, + "theoretical_loss": 3.419546414932519, + "tokens_seen": 2300903424 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024481444332998995, + "loss": 1.3097, + "theoretical_loss": 3.419538581992363, + "tokens_seen": 2300968960 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024479839518555665, + "loss": 1.1115, + "theoretical_loss": 3.419530749337767, + "tokens_seen": 2301034496 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024478234704112335, + "loss": 1.2281, + "theoretical_loss": 3.4195229169687105, + "tokens_seen": 2301100032 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024476629889669005, + "loss": 1.3452, + "theoretical_loss": 3.4195150848851767, + "tokens_seen": 2301165568 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024475025075225675, + "loss": 1.2994, + "theoretical_loss": 3.419507253087146, + "tokens_seen": 2301231104 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024473420260782345, + "loss": 1.2345, + "theoretical_loss": 3.4194994215746006, + "tokens_seen": 2301296640 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024471815446339016, + "loss": 1.3042, + "theoretical_loss": 3.419491590347522, + "tokens_seen": 2301362176 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024470210631895686, + "loss": 1.1382, + "theoretical_loss": 3.41948375940589, + "tokens_seen": 2301427712 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024468605817452356, + "loss": 1.3672, + "theoretical_loss": 3.4194759287496885, + "tokens_seen": 2301493248 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024467001003009026, + "loss": 1.1663, + "theoretical_loss": 3.4194680983788976, + "tokens_seen": 2301558784 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024465396188565696, + "loss": 1.2034, + "theoretical_loss": 3.419460268293499, + "tokens_seen": 2301624320 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024463791374122366, + "loss": 1.2607, + "theoretical_loss": 3.4194524384934737, + "tokens_seen": 2301689856 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024462186559679036, + "loss": 1.2202, + "theoretical_loss": 3.419444608978804, + "tokens_seen": 2301755392 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024460581745235706, + "loss": 1.1679, + "theoretical_loss": 3.4194367797494714, + "tokens_seen": 2301820928 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024458976930792376, + "loss": 1.2208, + "theoretical_loss": 3.4194289508054565, + "tokens_seen": 2301886464 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024457372116349046, + "loss": 1.2726, + "theoretical_loss": 3.4194211221467414, + "tokens_seen": 2301952000 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024455767301905716, + "loss": 1.2151, + "theoretical_loss": 3.419413293773307, + "tokens_seen": 2302017536 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024454162487462386, + "loss": 1.3508, + "theoretical_loss": 3.419405465685136, + "tokens_seen": 2302083072 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024452557673019056, + "loss": 1.2986, + "theoretical_loss": 3.4193976378822084, + "tokens_seen": 2302148608 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024450952858575726, + "loss": 1.135, + "theoretical_loss": 3.419389810364507, + "tokens_seen": 2302214144 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024449348044132396, + "loss": 1.1958, + "theoretical_loss": 3.4193819831320127, + "tokens_seen": 2302279680 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024447743229689066, + "loss": 1.1845, + "theoretical_loss": 3.4193741561847064, + "tokens_seen": 2302345216 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024446138415245737, + "loss": 1.2369, + "theoretical_loss": 3.4193663295225707, + "tokens_seen": 2302410752 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024444533600802407, + "loss": 1.193, + "theoretical_loss": 3.4193585031455864, + "tokens_seen": 2302476288 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002444292878635908, + "loss": 1.267, + "theoretical_loss": 3.4193506770537354, + "tokens_seen": 2302541824 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002444132397191575, + "loss": 1.1866, + "theoretical_loss": 3.4193428512469985, + "tokens_seen": 2302607360 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002443971915747242, + "loss": 1.2464, + "theoretical_loss": 3.419335025725358, + "tokens_seen": 2302672896 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002443811434302909, + "loss": 1.1643, + "theoretical_loss": 3.4193272004887953, + "tokens_seen": 2302738432 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002443650952858576, + "loss": 1.3318, + "theoretical_loss": 3.419319375537291, + "tokens_seen": 2302803968 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002443490471414243, + "loss": 1.1503, + "theoretical_loss": 3.4193115508708276, + "tokens_seen": 2302869504 + }, + { + "epoch": 0.7, + "learning_rate": 0.000244332998996991, + "loss": 1.2373, + "theoretical_loss": 3.419303726489386, + "tokens_seen": 2302935040 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002443169508525577, + "loss": 1.3668, + "theoretical_loss": 3.419295902392949, + "tokens_seen": 2303000576 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002443009027081244, + "loss": 1.169, + "theoretical_loss": 3.419288078581496, + "tokens_seen": 2303066112 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002442848545636911, + "loss": 1.1522, + "theoretical_loss": 3.4192802550550097, + "tokens_seen": 2303131648 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002442688064192578, + "loss": 1.329, + "theoretical_loss": 3.419272431813472, + "tokens_seen": 2303197184 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002442527582748245, + "loss": 1.2005, + "theoretical_loss": 3.4192646088568637, + "tokens_seen": 2303262720 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002442367101303912, + "loss": 1.1958, + "theoretical_loss": 3.4192567861851666, + "tokens_seen": 2303328256 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002442206619859579, + "loss": 1.2947, + "theoretical_loss": 3.419248963798362, + "tokens_seen": 2303393792 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002442046138415246, + "loss": 1.2765, + "theoretical_loss": 3.419241141696432, + "tokens_seen": 2303459328 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002441885656970913, + "loss": 1.2166, + "theoretical_loss": 3.419233319879357, + "tokens_seen": 2303524864 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 770522, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3516998291015625, + "objective/train/theoretical_loss": 3.41922549834712, + "objective/train/tokens_used": 2324050400, + "theoretical_loss": 3.41922549834712, + "tokens_seen": 2303590400 + }, + { + "epoch": 0.7, + "learning_rate": 0.000244172517552658, + "loss": 1.3416, + "theoretical_loss": 3.41922549834712, + "tokens_seen": 2303590400 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002441564694082247, + "loss": 1.2954, + "theoretical_loss": 3.4192176770997014, + "tokens_seen": 2303655936 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002441404212637914, + "loss": 1.2578, + "theoretical_loss": 3.4192098561370825, + "tokens_seen": 2303721472 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002441243731193581, + "loss": 1.1514, + "theoretical_loss": 3.4192020354592465, + "tokens_seen": 2303787008 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002441083249749248, + "loss": 1.1638, + "theoretical_loss": 3.419194215066173, + "tokens_seen": 2303852544 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002440922768304915, + "loss": 1.2296, + "theoretical_loss": 3.4191863949578445, + "tokens_seen": 2303918080 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002440762286860582, + "loss": 1.1848, + "theoretical_loss": 3.4191785751342425, + "tokens_seen": 2303983616 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024406018054162488, + "loss": 1.2919, + "theoretical_loss": 3.4191707555953483, + "tokens_seen": 2304049152 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024404413239719158, + "loss": 1.263, + "theoretical_loss": 3.419162936341144, + "tokens_seen": 2304114688 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024402808425275828, + "loss": 1.3238, + "theoretical_loss": 3.4191551173716106, + "tokens_seen": 2304180224 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024401203610832498, + "loss": 1.333, + "theoretical_loss": 3.4191472986867293, + "tokens_seen": 2304245760 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024399598796389168, + "loss": 1.2272, + "theoretical_loss": 3.4191394802864825, + "tokens_seen": 2304311296 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024397993981945838, + "loss": 1.2379, + "theoretical_loss": 3.419131662170851, + "tokens_seen": 2304376832 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024396389167502508, + "loss": 1.2953, + "theoretical_loss": 3.419123844339817, + "tokens_seen": 2304442368 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024394784353059179, + "loss": 1.2376, + "theoretical_loss": 3.4191160267933616, + "tokens_seen": 2304507904 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024393179538615849, + "loss": 1.1884, + "theoretical_loss": 3.4191082095314664, + "tokens_seen": 2304573440 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002439157472417252, + "loss": 1.1712, + "theoretical_loss": 3.419100392554113, + "tokens_seen": 2304638976 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002438996990972919, + "loss": 1.3699, + "theoretical_loss": 3.419092575861283, + "tokens_seen": 2304704512 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002438836509528586, + "loss": 1.2194, + "theoretical_loss": 3.419084759452958, + "tokens_seen": 2304770048 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002438676028084253, + "loss": 1.1735, + "theoretical_loss": 3.4190769433291197, + "tokens_seen": 2304835584 + }, + { + "epoch": 0.7, + "learning_rate": 0.000243851554663992, + "loss": 1.2454, + "theoretical_loss": 3.419069127489749, + "tokens_seen": 2304901120 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002438355065195587, + "loss": 1.2103, + "theoretical_loss": 3.4190613119348283, + "tokens_seen": 2304966656 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002438194583751254, + "loss": 1.2282, + "theoretical_loss": 3.4190534966643384, + "tokens_seen": 2305032192 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002438034102306921, + "loss": 1.3197, + "theoretical_loss": 3.4190456816782615, + "tokens_seen": 2305097728 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002437873620862588, + "loss": 1.1784, + "theoretical_loss": 3.419037866976579, + "tokens_seen": 2305163264 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002437713139418255, + "loss": 1.3009, + "theoretical_loss": 3.419030052559272, + "tokens_seen": 2305228800 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002437552657973922, + "loss": 1.2582, + "theoretical_loss": 3.4190222384263222, + "tokens_seen": 2305294336 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002437392176529589, + "loss": 1.2733, + "theoretical_loss": 3.419014424577712, + "tokens_seen": 2305359872 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002437231695085256, + "loss": 1.2621, + "theoretical_loss": 3.4190066110134216, + "tokens_seen": 2305425408 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002437071213640923, + "loss": 1.1991, + "theoretical_loss": 3.418998797733434, + "tokens_seen": 2305490944 + }, + { + "epoch": 0.7, + "learning_rate": 0.000243691073219659, + "loss": 1.1619, + "theoretical_loss": 3.41899098473773, + "tokens_seen": 2305556480 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024367502507522567, + "loss": 1.2237, + "theoretical_loss": 3.418983172026291, + "tokens_seen": 2305622016 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024365897693079237, + "loss": 1.2681, + "theoretical_loss": 3.4189753595990995, + "tokens_seen": 2305687552 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024364292878635907, + "loss": 1.2892, + "theoretical_loss": 3.418967547456136, + "tokens_seen": 2305753088 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024362688064192577, + "loss": 1.1772, + "theoretical_loss": 3.418959735597382, + "tokens_seen": 2305818624 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024361083249749247, + "loss": 1.2196, + "theoretical_loss": 3.4189519240228203, + "tokens_seen": 2305884160 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024359478435305917, + "loss": 1.1519, + "theoretical_loss": 3.4189441127324316, + "tokens_seen": 2305949696 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024357873620862587, + "loss": 1.1881, + "theoretical_loss": 3.4189363017261973, + "tokens_seen": 2306015232 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024356268806419257, + "loss": 1.1952, + "theoretical_loss": 3.4189284910041, + "tokens_seen": 2306080768 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024354663991975927, + "loss": 1.28, + "theoretical_loss": 3.4189206805661203, + "tokens_seen": 2306146304 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024353059177532597, + "loss": 1.3052, + "theoretical_loss": 3.4189128704122402, + "tokens_seen": 2306211840 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024351454363089268, + "loss": 1.0519, + "theoretical_loss": 3.4189050605424414, + "tokens_seen": 2306277376 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024349849548645938, + "loss": 1.215, + "theoretical_loss": 3.418897250956705, + "tokens_seen": 2306342912 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024348244734202608, + "loss": 1.2355, + "theoretical_loss": 3.418889441655013, + "tokens_seen": 2306408448 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024346639919759278, + "loss": 1.3166, + "theoretical_loss": 3.4188816326373472, + "tokens_seen": 2306473984 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024345035105315948, + "loss": 1.3523, + "theoretical_loss": 3.418873823903689, + "tokens_seen": 2306539520 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024343430290872618, + "loss": 1.1571, + "theoretical_loss": 3.41886601545402, + "tokens_seen": 2306605056 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024341825476429288, + "loss": 1.2541, + "theoretical_loss": 3.418858207288321, + "tokens_seen": 2306670592 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024340220661985958, + "loss": 1.2924, + "theoretical_loss": 3.4188503994065744, + "tokens_seen": 2306736128 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024338615847542628, + "loss": 1.1861, + "theoretical_loss": 3.4188425918087626, + "tokens_seen": 2306801664 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 771663, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1629533767700195, + "objective/train/theoretical_loss": 3.4188347844948654, + "objective/train/tokens_used": 2327327200, + "theoretical_loss": 3.4188347844948654, + "tokens_seen": 2306867200 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024337011033099298, + "loss": 1.1111, + "theoretical_loss": 3.4188347844948654, + "tokens_seen": 2306867200 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024335406218655968, + "loss": 1.3076, + "theoretical_loss": 3.418826977464866, + "tokens_seen": 2306932736 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024333801404212638, + "loss": 1.2254, + "theoretical_loss": 3.418819170718745, + "tokens_seen": 2306998272 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024332196589769308, + "loss": 1.1884, + "theoretical_loss": 3.418811364256485, + "tokens_seen": 2307063808 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024330591775325978, + "loss": 1.316, + "theoretical_loss": 3.4188035580780665, + "tokens_seen": 2307129344 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024328986960882646, + "loss": 1.3396, + "theoretical_loss": 3.4187957521834713, + "tokens_seen": 2307194880 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024327382146439316, + "loss": 1.2228, + "theoretical_loss": 3.418787946572682, + "tokens_seen": 2307260416 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024325777331995986, + "loss": 1.1799, + "theoretical_loss": 3.418780141245679, + "tokens_seen": 2307325952 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002432417251755266, + "loss": 1.2019, + "theoretical_loss": 3.418772336202445, + "tokens_seen": 2307391488 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024322567703109331, + "loss": 1.204, + "theoretical_loss": 3.418764531442961, + "tokens_seen": 2307457024 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024320962888666001, + "loss": 1.2013, + "theoretical_loss": 3.4187567269672083, + "tokens_seen": 2307522560 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024319358074222672, + "loss": 1.1461, + "theoretical_loss": 3.4187489227751695, + "tokens_seen": 2307588096 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024317753259779342, + "loss": 1.1785, + "theoretical_loss": 3.4187411188668255, + "tokens_seen": 2307653632 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024316148445336012, + "loss": 1.2037, + "theoretical_loss": 3.418733315242158, + "tokens_seen": 2307719168 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024314543630892682, + "loss": 1.3124, + "theoretical_loss": 3.418725511901149, + "tokens_seen": 2307784704 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024312938816449352, + "loss": 1.2337, + "theoretical_loss": 3.4187177088437797, + "tokens_seen": 2307850240 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024311334002006022, + "loss": 1.1818, + "theoretical_loss": 3.4187099060700317, + "tokens_seen": 2307915776 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024309729187562692, + "loss": 1.313, + "theoretical_loss": 3.418702103579887, + "tokens_seen": 2307981312 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024308124373119362, + "loss": 1.217, + "theoretical_loss": 3.4186943013733275, + "tokens_seen": 2308046848 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024306519558676032, + "loss": 1.2899, + "theoretical_loss": 3.418686499450334, + "tokens_seen": 2308112384 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024304914744232702, + "loss": 1.2581, + "theoretical_loss": 3.418678697810889, + "tokens_seen": 2308177920 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002430330992978937, + "loss": 1.2201, + "theoretical_loss": 3.418670896454973, + "tokens_seen": 2308243456 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002430170511534604, + "loss": 1.2428, + "theoretical_loss": 3.418663095382569, + "tokens_seen": 2308308992 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002430010030090271, + "loss": 1.2345, + "theoretical_loss": 3.418655294593658, + "tokens_seen": 2308374528 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002429849548645938, + "loss": 1.1857, + "theoretical_loss": 3.4186474940882214, + "tokens_seen": 2308440064 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002429689067201605, + "loss": 1.1638, + "theoretical_loss": 3.418639693866241, + "tokens_seen": 2308505600 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002429528585757272, + "loss": 1.1704, + "theoretical_loss": 3.418631893927699, + "tokens_seen": 2308571136 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002429368104312939, + "loss": 1.2958, + "theoretical_loss": 3.4186240942725767, + "tokens_seen": 2308636672 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002429207622868606, + "loss": 1.1853, + "theoretical_loss": 3.418616294900855, + "tokens_seen": 2308702208 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002429047141424273, + "loss": 1.1513, + "theoretical_loss": 3.418608495812517, + "tokens_seen": 2308767744 + }, + { + "epoch": 0.7, + "learning_rate": 0.000242888665997994, + "loss": 1.1231, + "theoretical_loss": 3.4186006970075433, + "tokens_seen": 2308833280 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002428726178535607, + "loss": 1.2092, + "theoretical_loss": 3.4185928984859157, + "tokens_seen": 2308898816 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002428565697091274, + "loss": 1.2945, + "theoretical_loss": 3.418585100247616, + "tokens_seen": 2308964352 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002428405215646941, + "loss": 1.252, + "theoretical_loss": 3.4185773022926265, + "tokens_seen": 2309029888 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002428244734202608, + "loss": 1.2378, + "theoretical_loss": 3.418569504620928, + "tokens_seen": 2309095424 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002428084252758275, + "loss": 1.188, + "theoretical_loss": 3.4185617072325023, + "tokens_seen": 2309160960 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002427923771313942, + "loss": 1.2108, + "theoretical_loss": 3.418553910127331, + "tokens_seen": 2309226496 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002427763289869609, + "loss": 1.3484, + "theoretical_loss": 3.4185461133053963, + "tokens_seen": 2309292032 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002427602808425276, + "loss": 1.3047, + "theoretical_loss": 3.4185383167666794, + "tokens_seen": 2309357568 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002427442326980943, + "loss": 1.2167, + "theoretical_loss": 3.418530520511162, + "tokens_seen": 2309423104 + }, + { + "epoch": 0.7, + "learning_rate": 0.000242728184553661, + "loss": 1.268, + "theoretical_loss": 3.418522724538826, + "tokens_seen": 2309488640 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002427121364092277, + "loss": 1.3635, + "theoretical_loss": 3.4185149288496532, + "tokens_seen": 2309554176 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002426960882647944, + "loss": 1.2457, + "theoretical_loss": 3.4185071334436246, + "tokens_seen": 2309619712 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002426800401203611, + "loss": 1.1987, + "theoretical_loss": 3.4184993383207227, + "tokens_seen": 2309685248 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002426639919759278, + "loss": 1.3117, + "theoretical_loss": 3.4184915434809287, + "tokens_seen": 2309750784 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024264794383149448, + "loss": 1.1847, + "theoretical_loss": 3.4184837489242246, + "tokens_seen": 2309816320 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024263189568706118, + "loss": 1.1902, + "theoretical_loss": 3.4184759546505914, + "tokens_seen": 2309881856 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024261584754262788, + "loss": 1.1751, + "theoretical_loss": 3.418468160660012, + "tokens_seen": 2309947392 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024259979939819458, + "loss": 1.3326, + "theoretical_loss": 3.4184603669524667, + "tokens_seen": 2310012928 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024258375125376129, + "loss": 1.2949, + "theoretical_loss": 3.4184525735279383, + "tokens_seen": 2310078464 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 773000, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2139942646026611, + "objective/train/theoretical_loss": 3.418444780386408, + "objective/train/tokens_used": 2330604000, + "theoretical_loss": 3.418444780386408, + "tokens_seen": 2310144000 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024256770310932799, + "loss": 1.1897, + "theoretical_loss": 3.418444780386408, + "tokens_seen": 2310144000 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002425516549648947, + "loss": 1.2346, + "theoretical_loss": 3.4184369875278575, + "tokens_seen": 2310209536 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002425356068204614, + "loss": 1.2794, + "theoretical_loss": 3.4184291949522683, + "tokens_seen": 2310275072 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002425195586760281, + "loss": 1.2384, + "theoretical_loss": 3.418421402659623, + "tokens_seen": 2310340608 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002425035105315948, + "loss": 1.1777, + "theoretical_loss": 3.4184136106499023, + "tokens_seen": 2310406144 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002424874623871615, + "loss": 1.3348, + "theoretical_loss": 3.418405818923088, + "tokens_seen": 2310471680 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002424714142427282, + "loss": 1.2096, + "theoretical_loss": 3.4183980274791623, + "tokens_seen": 2310537216 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002424553660982949, + "loss": 1.2266, + "theoretical_loss": 3.418390236318107, + "tokens_seen": 2310602752 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002424393179538616, + "loss": 1.224, + "theoretical_loss": 3.418382445439903, + "tokens_seen": 2310668288 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002424232698094283, + "loss": 1.2701, + "theoretical_loss": 3.4183746548445324, + "tokens_seen": 2310733824 + }, + { + "epoch": 0.7, + "learning_rate": 0.000242407221664995, + "loss": 1.2155, + "theoretical_loss": 3.4183668645319774, + "tokens_seen": 2310799360 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002423911735205617, + "loss": 1.3593, + "theoretical_loss": 3.418359074502219, + "tokens_seen": 2310864896 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002423751253761284, + "loss": 1.2234, + "theoretical_loss": 3.4183512847552393, + "tokens_seen": 2310930432 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002423590772316951, + "loss": 1.2543, + "theoretical_loss": 3.41834349529102, + "tokens_seen": 2310995968 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002423430290872618, + "loss": 1.2913, + "theoretical_loss": 3.4183357061095427, + "tokens_seen": 2311061504 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002423269809428285, + "loss": 1.2047, + "theoretical_loss": 3.418327917210789, + "tokens_seen": 2311127040 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002423109327983952, + "loss": 1.1791, + "theoretical_loss": 3.4183201285947415, + "tokens_seen": 2311192576 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002422948846539619, + "loss": 1.2353, + "theoretical_loss": 3.4183123402613806, + "tokens_seen": 2311258112 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002422788365095286, + "loss": 1.2173, + "theoretical_loss": 3.4183045522106887, + "tokens_seen": 2311323648 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024226278836509527, + "loss": 1.2546, + "theoretical_loss": 3.4182967644426476, + "tokens_seen": 2311389184 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024224674022066197, + "loss": 1.1064, + "theoretical_loss": 3.418288976957239, + "tokens_seen": 2311454720 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024223069207622867, + "loss": 1.217, + "theoretical_loss": 3.4182811897544445, + "tokens_seen": 2311520256 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024221464393179537, + "loss": 1.23, + "theoretical_loss": 3.418273402834246, + "tokens_seen": 2311585792 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024219859578736207, + "loss": 1.2094, + "theoretical_loss": 3.4182656161966247, + "tokens_seen": 2311651328 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024218254764292877, + "loss": 1.361, + "theoretical_loss": 3.418257829841563, + "tokens_seen": 2311716864 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024216649949849547, + "loss": 1.2967, + "theoretical_loss": 3.418250043769042, + "tokens_seen": 2311782400 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024215045135406218, + "loss": 1.2401, + "theoretical_loss": 3.418242257979044, + "tokens_seen": 2311847936 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024213440320962888, + "loss": 1.2797, + "theoretical_loss": 3.4182344724715508, + "tokens_seen": 2311913472 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024211835506519558, + "loss": 1.1904, + "theoretical_loss": 3.418226687246544, + "tokens_seen": 2311979008 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024210230692076228, + "loss": 1.1804, + "theoretical_loss": 3.418218902304005, + "tokens_seen": 2312044544 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024208625877632898, + "loss": 1.2188, + "theoretical_loss": 3.4182111176439154, + "tokens_seen": 2312110080 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024207021063189568, + "loss": 1.307, + "theoretical_loss": 3.418203333266258, + "tokens_seen": 2312175616 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024205416248746238, + "loss": 1.2398, + "theoretical_loss": 3.4181955491710134, + "tokens_seen": 2312241152 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024203811434302913, + "loss": 1.2119, + "theoretical_loss": 3.418187765358164, + "tokens_seen": 2312306688 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002420220661985958, + "loss": 1.1762, + "theoretical_loss": 3.4181799818276914, + "tokens_seen": 2312372224 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002420060180541625, + "loss": 1.2232, + "theoretical_loss": 3.418172198579577, + "tokens_seen": 2312437760 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002419899699097292, + "loss": 1.2855, + "theoretical_loss": 3.4181644156138034, + "tokens_seen": 2312503296 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002419739217652959, + "loss": 1.243, + "theoretical_loss": 3.4181566329303514, + "tokens_seen": 2312568832 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002419578736208626, + "loss": 1.16, + "theoretical_loss": 3.4181488505292035, + "tokens_seen": 2312634368 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002419418254764293, + "loss": 1.2766, + "theoretical_loss": 3.418141068410341, + "tokens_seen": 2312699904 + }, + { + "epoch": 0.7, + "learning_rate": 0.000241925777331996, + "loss": 1.0887, + "theoretical_loss": 3.4181332865737457, + "tokens_seen": 2312765440 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002419097291875627, + "loss": 1.2448, + "theoretical_loss": 3.4181255050194, + "tokens_seen": 2312830976 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002418936810431294, + "loss": 1.1304, + "theoretical_loss": 3.418117723747285, + "tokens_seen": 2312896512 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002418776328986961, + "loss": 1.1748, + "theoretical_loss": 3.418109942757382, + "tokens_seen": 2312962048 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024186158475426281, + "loss": 1.1968, + "theoretical_loss": 3.418102162049674, + "tokens_seen": 2313027584 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024184553660982951, + "loss": 1.191, + "theoretical_loss": 3.418094381624142, + "tokens_seen": 2313093120 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024182948846539622, + "loss": 1.2321, + "theoretical_loss": 3.418086601480768, + "tokens_seen": 2313158656 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024181344032096292, + "loss": 1.2134, + "theoretical_loss": 3.4180788216195332, + "tokens_seen": 2313224192 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024179739217652962, + "loss": 1.2907, + "theoretical_loss": 3.4180710420404203, + "tokens_seen": 2313289728 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024178134403209632, + "loss": 1.2111, + "theoretical_loss": 3.4180632627434107, + "tokens_seen": 2313355264 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 773923, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1678115129470825, + "objective/train/theoretical_loss": 3.418055483728486, + "objective/train/tokens_used": 2333880800, + "theoretical_loss": 3.418055483728486, + "tokens_seen": 2313420800 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024176529588766302, + "loss": 1.2176, + "theoretical_loss": 3.418055483728486, + "tokens_seen": 2313420800 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024174924774322972, + "loss": 1.3111, + "theoretical_loss": 3.418047704995628, + "tokens_seen": 2313486336 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024173319959879642, + "loss": 1.208, + "theoretical_loss": 3.418039926544819, + "tokens_seen": 2313551872 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024171715145436312, + "loss": 1.2033, + "theoretical_loss": 3.41803214837604, + "tokens_seen": 2313617408 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024170110330992982, + "loss": 1.2241, + "theoretical_loss": 3.4180243704892734, + "tokens_seen": 2313682944 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024168505516549652, + "loss": 1.2007, + "theoretical_loss": 3.4180165928845008, + "tokens_seen": 2313748480 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024166900702106322, + "loss": 1.2143, + "theoretical_loss": 3.418008815561704, + "tokens_seen": 2313814016 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024165295887662992, + "loss": 1.1417, + "theoretical_loss": 3.418001038520864, + "tokens_seen": 2313879552 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002416369107321966, + "loss": 1.2981, + "theoretical_loss": 3.417993261761964, + "tokens_seen": 2313945088 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002416208625877633, + "loss": 1.1643, + "theoretical_loss": 3.4179854852849854, + "tokens_seen": 2314010624 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024160481444333, + "loss": 1.2872, + "theoretical_loss": 3.417977709089909, + "tokens_seen": 2314076160 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002415887662988967, + "loss": 1.1273, + "theoretical_loss": 3.417969933176718, + "tokens_seen": 2314141696 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002415727181544634, + "loss": 1.2216, + "theoretical_loss": 3.417962157545393, + "tokens_seen": 2314207232 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002415566700100301, + "loss": 1.179, + "theoretical_loss": 3.4179543821959166, + "tokens_seen": 2314272768 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002415406218655968, + "loss": 1.2121, + "theoretical_loss": 3.41794660712827, + "tokens_seen": 2314338304 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002415245737211635, + "loss": 1.1965, + "theoretical_loss": 3.4179388323424353, + "tokens_seen": 2314403840 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002415085255767302, + "loss": 1.2813, + "theoretical_loss": 3.417931057838395, + "tokens_seen": 2314469376 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002414924774322969, + "loss": 1.2938, + "theoretical_loss": 3.4179232836161297, + "tokens_seen": 2314534912 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002414764292878636, + "loss": 1.1113, + "theoretical_loss": 3.4179155096756215, + "tokens_seen": 2314600448 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002414603811434303, + "loss": 1.1911, + "theoretical_loss": 3.417907736016853, + "tokens_seen": 2314665984 + }, + { + "epoch": 0.7, + "learning_rate": 0.000241444332998997, + "loss": 1.2883, + "theoretical_loss": 3.417899962639805, + "tokens_seen": 2314731520 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002414282848545637, + "loss": 1.3057, + "theoretical_loss": 3.4178921895444603, + "tokens_seen": 2314797056 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002414122367101304, + "loss": 1.2054, + "theoretical_loss": 3.4178844167308, + "tokens_seen": 2314862592 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002413961885656971, + "loss": 1.3421, + "theoretical_loss": 3.417876644198806, + "tokens_seen": 2314928128 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002413801404212638, + "loss": 1.1826, + "theoretical_loss": 3.41786887194846, + "tokens_seen": 2314993664 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002413640922768305, + "loss": 1.2321, + "theoretical_loss": 3.4178610999797447, + "tokens_seen": 2315059200 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002413480441323972, + "loss": 1.2188, + "theoretical_loss": 3.4178533282926407, + "tokens_seen": 2315124736 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002413319959879639, + "loss": 1.286, + "theoretical_loss": 3.417845556887131, + "tokens_seen": 2315190272 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002413159478435306, + "loss": 1.171, + "theoretical_loss": 3.4178377857631963, + "tokens_seen": 2315255808 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002412998996990973, + "loss": 1.2522, + "theoretical_loss": 3.4178300149208187, + "tokens_seen": 2315321344 + }, + { + "epoch": 0.7, + "learning_rate": 0.000241283851554664, + "loss": 1.2135, + "theoretical_loss": 3.4178222443599813, + "tokens_seen": 2315386880 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002412678034102307, + "loss": 1.3311, + "theoretical_loss": 3.417814474080664, + "tokens_seen": 2315452416 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024125175526579738, + "loss": 1.2337, + "theoretical_loss": 3.41780670408285, + "tokens_seen": 2315517952 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024123570712136408, + "loss": 1.325, + "theoretical_loss": 3.4177989343665205, + "tokens_seen": 2315583488 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024121965897693078, + "loss": 1.2764, + "theoretical_loss": 3.417791164931658, + "tokens_seen": 2315649024 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024120361083249749, + "loss": 1.1479, + "theoretical_loss": 3.417783395778243, + "tokens_seen": 2315714560 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024118756268806419, + "loss": 1.1879, + "theoretical_loss": 3.4177756269062587, + "tokens_seen": 2315780096 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002411715145436309, + "loss": 1.1476, + "theoretical_loss": 3.4177678583156865, + "tokens_seen": 2315845632 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002411554663991976, + "loss": 1.1463, + "theoretical_loss": 3.417760090006508, + "tokens_seen": 2315911168 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002411394182547643, + "loss": 1.206, + "theoretical_loss": 3.4177523219787056, + "tokens_seen": 2315976704 + }, + { + "epoch": 0.7, + "learning_rate": 0.000241123370110331, + "loss": 1.1195, + "theoretical_loss": 3.4177445542322604, + "tokens_seen": 2316042240 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002411073219658977, + "loss": 1.2057, + "theoretical_loss": 3.4177367867671546, + "tokens_seen": 2316107776 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002410912738214644, + "loss": 1.2296, + "theoretical_loss": 3.4177290195833705, + "tokens_seen": 2316173312 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002410752256770311, + "loss": 1.1976, + "theoretical_loss": 3.417721252680889, + "tokens_seen": 2316238848 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002410591775325978, + "loss": 1.3124, + "theoretical_loss": 3.417713486059693, + "tokens_seen": 2316304384 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002410431293881645, + "loss": 1.1993, + "theoretical_loss": 3.4177057197197636, + "tokens_seen": 2316369920 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002410270812437312, + "loss": 1.2105, + "theoretical_loss": 3.417697953661083, + "tokens_seen": 2316435456 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002410110330992979, + "loss": 1.303, + "theoretical_loss": 3.4176901878836325, + "tokens_seen": 2316500992 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002409949849548646, + "loss": 1.1791, + "theoretical_loss": 3.417682422387395, + "tokens_seen": 2316566528 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002409789368104313, + "loss": 1.1918, + "theoretical_loss": 3.417674657172351, + "tokens_seen": 2316632064 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 774990, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1873466968536377, + "objective/train/theoretical_loss": 3.417666892238484, + "objective/train/tokens_used": 2337157600, + "theoretical_loss": 3.417666892238484, + "tokens_seen": 2316697600 + }, + { + "epoch": 0.7, + "learning_rate": 0.000240962888665998, + "loss": 1.2267, + "theoretical_loss": 3.417666892238484, + "tokens_seen": 2316697600 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002409468405215647, + "loss": 1.1805, + "theoretical_loss": 3.4176591275857744, + "tokens_seen": 2316763136 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002409307923771314, + "loss": 1.3056, + "theoretical_loss": 3.417651363214205, + "tokens_seen": 2316828672 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002409147442326981, + "loss": 1.2232, + "theoretical_loss": 3.4176435991237577, + "tokens_seen": 2316894208 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002408986960882648, + "loss": 1.053, + "theoretical_loss": 3.4176358353144134, + "tokens_seen": 2316959744 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002408826479438315, + "loss": 1.2256, + "theoretical_loss": 3.4176280717861545, + "tokens_seen": 2317025280 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024086659979939817, + "loss": 1.167, + "theoretical_loss": 3.4176203085389636, + "tokens_seen": 2317090816 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024085055165496487, + "loss": 1.2503, + "theoretical_loss": 3.4176125455728212, + "tokens_seen": 2317156352 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024083450351053163, + "loss": 1.1633, + "theoretical_loss": 3.41760478288771, + "tokens_seen": 2317221888 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024081845536609833, + "loss": 1.2474, + "theoretical_loss": 3.4175970204836124, + "tokens_seen": 2317287424 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024080240722166503, + "loss": 1.2162, + "theoretical_loss": 3.4175892583605094, + "tokens_seen": 2317352960 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024078635907723173, + "loss": 1.2649, + "theoretical_loss": 3.417581496518383, + "tokens_seen": 2317418496 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024077031093279843, + "loss": 1.2525, + "theoretical_loss": 3.417573734957215, + "tokens_seen": 2317484032 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024075426278836513, + "loss": 1.2205, + "theoretical_loss": 3.4175659736769877, + "tokens_seen": 2317549568 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024073821464393183, + "loss": 1.174, + "theoretical_loss": 3.417558212677683, + "tokens_seen": 2317615104 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024072216649949853, + "loss": 1.2988, + "theoretical_loss": 3.4175504519592828, + "tokens_seen": 2317680640 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024070611835506523, + "loss": 1.2362, + "theoretical_loss": 3.417542691521768, + "tokens_seen": 2317746176 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024069007021063193, + "loss": 1.2468, + "theoretical_loss": 3.4175349313651218, + "tokens_seen": 2317811712 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024067402206619863, + "loss": 1.2884, + "theoretical_loss": 3.4175271714893256, + "tokens_seen": 2317877248 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024065797392176533, + "loss": 1.244, + "theoretical_loss": 3.4175194118943613, + "tokens_seen": 2317942784 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024064192577733203, + "loss": 1.3247, + "theoretical_loss": 3.4175116525802105, + "tokens_seen": 2318008320 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002406258776328987, + "loss": 1.1485, + "theoretical_loss": 3.4175038935468556, + "tokens_seen": 2318073856 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002406098294884654, + "loss": 1.1154, + "theoretical_loss": 3.417496134794278, + "tokens_seen": 2318139392 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002405937813440321, + "loss": 1.2538, + "theoretical_loss": 3.41748837632246, + "tokens_seen": 2318204928 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002405777331995988, + "loss": 1.2356, + "theoretical_loss": 3.4174806181313833, + "tokens_seen": 2318270464 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002405616850551655, + "loss": 1.1618, + "theoretical_loss": 3.41747286022103, + "tokens_seen": 2318336000 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002405456369107322, + "loss": 1.2046, + "theoretical_loss": 3.4174651025913816, + "tokens_seen": 2318401536 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002405295887662989, + "loss": 1.1598, + "theoretical_loss": 3.4174573452424206, + "tokens_seen": 2318467072 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002405135406218656, + "loss": 1.218, + "theoretical_loss": 3.4174495881741285, + "tokens_seen": 2318532608 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002404974924774323, + "loss": 1.207, + "theoretical_loss": 3.4174418313864874, + "tokens_seen": 2318598144 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024048144433299901, + "loss": 1.2691, + "theoretical_loss": 3.4174340748794787, + "tokens_seen": 2318663680 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024046539618856571, + "loss": 1.2642, + "theoretical_loss": 3.417426318653085, + "tokens_seen": 2318729216 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024044934804413242, + "loss": 1.1647, + "theoretical_loss": 3.4174185627072884, + "tokens_seen": 2318794752 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024043329989969912, + "loss": 1.1665, + "theoretical_loss": 3.41741080704207, + "tokens_seen": 2318860288 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024041725175526582, + "loss": 1.2644, + "theoretical_loss": 3.417403051657412, + "tokens_seen": 2318925824 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024040120361083252, + "loss": 1.2616, + "theoretical_loss": 3.4173952965532965, + "tokens_seen": 2318991360 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024038515546639922, + "loss": 1.3224, + "theoretical_loss": 3.4173875417297053, + "tokens_seen": 2319056896 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024036910732196592, + "loss": 1.2479, + "theoretical_loss": 3.4173797871866203, + "tokens_seen": 2319122432 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024035305917753262, + "loss": 1.2857, + "theoretical_loss": 3.417372032924024, + "tokens_seen": 2319187968 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024033701103309932, + "loss": 1.191, + "theoretical_loss": 3.417364278941897, + "tokens_seen": 2319253504 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024032096288866602, + "loss": 1.3059, + "theoretical_loss": 3.4173565252402223, + "tokens_seen": 2319319040 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024030491474423272, + "loss": 1.3087, + "theoretical_loss": 3.4173487718189817, + "tokens_seen": 2319384576 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024028886659979942, + "loss": 1.2734, + "theoretical_loss": 3.417341018678157, + "tokens_seen": 2319450112 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024027281845536612, + "loss": 1.2308, + "theoretical_loss": 3.4173332658177302, + "tokens_seen": 2319515648 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024025677031093282, + "loss": 1.2498, + "theoretical_loss": 3.4173255132376834, + "tokens_seen": 2319581184 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002402407221664995, + "loss": 1.2062, + "theoretical_loss": 3.417317760937998, + "tokens_seen": 2319646720 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002402246740220662, + "loss": 1.1652, + "theoretical_loss": 3.4173100089186565, + "tokens_seen": 2319712256 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002402086258776329, + "loss": 1.2413, + "theoretical_loss": 3.4173022571796405, + "tokens_seen": 2319777792 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002401925777331996, + "loss": 1.1894, + "theoretical_loss": 3.417294505720932, + "tokens_seen": 2319843328 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002401765295887663, + "loss": 1.1982, + "theoretical_loss": 3.417286754542513, + "tokens_seen": 2319908864 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 775797, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4274017810821533, + "objective/train/theoretical_loss": 3.4172790036443654, + "objective/train/tokens_used": 2340434400, + "theoretical_loss": 3.4172790036443654, + "tokens_seen": 2319974400 + }, + { + "epoch": 0.7, + "learning_rate": 0.000240160481444333, + "loss": 1.2221, + "theoretical_loss": 3.4172790036443654, + "tokens_seen": 2319974400 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002401444332998997, + "loss": 1.1468, + "theoretical_loss": 3.4172712530264713, + "tokens_seen": 2320039936 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002401283851554664, + "loss": 1.2992, + "theoretical_loss": 3.4172635026888125, + "tokens_seen": 2320105472 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002401123370110331, + "loss": 1.271, + "theoretical_loss": 3.4172557526313714, + "tokens_seen": 2320171008 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002400962888665998, + "loss": 1.2403, + "theoretical_loss": 3.417248002854129, + "tokens_seen": 2320236544 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002400802407221665, + "loss": 1.2454, + "theoretical_loss": 3.4172402533570683, + "tokens_seen": 2320302080 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002400641925777332, + "loss": 1.2685, + "theoretical_loss": 3.4172325041401703, + "tokens_seen": 2320367616 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002400481444332999, + "loss": 1.23, + "theoretical_loss": 3.4172247552034176, + "tokens_seen": 2320433152 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002400320962888666, + "loss": 1.2113, + "theoretical_loss": 3.417217006546792, + "tokens_seen": 2320498688 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002400160481444333, + "loss": 1.2328, + "theoretical_loss": 3.4172092581702755, + "tokens_seen": 2320564224 + }, + { + "epoch": 0.7, + "learning_rate": 0.00024, + "loss": 1.2716, + "theoretical_loss": 3.4172015100738498, + "tokens_seen": 2320629760 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002399839518555667, + "loss": 1.1896, + "theoretical_loss": 3.4171937622574973, + "tokens_seen": 2320695296 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002399679037111334, + "loss": 1.2135, + "theoretical_loss": 3.4171860147211994, + "tokens_seen": 2320760832 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002399518555667001, + "loss": 1.2511, + "theoretical_loss": 3.417178267464939, + "tokens_seen": 2320826368 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002399358074222668, + "loss": 1.1029, + "theoretical_loss": 3.417170520488697, + "tokens_seen": 2320891904 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002399197592778335, + "loss": 1.2411, + "theoretical_loss": 3.417162773792456, + "tokens_seen": 2320957440 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002399037111334002, + "loss": 1.2512, + "theoretical_loss": 3.4171550273761984, + "tokens_seen": 2321022976 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002398876629889669, + "loss": 1.2865, + "theoretical_loss": 3.4171472812399046, + "tokens_seen": 2321088512 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002398716148445336, + "loss": 1.1996, + "theoretical_loss": 3.4171395353835585, + "tokens_seen": 2321154048 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023985556670010028, + "loss": 1.2874, + "theoretical_loss": 3.4171317898071405, + "tokens_seen": 2321219584 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023983951855566699, + "loss": 1.2396, + "theoretical_loss": 3.4171240445106337, + "tokens_seen": 2321285120 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023982347041123369, + "loss": 1.2183, + "theoretical_loss": 3.417116299494019, + "tokens_seen": 2321350656 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002398074222668004, + "loss": 1.2177, + "theoretical_loss": 3.4171085547572795, + "tokens_seen": 2321416192 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002397913741223671, + "loss": 1.2373, + "theoretical_loss": 3.4171008103003966, + "tokens_seen": 2321481728 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002397753259779338, + "loss": 1.2279, + "theoretical_loss": 3.4170930661233525, + "tokens_seen": 2321547264 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002397592778335005, + "loss": 1.2978, + "theoretical_loss": 3.417085322226129, + "tokens_seen": 2321612800 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002397432296890672, + "loss": 1.2702, + "theoretical_loss": 3.417077578608708, + "tokens_seen": 2321678336 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002397271815446339, + "loss": 1.2528, + "theoretical_loss": 3.4170698352710716, + "tokens_seen": 2321743872 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002397111334002006, + "loss": 1.2094, + "theoretical_loss": 3.4170620922132025, + "tokens_seen": 2321809408 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002396950852557673, + "loss": 1.1891, + "theoretical_loss": 3.417054349435081, + "tokens_seen": 2321874944 + }, + { + "epoch": 0.7, + "learning_rate": 0.000239679037111334, + "loss": 1.2374, + "theoretical_loss": 3.417046606936691, + "tokens_seen": 2321940480 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002396629889669007, + "loss": 1.2481, + "theoretical_loss": 3.4170388647180134, + "tokens_seen": 2322006016 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002396469408224674, + "loss": 1.2104, + "theoretical_loss": 3.41703112277903, + "tokens_seen": 2322071552 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023963089267803415, + "loss": 1.336, + "theoretical_loss": 3.417023381119724, + "tokens_seen": 2322137088 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023961484453360085, + "loss": 1.2554, + "theoretical_loss": 3.4170156397400766, + "tokens_seen": 2322202624 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023959879638916752, + "loss": 1.2703, + "theoretical_loss": 3.4170078986400694, + "tokens_seen": 2322268160 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023958274824473422, + "loss": 1.2716, + "theoretical_loss": 3.417000157819685, + "tokens_seen": 2322333696 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023956670010030092, + "loss": 1.2542, + "theoretical_loss": 3.4169924172789052, + "tokens_seen": 2322399232 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023955065195586762, + "loss": 1.2212, + "theoretical_loss": 3.4169846770177124, + "tokens_seen": 2322464768 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023953460381143432, + "loss": 1.2614, + "theoretical_loss": 3.416976937036088, + "tokens_seen": 2322530304 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023951855566700103, + "loss": 1.3248, + "theoretical_loss": 3.4169691973340144, + "tokens_seen": 2322595840 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023950250752256773, + "loss": 1.2007, + "theoretical_loss": 3.4169614579114733, + "tokens_seen": 2322661376 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023948645937813443, + "loss": 1.2513, + "theoretical_loss": 3.4169537187684473, + "tokens_seen": 2322726912 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023947041123370113, + "loss": 1.2448, + "theoretical_loss": 3.4169459799049178, + "tokens_seen": 2322792448 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023945436308926783, + "loss": 1.1328, + "theoretical_loss": 3.4169382413208673, + "tokens_seen": 2322857984 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023943831494483453, + "loss": 1.2669, + "theoretical_loss": 3.4169305030162773, + "tokens_seen": 2322923520 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023942226680040123, + "loss": 1.2887, + "theoretical_loss": 3.4169227649911305, + "tokens_seen": 2322989056 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023940621865596793, + "loss": 1.2077, + "theoretical_loss": 3.4169150272454085, + "tokens_seen": 2323054592 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023939017051153463, + "loss": 1.26, + "theoretical_loss": 3.4169072897790933, + "tokens_seen": 2323120128 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023937412236710133, + "loss": 1.2425, + "theoretical_loss": 3.4168995525921666, + "tokens_seen": 2323185664 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 777104, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3013246059417725, + "objective/train/theoretical_loss": 3.4168918156846115, + "objective/train/tokens_used": 2343711200, + "theoretical_loss": 3.4168918156846115, + "tokens_seen": 2323251200 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023935807422266803, + "loss": 1.1993, + "theoretical_loss": 3.4168918156846115, + "tokens_seen": 2323251200 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023934202607823473, + "loss": 1.2379, + "theoretical_loss": 3.416884079056409, + "tokens_seen": 2323316736 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023932597793380143, + "loss": 1.2192, + "theoretical_loss": 3.4168763427075417, + "tokens_seen": 2323382272 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023930992978936813, + "loss": 1.3569, + "theoretical_loss": 3.4168686066379914, + "tokens_seen": 2323447808 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023929388164493483, + "loss": 1.2539, + "theoretical_loss": 3.41686087084774, + "tokens_seen": 2323513344 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023927783350050153, + "loss": 1.2552, + "theoretical_loss": 3.41685313533677, + "tokens_seen": 2323578880 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023926178535606824, + "loss": 1.215, + "theoretical_loss": 3.4168454001050628, + "tokens_seen": 2323644416 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023924573721163494, + "loss": 1.1481, + "theoretical_loss": 3.4168376651526007, + "tokens_seen": 2323709952 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023922968906720164, + "loss": 1.0823, + "theoretical_loss": 3.4168299304793663, + "tokens_seen": 2323775488 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002392136409227683, + "loss": 1.2402, + "theoretical_loss": 3.416822196085341, + "tokens_seen": 2323841024 + }, + { + "epoch": 0.7, + "learning_rate": 0.000239197592778335, + "loss": 1.2029, + "theoretical_loss": 3.416814461970507, + "tokens_seen": 2323906560 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002391815446339017, + "loss": 1.239, + "theoretical_loss": 3.416806728134847, + "tokens_seen": 2323972096 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002391654964894684, + "loss": 1.1811, + "theoretical_loss": 3.4167989945783415, + "tokens_seen": 2324037632 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002391494483450351, + "loss": 1.2025, + "theoretical_loss": 3.4167912613009737, + "tokens_seen": 2324103168 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002391334002006018, + "loss": 1.1869, + "theoretical_loss": 3.4167835283027257, + "tokens_seen": 2324168704 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023911735205616851, + "loss": 1.2109, + "theoretical_loss": 3.4167757955835794, + "tokens_seen": 2324234240 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023910130391173521, + "loss": 1.2326, + "theoretical_loss": 3.4167680631435164, + "tokens_seen": 2324299776 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023908525576730192, + "loss": 1.3234, + "theoretical_loss": 3.4167603309825196, + "tokens_seen": 2324365312 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023906920762286862, + "loss": 1.2298, + "theoretical_loss": 3.41675259910057, + "tokens_seen": 2324430848 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023905315947843532, + "loss": 1.2529, + "theoretical_loss": 3.4167448674976506, + "tokens_seen": 2324496384 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023903711133400202, + "loss": 1.2117, + "theoretical_loss": 3.416737136173743, + "tokens_seen": 2324561920 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023902106318956872, + "loss": 1.3468, + "theoretical_loss": 3.4167294051288293, + "tokens_seen": 2324627456 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023900501504513542, + "loss": 1.2653, + "theoretical_loss": 3.416721674362892, + "tokens_seen": 2324692992 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023898896690070212, + "loss": 1.2799, + "theoretical_loss": 3.416713943875912, + "tokens_seen": 2324758528 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023897291875626882, + "loss": 1.1944, + "theoretical_loss": 3.416706213667873, + "tokens_seen": 2324824064 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023895687061183552, + "loss": 1.2337, + "theoretical_loss": 3.416698483738756, + "tokens_seen": 2324889600 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023894082246740222, + "loss": 1.2246, + "theoretical_loss": 3.416690754088543, + "tokens_seen": 2324955136 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023892477432296892, + "loss": 1.2824, + "theoretical_loss": 3.4166830247172166, + "tokens_seen": 2325020672 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023890872617853562, + "loss": 1.2872, + "theoretical_loss": 3.416675295624759, + "tokens_seen": 2325086208 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023889267803410232, + "loss": 1.2534, + "theoretical_loss": 3.4166675668111517, + "tokens_seen": 2325151744 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023887662988966902, + "loss": 1.2224, + "theoretical_loss": 3.4166598382763773, + "tokens_seen": 2325217280 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023886058174523572, + "loss": 1.1143, + "theoretical_loss": 3.4166521100204172, + "tokens_seen": 2325282816 + }, + { + "epoch": 0.7, + "learning_rate": 0.00023884453360080242, + "loss": 1.2768, + "theoretical_loss": 3.4166443820432537, + "tokens_seen": 2325348352 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002388284854563691, + "loss": 1.158, + "theoretical_loss": 3.4166366543448694, + "tokens_seen": 2325413888 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002388124373119358, + "loss": 1.0745, + "theoretical_loss": 3.416628926925246, + "tokens_seen": 2325479424 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002387963891675025, + "loss": 1.153, + "theoretical_loss": 3.416621199784366, + "tokens_seen": 2325544960 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002387803410230692, + "loss": 1.3482, + "theoretical_loss": 3.416613472922211, + "tokens_seen": 2325610496 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002387642928786359, + "loss": 1.2649, + "theoretical_loss": 3.4166057463387633, + "tokens_seen": 2325676032 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002387482447342026, + "loss": 1.2798, + "theoretical_loss": 3.4165980200340047, + "tokens_seen": 2325741568 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002387321965897693, + "loss": 1.2071, + "theoretical_loss": 3.4165902940079174, + "tokens_seen": 2325807104 + }, + { + "epoch": 0.7, + "learning_rate": 0.000238716148445336, + "loss": 1.1782, + "theoretical_loss": 3.416582568260484, + "tokens_seen": 2325872640 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002387001003009027, + "loss": 1.2734, + "theoretical_loss": 3.416574842791686, + "tokens_seen": 2325938176 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002386840521564694, + "loss": 1.2, + "theoretical_loss": 3.416567117601506, + "tokens_seen": 2326003712 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002386680040120361, + "loss": 1.1891, + "theoretical_loss": 3.4165593926899254, + "tokens_seen": 2326069248 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002386519558676028, + "loss": 1.249, + "theoretical_loss": 3.4165516680569272, + "tokens_seen": 2326134784 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002386359077231695, + "loss": 1.2563, + "theoretical_loss": 3.4165439437024925, + "tokens_seen": 2326200320 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002386198595787362, + "loss": 1.4344, + "theoretical_loss": 3.4165362196266043, + "tokens_seen": 2326265856 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002386038114343029, + "loss": 1.2375, + "theoretical_loss": 3.4165284958292443, + "tokens_seen": 2326331392 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002385877632898696, + "loss": 1.2933, + "theoretical_loss": 3.4165207723103945, + "tokens_seen": 2326396928 + }, + { + "epoch": 0.7, + "learning_rate": 0.0002385717151454363, + "loss": 1.1979, + "theoretical_loss": 3.4165130490700375, + "tokens_seen": 2326462464 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 777937, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2629358768463135, + "objective/train/theoretical_loss": 3.4165053261081546, + "objective/train/tokens_used": 2346988000, + "theoretical_loss": 3.4165053261081546, + "tokens_seen": 2326528000 + }, + { + "epoch": 0.71, + "learning_rate": 0.000238555667001003, + "loss": 1.2397, + "theoretical_loss": 3.4165053261081546, + "tokens_seen": 2326528000 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002385396188565697, + "loss": 1.1785, + "theoretical_loss": 3.416497603424729, + "tokens_seen": 2326593536 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002385235707121364, + "loss": 1.2778, + "theoretical_loss": 3.416489881019742, + "tokens_seen": 2326659072 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002385075225677031, + "loss": 1.2964, + "theoretical_loss": 3.4164821588931757, + "tokens_seen": 2326724608 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002384914744232698, + "loss": 1.2518, + "theoretical_loss": 3.4164744370450126, + "tokens_seen": 2326790144 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002384754262788365, + "loss": 1.2052, + "theoretical_loss": 3.416466715475235, + "tokens_seen": 2326855680 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002384593781344032, + "loss": 1.2613, + "theoretical_loss": 3.416458994183824, + "tokens_seen": 2326921216 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023844332998996989, + "loss": 1.2501, + "theoretical_loss": 3.416451273170763, + "tokens_seen": 2326986752 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023842728184553664, + "loss": 1.272, + "theoretical_loss": 3.4164435524360335, + "tokens_seen": 2327052288 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023841123370110334, + "loss": 1.2377, + "theoretical_loss": 3.4164358319796175, + "tokens_seen": 2327117824 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023839518555667004, + "loss": 1.2464, + "theoretical_loss": 3.4164281118014976, + "tokens_seen": 2327183360 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023837913741223674, + "loss": 1.2954, + "theoretical_loss": 3.4164203919016556, + "tokens_seen": 2327248896 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023836308926780344, + "loss": 1.2255, + "theoretical_loss": 3.416412672280073, + "tokens_seen": 2327314432 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023834704112337014, + "loss": 1.2597, + "theoretical_loss": 3.4164049529367335, + "tokens_seen": 2327379968 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023833099297893684, + "loss": 1.1865, + "theoretical_loss": 3.416397233871618, + "tokens_seen": 2327445504 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023831494483450355, + "loss": 1.1867, + "theoretical_loss": 3.416389515084709, + "tokens_seen": 2327511040 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023829889669007025, + "loss": 1.2868, + "theoretical_loss": 3.4163817965759886, + "tokens_seen": 2327576576 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023828284854563695, + "loss": 1.2104, + "theoretical_loss": 3.416374078345439, + "tokens_seen": 2327642112 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023826680040120365, + "loss": 1.2571, + "theoretical_loss": 3.4163663603930425, + "tokens_seen": 2327707648 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023825075225677035, + "loss": 1.2077, + "theoretical_loss": 3.4163586427187806, + "tokens_seen": 2327773184 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023823470411233705, + "loss": 1.1783, + "theoretical_loss": 3.4163509253226363, + "tokens_seen": 2327838720 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023821865596790375, + "loss": 1.1714, + "theoretical_loss": 3.4163432082045913, + "tokens_seen": 2327904256 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023820260782347042, + "loss": 1.224, + "theoretical_loss": 3.4163354913646273, + "tokens_seen": 2327969792 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023818655967903712, + "loss": 1.3062, + "theoretical_loss": 3.4163277748027276, + "tokens_seen": 2328035328 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023817051153460382, + "loss": 1.3114, + "theoretical_loss": 3.4163200585188735, + "tokens_seen": 2328100864 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023815446339017052, + "loss": 1.2085, + "theoretical_loss": 3.4163123425130473, + "tokens_seen": 2328166400 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023813841524573723, + "loss": 1.2192, + "theoretical_loss": 3.4163046267852315, + "tokens_seen": 2328231936 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023812236710130393, + "loss": 1.2539, + "theoretical_loss": 3.4162969113354076, + "tokens_seen": 2328297472 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023810631895687063, + "loss": 1.2111, + "theoretical_loss": 3.416289196163558, + "tokens_seen": 2328363008 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023809027081243733, + "loss": 1.243, + "theoretical_loss": 3.4162814812696656, + "tokens_seen": 2328428544 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023807422266800403, + "loss": 1.1425, + "theoretical_loss": 3.416273766653712, + "tokens_seen": 2328494080 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023805817452357073, + "loss": 1.1676, + "theoretical_loss": 3.4162660523156787, + "tokens_seen": 2328559616 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023804212637913743, + "loss": 1.254, + "theoretical_loss": 3.416258338255549, + "tokens_seen": 2328625152 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023802607823470413, + "loss": 1.1355, + "theoretical_loss": 3.416250624473304, + "tokens_seen": 2328690688 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023801003009027083, + "loss": 1.3077, + "theoretical_loss": 3.416242910968927, + "tokens_seen": 2328756224 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023799398194583753, + "loss": 1.2697, + "theoretical_loss": 3.4162351977423993, + "tokens_seen": 2328821760 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023797793380140423, + "loss": 1.2941, + "theoretical_loss": 3.4162274847937035, + "tokens_seen": 2328887296 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023796188565697093, + "loss": 1.245, + "theoretical_loss": 3.4162197721228216, + "tokens_seen": 2328952832 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023794583751253763, + "loss": 1.1329, + "theoretical_loss": 3.4162120597297356, + "tokens_seen": 2329018368 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023792978936810433, + "loss": 1.258, + "theoretical_loss": 3.4162043476144284, + "tokens_seen": 2329083904 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023791374122367103, + "loss": 1.1703, + "theoretical_loss": 3.4161966357768816, + "tokens_seen": 2329149440 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023789769307923773, + "loss": 1.2445, + "theoretical_loss": 3.416188924217077, + "tokens_seen": 2329214976 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023788164493480444, + "loss": 1.3041, + "theoretical_loss": 3.4161812129349975, + "tokens_seen": 2329280512 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023786559679037114, + "loss": 1.2789, + "theoretical_loss": 3.416173501930625, + "tokens_seen": 2329346048 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023784954864593784, + "loss": 1.2527, + "theoretical_loss": 3.416165791203942, + "tokens_seen": 2329411584 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023783350050150454, + "loss": 1.2361, + "theoretical_loss": 3.41615808075493, + "tokens_seen": 2329477120 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002378174523570712, + "loss": 1.1774, + "theoretical_loss": 3.416150370583572, + "tokens_seen": 2329542656 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002378014042126379, + "loss": 1.1653, + "theoretical_loss": 3.4161426606898493, + "tokens_seen": 2329608192 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002377853560682046, + "loss": 1.2219, + "theoretical_loss": 3.416134951073745, + "tokens_seen": 2329673728 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002377693079237713, + "loss": 1.3072, + "theoretical_loss": 3.416127241735241, + "tokens_seen": 2329739264 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 779113, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2098369598388672, + "objective/train/theoretical_loss": 3.416119532674319, + "objective/train/tokens_used": 2350264800, + "theoretical_loss": 3.416119532674319, + "tokens_seen": 2329804800 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023775325977933801, + "loss": 1.2314, + "theoretical_loss": 3.416119532674319, + "tokens_seen": 2329804800 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023773721163490471, + "loss": 1.1416, + "theoretical_loss": 3.4161118238909616, + "tokens_seen": 2329870336 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023772116349047141, + "loss": 1.2683, + "theoretical_loss": 3.416104115385151, + "tokens_seen": 2329935872 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023770511534603812, + "loss": 1.1155, + "theoretical_loss": 3.4160964071568696, + "tokens_seen": 2330001408 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023768906720160482, + "loss": 1.2445, + "theoretical_loss": 3.416088699206099, + "tokens_seen": 2330066944 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023767301905717152, + "loss": 1.1785, + "theoretical_loss": 3.416080991532822, + "tokens_seen": 2330132480 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023765697091273822, + "loss": 1.2796, + "theoretical_loss": 3.4160732841370205, + "tokens_seen": 2330198016 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023764092276830492, + "loss": 1.2182, + "theoretical_loss": 3.416065577018677, + "tokens_seen": 2330263552 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023762487462387162, + "loss": 1.2349, + "theoretical_loss": 3.416057870177773, + "tokens_seen": 2330329088 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023760882647943832, + "loss": 1.2582, + "theoretical_loss": 3.4160501636142913, + "tokens_seen": 2330394624 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023759277833500502, + "loss": 1.2843, + "theoretical_loss": 3.4160424573282144, + "tokens_seen": 2330460160 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023757673019057172, + "loss": 1.259, + "theoretical_loss": 3.4160347513195237, + "tokens_seen": 2330525696 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023756068204613842, + "loss": 1.1507, + "theoretical_loss": 3.4160270455882022, + "tokens_seen": 2330591232 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023754463390170512, + "loss": 1.1515, + "theoretical_loss": 3.416019340134232, + "tokens_seen": 2330656768 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023752858575727182, + "loss": 1.2914, + "theoretical_loss": 3.4160116349575946, + "tokens_seen": 2330722304 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023751253761283852, + "loss": 1.2567, + "theoretical_loss": 3.416003930058273, + "tokens_seen": 2330787840 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023749648946840522, + "loss": 1.2134, + "theoretical_loss": 3.415996225436249, + "tokens_seen": 2330853376 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023748044132397192, + "loss": 1.1854, + "theoretical_loss": 3.4159885210915046, + "tokens_seen": 2330918912 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023746439317953863, + "loss": 1.1824, + "theoretical_loss": 3.415980817024023, + "tokens_seen": 2330984448 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023744834503510533, + "loss": 1.1746, + "theoretical_loss": 3.415973113233785, + "tokens_seen": 2331049984 + }, + { + "epoch": 0.71, + "learning_rate": 0.000237432296890672, + "loss": 1.3284, + "theoretical_loss": 3.4159654097207746, + "tokens_seen": 2331115520 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002374162487462387, + "loss": 1.197, + "theoretical_loss": 3.4159577064849724, + "tokens_seen": 2331181056 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002374002006018054, + "loss": 1.2251, + "theoretical_loss": 3.415950003526362, + "tokens_seen": 2331246592 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002373841524573721, + "loss": 1.1673, + "theoretical_loss": 3.415942300844924, + "tokens_seen": 2331312128 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002373681043129388, + "loss": 1.2353, + "theoretical_loss": 3.415934598440642, + "tokens_seen": 2331377664 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002373520561685055, + "loss": 1.1379, + "theoretical_loss": 3.415926896313498, + "tokens_seen": 2331443200 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002373360080240722, + "loss": 1.2357, + "theoretical_loss": 3.4159191944634735, + "tokens_seen": 2331508736 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002373199598796389, + "loss": 1.2638, + "theoretical_loss": 3.415911492890552, + "tokens_seen": 2331574272 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002373039117352056, + "loss": 1.1949, + "theoretical_loss": 3.4159037915947144, + "tokens_seen": 2331639808 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002372878635907723, + "loss": 1.1427, + "theoretical_loss": 3.4158960905759437, + "tokens_seen": 2331705344 + }, + { + "epoch": 0.71, + "learning_rate": 0.000237271815446339, + "loss": 1.2115, + "theoretical_loss": 3.4158883898342225, + "tokens_seen": 2331770880 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002372557673019057, + "loss": 1.3252, + "theoretical_loss": 3.4158806893695317, + "tokens_seen": 2331836416 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002372397191574724, + "loss": 1.1431, + "theoretical_loss": 3.415872989181855, + "tokens_seen": 2331901952 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023722367101303916, + "loss": 1.313, + "theoretical_loss": 3.415865289271174, + "tokens_seen": 2331967488 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023720762286860586, + "loss": 1.195, + "theoretical_loss": 3.4158575896374708, + "tokens_seen": 2332033024 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023719157472417256, + "loss": 1.1401, + "theoretical_loss": 3.415849890280728, + "tokens_seen": 2332098560 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023717552657973924, + "loss": 1.2614, + "theoretical_loss": 3.4158421912009276, + "tokens_seen": 2332164096 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023715947843530594, + "loss": 1.1656, + "theoretical_loss": 3.4158344923980524, + "tokens_seen": 2332229632 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023714343029087264, + "loss": 1.2996, + "theoretical_loss": 3.4158267938720837, + "tokens_seen": 2332295168 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023712738214643934, + "loss": 1.2282, + "theoretical_loss": 3.4158190956230046, + "tokens_seen": 2332360704 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023711133400200604, + "loss": 1.2035, + "theoretical_loss": 3.415811397650797, + "tokens_seen": 2332426240 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023709528585757274, + "loss": 1.2083, + "theoretical_loss": 3.4158036999554433, + "tokens_seen": 2332491776 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023707923771313944, + "loss": 1.2622, + "theoretical_loss": 3.4157960025369256, + "tokens_seen": 2332557312 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023706318956870614, + "loss": 1.0885, + "theoretical_loss": 3.415788305395226, + "tokens_seen": 2332622848 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023704714142427284, + "loss": 1.2815, + "theoretical_loss": 3.4157806085303273, + "tokens_seen": 2332688384 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023703109327983954, + "loss": 1.0539, + "theoretical_loss": 3.415772911942211, + "tokens_seen": 2332753920 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023701504513540624, + "loss": 1.2252, + "theoretical_loss": 3.41576521563086, + "tokens_seen": 2332819456 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023699899699097294, + "loss": 1.2041, + "theoretical_loss": 3.415757519596257, + "tokens_seen": 2332884992 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023698294884653964, + "loss": 1.1514, + "theoretical_loss": 3.415749823838383, + "tokens_seen": 2332950528 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023696690070210634, + "loss": 1.3344, + "theoretical_loss": 3.4157421283572216, + "tokens_seen": 2333016064 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 779969, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.34850013256073, + "objective/train/theoretical_loss": 3.415734433152754, + "objective/train/tokens_used": 2353541600, + "theoretical_loss": 3.415734433152754, + "tokens_seen": 2333081600 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023695085255767305, + "loss": 1.1848, + "theoretical_loss": 3.415734433152754, + "tokens_seen": 2333081600 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023693480441323975, + "loss": 1.2165, + "theoretical_loss": 3.4157267382249636, + "tokens_seen": 2333147136 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023691875626880645, + "loss": 1.141, + "theoretical_loss": 3.4157190435738314, + "tokens_seen": 2333212672 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023690270812437315, + "loss": 1.289, + "theoretical_loss": 3.4157113491993405, + "tokens_seen": 2333278208 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023688665997993985, + "loss": 1.1714, + "theoretical_loss": 3.415703655101473, + "tokens_seen": 2333343744 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023687061183550655, + "loss": 1.1581, + "theoretical_loss": 3.415695961280211, + "tokens_seen": 2333409280 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023685456369107325, + "loss": 1.1589, + "theoretical_loss": 3.415688267735537, + "tokens_seen": 2333474816 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023683851554663995, + "loss": 1.319, + "theoretical_loss": 3.4156805744674337, + "tokens_seen": 2333540352 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023682246740220665, + "loss": 1.224, + "theoretical_loss": 3.4156728814758823, + "tokens_seen": 2333605888 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023680641925777335, + "loss": 1.2391, + "theoretical_loss": 3.415665188760866, + "tokens_seen": 2333671424 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023679037111334002, + "loss": 1.2196, + "theoretical_loss": 3.4156574963223667, + "tokens_seen": 2333736960 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023677432296890673, + "loss": 1.2348, + "theoretical_loss": 3.415649804160367, + "tokens_seen": 2333802496 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023675827482447343, + "loss": 1.2407, + "theoretical_loss": 3.415642112274849, + "tokens_seen": 2333868032 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023674222668004013, + "loss": 1.3505, + "theoretical_loss": 3.4156344206657945, + "tokens_seen": 2333933568 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023672617853560683, + "loss": 1.1753, + "theoretical_loss": 3.4156267293331872, + "tokens_seen": 2333999104 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023671013039117353, + "loss": 1.2209, + "theoretical_loss": 3.4156190382770077, + "tokens_seen": 2334064640 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023669408224674023, + "loss": 1.2361, + "theoretical_loss": 3.4156113474972396, + "tokens_seen": 2334130176 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023667803410230693, + "loss": 1.1704, + "theoretical_loss": 3.4156036569938646, + "tokens_seen": 2334195712 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023666198595787363, + "loss": 1.1813, + "theoretical_loss": 3.4155959667668654, + "tokens_seen": 2334261248 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023664593781344033, + "loss": 1.2015, + "theoretical_loss": 3.415588276816224, + "tokens_seen": 2334326784 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023662988966900703, + "loss": 1.2214, + "theoretical_loss": 3.415580587141922, + "tokens_seen": 2334392320 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023661384152457373, + "loss": 1.2365, + "theoretical_loss": 3.4155728977439432, + "tokens_seen": 2334457856 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023659779338014043, + "loss": 1.2475, + "theoretical_loss": 3.415565208622269, + "tokens_seen": 2334523392 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023658174523570713, + "loss": 1.1534, + "theoretical_loss": 3.415557519776882, + "tokens_seen": 2334588928 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023656569709127383, + "loss": 1.276, + "theoretical_loss": 3.4155498312077643, + "tokens_seen": 2334654464 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023654964894684053, + "loss": 1.1967, + "theoretical_loss": 3.4155421429148984, + "tokens_seen": 2334720000 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023653360080240723, + "loss": 1.1333, + "theoretical_loss": 3.4155344548982667, + "tokens_seen": 2334785536 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023651755265797394, + "loss": 1.2213, + "theoretical_loss": 3.415526767157851, + "tokens_seen": 2334851072 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023650150451354064, + "loss": 1.2263, + "theoretical_loss": 3.415519079693634, + "tokens_seen": 2334916608 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023648545636910734, + "loss": 1.2838, + "theoretical_loss": 3.415511392505598, + "tokens_seen": 2334982144 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023646940822467404, + "loss": 1.2802, + "theoretical_loss": 3.415503705593726, + "tokens_seen": 2335047680 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023645336008024074, + "loss": 1.2428, + "theoretical_loss": 3.415496018957999, + "tokens_seen": 2335113216 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023643731193580744, + "loss": 1.2283, + "theoretical_loss": 3.4154883325984002, + "tokens_seen": 2335178752 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023642126379137414, + "loss": 1.2336, + "theoretical_loss": 3.415480646514912, + "tokens_seen": 2335244288 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002364052156469408, + "loss": 1.3312, + "theoretical_loss": 3.415472960707516, + "tokens_seen": 2335309824 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002363891675025075, + "loss": 1.2251, + "theoretical_loss": 3.4154652751761954, + "tokens_seen": 2335375360 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023637311935807421, + "loss": 1.2917, + "theoretical_loss": 3.415457589920932, + "tokens_seen": 2335440896 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023635707121364091, + "loss": 1.1916, + "theoretical_loss": 3.415449904941708, + "tokens_seen": 2335506432 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023634102306920762, + "loss": 1.1994, + "theoretical_loss": 3.4154422202385066, + "tokens_seen": 2335571968 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023632497492477432, + "loss": 1.1953, + "theoretical_loss": 3.4154345358113094, + "tokens_seen": 2335637504 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023630892678034102, + "loss": 1.2497, + "theoretical_loss": 3.4154268516600985, + "tokens_seen": 2335703040 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023629287863590772, + "loss": 1.1955, + "theoretical_loss": 3.415419167784857, + "tokens_seen": 2335768576 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023627683049147442, + "loss": 1.2891, + "theoretical_loss": 3.415411484185567, + "tokens_seen": 2335834112 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023626078234704112, + "loss": 1.2492, + "theoretical_loss": 3.4154038008622103, + "tokens_seen": 2335899648 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023624473420260782, + "loss": 1.1559, + "theoretical_loss": 3.41539611781477, + "tokens_seen": 2335965184 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023622868605817452, + "loss": 1.2286, + "theoretical_loss": 3.4153884350432278, + "tokens_seen": 2336030720 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023621263791374122, + "loss": 1.2079, + "theoretical_loss": 3.415380752547567, + "tokens_seen": 2336096256 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023619658976930792, + "loss": 1.2148, + "theoretical_loss": 3.415373070327769, + "tokens_seen": 2336161792 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023618054162487462, + "loss": 1.2845, + "theoretical_loss": 3.415365388383816, + "tokens_seen": 2336227328 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023616449348044132, + "loss": 1.2171, + "theoretical_loss": 3.415357706715692, + "tokens_seen": 2336292864 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 781226, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1148186922073364, + "objective/train/theoretical_loss": 3.4153500253233773, + "objective/train/tokens_used": 2356818400, + "theoretical_loss": 3.4153500253233773, + "tokens_seen": 2336358400 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023614844533600802, + "loss": 1.1695, + "theoretical_loss": 3.4153500253233773, + "tokens_seen": 2336358400 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023613239719157472, + "loss": 1.3498, + "theoretical_loss": 3.415342344206856, + "tokens_seen": 2336423936 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023611634904714142, + "loss": 1.2823, + "theoretical_loss": 3.415334663366109, + "tokens_seen": 2336489472 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023610030090270812, + "loss": 1.0882, + "theoretical_loss": 3.415326982801119, + "tokens_seen": 2336555008 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023608425275827483, + "loss": 1.2933, + "theoretical_loss": 3.415319302511869, + "tokens_seen": 2336620544 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023606820461384153, + "loss": 1.1677, + "theoretical_loss": 3.4153116224983417, + "tokens_seen": 2336686080 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023605215646940823, + "loss": 1.2958, + "theoretical_loss": 3.4153039427605183, + "tokens_seen": 2336751616 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023603610832497493, + "loss": 1.1604, + "theoretical_loss": 3.4152962632983814, + "tokens_seen": 2336817152 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023602006018054166, + "loss": 1.2628, + "theoretical_loss": 3.4152885841119143, + "tokens_seen": 2336882688 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023600401203610836, + "loss": 1.1698, + "theoretical_loss": 3.415280905201098, + "tokens_seen": 2336948224 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023598796389167506, + "loss": 1.0888, + "theoretical_loss": 3.4152732265659163, + "tokens_seen": 2337013760 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023597191574724176, + "loss": 1.2245, + "theoretical_loss": 3.4152655482063503, + "tokens_seen": 2337079296 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023595586760280846, + "loss": 1.383, + "theoretical_loss": 3.4152578701223835, + "tokens_seen": 2337144832 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023593981945837516, + "loss": 1.2602, + "theoretical_loss": 3.415250192313997, + "tokens_seen": 2337210368 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023592377131394186, + "loss": 1.1818, + "theoretical_loss": 3.4152425147811747, + "tokens_seen": 2337275904 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023590772316950856, + "loss": 1.1751, + "theoretical_loss": 3.415234837523898, + "tokens_seen": 2337341440 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023589167502507526, + "loss": 1.1655, + "theoretical_loss": 3.415227160542149, + "tokens_seen": 2337406976 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023587562688064196, + "loss": 1.2185, + "theoretical_loss": 3.4152194838359113, + "tokens_seen": 2337472512 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023585957873620866, + "loss": 1.1316, + "theoretical_loss": 3.415211807405166, + "tokens_seen": 2337538048 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023584353059177536, + "loss": 1.2174, + "theoretical_loss": 3.4152041312498964, + "tokens_seen": 2337603584 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023582748244734206, + "loss": 1.2716, + "theoretical_loss": 3.4151964553700846, + "tokens_seen": 2337669120 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023581143430290876, + "loss": 1.1816, + "theoretical_loss": 3.4151887797657126, + "tokens_seen": 2337734656 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023579538615847546, + "loss": 1.1557, + "theoretical_loss": 3.4151811044367633, + "tokens_seen": 2337800192 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023577933801404214, + "loss": 1.1639, + "theoretical_loss": 3.415173429383219, + "tokens_seen": 2337865728 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023576328986960884, + "loss": 1.2042, + "theoretical_loss": 3.415165754605062, + "tokens_seen": 2337931264 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023574724172517554, + "loss": 1.1896, + "theoretical_loss": 3.415158080102275, + "tokens_seen": 2337996800 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023573119358074224, + "loss": 1.3127, + "theoretical_loss": 3.4151504058748396, + "tokens_seen": 2338062336 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023571514543630894, + "loss": 1.208, + "theoretical_loss": 3.4151427319227388, + "tokens_seen": 2338127872 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023569909729187564, + "loss": 1.1695, + "theoretical_loss": 3.4151350582459554, + "tokens_seen": 2338193408 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023568304914744234, + "loss": 1.3035, + "theoretical_loss": 3.415127384844471, + "tokens_seen": 2338258944 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023566700100300904, + "loss": 1.2315, + "theoretical_loss": 3.4151197117182686, + "tokens_seen": 2338324480 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023565095285857574, + "loss": 1.2985, + "theoretical_loss": 3.41511203886733, + "tokens_seen": 2338390016 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023563490471414244, + "loss": 1.2176, + "theoretical_loss": 3.4151043662916383, + "tokens_seen": 2338455552 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023561885656970914, + "loss": 1.1845, + "theoretical_loss": 3.4150966939911753, + "tokens_seen": 2338521088 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023560280842527584, + "loss": 1.2156, + "theoretical_loss": 3.415089021965924, + "tokens_seen": 2338586624 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023558676028084255, + "loss": 1.2396, + "theoretical_loss": 3.4150813502158663, + "tokens_seen": 2338652160 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023557071213640925, + "loss": 1.2638, + "theoretical_loss": 3.4150736787409848, + "tokens_seen": 2338717696 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023555466399197595, + "loss": 1.1818, + "theoretical_loss": 3.415066007541262, + "tokens_seen": 2338783232 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023553861584754265, + "loss": 1.3052, + "theoretical_loss": 3.4150583366166805, + "tokens_seen": 2338848768 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023552256770310935, + "loss": 1.2165, + "theoretical_loss": 3.4150506659672226, + "tokens_seen": 2338914304 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023550651955867605, + "loss": 1.2539, + "theoretical_loss": 3.41504299559287, + "tokens_seen": 2338979840 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023549047141424275, + "loss": 1.2687, + "theoretical_loss": 3.4150353254936063, + "tokens_seen": 2339045376 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023547442326980945, + "loss": 1.1701, + "theoretical_loss": 3.4150276556694132, + "tokens_seen": 2339110912 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023545837512537615, + "loss": 1.1945, + "theoretical_loss": 3.4150199861202735, + "tokens_seen": 2339176448 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023544232698094285, + "loss": 1.1932, + "theoretical_loss": 3.415012316846169, + "tokens_seen": 2339241984 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023542627883650955, + "loss": 1.2808, + "theoretical_loss": 3.415004647847083, + "tokens_seen": 2339307520 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023541023069207625, + "loss": 1.0831, + "theoretical_loss": 3.4149969791229973, + "tokens_seen": 2339373056 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023539418254764293, + "loss": 1.1572, + "theoretical_loss": 3.414989310673894, + "tokens_seen": 2339438592 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023537813440320963, + "loss": 1.1823, + "theoretical_loss": 3.4149816424997566, + "tokens_seen": 2339504128 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023536208625877633, + "loss": 1.0633, + "theoretical_loss": 3.414973974600567, + "tokens_seen": 2339569664 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 782115, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.165141224861145, + "objective/train/theoretical_loss": 3.414966306976308, + "objective/train/tokens_used": 2360095200, + "theoretical_loss": 3.414966306976308, + "tokens_seen": 2339635200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023534603811434303, + "loss": 1.1655, + "theoretical_loss": 3.414966306976308, + "tokens_seen": 2339635200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023532998996990973, + "loss": 1.2571, + "theoretical_loss": 3.414958639626961, + "tokens_seen": 2339700736 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023531394182547643, + "loss": 1.2492, + "theoretical_loss": 3.41495097255251, + "tokens_seen": 2339766272 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023529789368104313, + "loss": 1.0723, + "theoretical_loss": 3.414943305752936, + "tokens_seen": 2339831808 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023528184553660983, + "loss": 1.1744, + "theoretical_loss": 3.4149356392282217, + "tokens_seen": 2339897344 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023526579739217653, + "loss": 1.212, + "theoretical_loss": 3.4149279729783504, + "tokens_seen": 2339962880 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023524974924774323, + "loss": 1.173, + "theoretical_loss": 3.414920307003304, + "tokens_seen": 2340028416 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023523370110330993, + "loss": 1.2268, + "theoretical_loss": 3.4149126413030646, + "tokens_seen": 2340093952 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023521765295887663, + "loss": 1.1959, + "theoretical_loss": 3.4149049758776155, + "tokens_seen": 2340159488 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023520160481444333, + "loss": 1.1722, + "theoretical_loss": 3.4148973107269383, + "tokens_seen": 2340225024 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023518555667001003, + "loss": 1.2482, + "theoretical_loss": 3.4148896458510163, + "tokens_seen": 2340290560 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023516950852557673, + "loss": 1.2382, + "theoretical_loss": 3.414881981249831, + "tokens_seen": 2340356096 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023515346038114344, + "loss": 1.2398, + "theoretical_loss": 3.4148743169233655, + "tokens_seen": 2340421632 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023513741223671014, + "loss": 1.1895, + "theoretical_loss": 3.414866652871602, + "tokens_seen": 2340487168 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023512136409227684, + "loss": 1.2001, + "theoretical_loss": 3.4148589890945233, + "tokens_seen": 2340552704 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023510531594784354, + "loss": 1.1978, + "theoretical_loss": 3.414851325592112, + "tokens_seen": 2340618240 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023508926780341024, + "loss": 1.1235, + "theoretical_loss": 3.4148436623643494, + "tokens_seen": 2340683776 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023507321965897694, + "loss": 1.2128, + "theoretical_loss": 3.4148359994112196, + "tokens_seen": 2340749312 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023505717151454364, + "loss": 1.1256, + "theoretical_loss": 3.414828336732704, + "tokens_seen": 2340814848 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023504112337011034, + "loss": 1.2779, + "theoretical_loss": 3.414820674328785, + "tokens_seen": 2340880384 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023502507522567704, + "loss": 1.208, + "theoretical_loss": 3.4148130121994456, + "tokens_seen": 2340945920 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023500902708124371, + "loss": 1.2995, + "theoretical_loss": 3.4148053503446683, + "tokens_seen": 2341011456 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023499297893681041, + "loss": 1.2244, + "theoretical_loss": 3.414797688764435, + "tokens_seen": 2341076992 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023497693079237711, + "loss": 1.1991, + "theoretical_loss": 3.414790027458728, + "tokens_seen": 2341142528 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023496088264794382, + "loss": 1.1807, + "theoretical_loss": 3.414782366427531, + "tokens_seen": 2341208064 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023494483450351052, + "loss": 1.2335, + "theoretical_loss": 3.414774705670826, + "tokens_seen": 2341273600 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023492878635907722, + "loss": 1.3009, + "theoretical_loss": 3.414767045188595, + "tokens_seen": 2341339136 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023491273821464392, + "loss": 1.1905, + "theoretical_loss": 3.4147593849808207, + "tokens_seen": 2341404672 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023489669007021062, + "loss": 1.2399, + "theoretical_loss": 3.4147517250474855, + "tokens_seen": 2341470208 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023488064192577732, + "loss": 1.2147, + "theoretical_loss": 3.414744065388572, + "tokens_seen": 2341535744 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023486459378134402, + "loss": 1.284, + "theoretical_loss": 3.414736406004063, + "tokens_seen": 2341601280 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023484854563691072, + "loss": 1.0707, + "theoretical_loss": 3.4147287468939407, + "tokens_seen": 2341666816 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023483249749247747, + "loss": 1.2434, + "theoretical_loss": 3.414721088058187, + "tokens_seen": 2341732352 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023481644934804418, + "loss": 1.2338, + "theoretical_loss": 3.4147134294967856, + "tokens_seen": 2341797888 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023480040120361088, + "loss": 1.207, + "theoretical_loss": 3.414705771209718, + "tokens_seen": 2341863424 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023478435305917758, + "loss": 1.168, + "theoretical_loss": 3.414698113196967, + "tokens_seen": 2341928960 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023476830491474425, + "loss": 1.3546, + "theoretical_loss": 3.4146904554585156, + "tokens_seen": 2341994496 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023475225677031095, + "loss": 1.3034, + "theoretical_loss": 3.4146827979943457, + "tokens_seen": 2342060032 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023473620862587765, + "loss": 1.244, + "theoretical_loss": 3.41467514080444, + "tokens_seen": 2342125568 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023472016048144435, + "loss": 1.2391, + "theoretical_loss": 3.4146674838887807, + "tokens_seen": 2342191104 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023470411233701105, + "loss": 1.3049, + "theoretical_loss": 3.414659827247351, + "tokens_seen": 2342256640 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023468806419257775, + "loss": 1.1441, + "theoretical_loss": 3.4146521708801325, + "tokens_seen": 2342322176 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023467201604814445, + "loss": 1.3079, + "theoretical_loss": 3.4146445147871085, + "tokens_seen": 2342387712 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023465596790371115, + "loss": 1.3107, + "theoretical_loss": 3.414636858968261, + "tokens_seen": 2342453248 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023463991975927786, + "loss": 1.2952, + "theoretical_loss": 3.414629203423573, + "tokens_seen": 2342518784 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023462387161484456, + "loss": 1.3347, + "theoretical_loss": 3.4146215481530264, + "tokens_seen": 2342584320 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023460782347041126, + "loss": 1.2027, + "theoretical_loss": 3.4146138931566044, + "tokens_seen": 2342649856 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023459177532597796, + "loss": 1.1749, + "theoretical_loss": 3.4146062384342892, + "tokens_seen": 2342715392 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023457572718154466, + "loss": 1.1367, + "theoretical_loss": 3.414598583986063, + "tokens_seen": 2342780928 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023455967903711136, + "loss": 1.1252, + "theoretical_loss": 3.4145909298119084, + "tokens_seen": 2342846464 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 783307, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.293138861656189, + "objective/train/theoretical_loss": 3.4145832759118084, + "objective/train/tokens_used": 2363372000, + "theoretical_loss": 3.4145832759118084, + "tokens_seen": 2342912000 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023454363089267806, + "loss": 1.2362, + "theoretical_loss": 3.4145832759118084, + "tokens_seen": 2342912000 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023452758274824476, + "loss": 1.1641, + "theoretical_loss": 3.4145756222857453, + "tokens_seen": 2342977536 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023451153460381146, + "loss": 1.1295, + "theoretical_loss": 3.414567968933701, + "tokens_seen": 2343043072 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023449548645937816, + "loss": 1.3184, + "theoretical_loss": 3.4145603158556592, + "tokens_seen": 2343108608 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023447943831494486, + "loss": 1.1716, + "theoretical_loss": 3.4145526630516017, + "tokens_seen": 2343174144 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023446339017051156, + "loss": 1.2537, + "theoretical_loss": 3.414545010521511, + "tokens_seen": 2343239680 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023444734202607826, + "loss": 1.2144, + "theoretical_loss": 3.41453735826537, + "tokens_seen": 2343305216 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023443129388164496, + "loss": 1.1962, + "theoretical_loss": 3.4145297062831608, + "tokens_seen": 2343370752 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023441524573721166, + "loss": 1.2483, + "theoretical_loss": 3.4145220545748662, + "tokens_seen": 2343436288 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023439919759277836, + "loss": 1.1948, + "theoretical_loss": 3.4145144031404686, + "tokens_seen": 2343501824 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023438314944834504, + "loss": 1.1629, + "theoretical_loss": 3.4145067519799506, + "tokens_seen": 2343567360 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023436710130391174, + "loss": 1.2492, + "theoretical_loss": 3.414499101093295, + "tokens_seen": 2343632896 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023435105315947844, + "loss": 1.2816, + "theoretical_loss": 3.4144914504804835, + "tokens_seen": 2343698432 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023433500501504514, + "loss": 1.2123, + "theoretical_loss": 3.4144838001414994, + "tokens_seen": 2343763968 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023431895687061184, + "loss": 1.2527, + "theoretical_loss": 3.4144761500763257, + "tokens_seen": 2343829504 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023430290872617854, + "loss": 1.1994, + "theoretical_loss": 3.4144685002849435, + "tokens_seen": 2343895040 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023428686058174524, + "loss": 1.1923, + "theoretical_loss": 3.4144608507673366, + "tokens_seen": 2343960576 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023427081243731194, + "loss": 1.2197, + "theoretical_loss": 3.414453201523487, + "tokens_seen": 2344026112 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023425476429287864, + "loss": 1.2069, + "theoretical_loss": 3.4144455525533774, + "tokens_seen": 2344091648 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023423871614844534, + "loss": 1.1677, + "theoretical_loss": 3.4144379038569905, + "tokens_seen": 2344157184 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023422266800401204, + "loss": 1.3352, + "theoretical_loss": 3.4144302554343082, + "tokens_seen": 2344222720 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023420661985957875, + "loss": 1.1749, + "theoretical_loss": 3.4144226072853137, + "tokens_seen": 2344288256 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023419057171514545, + "loss": 1.2767, + "theoretical_loss": 3.4144149594099895, + "tokens_seen": 2344353792 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023417452357071215, + "loss": 1.2482, + "theoretical_loss": 3.4144073118083176, + "tokens_seen": 2344419328 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023415847542627885, + "loss": 1.2943, + "theoretical_loss": 3.414399664480282, + "tokens_seen": 2344484864 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023414242728184555, + "loss": 1.2046, + "theoretical_loss": 3.414392017425863, + "tokens_seen": 2344550400 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023412637913741225, + "loss": 1.2131, + "theoretical_loss": 3.414384370645045, + "tokens_seen": 2344615936 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023411033099297895, + "loss": 1.1986, + "theoretical_loss": 3.41437672413781, + "tokens_seen": 2344681472 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023409428284854565, + "loss": 1.2144, + "theoretical_loss": 3.4143690779041407, + "tokens_seen": 2344747008 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023407823470411235, + "loss": 1.2912, + "theoretical_loss": 3.414361431944019, + "tokens_seen": 2344812544 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023406218655967905, + "loss": 1.2908, + "theoretical_loss": 3.4143537862574282, + "tokens_seen": 2344878080 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023404613841524575, + "loss": 1.1999, + "theoretical_loss": 3.4143461408443505, + "tokens_seen": 2344943616 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023403009027081245, + "loss": 1.2522, + "theoretical_loss": 3.414338495704769, + "tokens_seen": 2345009152 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023401404212637915, + "loss": 1.2103, + "theoretical_loss": 3.414330850838666, + "tokens_seen": 2345074688 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023399799398194583, + "loss": 1.1813, + "theoretical_loss": 3.4143232062460234, + "tokens_seen": 2345140224 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023398194583751253, + "loss": 1.2397, + "theoretical_loss": 3.4143155619268244, + "tokens_seen": 2345205760 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023396589769307923, + "loss": 1.2384, + "theoretical_loss": 3.414307917881052, + "tokens_seen": 2345271296 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023394984954864593, + "loss": 1.2147, + "theoretical_loss": 3.4143002741086876, + "tokens_seen": 2345336832 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023393380140421263, + "loss": 1.306, + "theoretical_loss": 3.414292630609715, + "tokens_seen": 2345402368 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023391775325977933, + "loss": 1.1879, + "theoretical_loss": 3.414284987384116, + "tokens_seen": 2345467904 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023390170511534603, + "loss": 1.2373, + "theoretical_loss": 3.414277344431874, + "tokens_seen": 2345533440 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023388565697091273, + "loss": 1.2236, + "theoretical_loss": 3.4142697017529704, + "tokens_seen": 2345598976 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023386960882647943, + "loss": 1.2648, + "theoretical_loss": 3.4142620593473887, + "tokens_seen": 2345664512 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023385356068204613, + "loss": 1.1546, + "theoretical_loss": 3.414254417215111, + "tokens_seen": 2345730048 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023383751253761283, + "loss": 1.2093, + "theoretical_loss": 3.4142467753561205, + "tokens_seen": 2345795584 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023382146439317953, + "loss": 1.2563, + "theoretical_loss": 3.414239133770399, + "tokens_seen": 2345861120 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023380541624874623, + "loss": 1.2515, + "theoretical_loss": 3.4142314924579296, + "tokens_seen": 2345926656 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023378936810431293, + "loss": 1.1926, + "theoretical_loss": 3.414223851418695, + "tokens_seen": 2345992192 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023377331995987964, + "loss": 1.2683, + "theoretical_loss": 3.414216210652677, + "tokens_seen": 2346057728 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023375727181544634, + "loss": 1.2123, + "theoretical_loss": 3.4142085701598592, + "tokens_seen": 2346123264 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 784199, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3218762874603271, + "objective/train/theoretical_loss": 3.414200929940224, + "objective/train/tokens_used": 2366648800, + "theoretical_loss": 3.414200929940224, + "tokens_seen": 2346188800 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023374122367101304, + "loss": 1.3144, + "theoretical_loss": 3.414200929940224, + "tokens_seen": 2346188800 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023372517552657974, + "loss": 1.2493, + "theoretical_loss": 3.4141932899937535, + "tokens_seen": 2346254336 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023370912738214644, + "loss": 1.2883, + "theoretical_loss": 3.4141856503204306, + "tokens_seen": 2346319872 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023369307923771314, + "loss": 1.2402, + "theoretical_loss": 3.4141780109202378, + "tokens_seen": 2346385408 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023367703109327984, + "loss": 1.2103, + "theoretical_loss": 3.414170371793158, + "tokens_seen": 2346450944 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023366098294884654, + "loss": 1.2279, + "theoretical_loss": 3.4141627329391735, + "tokens_seen": 2346516480 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023364493480441324, + "loss": 1.1865, + "theoretical_loss": 3.414155094358267, + "tokens_seen": 2346582016 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023362888665997997, + "loss": 1.2566, + "theoretical_loss": 3.4141474560504212, + "tokens_seen": 2346647552 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023361283851554667, + "loss": 1.2841, + "theoretical_loss": 3.4141398180156184, + "tokens_seen": 2346713088 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023359679037111337, + "loss": 1.3069, + "theoretical_loss": 3.4141321802538416, + "tokens_seen": 2346778624 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023358074222668007, + "loss": 1.2193, + "theoretical_loss": 3.4141245427650735, + "tokens_seen": 2346844160 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023356469408224677, + "loss": 1.2489, + "theoretical_loss": 3.414116905549296, + "tokens_seen": 2346909696 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023354864593781347, + "loss": 1.2378, + "theoretical_loss": 3.4141092686064924, + "tokens_seen": 2346975232 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023353259779338017, + "loss": 1.2603, + "theoretical_loss": 3.4141016319366457, + "tokens_seen": 2347040768 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023351654964894687, + "loss": 1.1539, + "theoretical_loss": 3.414093995539737, + "tokens_seen": 2347106304 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023350050150451357, + "loss": 1.2245, + "theoretical_loss": 3.4140863594157507, + "tokens_seen": 2347171840 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023348445336008027, + "loss": 1.3484, + "theoretical_loss": 3.414078723564668, + "tokens_seen": 2347237376 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023346840521564697, + "loss": 1.1816, + "theoretical_loss": 3.4140710879864726, + "tokens_seen": 2347302912 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023345235707121368, + "loss": 1.2687, + "theoretical_loss": 3.4140634526811464, + "tokens_seen": 2347368448 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023343630892678038, + "loss": 1.2833, + "theoretical_loss": 3.4140558176486726, + "tokens_seen": 2347433984 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023342026078234708, + "loss": 1.2545, + "theoretical_loss": 3.414048182889033, + "tokens_seen": 2347499520 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023340421263791378, + "loss": 1.2099, + "theoretical_loss": 3.414040548402211, + "tokens_seen": 2347565056 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023338816449348048, + "loss": 1.2287, + "theoretical_loss": 3.414032914188189, + "tokens_seen": 2347630592 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023337211634904718, + "loss": 1.3031, + "theoretical_loss": 3.4140252802469497, + "tokens_seen": 2347696128 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023335606820461385, + "loss": 1.3304, + "theoretical_loss": 3.4140176465784755, + "tokens_seen": 2347761664 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023334002006018055, + "loss": 1.1592, + "theoretical_loss": 3.4140100131827493, + "tokens_seen": 2347827200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023332397191574725, + "loss": 1.2097, + "theoretical_loss": 3.414002380059754, + "tokens_seen": 2347892736 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023330792377131395, + "loss": 1.1487, + "theoretical_loss": 3.4139947472094714, + "tokens_seen": 2347958272 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023329187562688065, + "loss": 1.1334, + "theoretical_loss": 3.413987114631885, + "tokens_seen": 2348023808 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023327582748244736, + "loss": 1.1415, + "theoretical_loss": 3.413979482326977, + "tokens_seen": 2348089344 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023325977933801406, + "loss": 1.3036, + "theoretical_loss": 3.41397185029473, + "tokens_seen": 2348154880 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023324373119358076, + "loss": 1.2213, + "theoretical_loss": 3.413964218535127, + "tokens_seen": 2348220416 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023322768304914746, + "loss": 1.2219, + "theoretical_loss": 3.41395658704815, + "tokens_seen": 2348285952 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023321163490471416, + "loss": 1.236, + "theoretical_loss": 3.4139489558337823, + "tokens_seen": 2348351488 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023319558676028086, + "loss": 1.1862, + "theoretical_loss": 3.4139413248920065, + "tokens_seen": 2348417024 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023317953861584756, + "loss": 1.2423, + "theoretical_loss": 3.4139336942228047, + "tokens_seen": 2348482560 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023316349047141426, + "loss": 1.2459, + "theoretical_loss": 3.4139260638261604, + "tokens_seen": 2348548096 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023314744232698096, + "loss": 1.2503, + "theoretical_loss": 3.4139184337020554, + "tokens_seen": 2348613632 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023313139418254766, + "loss": 1.1317, + "theoretical_loss": 3.4139108038504733, + "tokens_seen": 2348679168 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023311534603811436, + "loss": 1.2063, + "theoretical_loss": 3.413903174271396, + "tokens_seen": 2348744704 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023309929789368106, + "loss": 1.1988, + "theoretical_loss": 3.4138955449648063, + "tokens_seen": 2348810240 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023308324974924776, + "loss": 1.1637, + "theoretical_loss": 3.4138879159306867, + "tokens_seen": 2348875776 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023306720160481446, + "loss": 1.2081, + "theoretical_loss": 3.413880287169021, + "tokens_seen": 2348941312 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023305115346038116, + "loss": 1.1337, + "theoretical_loss": 3.4138726586797903, + "tokens_seen": 2349006848 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023303510531594786, + "loss": 1.2688, + "theoretical_loss": 3.4138650304629783, + "tokens_seen": 2349072384 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023301905717151457, + "loss": 1.1101, + "theoretical_loss": 3.413857402518567, + "tokens_seen": 2349137920 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023300300902708127, + "loss": 1.1289, + "theoretical_loss": 3.4138497748465397, + "tokens_seen": 2349203456 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023298696088264797, + "loss": 1.2333, + "theoretical_loss": 3.4138421474468785, + "tokens_seen": 2349268992 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023297091273821464, + "loss": 1.2767, + "theoretical_loss": 3.413834520319567, + "tokens_seen": 2349334528 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023295486459378134, + "loss": 1.2191, + "theoretical_loss": 3.4138268934645866, + "tokens_seen": 2349400064 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 785442, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0939662456512451, + "objective/train/theoretical_loss": 3.413819266881921, + "objective/train/tokens_used": 2369925600, + "theoretical_loss": 3.413819266881921, + "tokens_seen": 2349465600 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023293881644934804, + "loss": 1.1555, + "theoretical_loss": 3.413819266881921, + "tokens_seen": 2349465600 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023292276830491474, + "loss": 1.2651, + "theoretical_loss": 3.4138116405715526, + "tokens_seen": 2349531136 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023290672016048144, + "loss": 1.2048, + "theoretical_loss": 3.4138040145334636, + "tokens_seen": 2349596672 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023289067201604814, + "loss": 1.3099, + "theoretical_loss": 3.4137963887676372, + "tokens_seen": 2349662208 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023287462387161484, + "loss": 1.2072, + "theoretical_loss": 3.4137887632740562, + "tokens_seen": 2349727744 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023285857572718154, + "loss": 1.2773, + "theoretical_loss": 3.413781138052703, + "tokens_seen": 2349793280 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023284252758274825, + "loss": 1.2624, + "theoretical_loss": 3.41377351310356, + "tokens_seen": 2349858816 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023282647943831495, + "loss": 1.1926, + "theoretical_loss": 3.4137658884266107, + "tokens_seen": 2349924352 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023281043129388165, + "loss": 1.2331, + "theoretical_loss": 3.413758264021837, + "tokens_seen": 2349989888 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023279438314944835, + "loss": 1.2512, + "theoretical_loss": 3.4137506398892223, + "tokens_seen": 2350055424 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023277833500501505, + "loss": 1.1192, + "theoretical_loss": 3.4137430160287483, + "tokens_seen": 2350120960 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023276228686058175, + "loss": 1.1886, + "theoretical_loss": 3.413735392440399, + "tokens_seen": 2350186496 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023274623871614845, + "loss": 1.1554, + "theoretical_loss": 3.413727769124156, + "tokens_seen": 2350252032 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023273019057171515, + "loss": 1.2255, + "theoretical_loss": 3.413720146080003, + "tokens_seen": 2350317568 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023271414242728185, + "loss": 1.2293, + "theoretical_loss": 3.4137125233079213, + "tokens_seen": 2350383104 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023269809428284855, + "loss": 1.3214, + "theoretical_loss": 3.4137049008078946, + "tokens_seen": 2350448640 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023268204613841525, + "loss": 1.2422, + "theoretical_loss": 3.413697278579906, + "tokens_seen": 2350514176 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023266599799398195, + "loss": 1.3053, + "theoretical_loss": 3.4136896566239368, + "tokens_seen": 2350579712 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023264994984954865, + "loss": 1.2372, + "theoretical_loss": 3.413682034939971, + "tokens_seen": 2350645248 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023263390170511535, + "loss": 1.1928, + "theoretical_loss": 3.413674413527991, + "tokens_seen": 2350710784 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023261785356068205, + "loss": 1.2759, + "theoretical_loss": 3.4136667923879793, + "tokens_seen": 2350776320 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023260180541624875, + "loss": 1.2787, + "theoretical_loss": 3.4136591715199183, + "tokens_seen": 2350841856 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023258575727181543, + "loss": 1.1926, + "theoretical_loss": 3.4136515509237912, + "tokens_seen": 2350907392 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023256970912738213, + "loss": 1.2183, + "theoretical_loss": 3.413643930599581, + "tokens_seen": 2350972928 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023255366098294883, + "loss": 1.3204, + "theoretical_loss": 3.4136363105472696, + "tokens_seen": 2351038464 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023253761283851553, + "loss": 1.2239, + "theoretical_loss": 3.4136286907668403, + "tokens_seen": 2351104000 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023252156469408223, + "loss": 1.2035, + "theoretical_loss": 3.4136210712582757, + "tokens_seen": 2351169536 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023250551654964893, + "loss": 1.2894, + "theoretical_loss": 3.4136134520215586, + "tokens_seen": 2351235072 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023248946840521563, + "loss": 1.1739, + "theoretical_loss": 3.413605833056671, + "tokens_seen": 2351300608 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023247342026078233, + "loss": 1.2308, + "theoretical_loss": 3.4135982143635974, + "tokens_seen": 2351366144 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023245737211634903, + "loss": 1.3007, + "theoretical_loss": 3.4135905959423187, + "tokens_seen": 2351431680 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023244132397191573, + "loss": 1.2635, + "theoretical_loss": 3.413582977792818, + "tokens_seen": 2351497216 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002324252758274825, + "loss": 1.2194, + "theoretical_loss": 3.413575359915079, + "tokens_seen": 2351562752 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002324092276830492, + "loss": 1.193, + "theoretical_loss": 3.413567742309083, + "tokens_seen": 2351628288 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002323931795386159, + "loss": 1.2193, + "theoretical_loss": 3.413560124974814, + "tokens_seen": 2351693824 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002323771313941826, + "loss": 1.2819, + "theoretical_loss": 3.413552507912254, + "tokens_seen": 2351759360 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002323610832497493, + "loss": 1.2957, + "theoretical_loss": 3.4135448911213864, + "tokens_seen": 2351824896 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023234503510531596, + "loss": 1.2032, + "theoretical_loss": 3.4135372746021932, + "tokens_seen": 2351890432 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023232898696088267, + "loss": 1.121, + "theoretical_loss": 3.4135296583546575, + "tokens_seen": 2351955968 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023231293881644937, + "loss": 1.2737, + "theoretical_loss": 3.413522042378762, + "tokens_seen": 2352021504 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023229689067201607, + "loss": 1.2542, + "theoretical_loss": 3.4135144266744892, + "tokens_seen": 2352087040 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023228084252758277, + "loss": 1.2555, + "theoretical_loss": 3.413506811241822, + "tokens_seen": 2352152576 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023226479438314947, + "loss": 1.2045, + "theoretical_loss": 3.4134991960807435, + "tokens_seen": 2352218112 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023224874623871617, + "loss": 1.1975, + "theoretical_loss": 3.413491581191236, + "tokens_seen": 2352283648 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023223269809428287, + "loss": 1.3423, + "theoretical_loss": 3.413483966573283, + "tokens_seen": 2352349184 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023221664994984957, + "loss": 1.2084, + "theoretical_loss": 3.413476352226866, + "tokens_seen": 2352414720 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023220060180541627, + "loss": 1.245, + "theoretical_loss": 3.413468738151969, + "tokens_seen": 2352480256 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023218455366098297, + "loss": 1.1807, + "theoretical_loss": 3.4134611243485735, + "tokens_seen": 2352545792 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023216850551654967, + "loss": 1.2, + "theoretical_loss": 3.4134535108166633, + "tokens_seen": 2352611328 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023215245737211637, + "loss": 1.185, + "theoretical_loss": 3.4134458975562207, + "tokens_seen": 2352676864 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 786316, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2228635549545288, + "objective/train/theoretical_loss": 3.4134382845672286, + "objective/train/tokens_used": 2373202400, + "theoretical_loss": 3.4134382845672286, + "tokens_seen": 2352742400 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023213640922768307, + "loss": 1.2712, + "theoretical_loss": 3.4134382845672286, + "tokens_seen": 2352742400 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023212036108324977, + "loss": 1.2324, + "theoretical_loss": 3.4134306718496696, + "tokens_seen": 2352807936 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023210431293881647, + "loss": 1.3016, + "theoretical_loss": 3.4134230594035264, + "tokens_seen": 2352873472 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023208826479438317, + "loss": 1.2406, + "theoretical_loss": 3.4134154472287825, + "tokens_seen": 2352939008 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023207221664994988, + "loss": 1.1648, + "theoretical_loss": 3.41340783532542, + "tokens_seen": 2353004544 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023205616850551658, + "loss": 1.2235, + "theoretical_loss": 3.4134002236934213, + "tokens_seen": 2353070080 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023204012036108328, + "loss": 1.2541, + "theoretical_loss": 3.41339261233277, + "tokens_seen": 2353135616 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023202407221664998, + "loss": 1.1816, + "theoretical_loss": 3.413385001243448, + "tokens_seen": 2353201152 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023200802407221668, + "loss": 1.1229, + "theoretical_loss": 3.413377390425439, + "tokens_seen": 2353266688 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023199197592778338, + "loss": 1.1939, + "theoretical_loss": 3.4133697798787255, + "tokens_seen": 2353332224 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023197592778335008, + "loss": 1.1541, + "theoretical_loss": 3.41336216960329, + "tokens_seen": 2353397760 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023195987963891675, + "loss": 1.2375, + "theoretical_loss": 3.4133545595991155, + "tokens_seen": 2353463296 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023194383149448345, + "loss": 1.1853, + "theoretical_loss": 3.413346949866184, + "tokens_seen": 2353528832 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023192778335005015, + "loss": 1.2286, + "theoretical_loss": 3.41333934040448, + "tokens_seen": 2353594368 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023191173520561685, + "loss": 1.2181, + "theoretical_loss": 3.413331731213985, + "tokens_seen": 2353659904 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023189568706118356, + "loss": 1.2701, + "theoretical_loss": 3.4133241222946813, + "tokens_seen": 2353725440 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023187963891675026, + "loss": 1.2919, + "theoretical_loss": 3.413316513646553, + "tokens_seen": 2353790976 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023186359077231696, + "loss": 1.2744, + "theoretical_loss": 3.4133089052695818, + "tokens_seen": 2353856512 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023184754262788366, + "loss": 1.1734, + "theoretical_loss": 3.4133012971637515, + "tokens_seen": 2353922048 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023183149448345036, + "loss": 1.3188, + "theoretical_loss": 3.413293689329044, + "tokens_seen": 2353987584 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023181544633901706, + "loss": 1.2333, + "theoretical_loss": 3.4132860817654427, + "tokens_seen": 2354053120 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023179939819458376, + "loss": 1.2989, + "theoretical_loss": 3.4132784744729303, + "tokens_seen": 2354118656 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023178335005015046, + "loss": 1.2331, + "theoretical_loss": 3.413270867451489, + "tokens_seen": 2354184192 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023176730190571716, + "loss": 1.2665, + "theoretical_loss": 3.413263260701102, + "tokens_seen": 2354249728 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023175125376128386, + "loss": 1.2293, + "theoretical_loss": 3.4132556542217527, + "tokens_seen": 2354315264 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023173520561685056, + "loss": 1.3399, + "theoretical_loss": 3.4132480480134233, + "tokens_seen": 2354380800 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023171915747241726, + "loss": 1.1807, + "theoretical_loss": 3.4132404420760962, + "tokens_seen": 2354446336 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023170310932798396, + "loss": 1.2153, + "theoretical_loss": 3.4132328364097546, + "tokens_seen": 2354511872 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023168706118355066, + "loss": 1.1957, + "theoretical_loss": 3.413225231014382, + "tokens_seen": 2354577408 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023167101303911736, + "loss": 1.2111, + "theoretical_loss": 3.41321762588996, + "tokens_seen": 2354642944 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023165496489468407, + "loss": 1.3073, + "theoretical_loss": 3.413210021036472, + "tokens_seen": 2354708480 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023163891675025077, + "loss": 1.2734, + "theoretical_loss": 3.4132024164539008, + "tokens_seen": 2354774016 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023162286860581747, + "loss": 1.2869, + "theoretical_loss": 3.413194812142229, + "tokens_seen": 2354839552 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023160682046138417, + "loss": 1.2907, + "theoretical_loss": 3.41318720810144, + "tokens_seen": 2354905088 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023159077231695087, + "loss": 1.1976, + "theoretical_loss": 3.413179604331516, + "tokens_seen": 2354970624 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023157472417251754, + "loss": 1.1686, + "theoretical_loss": 3.4131720008324398, + "tokens_seen": 2355036160 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023155867602808424, + "loss": 1.3465, + "theoretical_loss": 3.4131643976041945, + "tokens_seen": 2355101696 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023154262788365094, + "loss": 1.2006, + "theoretical_loss": 3.413156794646763, + "tokens_seen": 2355167232 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023152657973921764, + "loss": 1.3053, + "theoretical_loss": 3.4131491919601276, + "tokens_seen": 2355232768 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023151053159478434, + "loss": 1.3158, + "theoretical_loss": 3.413141589544272, + "tokens_seen": 2355298304 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023149448345035104, + "loss": 1.2637, + "theoretical_loss": 3.413133987399178, + "tokens_seen": 2355363840 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023147843530591774, + "loss": 1.1707, + "theoretical_loss": 3.4131263855248295, + "tokens_seen": 2355429376 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023146238716148445, + "loss": 1.1645, + "theoretical_loss": 3.4131187839212083, + "tokens_seen": 2355494912 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023144633901705115, + "loss": 1.178, + "theoretical_loss": 3.4131111825882976, + "tokens_seen": 2355560448 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023143029087261785, + "loss": 1.231, + "theoretical_loss": 3.4131035815260806, + "tokens_seen": 2355625984 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023141424272818455, + "loss": 1.1212, + "theoretical_loss": 3.4130959807345396, + "tokens_seen": 2355691520 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023139819458375125, + "loss": 1.2088, + "theoretical_loss": 3.413088380213658, + "tokens_seen": 2355757056 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023138214643931795, + "loss": 1.3059, + "theoretical_loss": 3.4130807799634177, + "tokens_seen": 2355822592 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023136609829488465, + "loss": 1.2317, + "theoretical_loss": 3.4130731799838028, + "tokens_seen": 2355888128 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023135005015045135, + "loss": 1.1628, + "theoretical_loss": 3.413065580274795, + "tokens_seen": 2355953664 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 787703, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3746349811553955, + "objective/train/theoretical_loss": 3.4130579808363777, + "objective/train/tokens_used": 2376479200, + "theoretical_loss": 3.4130579808363777, + "tokens_seen": 2356019200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023133400200601805, + "loss": 1.3168, + "theoretical_loss": 3.4130579808363777, + "tokens_seen": 2356019200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023131795386158475, + "loss": 1.2377, + "theoretical_loss": 3.413050381668534, + "tokens_seen": 2356084736 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023130190571715145, + "loss": 1.1867, + "theoretical_loss": 3.413042782771246, + "tokens_seen": 2356150272 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023128585757271815, + "loss": 1.2846, + "theoretical_loss": 3.413035184144497, + "tokens_seen": 2356215808 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023126980942828485, + "loss": 1.2759, + "theoretical_loss": 3.4130275857882695, + "tokens_seen": 2356281344 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023125376128385155, + "loss": 1.2563, + "theoretical_loss": 3.4130199877025467, + "tokens_seen": 2356346880 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023123771313941825, + "loss": 1.231, + "theoretical_loss": 3.413012389887312, + "tokens_seen": 2356412416 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023122166499498498, + "loss": 1.3347, + "theoretical_loss": 3.413004792342547, + "tokens_seen": 2356477952 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023120561685055168, + "loss": 1.2702, + "theoretical_loss": 3.412997195068235, + "tokens_seen": 2356543488 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023118956870611838, + "loss": 1.2081, + "theoretical_loss": 3.4129895980643594, + "tokens_seen": 2356609024 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023117352056168508, + "loss": 1.135, + "theoretical_loss": 3.412982001330903, + "tokens_seen": 2356674560 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023115747241725178, + "loss": 1.263, + "theoretical_loss": 3.4129744048678474, + "tokens_seen": 2356740096 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023114142427281849, + "loss": 1.2018, + "theoretical_loss": 3.412966808675177, + "tokens_seen": 2356805632 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023112537612838519, + "loss": 1.2313, + "theoretical_loss": 3.412959212752874, + "tokens_seen": 2356871168 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002311093279839519, + "loss": 1.221, + "theoretical_loss": 3.412951617100921, + "tokens_seen": 2356936704 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002310932798395186, + "loss": 1.3483, + "theoretical_loss": 3.4129440217193014, + "tokens_seen": 2357002240 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002310772316950853, + "loss": 1.3294, + "theoretical_loss": 3.412936426607998, + "tokens_seen": 2357067776 + }, + { + "epoch": 0.71, + "learning_rate": 0.000231061183550652, + "loss": 1.2395, + "theoretical_loss": 3.412928831766993, + "tokens_seen": 2357133312 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002310451354062187, + "loss": 1.3221, + "theoretical_loss": 3.41292123719627, + "tokens_seen": 2357198848 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002310290872617854, + "loss": 1.1959, + "theoretical_loss": 3.412913642895812, + "tokens_seen": 2357264384 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002310130391173521, + "loss": 1.2143, + "theoretical_loss": 3.412906048865601, + "tokens_seen": 2357329920 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002309969909729188, + "loss": 1.3165, + "theoretical_loss": 3.41289845510562, + "tokens_seen": 2357395456 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002309809428284855, + "loss": 1.224, + "theoretical_loss": 3.4128908616158524, + "tokens_seen": 2357460992 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002309648946840522, + "loss": 1.1545, + "theoretical_loss": 3.4128832683962815, + "tokens_seen": 2357526528 + }, + { + "epoch": 0.71, + "learning_rate": 0.0002309488465396189, + "loss": 1.3043, + "theoretical_loss": 3.412875675446889, + "tokens_seen": 2357592064 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023093279839518557, + "loss": 1.2761, + "theoretical_loss": 3.412868082767659, + "tokens_seen": 2357657600 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023091675025075227, + "loss": 1.2987, + "theoretical_loss": 3.4128604903585726, + "tokens_seen": 2357723136 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023090070210631897, + "loss": 1.3419, + "theoretical_loss": 3.412852898219615, + "tokens_seen": 2357788672 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023088465396188567, + "loss": 1.2013, + "theoretical_loss": 3.4128453063507673, + "tokens_seen": 2357854208 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023086860581745237, + "loss": 1.2646, + "theoretical_loss": 3.412837714752013, + "tokens_seen": 2357919744 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023085255767301907, + "loss": 1.2016, + "theoretical_loss": 3.412830123423335, + "tokens_seen": 2357985280 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023083650952858577, + "loss": 1.1522, + "theoretical_loss": 3.412822532364716, + "tokens_seen": 2358050816 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023082046138415247, + "loss": 1.2405, + "theoretical_loss": 3.4128149415761393, + "tokens_seen": 2358116352 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023080441323971917, + "loss": 1.2846, + "theoretical_loss": 3.4128073510575874, + "tokens_seen": 2358181888 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023078836509528587, + "loss": 1.257, + "theoretical_loss": 3.4127997608090435, + "tokens_seen": 2358247424 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023077231695085257, + "loss": 1.2674, + "theoretical_loss": 3.41279217083049, + "tokens_seen": 2358312960 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023075626880641927, + "loss": 1.3299, + "theoretical_loss": 3.41278458112191, + "tokens_seen": 2358378496 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023074022066198597, + "loss": 1.3131, + "theoretical_loss": 3.4127769916832866, + "tokens_seen": 2358444032 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023072417251755267, + "loss": 1.1599, + "theoretical_loss": 3.412769402514603, + "tokens_seen": 2358509568 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023070812437311938, + "loss": 1.2204, + "theoretical_loss": 3.412761813615841, + "tokens_seen": 2358575104 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023069207622868608, + "loss": 1.3053, + "theoretical_loss": 3.4127542249869847, + "tokens_seen": 2358640640 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023067602808425278, + "loss": 1.1934, + "theoretical_loss": 3.4127466366280164, + "tokens_seen": 2358706176 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023065997993981948, + "loss": 1.2287, + "theoretical_loss": 3.412739048538919, + "tokens_seen": 2358771712 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023064393179538618, + "loss": 1.2404, + "theoretical_loss": 3.4127314607196753, + "tokens_seen": 2358837248 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023062788365095288, + "loss": 1.2639, + "theoretical_loss": 3.4127238731702687, + "tokens_seen": 2358902784 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023061183550651958, + "loss": 1.147, + "theoretical_loss": 3.412716285890682, + "tokens_seen": 2358968320 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023059578736208628, + "loss": 1.2215, + "theoretical_loss": 3.4127086988808975, + "tokens_seen": 2359033856 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023057973921765298, + "loss": 1.2203, + "theoretical_loss": 3.4127011121408986, + "tokens_seen": 2359099392 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023056369107321968, + "loss": 1.1661, + "theoretical_loss": 3.412693525670668, + "tokens_seen": 2359164928 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023054764292878635, + "loss": 1.1706, + "theoretical_loss": 3.412685939470189, + "tokens_seen": 2359230464 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 788845, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.213777780532837, + "objective/train/theoretical_loss": 3.412678353539444, + "objective/train/tokens_used": 2379756000, + "theoretical_loss": 3.412678353539444, + "tokens_seen": 2359296000 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023053159478435306, + "loss": 1.1956, + "theoretical_loss": 3.412678353539444, + "tokens_seen": 2359296000 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023051554663991976, + "loss": 1.1663, + "theoretical_loss": 3.4126707678784167, + "tokens_seen": 2359361536 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023049949849548646, + "loss": 1.2537, + "theoretical_loss": 3.412663182487089, + "tokens_seen": 2359427072 + }, + { + "epoch": 0.71, + "learning_rate": 0.00023048345035105316, + "loss": 1.2581, + "theoretical_loss": 3.4126555973654447, + "tokens_seen": 2359492608 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023046740220661986, + "loss": 1.2683, + "theoretical_loss": 3.412648012513466, + "tokens_seen": 2359558144 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023045135406218656, + "loss": 1.2868, + "theoretical_loss": 3.4126404279311364, + "tokens_seen": 2359623680 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023043530591775326, + "loss": 1.2184, + "theoretical_loss": 3.4126328436184385, + "tokens_seen": 2359689216 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023041925777331996, + "loss": 1.1353, + "theoretical_loss": 3.4126252595753552, + "tokens_seen": 2359754752 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023040320962888666, + "loss": 1.1895, + "theoretical_loss": 3.41261767580187, + "tokens_seen": 2359820288 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023038716148445336, + "loss": 1.1818, + "theoretical_loss": 3.412610092297965, + "tokens_seen": 2359885824 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023037111334002006, + "loss": 1.2427, + "theoretical_loss": 3.412602509063624, + "tokens_seen": 2359951360 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023035506519558676, + "loss": 1.1537, + "theoretical_loss": 3.412594926098829, + "tokens_seen": 2360016896 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023033901705115346, + "loss": 1.1345, + "theoretical_loss": 3.412587343403563, + "tokens_seen": 2360082432 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023032296890672016, + "loss": 1.1786, + "theoretical_loss": 3.4125797609778097, + "tokens_seen": 2360147968 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023030692076228686, + "loss": 1.3433, + "theoretical_loss": 3.412572178821552, + "tokens_seen": 2360213504 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023029087261785356, + "loss": 1.2205, + "theoretical_loss": 3.4125645969347724, + "tokens_seen": 2360279040 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023027482447342027, + "loss": 1.0895, + "theoretical_loss": 3.4125570153174536, + "tokens_seen": 2360344576 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023025877632898697, + "loss": 1.3115, + "theoretical_loss": 3.4125494339695788, + "tokens_seen": 2360410112 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023024272818455367, + "loss": 1.2873, + "theoretical_loss": 3.412541852891131, + "tokens_seen": 2360475648 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023022668004012037, + "loss": 1.3395, + "theoretical_loss": 3.4125342720820937, + "tokens_seen": 2360541184 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023021063189568707, + "loss": 1.1301, + "theoretical_loss": 3.412526691542449, + "tokens_seen": 2360606720 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023019458375125377, + "loss": 1.197, + "theoretical_loss": 3.41251911127218, + "tokens_seen": 2360672256 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023017853560682047, + "loss": 1.223, + "theoretical_loss": 3.41251153127127, + "tokens_seen": 2360737792 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023016248746238714, + "loss": 1.2352, + "theoretical_loss": 3.4125039515397018, + "tokens_seen": 2360803328 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023014643931795384, + "loss": 1.1167, + "theoretical_loss": 3.412496372077458, + "tokens_seen": 2360868864 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023013039117352054, + "loss": 1.2319, + "theoretical_loss": 3.4124887928845222, + "tokens_seen": 2360934400 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023011434302908724, + "loss": 1.3013, + "theoretical_loss": 3.412481213960877, + "tokens_seen": 2360999936 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023009829488465395, + "loss": 1.282, + "theoretical_loss": 3.412473635306505, + "tokens_seen": 2361065472 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023008224674022065, + "loss": 1.2048, + "theoretical_loss": 3.41246605692139, + "tokens_seen": 2361131008 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023006619859578735, + "loss": 1.2246, + "theoretical_loss": 3.4124584788055143, + "tokens_seen": 2361196544 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023005015045135405, + "loss": 1.2444, + "theoretical_loss": 3.412450900958861, + "tokens_seen": 2361262080 + }, + { + "epoch": 0.72, + "learning_rate": 0.00023003410230692075, + "loss": 1.3942, + "theoretical_loss": 3.4124433233814133, + "tokens_seen": 2361327616 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002300180541624875, + "loss": 1.1909, + "theoretical_loss": 3.412435746073154, + "tokens_seen": 2361393152 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002300020060180542, + "loss": 1.2456, + "theoretical_loss": 3.4124281690340657, + "tokens_seen": 2361458688 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002299859578736209, + "loss": 1.2122, + "theoretical_loss": 3.412420592264132, + "tokens_seen": 2361524224 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002299699097291876, + "loss": 1.1769, + "theoretical_loss": 3.4124130157633354, + "tokens_seen": 2361589760 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002299538615847543, + "loss": 1.2319, + "theoretical_loss": 3.4124054395316596, + "tokens_seen": 2361655296 + }, + { + "epoch": 0.72, + "learning_rate": 0.000229937813440321, + "loss": 1.1807, + "theoretical_loss": 3.412397863569087, + "tokens_seen": 2361720832 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022992176529588768, + "loss": 1.215, + "theoretical_loss": 3.4123902878756, + "tokens_seen": 2361786368 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022990571715145438, + "loss": 1.1909, + "theoretical_loss": 3.4123827124511825, + "tokens_seen": 2361851904 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022988966900702108, + "loss": 1.1894, + "theoretical_loss": 3.412375137295817, + "tokens_seen": 2361917440 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022987362086258778, + "loss": 1.1633, + "theoretical_loss": 3.4123675624094867, + "tokens_seen": 2361982976 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022985757271815448, + "loss": 1.2018, + "theoretical_loss": 3.4123599877921746, + "tokens_seen": 2362048512 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022984152457372118, + "loss": 1.3097, + "theoretical_loss": 3.4123524134438634, + "tokens_seen": 2362114048 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022982547642928788, + "loss": 1.2144, + "theoretical_loss": 3.412344839364537, + "tokens_seen": 2362179584 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022980942828485458, + "loss": 1.1674, + "theoretical_loss": 3.412337265554177, + "tokens_seen": 2362245120 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022979338014042128, + "loss": 1.1621, + "theoretical_loss": 3.4123296920127673, + "tokens_seen": 2362310656 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022977733199598799, + "loss": 1.2634, + "theoretical_loss": 3.4123221187402906, + "tokens_seen": 2362376192 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022976128385155469, + "loss": 1.2931, + "theoretical_loss": 3.41231454573673, + "tokens_seen": 2362441728 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022974523570712139, + "loss": 1.2438, + "theoretical_loss": 3.4123069730020683, + "tokens_seen": 2362507264 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 789852, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4360320568084717, + "objective/train/theoretical_loss": 3.4122994005362886, + "objective/train/tokens_used": 2383032800, + "theoretical_loss": 3.4122994005362886, + "tokens_seen": 2362572800 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002297291875626881, + "loss": 1.3079, + "theoretical_loss": 3.4122994005362886, + "tokens_seen": 2362572800 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002297131394182548, + "loss": 1.1812, + "theoretical_loss": 3.412291828339374, + "tokens_seen": 2362638336 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002296970912738215, + "loss": 1.2449, + "theoretical_loss": 3.412284256411308, + "tokens_seen": 2362703872 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002296810431293882, + "loss": 1.1647, + "theoretical_loss": 3.4122766847520722, + "tokens_seen": 2362769408 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002296649949849549, + "loss": 1.2091, + "theoretical_loss": 3.4122691133616505, + "tokens_seen": 2362834944 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002296489468405216, + "loss": 1.2829, + "theoretical_loss": 3.412261542240026, + "tokens_seen": 2362900480 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002296328986960883, + "loss": 1.2731, + "theoretical_loss": 3.4122539713871816, + "tokens_seen": 2362966016 + }, + { + "epoch": 0.72, + "learning_rate": 0.000229616850551655, + "loss": 1.2001, + "theoretical_loss": 3.4122464008031, + "tokens_seen": 2363031552 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002296008024072217, + "loss": 1.2437, + "theoretical_loss": 3.412238830487765, + "tokens_seen": 2363097088 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002295847542627884, + "loss": 1.1921, + "theoretical_loss": 3.4122312604411587, + "tokens_seen": 2363162624 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002295687061183551, + "loss": 1.3145, + "theoretical_loss": 3.4122236906632644, + "tokens_seen": 2363228160 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002295526579739218, + "loss": 1.3032, + "theoretical_loss": 3.412216121154065, + "tokens_seen": 2363293696 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022953660982948847, + "loss": 1.2727, + "theoretical_loss": 3.412208551913544, + "tokens_seen": 2363359232 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022952056168505517, + "loss": 1.3151, + "theoretical_loss": 3.4122009829416835, + "tokens_seen": 2363424768 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022950451354062187, + "loss": 1.2459, + "theoretical_loss": 3.412193414238468, + "tokens_seen": 2363490304 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022948846539618857, + "loss": 1.2052, + "theoretical_loss": 3.4121858458038785, + "tokens_seen": 2363555840 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022947241725175527, + "loss": 1.1967, + "theoretical_loss": 3.4121782776378997, + "tokens_seen": 2363621376 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022945636910732197, + "loss": 1.1985, + "theoretical_loss": 3.412170709740514, + "tokens_seen": 2363686912 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022944032096288867, + "loss": 1.1746, + "theoretical_loss": 3.412163142111705, + "tokens_seen": 2363752448 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022942427281845537, + "loss": 1.1932, + "theoretical_loss": 3.4121555747514547, + "tokens_seen": 2363817984 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022940822467402207, + "loss": 1.2518, + "theoretical_loss": 3.412148007659747, + "tokens_seen": 2363883520 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022939217652958877, + "loss": 1.3093, + "theoretical_loss": 3.4121404408365636, + "tokens_seen": 2363949056 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022937612838515547, + "loss": 1.2148, + "theoretical_loss": 3.4121328742818893, + "tokens_seen": 2364014592 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022936008024072217, + "loss": 1.2189, + "theoretical_loss": 3.412125307995706, + "tokens_seen": 2364080128 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022934403209628888, + "loss": 1.2437, + "theoretical_loss": 3.412117741977997, + "tokens_seen": 2364145664 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022932798395185558, + "loss": 1.2613, + "theoretical_loss": 3.4121101762287456, + "tokens_seen": 2364211200 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022931193580742228, + "loss": 1.3464, + "theoretical_loss": 3.4121026107479344, + "tokens_seen": 2364276736 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022929588766298898, + "loss": 1.2408, + "theoretical_loss": 3.4120950455355468, + "tokens_seen": 2364342272 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022927983951855568, + "loss": 1.2704, + "theoretical_loss": 3.4120874805915653, + "tokens_seen": 2364407808 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022926379137412238, + "loss": 1.385, + "theoretical_loss": 3.4120799159159736, + "tokens_seen": 2364473344 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022924774322968908, + "loss": 1.1175, + "theoretical_loss": 3.4120723515087548, + "tokens_seen": 2364538880 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022923169508525578, + "loss": 1.2194, + "theoretical_loss": 3.412064787369891, + "tokens_seen": 2364604416 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022921564694082248, + "loss": 1.2183, + "theoretical_loss": 3.412057223499366, + "tokens_seen": 2364669952 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022919959879638918, + "loss": 1.3321, + "theoretical_loss": 3.412049659897163, + "tokens_seen": 2364735488 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022918355065195588, + "loss": 1.1521, + "theoretical_loss": 3.412042096563264, + "tokens_seen": 2364801024 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022916750250752258, + "loss": 1.2474, + "theoretical_loss": 3.4120345334976534, + "tokens_seen": 2364866560 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022915145436308926, + "loss": 1.1373, + "theoretical_loss": 3.412026970700313, + "tokens_seen": 2364932096 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022913540621865596, + "loss": 1.3293, + "theoretical_loss": 3.412019408171227, + "tokens_seen": 2364997632 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022911935807422266, + "loss": 1.1929, + "theoretical_loss": 3.412011845910378, + "tokens_seen": 2365063168 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022910330992978936, + "loss": 1.1794, + "theoretical_loss": 3.4120042839177485, + "tokens_seen": 2365128704 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022908726178535606, + "loss": 1.1169, + "theoretical_loss": 3.4119967221933223, + "tokens_seen": 2365194240 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022907121364092276, + "loss": 1.1822, + "theoretical_loss": 3.411989160737082, + "tokens_seen": 2365259776 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022905516549648946, + "loss": 1.1998, + "theoretical_loss": 3.4119815995490113, + "tokens_seen": 2365325312 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022903911735205616, + "loss": 1.244, + "theoretical_loss": 3.411974038629092, + "tokens_seen": 2365390848 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022902306920762286, + "loss": 1.1379, + "theoretical_loss": 3.411966477977309, + "tokens_seen": 2365456384 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022900702106318956, + "loss": 1.3107, + "theoretical_loss": 3.4119589175936436, + "tokens_seen": 2365521920 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022899097291875626, + "loss": 1.2924, + "theoretical_loss": 3.4119513574780798, + "tokens_seen": 2365587456 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022897492477432296, + "loss": 1.2933, + "theoretical_loss": 3.4119437976306, + "tokens_seen": 2365652992 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022895887662988966, + "loss": 1.2681, + "theoretical_loss": 3.4119362380511884, + "tokens_seen": 2365718528 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022894282848545636, + "loss": 1.2153, + "theoretical_loss": 3.411928678739827, + "tokens_seen": 2365784064 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 790849, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1800425052642822, + "objective/train/theoretical_loss": 3.4119211196964994, + "objective/train/tokens_used": 2386309600, + "theoretical_loss": 3.4119211196964994, + "tokens_seen": 2365849600 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022892678034102306, + "loss": 1.2147, + "theoretical_loss": 3.4119211196964994, + "tokens_seen": 2365849600 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022891073219658977, + "loss": 1.2565, + "theoretical_loss": 3.4119135609211884, + "tokens_seen": 2365915136 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022889468405215647, + "loss": 1.2411, + "theoretical_loss": 3.411906002413877, + "tokens_seen": 2365980672 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022887863590772317, + "loss": 1.3014, + "theoretical_loss": 3.411898444174549, + "tokens_seen": 2366046208 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022886258776328987, + "loss": 1.2955, + "theoretical_loss": 3.411890886203187, + "tokens_seen": 2366111744 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022884653961885657, + "loss": 1.2318, + "theoretical_loss": 3.4118833284997736, + "tokens_seen": 2366177280 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022883049147442327, + "loss": 1.3338, + "theoretical_loss": 3.4118757710642926, + "tokens_seen": 2366242816 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022881444332999, + "loss": 1.2493, + "theoretical_loss": 3.4118682138967262, + "tokens_seen": 2366308352 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002287983951855567, + "loss": 1.1439, + "theoretical_loss": 3.411860656997059, + "tokens_seen": 2366373888 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002287823470411234, + "loss": 1.2419, + "theoretical_loss": 3.4118531003652723, + "tokens_seen": 2366439424 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002287662988966901, + "loss": 1.2425, + "theoretical_loss": 3.4118455440013506, + "tokens_seen": 2366504960 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002287502507522568, + "loss": 1.3039, + "theoretical_loss": 3.411837987905276, + "tokens_seen": 2366570496 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002287342026078235, + "loss": 1.259, + "theoretical_loss": 3.4118304320770325, + "tokens_seen": 2366636032 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002287181544633902, + "loss": 1.2358, + "theoretical_loss": 3.4118228765166023, + "tokens_seen": 2366701568 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002287021063189569, + "loss": 1.1944, + "theoretical_loss": 3.4118153212239695, + "tokens_seen": 2366767104 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002286860581745236, + "loss": 1.2959, + "theoretical_loss": 3.411807766199116, + "tokens_seen": 2366832640 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002286700100300903, + "loss": 1.3148, + "theoretical_loss": 3.4118002114420256, + "tokens_seen": 2366898176 + }, + { + "epoch": 0.72, + "learning_rate": 0.000228653961885657, + "loss": 1.2839, + "theoretical_loss": 3.4117926569526813, + "tokens_seen": 2366963712 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002286379137412237, + "loss": 1.1856, + "theoretical_loss": 3.4117851027310664, + "tokens_seen": 2367029248 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002286218655967904, + "loss": 1.3219, + "theoretical_loss": 3.4117775487771635, + "tokens_seen": 2367094784 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002286058174523571, + "loss": 1.1605, + "theoretical_loss": 3.4117699950909564, + "tokens_seen": 2367160320 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002285897693079238, + "loss": 1.2291, + "theoretical_loss": 3.4117624416724275, + "tokens_seen": 2367225856 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002285737211634905, + "loss": 1.1934, + "theoretical_loss": 3.41175488852156, + "tokens_seen": 2367291392 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002285576730190572, + "loss": 1.2652, + "theoretical_loss": 3.4117473356383377, + "tokens_seen": 2367356928 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002285416248746239, + "loss": 1.2991, + "theoretical_loss": 3.411739783022743, + "tokens_seen": 2367422464 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022852557673019058, + "loss": 1.3153, + "theoretical_loss": 3.411732230674759, + "tokens_seen": 2367488000 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022850952858575728, + "loss": 1.2223, + "theoretical_loss": 3.411724678594369, + "tokens_seen": 2367553536 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022849348044132398, + "loss": 1.2912, + "theoretical_loss": 3.4117171267815563, + "tokens_seen": 2367619072 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022847743229689068, + "loss": 1.2253, + "theoretical_loss": 3.411709575236304, + "tokens_seen": 2367684608 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022846138415245738, + "loss": 1.1495, + "theoretical_loss": 3.411702023958595, + "tokens_seen": 2367750144 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022844533600802408, + "loss": 1.1605, + "theoretical_loss": 3.411694472948412, + "tokens_seen": 2367815680 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022842928786359078, + "loss": 1.1421, + "theoretical_loss": 3.4116869222057393, + "tokens_seen": 2367881216 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022841323971915748, + "loss": 1.2383, + "theoretical_loss": 3.411679371730559, + "tokens_seen": 2367946752 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022839719157472419, + "loss": 1.315, + "theoretical_loss": 3.4116718215228543, + "tokens_seen": 2368012288 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022838114343029089, + "loss": 1.1597, + "theoretical_loss": 3.411664271582609, + "tokens_seen": 2368077824 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002283650952858576, + "loss": 1.1837, + "theoretical_loss": 3.4116567219098055, + "tokens_seen": 2368143360 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002283490471414243, + "loss": 1.1976, + "theoretical_loss": 3.4116491725044273, + "tokens_seen": 2368208896 + }, + { + "epoch": 0.72, + "learning_rate": 0.000228332998996991, + "loss": 1.1807, + "theoretical_loss": 3.4116416233664575, + "tokens_seen": 2368274432 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002283169508525577, + "loss": 1.1739, + "theoretical_loss": 3.411634074495879, + "tokens_seen": 2368339968 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002283009027081244, + "loss": 1.2166, + "theoretical_loss": 3.4116265258926752, + "tokens_seen": 2368405504 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002282848545636911, + "loss": 1.2574, + "theoretical_loss": 3.4116189775568295, + "tokens_seen": 2368471040 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002282688064192578, + "loss": 1.2418, + "theoretical_loss": 3.411611429488324, + "tokens_seen": 2368536576 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002282527582748245, + "loss": 1.1883, + "theoretical_loss": 3.4116038816871432, + "tokens_seen": 2368602112 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002282367101303912, + "loss": 1.1778, + "theoretical_loss": 3.411596334153269, + "tokens_seen": 2368667648 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002282206619859579, + "loss": 1.2314, + "theoretical_loss": 3.411588786886685, + "tokens_seen": 2368733184 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002282046138415246, + "loss": 1.3404, + "theoretical_loss": 3.411581239887375, + "tokens_seen": 2368798720 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002281885656970913, + "loss": 1.2689, + "theoretical_loss": 3.411573693155321, + "tokens_seen": 2368864256 + }, + { + "epoch": 0.72, + "learning_rate": 0.000228172517552658, + "loss": 1.1791, + "theoretical_loss": 3.411566146690507, + "tokens_seen": 2368929792 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002281564694082247, + "loss": 1.1527, + "theoretical_loss": 3.4115586004929157, + "tokens_seen": 2368995328 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022814042126379137, + "loss": 1.287, + "theoretical_loss": 3.4115510545625303, + "tokens_seen": 2369060864 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 791985, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1610465049743652, + "objective/train/theoretical_loss": 3.411543508899334, + "objective/train/tokens_used": 2389586400, + "theoretical_loss": 3.411543508899334, + "tokens_seen": 2369126400 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022812437311935807, + "loss": 1.1473, + "theoretical_loss": 3.411543508899334, + "tokens_seen": 2369126400 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022810832497492477, + "loss": 1.1854, + "theoretical_loss": 3.4115359635033102, + "tokens_seen": 2369191936 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022809227683049147, + "loss": 1.2386, + "theoretical_loss": 3.411528418374442, + "tokens_seen": 2369257472 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022807622868605817, + "loss": 1.1074, + "theoretical_loss": 3.411520873512712, + "tokens_seen": 2369323008 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022806018054162487, + "loss": 1.2091, + "theoretical_loss": 3.4115133289181037, + "tokens_seen": 2369388544 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022804413239719157, + "loss": 1.1968, + "theoretical_loss": 3.4115057845906005, + "tokens_seen": 2369454080 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022802808425275827, + "loss": 1.2168, + "theoretical_loss": 3.411498240530185, + "tokens_seen": 2369519616 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022801203610832497, + "loss": 1.1901, + "theoretical_loss": 3.4114906967368412, + "tokens_seen": 2369585152 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022799598796389167, + "loss": 1.2234, + "theoretical_loss": 3.4114831532105514, + "tokens_seen": 2369650688 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022797993981945837, + "loss": 1.2362, + "theoretical_loss": 3.4114756099512995, + "tokens_seen": 2369716224 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022796389167502508, + "loss": 1.2446, + "theoretical_loss": 3.411468066959068, + "tokens_seen": 2369781760 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022794784353059178, + "loss": 1.2218, + "theoretical_loss": 3.4114605242338403, + "tokens_seen": 2369847296 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022793179538615848, + "loss": 1.2482, + "theoretical_loss": 3.4114529817756, + "tokens_seen": 2369912832 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022791574724172518, + "loss": 1.2195, + "theoretical_loss": 3.4114454395843294, + "tokens_seen": 2369978368 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022789969909729188, + "loss": 1.1934, + "theoretical_loss": 3.4114378976600124, + "tokens_seen": 2370043904 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022788365095285858, + "loss": 1.1421, + "theoretical_loss": 3.411430356002632, + "tokens_seen": 2370109440 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022786760280842528, + "loss": 1.2698, + "theoretical_loss": 3.411422814612171, + "tokens_seen": 2370174976 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022785155466399198, + "loss": 1.227, + "theoretical_loss": 3.411415273488613, + "tokens_seen": 2370240512 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022783550651955868, + "loss": 1.2715, + "theoretical_loss": 3.4114077326319414, + "tokens_seen": 2370306048 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022781945837512538, + "loss": 1.2606, + "theoretical_loss": 3.4114001920421386, + "tokens_seen": 2370371584 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022780341023069208, + "loss": 1.2148, + "theoretical_loss": 3.411392651719188, + "tokens_seen": 2370437120 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022778736208625878, + "loss": 1.2027, + "theoretical_loss": 3.4113851116630736, + "tokens_seen": 2370502656 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022777131394182548, + "loss": 1.2085, + "theoretical_loss": 3.4113775718737775, + "tokens_seen": 2370568192 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022775526579739216, + "loss": 1.2653, + "theoretical_loss": 3.411370032351283, + "tokens_seen": 2370633728 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022773921765295886, + "loss": 1.2206, + "theoretical_loss": 3.411362493095574, + "tokens_seen": 2370699264 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022772316950852556, + "loss": 1.2067, + "theoretical_loss": 3.411354954106634, + "tokens_seen": 2370764800 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022770712136409226, + "loss": 1.1931, + "theoretical_loss": 3.4113474153844443, + "tokens_seen": 2370830336 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022769107321965896, + "loss": 1.3109, + "theoretical_loss": 3.4113398769289898, + "tokens_seen": 2370895872 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022767502507522566, + "loss": 1.2778, + "theoretical_loss": 3.4113323387402534, + "tokens_seen": 2370961408 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022765897693079236, + "loss": 1.2098, + "theoretical_loss": 3.411324800818218, + "tokens_seen": 2371026944 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022764292878635906, + "loss": 1.2, + "theoretical_loss": 3.4113172631628665, + "tokens_seen": 2371092480 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022762688064192582, + "loss": 1.2634, + "theoretical_loss": 3.4113097257741827, + "tokens_seen": 2371158016 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022761083249749252, + "loss": 1.1124, + "theoretical_loss": 3.4113021886521495, + "tokens_seen": 2371223552 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022759478435305922, + "loss": 1.1771, + "theoretical_loss": 3.41129465179675, + "tokens_seen": 2371289088 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022757873620862592, + "loss": 1.2522, + "theoretical_loss": 3.411287115207968, + "tokens_seen": 2371354624 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022756268806419262, + "loss": 1.1815, + "theoretical_loss": 3.411279578885786, + "tokens_seen": 2371420160 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022754663991975932, + "loss": 1.2734, + "theoretical_loss": 3.411272042830187, + "tokens_seen": 2371485696 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022753059177532602, + "loss": 1.2423, + "theoretical_loss": 3.411264507041155, + "tokens_seen": 2371551232 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022751454363089272, + "loss": 1.234, + "theoretical_loss": 3.411256971518673, + "tokens_seen": 2371616768 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002274984954864594, + "loss": 1.3039, + "theoretical_loss": 3.4112494362627235, + "tokens_seen": 2371682304 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002274824473420261, + "loss": 1.3046, + "theoretical_loss": 3.4112419012732906, + "tokens_seen": 2371747840 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002274663991975928, + "loss": 1.2805, + "theoretical_loss": 3.4112343665503575, + "tokens_seen": 2371813376 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002274503510531595, + "loss": 1.2336, + "theoretical_loss": 3.4112268320939068, + "tokens_seen": 2371878912 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002274343029087262, + "loss": 1.2039, + "theoretical_loss": 3.411219297903922, + "tokens_seen": 2371944448 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002274182547642929, + "loss": 1.218, + "theoretical_loss": 3.4112117639803863, + "tokens_seen": 2372009984 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002274022066198596, + "loss": 1.2331, + "theoretical_loss": 3.411204230323283, + "tokens_seen": 2372075520 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002273861584754263, + "loss": 1.2391, + "theoretical_loss": 3.411196696932595, + "tokens_seen": 2372141056 + }, + { + "epoch": 0.72, + "learning_rate": 0.000227370110330993, + "loss": 1.1919, + "theoretical_loss": 3.411189163808306, + "tokens_seen": 2372206592 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002273540621865597, + "loss": 1.1886, + "theoretical_loss": 3.4111816309503986, + "tokens_seen": 2372272128 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002273380140421264, + "loss": 1.1765, + "theoretical_loss": 3.411174098358857, + "tokens_seen": 2372337664 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 793283, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2480533123016357, + "objective/train/theoretical_loss": 3.4111665660336636, + "objective/train/tokens_used": 2392863200, + "theoretical_loss": 3.4111665660336636, + "tokens_seen": 2372403200 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002273219658976931, + "loss": 1.207, + "theoretical_loss": 3.4111665660336636, + "tokens_seen": 2372403200 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002273059177532598, + "loss": 1.2152, + "theoretical_loss": 3.4111590339748017, + "tokens_seen": 2372468736 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002272898696088265, + "loss": 1.3499, + "theoretical_loss": 3.4111515021822547, + "tokens_seen": 2372534272 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002272738214643932, + "loss": 1.2498, + "theoretical_loss": 3.411143970656006, + "tokens_seen": 2372599808 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002272577733199599, + "loss": 1.1974, + "theoretical_loss": 3.4111364393960386, + "tokens_seen": 2372665344 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002272417251755266, + "loss": 1.2408, + "theoretical_loss": 3.4111289084023357, + "tokens_seen": 2372730880 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002272256770310933, + "loss": 1.2875, + "theoretical_loss": 3.4111213776748803, + "tokens_seen": 2372796416 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022720962888666, + "loss": 1.1829, + "theoretical_loss": 3.4111138472136564, + "tokens_seen": 2372861952 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002271935807422267, + "loss": 1.2387, + "theoretical_loss": 3.4111063170186466, + "tokens_seen": 2372927488 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002271775325977934, + "loss": 1.3391, + "theoretical_loss": 3.411098787089834, + "tokens_seen": 2372993024 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002271614844533601, + "loss": 1.309, + "theoretical_loss": 3.411091257427203, + "tokens_seen": 2373058560 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002271454363089268, + "loss": 1.2755, + "theoretical_loss": 3.411083728030735, + "tokens_seen": 2373124096 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002271293881644935, + "loss": 1.2005, + "theoretical_loss": 3.411076198900415, + "tokens_seen": 2373189632 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022711334002006018, + "loss": 1.2996, + "theoretical_loss": 3.411068670036225, + "tokens_seen": 2373255168 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022709729187562688, + "loss": 1.0836, + "theoretical_loss": 3.411061141438149, + "tokens_seen": 2373320704 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022708124373119358, + "loss": 1.2831, + "theoretical_loss": 3.4110536131061693, + "tokens_seen": 2373386240 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022706519558676028, + "loss": 1.1903, + "theoretical_loss": 3.4110460850402706, + "tokens_seen": 2373451776 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022704914744232698, + "loss": 1.2109, + "theoretical_loss": 3.4110385572404347, + "tokens_seen": 2373517312 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022703309929789369, + "loss": 1.2999, + "theoretical_loss": 3.411031029706646, + "tokens_seen": 2373582848 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022701705115346039, + "loss": 1.2261, + "theoretical_loss": 3.411023502438887, + "tokens_seen": 2373648384 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022700100300902709, + "loss": 1.2605, + "theoretical_loss": 3.4110159754371416, + "tokens_seen": 2373713920 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002269849548645938, + "loss": 1.3836, + "theoretical_loss": 3.4110084487013923, + "tokens_seen": 2373779456 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002269689067201605, + "loss": 1.2666, + "theoretical_loss": 3.4110009222316227, + "tokens_seen": 2373844992 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002269528585757272, + "loss": 1.2384, + "theoretical_loss": 3.4109933960278163, + "tokens_seen": 2373910528 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002269368104312939, + "loss": 1.1957, + "theoretical_loss": 3.410985870089956, + "tokens_seen": 2373976064 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002269207622868606, + "loss": 1.2826, + "theoretical_loss": 3.4109783444180253, + "tokens_seen": 2374041600 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002269047141424273, + "loss": 1.2183, + "theoretical_loss": 3.4109708190120074, + "tokens_seen": 2374107136 + }, + { + "epoch": 0.72, + "learning_rate": 0.000226888665997994, + "loss": 1.2171, + "theoretical_loss": 3.4109632938718852, + "tokens_seen": 2374172672 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002268726178535607, + "loss": 1.2713, + "theoretical_loss": 3.4109557689976424, + "tokens_seen": 2374238208 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002268565697091274, + "loss": 1.2061, + "theoretical_loss": 3.4109482443892625, + "tokens_seen": 2374303744 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002268405215646941, + "loss": 1.2888, + "theoretical_loss": 3.410940720046728, + "tokens_seen": 2374369280 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002268244734202608, + "loss": 1.2707, + "theoretical_loss": 3.410933195970023, + "tokens_seen": 2374434816 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002268084252758275, + "loss": 1.1276, + "theoretical_loss": 3.41092567215913, + "tokens_seen": 2374500352 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002267923771313942, + "loss": 1.3248, + "theoretical_loss": 3.4109181486140328, + "tokens_seen": 2374565888 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002267763289869609, + "loss": 1.2076, + "theoretical_loss": 3.4109106253347146, + "tokens_seen": 2374631424 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002267602808425276, + "loss": 1.2211, + "theoretical_loss": 3.4109031023211585, + "tokens_seen": 2374696960 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002267442326980943, + "loss": 1.2325, + "theoretical_loss": 3.410895579573348, + "tokens_seen": 2374762496 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022672818455366097, + "loss": 1.2205, + "theoretical_loss": 3.4108880570912663, + "tokens_seen": 2374828032 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022671213640922767, + "loss": 1.2713, + "theoretical_loss": 3.410880534874896, + "tokens_seen": 2374893568 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022669608826479437, + "loss": 1.1759, + "theoretical_loss": 3.4108730129242217, + "tokens_seen": 2374959104 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022668004012036107, + "loss": 1.1801, + "theoretical_loss": 3.410865491239226, + "tokens_seen": 2375024640 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022666399197592777, + "loss": 1.186, + "theoretical_loss": 3.410857969819892, + "tokens_seen": 2375090176 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022664794383149447, + "loss": 1.1979, + "theoretical_loss": 3.4108504486662032, + "tokens_seen": 2375155712 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022663189568706117, + "loss": 1.3107, + "theoretical_loss": 3.410842927778143, + "tokens_seen": 2375221248 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022661584754262787, + "loss": 1.229, + "theoretical_loss": 3.410835407155694, + "tokens_seen": 2375286784 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022659979939819458, + "loss": 1.2506, + "theoretical_loss": 3.410827886798841, + "tokens_seen": 2375352320 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022658375125376128, + "loss": 1.2827, + "theoretical_loss": 3.4108203667075654, + "tokens_seen": 2375417856 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022656770310932798, + "loss": 1.0998, + "theoretical_loss": 3.4108128468818517, + "tokens_seen": 2375483392 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022655165496489468, + "loss": 1.2627, + "theoretical_loss": 3.4108053273216834, + "tokens_seen": 2375548928 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022653560682046138, + "loss": 1.1509, + "theoretical_loss": 3.4107978080270427, + "tokens_seen": 2375614464 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 794143, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2268905639648438, + "objective/train/theoretical_loss": 3.410790288997914, + "objective/train/tokens_used": 2396140000, + "theoretical_loss": 3.410790288997914, + "tokens_seen": 2375680000 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022651955867602808, + "loss": 1.2947, + "theoretical_loss": 3.410790288997914, + "tokens_seen": 2375680000 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022650351053159478, + "loss": 1.2449, + "theoretical_loss": 3.4107827702342797, + "tokens_seen": 2375745536 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022648746238716148, + "loss": 1.2793, + "theoretical_loss": 3.4107752517361236, + "tokens_seen": 2375811072 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022647141424272818, + "loss": 1.2137, + "theoretical_loss": 3.4107677335034294, + "tokens_seen": 2375876608 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022645536609829488, + "loss": 1.2129, + "theoretical_loss": 3.4107602155361794, + "tokens_seen": 2375942144 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022643931795386158, + "loss": 1.1991, + "theoretical_loss": 3.410752697834358, + "tokens_seen": 2376007680 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002264232698094283, + "loss": 1.214, + "theoretical_loss": 3.410745180397947, + "tokens_seen": 2376073216 + }, + { + "epoch": 0.72, + "learning_rate": 0.000226407221664995, + "loss": 1.2258, + "theoretical_loss": 3.4107376632269313, + "tokens_seen": 2376138752 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002263911735205617, + "loss": 1.1537, + "theoretical_loss": 3.410730146321294, + "tokens_seen": 2376204288 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002263751253761284, + "loss": 1.2582, + "theoretical_loss": 3.4107226296810174, + "tokens_seen": 2376269824 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002263590772316951, + "loss": 1.1709, + "theoretical_loss": 3.410715113306085, + "tokens_seen": 2376335360 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002263430290872618, + "loss": 1.2334, + "theoretical_loss": 3.410707597196481, + "tokens_seen": 2376400896 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002263269809428285, + "loss": 1.2121, + "theoretical_loss": 3.4107000813521884, + "tokens_seen": 2376466432 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022631093279839521, + "loss": 1.3528, + "theoretical_loss": 3.41069256577319, + "tokens_seen": 2376531968 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022629488465396191, + "loss": 1.2494, + "theoretical_loss": 3.41068505045947, + "tokens_seen": 2376597504 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022627883650952861, + "loss": 1.2996, + "theoretical_loss": 3.410677535411011, + "tokens_seen": 2376663040 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022626278836509532, + "loss": 1.197, + "theoretical_loss": 3.410670020627796, + "tokens_seen": 2376728576 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022624674022066202, + "loss": 1.1699, + "theoretical_loss": 3.4106625061098095, + "tokens_seen": 2376794112 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022623069207622872, + "loss": 1.1868, + "theoretical_loss": 3.4106549918570335, + "tokens_seen": 2376859648 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022621464393179542, + "loss": 1.2734, + "theoretical_loss": 3.4106474778694524, + "tokens_seen": 2376925184 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022619859578736212, + "loss": 1.2605, + "theoretical_loss": 3.410639964147049, + "tokens_seen": 2376990720 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022618254764292882, + "loss": 1.2087, + "theoretical_loss": 3.410632450689807, + "tokens_seen": 2377056256 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022616649949849552, + "loss": 1.1526, + "theoretical_loss": 3.410624937497709, + "tokens_seen": 2377121792 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022615045135406222, + "loss": 1.209, + "theoretical_loss": 3.4106174245707397, + "tokens_seen": 2377187328 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022613440320962892, + "loss": 1.3085, + "theoretical_loss": 3.410609911908881, + "tokens_seen": 2377252864 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022611835506519562, + "loss": 1.3448, + "theoretical_loss": 3.4106023995121166, + "tokens_seen": 2377318400 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002261023069207623, + "loss": 1.2383, + "theoretical_loss": 3.41059488738043, + "tokens_seen": 2377383936 + }, + { + "epoch": 0.72, + "learning_rate": 0.000226086258776329, + "loss": 1.3249, + "theoretical_loss": 3.410587375513805, + "tokens_seen": 2377449472 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002260702106318957, + "loss": 1.1943, + "theoretical_loss": 3.4105798639122242, + "tokens_seen": 2377515008 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002260541624874624, + "loss": 1.171, + "theoretical_loss": 3.410572352575671, + "tokens_seen": 2377580544 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002260381143430291, + "loss": 1.2519, + "theoretical_loss": 3.4105648415041294, + "tokens_seen": 2377646080 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002260220661985958, + "loss": 1.245, + "theoretical_loss": 3.4105573306975825, + "tokens_seen": 2377711616 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002260060180541625, + "loss": 1.142, + "theoretical_loss": 3.410549820156013, + "tokens_seen": 2377777152 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002259899699097292, + "loss": 1.3106, + "theoretical_loss": 3.410542309879405, + "tokens_seen": 2377842688 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002259739217652959, + "loss": 1.2829, + "theoretical_loss": 3.4105347998677416, + "tokens_seen": 2377908224 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002259578736208626, + "loss": 1.1594, + "theoretical_loss": 3.4105272901210064, + "tokens_seen": 2377973760 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002259418254764293, + "loss": 1.2976, + "theoretical_loss": 3.410519780639182, + "tokens_seen": 2378039296 + }, + { + "epoch": 0.72, + "learning_rate": 0.000225925777331996, + "loss": 1.2219, + "theoretical_loss": 3.4105122714222524, + "tokens_seen": 2378104832 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002259097291875627, + "loss": 1.2078, + "theoretical_loss": 3.410504762470201, + "tokens_seen": 2378170368 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002258936810431294, + "loss": 1.1857, + "theoretical_loss": 3.410497253783011, + "tokens_seen": 2378235904 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002258776328986961, + "loss": 1.171, + "theoretical_loss": 3.4104897453606653, + "tokens_seen": 2378301440 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002258615847542628, + "loss": 1.2654, + "theoretical_loss": 3.410482237203148, + "tokens_seen": 2378366976 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002258455366098295, + "loss": 1.2392, + "theoretical_loss": 3.410474729310442, + "tokens_seen": 2378432512 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002258294884653962, + "loss": 1.2609, + "theoretical_loss": 3.410467221682531, + "tokens_seen": 2378498048 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002258134403209629, + "loss": 1.2041, + "theoretical_loss": 3.410459714319398, + "tokens_seen": 2378563584 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002257973921765296, + "loss": 1.2164, + "theoretical_loss": 3.4104522072210264, + "tokens_seen": 2378629120 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002257813440320963, + "loss": 1.2834, + "theoretical_loss": 3.4104447003874, + "tokens_seen": 2378694656 + }, + { + "epoch": 0.72, + "learning_rate": 0.000225765295887663, + "loss": 1.2135, + "theoretical_loss": 3.4104371938185016, + "tokens_seen": 2378760192 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002257492477432297, + "loss": 1.2285, + "theoretical_loss": 3.4104296875143154, + "tokens_seen": 2378825728 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002257331995987964, + "loss": 1.2667, + "theoretical_loss": 3.4104221814748237, + "tokens_seen": 2378891264 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 795373, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1782634258270264, + "objective/train/theoretical_loss": 3.410414675700011, + "objective/train/tokens_used": 2399416800, + "theoretical_loss": 3.410414675700011, + "tokens_seen": 2378956800 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022571715145436308, + "loss": 1.2295, + "theoretical_loss": 3.410414675700011, + "tokens_seen": 2378956800 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022570110330992978, + "loss": 1.1878, + "theoretical_loss": 3.4104071701898593, + "tokens_seen": 2379022336 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022568505516549648, + "loss": 1.2206, + "theoretical_loss": 3.410399664944353, + "tokens_seen": 2379087872 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022566900702106318, + "loss": 1.3167, + "theoretical_loss": 3.4103921599634752, + "tokens_seen": 2379153408 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022565295887662989, + "loss": 1.2981, + "theoretical_loss": 3.4103846552472095, + "tokens_seen": 2379218944 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022563691073219659, + "loss": 1.2547, + "theoretical_loss": 3.410377150795539, + "tokens_seen": 2379284480 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002256208625877633, + "loss": 1.3127, + "theoretical_loss": 3.4103696466084474, + "tokens_seen": 2379350016 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022560481444333, + "loss": 1.2246, + "theoretical_loss": 3.4103621426859174, + "tokens_seen": 2379415552 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002255887662988967, + "loss": 1.1684, + "theoretical_loss": 3.410354639027933, + "tokens_seen": 2379481088 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002255727181544634, + "loss": 1.2786, + "theoretical_loss": 3.4103471356344777, + "tokens_seen": 2379546624 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002255566700100301, + "loss": 1.2614, + "theoretical_loss": 3.4103396325055346, + "tokens_seen": 2379612160 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002255406218655968, + "loss": 1.2885, + "theoretical_loss": 3.4103321296410867, + "tokens_seen": 2379677696 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002255245737211635, + "loss": 1.2694, + "theoretical_loss": 3.4103246270411183, + "tokens_seen": 2379743232 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002255085255767302, + "loss": 1.3012, + "theoretical_loss": 3.410317124705612, + "tokens_seen": 2379808768 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002254924774322969, + "loss": 1.2191, + "theoretical_loss": 3.4103096226345517, + "tokens_seen": 2379874304 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002254764292878636, + "loss": 1.153, + "theoretical_loss": 3.4103021208279203, + "tokens_seen": 2379939840 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002254603811434303, + "loss": 1.2488, + "theoretical_loss": 3.4102946192857018, + "tokens_seen": 2380005376 + }, + { + "epoch": 0.72, + "learning_rate": 0.000225444332998997, + "loss": 1.1691, + "theoretical_loss": 3.4102871180078793, + "tokens_seen": 2380070912 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002254282848545637, + "loss": 1.1865, + "theoretical_loss": 3.410279616994436, + "tokens_seen": 2380136448 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002254122367101304, + "loss": 1.2687, + "theoretical_loss": 3.4102721162453555, + "tokens_seen": 2380201984 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002253961885656971, + "loss": 1.2199, + "theoretical_loss": 3.410264615760621, + "tokens_seen": 2380267520 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002253801404212638, + "loss": 1.2146, + "theoretical_loss": 3.4102571155402166, + "tokens_seen": 2380333056 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002253640922768305, + "loss": 1.2565, + "theoretical_loss": 3.410249615584125, + "tokens_seen": 2380398592 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002253480441323972, + "loss": 1.1584, + "theoretical_loss": 3.4102421158923297, + "tokens_seen": 2380464128 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022533199598796387, + "loss": 1.2597, + "theoretical_loss": 3.4102346164648143, + "tokens_seen": 2380529664 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022531594784353057, + "loss": 1.1658, + "theoretical_loss": 3.410227117301562, + "tokens_seen": 2380595200 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022529989969909727, + "loss": 1.33, + "theoretical_loss": 3.4102196184025564, + "tokens_seen": 2380660736 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022528385155466397, + "loss": 1.3188, + "theoretical_loss": 3.410212119767781, + "tokens_seen": 2380726272 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022526780341023067, + "loss": 1.3587, + "theoretical_loss": 3.4102046213972192, + "tokens_seen": 2380791808 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022525175526579737, + "loss": 1.2253, + "theoretical_loss": 3.410197123290854, + "tokens_seen": 2380857344 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022523570712136407, + "loss": 1.2991, + "theoretical_loss": 3.4101896254486697, + "tokens_seen": 2380922880 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022521965897693083, + "loss": 1.2683, + "theoretical_loss": 3.4101821278706486, + "tokens_seen": 2380988416 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022520361083249753, + "loss": 1.2655, + "theoretical_loss": 3.410174630556775, + "tokens_seen": 2381053952 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022518756268806423, + "loss": 1.3182, + "theoretical_loss": 3.4101671335070316, + "tokens_seen": 2381119488 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022517151454363093, + "loss": 1.1117, + "theoretical_loss": 3.4101596367214024, + "tokens_seen": 2381185024 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022515546639919763, + "loss": 1.2422, + "theoretical_loss": 3.410152140199871, + "tokens_seen": 2381250560 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022513941825476433, + "loss": 1.2949, + "theoretical_loss": 3.41014464394242, + "tokens_seen": 2381316096 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022512337011033103, + "loss": 1.188, + "theoretical_loss": 3.410137147949033, + "tokens_seen": 2381381632 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022510732196589773, + "loss": 1.0207, + "theoretical_loss": 3.4101296522196947, + "tokens_seen": 2381447168 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002250912738214644, + "loss": 1.279, + "theoretical_loss": 3.4101221567543867, + "tokens_seen": 2381512704 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002250752256770311, + "loss": 1.1728, + "theoretical_loss": 3.410114661553094, + "tokens_seen": 2381578240 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002250591775325978, + "loss": 1.2417, + "theoretical_loss": 3.410107166615799, + "tokens_seen": 2381643776 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002250431293881645, + "loss": 1.1738, + "theoretical_loss": 3.4100996719424854, + "tokens_seen": 2381709312 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002250270812437312, + "loss": 1.2056, + "theoretical_loss": 3.4100921775331368, + "tokens_seen": 2381774848 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002250110330992979, + "loss": 1.3162, + "theoretical_loss": 3.4100846833877365, + "tokens_seen": 2381840384 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002249949849548646, + "loss": 1.2366, + "theoretical_loss": 3.410077189506268, + "tokens_seen": 2381905920 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002249789368104313, + "loss": 1.3271, + "theoretical_loss": 3.4100696958887147, + "tokens_seen": 2381971456 + }, + { + "epoch": 0.72, + "learning_rate": 0.000224962888665998, + "loss": 1.2511, + "theoretical_loss": 3.41006220253506, + "tokens_seen": 2382036992 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002249468405215647, + "loss": 1.1874, + "theoretical_loss": 3.410054709445288, + "tokens_seen": 2382102528 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022493079237713141, + "loss": 1.3549, + "theoretical_loss": 3.410047216619381, + "tokens_seen": 2382168064 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 796235, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.25529146194458, + "objective/train/theoretical_loss": 3.4100397240573233, + "objective/train/tokens_used": 2402693600, + "theoretical_loss": 3.4100397240573233, + "tokens_seen": 2382233600 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022491474423269811, + "loss": 1.2103, + "theoretical_loss": 3.4100397240573233, + "tokens_seen": 2382233600 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022489869608826482, + "loss": 1.2223, + "theoretical_loss": 3.410032231759098, + "tokens_seen": 2382299136 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022488264794383152, + "loss": 1.1529, + "theoretical_loss": 3.4100247397246886, + "tokens_seen": 2382364672 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022486659979939822, + "loss": 1.1752, + "theoretical_loss": 3.4100172479540785, + "tokens_seen": 2382430208 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022485055165496492, + "loss": 1.2051, + "theoretical_loss": 3.4100097564472516, + "tokens_seen": 2382495744 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022483450351053162, + "loss": 1.2359, + "theoretical_loss": 3.4100022652041906, + "tokens_seen": 2382561280 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022481845536609832, + "loss": 1.3021, + "theoretical_loss": 3.4099947742248795, + "tokens_seen": 2382626816 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022480240722166502, + "loss": 1.2265, + "theoretical_loss": 3.4099872835093015, + "tokens_seen": 2382692352 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022478635907723172, + "loss": 1.0809, + "theoretical_loss": 3.40997979305744, + "tokens_seen": 2382757888 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022477031093279842, + "loss": 1.2624, + "theoretical_loss": 3.409972302869279, + "tokens_seen": 2382823424 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022475426278836512, + "loss": 1.2082, + "theoretical_loss": 3.4099648129448017, + "tokens_seen": 2382888960 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022473821464393182, + "loss": 1.2004, + "theoretical_loss": 3.409957323283991, + "tokens_seen": 2382954496 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022472216649949852, + "loss": 1.3032, + "theoretical_loss": 3.4099498338868313, + "tokens_seen": 2383020032 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002247061183550652, + "loss": 1.27, + "theoretical_loss": 3.4099423447533055, + "tokens_seen": 2383085568 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002246900702106319, + "loss": 1.2191, + "theoretical_loss": 3.4099348558833973, + "tokens_seen": 2383151104 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002246740220661986, + "loss": 1.312, + "theoretical_loss": 3.40992736727709, + "tokens_seen": 2383216640 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002246579739217653, + "loss": 1.1118, + "theoretical_loss": 3.4099198789343665, + "tokens_seen": 2383282176 + }, + { + "epoch": 0.72, + "learning_rate": 0.000224641925777332, + "loss": 1.2676, + "theoretical_loss": 3.4099123908552116, + "tokens_seen": 2383347712 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002246258776328987, + "loss": 1.2293, + "theoretical_loss": 3.4099049030396076, + "tokens_seen": 2383413248 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002246098294884654, + "loss": 1.2976, + "theoretical_loss": 3.4098974154875386, + "tokens_seen": 2383478784 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002245937813440321, + "loss": 1.2863, + "theoretical_loss": 3.409889928198988, + "tokens_seen": 2383544320 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002245777331995988, + "loss": 1.2274, + "theoretical_loss": 3.4098824411739392, + "tokens_seen": 2383609856 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002245616850551655, + "loss": 1.1859, + "theoretical_loss": 3.409874954412376, + "tokens_seen": 2383675392 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002245456369107322, + "loss": 1.2085, + "theoretical_loss": 3.409867467914281, + "tokens_seen": 2383740928 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002245295887662989, + "loss": 1.2198, + "theoretical_loss": 3.4098599816796384, + "tokens_seen": 2383806464 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002245135406218656, + "loss": 1.2939, + "theoretical_loss": 3.409852495708432, + "tokens_seen": 2383872000 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002244974924774323, + "loss": 1.2061, + "theoretical_loss": 3.4098450100006445, + "tokens_seen": 2383937536 + }, + { + "epoch": 0.72, + "learning_rate": 0.000224481444332999, + "loss": 1.1416, + "theoretical_loss": 3.4098375245562593, + "tokens_seen": 2384003072 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002244653961885657, + "loss": 1.2201, + "theoretical_loss": 3.4098300393752607, + "tokens_seen": 2384068608 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002244493480441324, + "loss": 1.2717, + "theoretical_loss": 3.409822554457632, + "tokens_seen": 2384134144 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002244332998996991, + "loss": 1.2119, + "theoretical_loss": 3.4098150698033565, + "tokens_seen": 2384199680 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002244172517552658, + "loss": 1.2526, + "theoretical_loss": 3.4098075854124175, + "tokens_seen": 2384265216 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002244012036108325, + "loss": 1.2875, + "theoretical_loss": 3.409800101284799, + "tokens_seen": 2384330752 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002243851554663992, + "loss": 1.1876, + "theoretical_loss": 3.4097926174204836, + "tokens_seen": 2384396288 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002243691073219659, + "loss": 1.2671, + "theoretical_loss": 3.409785133819456, + "tokens_seen": 2384461824 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002243530591775326, + "loss": 1.2548, + "theoretical_loss": 3.409777650481699, + "tokens_seen": 2384527360 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002243370110330993, + "loss": 1.265, + "theoretical_loss": 3.4097701674071956, + "tokens_seen": 2384592896 + }, + { + "epoch": 0.72, + "learning_rate": 0.000224320962888666, + "loss": 1.1296, + "theoretical_loss": 3.4097626845959303, + "tokens_seen": 2384658432 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022430491474423268, + "loss": 1.243, + "theoretical_loss": 3.409755202047886, + "tokens_seen": 2384723968 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022428886659979939, + "loss": 1.2226, + "theoretical_loss": 3.409747719763047, + "tokens_seen": 2384789504 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022427281845536609, + "loss": 1.2694, + "theoretical_loss": 3.4097402377413957, + "tokens_seen": 2384855040 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002242567703109328, + "loss": 1.1591, + "theoretical_loss": 3.4097327559829163, + "tokens_seen": 2384920576 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002242407221664995, + "loss": 1.2342, + "theoretical_loss": 3.4097252744875917, + "tokens_seen": 2384986112 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002242246740220662, + "loss": 1.2674, + "theoretical_loss": 3.4097177932554064, + "tokens_seen": 2385051648 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002242086258776329, + "loss": 1.2491, + "theoretical_loss": 3.409710312286343, + "tokens_seen": 2385117184 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002241925777331996, + "loss": 1.2174, + "theoretical_loss": 3.4097028315803857, + "tokens_seen": 2385182720 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002241765295887663, + "loss": 1.3088, + "theoretical_loss": 3.409695351137518, + "tokens_seen": 2385248256 + }, + { + "epoch": 0.72, + "learning_rate": 0.000224160481444333, + "loss": 1.287, + "theoretical_loss": 3.4096878709577223, + "tokens_seen": 2385313792 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002241444332998997, + "loss": 1.2305, + "theoretical_loss": 3.4096803910409834, + "tokens_seen": 2385379328 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002241283851554664, + "loss": 1.279, + "theoretical_loss": 3.4096729113872843, + "tokens_seen": 2385444864 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 797452, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2669398784637451, + "objective/train/theoretical_loss": 3.4096654319966087, + "objective/train/tokens_used": 2405970400, + "theoretical_loss": 3.4096654319966087, + "tokens_seen": 2385510400 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002241123370110331, + "loss": 1.1906, + "theoretical_loss": 3.4096654319966087, + "tokens_seen": 2385510400 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002240962888665998, + "loss": 1.2593, + "theoretical_loss": 3.40965795286894, + "tokens_seen": 2385575936 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002240802407221665, + "loss": 1.1316, + "theoretical_loss": 3.4096504740042617, + "tokens_seen": 2385641472 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002240641925777332, + "loss": 1.2538, + "theoretical_loss": 3.4096429954025576, + "tokens_seen": 2385707008 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002240481444332999, + "loss": 1.2404, + "theoretical_loss": 3.4096355170638106, + "tokens_seen": 2385772544 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002240320962888666, + "loss": 1.2403, + "theoretical_loss": 3.409628038988005, + "tokens_seen": 2385838080 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022401604814443332, + "loss": 1.1189, + "theoretical_loss": 3.4096205611751236, + "tokens_seen": 2385903616 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022400000000000002, + "loss": 1.2319, + "theoretical_loss": 3.4096130836251506, + "tokens_seen": 2385969152 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022398395185556672, + "loss": 1.2988, + "theoretical_loss": 3.409605606338069, + "tokens_seen": 2386034688 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022396790371113343, + "loss": 1.2374, + "theoretical_loss": 3.4095981293138626, + "tokens_seen": 2386100224 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022395185556670013, + "loss": 1.2335, + "theoretical_loss": 3.409590652552515, + "tokens_seen": 2386165760 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022393580742226683, + "loss": 1.1489, + "theoretical_loss": 3.4095831760540096, + "tokens_seen": 2386231296 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022391975927783353, + "loss": 1.3527, + "theoretical_loss": 3.4095756998183298, + "tokens_seen": 2386296832 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022390371113340023, + "loss": 1.1674, + "theoretical_loss": 3.4095682238454597, + "tokens_seen": 2386362368 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022388766298896693, + "loss": 1.3141, + "theoretical_loss": 3.4095607481353825, + "tokens_seen": 2386427904 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022387161484453363, + "loss": 1.2816, + "theoretical_loss": 3.4095532726880813, + "tokens_seen": 2386493440 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022385556670010033, + "loss": 1.2809, + "theoretical_loss": 3.4095457975035406, + "tokens_seen": 2386558976 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022383951855566703, + "loss": 1.1744, + "theoretical_loss": 3.409538322581743, + "tokens_seen": 2386624512 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022382347041123373, + "loss": 1.2131, + "theoretical_loss": 3.4095308479226727, + "tokens_seen": 2386690048 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022380742226680043, + "loss": 1.2657, + "theoretical_loss": 3.4095233735263126, + "tokens_seen": 2386755584 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022379137412236713, + "loss": 1.2441, + "theoretical_loss": 3.4095158993926473, + "tokens_seen": 2386821120 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022377532597793383, + "loss": 1.3152, + "theoretical_loss": 3.4095084255216594, + "tokens_seen": 2386886656 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022375927783350053, + "loss": 1.2758, + "theoretical_loss": 3.4095009519133326, + "tokens_seen": 2386952192 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022374322968906723, + "loss": 1.1604, + "theoretical_loss": 3.409493478567651, + "tokens_seen": 2387017728 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022372718154463393, + "loss": 1.2376, + "theoretical_loss": 3.409486005484598, + "tokens_seen": 2387083264 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022371113340020064, + "loss": 1.2404, + "theoretical_loss": 3.4094785326641563, + "tokens_seen": 2387148800 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022369508525576734, + "loss": 1.2257, + "theoretical_loss": 3.4094710601063105, + "tokens_seen": 2387214336 + }, + { + "epoch": 0.72, + "learning_rate": 0.000223679037111334, + "loss": 1.2438, + "theoretical_loss": 3.409463587811044, + "tokens_seen": 2387279872 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002236629889669007, + "loss": 1.2165, + "theoretical_loss": 3.40945611577834, + "tokens_seen": 2387345408 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002236469408224674, + "loss": 1.315, + "theoretical_loss": 3.409448644008182, + "tokens_seen": 2387410944 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002236308926780341, + "loss": 1.1477, + "theoretical_loss": 3.409441172500554, + "tokens_seen": 2387476480 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002236148445336008, + "loss": 1.2359, + "theoretical_loss": 3.4094337012554394, + "tokens_seen": 2387542016 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002235987963891675, + "loss": 1.2446, + "theoretical_loss": 3.4094262302728215, + "tokens_seen": 2387607552 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002235827482447342, + "loss": 1.1663, + "theoretical_loss": 3.4094187595526844, + "tokens_seen": 2387673088 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022356670010030091, + "loss": 1.3253, + "theoretical_loss": 3.409411289095011, + "tokens_seen": 2387738624 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022355065195586761, + "loss": 1.2764, + "theoretical_loss": 3.4094038188997855, + "tokens_seen": 2387804160 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022353460381143432, + "loss": 1.2126, + "theoretical_loss": 3.409396348966991, + "tokens_seen": 2387869696 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022351855566700102, + "loss": 1.2194, + "theoretical_loss": 3.409388879296612, + "tokens_seen": 2387935232 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022350250752256772, + "loss": 1.1356, + "theoretical_loss": 3.4093814098886304, + "tokens_seen": 2388000768 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022348645937813442, + "loss": 1.3522, + "theoretical_loss": 3.4093739407430315, + "tokens_seen": 2388066304 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022347041123370112, + "loss": 1.3022, + "theoretical_loss": 3.409366471859798, + "tokens_seen": 2388131840 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022345436308926782, + "loss": 1.3022, + "theoretical_loss": 3.4093590032389134, + "tokens_seen": 2388197376 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022343831494483452, + "loss": 1.3056, + "theoretical_loss": 3.409351534880362, + "tokens_seen": 2388262912 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022342226680040122, + "loss": 1.2872, + "theoretical_loss": 3.4093440667841266, + "tokens_seen": 2388328448 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022340621865596792, + "loss": 1.2934, + "theoretical_loss": 3.409336598950191, + "tokens_seen": 2388393984 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022339017051153462, + "loss": 1.3046, + "theoretical_loss": 3.4093291313785388, + "tokens_seen": 2388459520 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022337412236710132, + "loss": 1.205, + "theoretical_loss": 3.409321664069154, + "tokens_seen": 2388525056 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022335807422266802, + "loss": 1.2247, + "theoretical_loss": 3.4093141970220198, + "tokens_seen": 2388590592 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022334202607823472, + "loss": 1.3195, + "theoretical_loss": 3.4093067302371196, + "tokens_seen": 2388656128 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022332597793380142, + "loss": 1.2349, + "theoretical_loss": 3.4092992637144377, + "tokens_seen": 2388721664 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 798410, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2534170150756836, + "objective/train/theoretical_loss": 3.409291797453957, + "objective/train/tokens_used": 2409247200, + "theoretical_loss": 3.409291797453957, + "tokens_seen": 2388787200 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022330992978936812, + "loss": 1.2783, + "theoretical_loss": 3.409291797453957, + "tokens_seen": 2388787200 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002232938816449348, + "loss": 1.2981, + "theoretical_loss": 3.4092843314556616, + "tokens_seen": 2388852736 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002232778335005015, + "loss": 1.2042, + "theoretical_loss": 3.4092768657195345, + "tokens_seen": 2388918272 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002232617853560682, + "loss": 1.2259, + "theoretical_loss": 3.40926940024556, + "tokens_seen": 2388983808 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002232457372116349, + "loss": 1.272, + "theoretical_loss": 3.4092619350337214, + "tokens_seen": 2389049344 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002232296890672016, + "loss": 1.2645, + "theoretical_loss": 3.4092544700840017, + "tokens_seen": 2389114880 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002232136409227683, + "loss": 1.2495, + "theoretical_loss": 3.4092470053963857, + "tokens_seen": 2389180416 + }, + { + "epoch": 0.72, + "learning_rate": 0.000223197592778335, + "loss": 1.3051, + "theoretical_loss": 3.4092395409708565, + "tokens_seen": 2389245952 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002231815446339017, + "loss": 1.2099, + "theoretical_loss": 3.409232076807397, + "tokens_seen": 2389311488 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002231654964894684, + "loss": 1.3029, + "theoretical_loss": 3.4092246129059918, + "tokens_seen": 2389377024 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002231494483450351, + "loss": 1.2125, + "theoretical_loss": 3.409217149266624, + "tokens_seen": 2389442560 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002231334002006018, + "loss": 1.2874, + "theoretical_loss": 3.4092096858892775, + "tokens_seen": 2389508096 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002231173520561685, + "loss": 1.2268, + "theoretical_loss": 3.4092022227739354, + "tokens_seen": 2389573632 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002231013039117352, + "loss": 1.2889, + "theoretical_loss": 3.409194759920582, + "tokens_seen": 2389639168 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002230852557673019, + "loss": 1.3325, + "theoretical_loss": 3.4091872973292006, + "tokens_seen": 2389704704 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002230692076228686, + "loss": 1.2182, + "theoretical_loss": 3.4091798349997746, + "tokens_seen": 2389770240 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002230531594784353, + "loss": 1.2603, + "theoretical_loss": 3.409172372932288, + "tokens_seen": 2389835776 + }, + { + "epoch": 0.72, + "learning_rate": 0.000223037111334002, + "loss": 1.2273, + "theoretical_loss": 3.409164911126724, + "tokens_seen": 2389901312 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002230210631895687, + "loss": 1.1944, + "theoretical_loss": 3.4091574495830668, + "tokens_seen": 2389966848 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002230050150451354, + "loss": 1.1673, + "theoretical_loss": 3.4091499883012997, + "tokens_seen": 2390032384 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002229889669007021, + "loss": 1.1867, + "theoretical_loss": 3.409142527281406, + "tokens_seen": 2390097920 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002229729187562688, + "loss": 1.1394, + "theoretical_loss": 3.40913506652337, + "tokens_seen": 2390163456 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002229568706118355, + "loss": 1.2844, + "theoretical_loss": 3.409127606027175, + "tokens_seen": 2390228992 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002229408224674022, + "loss": 1.2896, + "theoretical_loss": 3.409120145792804, + "tokens_seen": 2390294528 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002229247743229689, + "loss": 1.175, + "theoretical_loss": 3.409112685820242, + "tokens_seen": 2390360064 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022290872617853559, + "loss": 1.1948, + "theoretical_loss": 3.4091052261094714, + "tokens_seen": 2390425600 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022289267803410229, + "loss": 1.2125, + "theoretical_loss": 3.4090977666604765, + "tokens_seen": 2390491136 + }, + { + "epoch": 0.72, + "learning_rate": 0.000222876629889669, + "loss": 1.2967, + "theoretical_loss": 3.4090903074732406, + "tokens_seen": 2390556672 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002228605817452357, + "loss": 1.2522, + "theoretical_loss": 3.409082848547748, + "tokens_seen": 2390622208 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002228445336008024, + "loss": 1.2405, + "theoretical_loss": 3.409075389883981, + "tokens_seen": 2390687744 + }, + { + "epoch": 0.72, + "learning_rate": 0.0002228284854563691, + "loss": 1.145, + "theoretical_loss": 3.4090679314819248, + "tokens_seen": 2390753280 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022281243731193584, + "loss": 1.2426, + "theoretical_loss": 3.409060473341562, + "tokens_seen": 2390818816 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022279638916750254, + "loss": 1.3112, + "theoretical_loss": 3.4090530154628773, + "tokens_seen": 2390884352 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022278034102306924, + "loss": 1.2625, + "theoretical_loss": 3.409045557845853, + "tokens_seen": 2390949888 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022276429287863595, + "loss": 1.2141, + "theoretical_loss": 3.409038100490473, + "tokens_seen": 2391015424 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022274824473420265, + "loss": 1.1297, + "theoretical_loss": 3.4090306433967217, + "tokens_seen": 2391080960 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022273219658976935, + "loss": 1.2906, + "theoretical_loss": 3.4090231865645824, + "tokens_seen": 2391146496 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022271614844533605, + "loss": 1.2714, + "theoretical_loss": 3.4090157299940387, + "tokens_seen": 2391212032 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022270010030090275, + "loss": 1.1742, + "theoretical_loss": 3.4090082736850746, + "tokens_seen": 2391277568 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022268405215646945, + "loss": 1.2814, + "theoretical_loss": 3.409000817637673, + "tokens_seen": 2391343104 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022266800401203612, + "loss": 1.2991, + "theoretical_loss": 3.4089933618518176, + "tokens_seen": 2391408640 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022265195586760282, + "loss": 1.3264, + "theoretical_loss": 3.4089859063274934, + "tokens_seen": 2391474176 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022263590772316952, + "loss": 1.218, + "theoretical_loss": 3.4089784510646823, + "tokens_seen": 2391539712 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022261985957873622, + "loss": 1.2272, + "theoretical_loss": 3.408970996063369, + "tokens_seen": 2391605248 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022260381143430292, + "loss": 1.214, + "theoretical_loss": 3.408963541323537, + "tokens_seen": 2391670784 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022258776328986963, + "loss": 1.2642, + "theoretical_loss": 3.40895608684517, + "tokens_seen": 2391736320 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022257171514543633, + "loss": 1.2275, + "theoretical_loss": 3.4089486326282517, + "tokens_seen": 2391801856 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022255566700100303, + "loss": 1.2221, + "theoretical_loss": 3.408941178672765, + "tokens_seen": 2391867392 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022253961885656973, + "loss": 1.19, + "theoretical_loss": 3.4089337249786946, + "tokens_seen": 2391932928 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022252357071213643, + "loss": 1.258, + "theoretical_loss": 3.408926271546024, + "tokens_seen": 2391998464 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 799627, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3175032138824463, + "objective/train/theoretical_loss": 3.4089188183747363, + "objective/train/tokens_used": 2412524000, + "theoretical_loss": 3.4089188183747363, + "tokens_seen": 2392064000 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022250752256770313, + "loss": 1.2993, + "theoretical_loss": 3.4089188183747363, + "tokens_seen": 2392064000 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022249147442326983, + "loss": 1.3432, + "theoretical_loss": 3.4089113654648155, + "tokens_seen": 2392129536 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022247542627883653, + "loss": 1.2474, + "theoretical_loss": 3.4089039128162453, + "tokens_seen": 2392195072 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022245937813440323, + "loss": 1.3012, + "theoretical_loss": 3.408896460429009, + "tokens_seen": 2392260608 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022244332998996993, + "loss": 1.2082, + "theoretical_loss": 3.4088890083030914, + "tokens_seen": 2392326144 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022242728184553663, + "loss": 1.2625, + "theoretical_loss": 3.408881556438475, + "tokens_seen": 2392391680 + }, + { + "epoch": 0.72, + "learning_rate": 0.00022241123370110333, + "loss": 1.2528, + "theoretical_loss": 3.408874104835144, + "tokens_seen": 2392457216 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022239518555667003, + "loss": 1.2425, + "theoretical_loss": 3.4088666534930816, + "tokens_seen": 2392522752 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022237913741223673, + "loss": 1.2028, + "theoretical_loss": 3.4088592024122724, + "tokens_seen": 2392588288 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022236308926780343, + "loss": 1.1418, + "theoretical_loss": 3.408851751592699, + "tokens_seen": 2392653824 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022234704112337013, + "loss": 1.1811, + "theoretical_loss": 3.408844301034346, + "tokens_seen": 2392719360 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022233099297893684, + "loss": 1.2287, + "theoretical_loss": 3.4088368507371967, + "tokens_seen": 2392784896 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022231494483450354, + "loss": 1.1706, + "theoretical_loss": 3.4088294007012347, + "tokens_seen": 2392850432 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022229889669007024, + "loss": 1.3106, + "theoretical_loss": 3.4088219509264435, + "tokens_seen": 2392915968 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002222828485456369, + "loss": 1.3347, + "theoretical_loss": 3.4088145014128077, + "tokens_seen": 2392981504 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002222668004012036, + "loss": 1.2969, + "theoretical_loss": 3.4088070521603098, + "tokens_seen": 2393047040 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002222507522567703, + "loss": 1.2002, + "theoretical_loss": 3.4087996031689345, + "tokens_seen": 2393112576 + }, + { + "epoch": 0.73, + "learning_rate": 0.000222234704112337, + "loss": 1.1961, + "theoretical_loss": 3.408792154438665, + "tokens_seen": 2393178112 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002222186559679037, + "loss": 1.0998, + "theoretical_loss": 3.408784705969485, + "tokens_seen": 2393243648 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002222026078234704, + "loss": 1.1879, + "theoretical_loss": 3.408777257761378, + "tokens_seen": 2393309184 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022218655967903711, + "loss": 1.2168, + "theoretical_loss": 3.408769809814328, + "tokens_seen": 2393374720 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022217051153460381, + "loss": 1.2903, + "theoretical_loss": 3.408762362128319, + "tokens_seen": 2393440256 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022215446339017052, + "loss": 1.185, + "theoretical_loss": 3.4087549147033345, + "tokens_seen": 2393505792 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022213841524573722, + "loss": 1.2352, + "theoretical_loss": 3.4087474675393574, + "tokens_seen": 2393571328 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022212236710130392, + "loss": 1.2981, + "theoretical_loss": 3.4087400206363725, + "tokens_seen": 2393636864 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022210631895687062, + "loss": 1.1953, + "theoretical_loss": 3.4087325739943632, + "tokens_seen": 2393702400 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022209027081243732, + "loss": 1.2426, + "theoretical_loss": 3.408725127613313, + "tokens_seen": 2393767936 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022207422266800402, + "loss": 1.2118, + "theoretical_loss": 3.4087176814932056, + "tokens_seen": 2393833472 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022205817452357072, + "loss": 1.3016, + "theoretical_loss": 3.408710235634025, + "tokens_seen": 2393899008 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022204212637913742, + "loss": 1.2099, + "theoretical_loss": 3.4087027900357545, + "tokens_seen": 2393964544 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022202607823470412, + "loss": 1.2484, + "theoretical_loss": 3.408695344698378, + "tokens_seen": 2394030080 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022201003009027082, + "loss": 1.2916, + "theoretical_loss": 3.4086878996218797, + "tokens_seen": 2394095616 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022199398194583752, + "loss": 1.2989, + "theoretical_loss": 3.408680454806243, + "tokens_seen": 2394161152 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022197793380140422, + "loss": 1.1716, + "theoretical_loss": 3.408673010251451, + "tokens_seen": 2394226688 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022196188565697092, + "loss": 1.179, + "theoretical_loss": 3.408665565957488, + "tokens_seen": 2394292224 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022194583751253762, + "loss": 1.2377, + "theoretical_loss": 3.4086581219243377, + "tokens_seen": 2394357760 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022192978936810432, + "loss": 1.187, + "theoretical_loss": 3.4086506781519836, + "tokens_seen": 2394423296 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022191374122367102, + "loss": 1.1971, + "theoretical_loss": 3.40864323464041, + "tokens_seen": 2394488832 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002218976930792377, + "loss": 1.1876, + "theoretical_loss": 3.4086357913896, + "tokens_seen": 2394554368 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002218816449348044, + "loss": 1.2174, + "theoretical_loss": 3.408628348399538, + "tokens_seen": 2394619904 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002218655967903711, + "loss": 1.2872, + "theoretical_loss": 3.4086209056702064, + "tokens_seen": 2394685440 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002218495486459378, + "loss": 1.2512, + "theoretical_loss": 3.4086134632015903, + "tokens_seen": 2394750976 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002218335005015045, + "loss": 1.2803, + "theoretical_loss": 3.4086060209936733, + "tokens_seen": 2394816512 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002218174523570712, + "loss": 1.2183, + "theoretical_loss": 3.4085985790464384, + "tokens_seen": 2394882048 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002218014042126379, + "loss": 1.174, + "theoretical_loss": 3.4085911373598696, + "tokens_seen": 2394947584 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002217853560682046, + "loss": 1.1897, + "theoretical_loss": 3.4085836959339506, + "tokens_seen": 2395013120 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002217693079237713, + "loss": 1.1218, + "theoretical_loss": 3.4085762547686658, + "tokens_seen": 2395078656 + }, + { + "epoch": 0.73, + "learning_rate": 0.000221753259779338, + "loss": 1.2278, + "theoretical_loss": 3.4085688138639982, + "tokens_seen": 2395144192 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002217372116349047, + "loss": 1.2749, + "theoretical_loss": 3.408561373219932, + "tokens_seen": 2395209728 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002217211634904714, + "loss": 1.2569, + "theoretical_loss": 3.4085539328364503, + "tokens_seen": 2395275264 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 800556, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0488104820251465, + "objective/train/theoretical_loss": 3.4085464927135374, + "objective/train/tokens_used": 2415800800, + "theoretical_loss": 3.4085464927135374, + "tokens_seen": 2395340800 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002217051153460381, + "loss": 1.0931, + "theoretical_loss": 3.4085464927135374, + "tokens_seen": 2395340800 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002216890672016048, + "loss": 1.2178, + "theoretical_loss": 3.408539052851177, + "tokens_seen": 2395406336 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002216730190571715, + "loss": 1.1604, + "theoretical_loss": 3.408531613249353, + "tokens_seen": 2395471872 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002216569709127382, + "loss": 1.2482, + "theoretical_loss": 3.4085241739080487, + "tokens_seen": 2395537408 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002216409227683049, + "loss": 1.2212, + "theoretical_loss": 3.4085167348272476, + "tokens_seen": 2395602944 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002216248746238716, + "loss": 1.0609, + "theoretical_loss": 3.4085092960069345, + "tokens_seen": 2395668480 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022160882647943834, + "loss": 1.2801, + "theoretical_loss": 3.408501857447092, + "tokens_seen": 2395734016 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022159277833500504, + "loss": 1.3119, + "theoretical_loss": 3.408494419147705, + "tokens_seen": 2395799552 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022157673019057174, + "loss": 1.2373, + "theoretical_loss": 3.408486981108757, + "tokens_seen": 2395865088 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022156068204613844, + "loss": 1.1034, + "theoretical_loss": 3.4084795433302304, + "tokens_seen": 2395930624 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022154463390170514, + "loss": 1.2031, + "theoretical_loss": 3.408472105812111, + "tokens_seen": 2395996160 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022152858575727184, + "loss": 1.2993, + "theoretical_loss": 3.4084646685543807, + "tokens_seen": 2396061696 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022151253761283854, + "loss": 1.3134, + "theoretical_loss": 3.4084572315570245, + "tokens_seen": 2396127232 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022149648946840524, + "loss": 1.2956, + "theoretical_loss": 3.408449794820026, + "tokens_seen": 2396192768 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022148044132397194, + "loss": 1.315, + "theoretical_loss": 3.4084423583433683, + "tokens_seen": 2396258304 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022146439317953864, + "loss": 1.2522, + "theoretical_loss": 3.408434922127036, + "tokens_seen": 2396323840 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022144834503510534, + "loss": 1.2484, + "theoretical_loss": 3.408427486171012, + "tokens_seen": 2396389376 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022143229689067204, + "loss": 1.2024, + "theoretical_loss": 3.408420050475281, + "tokens_seen": 2396454912 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022141624874623874, + "loss": 1.1701, + "theoretical_loss": 3.4084126150398255, + "tokens_seen": 2396520448 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022140020060180545, + "loss": 1.2244, + "theoretical_loss": 3.408405179864631, + "tokens_seen": 2396585984 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022138415245737215, + "loss": 1.2387, + "theoretical_loss": 3.40839774494968, + "tokens_seen": 2396651520 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022136810431293885, + "loss": 1.1914, + "theoretical_loss": 3.4083903102949566, + "tokens_seen": 2396717056 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022135205616850555, + "loss": 1.2835, + "theoretical_loss": 3.408382875900445, + "tokens_seen": 2396782592 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022133600802407225, + "loss": 1.2794, + "theoretical_loss": 3.408375441766128, + "tokens_seen": 2396848128 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022131995987963895, + "loss": 1.1989, + "theoretical_loss": 3.4083680078919905, + "tokens_seen": 2396913664 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022130391173520565, + "loss": 1.211, + "theoretical_loss": 3.4083605742780154, + "tokens_seen": 2396979200 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022128786359077235, + "loss": 1.2108, + "theoretical_loss": 3.408353140924187, + "tokens_seen": 2397044736 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022127181544633905, + "loss": 1.2058, + "theoretical_loss": 3.4083457078304886, + "tokens_seen": 2397110272 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022125576730190572, + "loss": 1.3112, + "theoretical_loss": 3.408338274996905, + "tokens_seen": 2397175808 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022123971915747242, + "loss": 1.2715, + "theoretical_loss": 3.4083308424234184, + "tokens_seen": 2397241344 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022122367101303913, + "loss": 1.1728, + "theoretical_loss": 3.408323410110014, + "tokens_seen": 2397306880 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022120762286860583, + "loss": 1.2145, + "theoretical_loss": 3.4083159780566747, + "tokens_seen": 2397372416 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022119157472417253, + "loss": 1.2114, + "theoretical_loss": 3.408308546263385, + "tokens_seen": 2397437952 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022117552657973923, + "loss": 1.2763, + "theoretical_loss": 3.4083011147301283, + "tokens_seen": 2397503488 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022115947843530593, + "loss": 1.2531, + "theoretical_loss": 3.4082936834568884, + "tokens_seen": 2397569024 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022114343029087263, + "loss": 1.2448, + "theoretical_loss": 3.4082862524436495, + "tokens_seen": 2397634560 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022112738214643933, + "loss": 1.2824, + "theoretical_loss": 3.4082788216903945, + "tokens_seen": 2397700096 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022111133400200603, + "loss": 1.3908, + "theoretical_loss": 3.4082713911971076, + "tokens_seen": 2397765632 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022109528585757273, + "loss": 1.2614, + "theoretical_loss": 3.4082639609637733, + "tokens_seen": 2397831168 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022107923771313943, + "loss": 1.2747, + "theoretical_loss": 3.408256530990375, + "tokens_seen": 2397896704 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022106318956870613, + "loss": 1.1815, + "theoretical_loss": 3.408249101276896, + "tokens_seen": 2397962240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022104714142427283, + "loss": 1.1686, + "theoretical_loss": 3.40824167182332, + "tokens_seen": 2398027776 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022103109327983953, + "loss": 1.2351, + "theoretical_loss": 3.4082342426296317, + "tokens_seen": 2398093312 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022101504513540623, + "loss": 1.1671, + "theoretical_loss": 3.4082268136958143, + "tokens_seen": 2398158848 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022099899699097293, + "loss": 1.245, + "theoretical_loss": 3.4082193850218516, + "tokens_seen": 2398224384 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022098294884653963, + "loss": 1.1604, + "theoretical_loss": 3.408211956607728, + "tokens_seen": 2398289920 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022096690070210634, + "loss": 1.2951, + "theoretical_loss": 3.408204528453427, + "tokens_seen": 2398355456 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022095085255767304, + "loss": 1.21, + "theoretical_loss": 3.4081971005589318, + "tokens_seen": 2398420992 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022093480441323974, + "loss": 1.2221, + "theoretical_loss": 3.4081896729242267, + "tokens_seen": 2398486528 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022091875626880644, + "loss": 1.305, + "theoretical_loss": 3.4081822455492956, + "tokens_seen": 2398552064 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 801784, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2902165651321411, + "objective/train/theoretical_loss": 3.4081748184341225, + "objective/train/tokens_used": 2419077600, + "theoretical_loss": 3.4081748184341225, + "tokens_seen": 2398617600 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022090270812437314, + "loss": 1.2427, + "theoretical_loss": 3.4081748184341225, + "tokens_seen": 2398617600 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022088665997993984, + "loss": 1.2642, + "theoretical_loss": 3.408167391578691, + "tokens_seen": 2398683136 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002208706118355065, + "loss": 1.3026, + "theoretical_loss": 3.4081599649829846, + "tokens_seen": 2398748672 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002208545636910732, + "loss": 1.2524, + "theoretical_loss": 3.4081525386469878, + "tokens_seen": 2398814208 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002208385155466399, + "loss": 1.2735, + "theoretical_loss": 3.4081451125706836, + "tokens_seen": 2398879744 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022082246740220661, + "loss": 1.2383, + "theoretical_loss": 3.4081376867540563, + "tokens_seen": 2398945280 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022080641925777331, + "loss": 1.2326, + "theoretical_loss": 3.4081302611970896, + "tokens_seen": 2399010816 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022079037111334002, + "loss": 1.2457, + "theoretical_loss": 3.4081228358997677, + "tokens_seen": 2399076352 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022077432296890672, + "loss": 1.1637, + "theoretical_loss": 3.408115410862074, + "tokens_seen": 2399141888 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022075827482447342, + "loss": 1.2047, + "theoretical_loss": 3.4081079860839925, + "tokens_seen": 2399207424 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022074222668004012, + "loss": 1.227, + "theoretical_loss": 3.408100561565507, + "tokens_seen": 2399272960 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022072617853560682, + "loss": 1.28, + "theoretical_loss": 3.4080931373066012, + "tokens_seen": 2399338496 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022071013039117352, + "loss": 1.1592, + "theoretical_loss": 3.4080857133072593, + "tokens_seen": 2399404032 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022069408224674022, + "loss": 1.3136, + "theoretical_loss": 3.408078289567465, + "tokens_seen": 2399469568 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022067803410230692, + "loss": 1.2349, + "theoretical_loss": 3.4080708660872014, + "tokens_seen": 2399535104 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022066198595787362, + "loss": 1.2151, + "theoretical_loss": 3.4080634428664536, + "tokens_seen": 2399600640 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022064593781344032, + "loss": 1.227, + "theoretical_loss": 3.408056019905205, + "tokens_seen": 2399666176 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022062988966900702, + "loss": 1.2477, + "theoretical_loss": 3.4080485972034387, + "tokens_seen": 2399731712 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022061384152457372, + "loss": 1.2814, + "theoretical_loss": 3.4080411747611388, + "tokens_seen": 2399797248 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022059779338014042, + "loss": 1.2047, + "theoretical_loss": 3.40803375257829, + "tokens_seen": 2399862784 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022058174523570712, + "loss": 1.3245, + "theoretical_loss": 3.4080263306548755, + "tokens_seen": 2399928320 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022056569709127382, + "loss": 1.2007, + "theoretical_loss": 3.4080189089908792, + "tokens_seen": 2399993856 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022054964894684052, + "loss": 1.3298, + "theoretical_loss": 3.4080114875862852, + "tokens_seen": 2400059392 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022053360080240723, + "loss": 1.344, + "theoretical_loss": 3.408004066441077, + "tokens_seen": 2400124928 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022051755265797393, + "loss": 1.2251, + "theoretical_loss": 3.4079966455552384, + "tokens_seen": 2400190464 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022050150451354063, + "loss": 1.3187, + "theoretical_loss": 3.4079892249287536, + "tokens_seen": 2400256000 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002204854563691073, + "loss": 1.2017, + "theoretical_loss": 3.407981804561606, + "tokens_seen": 2400321536 + }, + { + "epoch": 0.73, + "learning_rate": 0.000220469408224674, + "loss": 1.3254, + "theoretical_loss": 3.40797438445378, + "tokens_seen": 2400387072 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002204533600802407, + "loss": 1.2585, + "theoretical_loss": 3.4079669646052597, + "tokens_seen": 2400452608 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002204373119358074, + "loss": 1.2464, + "theoretical_loss": 3.407959545016028, + "tokens_seen": 2400518144 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002204212637913741, + "loss": 1.2757, + "theoretical_loss": 3.4079521256860694, + "tokens_seen": 2400583680 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022040521564694086, + "loss": 1.1847, + "theoretical_loss": 3.407944706615367, + "tokens_seen": 2400649216 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022038916750250756, + "loss": 1.1446, + "theoretical_loss": 3.407937287803906, + "tokens_seen": 2400714752 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022037311935807426, + "loss": 1.2517, + "theoretical_loss": 3.4079298692516695, + "tokens_seen": 2400780288 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022035707121364096, + "loss": 1.2751, + "theoretical_loss": 3.407922450958641, + "tokens_seen": 2400845824 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022034102306920766, + "loss": 1.0793, + "theoretical_loss": 3.407915032924805, + "tokens_seen": 2400911360 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022032497492477436, + "loss": 1.1774, + "theoretical_loss": 3.407907615150145, + "tokens_seen": 2400976896 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022030892678034106, + "loss": 1.3753, + "theoretical_loss": 3.407900197634645, + "tokens_seen": 2401042432 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022029287863590776, + "loss": 1.2475, + "theoretical_loss": 3.4078927803782886, + "tokens_seen": 2401107968 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022027683049147446, + "loss": 1.2529, + "theoretical_loss": 3.4078853633810606, + "tokens_seen": 2401173504 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022026078234704116, + "loss": 1.209, + "theoretical_loss": 3.4078779466429436, + "tokens_seen": 2401239040 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022024473420260784, + "loss": 1.2558, + "theoretical_loss": 3.4078705301639225, + "tokens_seen": 2401304576 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022022868605817454, + "loss": 1.2546, + "theoretical_loss": 3.4078631139439803, + "tokens_seen": 2401370112 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022021263791374124, + "loss": 1.3184, + "theoretical_loss": 3.407855697983102, + "tokens_seen": 2401435648 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022019658976930794, + "loss": 1.1715, + "theoretical_loss": 3.407848282281271, + "tokens_seen": 2401501184 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022018054162487464, + "loss": 1.0906, + "theoretical_loss": 3.40784086683847, + "tokens_seen": 2401566720 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022016449348044134, + "loss": 1.2322, + "theoretical_loss": 3.4078334516546844, + "tokens_seen": 2401632256 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022014844533600804, + "loss": 1.3676, + "theoretical_loss": 3.407826036729898, + "tokens_seen": 2401697792 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022013239719157474, + "loss": 1.193, + "theoretical_loss": 3.4078186220640934, + "tokens_seen": 2401763328 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022011634904714144, + "loss": 1.2047, + "theoretical_loss": 3.407811207657256, + "tokens_seen": 2401828864 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 802719, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1629681587219238, + "objective/train/theoretical_loss": 3.407803793509369, + "objective/train/tokens_used": 2422354400, + "theoretical_loss": 3.407803793509369, + "tokens_seen": 2401894400 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022010030090270814, + "loss": 1.2282, + "theoretical_loss": 3.407803793509369, + "tokens_seen": 2401894400 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022008425275827484, + "loss": 1.2457, + "theoretical_loss": 3.4077963796204163, + "tokens_seen": 2401959936 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022006820461384154, + "loss": 1.2122, + "theoretical_loss": 3.4077889659903815, + "tokens_seen": 2402025472 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022005215646940824, + "loss": 1.2799, + "theoretical_loss": 3.4077815526192494, + "tokens_seen": 2402091008 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022003610832497494, + "loss": 1.2321, + "theoretical_loss": 3.407774139507003, + "tokens_seen": 2402156544 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022002006018054165, + "loss": 1.2217, + "theoretical_loss": 3.4077667266536267, + "tokens_seen": 2402222080 + }, + { + "epoch": 0.73, + "learning_rate": 0.00022000401203610835, + "loss": 1.1387, + "theoretical_loss": 3.4077593140591036, + "tokens_seen": 2402287616 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021998796389167505, + "loss": 1.3249, + "theoretical_loss": 3.407751901723419, + "tokens_seen": 2402353152 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021997191574724175, + "loss": 1.2354, + "theoretical_loss": 3.4077444896465554, + "tokens_seen": 2402418688 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021995586760280845, + "loss": 1.3425, + "theoretical_loss": 3.407737077828498, + "tokens_seen": 2402484224 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021993981945837515, + "loss": 1.216, + "theoretical_loss": 3.4077296662692294, + "tokens_seen": 2402549760 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021992377131394185, + "loss": 1.2117, + "theoretical_loss": 3.4077222549687343, + "tokens_seen": 2402615296 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021990772316950855, + "loss": 1.0909, + "theoretical_loss": 3.4077148439269966, + "tokens_seen": 2402680832 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021989167502507525, + "loss": 1.1767, + "theoretical_loss": 3.407707433144, + "tokens_seen": 2402746368 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021987562688064195, + "loss": 1.2863, + "theoretical_loss": 3.4077000226197285, + "tokens_seen": 2402811904 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021985957873620862, + "loss": 1.3756, + "theoretical_loss": 3.4076926123541655, + "tokens_seen": 2402877440 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021984353059177533, + "loss": 1.242, + "theoretical_loss": 3.407685202347296, + "tokens_seen": 2402942976 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021982748244734203, + "loss": 1.3055, + "theoretical_loss": 3.407677792599103, + "tokens_seen": 2403008512 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021981143430290873, + "loss": 1.2478, + "theoretical_loss": 3.4076703831095707, + "tokens_seen": 2403074048 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021979538615847543, + "loss": 1.2237, + "theoretical_loss": 3.4076629738786828, + "tokens_seen": 2403139584 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021977933801404213, + "loss": 1.2059, + "theoretical_loss": 3.407655564906424, + "tokens_seen": 2403205120 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021976328986960883, + "loss": 1.2108, + "theoretical_loss": 3.4076481561927774, + "tokens_seen": 2403270656 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021974724172517553, + "loss": 1.1795, + "theoretical_loss": 3.407640747737727, + "tokens_seen": 2403336192 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021973119358074223, + "loss": 1.1827, + "theoretical_loss": 3.407633339541257, + "tokens_seen": 2403401728 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021971514543630893, + "loss": 1.1742, + "theoretical_loss": 3.4076259316033513, + "tokens_seen": 2403467264 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021969909729187563, + "loss": 1.3358, + "theoretical_loss": 3.407618523923994, + "tokens_seen": 2403532800 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021968304914744233, + "loss": 1.2418, + "theoretical_loss": 3.4076111165031686, + "tokens_seen": 2403598336 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021966700100300903, + "loss": 1.2747, + "theoretical_loss": 3.407603709340859, + "tokens_seen": 2403663872 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021965095285857573, + "loss": 1.2629, + "theoretical_loss": 3.4075963024370495, + "tokens_seen": 2403729408 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021963490471414243, + "loss": 1.1858, + "theoretical_loss": 3.407588895791724, + "tokens_seen": 2403794944 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021961885656970913, + "loss": 1.3022, + "theoretical_loss": 3.407581489404866, + "tokens_seen": 2403860480 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021960280842527583, + "loss": 1.2555, + "theoretical_loss": 3.40757408327646, + "tokens_seen": 2403926016 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021958676028084254, + "loss": 1.1274, + "theoretical_loss": 3.407566677406489, + "tokens_seen": 2403991552 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021957071213640924, + "loss": 1.3347, + "theoretical_loss": 3.4075592717949386, + "tokens_seen": 2404057088 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021955466399197594, + "loss": 1.281, + "theoretical_loss": 3.4075518664417914, + "tokens_seen": 2404122624 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021953861584754264, + "loss": 1.1966, + "theoretical_loss": 3.4075444613470314, + "tokens_seen": 2404188160 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021952256770310934, + "loss": 1.3001, + "theoretical_loss": 3.4075370565106433, + "tokens_seen": 2404253696 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021950651955867604, + "loss": 1.211, + "theoretical_loss": 3.4075296519326104, + "tokens_seen": 2404319232 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021949047141424274, + "loss": 1.3744, + "theoretical_loss": 3.4075222476129166, + "tokens_seen": 2404384768 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002194744232698094, + "loss": 1.1826, + "theoretical_loss": 3.4075148435515463, + "tokens_seen": 2404450304 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021945837512537611, + "loss": 1.3, + "theoretical_loss": 3.407507439748483, + "tokens_seen": 2404515840 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021944232698094281, + "loss": 1.292, + "theoretical_loss": 3.407500036203711, + "tokens_seen": 2404581376 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021942627883650951, + "loss": 1.3132, + "theoretical_loss": 3.407492632917214, + "tokens_seen": 2404646912 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021941023069207622, + "loss": 1.2438, + "theoretical_loss": 3.407485229888976, + "tokens_seen": 2404712448 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021939418254764292, + "loss": 1.2597, + "theoretical_loss": 3.4074778271189814, + "tokens_seen": 2404777984 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021937813440320962, + "loss": 1.3085, + "theoretical_loss": 3.4074704246072134, + "tokens_seen": 2404843520 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021936208625877632, + "loss": 1.2801, + "theoretical_loss": 3.4074630223536566, + "tokens_seen": 2404909056 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021934603811434302, + "loss": 1.2171, + "theoretical_loss": 3.407455620358294, + "tokens_seen": 2404974592 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021932998996990972, + "loss": 1.1716, + "theoretical_loss": 3.407448218621111, + "tokens_seen": 2405040128 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021931394182547642, + "loss": 1.3297, + "theoretical_loss": 3.4074408171420907, + "tokens_seen": 2405105664 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 804018, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.286693811416626, + "objective/train/theoretical_loss": 3.407433415921217, + "objective/train/tokens_used": 2425631200, + "theoretical_loss": 3.407433415921217, + "tokens_seen": 2405171200 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021929789368104312, + "loss": 1.2416, + "theoretical_loss": 3.407433415921217, + "tokens_seen": 2405171200 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021928184553660982, + "loss": 1.2942, + "theoretical_loss": 3.407426014958474, + "tokens_seen": 2405236736 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021926579739217652, + "loss": 1.1975, + "theoretical_loss": 3.407418614253846, + "tokens_seen": 2405302272 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021924974924774322, + "loss": 1.2337, + "theoretical_loss": 3.407411213807316, + "tokens_seen": 2405367808 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021923370110330992, + "loss": 1.2871, + "theoretical_loss": 3.407403813618869, + "tokens_seen": 2405433344 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021921765295887665, + "loss": 1.1959, + "theoretical_loss": 3.4073964136884887, + "tokens_seen": 2405498880 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021920160481444335, + "loss": 1.3029, + "theoretical_loss": 3.407389014016159, + "tokens_seen": 2405564416 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021918555667001005, + "loss": 1.2495, + "theoretical_loss": 3.4073816146018636, + "tokens_seen": 2405629952 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021916950852557675, + "loss": 1.2344, + "theoretical_loss": 3.4073742154455866, + "tokens_seen": 2405695488 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021915346038114345, + "loss": 1.3191, + "theoretical_loss": 3.407366816547312, + "tokens_seen": 2405761024 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021913741223671015, + "loss": 1.2234, + "theoretical_loss": 3.4073594179070246, + "tokens_seen": 2405826560 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021912136409227685, + "loss": 1.2481, + "theoretical_loss": 3.407352019524707, + "tokens_seen": 2405892096 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021910531594784355, + "loss": 1.2132, + "theoretical_loss": 3.4073446214003438, + "tokens_seen": 2405957632 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021908926780341026, + "loss": 1.226, + "theoretical_loss": 3.4073372235339194, + "tokens_seen": 2406023168 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021907321965897696, + "loss": 1.3179, + "theoretical_loss": 3.4073298259254168, + "tokens_seen": 2406088704 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021905717151454366, + "loss": 1.2909, + "theoretical_loss": 3.407322428574821, + "tokens_seen": 2406154240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021904112337011036, + "loss": 1.2786, + "theoretical_loss": 3.4073150314821152, + "tokens_seen": 2406219776 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021902507522567706, + "loss": 1.2361, + "theoretical_loss": 3.4073076346472844, + "tokens_seen": 2406285312 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021900902708124376, + "loss": 1.216, + "theoretical_loss": 3.4073002380703112, + "tokens_seen": 2406350848 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021899297893681046, + "loss": 1.1832, + "theoretical_loss": 3.4072928417511807, + "tokens_seen": 2406416384 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021897693079237716, + "loss": 1.2809, + "theoretical_loss": 3.407285445689876, + "tokens_seen": 2406481920 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021896088264794386, + "loss": 1.2102, + "theoretical_loss": 3.407278049886382, + "tokens_seen": 2406547456 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021894483450351056, + "loss": 1.2919, + "theoretical_loss": 3.407270654340682, + "tokens_seen": 2406612992 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021892878635907726, + "loss": 1.2262, + "theoretical_loss": 3.4072632590527605, + "tokens_seen": 2406678528 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021891273821464396, + "loss": 1.2917, + "theoretical_loss": 3.407255864022601, + "tokens_seen": 2406744064 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021889669007021066, + "loss": 1.2023, + "theoretical_loss": 3.407248469250188, + "tokens_seen": 2406809600 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021888064192577736, + "loss": 1.1954, + "theoretical_loss": 3.4072410747355057, + "tokens_seen": 2406875136 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021886459378134406, + "loss": 1.2671, + "theoretical_loss": 3.4072336804785373, + "tokens_seen": 2406940672 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021884854563691074, + "loss": 1.2969, + "theoretical_loss": 3.4072262864792666, + "tokens_seen": 2407006208 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021883249749247744, + "loss": 1.3416, + "theoretical_loss": 3.4072188927376788, + "tokens_seen": 2407071744 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021881644934804414, + "loss": 1.3323, + "theoretical_loss": 3.4072114992537568, + "tokens_seen": 2407137280 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021880040120361084, + "loss": 1.2284, + "theoretical_loss": 3.4072041060274856, + "tokens_seen": 2407202816 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021878435305917754, + "loss": 1.2746, + "theoretical_loss": 3.4071967130588483, + "tokens_seen": 2407268352 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021876830491474424, + "loss": 1.2558, + "theoretical_loss": 3.4071893203478294, + "tokens_seen": 2407333888 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021875225677031094, + "loss": 1.185, + "theoretical_loss": 3.407181927894413, + "tokens_seen": 2407399424 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021873620862587764, + "loss": 1.0756, + "theoretical_loss": 3.4071745356985823, + "tokens_seen": 2407464960 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021872016048144434, + "loss": 1.193, + "theoretical_loss": 3.4071671437603226, + "tokens_seen": 2407530496 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021870411233701104, + "loss": 1.1683, + "theoretical_loss": 3.407159752079617, + "tokens_seen": 2407596032 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021868806419257774, + "loss": 1.2234, + "theoretical_loss": 3.4071523606564496, + "tokens_seen": 2407661568 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021867201604814444, + "loss": 1.1158, + "theoretical_loss": 3.4071449694908047, + "tokens_seen": 2407727104 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021865596790371115, + "loss": 1.1532, + "theoretical_loss": 3.4071375785826663, + "tokens_seen": 2407792640 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021863991975927785, + "loss": 1.2889, + "theoretical_loss": 3.407130187932018, + "tokens_seen": 2407858176 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021862387161484455, + "loss": 1.2277, + "theoretical_loss": 3.4071227975388445, + "tokens_seen": 2407923712 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021860782347041125, + "loss": 1.1693, + "theoretical_loss": 3.407115407403129, + "tokens_seen": 2407989248 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021859177532597795, + "loss": 1.2897, + "theoretical_loss": 3.4071080175248563, + "tokens_seen": 2408054784 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021857572718154465, + "loss": 1.2243, + "theoretical_loss": 3.4071006279040104, + "tokens_seen": 2408120320 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021855967903711135, + "loss": 1.1861, + "theoretical_loss": 3.4070932385405746, + "tokens_seen": 2408185856 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021854363089267805, + "loss": 1.2689, + "theoretical_loss": 3.4070858494345333, + "tokens_seen": 2408251392 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021852758274824475, + "loss": 1.2051, + "theoretical_loss": 3.4070784605858706, + "tokens_seen": 2408316928 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021851153460381145, + "loss": 1.1992, + "theoretical_loss": 3.407071071994571, + "tokens_seen": 2408382464 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 804944, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1068869829177856, + "objective/train/theoretical_loss": 3.4070636836606174, + "objective/train/tokens_used": 2428908000, + "theoretical_loss": 3.4070636836606174, + "tokens_seen": 2408448000 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021849548645937815, + "loss": 1.1614, + "theoretical_loss": 3.4070636836606174, + "tokens_seen": 2408448000 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021847943831494485, + "loss": 1.3202, + "theoretical_loss": 3.407056295583995, + "tokens_seen": 2408513536 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021846339017051153, + "loss": 1.1991, + "theoretical_loss": 3.407048907764687, + "tokens_seen": 2408579072 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021844734202607823, + "loss": 1.2757, + "theoretical_loss": 3.407041520202678, + "tokens_seen": 2408644608 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021843129388164493, + "loss": 1.2079, + "theoretical_loss": 3.4070341328979517, + "tokens_seen": 2408710144 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021841524573721163, + "loss": 1.1319, + "theoretical_loss": 3.407026745850492, + "tokens_seen": 2408775680 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021839919759277833, + "loss": 1.1695, + "theoretical_loss": 3.4070193590602833, + "tokens_seen": 2408841216 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021838314944834503, + "loss": 1.2489, + "theoretical_loss": 3.40701197252731, + "tokens_seen": 2408906752 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021836710130391173, + "loss": 1.3349, + "theoretical_loss": 3.407004586251555, + "tokens_seen": 2408972288 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021835105315947843, + "loss": 1.4175, + "theoretical_loss": 3.406997200233003, + "tokens_seen": 2409037824 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021833500501504513, + "loss": 1.2911, + "theoretical_loss": 3.406989814471639, + "tokens_seen": 2409103360 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021831895687061183, + "loss": 1.1659, + "theoretical_loss": 3.4069824289674453, + "tokens_seen": 2409168896 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021830290872617853, + "loss": 1.2157, + "theoretical_loss": 3.406975043720407, + "tokens_seen": 2409234432 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021828686058174523, + "loss": 1.1952, + "theoretical_loss": 3.4069676587305078, + "tokens_seen": 2409299968 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021827081243731193, + "loss": 1.1623, + "theoretical_loss": 3.406960273997732, + "tokens_seen": 2409365504 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021825476429287863, + "loss": 1.143, + "theoretical_loss": 3.406952889522063, + "tokens_seen": 2409431040 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021823871614844533, + "loss": 1.3046, + "theoretical_loss": 3.406945505303486, + "tokens_seen": 2409496576 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021822266800401204, + "loss": 1.2892, + "theoretical_loss": 3.4069381213419843, + "tokens_seen": 2409562112 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021820661985957874, + "loss": 1.2039, + "theoretical_loss": 3.406930737637542, + "tokens_seen": 2409627648 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021819057171514544, + "loss": 1.2999, + "theoretical_loss": 3.4069233541901434, + "tokens_seen": 2409693184 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021817452357071214, + "loss": 1.2166, + "theoretical_loss": 3.406915970999772, + "tokens_seen": 2409758720 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021815847542627884, + "loss": 1.1984, + "theoretical_loss": 3.4069085880664125, + "tokens_seen": 2409824256 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021814242728184554, + "loss": 1.1704, + "theoretical_loss": 3.406901205390049, + "tokens_seen": 2409889792 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021812637913741224, + "loss": 1.2759, + "theoretical_loss": 3.406893822970665, + "tokens_seen": 2409955328 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021811033099297894, + "loss": 1.1901, + "theoretical_loss": 3.406886440808245, + "tokens_seen": 2410020864 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021809428284854564, + "loss": 1.1847, + "theoretical_loss": 3.406879058902773, + "tokens_seen": 2410086400 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021807823470411231, + "loss": 1.2611, + "theoretical_loss": 3.406871677254233, + "tokens_seen": 2410151936 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021806218655967901, + "loss": 1.2936, + "theoretical_loss": 3.4068642958626087, + "tokens_seen": 2410217472 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021804613841524572, + "loss": 1.3191, + "theoretical_loss": 3.406856914727885, + "tokens_seen": 2410283008 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021803009027081242, + "loss": 1.1937, + "theoretical_loss": 3.4068495338500453, + "tokens_seen": 2410348544 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021801404212637917, + "loss": 1.3084, + "theoretical_loss": 3.406842153229074, + "tokens_seen": 2410414080 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021799799398194587, + "loss": 1.1911, + "theoretical_loss": 3.406834772864955, + "tokens_seen": 2410479616 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021798194583751257, + "loss": 1.2741, + "theoretical_loss": 3.4068273927576724, + "tokens_seen": 2410545152 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021796589769307927, + "loss": 1.3018, + "theoretical_loss": 3.40682001290721, + "tokens_seen": 2410610688 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021794984954864597, + "loss": 1.2263, + "theoretical_loss": 3.406812633313553, + "tokens_seen": 2410676224 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021793380140421267, + "loss": 1.2147, + "theoretical_loss": 3.4068052539766844, + "tokens_seen": 2410741760 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021791775325977937, + "loss": 1.1811, + "theoretical_loss": 3.406797874896588, + "tokens_seen": 2410807296 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021790170511534608, + "loss": 1.198, + "theoretical_loss": 3.406790496073249, + "tokens_seen": 2410872832 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021788565697091278, + "loss": 1.2642, + "theoretical_loss": 3.4067831175066514, + "tokens_seen": 2410938368 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021786960882647948, + "loss": 1.2298, + "theoretical_loss": 3.406775739196778, + "tokens_seen": 2411003904 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021785356068204618, + "loss": 1.1643, + "theoretical_loss": 3.406768361143614, + "tokens_seen": 2411069440 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021783751253761288, + "loss": 1.2284, + "theoretical_loss": 3.406760983347144, + "tokens_seen": 2411134976 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021782146439317955, + "loss": 1.2323, + "theoretical_loss": 3.4067536058073506, + "tokens_seen": 2411200512 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021780541624874625, + "loss": 1.1574, + "theoretical_loss": 3.4067462285242187, + "tokens_seen": 2411266048 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021778936810431295, + "loss": 1.2621, + "theoretical_loss": 3.406738851497732, + "tokens_seen": 2411331584 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021777331995987965, + "loss": 1.291, + "theoretical_loss": 3.4067314747278754, + "tokens_seen": 2411397120 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021775727181544635, + "loss": 1.2088, + "theoretical_loss": 3.406724098214632, + "tokens_seen": 2411462656 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021774122367101305, + "loss": 1.2948, + "theoretical_loss": 3.406716721957987, + "tokens_seen": 2411528192 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021772517552657976, + "loss": 1.1687, + "theoretical_loss": 3.406709345957924, + "tokens_seen": 2411593728 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021770912738214646, + "loss": 1.1936, + "theoretical_loss": 3.406701970214426, + "tokens_seen": 2411659264 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 806106, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.20001220703125, + "objective/train/theoretical_loss": 3.4066945947274787, + "objective/train/tokens_used": 2432184800, + "theoretical_loss": 3.4066945947274787, + "tokens_seen": 2411724800 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021769307923771316, + "loss": 1.2759, + "theoretical_loss": 3.4066945947274787, + "tokens_seen": 2411724800 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021767703109327986, + "loss": 1.3005, + "theoretical_loss": 3.4066872194970657, + "tokens_seen": 2411790336 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021766098294884656, + "loss": 1.3568, + "theoretical_loss": 3.406679844523171, + "tokens_seen": 2411855872 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021764493480441326, + "loss": 1.2956, + "theoretical_loss": 3.4066724698057786, + "tokens_seen": 2411921408 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021762888665997996, + "loss": 1.3647, + "theoretical_loss": 3.4066650953448727, + "tokens_seen": 2411986944 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021761283851554666, + "loss": 1.2306, + "theoretical_loss": 3.4066577211404376, + "tokens_seen": 2412052480 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021759679037111336, + "loss": 1.3112, + "theoretical_loss": 3.4066503471924574, + "tokens_seen": 2412118016 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021758074222668006, + "loss": 1.2315, + "theoretical_loss": 3.406642973500916, + "tokens_seen": 2412183552 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021756469408224676, + "loss": 1.2179, + "theoretical_loss": 3.4066356000657976, + "tokens_seen": 2412249088 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021754864593781346, + "loss": 1.2854, + "theoretical_loss": 3.406628226887086, + "tokens_seen": 2412314624 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021753259779338016, + "loss": 1.2285, + "theoretical_loss": 3.4066208539647658, + "tokens_seen": 2412380160 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021751654964894686, + "loss": 1.2477, + "theoretical_loss": 3.406613481298821, + "tokens_seen": 2412445696 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021750050150451356, + "loss": 1.2439, + "theoretical_loss": 3.406606108889236, + "tokens_seen": 2412511232 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021748445336008026, + "loss": 1.3353, + "theoretical_loss": 3.406598736735994, + "tokens_seen": 2412576768 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021746840521564697, + "loss": 1.1836, + "theoretical_loss": 3.40659136483908, + "tokens_seen": 2412642304 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021745235707121367, + "loss": 1.1719, + "theoretical_loss": 3.4065839931984776, + "tokens_seen": 2412707840 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021743630892678034, + "loss": 1.1542, + "theoretical_loss": 3.4065766218141715, + "tokens_seen": 2412773376 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021742026078234704, + "loss": 1.274, + "theoretical_loss": 3.4065692506861454, + "tokens_seen": 2412838912 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021740421263791374, + "loss": 1.196, + "theoretical_loss": 3.4065618798143835, + "tokens_seen": 2412904448 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021738816449348044, + "loss": 1.2472, + "theoretical_loss": 3.4065545091988696, + "tokens_seen": 2412969984 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021737211634904714, + "loss": 1.2604, + "theoretical_loss": 3.406547138839589, + "tokens_seen": 2413035520 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021735606820461384, + "loss": 1.2249, + "theoretical_loss": 3.406539768736524, + "tokens_seen": 2413101056 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021734002006018054, + "loss": 1.202, + "theoretical_loss": 3.4065323988896603, + "tokens_seen": 2413166592 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021732397191574724, + "loss": 1.297, + "theoretical_loss": 3.4065250292989817, + "tokens_seen": 2413232128 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021730792377131394, + "loss": 1.2883, + "theoretical_loss": 3.406517659964472, + "tokens_seen": 2413297664 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021729187562688065, + "loss": 1.1252, + "theoretical_loss": 3.406510290886115, + "tokens_seen": 2413363200 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021727582748244735, + "loss": 1.1489, + "theoretical_loss": 3.406502922063896, + "tokens_seen": 2413428736 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021725977933801405, + "loss": 1.1782, + "theoretical_loss": 3.4064955534977983, + "tokens_seen": 2413494272 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021724373119358075, + "loss": 1.2845, + "theoretical_loss": 3.406488185187806, + "tokens_seen": 2413559808 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021722768304914745, + "loss": 1.1681, + "theoretical_loss": 3.4064808171339034, + "tokens_seen": 2413625344 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021721163490471415, + "loss": 1.1187, + "theoretical_loss": 3.4064734493360747, + "tokens_seen": 2413690880 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021719558676028085, + "loss": 1.1811, + "theoretical_loss": 3.4064660817943038, + "tokens_seen": 2413756416 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021717953861584755, + "loss": 1.261, + "theoretical_loss": 3.4064587145085756, + "tokens_seen": 2413821952 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021716349047141425, + "loss": 1.2773, + "theoretical_loss": 3.4064513474788733, + "tokens_seen": 2413887488 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021714744232698095, + "loss": 1.1646, + "theoretical_loss": 3.406443980705182, + "tokens_seen": 2413953024 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021713139418254765, + "loss": 1.1667, + "theoretical_loss": 3.4064366141874847, + "tokens_seen": 2414018560 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021711534603811435, + "loss": 1.181, + "theoretical_loss": 3.4064292479257667, + "tokens_seen": 2414084096 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021709929789368105, + "loss": 1.361, + "theoretical_loss": 3.4064218819200116, + "tokens_seen": 2414149632 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021708324974924775, + "loss": 1.1868, + "theoretical_loss": 3.4064145161702033, + "tokens_seen": 2414215168 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021706720160481445, + "loss": 1.308, + "theoretical_loss": 3.4064071506763263, + "tokens_seen": 2414280704 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021705115346038113, + "loss": 1.1809, + "theoretical_loss": 3.406399785438365, + "tokens_seen": 2414346240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021703510531594783, + "loss": 1.2798, + "theoretical_loss": 3.406392420456303, + "tokens_seen": 2414411776 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021701905717151453, + "loss": 1.2482, + "theoretical_loss": 3.4063850557301247, + "tokens_seen": 2414477312 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021700300902708123, + "loss": 1.2254, + "theoretical_loss": 3.4063776912598147, + "tokens_seen": 2414542848 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021698696088264793, + "loss": 1.3024, + "theoretical_loss": 3.4063703270453565, + "tokens_seen": 2414608384 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021697091273821463, + "loss": 1.2442, + "theoretical_loss": 3.406362963086735, + "tokens_seen": 2414673920 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021695486459378133, + "loss": 1.3155, + "theoretical_loss": 3.406355599383933, + "tokens_seen": 2414739456 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021693881644934803, + "loss": 1.2043, + "theoretical_loss": 3.4063482359369366, + "tokens_seen": 2414804992 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021692276830491473, + "loss": 1.254, + "theoretical_loss": 3.4063408727457283, + "tokens_seen": 2414870528 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021690672016048143, + "loss": 1.1903, + "theoretical_loss": 3.406333509810293, + "tokens_seen": 2414936064 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 806981, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.205592393875122, + "objective/train/theoretical_loss": 3.406326147130615, + "objective/train/tokens_used": 2435461600, + "theoretical_loss": 3.406326147130615, + "tokens_seen": 2415001600 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021689067201604813, + "loss": 1.1965, + "theoretical_loss": 3.406326147130615, + "tokens_seen": 2415001600 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021687462387161483, + "loss": 1.1723, + "theoretical_loss": 3.4063187847066785, + "tokens_seen": 2415067136 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021685857572718154, + "loss": 1.1292, + "theoretical_loss": 3.406311422538467, + "tokens_seen": 2415132672 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021684252758274824, + "loss": 1.1958, + "theoretical_loss": 3.4063040606259656, + "tokens_seen": 2415198208 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021682647943831494, + "loss": 1.3287, + "theoretical_loss": 3.4062966989691574, + "tokens_seen": 2415263744 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021681043129388166, + "loss": 1.219, + "theoretical_loss": 3.406289337568028, + "tokens_seen": 2415329280 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021679438314944836, + "loss": 1.1809, + "theoretical_loss": 3.40628197642256, + "tokens_seen": 2415394816 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021677833500501507, + "loss": 1.1934, + "theoretical_loss": 3.406274615532739, + "tokens_seen": 2415460352 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021676228686058177, + "loss": 1.206, + "theoretical_loss": 3.406267254898548, + "tokens_seen": 2415525888 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021674623871614847, + "loss": 1.1547, + "theoretical_loss": 3.4062598945199722, + "tokens_seen": 2415591424 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021673019057171517, + "loss": 1.1022, + "theoretical_loss": 3.406252534396995, + "tokens_seen": 2415656960 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021671414242728187, + "loss": 1.1831, + "theoretical_loss": 3.406245174529601, + "tokens_seen": 2415722496 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021669809428284857, + "loss": 1.1534, + "theoretical_loss": 3.4062378149177746, + "tokens_seen": 2415788032 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021668204613841527, + "loss": 1.1985, + "theoretical_loss": 3.4062304555614995, + "tokens_seen": 2415853568 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021666599799398197, + "loss": 1.3183, + "theoretical_loss": 3.40622309646076, + "tokens_seen": 2415919104 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021664994984954867, + "loss": 1.1317, + "theoretical_loss": 3.4062157376155406, + "tokens_seen": 2415984640 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021663390170511537, + "loss": 1.2001, + "theoretical_loss": 3.406208379025825, + "tokens_seen": 2416050176 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021661785356068207, + "loss": 1.2048, + "theoretical_loss": 3.406201020691598, + "tokens_seen": 2416115712 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021660180541624877, + "loss": 1.1123, + "theoretical_loss": 3.4061936626128433, + "tokens_seen": 2416181248 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021658575727181547, + "loss": 1.2456, + "theoretical_loss": 3.406186304789545, + "tokens_seen": 2416246784 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021656970912738217, + "loss": 1.2175, + "theoretical_loss": 3.4061789472216883, + "tokens_seen": 2416312320 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021655366098294887, + "loss": 1.2157, + "theoretical_loss": 3.406171589909256, + "tokens_seen": 2416377856 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021653761283851557, + "loss": 1.2315, + "theoretical_loss": 3.4061642328522335, + "tokens_seen": 2416443392 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021652156469408228, + "loss": 1.1321, + "theoretical_loss": 3.4061568760506042, + "tokens_seen": 2416508928 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021650551654964898, + "loss": 1.1914, + "theoretical_loss": 3.406149519504353, + "tokens_seen": 2416574464 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021648946840521568, + "loss": 1.2061, + "theoretical_loss": 3.406142163213463, + "tokens_seen": 2416640000 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021647342026078238, + "loss": 1.1527, + "theoretical_loss": 3.40613480717792, + "tokens_seen": 2416705536 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021645737211634908, + "loss": 1.3139, + "theoretical_loss": 3.406127451397707, + "tokens_seen": 2416771072 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021644132397191578, + "loss": 1.174, + "theoretical_loss": 3.4061200958728084, + "tokens_seen": 2416836608 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021642527582748245, + "loss": 1.2709, + "theoretical_loss": 3.406112740603209, + "tokens_seen": 2416902144 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021640922768304915, + "loss": 1.2287, + "theoretical_loss": 3.406105385588892, + "tokens_seen": 2416967680 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021639317953861585, + "loss": 1.178, + "theoretical_loss": 3.4060980308298427, + "tokens_seen": 2417033216 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021637713139418255, + "loss": 1.2428, + "theoretical_loss": 3.406090676326045, + "tokens_seen": 2417098752 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021636108324974925, + "loss": 1.2643, + "theoretical_loss": 3.406083322077482, + "tokens_seen": 2417164288 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021634503510531596, + "loss": 1.2671, + "theoretical_loss": 3.40607596808414, + "tokens_seen": 2417229824 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021632898696088266, + "loss": 1.2439, + "theoretical_loss": 3.4060686143460015, + "tokens_seen": 2417295360 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021631293881644936, + "loss": 1.1798, + "theoretical_loss": 3.4060612608630514, + "tokens_seen": 2417360896 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021629689067201606, + "loss": 1.2148, + "theoretical_loss": 3.406053907635274, + "tokens_seen": 2417426432 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021628084252758276, + "loss": 1.1562, + "theoretical_loss": 3.4060465546626535, + "tokens_seen": 2417491968 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021626479438314946, + "loss": 1.1926, + "theoretical_loss": 3.4060392019451737, + "tokens_seen": 2417557504 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021624874623871616, + "loss": 1.2193, + "theoretical_loss": 3.4060318494828192, + "tokens_seen": 2417623040 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021623269809428286, + "loss": 1.2266, + "theoretical_loss": 3.406024497275574, + "tokens_seen": 2417688576 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021621664994984956, + "loss": 1.3415, + "theoretical_loss": 3.406017145323423, + "tokens_seen": 2417754112 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021620060180541626, + "loss": 1.2108, + "theoretical_loss": 3.4060097936263496, + "tokens_seen": 2417819648 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021618455366098296, + "loss": 1.3139, + "theoretical_loss": 3.4060024421843385, + "tokens_seen": 2417885184 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021616850551654966, + "loss": 1.1279, + "theoretical_loss": 3.4059950909973735, + "tokens_seen": 2417950720 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021615245737211636, + "loss": 1.2319, + "theoretical_loss": 3.4059877400654397, + "tokens_seen": 2418016256 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021613640922768306, + "loss": 1.2424, + "theoretical_loss": 3.4059803893885205, + "tokens_seen": 2418081792 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021612036108324976, + "loss": 1.1979, + "theoretical_loss": 3.4059730389666005, + "tokens_seen": 2418147328 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021610431293881646, + "loss": 1.3228, + "theoretical_loss": 3.4059656887996637, + "tokens_seen": 2418212864 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 808156, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3060191869735718, + "objective/train/theoretical_loss": 3.405958338887695, + "objective/train/tokens_used": 2438738400, + "theoretical_loss": 3.405958338887695, + "tokens_seen": 2418278400 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021608826479438317, + "loss": 1.2718, + "theoretical_loss": 3.405958338887695, + "tokens_seen": 2418278400 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021607221664994987, + "loss": 1.3015, + "theoretical_loss": 3.4059509892306776, + "tokens_seen": 2418343936 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021605616850551657, + "loss": 1.2481, + "theoretical_loss": 3.4059436398285965, + "tokens_seen": 2418409472 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021604012036108324, + "loss": 1.2096, + "theoretical_loss": 3.405936290681436, + "tokens_seen": 2418475008 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021602407221664994, + "loss": 1.1896, + "theoretical_loss": 3.40592894178918, + "tokens_seen": 2418540544 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021600802407221664, + "loss": 1.1642, + "theoretical_loss": 3.4059215931518128, + "tokens_seen": 2418606080 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021599197592778334, + "loss": 1.2329, + "theoretical_loss": 3.4059142447693187, + "tokens_seen": 2418671616 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021597592778335004, + "loss": 1.2242, + "theoretical_loss": 3.405906896641682, + "tokens_seen": 2418737152 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021595987963891674, + "loss": 1.3611, + "theoretical_loss": 3.405899548768887, + "tokens_seen": 2418802688 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021594383149448344, + "loss": 1.1732, + "theoretical_loss": 3.405892201150918, + "tokens_seen": 2418868224 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021592778335005014, + "loss": 1.1405, + "theoretical_loss": 3.405884853787759, + "tokens_seen": 2418933760 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021591173520561685, + "loss": 1.1953, + "theoretical_loss": 3.4058775066793943, + "tokens_seen": 2418999296 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021589568706118355, + "loss": 1.2064, + "theoretical_loss": 3.4058701598258083, + "tokens_seen": 2419064832 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021587963891675025, + "loss": 1.2331, + "theoretical_loss": 3.4058628132269857, + "tokens_seen": 2419130368 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021586359077231695, + "loss": 1.2219, + "theoretical_loss": 3.40585546688291, + "tokens_seen": 2419195904 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021584754262788365, + "loss": 1.1949, + "theoretical_loss": 3.4058481207935656, + "tokens_seen": 2419261440 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021583149448345035, + "loss": 1.2762, + "theoretical_loss": 3.405840774958937, + "tokens_seen": 2419326976 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021581544633901705, + "loss": 1.3346, + "theoretical_loss": 3.4058334293790082, + "tokens_seen": 2419392512 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021579939819458375, + "loss": 1.2321, + "theoretical_loss": 3.405826084053764, + "tokens_seen": 2419458048 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021578335005015045, + "loss": 1.1883, + "theoretical_loss": 3.4058187389831884, + "tokens_seen": 2419523584 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021576730190571715, + "loss": 1.2577, + "theoretical_loss": 3.4058113941672654, + "tokens_seen": 2419589120 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021575125376128385, + "loss": 1.1782, + "theoretical_loss": 3.4058040496059796, + "tokens_seen": 2419654656 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021573520561685055, + "loss": 1.0571, + "theoretical_loss": 3.4057967052993154, + "tokens_seen": 2419720192 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021571915747241725, + "loss": 1.1607, + "theoretical_loss": 3.4057893612472565, + "tokens_seen": 2419785728 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021570310932798395, + "loss": 1.3551, + "theoretical_loss": 3.4057820174497877, + "tokens_seen": 2419851264 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021568706118355065, + "loss": 1.2453, + "theoretical_loss": 3.405774673906893, + "tokens_seen": 2419916800 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021567101303911735, + "loss": 1.269, + "theoretical_loss": 3.4057673306185565, + "tokens_seen": 2419982336 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021565496489468403, + "loss": 1.2418, + "theoretical_loss": 3.405759987584763, + "tokens_seen": 2420047872 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021563891675025073, + "loss": 1.2023, + "theoretical_loss": 3.4057526448054967, + "tokens_seen": 2420113408 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021562286860581743, + "loss": 1.238, + "theoretical_loss": 3.405745302280742, + "tokens_seen": 2420178944 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021560682046138418, + "loss": 1.2424, + "theoretical_loss": 3.4057379600104825, + "tokens_seen": 2420244480 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021559077231695089, + "loss": 1.3595, + "theoretical_loss": 3.405730617994703, + "tokens_seen": 2420310016 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021557472417251759, + "loss": 1.1866, + "theoretical_loss": 3.4057232762333878, + "tokens_seen": 2420375552 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002155586760280843, + "loss": 1.1627, + "theoretical_loss": 3.405715934726521, + "tokens_seen": 2420441088 + }, + { + "epoch": 0.73, + "learning_rate": 0.000215542627883651, + "loss": 1.2064, + "theoretical_loss": 3.4057085934740865, + "tokens_seen": 2420506624 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002155265797392177, + "loss": 1.2131, + "theoretical_loss": 3.40570125247607, + "tokens_seen": 2420572160 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002155105315947844, + "loss": 1.1337, + "theoretical_loss": 3.4056939117324543, + "tokens_seen": 2420637696 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002154944834503511, + "loss": 1.1833, + "theoretical_loss": 3.4056865712432245, + "tokens_seen": 2420703232 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002154784353059178, + "loss": 1.1638, + "theoretical_loss": 3.4056792310083646, + "tokens_seen": 2420768768 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002154623871614845, + "loss": 1.2633, + "theoretical_loss": 3.4056718910278585, + "tokens_seen": 2420834304 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002154463390170512, + "loss": 1.2134, + "theoretical_loss": 3.405664551301691, + "tokens_seen": 2420899840 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002154302908726179, + "loss": 1.1343, + "theoretical_loss": 3.405657211829847, + "tokens_seen": 2420965376 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002154142427281846, + "loss": 1.1838, + "theoretical_loss": 3.40564987261231, + "tokens_seen": 2421030912 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021539819458375127, + "loss": 1.2957, + "theoretical_loss": 3.4056425336490648, + "tokens_seen": 2421096448 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021538214643931797, + "loss": 1.2216, + "theoretical_loss": 3.4056351949400945, + "tokens_seen": 2421161984 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021536609829488467, + "loss": 1.2648, + "theoretical_loss": 3.405627856485385, + "tokens_seen": 2421227520 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021535005015045137, + "loss": 1.2796, + "theoretical_loss": 3.4056205182849197, + "tokens_seen": 2421293056 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021533400200601807, + "loss": 1.2723, + "theoretical_loss": 3.405613180338683, + "tokens_seen": 2421358592 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021531795386158477, + "loss": 1.2495, + "theoretical_loss": 3.4056058426466596, + "tokens_seen": 2421424128 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021530190571715147, + "loss": 1.2431, + "theoretical_loss": 3.405598505208833, + "tokens_seen": 2421489664 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 809199, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3278754949569702, + "objective/train/theoretical_loss": 3.4055911680251887, + "objective/train/tokens_used": 2442015200, + "theoretical_loss": 3.4055911680251887, + "tokens_seen": 2421555200 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021528585757271817, + "loss": 1.2899, + "theoretical_loss": 3.4055911680251887, + "tokens_seen": 2421555200 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021526980942828487, + "loss": 1.1465, + "theoretical_loss": 3.4055838310957096, + "tokens_seen": 2421620736 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021525376128385157, + "loss": 1.2543, + "theoretical_loss": 3.4055764944203815, + "tokens_seen": 2421686272 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021523771313941827, + "loss": 1.1722, + "theoretical_loss": 3.405569157999188, + "tokens_seen": 2421751808 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021522166499498497, + "loss": 1.2255, + "theoretical_loss": 3.4055618218321126, + "tokens_seen": 2421817344 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021520561685055167, + "loss": 1.3018, + "theoretical_loss": 3.405554485919141, + "tokens_seen": 2421882880 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021518956870611837, + "loss": 1.264, + "theoretical_loss": 3.405547150260257, + "tokens_seen": 2421948416 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021517352056168507, + "loss": 1.1722, + "theoretical_loss": 3.405539814855445, + "tokens_seen": 2422013952 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021515747241725178, + "loss": 1.2487, + "theoretical_loss": 3.405532479704689, + "tokens_seen": 2422079488 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021514142427281848, + "loss": 1.2918, + "theoretical_loss": 3.4055251448079735, + "tokens_seen": 2422145024 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021512537612838518, + "loss": 1.1913, + "theoretical_loss": 3.405517810165283, + "tokens_seen": 2422210560 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021510932798395188, + "loss": 1.2363, + "theoretical_loss": 3.4055104757766017, + "tokens_seen": 2422276096 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021509327983951858, + "loss": 1.305, + "theoretical_loss": 3.4055031416419137, + "tokens_seen": 2422341632 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021507723169508528, + "loss": 1.1993, + "theoretical_loss": 3.405495807761204, + "tokens_seen": 2422407168 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021506118355065198, + "loss": 1.1444, + "theoretical_loss": 3.405488474134456, + "tokens_seen": 2422472704 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021504513540621868, + "loss": 1.2683, + "theoretical_loss": 3.4054811407616548, + "tokens_seen": 2422538240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021502908726178538, + "loss": 1.2111, + "theoretical_loss": 3.405473807642785, + "tokens_seen": 2422603776 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021501303911735205, + "loss": 1.1786, + "theoretical_loss": 3.4054664747778296, + "tokens_seen": 2422669312 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021499699097291875, + "loss": 1.268, + "theoretical_loss": 3.405459142166774, + "tokens_seen": 2422734848 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021498094282848546, + "loss": 1.2715, + "theoretical_loss": 3.4054518098096023, + "tokens_seen": 2422800384 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021496489468405216, + "loss": 1.2586, + "theoretical_loss": 3.4054444777062987, + "tokens_seen": 2422865920 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021494884653961886, + "loss": 1.3042, + "theoretical_loss": 3.405437145856848, + "tokens_seen": 2422931456 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021493279839518556, + "loss": 1.1765, + "theoretical_loss": 3.405429814261234, + "tokens_seen": 2422996992 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021491675025075226, + "loss": 1.1783, + "theoretical_loss": 3.4054224829194415, + "tokens_seen": 2423062528 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021490070210631896, + "loss": 1.2545, + "theoretical_loss": 3.405415151831454, + "tokens_seen": 2423128064 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021488465396188566, + "loss": 1.1972, + "theoretical_loss": 3.405407820997257, + "tokens_seen": 2423193600 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021486860581745236, + "loss": 1.299, + "theoretical_loss": 3.4054004904168345, + "tokens_seen": 2423259136 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021485255767301906, + "loss": 1.1559, + "theoretical_loss": 3.40539316009017, + "tokens_seen": 2423324672 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021483650952858576, + "loss": 1.203, + "theoretical_loss": 3.405385830017249, + "tokens_seen": 2423390208 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021482046138415246, + "loss": 1.2275, + "theoretical_loss": 3.405378500198055, + "tokens_seen": 2423455744 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021480441323971916, + "loss": 1.1968, + "theoretical_loss": 3.4053711706325736, + "tokens_seen": 2423521280 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021478836509528586, + "loss": 1.1728, + "theoretical_loss": 3.4053638413207876, + "tokens_seen": 2423586816 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021477231695085256, + "loss": 1.3198, + "theoretical_loss": 3.4053565122626823, + "tokens_seen": 2423652352 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021475626880641926, + "loss": 1.2855, + "theoretical_loss": 3.4053491834582417, + "tokens_seen": 2423717888 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021474022066198596, + "loss": 1.2311, + "theoretical_loss": 3.40534185490745, + "tokens_seen": 2423783424 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021472417251755267, + "loss": 1.2111, + "theoretical_loss": 3.405334526610292, + "tokens_seen": 2423848960 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021470812437311937, + "loss": 1.1828, + "theoretical_loss": 3.405327198566752, + "tokens_seen": 2423914496 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021469207622868607, + "loss": 1.2072, + "theoretical_loss": 3.4053198707768146, + "tokens_seen": 2423980032 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021467602808425277, + "loss": 1.1728, + "theoretical_loss": 3.405312543240463, + "tokens_seen": 2424045568 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021465997993981947, + "loss": 1.2729, + "theoretical_loss": 3.405305215957683, + "tokens_seen": 2424111104 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021464393179538617, + "loss": 1.2762, + "theoretical_loss": 3.4052978889284584, + "tokens_seen": 2424176640 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021462788365095284, + "loss": 1.2795, + "theoretical_loss": 3.4052905621527736, + "tokens_seen": 2424242176 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021461183550651954, + "loss": 1.2214, + "theoretical_loss": 3.4052832356306126, + "tokens_seen": 2424307712 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021459578736208624, + "loss": 1.2644, + "theoretical_loss": 3.40527590936196, + "tokens_seen": 2424373248 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021457973921765294, + "loss": 1.1287, + "theoretical_loss": 3.4052685833468006, + "tokens_seen": 2424438784 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021456369107321964, + "loss": 1.1832, + "theoretical_loss": 3.4052612575851184, + "tokens_seen": 2424504320 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021454764292878635, + "loss": 1.2609, + "theoretical_loss": 3.4052539320768975, + "tokens_seen": 2424569856 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021453159478435305, + "loss": 1.2711, + "theoretical_loss": 3.405246606822123, + "tokens_seen": 2424635392 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021451554663991975, + "loss": 1.1784, + "theoretical_loss": 3.405239281820779, + "tokens_seen": 2424700928 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021449949849548645, + "loss": 1.203, + "theoretical_loss": 3.405231957072849, + "tokens_seen": 2424766464 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 810525, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1189380884170532, + "objective/train/theoretical_loss": 3.405224632578319, + "objective/train/tokens_used": 2445292000, + "theoretical_loss": 3.405224632578319, + "tokens_seen": 2424832000 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021448345035105315, + "loss": 1.2053, + "theoretical_loss": 3.405224632578319, + "tokens_seen": 2424832000 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021446740220661985, + "loss": 1.2598, + "theoretical_loss": 3.405217308337172, + "tokens_seen": 2424897536 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021445135406218655, + "loss": 1.2035, + "theoretical_loss": 3.4052099843493933, + "tokens_seen": 2424963072 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021443530591775325, + "loss": 1.2279, + "theoretical_loss": 3.405202660614967, + "tokens_seen": 2425028608 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021441925777331995, + "loss": 1.2426, + "theoretical_loss": 3.405195337133877, + "tokens_seen": 2425094144 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002144032096288867, + "loss": 1.1973, + "theoretical_loss": 3.405188013906108, + "tokens_seen": 2425159680 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021438716148445338, + "loss": 1.2866, + "theoretical_loss": 3.405180690931645, + "tokens_seen": 2425225216 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021437111334002008, + "loss": 1.2717, + "theoretical_loss": 3.4051733682104715, + "tokens_seen": 2425290752 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021435506519558678, + "loss": 1.2488, + "theoretical_loss": 3.405166045742572, + "tokens_seen": 2425356288 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021433901705115348, + "loss": 1.1991, + "theoretical_loss": 3.4051587235279315, + "tokens_seen": 2425421824 + }, + { + "epoch": 0.73, + "learning_rate": 0.00021432296890672018, + "loss": 1.2548, + "theoretical_loss": 3.4051514015665343, + "tokens_seen": 2425487360 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021430692076228688, + "loss": 1.2152, + "theoretical_loss": 3.4051440798583643, + "tokens_seen": 2425552896 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021429087261785358, + "loss": 1.2238, + "theoretical_loss": 3.405136758403406, + "tokens_seen": 2425618432 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021427482447342028, + "loss": 1.2284, + "theoretical_loss": 3.4051294372016443, + "tokens_seen": 2425683968 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021425877632898698, + "loss": 1.1835, + "theoretical_loss": 3.405122116253063, + "tokens_seen": 2425749504 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021424272818455368, + "loss": 1.2238, + "theoretical_loss": 3.405114795557647, + "tokens_seen": 2425815040 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021422668004012038, + "loss": 1.2675, + "theoretical_loss": 3.4051074751153805, + "tokens_seen": 2425880576 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021421063189568709, + "loss": 1.2621, + "theoretical_loss": 3.405100154926248, + "tokens_seen": 2425946112 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021419458375125379, + "loss": 1.2111, + "theoretical_loss": 3.405092834990233, + "tokens_seen": 2426011648 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002141785356068205, + "loss": 1.1756, + "theoretical_loss": 3.405085515307322, + "tokens_seen": 2426077184 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002141624874623872, + "loss": 1.3343, + "theoretical_loss": 3.4050781958774974, + "tokens_seen": 2426142720 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002141464393179539, + "loss": 1.2369, + "theoretical_loss": 3.4050708767007443, + "tokens_seen": 2426208256 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002141303911735206, + "loss": 1.3483, + "theoretical_loss": 3.405063557777047, + "tokens_seen": 2426273792 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002141143430290873, + "loss": 1.2461, + "theoretical_loss": 3.4050562391063903, + "tokens_seen": 2426339328 + }, + { + "epoch": 0.74, + "learning_rate": 0.000214098294884654, + "loss": 1.2098, + "theoretical_loss": 3.4050489206887584, + "tokens_seen": 2426404864 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002140822467402207, + "loss": 1.2222, + "theoretical_loss": 3.4050416025241357, + "tokens_seen": 2426470400 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002140661985957874, + "loss": 1.1978, + "theoretical_loss": 3.4050342846125066, + "tokens_seen": 2426535936 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002140501504513541, + "loss": 1.2462, + "theoretical_loss": 3.4050269669538555, + "tokens_seen": 2426601472 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002140341023069208, + "loss": 1.2623, + "theoretical_loss": 3.405019649548167, + "tokens_seen": 2426667008 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002140180541624875, + "loss": 1.3263, + "theoretical_loss": 3.405012332395425, + "tokens_seen": 2426732544 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021400200601805417, + "loss": 1.3777, + "theoretical_loss": 3.4050050154956146, + "tokens_seen": 2426798080 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021398595787362087, + "loss": 1.168, + "theoretical_loss": 3.40499769884872, + "tokens_seen": 2426863616 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021396990972918757, + "loss": 1.2123, + "theoretical_loss": 3.404990382454726, + "tokens_seen": 2426929152 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021395386158475427, + "loss": 1.2737, + "theoretical_loss": 3.4049830663136156, + "tokens_seen": 2426994688 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021393781344032097, + "loss": 1.3026, + "theoretical_loss": 3.404975750425375, + "tokens_seen": 2427060224 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021392176529588767, + "loss": 1.2274, + "theoretical_loss": 3.4049684347899873, + "tokens_seen": 2427125760 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021390571715145437, + "loss": 1.2076, + "theoretical_loss": 3.404961119407438, + "tokens_seen": 2427191296 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021388966900702107, + "loss": 1.2909, + "theoretical_loss": 3.404953804277711, + "tokens_seen": 2427256832 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021387362086258777, + "loss": 1.2192, + "theoretical_loss": 3.4049464894007904, + "tokens_seen": 2427322368 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021385757271815447, + "loss": 1.3397, + "theoretical_loss": 3.4049391747766613, + "tokens_seen": 2427387904 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021384152457372117, + "loss": 1.2544, + "theoretical_loss": 3.404931860405308, + "tokens_seen": 2427453440 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021382547642928787, + "loss": 1.3042, + "theoretical_loss": 3.4049245462867144, + "tokens_seen": 2427518976 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021380942828485457, + "loss": 1.145, + "theoretical_loss": 3.404917232420865, + "tokens_seen": 2427584512 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021379338014042127, + "loss": 1.1955, + "theoretical_loss": 3.4049099188077454, + "tokens_seen": 2427650048 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021377733199598798, + "loss": 1.2189, + "theoretical_loss": 3.404902605447339, + "tokens_seen": 2427715584 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021376128385155468, + "loss": 1.2325, + "theoretical_loss": 3.40489529233963, + "tokens_seen": 2427781120 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021374523570712138, + "loss": 1.1721, + "theoretical_loss": 3.4048879794846036, + "tokens_seen": 2427846656 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021372918756268808, + "loss": 1.2649, + "theoretical_loss": 3.404880666882244, + "tokens_seen": 2427912192 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021371313941825478, + "loss": 1.1599, + "theoretical_loss": 3.4048733545325356, + "tokens_seen": 2427977728 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021369709127382148, + "loss": 1.1854, + "theoretical_loss": 3.404866042435463, + "tokens_seen": 2428043264 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 811717, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1337710618972778, + "objective/train/theoretical_loss": 3.4048587305910103, + "objective/train/tokens_used": 2448568800, + "theoretical_loss": 3.4048587305910103, + "tokens_seen": 2428108800 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021368104312938818, + "loss": 1.1389, + "theoretical_loss": 3.4048587305910103, + "tokens_seen": 2428108800 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021366499498495488, + "loss": 1.2529, + "theoretical_loss": 3.4048514189991623, + "tokens_seen": 2428174336 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021364894684052158, + "loss": 1.2767, + "theoretical_loss": 3.404844107659903, + "tokens_seen": 2428239872 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021363289869608828, + "loss": 1.1604, + "theoretical_loss": 3.4048367965732176, + "tokens_seen": 2428305408 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021361685055165495, + "loss": 1.2239, + "theoretical_loss": 3.40482948573909, + "tokens_seen": 2428370944 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021360080240722166, + "loss": 1.1767, + "theoretical_loss": 3.4048221751575047, + "tokens_seen": 2428436480 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021358475426278836, + "loss": 1.1925, + "theoretical_loss": 3.404814864828446, + "tokens_seen": 2428502016 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021356870611835506, + "loss": 1.124, + "theoretical_loss": 3.4048075547518994, + "tokens_seen": 2428567552 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021355265797392176, + "loss": 1.2132, + "theoretical_loss": 3.404800244927848, + "tokens_seen": 2428633088 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021353660982948846, + "loss": 1.2134, + "theoretical_loss": 3.404792935356277, + "tokens_seen": 2428698624 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021352056168505516, + "loss": 1.1678, + "theoretical_loss": 3.404785626037171, + "tokens_seen": 2428764160 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021350451354062186, + "loss": 1.3398, + "theoretical_loss": 3.4047783169705137, + "tokens_seen": 2428829696 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021348846539618856, + "loss": 1.2887, + "theoretical_loss": 3.40477100815629, + "tokens_seen": 2428895232 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021347241725175526, + "loss": 1.2398, + "theoretical_loss": 3.404763699594485, + "tokens_seen": 2428960768 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021345636910732196, + "loss": 1.1675, + "theoretical_loss": 3.404756391285082, + "tokens_seen": 2429026304 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021344032096288866, + "loss": 1.2322, + "theoretical_loss": 3.4047490832280665, + "tokens_seen": 2429091840 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021342427281845536, + "loss": 1.2196, + "theoretical_loss": 3.4047417754234224, + "tokens_seen": 2429157376 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021340822467402206, + "loss": 1.1684, + "theoretical_loss": 3.404734467871134, + "tokens_seen": 2429222912 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021339217652958876, + "loss": 1.2434, + "theoretical_loss": 3.404727160571187, + "tokens_seen": 2429288448 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021337612838515546, + "loss": 1.2655, + "theoretical_loss": 3.4047198535235643, + "tokens_seen": 2429353984 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021336008024072216, + "loss": 1.1372, + "theoretical_loss": 3.404712546728251, + "tokens_seen": 2429419520 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021334403209628887, + "loss": 1.2468, + "theoretical_loss": 3.4047052401852316, + "tokens_seen": 2429485056 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021332798395185557, + "loss": 1.2789, + "theoretical_loss": 3.404697933894491, + "tokens_seen": 2429550592 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021331193580742227, + "loss": 1.2319, + "theoretical_loss": 3.4046906278560134, + "tokens_seen": 2429616128 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021329588766298897, + "loss": 1.2194, + "theoretical_loss": 3.4046833220697827, + "tokens_seen": 2429681664 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021327983951855567, + "loss": 1.2251, + "theoretical_loss": 3.4046760165357846, + "tokens_seen": 2429747200 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021326379137412237, + "loss": 1.1926, + "theoretical_loss": 3.404668711254002, + "tokens_seen": 2429812736 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021324774322968907, + "loss": 1.3904, + "theoretical_loss": 3.4046614062244207, + "tokens_seen": 2429878272 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021323169508525574, + "loss": 1.1188, + "theoretical_loss": 3.404654101447025, + "tokens_seen": 2429943808 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021321564694082244, + "loss": 1.1759, + "theoretical_loss": 3.404646796921799, + "tokens_seen": 2430009344 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002131995987963892, + "loss": 1.229, + "theoretical_loss": 3.404639492648727, + "tokens_seen": 2430074880 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002131835506519559, + "loss": 1.1467, + "theoretical_loss": 3.404632188627794, + "tokens_seen": 2430140416 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002131675025075226, + "loss": 1.1525, + "theoretical_loss": 3.4046248848589844, + "tokens_seen": 2430205952 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002131514543630893, + "loss": 1.2374, + "theoretical_loss": 3.4046175813422828, + "tokens_seen": 2430271488 + }, + { + "epoch": 0.74, + "learning_rate": 0.000213135406218656, + "loss": 1.2248, + "theoretical_loss": 3.4046102780776732, + "tokens_seen": 2430337024 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002131193580742227, + "loss": 1.2395, + "theoretical_loss": 3.4046029750651408, + "tokens_seen": 2430402560 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002131033099297894, + "loss": 1.3251, + "theoretical_loss": 3.4045956723046693, + "tokens_seen": 2430468096 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002130872617853561, + "loss": 1.247, + "theoretical_loss": 3.404588369796244, + "tokens_seen": 2430533632 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002130712136409228, + "loss": 1.2932, + "theoretical_loss": 3.404581067539849, + "tokens_seen": 2430599168 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002130551654964895, + "loss": 1.2816, + "theoretical_loss": 3.404573765535469, + "tokens_seen": 2430664704 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002130391173520562, + "loss": 1.2838, + "theoretical_loss": 3.404566463783088, + "tokens_seen": 2430730240 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002130230692076229, + "loss": 1.3536, + "theoretical_loss": 3.404559162282691, + "tokens_seen": 2430795776 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002130070210631896, + "loss": 1.276, + "theoretical_loss": 3.404551861034262, + "tokens_seen": 2430861312 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021299097291875628, + "loss": 1.2649, + "theoretical_loss": 3.404544560037787, + "tokens_seen": 2430926848 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021297492477432298, + "loss": 1.2465, + "theoretical_loss": 3.4045372592932486, + "tokens_seen": 2430992384 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021295887662988968, + "loss": 1.2106, + "theoretical_loss": 3.404529958800632, + "tokens_seen": 2431057920 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021294282848545638, + "loss": 1.1642, + "theoretical_loss": 3.404522658559922, + "tokens_seen": 2431123456 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021292678034102308, + "loss": 1.2176, + "theoretical_loss": 3.4045153585711034, + "tokens_seen": 2431188992 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021291073219658978, + "loss": 1.2425, + "theoretical_loss": 3.4045080588341596, + "tokens_seen": 2431254528 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021289468405215648, + "loss": 1.2658, + "theoretical_loss": 3.404500759349076, + "tokens_seen": 2431320064 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 812751, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2283116579055786, + "objective/train/theoretical_loss": 3.4044934601158374, + "objective/train/tokens_used": 2451845600, + "theoretical_loss": 3.4044934601158374, + "tokens_seen": 2431385600 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021287863590772318, + "loss": 1.2107, + "theoretical_loss": 3.4044934601158374, + "tokens_seen": 2431385600 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021286258776328988, + "loss": 1.1902, + "theoretical_loss": 3.404486161134427, + "tokens_seen": 2431451136 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021284653961885659, + "loss": 1.2542, + "theoretical_loss": 3.404478862404831, + "tokens_seen": 2431516672 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021283049147442329, + "loss": 1.1443, + "theoretical_loss": 3.4044715639270327, + "tokens_seen": 2431582208 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021281444332999, + "loss": 1.2933, + "theoretical_loss": 3.4044642657010167, + "tokens_seen": 2431647744 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002127983951855567, + "loss": 1.2687, + "theoretical_loss": 3.4044569677267686, + "tokens_seen": 2431713280 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002127823470411234, + "loss": 1.1241, + "theoretical_loss": 3.4044496700042717, + "tokens_seen": 2431778816 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002127662988966901, + "loss": 1.3111, + "theoretical_loss": 3.404442372533511, + "tokens_seen": 2431844352 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002127502507522568, + "loss": 1.2639, + "theoretical_loss": 3.404435075314471, + "tokens_seen": 2431909888 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002127342026078235, + "loss": 1.2626, + "theoretical_loss": 3.404427778347136, + "tokens_seen": 2431975424 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002127181544633902, + "loss": 1.3172, + "theoretical_loss": 3.4044204816314916, + "tokens_seen": 2432040960 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002127021063189569, + "loss": 1.2782, + "theoretical_loss": 3.4044131851675212, + "tokens_seen": 2432106496 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002126860581745236, + "loss": 1.2304, + "theoretical_loss": 3.4044058889552096, + "tokens_seen": 2432172032 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002126700100300903, + "loss": 1.2716, + "theoretical_loss": 3.4043985929945415, + "tokens_seen": 2432237568 + }, + { + "epoch": 0.74, + "learning_rate": 0.000212653961885657, + "loss": 1.2557, + "theoretical_loss": 3.404391297285501, + "tokens_seen": 2432303104 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002126379137412237, + "loss": 1.2172, + "theoretical_loss": 3.4043840018280735, + "tokens_seen": 2432368640 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002126218655967904, + "loss": 1.3045, + "theoretical_loss": 3.4043767066222426, + "tokens_seen": 2432434176 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021260581745235707, + "loss": 1.2566, + "theoretical_loss": 3.404369411667994, + "tokens_seen": 2432499712 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021258976930792377, + "loss": 1.2935, + "theoretical_loss": 3.4043621169653107, + "tokens_seen": 2432565248 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021257372116349047, + "loss": 1.2836, + "theoretical_loss": 3.4043548225141786, + "tokens_seen": 2432630784 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021255767301905717, + "loss": 1.2883, + "theoretical_loss": 3.404347528314582, + "tokens_seen": 2432696320 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021254162487462387, + "loss": 1.1908, + "theoretical_loss": 3.4043402343665043, + "tokens_seen": 2432761856 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021252557673019057, + "loss": 1.2137, + "theoretical_loss": 3.4043329406699314, + "tokens_seen": 2432827392 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021250952858575727, + "loss": 1.2071, + "theoretical_loss": 3.4043256472248475, + "tokens_seen": 2432892928 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021249348044132397, + "loss": 1.1696, + "theoretical_loss": 3.404318354031237, + "tokens_seen": 2432958464 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021247743229689067, + "loss": 1.2651, + "theoretical_loss": 3.4043110610890848, + "tokens_seen": 2433024000 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021246138415245737, + "loss": 1.2124, + "theoretical_loss": 3.404303768398375, + "tokens_seen": 2433089536 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021244533600802407, + "loss": 1.1757, + "theoretical_loss": 3.4042964759590917, + "tokens_seen": 2433155072 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021242928786359077, + "loss": 1.2648, + "theoretical_loss": 3.404289183771221, + "tokens_seen": 2433220608 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021241323971915748, + "loss": 1.2172, + "theoretical_loss": 3.4042818918347457, + "tokens_seen": 2433286144 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021239719157472418, + "loss": 1.2475, + "theoretical_loss": 3.4042746001496518, + "tokens_seen": 2433351680 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021238114343029088, + "loss": 1.2223, + "theoretical_loss": 3.404267308715923, + "tokens_seen": 2433417216 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021236509528585758, + "loss": 1.1876, + "theoretical_loss": 3.4042600175335442, + "tokens_seen": 2433482752 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021234904714142428, + "loss": 1.239, + "theoretical_loss": 3.4042527266024996, + "tokens_seen": 2433548288 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021233299899699098, + "loss": 1.2575, + "theoretical_loss": 3.4042454359227743, + "tokens_seen": 2433613824 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021231695085255768, + "loss": 1.2248, + "theoretical_loss": 3.404238145494353, + "tokens_seen": 2433679360 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021230090270812438, + "loss": 1.1558, + "theoretical_loss": 3.4042308553172194, + "tokens_seen": 2433744896 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021228485456369108, + "loss": 1.2034, + "theoretical_loss": 3.4042235653913586, + "tokens_seen": 2433810432 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021226880641925778, + "loss": 1.1756, + "theoretical_loss": 3.4042162757167556, + "tokens_seen": 2433875968 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021225275827482448, + "loss": 1.1321, + "theoretical_loss": 3.404208986293394, + "tokens_seen": 2433941504 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021223671013039118, + "loss": 1.2973, + "theoretical_loss": 3.404201697121259, + "tokens_seen": 2434007040 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021222066198595786, + "loss": 1.309, + "theoretical_loss": 3.4041944082003353, + "tokens_seen": 2434072576 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021220461384152456, + "loss": 1.2782, + "theoretical_loss": 3.404187119530607, + "tokens_seen": 2434138112 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021218856569709126, + "loss": 1.2751, + "theoretical_loss": 3.4041798311120592, + "tokens_seen": 2434203648 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021217251755265796, + "loss": 1.2166, + "theoretical_loss": 3.4041725429446763, + "tokens_seen": 2434269184 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021215646940822466, + "loss": 1.208, + "theoretical_loss": 3.4041652550284427, + "tokens_seen": 2434334720 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021214042126379136, + "loss": 1.1791, + "theoretical_loss": 3.404157967363343, + "tokens_seen": 2434400256 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021212437311935806, + "loss": 1.2623, + "theoretical_loss": 3.404150679949362, + "tokens_seen": 2434465792 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021210832497492476, + "loss": 1.2136, + "theoretical_loss": 3.404143392786484, + "tokens_seen": 2434531328 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021209227683049146, + "loss": 1.1994, + "theoretical_loss": 3.404136105874694, + "tokens_seen": 2434596864 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 814053, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2832529544830322, + "objective/train/theoretical_loss": 3.4041288192139763, + "objective/train/tokens_used": 2455122400, + "theoretical_loss": 3.4041288192139763, + "tokens_seen": 2434662400 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021207622868605816, + "loss": 1.2401, + "theoretical_loss": 3.4041288192139763, + "tokens_seen": 2434662400 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021206018054162486, + "loss": 1.2534, + "theoretical_loss": 3.4041215328043153, + "tokens_seen": 2434727936 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021204413239719156, + "loss": 1.2171, + "theoretical_loss": 3.4041142466456957, + "tokens_seen": 2434793472 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021202808425275826, + "loss": 1.2908, + "theoretical_loss": 3.4041069607381025, + "tokens_seen": 2434859008 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021201203610832496, + "loss": 1.1228, + "theoretical_loss": 3.40409967508152, + "tokens_seen": 2434924544 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021199598796389172, + "loss": 1.2748, + "theoretical_loss": 3.4040923896759328, + "tokens_seen": 2434990080 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021197993981945842, + "loss": 1.1732, + "theoretical_loss": 3.4040851045213256, + "tokens_seen": 2435055616 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002119638916750251, + "loss": 1.2375, + "theoretical_loss": 3.4040778196176826, + "tokens_seen": 2435121152 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002119478435305918, + "loss": 1.2046, + "theoretical_loss": 3.404070534964989, + "tokens_seen": 2435186688 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002119317953861585, + "loss": 1.1096, + "theoretical_loss": 3.404063250563229, + "tokens_seen": 2435252224 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002119157472417252, + "loss": 1.1968, + "theoretical_loss": 3.4040559664123875, + "tokens_seen": 2435317760 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002118996990972919, + "loss": 1.1899, + "theoretical_loss": 3.4040486825124487, + "tokens_seen": 2435383296 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002118836509528586, + "loss": 1.2778, + "theoretical_loss": 3.4040413988633973, + "tokens_seen": 2435448832 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002118676028084253, + "loss": 1.2325, + "theoretical_loss": 3.404034115465218, + "tokens_seen": 2435514368 + }, + { + "epoch": 0.74, + "learning_rate": 0.000211851554663992, + "loss": 1.2795, + "theoretical_loss": 3.404026832317896, + "tokens_seen": 2435579904 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002118355065195587, + "loss": 1.3063, + "theoretical_loss": 3.404019549421415, + "tokens_seen": 2435645440 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002118194583751254, + "loss": 1.3204, + "theoretical_loss": 3.40401226677576, + "tokens_seen": 2435710976 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002118034102306921, + "loss": 1.2707, + "theoretical_loss": 3.4040049843809155, + "tokens_seen": 2435776512 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002117873620862588, + "loss": 1.1912, + "theoretical_loss": 3.4039977022368664, + "tokens_seen": 2435842048 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002117713139418255, + "loss": 1.2375, + "theoretical_loss": 3.4039904203435967, + "tokens_seen": 2435907584 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002117552657973922, + "loss": 1.2157, + "theoretical_loss": 3.4039831387010917, + "tokens_seen": 2435973120 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002117392176529589, + "loss": 1.3422, + "theoretical_loss": 3.4039758573093355, + "tokens_seen": 2436038656 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002117231695085256, + "loss": 1.2587, + "theoretical_loss": 3.4039685761683134, + "tokens_seen": 2436104192 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002117071213640923, + "loss": 1.3304, + "theoretical_loss": 3.4039612952780094, + "tokens_seen": 2436169728 + }, + { + "epoch": 0.74, + "learning_rate": 0.000211691073219659, + "loss": 1.2743, + "theoretical_loss": 3.4039540146384084, + "tokens_seen": 2436235264 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002116750250752257, + "loss": 1.3233, + "theoretical_loss": 3.4039467342494945, + "tokens_seen": 2436300800 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002116589769307924, + "loss": 1.2461, + "theoretical_loss": 3.4039394541112533, + "tokens_seen": 2436366336 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002116429287863591, + "loss": 1.2261, + "theoretical_loss": 3.4039321742236686, + "tokens_seen": 2436431872 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002116268806419258, + "loss": 1.1847, + "theoretical_loss": 3.4039248945867255, + "tokens_seen": 2436497408 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002116108324974925, + "loss": 1.2524, + "theoretical_loss": 3.403917615200408, + "tokens_seen": 2436562944 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002115947843530592, + "loss": 1.181, + "theoretical_loss": 3.4039103360647016, + "tokens_seen": 2436628480 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021157873620862588, + "loss": 1.202, + "theoretical_loss": 3.4039030571795905, + "tokens_seen": 2436694016 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021156268806419258, + "loss": 1.2898, + "theoretical_loss": 3.403895778545059, + "tokens_seen": 2436759552 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021154663991975928, + "loss": 1.2485, + "theoretical_loss": 3.4038885001610923, + "tokens_seen": 2436825088 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021153059177532598, + "loss": 1.2855, + "theoretical_loss": 3.4038812220276746, + "tokens_seen": 2436890624 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021151454363089268, + "loss": 1.2716, + "theoretical_loss": 3.403873944144791, + "tokens_seen": 2436956160 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021149849548645938, + "loss": 1.2029, + "theoretical_loss": 3.403866666512426, + "tokens_seen": 2437021696 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021148244734202609, + "loss": 1.0867, + "theoretical_loss": 3.403859389130564, + "tokens_seen": 2437087232 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021146639919759279, + "loss": 1.1786, + "theoretical_loss": 3.4038521119991896, + "tokens_seen": 2437152768 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021145035105315949, + "loss": 1.1941, + "theoretical_loss": 3.4038448351182877, + "tokens_seen": 2437218304 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002114343029087262, + "loss": 1.2116, + "theoretical_loss": 3.403837558487843, + "tokens_seen": 2437283840 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002114182547642929, + "loss": 1.2469, + "theoretical_loss": 3.4038302821078394, + "tokens_seen": 2437349376 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002114022066198596, + "loss": 1.2363, + "theoretical_loss": 3.4038230059782624, + "tokens_seen": 2437414912 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002113861584754263, + "loss": 1.2721, + "theoretical_loss": 3.403815730099097, + "tokens_seen": 2437480448 + }, + { + "epoch": 0.74, + "learning_rate": 0.000211370110330993, + "loss": 1.2924, + "theoretical_loss": 3.403808454470327, + "tokens_seen": 2437545984 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002113540621865597, + "loss": 1.1763, + "theoretical_loss": 3.403801179091937, + "tokens_seen": 2437611520 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002113380140421264, + "loss": 1.2239, + "theoretical_loss": 3.403793903963912, + "tokens_seen": 2437677056 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002113219658976931, + "loss": 1.1886, + "theoretical_loss": 3.4037866290862366, + "tokens_seen": 2437742592 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002113059177532598, + "loss": 1.1461, + "theoretical_loss": 3.4037793544588952, + "tokens_seen": 2437808128 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002112898696088265, + "loss": 1.1742, + "theoretical_loss": 3.403772080081873, + "tokens_seen": 2437873664 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 815053, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3028794527053833, + "objective/train/theoretical_loss": 3.4037648059551544, + "objective/train/tokens_used": 2458399200, + "theoretical_loss": 3.4037648059551544, + "tokens_seen": 2437939200 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002112738214643932, + "loss": 1.2959, + "theoretical_loss": 3.4037648059551544, + "tokens_seen": 2437939200 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002112577733199599, + "loss": 1.2335, + "theoretical_loss": 3.403757532078724, + "tokens_seen": 2438004736 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002112417251755266, + "loss": 1.1654, + "theoretical_loss": 3.4037502584525665, + "tokens_seen": 2438070272 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002112256770310933, + "loss": 1.2656, + "theoretical_loss": 3.4037429850766667, + "tokens_seen": 2438135808 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021120962888666, + "loss": 1.2331, + "theoretical_loss": 3.403735711951009, + "tokens_seen": 2438201344 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021119358074222667, + "loss": 1.2808, + "theoretical_loss": 3.4037284390755778, + "tokens_seen": 2438266880 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021117753259779337, + "loss": 1.3389, + "theoretical_loss": 3.403721166450359, + "tokens_seen": 2438332416 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021116148445336007, + "loss": 1.2447, + "theoretical_loss": 3.4037138940753353, + "tokens_seen": 2438397952 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021114543630892677, + "loss": 1.1622, + "theoretical_loss": 3.4037066219504934, + "tokens_seen": 2438463488 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021112938816449347, + "loss": 1.3385, + "theoretical_loss": 3.403699350075817, + "tokens_seen": 2438529024 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021111334002006017, + "loss": 1.2154, + "theoretical_loss": 3.4036920784512903, + "tokens_seen": 2438594560 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021109729187562687, + "loss": 1.2222, + "theoretical_loss": 3.4036848070768984, + "tokens_seen": 2438660096 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021108124373119357, + "loss": 1.2631, + "theoretical_loss": 3.4036775359526263, + "tokens_seen": 2438725632 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021106519558676027, + "loss": 1.3168, + "theoretical_loss": 3.4036702650784587, + "tokens_seen": 2438791168 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021104914744232698, + "loss": 1.3053, + "theoretical_loss": 3.40366299445438, + "tokens_seen": 2438856704 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021103309929789368, + "loss": 1.2031, + "theoretical_loss": 3.403655724080375, + "tokens_seen": 2438922240 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021101705115346038, + "loss": 1.2824, + "theoretical_loss": 3.4036484539564276, + "tokens_seen": 2438987776 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021100100300902708, + "loss": 1.1969, + "theoretical_loss": 3.4036411840825234, + "tokens_seen": 2439053312 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021098495486459378, + "loss": 1.2629, + "theoretical_loss": 3.403633914458647, + "tokens_seen": 2439118848 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021096890672016048, + "loss": 1.1861, + "theoretical_loss": 3.403626645084783, + "tokens_seen": 2439184384 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021095285857572718, + "loss": 1.2027, + "theoretical_loss": 3.4036193759609157, + "tokens_seen": 2439249920 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021093681043129388, + "loss": 1.2814, + "theoretical_loss": 3.4036121070870307, + "tokens_seen": 2439315456 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021092076228686058, + "loss": 1.2502, + "theoretical_loss": 3.4036048384631115, + "tokens_seen": 2439380992 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021090471414242728, + "loss": 1.2428, + "theoretical_loss": 3.4035975700891434, + "tokens_seen": 2439446528 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021088866599799398, + "loss": 1.2376, + "theoretical_loss": 3.4035903019651115, + "tokens_seen": 2439512064 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021087261785356068, + "loss": 1.2151, + "theoretical_loss": 3.4035830340909996, + "tokens_seen": 2439577600 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021085656970912738, + "loss": 1.2245, + "theoretical_loss": 3.403575766466793, + "tokens_seen": 2439643136 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021084052156469408, + "loss": 1.1639, + "theoretical_loss": 3.4035684990924766, + "tokens_seen": 2439708672 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021082447342026078, + "loss": 1.1733, + "theoretical_loss": 3.403561231968034, + "tokens_seen": 2439774208 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002108084252758275, + "loss": 1.2355, + "theoretical_loss": 3.403553965093451, + "tokens_seen": 2439839744 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002107923771313942, + "loss": 1.25, + "theoretical_loss": 3.403546698468712, + "tokens_seen": 2439905280 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002107763289869609, + "loss": 1.2608, + "theoretical_loss": 3.4035394320938015, + "tokens_seen": 2439970816 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021076028084252761, + "loss": 1.2048, + "theoretical_loss": 3.403532165968705, + "tokens_seen": 2440036352 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021074423269809431, + "loss": 1.1948, + "theoretical_loss": 3.4035249000934056, + "tokens_seen": 2440101888 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021072818455366101, + "loss": 1.3336, + "theoretical_loss": 3.4035176344678892, + "tokens_seen": 2440167424 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021071213640922772, + "loss": 1.2303, + "theoretical_loss": 3.40351036909214, + "tokens_seen": 2440232960 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021069608826479442, + "loss": 1.1314, + "theoretical_loss": 3.4035031039661434, + "tokens_seen": 2440298496 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021068004012036112, + "loss": 1.2898, + "theoretical_loss": 3.4034958390898833, + "tokens_seen": 2440364032 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021066399197592782, + "loss": 1.3442, + "theoretical_loss": 3.403488574463345, + "tokens_seen": 2440429568 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021064794383149452, + "loss": 1.2493, + "theoretical_loss": 3.403481310086513, + "tokens_seen": 2440495104 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021063189568706122, + "loss": 1.2961, + "theoretical_loss": 3.403474045959372, + "tokens_seen": 2440560640 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021061584754262792, + "loss": 1.2053, + "theoretical_loss": 3.4034667820819067, + "tokens_seen": 2440626176 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021059979939819462, + "loss": 1.1157, + "theoretical_loss": 3.4034595184541017, + "tokens_seen": 2440691712 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021058375125376132, + "loss": 1.2675, + "theoretical_loss": 3.403452255075942, + "tokens_seen": 2440757248 + }, + { + "epoch": 0.74, + "learning_rate": 0.000210567703109328, + "loss": 1.2813, + "theoretical_loss": 3.4034449919474117, + "tokens_seen": 2440822784 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002105516549648947, + "loss": 1.2083, + "theoretical_loss": 3.4034377290684965, + "tokens_seen": 2440888320 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002105356068204614, + "loss": 1.2027, + "theoretical_loss": 3.40343046643918, + "tokens_seen": 2440953856 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002105195586760281, + "loss": 1.2281, + "theoretical_loss": 3.403423204059448, + "tokens_seen": 2441019392 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002105035105315948, + "loss": 1.2608, + "theoretical_loss": 3.4034159419292846, + "tokens_seen": 2441084928 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002104874623871615, + "loss": 1.1941, + "theoretical_loss": 3.4034086800486745, + "tokens_seen": 2441150464 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 816249, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.226874589920044, + "objective/train/theoretical_loss": 3.403401418417603, + "objective/train/tokens_used": 2461676000, + "theoretical_loss": 3.403401418417603, + "tokens_seen": 2441216000 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002104714142427282, + "loss": 1.214, + "theoretical_loss": 3.403401418417603, + "tokens_seen": 2441216000 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002104553660982949, + "loss": 1.1483, + "theoretical_loss": 3.403394157036054, + "tokens_seen": 2441281536 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002104393179538616, + "loss": 1.2136, + "theoretical_loss": 3.4033868959040126, + "tokens_seen": 2441347072 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002104232698094283, + "loss": 1.2692, + "theoretical_loss": 3.4033796350214636, + "tokens_seen": 2441412608 + }, + { + "epoch": 0.74, + "learning_rate": 0.000210407221664995, + "loss": 1.2736, + "theoretical_loss": 3.403372374388392, + "tokens_seen": 2441478144 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002103911735205617, + "loss": 1.2639, + "theoretical_loss": 3.4033651140047816, + "tokens_seen": 2441543680 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002103751253761284, + "loss": 1.1667, + "theoretical_loss": 3.4033578538706184, + "tokens_seen": 2441609216 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002103590772316951, + "loss": 1.243, + "theoretical_loss": 3.403350593985886, + "tokens_seen": 2441674752 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002103430290872618, + "loss": 1.2145, + "theoretical_loss": 3.40334333435057, + "tokens_seen": 2441740288 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002103269809428285, + "loss": 1.1826, + "theoretical_loss": 3.4033360749646544, + "tokens_seen": 2441805824 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002103109327983952, + "loss": 1.2529, + "theoretical_loss": 3.4033288158281247, + "tokens_seen": 2441871360 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002102948846539619, + "loss": 1.2879, + "theoretical_loss": 3.403321556940965, + "tokens_seen": 2441936896 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002102788365095286, + "loss": 1.256, + "theoretical_loss": 3.4033142983031603, + "tokens_seen": 2442002432 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002102627883650953, + "loss": 1.2301, + "theoretical_loss": 3.4033070399146954, + "tokens_seen": 2442067968 + }, + { + "epoch": 0.74, + "learning_rate": 0.000210246740220662, + "loss": 1.2517, + "theoretical_loss": 3.403299781775555, + "tokens_seen": 2442133504 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002102306920762287, + "loss": 1.1799, + "theoretical_loss": 3.4032925238857237, + "tokens_seen": 2442199040 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002102146439317954, + "loss": 1.2366, + "theoretical_loss": 3.4032852662451862, + "tokens_seen": 2442264576 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002101985957873621, + "loss": 1.1749, + "theoretical_loss": 3.4032780088539276, + "tokens_seen": 2442330112 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021018254764292878, + "loss": 1.2633, + "theoretical_loss": 3.4032707517119327, + "tokens_seen": 2442395648 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021016649949849548, + "loss": 1.2142, + "theoretical_loss": 3.4032634948191856, + "tokens_seen": 2442461184 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021015045135406218, + "loss": 1.2413, + "theoretical_loss": 3.403256238175672, + "tokens_seen": 2442526720 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021013440320962888, + "loss": 1.2829, + "theoretical_loss": 3.4032489817813754, + "tokens_seen": 2442592256 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021011835506519558, + "loss": 1.1604, + "theoretical_loss": 3.4032417256362817, + "tokens_seen": 2442657792 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021010230692076229, + "loss": 1.2321, + "theoretical_loss": 3.4032344697403754, + "tokens_seen": 2442723328 + }, + { + "epoch": 0.74, + "learning_rate": 0.00021008625877632899, + "loss": 1.1591, + "theoretical_loss": 3.403227214093641, + "tokens_seen": 2442788864 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002100702106318957, + "loss": 1.2773, + "theoretical_loss": 3.403219958696063, + "tokens_seen": 2442854400 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002100541624874624, + "loss": 1.1995, + "theoretical_loss": 3.4032127035476263, + "tokens_seen": 2442919936 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002100381143430291, + "loss": 1.2714, + "theoretical_loss": 3.4032054486483165, + "tokens_seen": 2442985472 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002100220661985958, + "loss": 1.1922, + "theoretical_loss": 3.4031981939981173, + "tokens_seen": 2443051008 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002100060180541625, + "loss": 1.318, + "theoretical_loss": 3.403190939597014, + "tokens_seen": 2443116544 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002099899699097292, + "loss": 1.1664, + "theoretical_loss": 3.4031836854449917, + "tokens_seen": 2443182080 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002099739217652959, + "loss": 1.1963, + "theoretical_loss": 3.4031764315420343, + "tokens_seen": 2443247616 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002099578736208626, + "loss": 1.2503, + "theoretical_loss": 3.4031691778881266, + "tokens_seen": 2443313152 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002099418254764293, + "loss": 1.2622, + "theoretical_loss": 3.4031619244832543, + "tokens_seen": 2443378688 + }, + { + "epoch": 0.74, + "learning_rate": 0.000209925777331996, + "loss": 1.161, + "theoretical_loss": 3.4031546713274015, + "tokens_seen": 2443444224 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002099097291875627, + "loss": 1.3132, + "theoretical_loss": 3.403147418420553, + "tokens_seen": 2443509760 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002098936810431294, + "loss": 1.363, + "theoretical_loss": 3.403140165762694, + "tokens_seen": 2443575296 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002098776328986961, + "loss": 1.1775, + "theoretical_loss": 3.403132913353809, + "tokens_seen": 2443640832 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002098615847542628, + "loss": 1.26, + "theoretical_loss": 3.403125661193882, + "tokens_seen": 2443706368 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002098455366098295, + "loss": 1.2412, + "theoretical_loss": 3.403118409282899, + "tokens_seen": 2443771904 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002098294884653962, + "loss": 1.2138, + "theoretical_loss": 3.4031111576208444, + "tokens_seen": 2443837440 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002098134403209629, + "loss": 1.1496, + "theoretical_loss": 3.4031039062077024, + "tokens_seen": 2443902976 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020979739217652957, + "loss": 1.201, + "theoretical_loss": 3.4030966550434587, + "tokens_seen": 2443968512 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020978134403209627, + "loss": 1.2342, + "theoretical_loss": 3.4030894041280977, + "tokens_seen": 2444034048 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020976529588766297, + "loss": 1.2555, + "theoretical_loss": 3.403082153461604, + "tokens_seen": 2444099584 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020974924774322967, + "loss": 1.2585, + "theoretical_loss": 3.4030749030439624, + "tokens_seen": 2444165120 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020973319959879637, + "loss": 1.2239, + "theoretical_loss": 3.4030676528751576, + "tokens_seen": 2444230656 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020971715145436307, + "loss": 1.2328, + "theoretical_loss": 3.4030604029551745, + "tokens_seen": 2444296192 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020970110330992977, + "loss": 1.2435, + "theoretical_loss": 3.4030531532839987, + "tokens_seen": 2444361728 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020968505516549647, + "loss": 1.2117, + "theoretical_loss": 3.4030459038616137, + "tokens_seen": 2444427264 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 817070, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.277119517326355, + "objective/train/theoretical_loss": 3.4030386546880047, + "objective/train/tokens_used": 2464952800, + "theoretical_loss": 3.4030386546880047, + "tokens_seen": 2444492800 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020966900702106318, + "loss": 1.2573, + "theoretical_loss": 3.4030386546880047, + "tokens_seen": 2444492800 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020965295887662988, + "loss": 1.2495, + "theoretical_loss": 3.4030314057631568, + "tokens_seen": 2444558336 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020963691073219658, + "loss": 1.2834, + "theoretical_loss": 3.4030241570870547, + "tokens_seen": 2444623872 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020962086258776328, + "loss": 1.2938, + "theoretical_loss": 3.403016908659683, + "tokens_seen": 2444689408 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020960481444333003, + "loss": 1.2004, + "theoretical_loss": 3.403009660481027, + "tokens_seen": 2444754944 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020958876629889673, + "loss": 1.3432, + "theoretical_loss": 3.4030024125510705, + "tokens_seen": 2444820480 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020957271815446343, + "loss": 1.2426, + "theoretical_loss": 3.4029951648697994, + "tokens_seen": 2444886016 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020955667001003013, + "loss": 1.2545, + "theoretical_loss": 3.4029879174371978, + "tokens_seen": 2444951552 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002095406218655968, + "loss": 1.1728, + "theoretical_loss": 3.402980670253251, + "tokens_seen": 2445017088 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002095245737211635, + "loss": 1.1433, + "theoretical_loss": 3.4029734233179436, + "tokens_seen": 2445082624 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002095085255767302, + "loss": 1.2018, + "theoretical_loss": 3.40296617663126, + "tokens_seen": 2445148160 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002094924774322969, + "loss": 1.2357, + "theoretical_loss": 3.4029589301931855, + "tokens_seen": 2445213696 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002094764292878636, + "loss": 1.3183, + "theoretical_loss": 3.4029516840037046, + "tokens_seen": 2445279232 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002094603811434303, + "loss": 1.32, + "theoretical_loss": 3.4029444380628027, + "tokens_seen": 2445344768 + }, + { + "epoch": 0.74, + "learning_rate": 0.000209444332998997, + "loss": 1.2979, + "theoretical_loss": 3.4029371923704637, + "tokens_seen": 2445410304 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002094282848545637, + "loss": 1.2137, + "theoretical_loss": 3.402929946926673, + "tokens_seen": 2445475840 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002094122367101304, + "loss": 1.2905, + "theoretical_loss": 3.4029227017314154, + "tokens_seen": 2445541376 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002093961885656971, + "loss": 1.2501, + "theoretical_loss": 3.4029154567846756, + "tokens_seen": 2445606912 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020938014042126381, + "loss": 1.1957, + "theoretical_loss": 3.4029082120864382, + "tokens_seen": 2445672448 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020936409227683051, + "loss": 1.3486, + "theoretical_loss": 3.402900967636689, + "tokens_seen": 2445737984 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020934804413239722, + "loss": 1.2664, + "theoretical_loss": 3.4028937234354113, + "tokens_seen": 2445803520 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020933199598796392, + "loss": 1.1844, + "theoretical_loss": 3.4028864794825906, + "tokens_seen": 2445869056 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020931594784353062, + "loss": 1.1847, + "theoretical_loss": 3.4028792357782125, + "tokens_seen": 2445934592 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020929989969909732, + "loss": 1.1499, + "theoretical_loss": 3.4028719923222606, + "tokens_seen": 2446000128 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020928385155466402, + "loss": 1.3032, + "theoretical_loss": 3.4028647491147206, + "tokens_seen": 2446065664 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020926780341023072, + "loss": 1.2617, + "theoretical_loss": 3.4028575061555766, + "tokens_seen": 2446131200 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020925175526579742, + "loss": 1.2375, + "theoretical_loss": 3.4028502634448143, + "tokens_seen": 2446196736 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020923570712136412, + "loss": 1.1323, + "theoretical_loss": 3.4028430209824174, + "tokens_seen": 2446262272 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020921965897693082, + "loss": 1.3822, + "theoretical_loss": 3.402835778768372, + "tokens_seen": 2446327808 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020920361083249752, + "loss": 1.2304, + "theoretical_loss": 3.402828536802662, + "tokens_seen": 2446393344 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020918756268806422, + "loss": 1.2206, + "theoretical_loss": 3.4028212950852725, + "tokens_seen": 2446458880 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020917151454363092, + "loss": 1.2382, + "theoretical_loss": 3.402814053616188, + "tokens_seen": 2446524416 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002091554663991976, + "loss": 1.272, + "theoretical_loss": 3.4028068123953945, + "tokens_seen": 2446589952 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002091394182547643, + "loss": 1.1713, + "theoretical_loss": 3.4027995714228756, + "tokens_seen": 2446655488 + }, + { + "epoch": 0.74, + "learning_rate": 0.000209123370110331, + "loss": 1.3244, + "theoretical_loss": 3.4027923306986168, + "tokens_seen": 2446721024 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002091073219658977, + "loss": 1.2806, + "theoretical_loss": 3.4027850902226024, + "tokens_seen": 2446786560 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002090912738214644, + "loss": 1.168, + "theoretical_loss": 3.402777849994818, + "tokens_seen": 2446852096 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002090752256770311, + "loss": 1.2777, + "theoretical_loss": 3.4027706100152475, + "tokens_seen": 2446917632 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002090591775325978, + "loss": 1.3528, + "theoretical_loss": 3.4027633702838767, + "tokens_seen": 2446983168 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002090431293881645, + "loss": 1.2697, + "theoretical_loss": 3.4027561308006895, + "tokens_seen": 2447048704 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002090270812437312, + "loss": 1.2678, + "theoretical_loss": 3.402748891565672, + "tokens_seen": 2447114240 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002090110330992979, + "loss": 1.2762, + "theoretical_loss": 3.4027416525788072, + "tokens_seen": 2447179776 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002089949849548646, + "loss": 1.2328, + "theoretical_loss": 3.402734413840082, + "tokens_seen": 2447245312 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002089789368104313, + "loss": 1.1153, + "theoretical_loss": 3.4027271753494794, + "tokens_seen": 2447310848 + }, + { + "epoch": 0.74, + "learning_rate": 0.000208962888665998, + "loss": 1.2687, + "theoretical_loss": 3.4027199371069856, + "tokens_seen": 2447376384 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002089468405215647, + "loss": 1.2612, + "theoretical_loss": 3.402712699112585, + "tokens_seen": 2447441920 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002089307923771314, + "loss": 1.2083, + "theoretical_loss": 3.4027054613662626, + "tokens_seen": 2447507456 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002089147442326981, + "loss": 1.2288, + "theoretical_loss": 3.402698223868003, + "tokens_seen": 2447572992 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002088986960882648, + "loss": 1.3482, + "theoretical_loss": 3.402690986617791, + "tokens_seen": 2447638528 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002088826479438315, + "loss": 1.23, + "theoretical_loss": 3.4026837496156115, + "tokens_seen": 2447704064 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 818155, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3235154151916504, + "objective/train/theoretical_loss": 3.4026765128614493, + "objective/train/tokens_used": 2468229600, + "theoretical_loss": 3.4026765128614493, + "tokens_seen": 2447769600 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002088665997993982, + "loss": 1.3005, + "theoretical_loss": 3.4026765128614493, + "tokens_seen": 2447769600 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002088505516549649, + "loss": 1.2108, + "theoretical_loss": 3.40266927635529, + "tokens_seen": 2447835136 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002088345035105316, + "loss": 1.1519, + "theoretical_loss": 3.4026620400971175, + "tokens_seen": 2447900672 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002088184553660983, + "loss": 1.2319, + "theoretical_loss": 3.402654804086917, + "tokens_seen": 2447966208 + }, + { + "epoch": 0.74, + "learning_rate": 0.000208802407221665, + "loss": 1.151, + "theoretical_loss": 3.4026475683246735, + "tokens_seen": 2448031744 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002087863590772317, + "loss": 1.1803, + "theoretical_loss": 3.402640332810372, + "tokens_seen": 2448097280 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020877031093279838, + "loss": 1.2636, + "theoretical_loss": 3.4026330975439967, + "tokens_seen": 2448162816 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020875426278836508, + "loss": 1.2013, + "theoretical_loss": 3.4026258625255332, + "tokens_seen": 2448228352 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020873821464393179, + "loss": 1.2461, + "theoretical_loss": 3.402618627754966, + "tokens_seen": 2448293888 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020872216649949849, + "loss": 1.2373, + "theoretical_loss": 3.40261139323228, + "tokens_seen": 2448359424 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020870611835506519, + "loss": 1.217, + "theoretical_loss": 3.4026041589574603, + "tokens_seen": 2448424960 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002086900702106319, + "loss": 1.3279, + "theoretical_loss": 3.4025969249304913, + "tokens_seen": 2448490496 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002086740220661986, + "loss": 1.2907, + "theoretical_loss": 3.4025896911513587, + "tokens_seen": 2448556032 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002086579739217653, + "loss": 1.1812, + "theoretical_loss": 3.4025824576200465, + "tokens_seen": 2448621568 + }, + { + "epoch": 0.74, + "learning_rate": 0.000208641925777332, + "loss": 1.2431, + "theoretical_loss": 3.4025752243365397, + "tokens_seen": 2448687104 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002086258776328987, + "loss": 1.2983, + "theoretical_loss": 3.4025679913008235, + "tokens_seen": 2448752640 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002086098294884654, + "loss": 1.3082, + "theoretical_loss": 3.402560758512883, + "tokens_seen": 2448818176 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002085937813440321, + "loss": 1.1862, + "theoretical_loss": 3.4025535259727024, + "tokens_seen": 2448883712 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002085777331995988, + "loss": 1.2106, + "theoretical_loss": 3.4025462936802673, + "tokens_seen": 2448949248 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002085616850551655, + "loss": 1.2551, + "theoretical_loss": 3.402539061635562, + "tokens_seen": 2449014784 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002085456369107322, + "loss": 1.2744, + "theoretical_loss": 3.402531829838572, + "tokens_seen": 2449080320 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002085295887662989, + "loss": 1.0943, + "theoretical_loss": 3.402524598289281, + "tokens_seen": 2449145856 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002085135406218656, + "loss": 1.2161, + "theoretical_loss": 3.4025173669876754, + "tokens_seen": 2449211392 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002084974924774323, + "loss": 1.2318, + "theoretical_loss": 3.402510135933739, + "tokens_seen": 2449276928 + }, + { + "epoch": 0.74, + "learning_rate": 0.000208481444332999, + "loss": 1.2265, + "theoretical_loss": 3.402502905127457, + "tokens_seen": 2449342464 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002084653961885657, + "loss": 1.2588, + "theoretical_loss": 3.402495674568815, + "tokens_seen": 2449408000 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002084493480441324, + "loss": 1.2115, + "theoretical_loss": 3.4024884442577967, + "tokens_seen": 2449473536 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002084332998996991, + "loss": 1.2941, + "theoretical_loss": 3.4024812141943874, + "tokens_seen": 2449539072 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002084172517552658, + "loss": 1.1251, + "theoretical_loss": 3.4024739843785725, + "tokens_seen": 2449604608 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020840120361083253, + "loss": 1.2314, + "theoretical_loss": 3.4024667548103364, + "tokens_seen": 2449670144 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020838515546639923, + "loss": 1.3059, + "theoretical_loss": 3.4024595254896646, + "tokens_seen": 2449735680 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020836910732196593, + "loss": 1.2533, + "theoretical_loss": 3.402452296416541, + "tokens_seen": 2449801216 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020835305917753263, + "loss": 1.2457, + "theoretical_loss": 3.4024450675909512, + "tokens_seen": 2449866752 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020833701103309933, + "loss": 1.2098, + "theoretical_loss": 3.4024378390128796, + "tokens_seen": 2449932288 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020832096288866603, + "loss": 1.1117, + "theoretical_loss": 3.402430610682312, + "tokens_seen": 2449997824 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020830491474423273, + "loss": 1.1781, + "theoretical_loss": 3.4024233825992325, + "tokens_seen": 2450063360 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020828886659979943, + "loss": 1.1087, + "theoretical_loss": 3.4024161547636265, + "tokens_seen": 2450128896 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020827281845536613, + "loss": 1.2669, + "theoretical_loss": 3.4024089271754785, + "tokens_seen": 2450194432 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020825677031093283, + "loss": 1.2313, + "theoretical_loss": 3.4024016998347735, + "tokens_seen": 2450259968 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020824072216649953, + "loss": 1.1969, + "theoretical_loss": 3.4023944727414968, + "tokens_seen": 2450325504 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020822467402206623, + "loss": 1.1854, + "theoretical_loss": 3.4023872458956324, + "tokens_seen": 2450391040 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020820862587763293, + "loss": 1.1056, + "theoretical_loss": 3.402380019297166, + "tokens_seen": 2450456576 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020819257773319963, + "loss": 1.2405, + "theoretical_loss": 3.4023727929460827, + "tokens_seen": 2450522112 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020817652958876633, + "loss": 1.0622, + "theoretical_loss": 3.402365566842367, + "tokens_seen": 2450587648 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020816048144433304, + "loss": 1.2725, + "theoretical_loss": 3.4023583409860034, + "tokens_seen": 2450653184 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002081444332998997, + "loss": 1.2608, + "theoretical_loss": 3.4023511153769777, + "tokens_seen": 2450718720 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002081283851554664, + "loss": 1.2369, + "theoretical_loss": 3.4023438900152745, + "tokens_seen": 2450784256 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002081123370110331, + "loss": 1.1828, + "theoretical_loss": 3.4023366649008784, + "tokens_seen": 2450849792 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002080962888665998, + "loss": 1.2195, + "theoretical_loss": 3.4023294400337747, + "tokens_seen": 2450915328 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002080802407221665, + "loss": 1.1741, + "theoretical_loss": 3.402322215413948, + "tokens_seen": 2450980864 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 819116, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2947700023651123, + "objective/train/theoretical_loss": 3.4023149910413832, + "objective/train/tokens_used": 2471506400, + "theoretical_loss": 3.4023149910413832, + "tokens_seen": 2451046400 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002080641925777332, + "loss": 1.2673, + "theoretical_loss": 3.4023149910413832, + "tokens_seen": 2451046400 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002080481444332999, + "loss": 1.2936, + "theoretical_loss": 3.402307766916066, + "tokens_seen": 2451111936 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002080320962888666, + "loss": 1.1986, + "theoretical_loss": 3.4023005430379802, + "tokens_seen": 2451177472 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020801604814443331, + "loss": 1.1496, + "theoretical_loss": 3.4022933194071117, + "tokens_seen": 2451243008 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020800000000000001, + "loss": 1.2418, + "theoretical_loss": 3.402286096023445, + "tokens_seen": 2451308544 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020798395185556671, + "loss": 1.098, + "theoretical_loss": 3.4022788728869644, + "tokens_seen": 2451374080 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020796790371113342, + "loss": 1.2987, + "theoretical_loss": 3.402271649997656, + "tokens_seen": 2451439616 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020795185556670012, + "loss": 1.2791, + "theoretical_loss": 3.4022644273555045, + "tokens_seen": 2451505152 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020793580742226682, + "loss": 1.3005, + "theoretical_loss": 3.402257204960494, + "tokens_seen": 2451570688 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020791975927783352, + "loss": 1.2636, + "theoretical_loss": 3.4022499828126103, + "tokens_seen": 2451636224 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020790371113340022, + "loss": 1.201, + "theoretical_loss": 3.4022427609118377, + "tokens_seen": 2451701760 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020788766298896692, + "loss": 1.142, + "theoretical_loss": 3.402235539258162, + "tokens_seen": 2451767296 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020787161484453362, + "loss": 1.163, + "theoretical_loss": 3.402228317851567, + "tokens_seen": 2451832832 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020785556670010032, + "loss": 1.2044, + "theoretical_loss": 3.4022210966920383, + "tokens_seen": 2451898368 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020783951855566702, + "loss": 1.2662, + "theoretical_loss": 3.402213875779561, + "tokens_seen": 2451963904 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020782347041123372, + "loss": 1.2094, + "theoretical_loss": 3.40220665511412, + "tokens_seen": 2452029440 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020780742226680042, + "loss": 1.2085, + "theoretical_loss": 3.4021994346957, + "tokens_seen": 2452094976 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020779137412236712, + "loss": 1.2771, + "theoretical_loss": 3.4021922145242858, + "tokens_seen": 2452160512 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020777532597793382, + "loss": 1.258, + "theoretical_loss": 3.4021849945998626, + "tokens_seen": 2452226048 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002077592778335005, + "loss": 1.1628, + "theoretical_loss": 3.4021777749224156, + "tokens_seen": 2452291584 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002077432296890672, + "loss": 1.2177, + "theoretical_loss": 3.4021705554919293, + "tokens_seen": 2452357120 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002077271815446339, + "loss": 1.2857, + "theoretical_loss": 3.4021633363083885, + "tokens_seen": 2452422656 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002077111334002006, + "loss": 1.2425, + "theoretical_loss": 3.4021561173717787, + "tokens_seen": 2452488192 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002076950852557673, + "loss": 1.1928, + "theoretical_loss": 3.402148898682085, + "tokens_seen": 2452553728 + }, + { + "epoch": 0.74, + "learning_rate": 0.000207679037111334, + "loss": 1.2478, + "theoretical_loss": 3.402141680239292, + "tokens_seen": 2452619264 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002076629889669007, + "loss": 1.164, + "theoretical_loss": 3.402134462043384, + "tokens_seen": 2452684800 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002076469408224674, + "loss": 1.2529, + "theoretical_loss": 3.402127244094347, + "tokens_seen": 2452750336 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002076308926780341, + "loss": 1.1733, + "theoretical_loss": 3.402120026392166, + "tokens_seen": 2452815872 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002076148445336008, + "loss": 1.2298, + "theoretical_loss": 3.402112808936825, + "tokens_seen": 2452881408 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002075987963891675, + "loss": 1.2405, + "theoretical_loss": 3.4021055917283096, + "tokens_seen": 2452946944 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002075827482447342, + "loss": 1.1828, + "theoretical_loss": 3.402098374766605, + "tokens_seen": 2453012480 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002075667001003009, + "loss": 1.2505, + "theoretical_loss": 3.4020911580516953, + "tokens_seen": 2453078016 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002075506519558676, + "loss": 1.1903, + "theoretical_loss": 3.402083941583566, + "tokens_seen": 2453143552 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002075346038114343, + "loss": 1.1833, + "theoretical_loss": 3.4020767253622024, + "tokens_seen": 2453209088 + }, + { + "epoch": 0.74, + "learning_rate": 0.000207518555667001, + "loss": 1.2091, + "theoretical_loss": 3.402069509387589, + "tokens_seen": 2453274624 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002075025075225677, + "loss": 1.2554, + "theoretical_loss": 3.402062293659711, + "tokens_seen": 2453340160 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002074864593781344, + "loss": 1.1478, + "theoretical_loss": 3.4020550781785532, + "tokens_seen": 2453405696 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002074704112337011, + "loss": 1.2385, + "theoretical_loss": 3.4020478629441007, + "tokens_seen": 2453471232 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002074543630892678, + "loss": 1.2135, + "theoretical_loss": 3.4020406479563383, + "tokens_seen": 2453536768 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002074383149448345, + "loss": 1.2746, + "theoretical_loss": 3.4020334332152515, + "tokens_seen": 2453602304 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002074222668004012, + "loss": 1.2242, + "theoretical_loss": 3.402026218720825, + "tokens_seen": 2453667840 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002074062186559679, + "loss": 1.235, + "theoretical_loss": 3.4020190044730425, + "tokens_seen": 2453733376 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002073901705115346, + "loss": 1.1238, + "theoretical_loss": 3.4020117904718914, + "tokens_seen": 2453798912 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020737412236710128, + "loss": 1.2187, + "theoretical_loss": 3.4020045767173546, + "tokens_seen": 2453864448 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020735807422266799, + "loss": 1.2126, + "theoretical_loss": 3.4019973632094183, + "tokens_seen": 2453929984 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020734202607823469, + "loss": 1.2886, + "theoretical_loss": 3.4019901499480674, + "tokens_seen": 2453995520 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002073259779338014, + "loss": 1.198, + "theoretical_loss": 3.401982936933286, + "tokens_seen": 2454061056 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002073099297893681, + "loss": 1.3155, + "theoretical_loss": 3.40197572416506, + "tokens_seen": 2454126592 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002072938816449348, + "loss": 1.2259, + "theoretical_loss": 3.401968511643374, + "tokens_seen": 2454192128 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002072778335005015, + "loss": 1.3032, + "theoretical_loss": 3.401961299368213, + "tokens_seen": 2454257664 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 820455, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.244768738746643, + "objective/train/theoretical_loss": 3.401954087339562, + "objective/train/tokens_used": 2474783200, + "theoretical_loss": 3.401954087339562, + "tokens_seen": 2454323200 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002072617853560682, + "loss": 1.208, + "theoretical_loss": 3.401954087339562, + "tokens_seen": 2454323200 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002072457372116349, + "loss": 1.2968, + "theoretical_loss": 3.4019468755574063, + "tokens_seen": 2454388736 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002072296890672016, + "loss": 1.2244, + "theoretical_loss": 3.4019396640217305, + "tokens_seen": 2454454272 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002072136409227683, + "loss": 1.2645, + "theoretical_loss": 3.4019324527325194, + "tokens_seen": 2454519808 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020719759277833505, + "loss": 1.2233, + "theoretical_loss": 3.4019252416897587, + "tokens_seen": 2454585344 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020718154463390175, + "loss": 1.2393, + "theoretical_loss": 3.401918030893433, + "tokens_seen": 2454650880 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020716549648946845, + "loss": 1.2546, + "theoretical_loss": 3.401910820343527, + "tokens_seen": 2454716416 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020714944834503515, + "loss": 1.2543, + "theoretical_loss": 3.401903610040026, + "tokens_seen": 2454781952 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020713340020060182, + "loss": 1.2841, + "theoretical_loss": 3.4018963999829155, + "tokens_seen": 2454847488 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020711735205616852, + "loss": 1.279, + "theoretical_loss": 3.4018891901721795, + "tokens_seen": 2454913024 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020710130391173522, + "loss": 1.2405, + "theoretical_loss": 3.401881980607804, + "tokens_seen": 2454978560 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020708525576730192, + "loss": 1.3013, + "theoretical_loss": 3.4018747712897732, + "tokens_seen": 2455044096 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020706920762286862, + "loss": 1.2081, + "theoretical_loss": 3.4018675622180723, + "tokens_seen": 2455109632 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020705315947843532, + "loss": 1.3198, + "theoretical_loss": 3.4018603533926868, + "tokens_seen": 2455175168 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020703711133400203, + "loss": 1.3187, + "theoretical_loss": 3.401853144813601, + "tokens_seen": 2455240704 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020702106318956873, + "loss": 1.3014, + "theoretical_loss": 3.4018459364808007, + "tokens_seen": 2455306240 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020700501504513543, + "loss": 1.0781, + "theoretical_loss": 3.4018387283942704, + "tokens_seen": 2455371776 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020698896690070213, + "loss": 1.1851, + "theoretical_loss": 3.401831520553995, + "tokens_seen": 2455437312 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020697291875626883, + "loss": 1.2065, + "theoretical_loss": 3.4018243129599597, + "tokens_seen": 2455502848 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020695687061183553, + "loss": 1.2741, + "theoretical_loss": 3.4018171056121496, + "tokens_seen": 2455568384 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020694082246740223, + "loss": 1.2952, + "theoretical_loss": 3.4018098985105496, + "tokens_seen": 2455633920 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020692477432296893, + "loss": 1.1389, + "theoretical_loss": 3.4018026916551447, + "tokens_seen": 2455699456 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020690872617853563, + "loss": 1.2579, + "theoretical_loss": 3.4017954850459198, + "tokens_seen": 2455764992 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020689267803410233, + "loss": 1.2653, + "theoretical_loss": 3.4017882786828606, + "tokens_seen": 2455830528 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020687662988966903, + "loss": 1.2538, + "theoretical_loss": 3.401781072565951, + "tokens_seen": 2455896064 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020686058174523573, + "loss": 1.2709, + "theoretical_loss": 3.4017738666951773, + "tokens_seen": 2455961600 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020684453360080243, + "loss": 1.2868, + "theoretical_loss": 3.4017666610705235, + "tokens_seen": 2456027136 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020682848545636913, + "loss": 1.2406, + "theoretical_loss": 3.401759455691975, + "tokens_seen": 2456092672 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020681243731193583, + "loss": 1.1935, + "theoretical_loss": 3.401752250559517, + "tokens_seen": 2456158208 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020679638916750253, + "loss": 1.3005, + "theoretical_loss": 3.401745045673134, + "tokens_seen": 2456223744 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020678034102306924, + "loss": 1.2398, + "theoretical_loss": 3.4017378410328116, + "tokens_seen": 2456289280 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020676429287863594, + "loss": 1.2245, + "theoretical_loss": 3.401730636638535, + "tokens_seen": 2456354816 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002067482447342026, + "loss": 1.213, + "theoretical_loss": 3.401723432490288, + "tokens_seen": 2456420352 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002067321965897693, + "loss": 1.2237, + "theoretical_loss": 3.4017162285880573, + "tokens_seen": 2456485888 + }, + { + "epoch": 0.74, + "learning_rate": 0.000206716148445336, + "loss": 1.3082, + "theoretical_loss": 3.4017090249318267, + "tokens_seen": 2456551424 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002067001003009027, + "loss": 1.2062, + "theoretical_loss": 3.4017018215215815, + "tokens_seen": 2456616960 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002066840521564694, + "loss": 1.1916, + "theoretical_loss": 3.4016946183573076, + "tokens_seen": 2456682496 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002066680040120361, + "loss": 1.1984, + "theoretical_loss": 3.4016874154389884, + "tokens_seen": 2456748032 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002066519558676028, + "loss": 1.2414, + "theoretical_loss": 3.4016802127666104, + "tokens_seen": 2456813568 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020663590772316951, + "loss": 1.1979, + "theoretical_loss": 3.4016730103401582, + "tokens_seen": 2456879104 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020661985957873621, + "loss": 1.2383, + "theoretical_loss": 3.4016658081596165, + "tokens_seen": 2456944640 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020660381143430292, + "loss": 1.2672, + "theoretical_loss": 3.401658606224971, + "tokens_seen": 2457010176 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020658776328986962, + "loss": 1.1914, + "theoretical_loss": 3.4016514045362056, + "tokens_seen": 2457075712 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020657171514543632, + "loss": 1.2629, + "theoretical_loss": 3.4016442030933067, + "tokens_seen": 2457141248 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020655566700100302, + "loss": 1.1803, + "theoretical_loss": 3.4016370018962587, + "tokens_seen": 2457206784 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020653961885656972, + "loss": 1.3264, + "theoretical_loss": 3.4016298009450465, + "tokens_seen": 2457272320 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020652357071213642, + "loss": 1.2146, + "theoretical_loss": 3.4016226002396555, + "tokens_seen": 2457337856 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020650752256770312, + "loss": 1.2644, + "theoretical_loss": 3.4016153997800704, + "tokens_seen": 2457403392 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020649147442326982, + "loss": 1.2561, + "theoretical_loss": 3.401608199566277, + "tokens_seen": 2457468928 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020647542627883652, + "loss": 1.1833, + "theoretical_loss": 3.4016009995982595, + "tokens_seen": 2457534464 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 821745, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2310079336166382, + "objective/train/theoretical_loss": 3.401593799876003, + "objective/train/tokens_used": 2478060000, + "theoretical_loss": 3.401593799876003, + "tokens_seen": 2457600000 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020645937813440322, + "loss": 1.1677, + "theoretical_loss": 3.401593799876003, + "tokens_seen": 2457600000 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020644332998996992, + "loss": 1.1647, + "theoretical_loss": 3.401586600399493, + "tokens_seen": 2457665536 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020642728184553662, + "loss": 1.3308, + "theoretical_loss": 3.401579401168714, + "tokens_seen": 2457731072 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020641123370110332, + "loss": 1.1624, + "theoretical_loss": 3.4015722021836523, + "tokens_seen": 2457796608 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020639518555667002, + "loss": 1.2597, + "theoretical_loss": 3.4015650034442917, + "tokens_seen": 2457862144 + }, + { + "epoch": 0.74, + "learning_rate": 0.00020637913741223672, + "loss": 1.3063, + "theoretical_loss": 3.4015578049506177, + "tokens_seen": 2457927680 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002063630892678034, + "loss": 1.224, + "theoretical_loss": 3.401550606702615, + "tokens_seen": 2457993216 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002063470411233701, + "loss": 1.1333, + "theoretical_loss": 3.4015434087002694, + "tokens_seen": 2458058752 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002063309929789368, + "loss": 1.3672, + "theoretical_loss": 3.4015362109435654, + "tokens_seen": 2458124288 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002063149448345035, + "loss": 1.1334, + "theoretical_loss": 3.401529013432488, + "tokens_seen": 2458189824 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002062988966900702, + "loss": 1.2159, + "theoretical_loss": 3.401521816167023, + "tokens_seen": 2458255360 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002062828485456369, + "loss": 1.2202, + "theoretical_loss": 3.4015146191471546, + "tokens_seen": 2458320896 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002062668004012036, + "loss": 1.1894, + "theoretical_loss": 3.4015074223728687, + "tokens_seen": 2458386432 + }, + { + "epoch": 0.74, + "learning_rate": 0.0002062507522567703, + "loss": 1.2767, + "theoretical_loss": 3.4015002258441496, + "tokens_seen": 2458451968 + }, + { + "epoch": 0.75, + "learning_rate": 0.000206234704112337, + "loss": 1.2532, + "theoretical_loss": 3.4014930295609824, + "tokens_seen": 2458517504 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002062186559679037, + "loss": 1.2043, + "theoretical_loss": 3.401485833523353, + "tokens_seen": 2458583040 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002062026078234704, + "loss": 1.23, + "theoretical_loss": 3.4014786377312456, + "tokens_seen": 2458648576 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002061865596790371, + "loss": 1.2265, + "theoretical_loss": 3.401471442184646, + "tokens_seen": 2458714112 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002061705115346038, + "loss": 1.2015, + "theoretical_loss": 3.4014642468835383, + "tokens_seen": 2458779648 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002061544633901705, + "loss": 1.24, + "theoretical_loss": 3.401457051827909, + "tokens_seen": 2458845184 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002061384152457372, + "loss": 1.1823, + "theoretical_loss": 3.4014498570177416, + "tokens_seen": 2458910720 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002061223671013039, + "loss": 1.2175, + "theoretical_loss": 3.4014426624530225, + "tokens_seen": 2458976256 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002061063189568706, + "loss": 1.2433, + "theoretical_loss": 3.401435468133736, + "tokens_seen": 2459041792 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002060902708124373, + "loss": 1.3019, + "theoretical_loss": 3.4014282740598674, + "tokens_seen": 2459107328 + }, + { + "epoch": 0.75, + "learning_rate": 0.000206074222668004, + "loss": 1.2126, + "theoretical_loss": 3.401421080231402, + "tokens_seen": 2459172864 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002060581745235707, + "loss": 1.268, + "theoretical_loss": 3.401413886648325, + "tokens_seen": 2459238400 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002060421263791374, + "loss": 1.2252, + "theoretical_loss": 3.40140669331062, + "tokens_seen": 2459303936 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002060260782347041, + "loss": 1.2317, + "theoretical_loss": 3.4013995002182744, + "tokens_seen": 2459369472 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002060100300902708, + "loss": 1.1872, + "theoretical_loss": 3.401392307371272, + "tokens_seen": 2459435008 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020599398194583754, + "loss": 1.2033, + "theoretical_loss": 3.401385114769598, + "tokens_seen": 2459500544 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020597793380140424, + "loss": 1.1539, + "theoretical_loss": 3.4013779224132374, + "tokens_seen": 2459566080 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020596188565697094, + "loss": 1.2724, + "theoretical_loss": 3.401370730302176, + "tokens_seen": 2459631616 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020594583751253764, + "loss": 1.332, + "theoretical_loss": 3.4013635384363976, + "tokens_seen": 2459697152 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020592978936810434, + "loss": 1.371, + "theoretical_loss": 3.4013563468158887, + "tokens_seen": 2459762688 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020591374122367104, + "loss": 1.1665, + "theoretical_loss": 3.4013491554406334, + "tokens_seen": 2459828224 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020589769307923774, + "loss": 1.2184, + "theoretical_loss": 3.4013419643106175, + "tokens_seen": 2459893760 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020588164493480444, + "loss": 1.2622, + "theoretical_loss": 3.4013347734258255, + "tokens_seen": 2459959296 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020586559679037114, + "loss": 1.2384, + "theoretical_loss": 3.401327582786243, + "tokens_seen": 2460024832 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020584954864593785, + "loss": 1.2298, + "theoretical_loss": 3.4013203923918547, + "tokens_seen": 2460090368 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020583350050150455, + "loss": 1.2958, + "theoretical_loss": 3.4013132022426458, + "tokens_seen": 2460155904 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020581745235707125, + "loss": 1.222, + "theoretical_loss": 3.4013060123386016, + "tokens_seen": 2460221440 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020580140421263795, + "loss": 1.1952, + "theoretical_loss": 3.401298822679707, + "tokens_seen": 2460286976 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020578535606820465, + "loss": 1.2134, + "theoretical_loss": 3.4012916332659477, + "tokens_seen": 2460352512 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020576930792377135, + "loss": 1.4222, + "theoretical_loss": 3.401284444097308, + "tokens_seen": 2460418048 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020575325977933805, + "loss": 1.3339, + "theoretical_loss": 3.4012772551737736, + "tokens_seen": 2460483584 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020573721163490475, + "loss": 1.1863, + "theoretical_loss": 3.4012700664953286, + "tokens_seen": 2460549120 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020572116349047142, + "loss": 1.1546, + "theoretical_loss": 3.40126287806196, + "tokens_seen": 2460614656 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020570511534603812, + "loss": 1.3119, + "theoretical_loss": 3.401255689873651, + "tokens_seen": 2460680192 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020568906720160482, + "loss": 1.2238, + "theoretical_loss": 3.4012485019303877, + "tokens_seen": 2460745728 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020567301905717153, + "loss": 1.2401, + "theoretical_loss": 3.401241314232155, + "tokens_seen": 2460811264 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 822717, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.288038730621338, + "objective/train/theoretical_loss": 3.401234126778938, + "objective/train/tokens_used": 2481336800, + "theoretical_loss": 3.401234126778938, + "tokens_seen": 2460876800 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020565697091273823, + "loss": 1.239, + "theoretical_loss": 3.401234126778938, + "tokens_seen": 2460876800 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020564092276830493, + "loss": 1.2188, + "theoretical_loss": 3.4012269395707224, + "tokens_seen": 2460942336 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020562487462387163, + "loss": 1.2133, + "theoretical_loss": 3.4012197526074925, + "tokens_seen": 2461007872 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020560882647943833, + "loss": 1.2739, + "theoretical_loss": 3.401212565889234, + "tokens_seen": 2461073408 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020559277833500503, + "loss": 1.2217, + "theoretical_loss": 3.401205379415931, + "tokens_seen": 2461138944 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020557673019057173, + "loss": 1.2564, + "theoretical_loss": 3.40119819318757, + "tokens_seen": 2461204480 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020556068204613843, + "loss": 1.2609, + "theoretical_loss": 3.4011910072041354, + "tokens_seen": 2461270016 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020554463390170513, + "loss": 1.349, + "theoretical_loss": 3.4011838214656125, + "tokens_seen": 2461335552 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020552858575727183, + "loss": 1.3105, + "theoretical_loss": 3.401176635971986, + "tokens_seen": 2461401088 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020551253761283853, + "loss": 1.1493, + "theoretical_loss": 3.4011694507232417, + "tokens_seen": 2461466624 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020549648946840523, + "loss": 1.2055, + "theoretical_loss": 3.4011622657193645, + "tokens_seen": 2461532160 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020548044132397193, + "loss": 1.2198, + "theoretical_loss": 3.40115508096034, + "tokens_seen": 2461597696 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020546439317953863, + "loss": 1.1566, + "theoretical_loss": 3.401147896446152, + "tokens_seen": 2461663232 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020544834503510533, + "loss": 1.2226, + "theoretical_loss": 3.4011407121767867, + "tokens_seen": 2461728768 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020543229689067203, + "loss": 1.1852, + "theoretical_loss": 3.4011335281522292, + "tokens_seen": 2461794304 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020541624874623874, + "loss": 1.2178, + "theoretical_loss": 3.4011263443724644, + "tokens_seen": 2461859840 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020540020060180544, + "loss": 1.2647, + "theoretical_loss": 3.401119160837477, + "tokens_seen": 2461925376 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020538415245737214, + "loss": 1.2907, + "theoretical_loss": 3.401111977547253, + "tokens_seen": 2461990912 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020536810431293884, + "loss": 1.2758, + "theoretical_loss": 3.4011047945017774, + "tokens_seen": 2462056448 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020535205616850554, + "loss": 1.2473, + "theoretical_loss": 3.401097611701035, + "tokens_seen": 2462121984 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002053360080240722, + "loss": 1.1714, + "theoretical_loss": 3.401090429145011, + "tokens_seen": 2462187520 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002053199598796389, + "loss": 1.2319, + "theoretical_loss": 3.4010832468336907, + "tokens_seen": 2462253056 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002053039117352056, + "loss": 1.2347, + "theoretical_loss": 3.4010760647670595, + "tokens_seen": 2462318592 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002052878635907723, + "loss": 1.2177, + "theoretical_loss": 3.4010688829451015, + "tokens_seen": 2462384128 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020527181544633901, + "loss": 1.1427, + "theoretical_loss": 3.4010617013678033, + "tokens_seen": 2462449664 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020525576730190571, + "loss": 1.2248, + "theoretical_loss": 3.4010545200351485, + "tokens_seen": 2462515200 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020523971915747242, + "loss": 1.2802, + "theoretical_loss": 3.401047338947124, + "tokens_seen": 2462580736 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020522367101303912, + "loss": 1.2035, + "theoretical_loss": 3.4010401581037133, + "tokens_seen": 2462646272 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020520762286860582, + "loss": 1.1558, + "theoretical_loss": 3.4010329775049026, + "tokens_seen": 2462711808 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020519157472417252, + "loss": 1.2238, + "theoretical_loss": 3.4010257971506768, + "tokens_seen": 2462777344 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020517552657973922, + "loss": 1.2378, + "theoretical_loss": 3.401018617041021, + "tokens_seen": 2462842880 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020515947843530592, + "loss": 1.2178, + "theoretical_loss": 3.4010114371759204, + "tokens_seen": 2462908416 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020514343029087262, + "loss": 1.2749, + "theoretical_loss": 3.40100425755536, + "tokens_seen": 2462973952 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020512738214643932, + "loss": 1.3236, + "theoretical_loss": 3.4009970781793255, + "tokens_seen": 2463039488 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020511133400200602, + "loss": 1.2713, + "theoretical_loss": 3.400989899047801, + "tokens_seen": 2463105024 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020509528585757272, + "loss": 1.1697, + "theoretical_loss": 3.4009827201607727, + "tokens_seen": 2463170560 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020507923771313942, + "loss": 1.1927, + "theoretical_loss": 3.4009755415182257, + "tokens_seen": 2463236096 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020506318956870612, + "loss": 1.2339, + "theoretical_loss": 3.4009683631201444, + "tokens_seen": 2463301632 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020504714142427282, + "loss": 1.268, + "theoretical_loss": 3.400961184966515, + "tokens_seen": 2463367168 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020503109327983952, + "loss": 1.1121, + "theoretical_loss": 3.400954007057322, + "tokens_seen": 2463432704 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020501504513540622, + "loss": 1.2205, + "theoretical_loss": 3.4009468293925504, + "tokens_seen": 2463498240 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020499899699097292, + "loss": 1.2604, + "theoretical_loss": 3.4009396519721857, + "tokens_seen": 2463563776 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020498294884653963, + "loss": 1.2815, + "theoretical_loss": 3.400932474796213, + "tokens_seen": 2463629312 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020496690070210633, + "loss": 1.314, + "theoretical_loss": 3.400925297864618, + "tokens_seen": 2463694848 + }, + { + "epoch": 0.75, + "learning_rate": 0.000204950852557673, + "loss": 1.265, + "theoretical_loss": 3.400918121177385, + "tokens_seen": 2463760384 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002049348044132397, + "loss": 1.2212, + "theoretical_loss": 3.4009109447344996, + "tokens_seen": 2463825920 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002049187562688064, + "loss": 1.3325, + "theoretical_loss": 3.4009037685359473, + "tokens_seen": 2463891456 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002049027081243731, + "loss": 1.23, + "theoretical_loss": 3.4008965925817125, + "tokens_seen": 2463956992 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002048866599799398, + "loss": 1.3348, + "theoretical_loss": 3.400889416871781, + "tokens_seen": 2464022528 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002048706118355065, + "loss": 1.2437, + "theoretical_loss": 3.400882241406138, + "tokens_seen": 2464088064 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 824064, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3477821350097656, + "objective/train/theoretical_loss": 3.4008750661847684, + "objective/train/tokens_used": 2484613600, + "theoretical_loss": 3.4008750661847684, + "tokens_seen": 2464153600 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002048545636910732, + "loss": 1.2699, + "theoretical_loss": 3.4008750661847684, + "tokens_seen": 2464153600 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002048385155466399, + "loss": 1.3319, + "theoretical_loss": 3.4008678912076573, + "tokens_seen": 2464219136 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002048224674022066, + "loss": 1.1806, + "theoretical_loss": 3.4008607164747904, + "tokens_seen": 2464284672 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002048064192577733, + "loss": 1.2831, + "theoretical_loss": 3.4008535419861525, + "tokens_seen": 2464350208 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020479037111334006, + "loss": 1.2485, + "theoretical_loss": 3.4008463677417287, + "tokens_seen": 2464415744 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020477432296890676, + "loss": 1.1256, + "theoretical_loss": 3.4008391937415046, + "tokens_seen": 2464481280 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020475827482447346, + "loss": 1.2369, + "theoretical_loss": 3.400832019985465, + "tokens_seen": 2464546816 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020474222668004016, + "loss": 1.231, + "theoretical_loss": 3.4008248464735953, + "tokens_seen": 2464612352 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020472617853560686, + "loss": 1.2005, + "theoretical_loss": 3.4008176732058804, + "tokens_seen": 2464677888 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020471013039117354, + "loss": 1.2592, + "theoretical_loss": 3.400810500182306, + "tokens_seen": 2464743424 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020469408224674024, + "loss": 1.3405, + "theoretical_loss": 3.4008033274028575, + "tokens_seen": 2464808960 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020467803410230694, + "loss": 1.2228, + "theoretical_loss": 3.400796154867519, + "tokens_seen": 2464874496 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020466198595787364, + "loss": 1.28, + "theoretical_loss": 3.4007889825762767, + "tokens_seen": 2464940032 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020464593781344034, + "loss": 1.2225, + "theoretical_loss": 3.400781810529115, + "tokens_seen": 2465005568 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020462988966900704, + "loss": 1.245, + "theoretical_loss": 3.40077463872602, + "tokens_seen": 2465071104 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020461384152457374, + "loss": 1.4661, + "theoretical_loss": 3.4007674671669763, + "tokens_seen": 2465136640 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020459779338014044, + "loss": 1.2308, + "theoretical_loss": 3.400760295851969, + "tokens_seen": 2465202176 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020458174523570714, + "loss": 1.2438, + "theoretical_loss": 3.400753124780984, + "tokens_seen": 2465267712 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020456569709127384, + "loss": 1.1934, + "theoretical_loss": 3.400745953954006, + "tokens_seen": 2465333248 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020454964894684054, + "loss": 1.2494, + "theoretical_loss": 3.40073878337102, + "tokens_seen": 2465398784 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020453360080240724, + "loss": 1.2292, + "theoretical_loss": 3.400731613032012, + "tokens_seen": 2465464320 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020451755265797394, + "loss": 1.3447, + "theoretical_loss": 3.4007244429369665, + "tokens_seen": 2465529856 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020450150451354064, + "loss": 1.1722, + "theoretical_loss": 3.400717273085869, + "tokens_seen": 2465595392 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020448545636910734, + "loss": 1.184, + "theoretical_loss": 3.4007101034787044, + "tokens_seen": 2465660928 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020446940822467405, + "loss": 1.2654, + "theoretical_loss": 3.4007029341154587, + "tokens_seen": 2465726464 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020445336008024075, + "loss": 1.3029, + "theoretical_loss": 3.400695764996116, + "tokens_seen": 2465792000 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020443731193580745, + "loss": 1.2573, + "theoretical_loss": 3.4006885961206623, + "tokens_seen": 2465857536 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020442126379137415, + "loss": 1.2895, + "theoretical_loss": 3.400681427489083, + "tokens_seen": 2465923072 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020440521564694085, + "loss": 1.1699, + "theoretical_loss": 3.4006742591013626, + "tokens_seen": 2465988608 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020438916750250755, + "loss": 1.1582, + "theoretical_loss": 3.4006670909574868, + "tokens_seen": 2466054144 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020437311935807425, + "loss": 1.2569, + "theoretical_loss": 3.4006599230574404, + "tokens_seen": 2466119680 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020435707121364095, + "loss": 1.18, + "theoretical_loss": 3.400652755401209, + "tokens_seen": 2466185216 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020434102306920765, + "loss": 1.3147, + "theoretical_loss": 3.4006455879887785, + "tokens_seen": 2466250752 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020432497492477432, + "loss": 1.246, + "theoretical_loss": 3.400638420820133, + "tokens_seen": 2466316288 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020430892678034102, + "loss": 1.1472, + "theoretical_loss": 3.4006312538952574, + "tokens_seen": 2466381824 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020429287863590773, + "loss": 1.3528, + "theoretical_loss": 3.4006240872141387, + "tokens_seen": 2466447360 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020427683049147443, + "loss": 1.2192, + "theoretical_loss": 3.4006169207767605, + "tokens_seen": 2466512896 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020426078234704113, + "loss": 1.3142, + "theoretical_loss": 3.4006097545831087, + "tokens_seen": 2466578432 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020424473420260783, + "loss": 1.2213, + "theoretical_loss": 3.4006025886331686, + "tokens_seen": 2466643968 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020422868605817453, + "loss": 1.2326, + "theoretical_loss": 3.400595422926925, + "tokens_seen": 2466709504 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020421263791374123, + "loss": 1.267, + "theoretical_loss": 3.4005882574643636, + "tokens_seen": 2466775040 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020419658976930793, + "loss": 1.2992, + "theoretical_loss": 3.40058109224547, + "tokens_seen": 2466840576 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020418054162487463, + "loss": 1.1801, + "theoretical_loss": 3.4005739272702282, + "tokens_seen": 2466906112 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020416449348044133, + "loss": 1.1433, + "theoretical_loss": 3.400566762538624, + "tokens_seen": 2466971648 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020414844533600803, + "loss": 1.2958, + "theoretical_loss": 3.400559598050643, + "tokens_seen": 2467037184 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020413239719157473, + "loss": 1.1669, + "theoretical_loss": 3.4005524338062707, + "tokens_seen": 2467102720 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020411634904714143, + "loss": 1.2069, + "theoretical_loss": 3.400545269805492, + "tokens_seen": 2467168256 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020410030090270813, + "loss": 1.2906, + "theoretical_loss": 3.400538106048291, + "tokens_seen": 2467233792 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020408425275827483, + "loss": 1.2204, + "theoretical_loss": 3.400530942534655, + "tokens_seen": 2467299328 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020406820461384153, + "loss": 1.2933, + "theoretical_loss": 3.4005237792645677, + "tokens_seen": 2467364864 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 825046, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1484394073486328, + "objective/train/theoretical_loss": 3.400516616238015, + "objective/train/tokens_used": 2487890400, + "theoretical_loss": 3.400516616238015, + "tokens_seen": 2467430400 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020405215646940823, + "loss": 1.177, + "theoretical_loss": 3.400516616238015, + "tokens_seen": 2467430400 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020403610832497494, + "loss": 1.2294, + "theoretical_loss": 3.4005094534549825, + "tokens_seen": 2467495936 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020402006018054164, + "loss": 1.1848, + "theoretical_loss": 3.4005022909154543, + "tokens_seen": 2467561472 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020400401203610834, + "loss": 1.1985, + "theoretical_loss": 3.4004951286194167, + "tokens_seen": 2467627008 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020398796389167504, + "loss": 1.2679, + "theoretical_loss": 3.400487966566854, + "tokens_seen": 2467692544 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020397191574724174, + "loss": 1.2474, + "theoretical_loss": 3.400480804757753, + "tokens_seen": 2467758080 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020395586760280844, + "loss": 1.2142, + "theoretical_loss": 3.4004736431920977, + "tokens_seen": 2467823616 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002039398194583751, + "loss": 1.2753, + "theoretical_loss": 3.4004664818698735, + "tokens_seen": 2467889152 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002039237713139418, + "loss": 1.1571, + "theoretical_loss": 3.400459320791066, + "tokens_seen": 2467954688 + }, + { + "epoch": 0.75, + "learning_rate": 0.0002039077231695085, + "loss": 1.2638, + "theoretical_loss": 3.4004521599556603, + "tokens_seen": 2468020224 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020389167502507521, + "loss": 1.2101, + "theoretical_loss": 3.4004449993636414, + "tokens_seen": 2468085760 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020387562688064191, + "loss": 1.2085, + "theoretical_loss": 3.400437839014995, + "tokens_seen": 2468151296 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020385957873620862, + "loss": 1.3225, + "theoretical_loss": 3.4004306789097063, + "tokens_seen": 2468216832 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020384353059177532, + "loss": 1.1514, + "theoretical_loss": 3.4004235190477603, + "tokens_seen": 2468282368 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020382748244734202, + "loss": 1.201, + "theoretical_loss": 3.4004163594291423, + "tokens_seen": 2468347904 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020381143430290872, + "loss": 1.2713, + "theoretical_loss": 3.400409200053838, + "tokens_seen": 2468413440 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020379538615847542, + "loss": 1.3468, + "theoretical_loss": 3.4004020409218323, + "tokens_seen": 2468478976 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020377933801404212, + "loss": 1.2268, + "theoretical_loss": 3.400394882033111, + "tokens_seen": 2468544512 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020376328986960882, + "loss": 1.2785, + "theoretical_loss": 3.4003877233876585, + "tokens_seen": 2468610048 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020374724172517552, + "loss": 1.2412, + "theoretical_loss": 3.4003805649854604, + "tokens_seen": 2468675584 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020373119358074222, + "loss": 1.2188, + "theoretical_loss": 3.400373406826502, + "tokens_seen": 2468741120 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020371514543630892, + "loss": 1.2024, + "theoretical_loss": 3.400366248910769, + "tokens_seen": 2468806656 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020369909729187562, + "loss": 1.2805, + "theoretical_loss": 3.4003590912382458, + "tokens_seen": 2468872192 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020368304914744232, + "loss": 1.2869, + "theoretical_loss": 3.4003519338089188, + "tokens_seen": 2468937728 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020366700100300902, + "loss": 1.2198, + "theoretical_loss": 3.4003447766227723, + "tokens_seen": 2469003264 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020365095285857572, + "loss": 1.1152, + "theoretical_loss": 3.4003376196797923, + "tokens_seen": 2469068800 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020363490471414242, + "loss": 1.2242, + "theoretical_loss": 3.400330462979963, + "tokens_seen": 2469134336 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020361885656970912, + "loss": 1.2391, + "theoretical_loss": 3.4003233065232714, + "tokens_seen": 2469199872 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020360280842527585, + "loss": 1.2412, + "theoretical_loss": 3.4003161503097017, + "tokens_seen": 2469265408 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020358676028084255, + "loss": 1.1966, + "theoretical_loss": 3.400308994339239, + "tokens_seen": 2469330944 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020357071213640925, + "loss": 1.2149, + "theoretical_loss": 3.400301838611869, + "tokens_seen": 2469396480 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020355466399197595, + "loss": 1.196, + "theoretical_loss": 3.400294683127577, + "tokens_seen": 2469462016 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020353861584754266, + "loss": 1.129, + "theoretical_loss": 3.400287527886348, + "tokens_seen": 2469527552 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020352256770310936, + "loss": 1.2521, + "theoretical_loss": 3.400280372888168, + "tokens_seen": 2469593088 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020350651955867606, + "loss": 1.2563, + "theoretical_loss": 3.400273218133021, + "tokens_seen": 2469658624 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020349047141424276, + "loss": 1.2738, + "theoretical_loss": 3.4002660636208937, + "tokens_seen": 2469724160 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020347442326980946, + "loss": 1.2064, + "theoretical_loss": 3.4002589093517703, + "tokens_seen": 2469789696 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020345837512537616, + "loss": 1.1881, + "theoretical_loss": 3.4002517553256366, + "tokens_seen": 2469855232 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020344232698094286, + "loss": 1.3745, + "theoretical_loss": 3.4002446015424783, + "tokens_seen": 2469920768 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020342627883650956, + "loss": 1.2314, + "theoretical_loss": 3.4002374480022803, + "tokens_seen": 2469986304 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020341023069207626, + "loss": 1.1338, + "theoretical_loss": 3.4002302947050276, + "tokens_seen": 2470051840 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020339418254764296, + "loss": 1.3465, + "theoretical_loss": 3.400223141650706, + "tokens_seen": 2470117376 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020337813440320966, + "loss": 1.217, + "theoretical_loss": 3.4002159888393004, + "tokens_seen": 2470182912 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020336208625877636, + "loss": 1.2215, + "theoretical_loss": 3.4002088362707963, + "tokens_seen": 2470248448 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020334603811434306, + "loss": 1.2032, + "theoretical_loss": 3.4002016839451787, + "tokens_seen": 2470313984 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020332998996990976, + "loss": 1.1877, + "theoretical_loss": 3.400194531862434, + "tokens_seen": 2470379520 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020331394182547646, + "loss": 1.163, + "theoretical_loss": 3.400187380022546, + "tokens_seen": 2470445056 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020329789368104314, + "loss": 1.348, + "theoretical_loss": 3.4001802284255014, + "tokens_seen": 2470510592 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020328184553660984, + "loss": 1.2937, + "theoretical_loss": 3.400173077071284, + "tokens_seen": 2470576128 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020326579739217654, + "loss": 1.2102, + "theoretical_loss": 3.4001659259598807, + "tokens_seen": 2470641664 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 826177, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2446273565292358, + "objective/train/theoretical_loss": 3.400158775091276, + "objective/train/tokens_used": 2491167200, + "theoretical_loss": 3.400158775091276, + "tokens_seen": 2470707200 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020324974924774324, + "loss": 1.224, + "theoretical_loss": 3.400158775091276, + "tokens_seen": 2470707200 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020323370110330994, + "loss": 1.2143, + "theoretical_loss": 3.4001516244654546, + "tokens_seen": 2470772736 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020321765295887664, + "loss": 1.1827, + "theoretical_loss": 3.400144474082403, + "tokens_seen": 2470838272 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020320160481444334, + "loss": 1.2172, + "theoretical_loss": 3.400137323942106, + "tokens_seen": 2470903808 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020318555667001004, + "loss": 1.2655, + "theoretical_loss": 3.4001301740445493, + "tokens_seen": 2470969344 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020316950852557674, + "loss": 1.2819, + "theoretical_loss": 3.4001230243897176, + "tokens_seen": 2471034880 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020315346038114344, + "loss": 1.2298, + "theoretical_loss": 3.400115874977596, + "tokens_seen": 2471100416 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020313741223671014, + "loss": 1.2408, + "theoretical_loss": 3.400108725808171, + "tokens_seen": 2471165952 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020312136409227684, + "loss": 1.2579, + "theoretical_loss": 3.400101576881427, + "tokens_seen": 2471231488 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020310531594784355, + "loss": 1.1748, + "theoretical_loss": 3.4000944281973493, + "tokens_seen": 2471297024 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020308926780341025, + "loss": 1.2703, + "theoretical_loss": 3.400087279755924, + "tokens_seen": 2471362560 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020307321965897695, + "loss": 1.2316, + "theoretical_loss": 3.4000801315571354, + "tokens_seen": 2471428096 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020305717151454365, + "loss": 1.23, + "theoretical_loss": 3.4000729836009698, + "tokens_seen": 2471493632 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020304112337011035, + "loss": 1.1598, + "theoretical_loss": 3.400065835887412, + "tokens_seen": 2471559168 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020302507522567705, + "loss": 1.1848, + "theoretical_loss": 3.400058688416447, + "tokens_seen": 2471624704 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020300902708124375, + "loss": 1.1619, + "theoretical_loss": 3.400051541188061, + "tokens_seen": 2471690240 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020299297893681045, + "loss": 1.2958, + "theoretical_loss": 3.4000443942022387, + "tokens_seen": 2471755776 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020297693079237715, + "loss": 1.1878, + "theoretical_loss": 3.4000372474589655, + "tokens_seen": 2471821312 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020296088264794385, + "loss": 1.1028, + "theoretical_loss": 3.400030100958227, + "tokens_seen": 2471886848 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020294483450351055, + "loss": 1.1892, + "theoretical_loss": 3.4000229547000087, + "tokens_seen": 2471952384 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020292878635907725, + "loss": 1.2539, + "theoretical_loss": 3.4000158086842953, + "tokens_seen": 2472017920 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020291273821464393, + "loss": 1.2367, + "theoretical_loss": 3.400008662911073, + "tokens_seen": 2472083456 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020289669007021063, + "loss": 1.1866, + "theoretical_loss": 3.4000015173803257, + "tokens_seen": 2472148992 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020288064192577733, + "loss": 1.2055, + "theoretical_loss": 3.3999943720920403, + "tokens_seen": 2472214528 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020286459378134403, + "loss": 1.3016, + "theoretical_loss": 3.3999872270462017, + "tokens_seen": 2472280064 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020284854563691073, + "loss": 1.1554, + "theoretical_loss": 3.3999800822427946, + "tokens_seen": 2472345600 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020283249749247743, + "loss": 1.3193, + "theoretical_loss": 3.399972937681805, + "tokens_seen": 2472411136 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020281644934804413, + "loss": 1.2168, + "theoretical_loss": 3.399965793363218, + "tokens_seen": 2472476672 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020280040120361083, + "loss": 1.2385, + "theoretical_loss": 3.399958649287019, + "tokens_seen": 2472542208 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020278435305917753, + "loss": 1.2101, + "theoretical_loss": 3.3999515054531937, + "tokens_seen": 2472607744 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020276830491474423, + "loss": 1.1638, + "theoretical_loss": 3.399944361861727, + "tokens_seen": 2472673280 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020275225677031093, + "loss": 1.2297, + "theoretical_loss": 3.399937218512604, + "tokens_seen": 2472738816 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020273620862587763, + "loss": 1.1639, + "theoretical_loss": 3.399930075405811, + "tokens_seen": 2472804352 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020272016048144433, + "loss": 1.0829, + "theoretical_loss": 3.3999229325413323, + "tokens_seen": 2472869888 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020270411233701103, + "loss": 1.2019, + "theoretical_loss": 3.3999157899191537, + "tokens_seen": 2472935424 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020268806419257773, + "loss": 1.2127, + "theoretical_loss": 3.399908647539261, + "tokens_seen": 2473000960 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020267201604814444, + "loss": 1.2104, + "theoretical_loss": 3.3999015054016386, + "tokens_seen": 2473066496 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020265596790371114, + "loss": 1.0949, + "theoretical_loss": 3.399894363506273, + "tokens_seen": 2473132032 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020263991975927784, + "loss": 1.3268, + "theoretical_loss": 3.399887221853149, + "tokens_seen": 2473197568 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020262387161484454, + "loss": 1.1291, + "theoretical_loss": 3.3998800804422515, + "tokens_seen": 2473263104 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020260782347041124, + "loss": 1.1697, + "theoretical_loss": 3.3998729392735667, + "tokens_seen": 2473328640 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020259177532597794, + "loss": 1.1637, + "theoretical_loss": 3.3998657983470792, + "tokens_seen": 2473394176 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020257572718154464, + "loss": 1.1975, + "theoretical_loss": 3.3998586576627754, + "tokens_seen": 2473459712 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020255967903711134, + "loss": 1.1958, + "theoretical_loss": 3.3998515172206396, + "tokens_seen": 2473525248 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020254363089267804, + "loss": 1.1177, + "theoretical_loss": 3.3998443770206577, + "tokens_seen": 2473590784 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020252758274824471, + "loss": 1.3374, + "theoretical_loss": 3.399837237062815, + "tokens_seen": 2473656320 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020251153460381141, + "loss": 1.2645, + "theoretical_loss": 3.3998300973470967, + "tokens_seen": 2473721856 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020249548645937812, + "loss": 1.299, + "theoretical_loss": 3.3998229578734884, + "tokens_seen": 2473787392 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020247943831494482, + "loss": 1.2866, + "theoretical_loss": 3.399815818641976, + "tokens_seen": 2473852928 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020246339017051152, + "loss": 1.3517, + "theoretical_loss": 3.3998086796525433, + "tokens_seen": 2473918464 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 827110, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.092146873474121, + "objective/train/theoretical_loss": 3.399801540905177, + "objective/train/tokens_used": 2494444000, + "theoretical_loss": 3.399801540905177, + "tokens_seen": 2473984000 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020244734202607822, + "loss": 1.1631, + "theoretical_loss": 3.399801540905177, + "tokens_seen": 2473984000 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020243129388164492, + "loss": 1.3172, + "theoretical_loss": 3.3997944023998623, + "tokens_seen": 2474049536 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020241524573721162, + "loss": 1.2592, + "theoretical_loss": 3.399787264136584, + "tokens_seen": 2474115072 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020239919759277837, + "loss": 1.3061, + "theoretical_loss": 3.399780126115328, + "tokens_seen": 2474180608 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020238314944834507, + "loss": 1.2104, + "theoretical_loss": 3.39977298833608, + "tokens_seen": 2474246144 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020236710130391177, + "loss": 1.3113, + "theoretical_loss": 3.399765850798825, + "tokens_seen": 2474311680 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020235105315947848, + "loss": 1.1805, + "theoretical_loss": 3.399758713503548, + "tokens_seen": 2474377216 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020233500501504518, + "loss": 1.2795, + "theoretical_loss": 3.399751576450235, + "tokens_seen": 2474442752 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020231895687061188, + "loss": 1.1756, + "theoretical_loss": 3.3997444396388707, + "tokens_seen": 2474508288 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020230290872617858, + "loss": 1.277, + "theoretical_loss": 3.3997373030694416, + "tokens_seen": 2474573824 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020228686058174525, + "loss": 1.1854, + "theoretical_loss": 3.399730166741932, + "tokens_seen": 2474639360 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020227081243731195, + "loss": 1.2728, + "theoretical_loss": 3.3997230306563275, + "tokens_seen": 2474704896 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020225476429287865, + "loss": 1.2201, + "theoretical_loss": 3.3997158948126143, + "tokens_seen": 2474770432 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020223871614844535, + "loss": 1.1787, + "theoretical_loss": 3.3997087592107764, + "tokens_seen": 2474835968 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020222266800401205, + "loss": 1.4076, + "theoretical_loss": 3.3997016238508007, + "tokens_seen": 2474901504 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020220661985957875, + "loss": 1.2577, + "theoretical_loss": 3.3996944887326714, + "tokens_seen": 2474967040 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020219057171514545, + "loss": 1.2906, + "theoretical_loss": 3.3996873538563745, + "tokens_seen": 2475032576 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020217452357071215, + "loss": 1.1519, + "theoretical_loss": 3.399680219221896, + "tokens_seen": 2475098112 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020215847542627886, + "loss": 1.2329, + "theoretical_loss": 3.3996730848292196, + "tokens_seen": 2475163648 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020214242728184556, + "loss": 1.2425, + "theoretical_loss": 3.399665950678332, + "tokens_seen": 2475229184 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020212637913741226, + "loss": 1.3295, + "theoretical_loss": 3.3996588167692186, + "tokens_seen": 2475294720 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020211033099297896, + "loss": 1.2554, + "theoretical_loss": 3.3996516831018644, + "tokens_seen": 2475360256 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020209428284854566, + "loss": 1.1474, + "theoretical_loss": 3.3996445496762546, + "tokens_seen": 2475425792 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020207823470411236, + "loss": 1.2736, + "theoretical_loss": 3.399637416492375, + "tokens_seen": 2475491328 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020206218655967906, + "loss": 1.115, + "theoretical_loss": 3.399630283550211, + "tokens_seen": 2475556864 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020204613841524576, + "loss": 1.092, + "theoretical_loss": 3.3996231508497483, + "tokens_seen": 2475622400 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020203009027081246, + "loss": 1.2549, + "theoretical_loss": 3.3996160183909714, + "tokens_seen": 2475687936 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020201404212637916, + "loss": 1.2399, + "theoretical_loss": 3.399608886173867, + "tokens_seen": 2475753472 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020199799398194586, + "loss": 1.2441, + "theoretical_loss": 3.399601754198419, + "tokens_seen": 2475819008 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020198194583751256, + "loss": 1.2462, + "theoretical_loss": 3.399594622464614, + "tokens_seen": 2475884544 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020196589769307926, + "loss": 1.1227, + "theoretical_loss": 3.399587490972437, + "tokens_seen": 2475950080 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020194984954864596, + "loss": 1.3241, + "theoretical_loss": 3.399580359721873, + "tokens_seen": 2476015616 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020193380140421266, + "loss": 1.2855, + "theoretical_loss": 3.3995732287129083, + "tokens_seen": 2476081152 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020191775325977937, + "loss": 1.2714, + "theoretical_loss": 3.3995660979455278, + "tokens_seen": 2476146688 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020190170511534604, + "loss": 1.194, + "theoretical_loss": 3.3995589674197166, + "tokens_seen": 2476212224 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020188565697091274, + "loss": 1.3153, + "theoretical_loss": 3.399551837135461, + "tokens_seen": 2476277760 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020186960882647944, + "loss": 1.2652, + "theoretical_loss": 3.399544707092746, + "tokens_seen": 2476343296 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020185356068204614, + "loss": 1.1842, + "theoretical_loss": 3.3995375772915564, + "tokens_seen": 2476408832 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020183751253761284, + "loss": 1.2272, + "theoretical_loss": 3.3995304477318786, + "tokens_seen": 2476474368 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020182146439317954, + "loss": 1.1926, + "theoretical_loss": 3.3995233184136975, + "tokens_seen": 2476539904 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020180541624874624, + "loss": 1.1573, + "theoretical_loss": 3.3995161893369987, + "tokens_seen": 2476605440 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020178936810431294, + "loss": 1.2365, + "theoretical_loss": 3.3995090605017677, + "tokens_seen": 2476670976 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020177331995987964, + "loss": 1.2127, + "theoretical_loss": 3.3995019319079898, + "tokens_seen": 2476736512 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020175727181544634, + "loss": 1.1979, + "theoretical_loss": 3.3994948035556503, + "tokens_seen": 2476802048 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020174122367101304, + "loss": 1.2216, + "theoretical_loss": 3.399487675444735, + "tokens_seen": 2476867584 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020172517552657975, + "loss": 1.2472, + "theoretical_loss": 3.3994805475752288, + "tokens_seen": 2476933120 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020170912738214645, + "loss": 1.2953, + "theoretical_loss": 3.3994734199471175, + "tokens_seen": 2476998656 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020169307923771315, + "loss": 1.2852, + "theoretical_loss": 3.399466292560387, + "tokens_seen": 2477064192 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020167703109327985, + "loss": 1.1975, + "theoretical_loss": 3.3994591654150215, + "tokens_seen": 2477129728 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020166098294884655, + "loss": 1.1297, + "theoretical_loss": 3.3994520385110074, + "tokens_seen": 2477195264 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 828337, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0806145668029785, + "objective/train/theoretical_loss": 3.39944491184833, + "objective/train/tokens_used": 2497720800, + "theoretical_loss": 3.39944491184833, + "tokens_seen": 2477260800 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020164493480441325, + "loss": 1.1918, + "theoretical_loss": 3.39944491184833, + "tokens_seen": 2477260800 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020162888665997995, + "loss": 1.1674, + "theoretical_loss": 3.399437785426975, + "tokens_seen": 2477326336 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020161283851554665, + "loss": 1.1511, + "theoretical_loss": 3.399430659246927, + "tokens_seen": 2477391872 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020159679037111335, + "loss": 1.2589, + "theoretical_loss": 3.3994235333081724, + "tokens_seen": 2477457408 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020158074222668005, + "loss": 1.2159, + "theoretical_loss": 3.3994164076106954, + "tokens_seen": 2477522944 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020156469408224675, + "loss": 1.2536, + "theoretical_loss": 3.3994092821544832, + "tokens_seen": 2477588480 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020154864593781345, + "loss": 1.3516, + "theoretical_loss": 3.3994021569395194, + "tokens_seen": 2477654016 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020153259779338015, + "loss": 1.3962, + "theoretical_loss": 3.399395031965791, + "tokens_seen": 2477719552 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020151654964894683, + "loss": 1.1767, + "theoretical_loss": 3.3993879072332827, + "tokens_seen": 2477785088 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020150050150451353, + "loss": 1.3202, + "theoretical_loss": 3.39938078274198, + "tokens_seen": 2477850624 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020148445336008023, + "loss": 1.183, + "theoretical_loss": 3.399373658491868, + "tokens_seen": 2477916160 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020146840521564693, + "loss": 1.1596, + "theoretical_loss": 3.399366534482933, + "tokens_seen": 2477981696 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020145235707121363, + "loss": 1.2108, + "theoretical_loss": 3.39935941071516, + "tokens_seen": 2478047232 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020143630892678033, + "loss": 1.1791, + "theoretical_loss": 3.3993522871885347, + "tokens_seen": 2478112768 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020142026078234703, + "loss": 1.2114, + "theoretical_loss": 3.3993451639030416, + "tokens_seen": 2478178304 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020140421263791373, + "loss": 1.1557, + "theoretical_loss": 3.3993380408586678, + "tokens_seen": 2478243840 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020138816449348043, + "loss": 1.2622, + "theoretical_loss": 3.399330918055397, + "tokens_seen": 2478309376 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020137211634904713, + "loss": 1.217, + "theoretical_loss": 3.399323795493216, + "tokens_seen": 2478374912 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020135606820461383, + "loss": 1.2067, + "theoretical_loss": 3.39931667317211, + "tokens_seen": 2478440448 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020134002006018053, + "loss": 1.0893, + "theoretical_loss": 3.3993095510920637, + "tokens_seen": 2478505984 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020132397191574723, + "loss": 1.2759, + "theoretical_loss": 3.3993024292530634, + "tokens_seen": 2478571520 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020130792377131393, + "loss": 1.3027, + "theoretical_loss": 3.3992953076550942, + "tokens_seen": 2478637056 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020129187562688064, + "loss": 1.2184, + "theoretical_loss": 3.3992881862981417, + "tokens_seen": 2478702592 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020127582748244734, + "loss": 1.175, + "theoretical_loss": 3.3992810651821914, + "tokens_seen": 2478768128 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020125977933801404, + "loss": 1.2264, + "theoretical_loss": 3.399273944307229, + "tokens_seen": 2478833664 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020124373119358074, + "loss": 1.1346, + "theoretical_loss": 3.3992668236732393, + "tokens_seen": 2478899200 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020122768304914744, + "loss": 1.2599, + "theoretical_loss": 3.399259703280208, + "tokens_seen": 2478964736 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020121163490471414, + "loss": 1.3759, + "theoretical_loss": 3.399252583128121, + "tokens_seen": 2479030272 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020119558676028087, + "loss": 1.2659, + "theoretical_loss": 3.3992454632169635, + "tokens_seen": 2479095808 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020117953861584757, + "loss": 1.2509, + "theoretical_loss": 3.399238343546721, + "tokens_seen": 2479161344 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020116349047141427, + "loss": 1.241, + "theoretical_loss": 3.3992312241173788, + "tokens_seen": 2479226880 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020114744232698097, + "loss": 1.3191, + "theoretical_loss": 3.3992241049289227, + "tokens_seen": 2479292416 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020113139418254767, + "loss": 1.2666, + "theoretical_loss": 3.399216985981338, + "tokens_seen": 2479357952 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020111534603811437, + "loss": 1.2927, + "theoretical_loss": 3.3992098672746103, + "tokens_seen": 2479423488 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020109929789368107, + "loss": 1.1192, + "theoretical_loss": 3.3992027488087246, + "tokens_seen": 2479489024 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020108324974924777, + "loss": 1.2918, + "theoretical_loss": 3.3991956305836672, + "tokens_seen": 2479554560 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020106720160481447, + "loss": 1.2369, + "theoretical_loss": 3.399188512599423, + "tokens_seen": 2479620096 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020105115346038117, + "loss": 1.2813, + "theoretical_loss": 3.3991813948559777, + "tokens_seen": 2479685632 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020103510531594787, + "loss": 1.2316, + "theoretical_loss": 3.3991742773533167, + "tokens_seen": 2479751168 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020101905717151457, + "loss": 1.2014, + "theoretical_loss": 3.3991671600914257, + "tokens_seen": 2479816704 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020100300902708127, + "loss": 1.2179, + "theoretical_loss": 3.3991600430702897, + "tokens_seen": 2479882240 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020098696088264797, + "loss": 1.2137, + "theoretical_loss": 3.3991529262898954, + "tokens_seen": 2479947776 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020097091273821468, + "loss": 1.2418, + "theoretical_loss": 3.3991458097502263, + "tokens_seen": 2480013312 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020095486459378138, + "loss": 1.1184, + "theoretical_loss": 3.3991386934512695, + "tokens_seen": 2480078848 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020093881644934808, + "loss": 1.2318, + "theoretical_loss": 3.3991315773930104, + "tokens_seen": 2480144384 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020092276830491478, + "loss": 1.245, + "theoretical_loss": 3.3991244615754335, + "tokens_seen": 2480209920 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020090672016048148, + "loss": 1.321, + "theoretical_loss": 3.3991173459985253, + "tokens_seen": 2480275456 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020089067201604815, + "loss": 1.2953, + "theoretical_loss": 3.399110230662271, + "tokens_seen": 2480340992 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020087462387161485, + "loss": 1.2491, + "theoretical_loss": 3.3991031155666556, + "tokens_seen": 2480406528 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020085857572718155, + "loss": 1.2179, + "theoretical_loss": 3.3990960007116655, + "tokens_seen": 2480472064 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 829255, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2619050741195679, + "objective/train/theoretical_loss": 3.399088886097285, + "objective/train/tokens_used": 2500997600, + "theoretical_loss": 3.399088886097285, + "tokens_seen": 2480537600 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020084252758274825, + "loss": 1.2409, + "theoretical_loss": 3.399088886097285, + "tokens_seen": 2480537600 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020082647943831495, + "loss": 1.1882, + "theoretical_loss": 3.399081771723501, + "tokens_seen": 2480603136 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020081043129388165, + "loss": 1.2541, + "theoretical_loss": 3.3990746575902984, + "tokens_seen": 2480668672 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020079438314944836, + "loss": 1.1639, + "theoretical_loss": 3.399067543697662, + "tokens_seen": 2480734208 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020077833500501506, + "loss": 1.2356, + "theoretical_loss": 3.3990604300455787, + "tokens_seen": 2480799744 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020076228686058176, + "loss": 1.2349, + "theoretical_loss": 3.3990533166340327, + "tokens_seen": 2480865280 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020074623871614846, + "loss": 1.2201, + "theoretical_loss": 3.3990462034630102, + "tokens_seen": 2480930816 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020073019057171516, + "loss": 1.2488, + "theoretical_loss": 3.399039090532497, + "tokens_seen": 2480996352 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020071414242728186, + "loss": 1.2834, + "theoretical_loss": 3.399031977842478, + "tokens_seen": 2481061888 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020069809428284856, + "loss": 1.212, + "theoretical_loss": 3.3990248653929385, + "tokens_seen": 2481127424 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020068204613841526, + "loss": 1.2744, + "theoretical_loss": 3.3990177531838652, + "tokens_seen": 2481192960 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020066599799398196, + "loss": 1.1677, + "theoretical_loss": 3.3990106412152423, + "tokens_seen": 2481258496 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020064994984954866, + "loss": 1.1946, + "theoretical_loss": 3.3990035294870564, + "tokens_seen": 2481324032 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020063390170511536, + "loss": 1.2133, + "theoretical_loss": 3.398996417999292, + "tokens_seen": 2481389568 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020061785356068206, + "loss": 1.1975, + "theoretical_loss": 3.3989893067519352, + "tokens_seen": 2481455104 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020060180541624876, + "loss": 1.1522, + "theoretical_loss": 3.3989821957449715, + "tokens_seen": 2481520640 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020058575727181546, + "loss": 1.2933, + "theoretical_loss": 3.3989750849783866, + "tokens_seen": 2481586176 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020056970912738216, + "loss": 1.2804, + "theoretical_loss": 3.398967974452166, + "tokens_seen": 2481651712 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020055366098294886, + "loss": 1.2739, + "theoretical_loss": 3.3989608641662947, + "tokens_seen": 2481717248 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020053761283851557, + "loss": 1.2873, + "theoretical_loss": 3.3989537541207584, + "tokens_seen": 2481782784 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020052156469408227, + "loss": 1.1431, + "theoretical_loss": 3.3989466443155427, + "tokens_seen": 2481848320 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020050551654964894, + "loss": 1.2132, + "theoretical_loss": 3.398939534750634, + "tokens_seen": 2481913856 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020048946840521564, + "loss": 1.1458, + "theoretical_loss": 3.3989324254260165, + "tokens_seen": 2481979392 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020047342026078234, + "loss": 1.2597, + "theoretical_loss": 3.398925316341676, + "tokens_seen": 2482044928 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020045737211634904, + "loss": 1.2372, + "theoretical_loss": 3.398918207497599, + "tokens_seen": 2482110464 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020044132397191574, + "loss": 1.2187, + "theoretical_loss": 3.39891109889377, + "tokens_seen": 2482176000 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020042527582748244, + "loss": 1.1225, + "theoretical_loss": 3.398903990530175, + "tokens_seen": 2482241536 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020040922768304914, + "loss": 1.2305, + "theoretical_loss": 3.3988968824067998, + "tokens_seen": 2482307072 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020039317953861584, + "loss": 1.2513, + "theoretical_loss": 3.398889774523629, + "tokens_seen": 2482372608 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020037713139418254, + "loss": 1.3474, + "theoretical_loss": 3.398882666880649, + "tokens_seen": 2482438144 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020036108324974925, + "loss": 1.227, + "theoretical_loss": 3.398875559477845, + "tokens_seen": 2482503680 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020034503510531595, + "loss": 1.1729, + "theoretical_loss": 3.3988684523152024, + "tokens_seen": 2482569216 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020032898696088265, + "loss": 1.111, + "theoretical_loss": 3.3988613453927075, + "tokens_seen": 2482634752 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020031293881644935, + "loss": 1.1301, + "theoretical_loss": 3.398854238710345, + "tokens_seen": 2482700288 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020029689067201605, + "loss": 1.2035, + "theoretical_loss": 3.3988471322681004, + "tokens_seen": 2482765824 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020028084252758275, + "loss": 1.1641, + "theoretical_loss": 3.39884002606596, + "tokens_seen": 2482831360 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020026479438314945, + "loss": 1.2287, + "theoretical_loss": 3.3988329201039087, + "tokens_seen": 2482896896 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020024874623871615, + "loss": 1.2242, + "theoretical_loss": 3.3988258143819325, + "tokens_seen": 2482962432 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020023269809428285, + "loss": 1.209, + "theoretical_loss": 3.398818708900017, + "tokens_seen": 2483027968 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020021664994984955, + "loss": 1.1486, + "theoretical_loss": 3.3988116036581473, + "tokens_seen": 2483093504 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020020060180541625, + "loss": 1.1233, + "theoretical_loss": 3.398804498656309, + "tokens_seen": 2483159040 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020018455366098295, + "loss": 1.1901, + "theoretical_loss": 3.3987973938944878, + "tokens_seen": 2483224576 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020016850551654965, + "loss": 1.2299, + "theoretical_loss": 3.3987902893726694, + "tokens_seen": 2483290112 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020015245737211635, + "loss": 1.2256, + "theoretical_loss": 3.3987831850908394, + "tokens_seen": 2483355648 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020013640922768305, + "loss": 1.2078, + "theoretical_loss": 3.398776081048983, + "tokens_seen": 2483421184 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020012036108324973, + "loss": 1.1614, + "theoretical_loss": 3.398768977247086, + "tokens_seen": 2483486720 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020010431293881643, + "loss": 1.285, + "theoretical_loss": 3.398761873685134, + "tokens_seen": 2483552256 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020008826479438313, + "loss": 1.3088, + "theoretical_loss": 3.3987547703631122, + "tokens_seen": 2483617792 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020007221664994983, + "loss": 1.2618, + "theoretical_loss": 3.398747667281007, + "tokens_seen": 2483683328 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020005616850551653, + "loss": 1.269, + "theoretical_loss": 3.398740564438803, + "tokens_seen": 2483748864 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 830439, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1394518613815308, + "objective/train/theoretical_loss": 3.3987334618364864, + "objective/train/tokens_used": 2504274400, + "theoretical_loss": 3.3987334618364864, + "tokens_seen": 2483814400 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020004012036108323, + "loss": 1.201, + "theoretical_loss": 3.3987334618364864, + "tokens_seen": 2483814400 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020002407221664993, + "loss": 1.1851, + "theoretical_loss": 3.3987263594740424, + "tokens_seen": 2483879936 + }, + { + "epoch": 0.75, + "learning_rate": 0.00020000802407221663, + "loss": 1.2609, + "theoretical_loss": 3.398719257351457, + "tokens_seen": 2483945472 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019999197592778336, + "loss": 1.3455, + "theoretical_loss": 3.398712155468715, + "tokens_seen": 2484011008 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019997592778335006, + "loss": 1.1757, + "theoretical_loss": 3.3987050538258026, + "tokens_seen": 2484076544 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019995987963891676, + "loss": 1.2225, + "theoretical_loss": 3.3986979524227054, + "tokens_seen": 2484142080 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019994383149448346, + "loss": 1.2049, + "theoretical_loss": 3.398690851259409, + "tokens_seen": 2484207616 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019992778335005016, + "loss": 1.3037, + "theoretical_loss": 3.3986837503358984, + "tokens_seen": 2484273152 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019991173520561686, + "loss": 1.2266, + "theoretical_loss": 3.3986766496521597, + "tokens_seen": 2484338688 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019989568706118356, + "loss": 1.237, + "theoretical_loss": 3.3986695492081784, + "tokens_seen": 2484404224 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019987963891675026, + "loss": 1.2933, + "theoretical_loss": 3.39866244900394, + "tokens_seen": 2484469760 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019986359077231696, + "loss": 1.255, + "theoretical_loss": 3.3986553490394305, + "tokens_seen": 2484535296 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019984754262788367, + "loss": 1.2548, + "theoretical_loss": 3.398648249314635, + "tokens_seen": 2484600832 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019983149448345037, + "loss": 1.2898, + "theoretical_loss": 3.3986411498295386, + "tokens_seen": 2484666368 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019981544633901707, + "loss": 1.3061, + "theoretical_loss": 3.398634050584128, + "tokens_seen": 2484731904 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019979939819458374, + "loss": 1.1572, + "theoretical_loss": 3.398626951578388, + "tokens_seen": 2484797440 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019978335005015044, + "loss": 1.4248, + "theoretical_loss": 3.3986198528123044, + "tokens_seen": 2484862976 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019976730190571714, + "loss": 1.2282, + "theoretical_loss": 3.398612754285863, + "tokens_seen": 2484928512 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019975125376128384, + "loss": 1.1543, + "theoretical_loss": 3.398605655999049, + "tokens_seen": 2484994048 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019973520561685054, + "loss": 1.2339, + "theoretical_loss": 3.3985985579518485, + "tokens_seen": 2485059584 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019971915747241724, + "loss": 1.2232, + "theoretical_loss": 3.398591460144247, + "tokens_seen": 2485125120 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019970310932798394, + "loss": 1.113, + "theoretical_loss": 3.398584362576229, + "tokens_seen": 2485190656 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019968706118355067, + "loss": 1.2763, + "theoretical_loss": 3.398577265247782, + "tokens_seen": 2485256192 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019967101303911737, + "loss": 1.2816, + "theoretical_loss": 3.39857016815889, + "tokens_seen": 2485321728 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019965496489468407, + "loss": 1.2113, + "theoretical_loss": 3.3985630713095394, + "tokens_seen": 2485387264 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019963891675025077, + "loss": 1.1842, + "theoretical_loss": 3.3985559746997156, + "tokens_seen": 2485452800 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019962286860581747, + "loss": 1.1257, + "theoretical_loss": 3.3985488783294038, + "tokens_seen": 2485518336 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019960682046138418, + "loss": 1.1704, + "theoretical_loss": 3.3985417821985906, + "tokens_seen": 2485583872 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019959077231695088, + "loss": 1.2482, + "theoretical_loss": 3.3985346863072605, + "tokens_seen": 2485649408 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019957472417251758, + "loss": 1.175, + "theoretical_loss": 3.3985275906553998, + "tokens_seen": 2485714944 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019955867602808428, + "loss": 1.1798, + "theoretical_loss": 3.398520495242994, + "tokens_seen": 2485780480 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019954262788365098, + "loss": 1.4069, + "theoretical_loss": 3.3985134000700286, + "tokens_seen": 2485846016 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019952657973921768, + "loss": 1.1513, + "theoretical_loss": 3.398506305136489, + "tokens_seen": 2485911552 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019951053159478438, + "loss": 1.2008, + "theoretical_loss": 3.3984992104423615, + "tokens_seen": 2485977088 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019949448345035108, + "loss": 1.25, + "theoretical_loss": 3.3984921159876307, + "tokens_seen": 2486042624 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019947843530591775, + "loss": 1.2065, + "theoretical_loss": 3.3984850217722826, + "tokens_seen": 2486108160 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019946238716148445, + "loss": 1.234, + "theoretical_loss": 3.3984779277963035, + "tokens_seen": 2486173696 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019944633901705115, + "loss": 1.2046, + "theoretical_loss": 3.3984708340596783, + "tokens_seen": 2486239232 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019943029087261786, + "loss": 1.1899, + "theoretical_loss": 3.3984637405623928, + "tokens_seen": 2486304768 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019941424272818456, + "loss": 1.2836, + "theoretical_loss": 3.3984566473044326, + "tokens_seen": 2486370304 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019939819458375126, + "loss": 1.3254, + "theoretical_loss": 3.398449554285783, + "tokens_seen": 2486435840 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019938214643931796, + "loss": 1.2892, + "theoretical_loss": 3.3984424615064306, + "tokens_seen": 2486501376 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019936609829488466, + "loss": 1.2191, + "theoretical_loss": 3.39843536896636, + "tokens_seen": 2486566912 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019935005015045136, + "loss": 1.4025, + "theoretical_loss": 3.398428276665557, + "tokens_seen": 2486632448 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019933400200601806, + "loss": 1.2142, + "theoretical_loss": 3.3984211846040075, + "tokens_seen": 2486697984 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019931795386158476, + "loss": 1.1669, + "theoretical_loss": 3.398414092781697, + "tokens_seen": 2486763520 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019930190571715146, + "loss": 1.2416, + "theoretical_loss": 3.398407001198611, + "tokens_seen": 2486829056 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019928585757271816, + "loss": 1.1906, + "theoretical_loss": 3.398399909854736, + "tokens_seen": 2486894592 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019926980942828486, + "loss": 1.2428, + "theoretical_loss": 3.3983928187500565, + "tokens_seen": 2486960128 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019925376128385156, + "loss": 1.3065, + "theoretical_loss": 3.398385727884558, + "tokens_seen": 2487025664 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 831332, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1451013088226318, + "objective/train/theoretical_loss": 3.398378637258227, + "objective/train/tokens_used": 2507551200, + "theoretical_loss": 3.398378637258227, + "tokens_seen": 2487091200 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019923771313941826, + "loss": 1.2082, + "theoretical_loss": 3.398378637258227, + "tokens_seen": 2487091200 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019922166499498496, + "loss": 1.1934, + "theoretical_loss": 3.3983715468710494, + "tokens_seen": 2487156736 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019920561685055166, + "loss": 1.2297, + "theoretical_loss": 3.3983644567230096, + "tokens_seen": 2487222272 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019918956870611836, + "loss": 1.2851, + "theoretical_loss": 3.398357366814094, + "tokens_seen": 2487287808 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019917352056168507, + "loss": 1.0724, + "theoretical_loss": 3.398350277144288, + "tokens_seen": 2487353344 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019915747241725177, + "loss": 1.2158, + "theoretical_loss": 3.3983431877135777, + "tokens_seen": 2487418880 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019914142427281847, + "loss": 1.2054, + "theoretical_loss": 3.398336098521948, + "tokens_seen": 2487484416 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019912537612838517, + "loss": 1.1671, + "theoretical_loss": 3.3983290095693848, + "tokens_seen": 2487549952 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019910932798395187, + "loss": 1.2764, + "theoretical_loss": 3.3983219208558744, + "tokens_seen": 2487615488 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019909327983951857, + "loss": 1.2087, + "theoretical_loss": 3.398314832381401, + "tokens_seen": 2487681024 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019907723169508527, + "loss": 1.1089, + "theoretical_loss": 3.398307744145952, + "tokens_seen": 2487746560 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019906118355065197, + "loss": 1.2529, + "theoretical_loss": 3.398300656149512, + "tokens_seen": 2487812096 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019904513540621867, + "loss": 1.2395, + "theoretical_loss": 3.3982935683920665, + "tokens_seen": 2487877632 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019902908726178537, + "loss": 1.1737, + "theoretical_loss": 3.398286480873602, + "tokens_seen": 2487943168 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019901303911735207, + "loss": 1.2631, + "theoretical_loss": 3.398279393594103, + "tokens_seen": 2488008704 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019899699097291877, + "loss": 1.2305, + "theoretical_loss": 3.3982723065535563, + "tokens_seen": 2488074240 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019898094282848547, + "loss": 1.2078, + "theoretical_loss": 3.3982652197519467, + "tokens_seen": 2488139776 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019896489468405217, + "loss": 1.2546, + "theoretical_loss": 3.3982581331892603, + "tokens_seen": 2488205312 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019894884653961887, + "loss": 1.2737, + "theoretical_loss": 3.3982510468654823, + "tokens_seen": 2488270848 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019893279839518557, + "loss": 1.2507, + "theoretical_loss": 3.398243960780599, + "tokens_seen": 2488336384 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019891675025075228, + "loss": 1.278, + "theoretical_loss": 3.3982368749345957, + "tokens_seen": 2488401920 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019890070210631898, + "loss": 1.2553, + "theoretical_loss": 3.3982297893274582, + "tokens_seen": 2488467456 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019888465396188568, + "loss": 1.1529, + "theoretical_loss": 3.398222703959172, + "tokens_seen": 2488532992 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019886860581745238, + "loss": 1.1808, + "theoretical_loss": 3.3982156188297226, + "tokens_seen": 2488598528 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019885255767301908, + "loss": 1.1966, + "theoretical_loss": 3.398208533939096, + "tokens_seen": 2488664064 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019883650952858578, + "loss": 1.239, + "theoretical_loss": 3.3982014492872774, + "tokens_seen": 2488729600 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019882046138415248, + "loss": 1.2778, + "theoretical_loss": 3.398194364874253, + "tokens_seen": 2488795136 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019880441323971918, + "loss": 1.1353, + "theoretical_loss": 3.398187280700008, + "tokens_seen": 2488860672 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019878836509528585, + "loss": 1.2361, + "theoretical_loss": 3.398180196764529, + "tokens_seen": 2488926208 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019877231695085255, + "loss": 1.2747, + "theoretical_loss": 3.398173113067801, + "tokens_seen": 2488991744 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019875626880641925, + "loss": 1.1917, + "theoretical_loss": 3.3981660296098086, + "tokens_seen": 2489057280 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019874022066198596, + "loss": 1.1691, + "theoretical_loss": 3.3981589463905393, + "tokens_seen": 2489122816 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019872417251755266, + "loss": 1.3089, + "theoretical_loss": 3.398151863409978, + "tokens_seen": 2489188352 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019870812437311936, + "loss": 1.2664, + "theoretical_loss": 3.39814478066811, + "tokens_seen": 2489253888 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019869207622868606, + "loss": 1.2682, + "theoretical_loss": 3.3981376981649216, + "tokens_seen": 2489319424 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019867602808425276, + "loss": 1.1665, + "theoretical_loss": 3.3981306159003983, + "tokens_seen": 2489384960 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019865997993981946, + "loss": 1.3145, + "theoretical_loss": 3.398123533874525, + "tokens_seen": 2489450496 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019864393179538616, + "loss": 1.2427, + "theoretical_loss": 3.398116452087289, + "tokens_seen": 2489516032 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019862788365095286, + "loss": 1.1563, + "theoretical_loss": 3.3981093705386742, + "tokens_seen": 2489581568 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019861183550651956, + "loss": 1.1608, + "theoretical_loss": 3.3981022892286674, + "tokens_seen": 2489647104 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019859578736208626, + "loss": 1.2877, + "theoretical_loss": 3.398095208157254, + "tokens_seen": 2489712640 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019857973921765296, + "loss": 1.1756, + "theoretical_loss": 3.39808812732442, + "tokens_seen": 2489778176 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019856369107321966, + "loss": 1.2563, + "theoretical_loss": 3.3980810467301508, + "tokens_seen": 2489843712 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019854764292878636, + "loss": 1.2139, + "theoretical_loss": 3.3980739663744313, + "tokens_seen": 2489909248 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019853159478435306, + "loss": 1.1579, + "theoretical_loss": 3.398066886257248, + "tokens_seen": 2489974784 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019851554663991976, + "loss": 1.1793, + "theoretical_loss": 3.398059806378587, + "tokens_seen": 2490040320 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019849949849548646, + "loss": 1.1124, + "theoretical_loss": 3.3980527267384337, + "tokens_seen": 2490105856 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001984834503510532, + "loss": 1.311, + "theoretical_loss": 3.398045647336773, + "tokens_seen": 2490171392 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019846740220661987, + "loss": 1.3087, + "theoretical_loss": 3.3980385681735914, + "tokens_seen": 2490236928 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019845135406218657, + "loss": 1.2547, + "theoretical_loss": 3.3980314892488743, + "tokens_seen": 2490302464 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 832559, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2697808742523193, + "objective/train/theoretical_loss": 3.3980244105626074, + "objective/train/tokens_used": 2510828000, + "theoretical_loss": 3.3980244105626074, + "tokens_seen": 2490368000 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019843530591775327, + "loss": 1.2602, + "theoretical_loss": 3.3980244105626074, + "tokens_seen": 2490368000 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019841925777331997, + "loss": 1.2989, + "theoretical_loss": 3.3980173321147764, + "tokens_seen": 2490433536 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019840320962888667, + "loss": 1.1108, + "theoretical_loss": 3.398010253905367, + "tokens_seen": 2490499072 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019838716148445337, + "loss": 1.324, + "theoretical_loss": 3.398003175934365, + "tokens_seen": 2490564608 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019837111334002007, + "loss": 1.2147, + "theoretical_loss": 3.397996098201756, + "tokens_seen": 2490630144 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019835506519558677, + "loss": 1.1787, + "theoretical_loss": 3.3979890207075263, + "tokens_seen": 2490695680 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019833901705115347, + "loss": 1.265, + "theoretical_loss": 3.3979819434516605, + "tokens_seen": 2490761216 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019832296890672017, + "loss": 1.2064, + "theoretical_loss": 3.3979748664341445, + "tokens_seen": 2490826752 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019830692076228687, + "loss": 1.1862, + "theoretical_loss": 3.397967789654965, + "tokens_seen": 2490892288 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019829087261785357, + "loss": 1.2933, + "theoretical_loss": 3.3979607131141067, + "tokens_seen": 2490957824 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019827482447342027, + "loss": 1.2547, + "theoretical_loss": 3.3979536368115557, + "tokens_seen": 2491023360 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019825877632898697, + "loss": 1.2084, + "theoretical_loss": 3.3979465607472976, + "tokens_seen": 2491088896 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019824272818455367, + "loss": 1.1839, + "theoretical_loss": 3.3979394849213183, + "tokens_seen": 2491154432 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019822668004012038, + "loss": 1.2297, + "theoretical_loss": 3.397932409333603, + "tokens_seen": 2491219968 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019821063189568708, + "loss": 1.3544, + "theoretical_loss": 3.397925333984138, + "tokens_seen": 2491285504 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019819458375125378, + "loss": 1.2595, + "theoretical_loss": 3.3979182588729087, + "tokens_seen": 2491351040 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019817853560682048, + "loss": 1.1311, + "theoretical_loss": 3.397911183999901, + "tokens_seen": 2491416576 + }, + { + "epoch": 0.75, + "learning_rate": 0.00019816248746238718, + "loss": 1.2664, + "theoretical_loss": 3.3979041093651006, + "tokens_seen": 2491482112 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019814643931795388, + "loss": 1.2052, + "theoretical_loss": 3.397897034968493, + "tokens_seen": 2491547648 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019813039117352058, + "loss": 1.2593, + "theoretical_loss": 3.3978899608100637, + "tokens_seen": 2491613184 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019811434302908728, + "loss": 1.2833, + "theoretical_loss": 3.3978828868897994, + "tokens_seen": 2491678720 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019809829488465398, + "loss": 1.2358, + "theoretical_loss": 3.3978758132076847, + "tokens_seen": 2491744256 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019808224674022065, + "loss": 1.2502, + "theoretical_loss": 3.397868739763706, + "tokens_seen": 2491809792 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019806619859578735, + "loss": 1.1732, + "theoretical_loss": 3.3978616665578487, + "tokens_seen": 2491875328 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019805015045135406, + "loss": 1.1261, + "theoretical_loss": 3.3978545935900986, + "tokens_seen": 2491940864 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019803410230692076, + "loss": 1.1845, + "theoretical_loss": 3.3978475208604415, + "tokens_seen": 2492006400 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019801805416248746, + "loss": 1.1471, + "theoretical_loss": 3.397840448368863, + "tokens_seen": 2492071936 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019800200601805416, + "loss": 1.2106, + "theoretical_loss": 3.397833376115349, + "tokens_seen": 2492137472 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019798595787362086, + "loss": 1.3668, + "theoretical_loss": 3.397826304099885, + "tokens_seen": 2492203008 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019796990972918756, + "loss": 1.2273, + "theoretical_loss": 3.397819232322457, + "tokens_seen": 2492268544 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019795386158475426, + "loss": 1.2069, + "theoretical_loss": 3.3978121607830505, + "tokens_seen": 2492334080 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019793781344032096, + "loss": 1.2428, + "theoretical_loss": 3.3978050894816514, + "tokens_seen": 2492399616 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019792176529588766, + "loss": 1.2217, + "theoretical_loss": 3.3977980184182455, + "tokens_seen": 2492465152 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019790571715145436, + "loss": 1.2452, + "theoretical_loss": 3.397790947592818, + "tokens_seen": 2492530688 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001978896690070211, + "loss": 1.2358, + "theoretical_loss": 3.397783877005355, + "tokens_seen": 2492596224 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001978736208625878, + "loss": 1.1849, + "theoretical_loss": 3.3977768066558425, + "tokens_seen": 2492661760 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001978575727181545, + "loss": 1.2653, + "theoretical_loss": 3.397769736544266, + "tokens_seen": 2492727296 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001978415245737212, + "loss": 1.1974, + "theoretical_loss": 3.3977626666706113, + "tokens_seen": 2492792832 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001978254764292879, + "loss": 1.1352, + "theoretical_loss": 3.3977555970348634, + "tokens_seen": 2492858368 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001978094282848546, + "loss": 1.3429, + "theoretical_loss": 3.3977485276370096, + "tokens_seen": 2492923904 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001977933801404213, + "loss": 1.4025, + "theoretical_loss": 3.397741458477034, + "tokens_seen": 2492989440 + }, + { + "epoch": 0.76, + "learning_rate": 0.000197777331995988, + "loss": 1.208, + "theoretical_loss": 3.3977343895549237, + "tokens_seen": 2493054976 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019776128385155467, + "loss": 1.2541, + "theoretical_loss": 3.397727320870663, + "tokens_seen": 2493120512 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019774523570712137, + "loss": 1.3009, + "theoretical_loss": 3.3977202524242394, + "tokens_seen": 2493186048 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019772918756268807, + "loss": 1.2071, + "theoretical_loss": 3.3977131842156374, + "tokens_seen": 2493251584 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019771313941825477, + "loss": 1.1655, + "theoretical_loss": 3.397706116244843, + "tokens_seen": 2493317120 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019769709127382147, + "loss": 1.1185, + "theoretical_loss": 3.397699048511842, + "tokens_seen": 2493382656 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019768104312938817, + "loss": 1.2865, + "theoretical_loss": 3.39769198101662, + "tokens_seen": 2493448192 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019766499498495487, + "loss": 1.1666, + "theoretical_loss": 3.3976849137591634, + "tokens_seen": 2493513728 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019764894684052157, + "loss": 1.2264, + "theoretical_loss": 3.397677846739457, + "tokens_seen": 2493579264 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 833727, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1852948665618896, + "objective/train/theoretical_loss": 3.397670779957487, + "objective/train/tokens_used": 2514104800, + "theoretical_loss": 3.397670779957487, + "tokens_seen": 2493644800 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019763289869608827, + "loss": 1.258, + "theoretical_loss": 3.397670779957487, + "tokens_seen": 2493644800 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019761685055165497, + "loss": 1.1733, + "theoretical_loss": 3.3976637134132393, + "tokens_seen": 2493710336 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019760080240722167, + "loss": 1.1952, + "theoretical_loss": 3.3976566471067, + "tokens_seen": 2493775872 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019758475426278837, + "loss": 1.2515, + "theoretical_loss": 3.3976495810378537, + "tokens_seen": 2493841408 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019756870611835507, + "loss": 1.2697, + "theoretical_loss": 3.397642515206687, + "tokens_seen": 2493906944 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019755265797392178, + "loss": 1.2078, + "theoretical_loss": 3.3976354496131855, + "tokens_seen": 2493972480 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019753660982948848, + "loss": 1.2375, + "theoretical_loss": 3.3976283842573354, + "tokens_seen": 2494038016 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019752056168505518, + "loss": 1.1955, + "theoretical_loss": 3.3976213191391214, + "tokens_seen": 2494103552 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019750451354062188, + "loss": 1.1719, + "theoretical_loss": 3.3976142542585306, + "tokens_seen": 2494169088 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019748846539618858, + "loss": 1.1389, + "theoretical_loss": 3.3976071896155475, + "tokens_seen": 2494234624 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019747241725175528, + "loss": 1.1905, + "theoretical_loss": 3.3976001252101584, + "tokens_seen": 2494300160 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019745636910732198, + "loss": 1.2328, + "theoretical_loss": 3.3975930610423495, + "tokens_seen": 2494365696 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019744032096288868, + "loss": 1.1742, + "theoretical_loss": 3.397585997112106, + "tokens_seen": 2494431232 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019742427281845538, + "loss": 1.1732, + "theoretical_loss": 3.3975789334194135, + "tokens_seen": 2494496768 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019740822467402208, + "loss": 1.1412, + "theoretical_loss": 3.3975718699642585, + "tokens_seen": 2494562304 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019739217652958878, + "loss": 1.2652, + "theoretical_loss": 3.397564806746626, + "tokens_seen": 2494627840 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019737612838515545, + "loss": 1.19, + "theoretical_loss": 3.3975577437665025, + "tokens_seen": 2494693376 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019736008024072216, + "loss": 1.1678, + "theoretical_loss": 3.3975506810238736, + "tokens_seen": 2494758912 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019734403209628886, + "loss": 1.2406, + "theoretical_loss": 3.3975436185187244, + "tokens_seen": 2494824448 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019732798395185556, + "loss": 1.2577, + "theoretical_loss": 3.3975365562510413, + "tokens_seen": 2494889984 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019731193580742226, + "loss": 1.2974, + "theoretical_loss": 3.39752949422081, + "tokens_seen": 2494955520 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019729588766298899, + "loss": 1.3592, + "theoretical_loss": 3.3975224324280164, + "tokens_seen": 2495021056 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019727983951855569, + "loss": 1.236, + "theoretical_loss": 3.3975153708726458, + "tokens_seen": 2495086592 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001972637913741224, + "loss": 1.2978, + "theoretical_loss": 3.3975083095546843, + "tokens_seen": 2495152128 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001972477432296891, + "loss": 1.185, + "theoretical_loss": 3.3975012484741183, + "tokens_seen": 2495217664 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001972316950852558, + "loss": 1.2044, + "theoretical_loss": 3.397494187630932, + "tokens_seen": 2495283200 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001972156469408225, + "loss": 1.201, + "theoretical_loss": 3.397487127025113, + "tokens_seen": 2495348736 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001971995987963892, + "loss": 1.2367, + "theoretical_loss": 3.397480066656646, + "tokens_seen": 2495414272 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001971835506519559, + "loss": 1.2433, + "theoretical_loss": 3.3974730065255168, + "tokens_seen": 2495479808 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001971675025075226, + "loss": 1.142, + "theoretical_loss": 3.397465946631711, + "tokens_seen": 2495545344 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001971514543630893, + "loss": 1.1151, + "theoretical_loss": 3.397458886975216, + "tokens_seen": 2495610880 + }, + { + "epoch": 0.76, + "learning_rate": 0.000197135406218656, + "loss": 1.2363, + "theoretical_loss": 3.397451827556015, + "tokens_seen": 2495676416 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001971193580742227, + "loss": 1.2027, + "theoretical_loss": 3.397444768374096, + "tokens_seen": 2495741952 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001971033099297894, + "loss": 1.2011, + "theoretical_loss": 3.397437709429444, + "tokens_seen": 2495807488 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001970872617853561, + "loss": 1.1527, + "theoretical_loss": 3.3974306507220446, + "tokens_seen": 2495873024 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019707121364092277, + "loss": 1.2068, + "theoretical_loss": 3.397423592251884, + "tokens_seen": 2495938560 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019705516549648947, + "loss": 1.249, + "theoretical_loss": 3.397416534018947, + "tokens_seen": 2496004096 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019703911735205617, + "loss": 1.3041, + "theoretical_loss": 3.397409476023221, + "tokens_seen": 2496069632 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019702306920762287, + "loss": 1.1836, + "theoretical_loss": 3.397402418264691, + "tokens_seen": 2496135168 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019700702106318957, + "loss": 1.3214, + "theoretical_loss": 3.3973953607433423, + "tokens_seen": 2496200704 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019699097291875627, + "loss": 1.2023, + "theoretical_loss": 3.397388303459161, + "tokens_seen": 2496266240 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019697492477432297, + "loss": 1.2813, + "theoretical_loss": 3.3973812464121336, + "tokens_seen": 2496331776 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019695887662988967, + "loss": 1.1223, + "theoretical_loss": 3.397374189602245, + "tokens_seen": 2496397312 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019694282848545637, + "loss": 1.0884, + "theoretical_loss": 3.3973671330294817, + "tokens_seen": 2496462848 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019692678034102307, + "loss": 1.1843, + "theoretical_loss": 3.397360076693829, + "tokens_seen": 2496528384 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019691073219658977, + "loss": 1.194, + "theoretical_loss": 3.397353020595273, + "tokens_seen": 2496593920 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019689468405215647, + "loss": 1.2526, + "theoretical_loss": 3.397345964733799, + "tokens_seen": 2496659456 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019687863590772317, + "loss": 1.1783, + "theoretical_loss": 3.3973389091093935, + "tokens_seen": 2496724992 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019686258776328988, + "loss": 1.2051, + "theoretical_loss": 3.3973318537220423, + "tokens_seen": 2496790528 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019684653961885658, + "loss": 1.1191, + "theoretical_loss": 3.3973247985717308, + "tokens_seen": 2496856064 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 834675, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.094529628753662, + "objective/train/theoretical_loss": 3.397317743658445, + "objective/train/tokens_used": 2517381600, + "theoretical_loss": 3.397317743658445, + "tokens_seen": 2496921600 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019683049147442328, + "loss": 1.1351, + "theoretical_loss": 3.397317743658445, + "tokens_seen": 2496921600 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019681444332998998, + "loss": 1.2244, + "theoretical_loss": 3.3973106889821705, + "tokens_seen": 2496987136 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019679839518555668, + "loss": 1.1887, + "theoretical_loss": 3.3973036345428933, + "tokens_seen": 2497052672 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019678234704112338, + "loss": 1.2425, + "theoretical_loss": 3.3972965803405994, + "tokens_seen": 2497118208 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019676629889669008, + "loss": 1.1659, + "theoretical_loss": 3.3972895263752743, + "tokens_seen": 2497183744 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019675025075225678, + "loss": 1.2185, + "theoretical_loss": 3.397282472646904, + "tokens_seen": 2497249280 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019673420260782348, + "loss": 1.2426, + "theoretical_loss": 3.3972754191554744, + "tokens_seen": 2497314816 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019671815446339018, + "loss": 1.2389, + "theoretical_loss": 3.3972683659009713, + "tokens_seen": 2497380352 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019670210631895688, + "loss": 1.163, + "theoretical_loss": 3.39726131288338, + "tokens_seen": 2497445888 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019668605817452358, + "loss": 1.2172, + "theoretical_loss": 3.3972542601026867, + "tokens_seen": 2497511424 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019667001003009028, + "loss": 1.2056, + "theoretical_loss": 3.3972472075588778, + "tokens_seen": 2497576960 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019665396188565698, + "loss": 1.2784, + "theoretical_loss": 3.397240155251938, + "tokens_seen": 2497642496 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019663791374122368, + "loss": 1.1979, + "theoretical_loss": 3.397233103181854, + "tokens_seen": 2497708032 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019662186559679038, + "loss": 1.179, + "theoretical_loss": 3.3972260513486114, + "tokens_seen": 2497773568 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019660581745235709, + "loss": 1.2527, + "theoretical_loss": 3.397218999752196, + "tokens_seen": 2497839104 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019658976930792379, + "loss": 1.2825, + "theoretical_loss": 3.3972119483925938, + "tokens_seen": 2497904640 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001965737211634905, + "loss": 1.1905, + "theoretical_loss": 3.3972048972697904, + "tokens_seen": 2497970176 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001965576730190572, + "loss": 1.1757, + "theoretical_loss": 3.3971978463837713, + "tokens_seen": 2498035712 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001965416248746239, + "loss": 1.2178, + "theoretical_loss": 3.3971907957345233, + "tokens_seen": 2498101248 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001965255767301906, + "loss": 1.3259, + "theoretical_loss": 3.3971837453220317, + "tokens_seen": 2498166784 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001965095285857573, + "loss": 1.363, + "theoretical_loss": 3.397176695146282, + "tokens_seen": 2498232320 + }, + { + "epoch": 0.76, + "learning_rate": 0.000196493480441324, + "loss": 1.257, + "theoretical_loss": 3.39716964520726, + "tokens_seen": 2498297856 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001964774322968907, + "loss": 1.2151, + "theoretical_loss": 3.3971625955049527, + "tokens_seen": 2498363392 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001964613841524574, + "loss": 1.2527, + "theoretical_loss": 3.3971555460393446, + "tokens_seen": 2498428928 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001964453360080241, + "loss": 1.2298, + "theoretical_loss": 3.397148496810422, + "tokens_seen": 2498494464 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001964292878635908, + "loss": 1.2827, + "theoretical_loss": 3.3971414478181714, + "tokens_seen": 2498560000 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001964132397191575, + "loss": 1.26, + "theoretical_loss": 3.3971343990625775, + "tokens_seen": 2498625536 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001963971915747242, + "loss": 1.1719, + "theoretical_loss": 3.3971273505436272, + "tokens_seen": 2498691072 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001963811434302909, + "loss": 1.2759, + "theoretical_loss": 3.3971203022613055, + "tokens_seen": 2498756608 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019636509528585757, + "loss": 1.3374, + "theoretical_loss": 3.397113254215599, + "tokens_seen": 2498822144 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019634904714142427, + "loss": 1.1912, + "theoretical_loss": 3.3971062064064927, + "tokens_seen": 2498887680 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019633299899699097, + "loss": 1.118, + "theoretical_loss": 3.397099158833973, + "tokens_seen": 2498953216 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019631695085255767, + "loss": 1.2384, + "theoretical_loss": 3.3970921114980257, + "tokens_seen": 2499018752 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019630090270812437, + "loss": 1.2525, + "theoretical_loss": 3.3970850643986368, + "tokens_seen": 2499084288 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019628485456369107, + "loss": 1.24, + "theoretical_loss": 3.397078017535792, + "tokens_seen": 2499149824 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019626880641925777, + "loss": 1.2153, + "theoretical_loss": 3.397070970909477, + "tokens_seen": 2499215360 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019625275827482447, + "loss": 1.213, + "theoretical_loss": 3.397063924519678, + "tokens_seen": 2499280896 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019623671013039117, + "loss": 1.2189, + "theoretical_loss": 3.3970568783663806, + "tokens_seen": 2499346432 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019622066198595787, + "loss": 1.1459, + "theoretical_loss": 3.3970498324495706, + "tokens_seen": 2499411968 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019620461384152457, + "loss": 1.2174, + "theoretical_loss": 3.397042786769234, + "tokens_seen": 2499477504 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019618856569709127, + "loss": 1.1497, + "theoretical_loss": 3.397035741325357, + "tokens_seen": 2499543040 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019617251755265798, + "loss": 1.1609, + "theoretical_loss": 3.397028696117925, + "tokens_seen": 2499608576 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019615646940822468, + "loss": 1.2077, + "theoretical_loss": 3.3970216511469236, + "tokens_seen": 2499674112 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019614042126379138, + "loss": 1.171, + "theoretical_loss": 3.3970146064123394, + "tokens_seen": 2499739648 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019612437311935808, + "loss": 1.1714, + "theoretical_loss": 3.3970075619141578, + "tokens_seen": 2499805184 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019610832497492478, + "loss": 1.1589, + "theoretical_loss": 3.3970005176523648, + "tokens_seen": 2499870720 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001960922768304915, + "loss": 1.2444, + "theoretical_loss": 3.3969934736269463, + "tokens_seen": 2499936256 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001960762286860582, + "loss": 1.2292, + "theoretical_loss": 3.396986429837888, + "tokens_seen": 2500001792 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001960601805416249, + "loss": 1.3088, + "theoretical_loss": 3.396979386285176, + "tokens_seen": 2500067328 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019604413239719158, + "loss": 1.1654, + "theoretical_loss": 3.396972342968796, + "tokens_seen": 2500132864 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 835943, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2011704444885254, + "objective/train/theoretical_loss": 3.396965299888734, + "objective/train/tokens_used": 2520658400, + "theoretical_loss": 3.396965299888734, + "tokens_seen": 2500198400 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019602808425275828, + "loss": 1.1695, + "theoretical_loss": 3.396965299888734, + "tokens_seen": 2500198400 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019601203610832498, + "loss": 1.1229, + "theoretical_loss": 3.396958257044976, + "tokens_seen": 2500263936 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019599598796389168, + "loss": 1.2418, + "theoretical_loss": 3.3969512144375074, + "tokens_seen": 2500329472 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019597993981945838, + "loss": 1.1921, + "theoretical_loss": 3.3969441720663145, + "tokens_seen": 2500395008 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019596389167502508, + "loss": 1.2576, + "theoretical_loss": 3.3969371299313833, + "tokens_seen": 2500460544 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019594784353059178, + "loss": 1.2226, + "theoretical_loss": 3.3969300880326996, + "tokens_seen": 2500526080 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019593179538615848, + "loss": 1.3133, + "theoretical_loss": 3.3969230463702487, + "tokens_seen": 2500591616 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019591574724172519, + "loss": 1.2329, + "theoretical_loss": 3.396916004944017, + "tokens_seen": 2500657152 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019589969909729189, + "loss": 1.1005, + "theoretical_loss": 3.39690896375399, + "tokens_seen": 2500722688 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001958836509528586, + "loss": 1.3547, + "theoretical_loss": 3.3969019228001547, + "tokens_seen": 2500788224 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001958676028084253, + "loss": 1.3067, + "theoretical_loss": 3.3968948820824956, + "tokens_seen": 2500853760 + }, + { + "epoch": 0.76, + "learning_rate": 0.000195851554663992, + "loss": 1.2653, + "theoretical_loss": 3.396887841600999, + "tokens_seen": 2500919296 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001958355065195587, + "loss": 1.2943, + "theoretical_loss": 3.3968808013556515, + "tokens_seen": 2500984832 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001958194583751254, + "loss": 1.3889, + "theoretical_loss": 3.3968737613464386, + "tokens_seen": 2501050368 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001958034102306921, + "loss": 1.1525, + "theoretical_loss": 3.3968667215733452, + "tokens_seen": 2501115904 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001957873620862588, + "loss": 1.3904, + "theoretical_loss": 3.396859682036359, + "tokens_seen": 2501181440 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001957713139418255, + "loss": 1.1764, + "theoretical_loss": 3.396852642735464, + "tokens_seen": 2501246976 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001957552657973922, + "loss": 1.2492, + "theoretical_loss": 3.3968456036706476, + "tokens_seen": 2501312512 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001957392176529589, + "loss": 1.342, + "theoretical_loss": 3.396838564841895, + "tokens_seen": 2501378048 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001957231695085256, + "loss": 1.2659, + "theoretical_loss": 3.3968315262491924, + "tokens_seen": 2501443584 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001957071213640923, + "loss": 1.2222, + "theoretical_loss": 3.396824487892525, + "tokens_seen": 2501509120 + }, + { + "epoch": 0.76, + "learning_rate": 0.000195691073219659, + "loss": 1.245, + "theoretical_loss": 3.3968174497718797, + "tokens_seen": 2501574656 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001956750250752257, + "loss": 1.2326, + "theoretical_loss": 3.396810411887242, + "tokens_seen": 2501640192 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019565897693079237, + "loss": 1.1844, + "theoretical_loss": 3.3968033742385972, + "tokens_seen": 2501705728 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019564292878635907, + "loss": 1.1537, + "theoretical_loss": 3.396796336825932, + "tokens_seen": 2501771264 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019562688064192577, + "loss": 1.23, + "theoretical_loss": 3.396789299649232, + "tokens_seen": 2501836800 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019561083249749247, + "loss": 1.1277, + "theoretical_loss": 3.3967822627084834, + "tokens_seen": 2501902336 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019559478435305917, + "loss": 1.2505, + "theoretical_loss": 3.3967752260036717, + "tokens_seen": 2501967872 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019557873620862587, + "loss": 1.2875, + "theoretical_loss": 3.396768189534783, + "tokens_seen": 2502033408 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019556268806419257, + "loss": 1.1977, + "theoretical_loss": 3.3967611533018034, + "tokens_seen": 2502098944 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019554663991975927, + "loss": 1.1657, + "theoretical_loss": 3.3967541173047184, + "tokens_seen": 2502164480 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019553059177532597, + "loss": 1.1731, + "theoretical_loss": 3.3967470815435137, + "tokens_seen": 2502230016 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019551454363089267, + "loss": 1.2082, + "theoretical_loss": 3.3967400460181763, + "tokens_seen": 2502295552 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019549849548645937, + "loss": 1.3192, + "theoretical_loss": 3.396733010728691, + "tokens_seen": 2502361088 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001954824473420261, + "loss": 1.1778, + "theoretical_loss": 3.3967259756750443, + "tokens_seen": 2502426624 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001954663991975928, + "loss": 1.2459, + "theoretical_loss": 3.3967189408572223, + "tokens_seen": 2502492160 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001954503510531595, + "loss": 1.1783, + "theoretical_loss": 3.39671190627521, + "tokens_seen": 2502557696 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001954343029087262, + "loss": 1.2286, + "theoretical_loss": 3.396704871928994, + "tokens_seen": 2502623232 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001954182547642929, + "loss": 1.3401, + "theoretical_loss": 3.3966978378185604, + "tokens_seen": 2502688768 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001954022066198596, + "loss": 1.2457, + "theoretical_loss": 3.3966908039438946, + "tokens_seen": 2502754304 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001953861584754263, + "loss": 1.139, + "theoretical_loss": 3.396683770304983, + "tokens_seen": 2502819840 + }, + { + "epoch": 0.76, + "learning_rate": 0.000195370110330993, + "loss": 1.2442, + "theoretical_loss": 3.396676736901811, + "tokens_seen": 2502885376 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001953540621865597, + "loss": 1.2214, + "theoretical_loss": 3.396669703734365, + "tokens_seen": 2502950912 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019533801404212638, + "loss": 1.3004, + "theoretical_loss": 3.3966626708026313, + "tokens_seen": 2503016448 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019532196589769308, + "loss": 1.2304, + "theoretical_loss": 3.3966556381065947, + "tokens_seen": 2503081984 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019530591775325978, + "loss": 1.2286, + "theoretical_loss": 3.3966486056462415, + "tokens_seen": 2503147520 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019528986960882648, + "loss": 1.1371, + "theoretical_loss": 3.396641573421558, + "tokens_seen": 2503213056 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019527382146439318, + "loss": 1.3062, + "theoretical_loss": 3.3966345414325305, + "tokens_seen": 2503278592 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019525777331995988, + "loss": 1.178, + "theoretical_loss": 3.396627509679144, + "tokens_seen": 2503344128 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019524172517552659, + "loss": 1.2415, + "theoretical_loss": 3.3966204781613847, + "tokens_seen": 2503409664 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 836801, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1851744651794434, + "objective/train/theoretical_loss": 3.396613446879239, + "objective/train/tokens_used": 2523935200, + "theoretical_loss": 3.396613446879239, + "tokens_seen": 2503475200 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019522567703109329, + "loss": 1.1437, + "theoretical_loss": 3.396613446879239, + "tokens_seen": 2503475200 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019520962888665999, + "loss": 1.1687, + "theoretical_loss": 3.3966064158326925, + "tokens_seen": 2503540736 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001951935807422267, + "loss": 1.1268, + "theoretical_loss": 3.396599385021731, + "tokens_seen": 2503606272 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001951775325977934, + "loss": 1.3568, + "theoretical_loss": 3.3965923544463408, + "tokens_seen": 2503671808 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001951614844533601, + "loss": 1.2383, + "theoretical_loss": 3.3965853241065074, + "tokens_seen": 2503737344 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001951454363089268, + "loss": 1.1945, + "theoretical_loss": 3.396578294002217, + "tokens_seen": 2503802880 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001951293881644935, + "loss": 1.2047, + "theoretical_loss": 3.396571264133456, + "tokens_seen": 2503868416 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001951133400200602, + "loss": 1.2576, + "theoretical_loss": 3.396564234500209, + "tokens_seen": 2503933952 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001950972918756269, + "loss": 1.2785, + "theoretical_loss": 3.3965572051024635, + "tokens_seen": 2503999488 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001950812437311936, + "loss": 1.3584, + "theoretical_loss": 3.3965501759402046, + "tokens_seen": 2504065024 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001950651955867603, + "loss": 1.264, + "theoretical_loss": 3.3965431470134186, + "tokens_seen": 2504130560 + }, + { + "epoch": 0.76, + "learning_rate": 0.000195049147442327, + "loss": 1.2903, + "theoretical_loss": 3.396536118322091, + "tokens_seen": 2504196096 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001950330992978937, + "loss": 1.2513, + "theoretical_loss": 3.396529089866208, + "tokens_seen": 2504261632 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001950170511534604, + "loss": 1.1821, + "theoretical_loss": 3.3965220616457557, + "tokens_seen": 2504327168 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001950010030090271, + "loss": 1.1958, + "theoretical_loss": 3.3965150336607195, + "tokens_seen": 2504392704 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001949849548645938, + "loss": 1.145, + "theoretical_loss": 3.3965080059110866, + "tokens_seen": 2504458240 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001949689067201605, + "loss": 1.3016, + "theoretical_loss": 3.3965009783968414, + "tokens_seen": 2504523776 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019495285857572717, + "loss": 1.1432, + "theoretical_loss": 3.396493951117971, + "tokens_seen": 2504589312 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019493681043129387, + "loss": 1.1899, + "theoretical_loss": 3.396486924074461, + "tokens_seen": 2504654848 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019492076228686057, + "loss": 1.2631, + "theoretical_loss": 3.396479897266297, + "tokens_seen": 2504720384 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019490471414242727, + "loss": 1.2164, + "theoretical_loss": 3.3964728706934655, + "tokens_seen": 2504785920 + }, + { + "epoch": 0.76, + "learning_rate": 0.000194888665997994, + "loss": 1.2142, + "theoretical_loss": 3.396465844355952, + "tokens_seen": 2504851456 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001948726178535607, + "loss": 1.1886, + "theoretical_loss": 3.396458818253743, + "tokens_seen": 2504916992 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001948565697091274, + "loss": 1.2463, + "theoretical_loss": 3.3964517923868236, + "tokens_seen": 2504982528 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001948405215646941, + "loss": 1.2308, + "theoretical_loss": 3.396444766755181, + "tokens_seen": 2505048064 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001948244734202608, + "loss": 1.279, + "theoretical_loss": 3.3964377413588, + "tokens_seen": 2505113600 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001948084252758275, + "loss": 1.2787, + "theoretical_loss": 3.396430716197667, + "tokens_seen": 2505179136 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001947923771313942, + "loss": 1.3077, + "theoretical_loss": 3.3964236912717682, + "tokens_seen": 2505244672 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001947763289869609, + "loss": 1.1953, + "theoretical_loss": 3.3964166665810898, + "tokens_seen": 2505310208 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001947602808425276, + "loss": 1.2642, + "theoretical_loss": 3.396409642125617, + "tokens_seen": 2505375744 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001947442326980943, + "loss": 1.2242, + "theoretical_loss": 3.396402617905336, + "tokens_seen": 2505441280 + }, + { + "epoch": 0.76, + "learning_rate": 0.000194728184553661, + "loss": 1.1512, + "theoretical_loss": 3.396395593920233, + "tokens_seen": 2505506816 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001947121364092277, + "loss": 1.2333, + "theoretical_loss": 3.396388570170294, + "tokens_seen": 2505572352 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001946960882647944, + "loss": 1.1901, + "theoretical_loss": 3.3963815466555047, + "tokens_seen": 2505637888 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001946800401203611, + "loss": 1.3146, + "theoretical_loss": 3.3963745233758513, + "tokens_seen": 2505703424 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001946639919759278, + "loss": 1.1958, + "theoretical_loss": 3.3963675003313196, + "tokens_seen": 2505768960 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019464794383149448, + "loss": 1.1709, + "theoretical_loss": 3.396360477521896, + "tokens_seen": 2505834496 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019463189568706118, + "loss": 1.1571, + "theoretical_loss": 3.396353454947566, + "tokens_seen": 2505900032 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019461584754262788, + "loss": 1.147, + "theoretical_loss": 3.3963464326083157, + "tokens_seen": 2505965568 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019459979939819458, + "loss": 1.2497, + "theoretical_loss": 3.3963394105041314, + "tokens_seen": 2506031104 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019458375125376128, + "loss": 1.3761, + "theoretical_loss": 3.396332388634998, + "tokens_seen": 2506096640 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019456770310932798, + "loss": 1.1774, + "theoretical_loss": 3.3963253670009035, + "tokens_seen": 2506162176 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019455165496489469, + "loss": 1.2169, + "theoretical_loss": 3.396318345601832, + "tokens_seen": 2506227712 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019453560682046139, + "loss": 1.2107, + "theoretical_loss": 3.3963113244377703, + "tokens_seen": 2506293248 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001945195586760281, + "loss": 1.3085, + "theoretical_loss": 3.396304303508704, + "tokens_seen": 2506358784 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001945035105315948, + "loss": 1.1103, + "theoretical_loss": 3.39629728281462, + "tokens_seen": 2506424320 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001944874623871615, + "loss": 1.1505, + "theoretical_loss": 3.396290262355503, + "tokens_seen": 2506489856 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001944714142427282, + "loss": 1.2547, + "theoretical_loss": 3.3962832421313403, + "tokens_seen": 2506555392 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001944553660982949, + "loss": 1.2668, + "theoretical_loss": 3.3962762221421166, + "tokens_seen": 2506620928 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001944393179538616, + "loss": 1.2191, + "theoretical_loss": 3.396269202387819, + "tokens_seen": 2506686464 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 837983, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3862534761428833, + "objective/train/theoretical_loss": 3.396262182868433, + "objective/train/tokens_used": 2527212000, + "theoretical_loss": 3.396262182868433, + "tokens_seen": 2506752000 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001944232698094283, + "loss": 1.3293, + "theoretical_loss": 3.396262182868433, + "tokens_seen": 2506752000 + }, + { + "epoch": 0.76, + "learning_rate": 0.000194407221664995, + "loss": 1.1991, + "theoretical_loss": 3.3962551635839446, + "tokens_seen": 2506817536 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001943911735205617, + "loss": 1.2173, + "theoretical_loss": 3.3962481445343395, + "tokens_seen": 2506883072 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001943751253761284, + "loss": 1.1538, + "theoretical_loss": 3.3962411257196043, + "tokens_seen": 2506948608 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001943590772316951, + "loss": 1.1887, + "theoretical_loss": 3.396234107139725, + "tokens_seen": 2507014144 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001943430290872618, + "loss": 1.2427, + "theoretical_loss": 3.396227088794687, + "tokens_seen": 2507079680 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001943269809428285, + "loss": 1.1937, + "theoretical_loss": 3.3962200706844765, + "tokens_seen": 2507145216 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001943109327983952, + "loss": 1.2303, + "theoretical_loss": 3.3962130528090797, + "tokens_seen": 2507210752 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001942948846539619, + "loss": 1.3851, + "theoretical_loss": 3.3962060351684826, + "tokens_seen": 2507276288 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001942788365095286, + "loss": 1.1938, + "theoretical_loss": 3.3961990177626715, + "tokens_seen": 2507341824 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001942627883650953, + "loss": 1.2786, + "theoretical_loss": 3.3961920005916317, + "tokens_seen": 2507407360 + }, + { + "epoch": 0.76, + "learning_rate": 0.000194246740220662, + "loss": 1.2737, + "theoretical_loss": 3.39618498365535, + "tokens_seen": 2507472896 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001942306920762287, + "loss": 1.2037, + "theoretical_loss": 3.3961779669538115, + "tokens_seen": 2507538432 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001942146439317954, + "loss": 1.1921, + "theoretical_loss": 3.3961709504870026, + "tokens_seen": 2507603968 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001941985957873621, + "loss": 1.2391, + "theoretical_loss": 3.39616393425491, + "tokens_seen": 2507669504 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001941825476429288, + "loss": 1.2024, + "theoretical_loss": 3.396156918257519, + "tokens_seen": 2507735040 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001941664994984955, + "loss": 1.3097, + "theoretical_loss": 3.396149902494815, + "tokens_seen": 2507800576 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001941504513540622, + "loss": 1.2206, + "theoretical_loss": 3.3961428869667856, + "tokens_seen": 2507866112 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001941344032096289, + "loss": 1.1699, + "theoretical_loss": 3.396135871673416, + "tokens_seen": 2507931648 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001941183550651956, + "loss": 1.3081, + "theoretical_loss": 3.3961288566146917, + "tokens_seen": 2507997184 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001941023069207623, + "loss": 1.2172, + "theoretical_loss": 3.3961218417905994, + "tokens_seen": 2508062720 + }, + { + "epoch": 0.76, + "learning_rate": 0.000194086258776329, + "loss": 1.161, + "theoretical_loss": 3.396114827201125, + "tokens_seen": 2508128256 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001940702106318957, + "loss": 1.2611, + "theoretical_loss": 3.396107812846254, + "tokens_seen": 2508193792 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001940541624874624, + "loss": 1.1761, + "theoretical_loss": 3.3961007987259735, + "tokens_seen": 2508259328 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001940381143430291, + "loss": 1.0773, + "theoretical_loss": 3.396093784840269, + "tokens_seen": 2508324864 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001940220661985958, + "loss": 1.1955, + "theoretical_loss": 3.396086771189126, + "tokens_seen": 2508390400 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001940060180541625, + "loss": 1.2193, + "theoretical_loss": 3.396079757772531, + "tokens_seen": 2508455936 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001939899699097292, + "loss": 1.1528, + "theoretical_loss": 3.39607274459047, + "tokens_seen": 2508521472 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001939739217652959, + "loss": 1.1683, + "theoretical_loss": 3.396065731642929, + "tokens_seen": 2508587008 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001939578736208626, + "loss": 1.2637, + "theoretical_loss": 3.3960587189298943, + "tokens_seen": 2508652544 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019394182547642928, + "loss": 1.1625, + "theoretical_loss": 3.3960517064513516, + "tokens_seen": 2508718080 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019392577733199598, + "loss": 1.2908, + "theoretical_loss": 3.396044694207287, + "tokens_seen": 2508783616 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019390972918756268, + "loss": 1.2296, + "theoretical_loss": 3.3960376821976865, + "tokens_seen": 2508849152 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019389368104312938, + "loss": 1.2768, + "theoretical_loss": 3.3960306704225363, + "tokens_seen": 2508914688 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019387763289869608, + "loss": 1.2666, + "theoretical_loss": 3.3960236588818224, + "tokens_seen": 2508980224 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019386158475426279, + "loss": 1.1791, + "theoretical_loss": 3.3960166475755305, + "tokens_seen": 2509045760 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019384553660982949, + "loss": 1.2245, + "theoretical_loss": 3.396009636503647, + "tokens_seen": 2509111296 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001938294884653962, + "loss": 1.2237, + "theoretical_loss": 3.3960026256661577, + "tokens_seen": 2509176832 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001938134403209629, + "loss": 1.1963, + "theoretical_loss": 3.395995615063049, + "tokens_seen": 2509242368 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001937973921765296, + "loss": 1.2684, + "theoretical_loss": 3.3959886046943066, + "tokens_seen": 2509307904 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001937813440320963, + "loss": 1.2072, + "theoretical_loss": 3.3959815945599168, + "tokens_seen": 2509373440 + }, + { + "epoch": 0.76, + "learning_rate": 0.000193765295887663, + "loss": 1.2022, + "theoretical_loss": 3.3959745846598657, + "tokens_seen": 2509438976 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001937492477432297, + "loss": 1.2486, + "theoretical_loss": 3.395967574994139, + "tokens_seen": 2509504512 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001937331995987964, + "loss": 1.1792, + "theoretical_loss": 3.395960565562723, + "tokens_seen": 2509570048 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001937171514543631, + "loss": 1.2311, + "theoretical_loss": 3.3959535563656034, + "tokens_seen": 2509635584 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001937011033099298, + "loss": 1.2832, + "theoretical_loss": 3.3959465474027666, + "tokens_seen": 2509701120 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019368505516549652, + "loss": 1.1687, + "theoretical_loss": 3.395939538674199, + "tokens_seen": 2509766656 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019366900702106322, + "loss": 1.2921, + "theoretical_loss": 3.3959325301798855, + "tokens_seen": 2509832192 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019365295887662992, + "loss": 1.2443, + "theoretical_loss": 3.3959255219198132, + "tokens_seen": 2509897728 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019363691073219662, + "loss": 1.1912, + "theoretical_loss": 3.3959185138939683, + "tokens_seen": 2509963264 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 838918, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.175501823425293, + "objective/train/theoretical_loss": 3.395911506102336, + "objective/train/tokens_used": 2530488800, + "theoretical_loss": 3.395911506102336, + "tokens_seen": 2510028800 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001936208625877633, + "loss": 1.205, + "theoretical_loss": 3.395911506102336, + "tokens_seen": 2510028800 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019360481444333, + "loss": 1.3494, + "theoretical_loss": 3.3959044985449025, + "tokens_seen": 2510094336 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001935887662988967, + "loss": 1.2559, + "theoretical_loss": 3.395897491221654, + "tokens_seen": 2510159872 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001935727181544634, + "loss": 1.216, + "theoretical_loss": 3.395890484132577, + "tokens_seen": 2510225408 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001935566700100301, + "loss": 1.323, + "theoretical_loss": 3.3958834772776574, + "tokens_seen": 2510290944 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001935406218655968, + "loss": 1.1334, + "theoretical_loss": 3.395876470656881, + "tokens_seen": 2510356480 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001935245737211635, + "loss": 1.224, + "theoretical_loss": 3.3958694642702336, + "tokens_seen": 2510422016 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001935085255767302, + "loss": 1.2977, + "theoretical_loss": 3.3958624581177017, + "tokens_seen": 2510487552 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001934924774322969, + "loss": 1.2504, + "theoretical_loss": 3.3958554521992714, + "tokens_seen": 2510553088 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001934764292878636, + "loss": 1.3186, + "theoretical_loss": 3.395848446514929, + "tokens_seen": 2510618624 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001934603811434303, + "loss": 1.2696, + "theoretical_loss": 3.39584144106466, + "tokens_seen": 2510684160 + }, + { + "epoch": 0.76, + "learning_rate": 0.000193444332998997, + "loss": 1.2094, + "theoretical_loss": 3.3958344358484505, + "tokens_seen": 2510749696 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001934282848545637, + "loss": 1.2751, + "theoretical_loss": 3.395827430866287, + "tokens_seen": 2510815232 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001934122367101304, + "loss": 1.2525, + "theoretical_loss": 3.3958204261181546, + "tokens_seen": 2510880768 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001933961885656971, + "loss": 1.2455, + "theoretical_loss": 3.395813421604041, + "tokens_seen": 2510946304 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001933801404212638, + "loss": 1.2042, + "theoretical_loss": 3.395806417323931, + "tokens_seen": 2511011840 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001933640922768305, + "loss": 1.2715, + "theoretical_loss": 3.395799413277811, + "tokens_seen": 2511077376 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001933480441323972, + "loss": 1.2831, + "theoretical_loss": 3.3957924094656673, + "tokens_seen": 2511142912 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001933319959879639, + "loss": 1.1423, + "theoretical_loss": 3.3957854058874855, + "tokens_seen": 2511208448 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001933159478435306, + "loss": 1.2279, + "theoretical_loss": 3.3957784025432525, + "tokens_seen": 2511273984 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001932998996990973, + "loss": 1.3087, + "theoretical_loss": 3.3957713994329533, + "tokens_seen": 2511339520 + }, + { + "epoch": 0.76, + "learning_rate": 0.000193283851554664, + "loss": 1.3316, + "theoretical_loss": 3.395764396556575, + "tokens_seen": 2511405056 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001932678034102307, + "loss": 1.3056, + "theoretical_loss": 3.395757393914103, + "tokens_seen": 2511470592 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001932517552657974, + "loss": 1.1784, + "theoretical_loss": 3.3957503915055236, + "tokens_seen": 2511536128 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019323570712136408, + "loss": 1.2264, + "theoretical_loss": 3.395743389330823, + "tokens_seen": 2511601664 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019321965897693078, + "loss": 1.27, + "theoretical_loss": 3.395736387389987, + "tokens_seen": 2511667200 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019320361083249748, + "loss": 1.3165, + "theoretical_loss": 3.395729385683002, + "tokens_seen": 2511732736 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019318756268806419, + "loss": 1.3195, + "theoretical_loss": 3.3957223842098543, + "tokens_seen": 2511798272 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019317151454363089, + "loss": 1.3362, + "theoretical_loss": 3.395715382970529, + "tokens_seen": 2511863808 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019315546639919759, + "loss": 1.2177, + "theoretical_loss": 3.3957083819650133, + "tokens_seen": 2511929344 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001931394182547643, + "loss": 1.2041, + "theoretical_loss": 3.395701381193293, + "tokens_seen": 2511994880 + }, + { + "epoch": 0.76, + "learning_rate": 0.000193123370110331, + "loss": 1.2994, + "theoretical_loss": 3.3956943806553532, + "tokens_seen": 2512060416 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001931073219658977, + "loss": 1.309, + "theoretical_loss": 3.3956873803511813, + "tokens_seen": 2512125952 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019309127382146442, + "loss": 1.2845, + "theoretical_loss": 3.395680380280763, + "tokens_seen": 2512191488 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019307522567703112, + "loss": 1.2844, + "theoretical_loss": 3.395673380444084, + "tokens_seen": 2512257024 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019305917753259782, + "loss": 1.3458, + "theoretical_loss": 3.3956663808411314, + "tokens_seen": 2512322560 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019304312938816452, + "loss": 1.1635, + "theoretical_loss": 3.39565938147189, + "tokens_seen": 2512388096 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019302708124373122, + "loss": 1.272, + "theoretical_loss": 3.3956523823363467, + "tokens_seen": 2512453632 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019301103309929792, + "loss": 1.2422, + "theoretical_loss": 3.3956453834344873, + "tokens_seen": 2512519168 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019299498495486462, + "loss": 1.1422, + "theoretical_loss": 3.3956383847662983, + "tokens_seen": 2512584704 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019297893681043132, + "loss": 1.1637, + "theoretical_loss": 3.395631386331765, + "tokens_seen": 2512650240 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019296288866599802, + "loss": 1.2324, + "theoretical_loss": 3.3956243881308747, + "tokens_seen": 2512715776 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019294684052156472, + "loss": 1.1093, + "theoretical_loss": 3.395617390163612, + "tokens_seen": 2512781312 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001929307923771314, + "loss": 1.2073, + "theoretical_loss": 3.3956103924299645, + "tokens_seen": 2512846848 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001929147442326981, + "loss": 1.218, + "theoretical_loss": 3.3956033949299176, + "tokens_seen": 2512912384 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001928986960882648, + "loss": 1.263, + "theoretical_loss": 3.395596397663457, + "tokens_seen": 2512977920 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001928826479438315, + "loss": 1.2031, + "theoretical_loss": 3.39558940063057, + "tokens_seen": 2513043456 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001928665997993982, + "loss": 1.1723, + "theoretical_loss": 3.3955824038312414, + "tokens_seen": 2513108992 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001928505516549649, + "loss": 1.3444, + "theoretical_loss": 3.395575407265458, + "tokens_seen": 2513174528 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001928345035105316, + "loss": 1.2819, + "theoretical_loss": 3.395568410933206, + "tokens_seen": 2513240064 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 840191, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3639895915985107, + "objective/train/theoretical_loss": 3.395561414834471, + "objective/train/tokens_used": 2533765600, + "theoretical_loss": 3.395561414834471, + "tokens_seen": 2513305600 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001928184553660983, + "loss": 1.3505, + "theoretical_loss": 3.395561414834471, + "tokens_seen": 2513305600 + }, + { + "epoch": 0.76, + "learning_rate": 0.000192802407221665, + "loss": 1.2204, + "theoretical_loss": 3.39555441896924, + "tokens_seen": 2513371136 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001927863590772317, + "loss": 1.2122, + "theoretical_loss": 3.395547423337498, + "tokens_seen": 2513436672 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001927703109327984, + "loss": 1.1819, + "theoretical_loss": 3.395540427939232, + "tokens_seen": 2513502208 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001927542627883651, + "loss": 1.2136, + "theoretical_loss": 3.3955334327744273, + "tokens_seen": 2513567744 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001927382146439318, + "loss": 1.3192, + "theoretical_loss": 3.395526437843071, + "tokens_seen": 2513633280 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001927221664994985, + "loss": 1.2008, + "theoretical_loss": 3.3955194431451488, + "tokens_seen": 2513698816 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001927061183550652, + "loss": 1.2369, + "theoretical_loss": 3.3955124486806465, + "tokens_seen": 2513764352 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001926900702106319, + "loss": 1.201, + "theoretical_loss": 3.3955054544495504, + "tokens_seen": 2513829888 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001926740220661986, + "loss": 1.2625, + "theoretical_loss": 3.3954984604518472, + "tokens_seen": 2513895424 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001926579739217653, + "loss": 1.3409, + "theoretical_loss": 3.395491466687522, + "tokens_seen": 2513960960 + }, + { + "epoch": 0.76, + "learning_rate": 0.000192641925777332, + "loss": 1.3522, + "theoretical_loss": 3.3954844731565617, + "tokens_seen": 2514026496 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001926258776328987, + "loss": 1.2753, + "theoretical_loss": 3.395477479858952, + "tokens_seen": 2514092032 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001926098294884654, + "loss": 1.3759, + "theoretical_loss": 3.3954704867946797, + "tokens_seen": 2514157568 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001925937813440321, + "loss": 1.1793, + "theoretical_loss": 3.3954634939637303, + "tokens_seen": 2514223104 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001925777331995988, + "loss": 1.1635, + "theoretical_loss": 3.3954565013660902, + "tokens_seen": 2514288640 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001925616850551655, + "loss": 1.2322, + "theoretical_loss": 3.3954495090017454, + "tokens_seen": 2514354176 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019254563691073218, + "loss": 1.2233, + "theoretical_loss": 3.395442516870682, + "tokens_seen": 2514419712 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019252958876629888, + "loss": 1.0636, + "theoretical_loss": 3.395435524972886, + "tokens_seen": 2514485248 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019251354062186558, + "loss": 1.3243, + "theoretical_loss": 3.395428533308344, + "tokens_seen": 2514550784 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019249749247743229, + "loss": 1.2092, + "theoretical_loss": 3.3954215418770417, + "tokens_seen": 2514616320 + }, + { + "epoch": 0.76, + "learning_rate": 0.000192481444332999, + "loss": 1.2317, + "theoretical_loss": 3.3954145506789657, + "tokens_seen": 2514681856 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019246539618856571, + "loss": 1.1753, + "theoretical_loss": 3.3954075597141014, + "tokens_seen": 2514747392 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019244934804413241, + "loss": 1.1243, + "theoretical_loss": 3.395400568982436, + "tokens_seen": 2514812928 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019243329989969911, + "loss": 1.3581, + "theoretical_loss": 3.395393578483955, + "tokens_seen": 2514878464 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019241725175526582, + "loss": 1.1478, + "theoretical_loss": 3.3953865882186443, + "tokens_seen": 2514944000 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019240120361083252, + "loss": 1.1222, + "theoretical_loss": 3.3953795981864903, + "tokens_seen": 2515009536 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019238515546639922, + "loss": 1.1794, + "theoretical_loss": 3.395372608387479, + "tokens_seen": 2515075072 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019236910732196592, + "loss": 1.2134, + "theoretical_loss": 3.3953656188215975, + "tokens_seen": 2515140608 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019235305917753262, + "loss": 1.1846, + "theoretical_loss": 3.3953586294888307, + "tokens_seen": 2515206144 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019233701103309932, + "loss": 1.3127, + "theoretical_loss": 3.3953516403891655, + "tokens_seen": 2515271680 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019232096288866602, + "loss": 1.1587, + "theoretical_loss": 3.3953446515225876, + "tokens_seen": 2515337216 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019230491474423272, + "loss": 1.3073, + "theoretical_loss": 3.3953376628890832, + "tokens_seen": 2515402752 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019228886659979942, + "loss": 1.1928, + "theoretical_loss": 3.3953306744886387, + "tokens_seen": 2515468288 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019227281845536612, + "loss": 1.2513, + "theoretical_loss": 3.3953236863212406, + "tokens_seen": 2515533824 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019225677031093282, + "loss": 1.1662, + "theoretical_loss": 3.3953166983868743, + "tokens_seen": 2515599360 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019224072216649952, + "loss": 1.2908, + "theoretical_loss": 3.395309710685526, + "tokens_seen": 2515664896 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001922246740220662, + "loss": 1.2539, + "theoretical_loss": 3.3953027232171826, + "tokens_seen": 2515730432 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001922086258776329, + "loss": 1.2325, + "theoretical_loss": 3.3952957359818297, + "tokens_seen": 2515795968 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001921925777331996, + "loss": 1.2379, + "theoretical_loss": 3.3952887489794534, + "tokens_seen": 2515861504 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001921765295887663, + "loss": 1.2414, + "theoretical_loss": 3.39528176221004, + "tokens_seen": 2515927040 + }, + { + "epoch": 0.76, + "learning_rate": 0.000192160481444333, + "loss": 1.2462, + "theoretical_loss": 3.395274775673576, + "tokens_seen": 2515992576 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001921444332998997, + "loss": 1.3523, + "theoretical_loss": 3.395267789370047, + "tokens_seen": 2516058112 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001921283851554664, + "loss": 1.2588, + "theoretical_loss": 3.39526080329944, + "tokens_seen": 2516123648 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001921123370110331, + "loss": 1.2847, + "theoretical_loss": 3.39525381746174, + "tokens_seen": 2516189184 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001920962888665998, + "loss": 1.3111, + "theoretical_loss": 3.3952468318569338, + "tokens_seen": 2516254720 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001920802407221665, + "loss": 1.2607, + "theoretical_loss": 3.3952398464850075, + "tokens_seen": 2516320256 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001920641925777332, + "loss": 1.1896, + "theoretical_loss": 3.3952328613459475, + "tokens_seen": 2516385792 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001920481444332999, + "loss": 1.2671, + "theoretical_loss": 3.39522587643974, + "tokens_seen": 2516451328 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001920320962888666, + "loss": 1.1638, + "theoretical_loss": 3.395218891766371, + "tokens_seen": 2516516864 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 841116, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0737740993499756, + "objective/train/theoretical_loss": 3.395211907325826, + "objective/train/tokens_used": 2537042400, + "theoretical_loss": 3.395211907325826, + "tokens_seen": 2516582400 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001920160481444333, + "loss": 1.1446, + "theoretical_loss": 3.395211907325826, + "tokens_seen": 2516582400 + }, + { + "epoch": 0.76, + "learning_rate": 0.000192, + "loss": 1.2054, + "theoretical_loss": 3.395204923118092, + "tokens_seen": 2516647936 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001919839518555667, + "loss": 1.3156, + "theoretical_loss": 3.395197939143155, + "tokens_seen": 2516713472 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001919679037111334, + "loss": 1.2779, + "theoretical_loss": 3.3951909554010014, + "tokens_seen": 2516779008 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001919518555667001, + "loss": 1.2375, + "theoretical_loss": 3.395183971891617, + "tokens_seen": 2516844544 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001919358074222668, + "loss": 1.238, + "theoretical_loss": 3.3951769886149883, + "tokens_seen": 2516910080 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001919197592778335, + "loss": 1.309, + "theoretical_loss": 3.3951700055711007, + "tokens_seen": 2516975616 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001919037111334002, + "loss": 1.3241, + "theoretical_loss": 3.3951630227599416, + "tokens_seen": 2517041152 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001918876629889669, + "loss": 1.3252, + "theoretical_loss": 3.3951560401814964, + "tokens_seen": 2517106688 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001918716148445336, + "loss": 1.2644, + "theoretical_loss": 3.3951490578357513, + "tokens_seen": 2517172224 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001918555667001003, + "loss": 1.2739, + "theoretical_loss": 3.395142075722693, + "tokens_seen": 2517237760 + }, + { + "epoch": 0.76, + "learning_rate": 0.000191839518555667, + "loss": 1.2622, + "theoretical_loss": 3.395135093842307, + "tokens_seen": 2517303296 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001918234704112337, + "loss": 1.2625, + "theoretical_loss": 3.39512811219458, + "tokens_seen": 2517368832 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001918074222668004, + "loss": 1.2306, + "theoretical_loss": 3.395121130779498, + "tokens_seen": 2517434368 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001917913741223671, + "loss": 1.1693, + "theoretical_loss": 3.3951141495970467, + "tokens_seen": 2517499904 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019177532597793381, + "loss": 1.2514, + "theoretical_loss": 3.3951071686472134, + "tokens_seen": 2517565440 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019175927783350051, + "loss": 1.2609, + "theoretical_loss": 3.395100187929984, + "tokens_seen": 2517630976 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019174322968906722, + "loss": 1.3038, + "theoretical_loss": 3.3950932074453433, + "tokens_seen": 2517696512 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019172718154463392, + "loss": 1.3479, + "theoretical_loss": 3.3950862271932793, + "tokens_seen": 2517762048 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019171113340020062, + "loss": 1.2301, + "theoretical_loss": 3.3950792471737774, + "tokens_seen": 2517827584 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019169508525576732, + "loss": 1.2599, + "theoretical_loss": 3.3950722673868237, + "tokens_seen": 2517893120 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019167903711133402, + "loss": 1.2017, + "theoretical_loss": 3.395065287832405, + "tokens_seen": 2517958656 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019166298896690072, + "loss": 1.2822, + "theoretical_loss": 3.3950583085105066, + "tokens_seen": 2518024192 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019164694082246742, + "loss": 1.1419, + "theoretical_loss": 3.3950513294211158, + "tokens_seen": 2518089728 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019163089267803412, + "loss": 1.1867, + "theoretical_loss": 3.3950443505642176, + "tokens_seen": 2518155264 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019161484453360082, + "loss": 1.1914, + "theoretical_loss": 3.395037371939799, + "tokens_seen": 2518220800 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019159879638916752, + "loss": 1.2774, + "theoretical_loss": 3.395030393547846, + "tokens_seen": 2518286336 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019158274824473422, + "loss": 1.2222, + "theoretical_loss": 3.3950234153883443, + "tokens_seen": 2518351872 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019156670010030092, + "loss": 1.2515, + "theoretical_loss": 3.3950164374612815, + "tokens_seen": 2518417408 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019155065195586762, + "loss": 1.3107, + "theoretical_loss": 3.3950094597666425, + "tokens_seen": 2518482944 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019153460381143432, + "loss": 1.2046, + "theoretical_loss": 3.3950024823044136, + "tokens_seen": 2518548480 + }, + { + "epoch": 0.76, + "learning_rate": 0.000191518555667001, + "loss": 1.2034, + "theoretical_loss": 3.394995505074582, + "tokens_seen": 2518614016 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001915025075225677, + "loss": 1.247, + "theoretical_loss": 3.394988528077133, + "tokens_seen": 2518679552 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001914864593781344, + "loss": 1.2045, + "theoretical_loss": 3.3949815513120525, + "tokens_seen": 2518745088 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001914704112337011, + "loss": 1.2915, + "theoretical_loss": 3.394974574779328, + "tokens_seen": 2518810624 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001914543630892678, + "loss": 1.337, + "theoretical_loss": 3.394967598478945, + "tokens_seen": 2518876160 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001914383149448345, + "loss": 1.2344, + "theoretical_loss": 3.3949606224108893, + "tokens_seen": 2518941696 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001914222668004012, + "loss": 1.2545, + "theoretical_loss": 3.3949536465751478, + "tokens_seen": 2519007232 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001914062186559679, + "loss": 1.2373, + "theoretical_loss": 3.394946670971706, + "tokens_seen": 2519072768 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001913901705115346, + "loss": 1.2537, + "theoretical_loss": 3.3949396956005513, + "tokens_seen": 2519138304 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001913741223671013, + "loss": 1.2587, + "theoretical_loss": 3.394932720461669, + "tokens_seen": 2519203840 + }, + { + "epoch": 0.76, + "learning_rate": 0.000191358074222668, + "loss": 1.3021, + "theoretical_loss": 3.394925745555045, + "tokens_seen": 2519269376 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001913420260782347, + "loss": 1.2178, + "theoretical_loss": 3.3949187708806665, + "tokens_seen": 2519334912 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001913259779338014, + "loss": 1.3025, + "theoretical_loss": 3.394911796438519, + "tokens_seen": 2519400448 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001913099297893681, + "loss": 1.3094, + "theoretical_loss": 3.3949048222285896, + "tokens_seen": 2519465984 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001912938816449348, + "loss": 1.2495, + "theoretical_loss": 3.394897848250863, + "tokens_seen": 2519531520 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019127783350050153, + "loss": 1.0934, + "theoretical_loss": 3.394890874505327, + "tokens_seen": 2519597056 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019126178535606823, + "loss": 1.1425, + "theoretical_loss": 3.3948839009919674, + "tokens_seen": 2519662592 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019124573721163493, + "loss": 1.2714, + "theoretical_loss": 3.3948769277107695, + "tokens_seen": 2519728128 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019122968906720164, + "loss": 1.1877, + "theoretical_loss": 3.3948699546617207, + "tokens_seen": 2519793664 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 842376, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2077178955078125, + "objective/train/theoretical_loss": 3.394862981844807, + "objective/train/tokens_used": 2540319200, + "theoretical_loss": 3.394862981844807, + "tokens_seen": 2519859200 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001912136409227683, + "loss": 1.2096, + "theoretical_loss": 3.394862981844807, + "tokens_seen": 2519859200 + }, + { + "epoch": 0.76, + "learning_rate": 0.000191197592778335, + "loss": 1.208, + "theoretical_loss": 3.394856009260014, + "tokens_seen": 2519924736 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001911815446339017, + "loss": 1.4849, + "theoretical_loss": 3.3948490369073285, + "tokens_seen": 2519990272 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001911654964894684, + "loss": 1.2556, + "theoretical_loss": 3.3948420647867366, + "tokens_seen": 2520055808 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001911494483450351, + "loss": 1.2083, + "theoretical_loss": 3.3948350928982247, + "tokens_seen": 2520121344 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001911334002006018, + "loss": 1.2162, + "theoretical_loss": 3.394828121241779, + "tokens_seen": 2520186880 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001911173520561685, + "loss": 1.1963, + "theoretical_loss": 3.3948211498173846, + "tokens_seen": 2520252416 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001911013039117352, + "loss": 1.2108, + "theoretical_loss": 3.3948141786250297, + "tokens_seen": 2520317952 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019108525576730191, + "loss": 1.3027, + "theoretical_loss": 3.3948072076646993, + "tokens_seen": 2520383488 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019106920762286861, + "loss": 1.2871, + "theoretical_loss": 3.39480023693638, + "tokens_seen": 2520449024 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019105315947843532, + "loss": 1.2297, + "theoretical_loss": 3.394793266440058, + "tokens_seen": 2520514560 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019103711133400202, + "loss": 1.2092, + "theoretical_loss": 3.39478629617572, + "tokens_seen": 2520580096 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019102106318956872, + "loss": 1.1718, + "theoretical_loss": 3.3947793261433508, + "tokens_seen": 2520645632 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019100501504513542, + "loss": 1.3035, + "theoretical_loss": 3.3947723563429384, + "tokens_seen": 2520711168 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019098896690070212, + "loss": 1.216, + "theoretical_loss": 3.394765386774468, + "tokens_seen": 2520776704 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019097291875626882, + "loss": 1.2002, + "theoretical_loss": 3.394758417437926, + "tokens_seen": 2520842240 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019095687061183552, + "loss": 1.1021, + "theoretical_loss": 3.3947514483332992, + "tokens_seen": 2520907776 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019094082246740222, + "loss": 1.3205, + "theoretical_loss": 3.394744479460573, + "tokens_seen": 2520973312 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019092477432296892, + "loss": 1.1865, + "theoretical_loss": 3.394737510819734, + "tokens_seen": 2521038848 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019090872617853562, + "loss": 1.2165, + "theoretical_loss": 3.394730542410769, + "tokens_seen": 2521104384 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019089267803410232, + "loss": 1.2311, + "theoretical_loss": 3.394723574233664, + "tokens_seen": 2521169920 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019087662988966902, + "loss": 1.2046, + "theoretical_loss": 3.3947166062884047, + "tokens_seen": 2521235456 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019086058174523572, + "loss": 1.2477, + "theoretical_loss": 3.3947096385749775, + "tokens_seen": 2521300992 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019084453360080242, + "loss": 1.2479, + "theoretical_loss": 3.3947026710933694, + "tokens_seen": 2521366528 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001908284854563691, + "loss": 1.2128, + "theoretical_loss": 3.394695703843566, + "tokens_seen": 2521432064 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001908124373119358, + "loss": 1.2739, + "theoretical_loss": 3.3946887368255534, + "tokens_seen": 2521497600 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001907963891675025, + "loss": 1.2322, + "theoretical_loss": 3.394681770039319, + "tokens_seen": 2521563136 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001907803410230692, + "loss": 1.2407, + "theoretical_loss": 3.3946748034848473, + "tokens_seen": 2521628672 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001907642928786359, + "loss": 1.2945, + "theoretical_loss": 3.3946678371621255, + "tokens_seen": 2521694208 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001907482447342026, + "loss": 1.2716, + "theoretical_loss": 3.3946608710711406, + "tokens_seen": 2521759744 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001907321965897693, + "loss": 1.2766, + "theoretical_loss": 3.394653905211878, + "tokens_seen": 2521825280 + }, + { + "epoch": 0.76, + "learning_rate": 0.000190716148445336, + "loss": 1.3418, + "theoretical_loss": 3.3946469395843235, + "tokens_seen": 2521890816 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001907001003009027, + "loss": 1.2245, + "theoretical_loss": 3.3946399741884647, + "tokens_seen": 2521956352 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019068405215646943, + "loss": 1.1483, + "theoretical_loss": 3.394633009024287, + "tokens_seen": 2522021888 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019066800401203613, + "loss": 1.2167, + "theoretical_loss": 3.3946260440917766, + "tokens_seen": 2522087424 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019065195586760283, + "loss": 1.3095, + "theoretical_loss": 3.39461907939092, + "tokens_seen": 2522152960 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019063590772316953, + "loss": 1.2455, + "theoretical_loss": 3.3946121149217037, + "tokens_seen": 2522218496 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019061985957873623, + "loss": 1.1952, + "theoretical_loss": 3.3946051506841135, + "tokens_seen": 2522284032 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019060381143430293, + "loss": 1.1452, + "theoretical_loss": 3.3945981866781363, + "tokens_seen": 2522349568 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019058776328986963, + "loss": 1.3721, + "theoretical_loss": 3.394591222903758, + "tokens_seen": 2522415104 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019057171514543633, + "loss": 1.2393, + "theoretical_loss": 3.3945842593609643, + "tokens_seen": 2522480640 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019055566700100303, + "loss": 1.3651, + "theoretical_loss": 3.394577296049743, + "tokens_seen": 2522546176 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019053961885656974, + "loss": 1.2264, + "theoretical_loss": 3.394570332970079, + "tokens_seen": 2522611712 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019052357071213644, + "loss": 1.2683, + "theoretical_loss": 3.3945633701219586, + "tokens_seen": 2522677248 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001905075225677031, + "loss": 1.3467, + "theoretical_loss": 3.394556407505369, + "tokens_seen": 2522742784 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001904914744232698, + "loss": 1.231, + "theoretical_loss": 3.3945494451202967, + "tokens_seen": 2522808320 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001904754262788365, + "loss": 1.2425, + "theoretical_loss": 3.3945424829667266, + "tokens_seen": 2522873856 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001904593781344032, + "loss": 1.1649, + "theoretical_loss": 3.3945355210446455, + "tokens_seen": 2522939392 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001904433299899699, + "loss": 1.2585, + "theoretical_loss": 3.39452855935404, + "tokens_seen": 2523004928 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001904272818455366, + "loss": 1.2358, + "theoretical_loss": 3.3945215978948964, + "tokens_seen": 2523070464 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 843632, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2696731090545654, + "objective/train/theoretical_loss": 3.394514636667201, + "objective/train/tokens_used": 2543596000, + "theoretical_loss": 3.394514636667201, + "tokens_seen": 2523136000 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001904112337011033, + "loss": 1.3735, + "theoretical_loss": 3.394514636667201, + "tokens_seen": 2523136000 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019039518555667001, + "loss": 1.3104, + "theoretical_loss": 3.39450767567094, + "tokens_seen": 2523201536 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019037913741223671, + "loss": 1.2269, + "theoretical_loss": 3.3945007149060995, + "tokens_seen": 2523267072 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019036308926780342, + "loss": 1.1875, + "theoretical_loss": 3.3944937543726663, + "tokens_seen": 2523332608 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019034704112337012, + "loss": 1.2556, + "theoretical_loss": 3.394486794070626, + "tokens_seen": 2523398144 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019033099297893682, + "loss": 1.1804, + "theoretical_loss": 3.394479833999965, + "tokens_seen": 2523463680 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019031494483450352, + "loss": 1.2697, + "theoretical_loss": 3.3944728741606705, + "tokens_seen": 2523529216 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019029889669007022, + "loss": 1.2328, + "theoretical_loss": 3.394465914552728, + "tokens_seen": 2523594752 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019028284854563692, + "loss": 1.1921, + "theoretical_loss": 3.3944589551761237, + "tokens_seen": 2523660288 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019026680040120362, + "loss": 1.2014, + "theoretical_loss": 3.394451996030844, + "tokens_seen": 2523725824 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019025075225677032, + "loss": 1.2345, + "theoretical_loss": 3.394445037116876, + "tokens_seen": 2523791360 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019023470411233702, + "loss": 1.2752, + "theoretical_loss": 3.394438078434205, + "tokens_seen": 2523856896 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019021865596790372, + "loss": 1.2722, + "theoretical_loss": 3.3944311199828174, + "tokens_seen": 2523922432 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019020260782347042, + "loss": 1.1993, + "theoretical_loss": 3.3944241617627005, + "tokens_seen": 2523987968 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019018655967903712, + "loss": 1.2526, + "theoretical_loss": 3.3944172037738394, + "tokens_seen": 2524053504 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019017051153460382, + "loss": 1.2317, + "theoretical_loss": 3.394410246016221, + "tokens_seen": 2524119040 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019015446339017052, + "loss": 1.1943, + "theoretical_loss": 3.3944032884898316, + "tokens_seen": 2524184576 + }, + { + "epoch": 0.76, + "learning_rate": 0.00019013841524573722, + "loss": 1.25, + "theoretical_loss": 3.394396331194657, + "tokens_seen": 2524250112 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001901223671013039, + "loss": 1.2104, + "theoretical_loss": 3.3943893741306845, + "tokens_seen": 2524315648 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001901063189568706, + "loss": 1.2524, + "theoretical_loss": 3.3943824172978996, + "tokens_seen": 2524381184 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001900902708124373, + "loss": 1.2413, + "theoretical_loss": 3.3943754606962893, + "tokens_seen": 2524446720 + }, + { + "epoch": 0.77, + "learning_rate": 0.00019007422266800403, + "loss": 1.1799, + "theoretical_loss": 3.394368504325839, + "tokens_seen": 2524512256 + }, + { + "epoch": 0.77, + "learning_rate": 0.00019005817452357073, + "loss": 1.185, + "theoretical_loss": 3.3943615481865352, + "tokens_seen": 2524577792 + }, + { + "epoch": 0.77, + "learning_rate": 0.00019004212637913743, + "loss": 1.2198, + "theoretical_loss": 3.394354592278365, + "tokens_seen": 2524643328 + }, + { + "epoch": 0.77, + "learning_rate": 0.00019002607823470413, + "loss": 1.2875, + "theoretical_loss": 3.3943476366013146, + "tokens_seen": 2524708864 + }, + { + "epoch": 0.77, + "learning_rate": 0.00019001003009027083, + "loss": 1.2503, + "theoretical_loss": 3.394340681155369, + "tokens_seen": 2524774400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018999398194583753, + "loss": 1.2372, + "theoretical_loss": 3.394333725940516, + "tokens_seen": 2524839936 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018997793380140423, + "loss": 1.2054, + "theoretical_loss": 3.394326770956742, + "tokens_seen": 2524905472 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018996188565697093, + "loss": 1.2249, + "theoretical_loss": 3.3943198162040322, + "tokens_seen": 2524971008 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018994583751253763, + "loss": 1.1985, + "theoretical_loss": 3.3943128616823737, + "tokens_seen": 2525036544 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018992978936810433, + "loss": 1.3384, + "theoretical_loss": 3.394305907391752, + "tokens_seen": 2525102080 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018991374122367103, + "loss": 1.2015, + "theoretical_loss": 3.3942989533321546, + "tokens_seen": 2525167616 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018989769307923773, + "loss": 1.3059, + "theoretical_loss": 3.394291999503567, + "tokens_seen": 2525233152 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018988164493480443, + "loss": 1.2896, + "theoretical_loss": 3.394285045905976, + "tokens_seen": 2525298688 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018986559679037114, + "loss": 1.2271, + "theoretical_loss": 3.3942780925393676, + "tokens_seen": 2525364224 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018984954864593784, + "loss": 1.2369, + "theoretical_loss": 3.394271139403728, + "tokens_seen": 2525429760 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018983350050150454, + "loss": 1.2166, + "theoretical_loss": 3.3942641864990444, + "tokens_seen": 2525495296 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018981745235707124, + "loss": 1.2171, + "theoretical_loss": 3.394257233825302, + "tokens_seen": 2525560832 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001898014042126379, + "loss": 1.1671, + "theoretical_loss": 3.3942502813824875, + "tokens_seen": 2525626368 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001897853560682046, + "loss": 1.2539, + "theoretical_loss": 3.394243329170588, + "tokens_seen": 2525691904 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001897693079237713, + "loss": 1.2905, + "theoretical_loss": 3.3942363771895887, + "tokens_seen": 2525757440 + }, + { + "epoch": 0.77, + "learning_rate": 0.000189753259779338, + "loss": 1.2241, + "theoretical_loss": 3.3942294254394767, + "tokens_seen": 2525822976 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001897372116349047, + "loss": 1.2278, + "theoretical_loss": 3.3942224739202382, + "tokens_seen": 2525888512 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018972116349047141, + "loss": 1.2543, + "theoretical_loss": 3.3942155226318595, + "tokens_seen": 2525954048 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018970511534603811, + "loss": 1.2532, + "theoretical_loss": 3.3942085715743264, + "tokens_seen": 2526019584 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018968906720160481, + "loss": 1.2555, + "theoretical_loss": 3.3942016207476264, + "tokens_seen": 2526085120 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018967301905717152, + "loss": 1.2781, + "theoretical_loss": 3.394194670151745, + "tokens_seen": 2526150656 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018965697091273822, + "loss": 1.2557, + "theoretical_loss": 3.3941877197866686, + "tokens_seen": 2526216192 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018964092276830492, + "loss": 1.2569, + "theoretical_loss": 3.394180769652384, + "tokens_seen": 2526281728 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018962487462387162, + "loss": 1.1863, + "theoretical_loss": 3.3941738197488767, + "tokens_seen": 2526347264 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 844616, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1497843265533447, + "objective/train/theoretical_loss": 3.394166870076134, + "objective/train/tokens_used": 2546872800, + "theoretical_loss": 3.394166870076134, + "tokens_seen": 2526412800 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018960882647943832, + "loss": 1.1828, + "theoretical_loss": 3.394166870076134, + "tokens_seen": 2526412800 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018959277833500502, + "loss": 1.2301, + "theoretical_loss": 3.3941599206341415, + "tokens_seen": 2526478336 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018957673019057172, + "loss": 1.1905, + "theoretical_loss": 3.394152971422886, + "tokens_seen": 2526543872 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018956068204613842, + "loss": 1.2673, + "theoretical_loss": 3.3941460224423543, + "tokens_seen": 2526609408 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018954463390170512, + "loss": 1.2414, + "theoretical_loss": 3.3941390736925316, + "tokens_seen": 2526674944 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018952858575727182, + "loss": 1.3804, + "theoretical_loss": 3.394132125173405, + "tokens_seen": 2526740480 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018951253761283852, + "loss": 1.127, + "theoretical_loss": 3.3941251768849607, + "tokens_seen": 2526806016 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018949648946840522, + "loss": 1.3409, + "theoretical_loss": 3.3941182288271854, + "tokens_seen": 2526871552 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018948044132397192, + "loss": 1.2422, + "theoretical_loss": 3.394111281000065, + "tokens_seen": 2526937088 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018946439317953862, + "loss": 1.2815, + "theoretical_loss": 3.3941043334035856, + "tokens_seen": 2527002624 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018944834503510532, + "loss": 1.3702, + "theoretical_loss": 3.3940973860377346, + "tokens_seen": 2527068160 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018943229689067203, + "loss": 1.1912, + "theoretical_loss": 3.3940904389024973, + "tokens_seen": 2527133696 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018941624874623873, + "loss": 1.1665, + "theoretical_loss": 3.3940834919978604, + "tokens_seen": 2527199232 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018940020060180543, + "loss": 1.1886, + "theoretical_loss": 3.3940765453238106, + "tokens_seen": 2527264768 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018938415245737213, + "loss": 1.2574, + "theoretical_loss": 3.3940695988803338, + "tokens_seen": 2527330304 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018936810431293883, + "loss": 1.2512, + "theoretical_loss": 3.394062652667417, + "tokens_seen": 2527395840 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018935205616850553, + "loss": 1.0968, + "theoretical_loss": 3.394055706685046, + "tokens_seen": 2527461376 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018933600802407223, + "loss": 1.1788, + "theoretical_loss": 3.394048760933207, + "tokens_seen": 2527526912 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018931995987963893, + "loss": 1.1762, + "theoretical_loss": 3.394041815411887, + "tokens_seen": 2527592448 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018930391173520563, + "loss": 1.2626, + "theoretical_loss": 3.394034870121072, + "tokens_seen": 2527657984 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018928786359077233, + "loss": 1.1391, + "theoretical_loss": 3.3940279250607484, + "tokens_seen": 2527723520 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018927181544633903, + "loss": 1.2426, + "theoretical_loss": 3.394020980230903, + "tokens_seen": 2527789056 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018925576730190573, + "loss": 1.1303, + "theoretical_loss": 3.394014035631521, + "tokens_seen": 2527854592 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018923971915747243, + "loss": 1.2078, + "theoretical_loss": 3.3940070912625906, + "tokens_seen": 2527920128 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018922367101303913, + "loss": 1.2496, + "theoretical_loss": 3.394000147124096, + "tokens_seen": 2527985664 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018920762286860583, + "loss": 1.2562, + "theoretical_loss": 3.393993203216026, + "tokens_seen": 2528051200 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018919157472417253, + "loss": 1.289, + "theoretical_loss": 3.393986259538365, + "tokens_seen": 2528116736 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018917552657973924, + "loss": 1.3237, + "theoretical_loss": 3.3939793160911, + "tokens_seen": 2528182272 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018915947843530594, + "loss": 1.1863, + "theoretical_loss": 3.3939723728742175, + "tokens_seen": 2528247808 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018914343029087264, + "loss": 1.3418, + "theoretical_loss": 3.393965429887704, + "tokens_seen": 2528313344 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018912738214643934, + "loss": 1.2635, + "theoretical_loss": 3.393958487131546, + "tokens_seen": 2528378880 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018911133400200604, + "loss": 1.1688, + "theoretical_loss": 3.3939515446057293, + "tokens_seen": 2528444416 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001890952858575727, + "loss": 1.2017, + "theoretical_loss": 3.393944602310241, + "tokens_seen": 2528509952 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001890792377131394, + "loss": 1.2106, + "theoretical_loss": 3.393937660245067, + "tokens_seen": 2528575488 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001890631895687061, + "loss": 1.222, + "theoretical_loss": 3.3939307184101932, + "tokens_seen": 2528641024 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001890471414242728, + "loss": 1.112, + "theoretical_loss": 3.3939237768056074, + "tokens_seen": 2528706560 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018903109327983951, + "loss": 1.2588, + "theoretical_loss": 3.3939168354312947, + "tokens_seen": 2528772096 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018901504513540621, + "loss": 1.1692, + "theoretical_loss": 3.3939098942872423, + "tokens_seen": 2528837632 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018899899699097292, + "loss": 1.1448, + "theoretical_loss": 3.393902953373436, + "tokens_seen": 2528903168 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018898294884653962, + "loss": 1.2601, + "theoretical_loss": 3.3938960126898623, + "tokens_seen": 2528968704 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018896690070210632, + "loss": 1.3064, + "theoretical_loss": 3.3938890722365085, + "tokens_seen": 2529034240 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018895085255767302, + "loss": 1.1788, + "theoretical_loss": 3.39388213201336, + "tokens_seen": 2529099776 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018893480441323972, + "loss": 1.2701, + "theoretical_loss": 3.393875192020403, + "tokens_seen": 2529165312 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018891875626880642, + "loss": 1.1069, + "theoretical_loss": 3.393868252257625, + "tokens_seen": 2529230848 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018890270812437312, + "loss": 1.1892, + "theoretical_loss": 3.393861312725011, + "tokens_seen": 2529296384 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018888665997993985, + "loss": 1.112, + "theoretical_loss": 3.393854373422549, + "tokens_seen": 2529361920 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018887061183550655, + "loss": 1.1283, + "theoretical_loss": 3.3938474343502243, + "tokens_seen": 2529427456 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018885456369107325, + "loss": 1.2236, + "theoretical_loss": 3.393840495508023, + "tokens_seen": 2529492992 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018883851554663995, + "loss": 1.2219, + "theoretical_loss": 3.393833556895933, + "tokens_seen": 2529558528 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018882246740220665, + "loss": 1.2756, + "theoretical_loss": 3.3938266185139394, + "tokens_seen": 2529624064 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 845397, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1955511569976807, + "objective/train/theoretical_loss": 3.3938196803620286, + "objective/train/tokens_used": 2550149600, + "theoretical_loss": 3.3938196803620286, + "tokens_seen": 2529689600 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018880641925777335, + "loss": 1.2375, + "theoretical_loss": 3.3938196803620286, + "tokens_seen": 2529689600 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018879037111334002, + "loss": 1.178, + "theoretical_loss": 3.393812742440188, + "tokens_seen": 2529755136 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018877432296890672, + "loss": 1.2921, + "theoretical_loss": 3.3938058047484034, + "tokens_seen": 2529820672 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018875827482447342, + "loss": 1.1552, + "theoretical_loss": 3.3937988672866606, + "tokens_seen": 2529886208 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018874222668004013, + "loss": 1.2997, + "theoretical_loss": 3.3937919300549475, + "tokens_seen": 2529951744 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018872617853560683, + "loss": 1.185, + "theoretical_loss": 3.393784993053249, + "tokens_seen": 2530017280 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018871013039117353, + "loss": 1.23, + "theoretical_loss": 3.393778056281553, + "tokens_seen": 2530082816 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018869408224674023, + "loss": 1.305, + "theoretical_loss": 3.3937711197398444, + "tokens_seen": 2530148352 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018867803410230693, + "loss": 1.2321, + "theoretical_loss": 3.3937641834281105, + "tokens_seen": 2530213888 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018866198595787363, + "loss": 1.3437, + "theoretical_loss": 3.3937572473463375, + "tokens_seen": 2530279424 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018864593781344033, + "loss": 1.2402, + "theoretical_loss": 3.393750311494512, + "tokens_seen": 2530344960 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018862988966900703, + "loss": 1.1274, + "theoretical_loss": 3.39374337587262, + "tokens_seen": 2530410496 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018861384152457373, + "loss": 1.3974, + "theoretical_loss": 3.3937364404806485, + "tokens_seen": 2530476032 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018859779338014043, + "loss": 1.1792, + "theoretical_loss": 3.3937295053185834, + "tokens_seen": 2530541568 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018858174523570713, + "loss": 1.2095, + "theoretical_loss": 3.3937225703864113, + "tokens_seen": 2530607104 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018856569709127383, + "loss": 1.177, + "theoretical_loss": 3.393715635684119, + "tokens_seen": 2530672640 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018854964894684053, + "loss": 1.245, + "theoretical_loss": 3.3937087012116924, + "tokens_seen": 2530738176 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018853360080240723, + "loss": 1.2846, + "theoretical_loss": 3.393701766969118, + "tokens_seen": 2530803712 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018851755265797393, + "loss": 1.2408, + "theoretical_loss": 3.3936948329563825, + "tokens_seen": 2530869248 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018850150451354063, + "loss": 1.2842, + "theoretical_loss": 3.3936878991734725, + "tokens_seen": 2530934784 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018848545636910734, + "loss": 1.1367, + "theoretical_loss": 3.3936809656203737, + "tokens_seen": 2531000320 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018846940822467404, + "loss": 1.1785, + "theoretical_loss": 3.393674032297073, + "tokens_seen": 2531065856 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018845336008024074, + "loss": 1.2866, + "theoretical_loss": 3.3936670992035567, + "tokens_seen": 2531131392 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018843731193580744, + "loss": 1.2279, + "theoretical_loss": 3.393660166339812, + "tokens_seen": 2531196928 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018842126379137414, + "loss": 1.1593, + "theoretical_loss": 3.3936532337058236, + "tokens_seen": 2531262464 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001884052156469408, + "loss": 1.2492, + "theoretical_loss": 3.3936463013015796, + "tokens_seen": 2531328000 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001883891675025075, + "loss": 1.23, + "theoretical_loss": 3.393639369127066, + "tokens_seen": 2531393536 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001883731193580742, + "loss": 1.2131, + "theoretical_loss": 3.393632437182269, + "tokens_seen": 2531459072 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001883570712136409, + "loss": 1.308, + "theoretical_loss": 3.393625505467175, + "tokens_seen": 2531524608 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018834102306920761, + "loss": 1.2405, + "theoretical_loss": 3.3936185739817706, + "tokens_seen": 2531590144 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018832497492477431, + "loss": 1.2531, + "theoretical_loss": 3.393611642726042, + "tokens_seen": 2531655680 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018830892678034102, + "loss": 1.3061, + "theoretical_loss": 3.393604711699976, + "tokens_seen": 2531721216 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018829287863590772, + "loss": 1.2617, + "theoretical_loss": 3.393597780903559, + "tokens_seen": 2531786752 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018827683049147444, + "loss": 1.2688, + "theoretical_loss": 3.393590850336777, + "tokens_seen": 2531852288 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018826078234704114, + "loss": 1.1987, + "theoretical_loss": 3.393583919999617, + "tokens_seen": 2531917824 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018824473420260784, + "loss": 1.2373, + "theoretical_loss": 3.393576989892065, + "tokens_seen": 2531983360 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018822868605817455, + "loss": 1.2014, + "theoretical_loss": 3.3935700600141083, + "tokens_seen": 2532048896 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018821263791374125, + "loss": 1.133, + "theoretical_loss": 3.3935631303657323, + "tokens_seen": 2532114432 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018819658976930795, + "loss": 1.2373, + "theoretical_loss": 3.3935562009469242, + "tokens_seen": 2532179968 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018818054162487465, + "loss": 1.2476, + "theoretical_loss": 3.39354927175767, + "tokens_seen": 2532245504 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018816449348044135, + "loss": 1.1011, + "theoretical_loss": 3.393542342797956, + "tokens_seen": 2532311040 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018814844533600805, + "loss": 1.2366, + "theoretical_loss": 3.3935354140677694, + "tokens_seen": 2532376576 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018813239719157475, + "loss": 1.1558, + "theoretical_loss": 3.3935284855670957, + "tokens_seen": 2532442112 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018811634904714145, + "loss": 1.2294, + "theoretical_loss": 3.393521557295922, + "tokens_seen": 2532507648 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018810030090270815, + "loss": 1.2017, + "theoretical_loss": 3.393514629254235, + "tokens_seen": 2532573184 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018808425275827482, + "loss": 1.2824, + "theoretical_loss": 3.3935077014420205, + "tokens_seen": 2532638720 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018806820461384152, + "loss": 1.2697, + "theoretical_loss": 3.393500773859265, + "tokens_seen": 2532704256 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018805215646940823, + "loss": 1.3512, + "theoretical_loss": 3.393493846505956, + "tokens_seen": 2532769792 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018803610832497493, + "loss": 1.2202, + "theoretical_loss": 3.3934869193820782, + "tokens_seen": 2532835328 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018802006018054163, + "loss": 1.2925, + "theoretical_loss": 3.39347999248762, + "tokens_seen": 2532900864 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 846552, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3343746662139893, + "objective/train/theoretical_loss": 3.3934730658225662, + "objective/train/tokens_used": 2553426400, + "theoretical_loss": 3.3934730658225662, + "tokens_seen": 2532966400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018800401203610833, + "loss": 1.2571, + "theoretical_loss": 3.3934730658225662, + "tokens_seen": 2532966400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018798796389167503, + "loss": 1.2587, + "theoretical_loss": 3.3934661393869043, + "tokens_seen": 2533031936 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018797191574724173, + "loss": 1.2321, + "theoretical_loss": 3.3934592131806207, + "tokens_seen": 2533097472 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018795586760280843, + "loss": 1.1838, + "theoretical_loss": 3.393452287203701, + "tokens_seen": 2533163008 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018793981945837513, + "loss": 1.3736, + "theoretical_loss": 3.393445361456133, + "tokens_seen": 2533228544 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018792377131394183, + "loss": 1.2323, + "theoretical_loss": 3.393438435937902, + "tokens_seen": 2533294080 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018790772316950853, + "loss": 1.2497, + "theoretical_loss": 3.3934315106489947, + "tokens_seen": 2533359616 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018789167502507523, + "loss": 1.2638, + "theoretical_loss": 3.393424585589398, + "tokens_seen": 2533425152 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018787562688064193, + "loss": 1.2749, + "theoretical_loss": 3.3934176607590985, + "tokens_seen": 2533490688 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018785957873620863, + "loss": 1.2644, + "theoretical_loss": 3.393410736158082, + "tokens_seen": 2533556224 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018784353059177533, + "loss": 1.2245, + "theoretical_loss": 3.3934038117863357, + "tokens_seen": 2533621760 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018782748244734203, + "loss": 1.2003, + "theoretical_loss": 3.393396887643845, + "tokens_seen": 2533687296 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018781143430290873, + "loss": 1.1759, + "theoretical_loss": 3.393389963730598, + "tokens_seen": 2533752832 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018779538615847544, + "loss": 1.2726, + "theoretical_loss": 3.3933830400465794, + "tokens_seen": 2533818368 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018777933801404214, + "loss": 1.2588, + "theoretical_loss": 3.393376116591777, + "tokens_seen": 2533883904 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018776328986960884, + "loss": 1.1765, + "theoretical_loss": 3.393369193366177, + "tokens_seen": 2533949440 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018774724172517554, + "loss": 1.1964, + "theoretical_loss": 3.3933622703697655, + "tokens_seen": 2534014976 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018773119358074224, + "loss": 1.1999, + "theoretical_loss": 3.3933553476025295, + "tokens_seen": 2534080512 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018771514543630894, + "loss": 1.3008, + "theoretical_loss": 3.393348425064455, + "tokens_seen": 2534146048 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001876990972918756, + "loss": 1.1588, + "theoretical_loss": 3.3933415027555287, + "tokens_seen": 2534211584 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018768304914744234, + "loss": 1.149, + "theoretical_loss": 3.393334580675737, + "tokens_seen": 2534277120 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018766700100300904, + "loss": 1.172, + "theoretical_loss": 3.3933276588250667, + "tokens_seen": 2534342656 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018765095285857574, + "loss": 1.197, + "theoretical_loss": 3.3933207372035037, + "tokens_seen": 2534408192 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018763490471414244, + "loss": 1.3048, + "theoretical_loss": 3.3933138158110348, + "tokens_seen": 2534473728 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018761885656970914, + "loss": 1.2108, + "theoretical_loss": 3.393306894647647, + "tokens_seen": 2534539264 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018760280842527584, + "loss": 1.3318, + "theoretical_loss": 3.393299973713326, + "tokens_seen": 2534604800 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018758676028084254, + "loss": 1.2412, + "theoretical_loss": 3.3932930530080587, + "tokens_seen": 2534670336 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018757071213640924, + "loss": 1.1867, + "theoretical_loss": 3.3932861325318315, + "tokens_seen": 2534735872 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018755466399197595, + "loss": 1.2762, + "theoretical_loss": 3.3932792122846314, + "tokens_seen": 2534801408 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018753861584754265, + "loss": 1.2688, + "theoretical_loss": 3.3932722922664436, + "tokens_seen": 2534866944 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018752256770310935, + "loss": 1.2418, + "theoretical_loss": 3.393265372477256, + "tokens_seen": 2534932480 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018750651955867605, + "loss": 1.1165, + "theoretical_loss": 3.3932584529170544, + "tokens_seen": 2534998016 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018749047141424275, + "loss": 1.1885, + "theoretical_loss": 3.3932515335858255, + "tokens_seen": 2535063552 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018747442326980945, + "loss": 1.2653, + "theoretical_loss": 3.3932446144835557, + "tokens_seen": 2535129088 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018745837512537615, + "loss": 1.2512, + "theoretical_loss": 3.3932376956102313, + "tokens_seen": 2535194624 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018744232698094285, + "loss": 1.2289, + "theoretical_loss": 3.393230776965839, + "tokens_seen": 2535260160 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018742627883650955, + "loss": 1.2001, + "theoretical_loss": 3.393223858550366, + "tokens_seen": 2535325696 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018741023069207625, + "loss": 1.2547, + "theoretical_loss": 3.3932169403637977, + "tokens_seen": 2535391232 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018739418254764295, + "loss": 1.2015, + "theoretical_loss": 3.3932100224061212, + "tokens_seen": 2535456768 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018737813440320962, + "loss": 1.1933, + "theoretical_loss": 3.393203104677323, + "tokens_seen": 2535522304 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018736208625877633, + "loss": 1.2338, + "theoretical_loss": 3.3931961871773892, + "tokens_seen": 2535587840 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018734603811434303, + "loss": 1.2327, + "theoretical_loss": 3.3931892699063066, + "tokens_seen": 2535653376 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018732998996990973, + "loss": 1.2881, + "theoretical_loss": 3.393182352864062, + "tokens_seen": 2535718912 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018731394182547643, + "loss": 1.3371, + "theoretical_loss": 3.3931754360506416, + "tokens_seen": 2535784448 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018729789368104313, + "loss": 1.2341, + "theoretical_loss": 3.393168519466032, + "tokens_seen": 2535849984 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018728184553660983, + "loss": 1.2079, + "theoretical_loss": 3.3931616031102196, + "tokens_seen": 2535915520 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018726579739217653, + "loss": 1.1949, + "theoretical_loss": 3.3931546869831912, + "tokens_seen": 2535981056 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018724974924774323, + "loss": 1.1935, + "theoretical_loss": 3.3931477710849327, + "tokens_seen": 2536046592 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018723370110330993, + "loss": 1.1775, + "theoretical_loss": 3.393140855415431, + "tokens_seen": 2536112128 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018721765295887663, + "loss": 1.1763, + "theoretical_loss": 3.393133939974673, + "tokens_seen": 2536177664 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 847558, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3550597429275513, + "objective/train/theoretical_loss": 3.393127024762645, + "objective/train/tokens_used": 2556703200, + "theoretical_loss": 3.393127024762645, + "tokens_seen": 2536243200 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018720160481444333, + "loss": 1.3461, + "theoretical_loss": 3.393127024762645, + "tokens_seen": 2536243200 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018718555667001003, + "loss": 1.2017, + "theoretical_loss": 3.393120109779333, + "tokens_seen": 2536308736 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018716950852557673, + "loss": 1.212, + "theoretical_loss": 3.393113195024724, + "tokens_seen": 2536374272 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018715346038114343, + "loss": 1.2324, + "theoretical_loss": 3.3931062804988046, + "tokens_seen": 2536439808 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018713741223671013, + "loss": 1.2445, + "theoretical_loss": 3.393099366201561, + "tokens_seen": 2536505344 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018712136409227684, + "loss": 1.2912, + "theoretical_loss": 3.39309245213298, + "tokens_seen": 2536570880 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018710531594784354, + "loss": 1.2254, + "theoretical_loss": 3.3930855382930485, + "tokens_seen": 2536636416 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018708926780341024, + "loss": 1.2006, + "theoretical_loss": 3.393078624681752, + "tokens_seen": 2536701952 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018707321965897694, + "loss": 1.2528, + "theoretical_loss": 3.3930717112990783, + "tokens_seen": 2536767488 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018705717151454364, + "loss": 1.1579, + "theoretical_loss": 3.3930647981450126, + "tokens_seen": 2536833024 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018704112337011034, + "loss": 1.1762, + "theoretical_loss": 3.393057885219542, + "tokens_seen": 2536898560 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018702507522567704, + "loss": 1.2945, + "theoretical_loss": 3.3930509725226536, + "tokens_seen": 2536964096 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018700902708124374, + "loss": 1.1739, + "theoretical_loss": 3.3930440600543332, + "tokens_seen": 2537029632 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018699297893681044, + "loss": 1.2822, + "theoretical_loss": 3.3930371478145673, + "tokens_seen": 2537095168 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018697693079237714, + "loss": 1.2705, + "theoretical_loss": 3.3930302358033435, + "tokens_seen": 2537160704 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018696088264794384, + "loss": 1.2908, + "theoretical_loss": 3.393023324020647, + "tokens_seen": 2537226240 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018694483450351054, + "loss": 1.2306, + "theoretical_loss": 3.3930164124664652, + "tokens_seen": 2537291776 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018692878635907724, + "loss": 1.1837, + "theoretical_loss": 3.393009501140784, + "tokens_seen": 2537357312 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018691273821464394, + "loss": 1.3636, + "theoretical_loss": 3.3930025900435905, + "tokens_seen": 2537422848 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018689669007021064, + "loss": 1.2431, + "theoretical_loss": 3.392995679174871, + "tokens_seen": 2537488384 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018688064192577734, + "loss": 1.2426, + "theoretical_loss": 3.3929887685346123, + "tokens_seen": 2537553920 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018686459378134405, + "loss": 1.2503, + "theoretical_loss": 3.3929818581228006, + "tokens_seen": 2537619456 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018684854563691075, + "loss": 1.1494, + "theoretical_loss": 3.3929749479394227, + "tokens_seen": 2537684992 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018683249749247745, + "loss": 1.2439, + "theoretical_loss": 3.392968037984465, + "tokens_seen": 2537750528 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018681644934804415, + "loss": 1.258, + "theoretical_loss": 3.3929611282579137, + "tokens_seen": 2537816064 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018680040120361085, + "loss": 1.2285, + "theoretical_loss": 3.392954218759756, + "tokens_seen": 2537881600 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018678435305917755, + "loss": 1.2453, + "theoretical_loss": 3.3929473094899785, + "tokens_seen": 2537947136 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018676830491474425, + "loss": 1.2704, + "theoretical_loss": 3.392940400448567, + "tokens_seen": 2538012672 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018675225677031095, + "loss": 1.2871, + "theoretical_loss": 3.392933491635509, + "tokens_seen": 2538078208 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018673620862587765, + "loss": 1.2006, + "theoretical_loss": 3.39292658305079, + "tokens_seen": 2538143744 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018672016048144435, + "loss": 1.2579, + "theoretical_loss": 3.3929196746943973, + "tokens_seen": 2538209280 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018670411233701105, + "loss": 1.2428, + "theoretical_loss": 3.3929127665663175, + "tokens_seen": 2538274816 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018668806419257773, + "loss": 1.0839, + "theoretical_loss": 3.392905858666537, + "tokens_seen": 2538340352 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018667201604814443, + "loss": 1.2616, + "theoretical_loss": 3.392898950995042, + "tokens_seen": 2538405888 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018665596790371113, + "loss": 1.1977, + "theoretical_loss": 3.3928920435518197, + "tokens_seen": 2538471424 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018663991975927783, + "loss": 1.2824, + "theoretical_loss": 3.392885136336856, + "tokens_seen": 2538536960 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018662387161484453, + "loss": 1.1505, + "theoretical_loss": 3.3928782293501376, + "tokens_seen": 2538602496 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018660782347041123, + "loss": 1.2781, + "theoretical_loss": 3.3928713225916516, + "tokens_seen": 2538668032 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018659177532597793, + "loss": 1.2953, + "theoretical_loss": 3.3928644160613843, + "tokens_seen": 2538733568 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018657572718154463, + "loss": 1.2341, + "theoretical_loss": 3.392857509759322, + "tokens_seen": 2538799104 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018655967903711133, + "loss": 1.2102, + "theoretical_loss": 3.3928506036854515, + "tokens_seen": 2538864640 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018654363089267803, + "loss": 1.1903, + "theoretical_loss": 3.3928436978397594, + "tokens_seen": 2538930176 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018652758274824473, + "loss": 1.2362, + "theoretical_loss": 3.3928367922222322, + "tokens_seen": 2538995712 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018651153460381143, + "loss": 1.1412, + "theoretical_loss": 3.392829886832857, + "tokens_seen": 2539061248 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018649548645937813, + "loss": 1.2274, + "theoretical_loss": 3.392822981671619, + "tokens_seen": 2539126784 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018647943831494486, + "loss": 1.2565, + "theoretical_loss": 3.3928160767385056, + "tokens_seen": 2539192320 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018646339017051156, + "loss": 1.1697, + "theoretical_loss": 3.3928091720335036, + "tokens_seen": 2539257856 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018644734202607826, + "loss": 1.1158, + "theoretical_loss": 3.3928022675565996, + "tokens_seen": 2539323392 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018643129388164496, + "loss": 1.3279, + "theoretical_loss": 3.39279536330778, + "tokens_seen": 2539388928 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018641524573721166, + "loss": 1.3171, + "theoretical_loss": 3.392788459287031, + "tokens_seen": 2539454464 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 848831, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.207207202911377, + "objective/train/theoretical_loss": 3.3927815554943397, + "objective/train/tokens_used": 2559980000, + "theoretical_loss": 3.3927815554943397, + "tokens_seen": 2539520000 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018639919759277836, + "loss": 1.1975, + "theoretical_loss": 3.3927815554943397, + "tokens_seen": 2539520000 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018638314944834506, + "loss": 1.1012, + "theoretical_loss": 3.3927746519296926, + "tokens_seen": 2539585536 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018636710130391174, + "loss": 1.3212, + "theoretical_loss": 3.3927677485930756, + "tokens_seen": 2539651072 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018635105315947844, + "loss": 1.4065, + "theoretical_loss": 3.3927608454844767, + "tokens_seen": 2539716608 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018633500501504514, + "loss": 1.2855, + "theoretical_loss": 3.392753942603881, + "tokens_seen": 2539782144 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018631895687061184, + "loss": 1.2198, + "theoretical_loss": 3.392747039951276, + "tokens_seen": 2539847680 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018630290872617854, + "loss": 1.2604, + "theoretical_loss": 3.392740137526648, + "tokens_seen": 2539913216 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018628686058174524, + "loss": 1.2049, + "theoretical_loss": 3.3927332353299837, + "tokens_seen": 2539978752 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018627081243731194, + "loss": 1.2307, + "theoretical_loss": 3.39272633336127, + "tokens_seen": 2540044288 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018625476429287864, + "loss": 1.3169, + "theoretical_loss": 3.3927194316204923, + "tokens_seen": 2540109824 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018623871614844534, + "loss": 1.1355, + "theoretical_loss": 3.392712530107638, + "tokens_seen": 2540175360 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018622266800401204, + "loss": 1.2241, + "theoretical_loss": 3.392705628822694, + "tokens_seen": 2540240896 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018620661985957874, + "loss": 1.1714, + "theoretical_loss": 3.3926987277656466, + "tokens_seen": 2540306432 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018619057171514544, + "loss": 1.1871, + "theoretical_loss": 3.3926918269364825, + "tokens_seen": 2540371968 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018617452357071215, + "loss": 1.1783, + "theoretical_loss": 3.3926849263351877, + "tokens_seen": 2540437504 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018615847542627885, + "loss": 1.1965, + "theoretical_loss": 3.3926780259617497, + "tokens_seen": 2540503040 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018614242728184555, + "loss": 1.1625, + "theoretical_loss": 3.3926711258161544, + "tokens_seen": 2540568576 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018612637913741225, + "loss": 1.1924, + "theoretical_loss": 3.392664225898389, + "tokens_seen": 2540634112 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018611033099297895, + "loss": 1.2037, + "theoretical_loss": 3.392657326208439, + "tokens_seen": 2540699648 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018609428284854565, + "loss": 1.2518, + "theoretical_loss": 3.3926504267462927, + "tokens_seen": 2540765184 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018607823470411235, + "loss": 1.2704, + "theoretical_loss": 3.392643527511935, + "tokens_seen": 2540830720 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018606218655967905, + "loss": 1.268, + "theoretical_loss": 3.3926366285053535, + "tokens_seen": 2540896256 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018604613841524575, + "loss": 1.2098, + "theoretical_loss": 3.3926297297265346, + "tokens_seen": 2540961792 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018603009027081245, + "loss": 1.2339, + "theoretical_loss": 3.392622831175465, + "tokens_seen": 2541027328 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018601404212637915, + "loss": 1.3431, + "theoretical_loss": 3.3926159328521313, + "tokens_seen": 2541092864 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018599799398194585, + "loss": 1.2883, + "theoretical_loss": 3.3926090347565196, + "tokens_seen": 2541158400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018598194583751253, + "loss": 1.1883, + "theoretical_loss": 3.3926021368886174, + "tokens_seen": 2541223936 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018596589769307923, + "loss": 1.2107, + "theoretical_loss": 3.3925952392484104, + "tokens_seen": 2541289472 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018594984954864593, + "loss": 1.1663, + "theoretical_loss": 3.3925883418358858, + "tokens_seen": 2541355008 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018593380140421263, + "loss": 1.2308, + "theoretical_loss": 3.39258144465103, + "tokens_seen": 2541420544 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018591775325977933, + "loss": 1.2113, + "theoretical_loss": 3.3925745476938296, + "tokens_seen": 2541486080 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018590170511534603, + "loss": 1.3116, + "theoretical_loss": 3.3925676509642715, + "tokens_seen": 2541551616 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018588565697091276, + "loss": 1.2396, + "theoretical_loss": 3.3925607544623415, + "tokens_seen": 2541617152 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018586960882647946, + "loss": 1.0856, + "theoretical_loss": 3.3925538581880272, + "tokens_seen": 2541682688 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018585356068204616, + "loss": 1.2278, + "theoretical_loss": 3.392546962141315, + "tokens_seen": 2541748224 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018583751253761286, + "loss": 1.2154, + "theoretical_loss": 3.392540066322191, + "tokens_seen": 2541813760 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018582146439317956, + "loss": 1.2223, + "theoretical_loss": 3.3925331707306423, + "tokens_seen": 2541879296 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018580541624874626, + "loss": 1.222, + "theoretical_loss": 3.392526275366656, + "tokens_seen": 2541944832 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018578936810431296, + "loss": 1.228, + "theoretical_loss": 3.392519380230217, + "tokens_seen": 2542010368 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018577331995987966, + "loss": 1.3536, + "theoretical_loss": 3.3925124853213138, + "tokens_seen": 2542075904 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018575727181544636, + "loss": 1.24, + "theoretical_loss": 3.392505590639932, + "tokens_seen": 2542141440 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018574122367101306, + "loss": 1.245, + "theoretical_loss": 3.3924986961860584, + "tokens_seen": 2542206976 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018572517552657976, + "loss": 1.309, + "theoretical_loss": 3.39249180195968, + "tokens_seen": 2542272512 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018570912738214646, + "loss": 1.2042, + "theoretical_loss": 3.392484907960783, + "tokens_seen": 2542338048 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018569307923771316, + "loss": 1.1865, + "theoretical_loss": 3.3924780141893542, + "tokens_seen": 2542403584 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018567703109327987, + "loss": 1.2568, + "theoretical_loss": 3.3924711206453804, + "tokens_seen": 2542469120 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018566098294884654, + "loss": 1.2161, + "theoretical_loss": 3.3924642273288477, + "tokens_seen": 2542534656 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018564493480441324, + "loss": 1.2403, + "theoretical_loss": 3.3924573342397433, + "tokens_seen": 2542600192 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018562888665997994, + "loss": 1.1861, + "theoretical_loss": 3.3924504413780534, + "tokens_seen": 2542665728 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018561283851554664, + "loss": 1.2911, + "theoretical_loss": 3.3924435487437647, + "tokens_seen": 2542731264 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 849771, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.192575454711914, + "objective/train/theoretical_loss": 3.3924366563368644, + "objective/train/tokens_used": 2563256800, + "theoretical_loss": 3.3924366563368644, + "tokens_seen": 2542796800 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018559679037111334, + "loss": 1.1147, + "theoretical_loss": 3.3924366563368644, + "tokens_seen": 2542796800 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018558074222668004, + "loss": 1.171, + "theoretical_loss": 3.3924297641573387, + "tokens_seen": 2542862336 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018556469408224674, + "loss": 1.2004, + "theoretical_loss": 3.392422872205174, + "tokens_seen": 2542927872 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018554864593781344, + "loss": 1.1737, + "theoretical_loss": 3.392415980480357, + "tokens_seen": 2542993408 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018553259779338014, + "loss": 1.2306, + "theoretical_loss": 3.392409088982875, + "tokens_seen": 2543058944 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018551654964894684, + "loss": 1.2507, + "theoretical_loss": 3.3924021977127143, + "tokens_seen": 2543124480 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018550050150451355, + "loss": 1.2864, + "theoretical_loss": 3.392395306669861, + "tokens_seen": 2543190016 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018548445336008025, + "loss": 1.1697, + "theoretical_loss": 3.3923884158543025, + "tokens_seen": 2543255552 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018546840521564695, + "loss": 1.1489, + "theoretical_loss": 3.3923815252660248, + "tokens_seen": 2543321088 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018545235707121365, + "loss": 1.1877, + "theoretical_loss": 3.3923746349050155, + "tokens_seen": 2543386624 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018543630892678035, + "loss": 1.2929, + "theoretical_loss": 3.39236774477126, + "tokens_seen": 2543452160 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018542026078234705, + "loss": 1.2211, + "theoretical_loss": 3.3923608548647457, + "tokens_seen": 2543517696 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018540421263791375, + "loss": 1.2752, + "theoretical_loss": 3.392353965185459, + "tokens_seen": 2543583232 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018538816449348045, + "loss": 1.2962, + "theoretical_loss": 3.3923470757333867, + "tokens_seen": 2543648768 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018537211634904715, + "loss": 1.2712, + "theoretical_loss": 3.3923401865085157, + "tokens_seen": 2543714304 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018535606820461385, + "loss": 1.1882, + "theoretical_loss": 3.3923332975108322, + "tokens_seen": 2543779840 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018534002006018055, + "loss": 1.185, + "theoretical_loss": 3.392326408740323, + "tokens_seen": 2543845376 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018532397191574725, + "loss": 1.3459, + "theoretical_loss": 3.392319520196975, + "tokens_seen": 2543910912 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018530792377131395, + "loss": 1.2366, + "theoretical_loss": 3.3923126318807744, + "tokens_seen": 2543976448 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018529187562688065, + "loss": 1.1826, + "theoretical_loss": 3.392305743791708, + "tokens_seen": 2544041984 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018527582748244735, + "loss": 1.3458, + "theoretical_loss": 3.392298855929763, + "tokens_seen": 2544107520 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018525977933801405, + "loss": 1.2044, + "theoretical_loss": 3.3922919682949253, + "tokens_seen": 2544173056 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018524373119358076, + "loss": 1.2424, + "theoretical_loss": 3.392285080887182, + "tokens_seen": 2544238592 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018522768304914746, + "loss": 1.2333, + "theoretical_loss": 3.3922781937065194, + "tokens_seen": 2544304128 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018521163490471416, + "loss": 1.1204, + "theoretical_loss": 3.392271306752925, + "tokens_seen": 2544369664 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018519558676028086, + "loss": 1.2526, + "theoretical_loss": 3.3922644200263843, + "tokens_seen": 2544435200 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018517953861584756, + "loss": 1.2265, + "theoretical_loss": 3.3922575335268847, + "tokens_seen": 2544500736 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018516349047141426, + "loss": 1.1826, + "theoretical_loss": 3.3922506472544125, + "tokens_seen": 2544566272 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018514744232698096, + "loss": 1.2365, + "theoretical_loss": 3.392243761208955, + "tokens_seen": 2544631808 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018513139418254766, + "loss": 1.2403, + "theoretical_loss": 3.3922368753904983, + "tokens_seen": 2544697344 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018511534603811436, + "loss": 1.1491, + "theoretical_loss": 3.3922299897990293, + "tokens_seen": 2544762880 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018509929789368106, + "loss": 1.2645, + "theoretical_loss": 3.3922231044345343, + "tokens_seen": 2544828416 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018508324974924776, + "loss": 1.2381, + "theoretical_loss": 3.3922162192970005, + "tokens_seen": 2544893952 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018506720160481446, + "loss": 1.2347, + "theoretical_loss": 3.3922093343864144, + "tokens_seen": 2544959488 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018505115346038116, + "loss": 1.2003, + "theoretical_loss": 3.3922024497027623, + "tokens_seen": 2545025024 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018503510531594786, + "loss": 1.3243, + "theoretical_loss": 3.3921955652460314, + "tokens_seen": 2545090560 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018501905717151456, + "loss": 1.3079, + "theoretical_loss": 3.3921886810162083, + "tokens_seen": 2545156096 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018500300902708126, + "loss": 1.0825, + "theoretical_loss": 3.3921817970132793, + "tokens_seen": 2545221632 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018498696088264797, + "loss": 1.3195, + "theoretical_loss": 3.3921749132372314, + "tokens_seen": 2545287168 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018497091273821464, + "loss": 1.176, + "theoretical_loss": 3.3921680296880514, + "tokens_seen": 2545352704 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018495486459378134, + "loss": 1.219, + "theoretical_loss": 3.392161146365726, + "tokens_seen": 2545418240 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018493881644934804, + "loss": 1.2417, + "theoretical_loss": 3.3921542632702413, + "tokens_seen": 2545483776 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018492276830491474, + "loss": 1.2832, + "theoretical_loss": 3.3921473804015845, + "tokens_seen": 2545549312 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018490672016048144, + "loss": 1.1562, + "theoretical_loss": 3.392140497759742, + "tokens_seen": 2545614848 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018489067201604814, + "loss": 1.1892, + "theoretical_loss": 3.3921336153447004, + "tokens_seen": 2545680384 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018487462387161484, + "loss": 1.2632, + "theoretical_loss": 3.392126733156447, + "tokens_seen": 2545745920 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018485857572718154, + "loss": 1.2464, + "theoretical_loss": 3.392119851194968, + "tokens_seen": 2545811456 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018484252758274824, + "loss": 1.2623, + "theoretical_loss": 3.3921129694602503, + "tokens_seen": 2545876992 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018482647943831494, + "loss": 1.3974, + "theoretical_loss": 3.39210608795228, + "tokens_seen": 2545942528 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018481043129388165, + "loss": 1.3174, + "theoretical_loss": 3.3920992066710447, + "tokens_seen": 2546008064 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 851047, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1998200416564941, + "objective/train/theoretical_loss": 3.392092325616531, + "objective/train/tokens_used": 2566533600, + "theoretical_loss": 3.392092325616531, + "tokens_seen": 2546073600 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018479438314944835, + "loss": 1.2766, + "theoretical_loss": 3.392092325616531, + "tokens_seen": 2546073600 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018477833500501505, + "loss": 1.309, + "theoretical_loss": 3.3920854447887248, + "tokens_seen": 2546139136 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018476228686058175, + "loss": 1.2097, + "theoretical_loss": 3.3920785641876137, + "tokens_seen": 2546204672 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018474623871614845, + "loss": 1.261, + "theoretical_loss": 3.3920716838131835, + "tokens_seen": 2546270208 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018473019057171515, + "loss": 1.246, + "theoretical_loss": 3.3920648036654213, + "tokens_seen": 2546335744 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018471414242728185, + "loss": 1.2942, + "theoretical_loss": 3.3920579237443143, + "tokens_seen": 2546401280 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018469809428284855, + "loss": 1.22, + "theoretical_loss": 3.3920510440498486, + "tokens_seen": 2546466816 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018468204613841528, + "loss": 1.2648, + "theoretical_loss": 3.392044164582011, + "tokens_seen": 2546532352 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018466599799398198, + "loss": 1.3019, + "theoretical_loss": 3.392037285340788, + "tokens_seen": 2546597888 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018464994984954865, + "loss": 1.2753, + "theoretical_loss": 3.392030406326167, + "tokens_seen": 2546663424 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018463390170511535, + "loss": 1.3624, + "theoretical_loss": 3.392023527538134, + "tokens_seen": 2546728960 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018461785356068205, + "loss": 1.1839, + "theoretical_loss": 3.3920166489766763, + "tokens_seen": 2546794496 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018460180541624875, + "loss": 1.1924, + "theoretical_loss": 3.39200977064178, + "tokens_seen": 2546860032 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018458575727181545, + "loss": 1.2276, + "theoretical_loss": 3.3920028925334322, + "tokens_seen": 2546925568 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018456970912738215, + "loss": 1.303, + "theoretical_loss": 3.3919960146516193, + "tokens_seen": 2546991104 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018455366098294886, + "loss": 1.3268, + "theoretical_loss": 3.3919891369963286, + "tokens_seen": 2547056640 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018453761283851556, + "loss": 1.3704, + "theoretical_loss": 3.391982259567546, + "tokens_seen": 2547122176 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018452156469408226, + "loss": 1.3116, + "theoretical_loss": 3.391975382365259, + "tokens_seen": 2547187712 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018450551654964896, + "loss": 1.187, + "theoretical_loss": 3.3919685053894537, + "tokens_seen": 2547253248 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018448946840521566, + "loss": 1.2531, + "theoretical_loss": 3.391961628640117, + "tokens_seen": 2547318784 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018447342026078236, + "loss": 1.1989, + "theoretical_loss": 3.391954752117236, + "tokens_seen": 2547384320 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018445737211634906, + "loss": 1.2643, + "theoretical_loss": 3.3919478758207973, + "tokens_seen": 2547449856 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018444132397191576, + "loss": 1.185, + "theoretical_loss": 3.391940999750787, + "tokens_seen": 2547515392 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018442527582748246, + "loss": 1.253, + "theoretical_loss": 3.391934123907192, + "tokens_seen": 2547580928 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018440922768304916, + "loss": 1.2845, + "theoretical_loss": 3.3919272482899996, + "tokens_seen": 2547646464 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018439317953861586, + "loss": 1.4003, + "theoretical_loss": 3.3919203728991962, + "tokens_seen": 2547712000 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018437713139418256, + "loss": 1.268, + "theoretical_loss": 3.3919134977347687, + "tokens_seen": 2547777536 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018436108324974926, + "loss": 1.1742, + "theoretical_loss": 3.391906622796703, + "tokens_seen": 2547843072 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018434503510531596, + "loss": 1.2059, + "theoretical_loss": 3.391899748084987, + "tokens_seen": 2547908608 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018432898696088266, + "loss": 1.2231, + "theoretical_loss": 3.3918928735996063, + "tokens_seen": 2547974144 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018431293881644936, + "loss": 1.2193, + "theoretical_loss": 3.391885999340549, + "tokens_seen": 2548039680 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018429689067201607, + "loss": 1.2512, + "theoretical_loss": 3.3918791253078004, + "tokens_seen": 2548105216 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018428084252758277, + "loss": 1.3363, + "theoretical_loss": 3.391872251501348, + "tokens_seen": 2548170752 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018426479438314944, + "loss": 1.3097, + "theoretical_loss": 3.3918653779211785, + "tokens_seen": 2548236288 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018424874623871614, + "loss": 1.224, + "theoretical_loss": 3.3918585045672787, + "tokens_seen": 2548301824 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018423269809428284, + "loss": 1.3264, + "theoretical_loss": 3.3918516314396348, + "tokens_seen": 2548367360 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018421664994984954, + "loss": 1.3602, + "theoretical_loss": 3.391844758538234, + "tokens_seen": 2548432896 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018420060180541624, + "loss": 1.2881, + "theoretical_loss": 3.391837885863063, + "tokens_seen": 2548498432 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018418455366098294, + "loss": 1.2621, + "theoretical_loss": 3.3918310134141083, + "tokens_seen": 2548563968 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018416850551654964, + "loss": 1.2291, + "theoretical_loss": 3.391824141191357, + "tokens_seen": 2548629504 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018415245737211634, + "loss": 1.2733, + "theoretical_loss": 3.3918172691947954, + "tokens_seen": 2548695040 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018413640922768304, + "loss": 1.184, + "theoretical_loss": 3.3918103974244107, + "tokens_seen": 2548760576 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018412036108324975, + "loss": 1.2642, + "theoretical_loss": 3.3918035258801895, + "tokens_seen": 2548826112 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018410431293881645, + "loss": 1.2345, + "theoretical_loss": 3.391796654562118, + "tokens_seen": 2548891648 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018408826479438315, + "loss": 1.2785, + "theoretical_loss": 3.391789783470184, + "tokens_seen": 2548957184 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018407221664994987, + "loss": 1.183, + "theoretical_loss": 3.391782912604373, + "tokens_seen": 2549022720 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018405616850551658, + "loss": 1.2723, + "theoretical_loss": 3.3917760419646727, + "tokens_seen": 2549088256 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018404012036108328, + "loss": 1.3081, + "theoretical_loss": 3.39176917155107, + "tokens_seen": 2549153792 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018402407221664998, + "loss": 1.1422, + "theoretical_loss": 3.3917623013635505, + "tokens_seen": 2549219328 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018400802407221668, + "loss": 1.132, + "theoretical_loss": 3.391755431402102, + "tokens_seen": 2549284864 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 852130, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1926579475402832, + "objective/train/theoretical_loss": 3.3917485616667107, + "objective/train/tokens_used": 2569810400, + "theoretical_loss": 3.3917485616667107, + "tokens_seen": 2549350400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018399197592778338, + "loss": 1.2137, + "theoretical_loss": 3.3917485616667107, + "tokens_seen": 2549350400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018397592778335008, + "loss": 1.3151, + "theoretical_loss": 3.3917416921573635, + "tokens_seen": 2549415936 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018395987963891678, + "loss": 1.2872, + "theoretical_loss": 3.391734822874047, + "tokens_seen": 2549481472 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018394383149448345, + "loss": 1.3052, + "theoretical_loss": 3.3917279538167486, + "tokens_seen": 2549547008 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018392778335005015, + "loss": 1.2793, + "theoretical_loss": 3.391721084985454, + "tokens_seen": 2549612544 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018391173520561685, + "loss": 1.2703, + "theoretical_loss": 3.391714216380151, + "tokens_seen": 2549678080 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018389568706118355, + "loss": 1.2326, + "theoretical_loss": 3.3917073480008257, + "tokens_seen": 2549743616 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018387963891675025, + "loss": 1.1803, + "theoretical_loss": 3.3917004798474655, + "tokens_seen": 2549809152 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018386359077231696, + "loss": 1.2649, + "theoretical_loss": 3.3916936119200565, + "tokens_seen": 2549874688 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018384754262788366, + "loss": 1.3245, + "theoretical_loss": 3.3916867442185854, + "tokens_seen": 2549940224 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018383149448345036, + "loss": 1.3108, + "theoretical_loss": 3.3916798767430394, + "tokens_seen": 2550005760 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018381544633901706, + "loss": 1.2477, + "theoretical_loss": 3.391673009493405, + "tokens_seen": 2550071296 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018379939819458376, + "loss": 1.1879, + "theoretical_loss": 3.3916661424696692, + "tokens_seen": 2550136832 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018378335005015046, + "loss": 1.2243, + "theoretical_loss": 3.3916592756718185, + "tokens_seen": 2550202368 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018376730190571716, + "loss": 1.2074, + "theoretical_loss": 3.39165240909984, + "tokens_seen": 2550267904 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018375125376128386, + "loss": 1.1012, + "theoretical_loss": 3.39164554275372, + "tokens_seen": 2550333440 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018373520561685056, + "loss": 1.2154, + "theoretical_loss": 3.3916386766334456, + "tokens_seen": 2550398976 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018371915747241726, + "loss": 1.2439, + "theoretical_loss": 3.3916318107390038, + "tokens_seen": 2550464512 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018370310932798396, + "loss": 1.2744, + "theoretical_loss": 3.3916249450703804, + "tokens_seen": 2550530048 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018368706118355066, + "loss": 1.2515, + "theoretical_loss": 3.3916180796275635, + "tokens_seen": 2550595584 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018367101303911736, + "loss": 1.2048, + "theoretical_loss": 3.391611214410539, + "tokens_seen": 2550661120 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018365496489468406, + "loss": 1.1926, + "theoretical_loss": 3.3916043494192936, + "tokens_seen": 2550726656 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018363891675025076, + "loss": 1.2411, + "theoretical_loss": 3.391597484653815, + "tokens_seen": 2550792192 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018362286860581747, + "loss": 1.2559, + "theoretical_loss": 3.3915906201140884, + "tokens_seen": 2550857728 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018360682046138417, + "loss": 1.3021, + "theoretical_loss": 3.391583755800102, + "tokens_seen": 2550923264 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018359077231695087, + "loss": 1.1949, + "theoretical_loss": 3.391576891711842, + "tokens_seen": 2550988800 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018357472417251757, + "loss": 1.2003, + "theoretical_loss": 3.391570027849295, + "tokens_seen": 2551054336 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018355867602808424, + "loss": 1.2282, + "theoretical_loss": 3.3915631642124486, + "tokens_seen": 2551119872 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018354262788365094, + "loss": 1.2162, + "theoretical_loss": 3.391556300801289, + "tokens_seen": 2551185408 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018352657973921764, + "loss": 1.2095, + "theoretical_loss": 3.3915494376158026, + "tokens_seen": 2551250944 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018351053159478434, + "loss": 1.1886, + "theoretical_loss": 3.391542574655977, + "tokens_seen": 2551316480 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018349448345035104, + "loss": 1.2627, + "theoretical_loss": 3.391535711921798, + "tokens_seen": 2551382016 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018347843530591777, + "loss": 1.2215, + "theoretical_loss": 3.391528849413253, + "tokens_seen": 2551447552 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018346238716148447, + "loss": 1.2091, + "theoretical_loss": 3.3915219871303295, + "tokens_seen": 2551513088 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018344633901705117, + "loss": 1.2368, + "theoretical_loss": 3.391515125073013, + "tokens_seen": 2551578624 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018343029087261787, + "loss": 1.2635, + "theoretical_loss": 3.3915082632412905, + "tokens_seen": 2551644160 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018341424272818457, + "loss": 1.2198, + "theoretical_loss": 3.3915014016351495, + "tokens_seen": 2551709696 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018339819458375127, + "loss": 1.3362, + "theoretical_loss": 3.3914945402545764, + "tokens_seen": 2551775232 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018338214643931797, + "loss": 1.1543, + "theoretical_loss": 3.3914876790995576, + "tokens_seen": 2551840768 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018336609829488468, + "loss": 1.1687, + "theoretical_loss": 3.3914808181700806, + "tokens_seen": 2551906304 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018335005015045138, + "loss": 1.3378, + "theoretical_loss": 3.391473957466132, + "tokens_seen": 2551971840 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018333400200601808, + "loss": 1.2873, + "theoretical_loss": 3.3914670969876983, + "tokens_seen": 2552037376 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018331795386158478, + "loss": 1.2233, + "theoretical_loss": 3.391460236734767, + "tokens_seen": 2552102912 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018330190571715148, + "loss": 1.1899, + "theoretical_loss": 3.3914533767073234, + "tokens_seen": 2552168448 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018328585757271818, + "loss": 1.2549, + "theoretical_loss": 3.391446516905356, + "tokens_seen": 2552233984 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018326980942828488, + "loss": 1.2398, + "theoretical_loss": 3.3914396573288506, + "tokens_seen": 2552299520 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018325376128385155, + "loss": 1.1484, + "theoretical_loss": 3.391432797977794, + "tokens_seen": 2552365056 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018323771313941825, + "loss": 1.1898, + "theoretical_loss": 3.3914259388521737, + "tokens_seen": 2552430592 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018322166499498495, + "loss": 1.2129, + "theoretical_loss": 3.3914190799519757, + "tokens_seen": 2552496128 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018320561685055165, + "loss": 1.2012, + "theoretical_loss": 3.3914122212771876, + "tokens_seen": 2552561664 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 852961, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0892187356948853, + "objective/train/theoretical_loss": 3.391405362827795, + "objective/train/tokens_used": 2573087200, + "theoretical_loss": 3.391405362827795, + "tokens_seen": 2552627200 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018318956870611836, + "loss": 1.1892, + "theoretical_loss": 3.391405362827795, + "tokens_seen": 2552627200 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018317352056168506, + "loss": 1.2731, + "theoretical_loss": 3.391398504603786, + "tokens_seen": 2552692736 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018315747241725176, + "loss": 1.2785, + "theoretical_loss": 3.391391646605147, + "tokens_seen": 2552758272 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018314142427281846, + "loss": 1.1491, + "theoretical_loss": 3.3913847888318647, + "tokens_seen": 2552823808 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018312537612838516, + "loss": 1.2727, + "theoretical_loss": 3.391377931283926, + "tokens_seen": 2552889344 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018310932798395186, + "loss": 1.2745, + "theoretical_loss": 3.3913710739613174, + "tokens_seen": 2552954880 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018309327983951856, + "loss": 1.2209, + "theoretical_loss": 3.3913642168640257, + "tokens_seen": 2553020416 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018307723169508526, + "loss": 1.3988, + "theoretical_loss": 3.391357359992038, + "tokens_seen": 2553085952 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018306118355065196, + "loss": 1.1938, + "theoretical_loss": 3.3913505033453415, + "tokens_seen": 2553151488 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018304513540621866, + "loss": 1.2882, + "theoretical_loss": 3.391343646923922, + "tokens_seen": 2553217024 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018302908726178536, + "loss": 1.2226, + "theoretical_loss": 3.391336790727767, + "tokens_seen": 2553282560 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018301303911735206, + "loss": 1.2052, + "theoretical_loss": 3.3913299347568637, + "tokens_seen": 2553348096 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018299699097291876, + "loss": 1.2964, + "theoretical_loss": 3.391323079011198, + "tokens_seen": 2553413632 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018298094282848546, + "loss": 1.2836, + "theoretical_loss": 3.3913162234907572, + "tokens_seen": 2553479168 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018296489468405216, + "loss": 1.2031, + "theoretical_loss": 3.391309368195528, + "tokens_seen": 2553544704 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018294884653961886, + "loss": 1.2805, + "theoretical_loss": 3.3913025131254977, + "tokens_seen": 2553610240 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018293279839518557, + "loss": 1.1786, + "theoretical_loss": 3.391295658280652, + "tokens_seen": 2553675776 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018291675025075227, + "loss": 1.2469, + "theoretical_loss": 3.391288803660979, + "tokens_seen": 2553741312 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018290070210631897, + "loss": 1.1115, + "theoretical_loss": 3.3912819492664648, + "tokens_seen": 2553806848 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018288465396188567, + "loss": 1.2825, + "theoretical_loss": 3.3912750950970962, + "tokens_seen": 2553872384 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018286860581745237, + "loss": 1.1998, + "theoretical_loss": 3.3912682411528605, + "tokens_seen": 2553937920 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018285255767301907, + "loss": 1.2509, + "theoretical_loss": 3.3912613874337443, + "tokens_seen": 2554003456 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018283650952858577, + "loss": 1.1624, + "theoretical_loss": 3.391254533939734, + "tokens_seen": 2554068992 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018282046138415247, + "loss": 1.2273, + "theoretical_loss": 3.3912476806708174, + "tokens_seen": 2554134528 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018280441323971917, + "loss": 1.2205, + "theoretical_loss": 3.39124082762698, + "tokens_seen": 2554200064 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018278836509528587, + "loss": 1.1506, + "theoretical_loss": 3.39123397480821, + "tokens_seen": 2554265600 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018277231695085257, + "loss": 1.2619, + "theoretical_loss": 3.391227122214493, + "tokens_seen": 2554331136 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018275626880641927, + "loss": 1.2738, + "theoretical_loss": 3.3912202698458165, + "tokens_seen": 2554396672 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018274022066198597, + "loss": 1.2275, + "theoretical_loss": 3.3912134177021676, + "tokens_seen": 2554462208 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018272417251755267, + "loss": 1.2459, + "theoretical_loss": 3.391206565783533, + "tokens_seen": 2554527744 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018270812437311937, + "loss": 1.1933, + "theoretical_loss": 3.3911997140898986, + "tokens_seen": 2554593280 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018269207622868607, + "loss": 1.1933, + "theoretical_loss": 3.391192862621253, + "tokens_seen": 2554658816 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018267602808425278, + "loss": 1.1571, + "theoretical_loss": 3.391186011377581, + "tokens_seen": 2554724352 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018265997993981948, + "loss": 1.2193, + "theoretical_loss": 3.3911791603588712, + "tokens_seen": 2554789888 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018264393179538618, + "loss": 1.192, + "theoretical_loss": 3.391172309565109, + "tokens_seen": 2554855424 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018262788365095288, + "loss": 1.2918, + "theoretical_loss": 3.3911654589962827, + "tokens_seen": 2554920960 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018261183550651958, + "loss": 1.3042, + "theoretical_loss": 3.391158608652378, + "tokens_seen": 2554986496 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018259578736208628, + "loss": 1.1547, + "theoretical_loss": 3.391151758533382, + "tokens_seen": 2555052032 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018257973921765298, + "loss": 1.1846, + "theoretical_loss": 3.391144908639282, + "tokens_seen": 2555117568 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018256369107321968, + "loss": 1.2734, + "theoretical_loss": 3.3911380589700646, + "tokens_seen": 2555183104 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018254764292878635, + "loss": 1.2575, + "theoretical_loss": 3.3911312095257165, + "tokens_seen": 2555248640 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018253159478435305, + "loss": 1.3138, + "theoretical_loss": 3.3911243603062244, + "tokens_seen": 2555314176 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018251554663991975, + "loss": 1.185, + "theoretical_loss": 3.3911175113115757, + "tokens_seen": 2555379712 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018249949849548646, + "loss": 1.1856, + "theoretical_loss": 3.391110662541757, + "tokens_seen": 2555445248 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018248345035105316, + "loss": 1.2174, + "theoretical_loss": 3.3911038139967546, + "tokens_seen": 2555510784 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018246740220661986, + "loss": 1.2622, + "theoretical_loss": 3.391096965676556, + "tokens_seen": 2555576320 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018245135406218656, + "loss": 1.2002, + "theoretical_loss": 3.3910901175811485, + "tokens_seen": 2555641856 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018243530591775326, + "loss": 1.3218, + "theoretical_loss": 3.391083269710518, + "tokens_seen": 2555707392 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018241925777331996, + "loss": 1.1216, + "theoretical_loss": 3.3910764220646517, + "tokens_seen": 2555772928 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018240320962888666, + "loss": 1.2135, + "theoretical_loss": 3.391069574643536, + "tokens_seen": 2555838464 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 854270, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.256941318511963, + "objective/train/theoretical_loss": 3.391062727447159, + "objective/train/tokens_used": 2576364000, + "theoretical_loss": 3.391062727447159, + "tokens_seen": 2555904000 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018238716148445336, + "loss": 1.1547, + "theoretical_loss": 3.391062727447159, + "tokens_seen": 2555904000 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018237111334002006, + "loss": 1.1315, + "theoretical_loss": 3.3910558804755064, + "tokens_seen": 2555969536 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018235506519558676, + "loss": 1.1719, + "theoretical_loss": 3.391049033728566, + "tokens_seen": 2556035072 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018233901705115346, + "loss": 1.19, + "theoretical_loss": 3.3910421872063234, + "tokens_seen": 2556100608 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018232296890672016, + "loss": 1.2329, + "theoretical_loss": 3.3910353409087666, + "tokens_seen": 2556166144 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018230692076228686, + "loss": 1.3025, + "theoretical_loss": 3.391028494835882, + "tokens_seen": 2556231680 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018229087261785356, + "loss": 1.2144, + "theoretical_loss": 3.391021648987657, + "tokens_seen": 2556297216 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001822748244734203, + "loss": 1.251, + "theoretical_loss": 3.3910148033640772, + "tokens_seen": 2556362752 + }, + { + "epoch": 0.77, + "learning_rate": 0.000182258776328987, + "loss": 1.2475, + "theoretical_loss": 3.3910079579651304, + "tokens_seen": 2556428288 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001822427281845537, + "loss": 1.1583, + "theoretical_loss": 3.391001112790804, + "tokens_seen": 2556493824 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018222668004012037, + "loss": 1.252, + "theoretical_loss": 3.3909942678410836, + "tokens_seen": 2556559360 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018221063189568707, + "loss": 1.1881, + "theoretical_loss": 3.390987423115957, + "tokens_seen": 2556624896 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018219458375125377, + "loss": 1.3174, + "theoretical_loss": 3.3909805786154106, + "tokens_seen": 2556690432 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018217853560682047, + "loss": 1.2002, + "theoretical_loss": 3.3909737343394313, + "tokens_seen": 2556755968 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018216248746238717, + "loss": 1.1978, + "theoretical_loss": 3.390966890288006, + "tokens_seen": 2556821504 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018214643931795387, + "loss": 1.2715, + "theoretical_loss": 3.3909600464611223, + "tokens_seen": 2556887040 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018213039117352057, + "loss": 1.2949, + "theoretical_loss": 3.390953202858766, + "tokens_seen": 2556952576 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018211434302908727, + "loss": 1.3087, + "theoretical_loss": 3.3909463594809246, + "tokens_seen": 2557018112 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018209829488465397, + "loss": 1.2588, + "theoretical_loss": 3.3909395163275846, + "tokens_seen": 2557083648 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018208224674022067, + "loss": 1.1062, + "theoretical_loss": 3.3909326733987335, + "tokens_seen": 2557149184 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018206619859578737, + "loss": 1.2165, + "theoretical_loss": 3.3909258306943575, + "tokens_seen": 2557214720 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018205015045135407, + "loss": 1.1035, + "theoretical_loss": 3.3909189882144437, + "tokens_seen": 2557280256 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018203410230692077, + "loss": 1.0944, + "theoretical_loss": 3.3909121459589793, + "tokens_seen": 2557345792 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018201805416248747, + "loss": 1.2671, + "theoretical_loss": 3.390905303927951, + "tokens_seen": 2557411328 + }, + { + "epoch": 0.77, + "learning_rate": 0.00018200200601805417, + "loss": 1.2414, + "theoretical_loss": 3.3908984621213456, + "tokens_seen": 2557476864 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018198595787362088, + "loss": 1.184, + "theoretical_loss": 3.3908916205391497, + "tokens_seen": 2557542400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018196990972918758, + "loss": 1.2357, + "theoretical_loss": 3.390884779181351, + "tokens_seen": 2557607936 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018195386158475428, + "loss": 1.272, + "theoretical_loss": 3.3908779380479355, + "tokens_seen": 2557673472 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018193781344032098, + "loss": 1.3094, + "theoretical_loss": 3.3908710971388905, + "tokens_seen": 2557739008 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018192176529588768, + "loss": 1.2334, + "theoretical_loss": 3.3908642564542033, + "tokens_seen": 2557804544 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018190571715145438, + "loss": 1.2013, + "theoretical_loss": 3.39085741599386, + "tokens_seen": 2557870080 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018188966900702108, + "loss": 1.2655, + "theoretical_loss": 3.390850575757848, + "tokens_seen": 2557935616 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018187362086258778, + "loss": 1.2648, + "theoretical_loss": 3.390843735746154, + "tokens_seen": 2558001152 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018185757271815448, + "loss": 1.1647, + "theoretical_loss": 3.3908368959587647, + "tokens_seen": 2558066688 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018184152457372115, + "loss": 1.1636, + "theoretical_loss": 3.390830056395668, + "tokens_seen": 2558132224 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018182547642928785, + "loss": 1.1806, + "theoretical_loss": 3.3908232170568495, + "tokens_seen": 2558197760 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018180942828485456, + "loss": 1.2388, + "theoretical_loss": 3.3908163779422966, + "tokens_seen": 2558263296 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018179338014042126, + "loss": 1.2125, + "theoretical_loss": 3.3908095390519963, + "tokens_seen": 2558328832 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018177733199598796, + "loss": 1.139, + "theoretical_loss": 3.390802700385936, + "tokens_seen": 2558394368 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018176128385155466, + "loss": 1.278, + "theoretical_loss": 3.3907958619441017, + "tokens_seen": 2558459904 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018174523570712136, + "loss": 1.2945, + "theoretical_loss": 3.39078902372648, + "tokens_seen": 2558525440 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018172918756268806, + "loss": 1.2162, + "theoretical_loss": 3.39078218573306, + "tokens_seen": 2558590976 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018171313941825476, + "loss": 1.1633, + "theoretical_loss": 3.390775347963826, + "tokens_seen": 2558656512 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018169709127382146, + "loss": 1.268, + "theoretical_loss": 3.3907685104187664, + "tokens_seen": 2558722048 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001816810431293882, + "loss": 1.2033, + "theoretical_loss": 3.390761673097867, + "tokens_seen": 2558787584 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001816649949849549, + "loss": 1.2641, + "theoretical_loss": 3.390754836001116, + "tokens_seen": 2558853120 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001816489468405216, + "loss": 1.2824, + "theoretical_loss": 3.3907479991284997, + "tokens_seen": 2558918656 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001816328986960883, + "loss": 1.216, + "theoretical_loss": 3.3907411624800052, + "tokens_seen": 2558984192 + }, + { + "epoch": 0.78, + "learning_rate": 0.000181616850551655, + "loss": 1.2461, + "theoretical_loss": 3.390734326055619, + "tokens_seen": 2559049728 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001816008024072217, + "loss": 1.2625, + "theoretical_loss": 3.3907274898553283, + "tokens_seen": 2559115264 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 855161, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.085707426071167, + "objective/train/theoretical_loss": 3.39072065387912, + "objective/train/tokens_used": 2579640800, + "theoretical_loss": 3.39072065387912, + "tokens_seen": 2559180800 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001815847542627884, + "loss": 1.0913, + "theoretical_loss": 3.39072065387912, + "tokens_seen": 2559180800 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001815687061183551, + "loss": 1.2379, + "theoretical_loss": 3.390713818126981, + "tokens_seen": 2559246336 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001815526579739218, + "loss": 1.265, + "theoretical_loss": 3.390706982598898, + "tokens_seen": 2559311872 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001815366098294885, + "loss": 1.2565, + "theoretical_loss": 3.390700147294859, + "tokens_seen": 2559377408 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018152056168505517, + "loss": 1.1769, + "theoretical_loss": 3.3906933122148493, + "tokens_seen": 2559442944 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018150451354062187, + "loss": 1.191, + "theoretical_loss": 3.3906864773588565, + "tokens_seen": 2559508480 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018148846539618857, + "loss": 1.3477, + "theoretical_loss": 3.3906796427268677, + "tokens_seen": 2559574016 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018147241725175527, + "loss": 1.2367, + "theoretical_loss": 3.39067280831887, + "tokens_seen": 2559639552 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018145636910732197, + "loss": 1.2564, + "theoretical_loss": 3.3906659741348495, + "tokens_seen": 2559705088 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018144032096288867, + "loss": 1.1895, + "theoretical_loss": 3.3906591401747943, + "tokens_seen": 2559770624 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018142427281845537, + "loss": 1.2772, + "theoretical_loss": 3.39065230643869, + "tokens_seen": 2559836160 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018140822467402207, + "loss": 1.1839, + "theoretical_loss": 3.3906454729265247, + "tokens_seen": 2559901696 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018139217652958877, + "loss": 1.1996, + "theoretical_loss": 3.390638639638285, + "tokens_seen": 2559967232 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018137612838515547, + "loss": 1.2065, + "theoretical_loss": 3.390631806573958, + "tokens_seen": 2560032768 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018136008024072217, + "loss": 1.2414, + "theoretical_loss": 3.3906249737335297, + "tokens_seen": 2560098304 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018134403209628887, + "loss": 1.2772, + "theoretical_loss": 3.3906181411169873, + "tokens_seen": 2560163840 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018132798395185557, + "loss": 1.2337, + "theoretical_loss": 3.390611308724319, + "tokens_seen": 2560229376 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018131193580742228, + "loss": 1.2546, + "theoretical_loss": 3.3906044765555103, + "tokens_seen": 2560294912 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018129588766298898, + "loss": 1.2422, + "theoretical_loss": 3.3905976446105486, + "tokens_seen": 2560360448 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018127983951855568, + "loss": 1.214, + "theoretical_loss": 3.390590812889421, + "tokens_seen": 2560425984 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018126379137412238, + "loss": 1.323, + "theoretical_loss": 3.3905839813921146, + "tokens_seen": 2560491520 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018124774322968908, + "loss": 1.2029, + "theoretical_loss": 3.3905771501186157, + "tokens_seen": 2560557056 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018123169508525578, + "loss": 1.2888, + "theoretical_loss": 3.3905703190689116, + "tokens_seen": 2560622592 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018121564694082248, + "loss": 1.2175, + "theoretical_loss": 3.3905634882429894, + "tokens_seen": 2560688128 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018119959879638918, + "loss": 1.1518, + "theoretical_loss": 3.3905566576408357, + "tokens_seen": 2560753664 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018118355065195588, + "loss": 1.1843, + "theoretical_loss": 3.390549827262438, + "tokens_seen": 2560819200 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018116750250752258, + "loss": 1.232, + "theoretical_loss": 3.3905429971077825, + "tokens_seen": 2560884736 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018115145436308928, + "loss": 1.2461, + "theoretical_loss": 3.390536167176857, + "tokens_seen": 2560950272 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018113540621865595, + "loss": 1.1495, + "theoretical_loss": 3.3905293374696477, + "tokens_seen": 2561015808 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018111935807422266, + "loss": 1.2908, + "theoretical_loss": 3.3905225079861414, + "tokens_seen": 2561081344 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018110330992978936, + "loss": 1.2805, + "theoretical_loss": 3.390515678726326, + "tokens_seen": 2561146880 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018108726178535606, + "loss": 1.2426, + "theoretical_loss": 3.3905088496901876, + "tokens_seen": 2561212416 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018107121364092278, + "loss": 1.2379, + "theoretical_loss": 3.3905020208777135, + "tokens_seen": 2561277952 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018105516549648949, + "loss": 1.1045, + "theoretical_loss": 3.390495192288891, + "tokens_seen": 2561343488 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018103911735205619, + "loss": 1.2663, + "theoretical_loss": 3.3904883639237062, + "tokens_seen": 2561409024 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001810230692076229, + "loss": 1.2875, + "theoretical_loss": 3.390481535782147, + "tokens_seen": 2561474560 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001810070210631896, + "loss": 1.2584, + "theoretical_loss": 3.3904747078641995, + "tokens_seen": 2561540096 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001809909729187563, + "loss": 1.2855, + "theoretical_loss": 3.390467880169851, + "tokens_seen": 2561605632 + }, + { + "epoch": 0.78, + "learning_rate": 0.000180974924774323, + "loss": 1.2838, + "theoretical_loss": 3.390461052699089, + "tokens_seen": 2561671168 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001809588766298897, + "loss": 1.1751, + "theoretical_loss": 3.390454225451899, + "tokens_seen": 2561736704 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001809428284854564, + "loss": 1.1917, + "theoretical_loss": 3.39044739842827, + "tokens_seen": 2561802240 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001809267803410231, + "loss": 1.2807, + "theoretical_loss": 3.3904405716281873, + "tokens_seen": 2561867776 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001809107321965898, + "loss": 1.3434, + "theoretical_loss": 3.3904337450516384, + "tokens_seen": 2561933312 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001808946840521565, + "loss": 1.3187, + "theoretical_loss": 3.3904269186986102, + "tokens_seen": 2561998848 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001808786359077232, + "loss": 1.1852, + "theoretical_loss": 3.39042009256909, + "tokens_seen": 2562064384 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001808625877632899, + "loss": 1.2866, + "theoretical_loss": 3.3904132666630646, + "tokens_seen": 2562129920 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001808465396188566, + "loss": 1.1946, + "theoretical_loss": 3.3904064409805206, + "tokens_seen": 2562195456 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018083049147442327, + "loss": 1.2232, + "theoretical_loss": 3.3903996155214458, + "tokens_seen": 2562260992 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018081444332998997, + "loss": 1.2519, + "theoretical_loss": 3.390392790285826, + "tokens_seen": 2562326528 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018079839518555667, + "loss": 1.227, + "theoretical_loss": 3.3903859652736488, + "tokens_seen": 2562392064 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 856477, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3286690711975098, + "objective/train/theoretical_loss": 3.3903791404849013, + "objective/train/tokens_used": 2582917600, + "theoretical_loss": 3.3903791404849013, + "tokens_seen": 2562457600 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018078234704112337, + "loss": 1.3219, + "theoretical_loss": 3.3903791404849013, + "tokens_seen": 2562457600 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018076629889669007, + "loss": 1.2258, + "theoretical_loss": 3.3903723159195707, + "tokens_seen": 2562523136 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018075025075225677, + "loss": 1.1858, + "theoretical_loss": 3.3903654915776436, + "tokens_seen": 2562588672 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018073420260782347, + "loss": 1.2045, + "theoretical_loss": 3.3903586674591066, + "tokens_seen": 2562654208 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018071815446339017, + "loss": 1.2685, + "theoretical_loss": 3.390351843563947, + "tokens_seen": 2562719744 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018070210631895687, + "loss": 1.272, + "theoretical_loss": 3.390345019892152, + "tokens_seen": 2562785280 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018068605817452357, + "loss": 1.2652, + "theoretical_loss": 3.3903381964437083, + "tokens_seen": 2562850816 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018067001003009027, + "loss": 1.2318, + "theoretical_loss": 3.3903313732186033, + "tokens_seen": 2562916352 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018065396188565697, + "loss": 1.2112, + "theoretical_loss": 3.3903245502168238, + "tokens_seen": 2562981888 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018063791374122367, + "loss": 1.2324, + "theoretical_loss": 3.3903177274383562, + "tokens_seen": 2563047424 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018062186559679038, + "loss": 1.2, + "theoretical_loss": 3.390310904883188, + "tokens_seen": 2563112960 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018060581745235708, + "loss": 1.2099, + "theoretical_loss": 3.390304082551306, + "tokens_seen": 2563178496 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018058976930792378, + "loss": 1.2464, + "theoretical_loss": 3.390297260442698, + "tokens_seen": 2563244032 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018057372116349048, + "loss": 1.2781, + "theoretical_loss": 3.3902904385573494, + "tokens_seen": 2563309568 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018055767301905718, + "loss": 1.2602, + "theoretical_loss": 3.3902836168952484, + "tokens_seen": 2563375104 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018054162487462388, + "loss": 1.2748, + "theoretical_loss": 3.3902767954563817, + "tokens_seen": 2563440640 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018052557673019058, + "loss": 1.3444, + "theoretical_loss": 3.390269974240736, + "tokens_seen": 2563506176 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018050952858575728, + "loss": 1.1388, + "theoretical_loss": 3.390263153248299, + "tokens_seen": 2563571712 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018049348044132398, + "loss": 1.2426, + "theoretical_loss": 3.390256332479057, + "tokens_seen": 2563637248 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018047743229689068, + "loss": 1.233, + "theoretical_loss": 3.390249511932997, + "tokens_seen": 2563702784 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018046138415245738, + "loss": 1.2018, + "theoretical_loss": 3.3902426916101067, + "tokens_seen": 2563768320 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018044533600802408, + "loss": 1.2688, + "theoretical_loss": 3.390235871510372, + "tokens_seen": 2563833856 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018042928786359078, + "loss": 1.2421, + "theoretical_loss": 3.390229051633781, + "tokens_seen": 2563899392 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018041323971915748, + "loss": 1.1542, + "theoretical_loss": 3.39022223198032, + "tokens_seen": 2563964928 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018039719157472418, + "loss": 1.215, + "theoretical_loss": 3.390215412549976, + "tokens_seen": 2564030464 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018038114343029088, + "loss": 1.2216, + "theoretical_loss": 3.3902085933427366, + "tokens_seen": 2564096000 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018036509528585759, + "loss": 1.2335, + "theoretical_loss": 3.390201774358588, + "tokens_seen": 2564161536 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018034904714142429, + "loss": 1.2288, + "theoretical_loss": 3.3901949555975177, + "tokens_seen": 2564227072 + }, + { + "epoch": 0.78, + "learning_rate": 0.000180332998996991, + "loss": 1.4146, + "theoretical_loss": 3.3901881370595124, + "tokens_seen": 2564292608 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001803169508525577, + "loss": 1.1692, + "theoretical_loss": 3.3901813187445597, + "tokens_seen": 2564358144 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001803009027081244, + "loss": 1.2303, + "theoretical_loss": 3.390174500652646, + "tokens_seen": 2564423680 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001802848545636911, + "loss": 1.2277, + "theoretical_loss": 3.3901676827837584, + "tokens_seen": 2564489216 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001802688064192578, + "loss": 1.2241, + "theoretical_loss": 3.3901608651378843, + "tokens_seen": 2564554752 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001802527582748245, + "loss": 1.181, + "theoretical_loss": 3.39015404771501, + "tokens_seen": 2564620288 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001802367101303912, + "loss": 1.2339, + "theoretical_loss": 3.3901472305151232, + "tokens_seen": 2564685824 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001802206619859579, + "loss": 1.227, + "theoretical_loss": 3.3901404135382105, + "tokens_seen": 2564751360 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001802046138415246, + "loss": 1.221, + "theoretical_loss": 3.390133596784259, + "tokens_seen": 2564816896 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001801885656970913, + "loss": 1.2349, + "theoretical_loss": 3.3901267802532558, + "tokens_seen": 2564882432 + }, + { + "epoch": 0.78, + "learning_rate": 0.000180172517552658, + "loss": 1.2044, + "theoretical_loss": 3.3901199639451876, + "tokens_seen": 2564947968 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001801564694082247, + "loss": 1.1673, + "theoretical_loss": 3.390113147860042, + "tokens_seen": 2565013504 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001801404212637914, + "loss": 1.2772, + "theoretical_loss": 3.3901063319978055, + "tokens_seen": 2565079040 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018012437311935807, + "loss": 1.2857, + "theoretical_loss": 3.3900995163584655, + "tokens_seen": 2565144576 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018010832497492477, + "loss": 1.2228, + "theoretical_loss": 3.3900927009420085, + "tokens_seen": 2565210112 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018009227683049147, + "loss": 1.1032, + "theoretical_loss": 3.390085885748422, + "tokens_seen": 2565275648 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018007622868605817, + "loss": 1.2064, + "theoretical_loss": 3.390079070777693, + "tokens_seen": 2565341184 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018006018054162487, + "loss": 1.2457, + "theoretical_loss": 3.390072256029808, + "tokens_seen": 2565406720 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018004413239719157, + "loss": 1.1927, + "theoretical_loss": 3.390065441504755, + "tokens_seen": 2565472256 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018002808425275827, + "loss": 1.2141, + "theoretical_loss": 3.39005862720252, + "tokens_seen": 2565537792 + }, + { + "epoch": 0.78, + "learning_rate": 0.00018001203610832497, + "loss": 1.2668, + "theoretical_loss": 3.3900518131230903, + "tokens_seen": 2565603328 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017999598796389167, + "loss": 1.207, + "theoretical_loss": 3.390044999266453, + "tokens_seen": 2565668864 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 857286, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3080354928970337, + "objective/train/theoretical_loss": 3.3900381856325956, + "objective/train/tokens_used": 2586194400, + "theoretical_loss": 3.3900381856325956, + "tokens_seen": 2565734400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017997993981945837, + "loss": 1.2539, + "theoretical_loss": 3.3900381856325956, + "tokens_seen": 2565734400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017996389167502507, + "loss": 1.1962, + "theoretical_loss": 3.3900313722215043, + "tokens_seen": 2565799936 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017994784353059177, + "loss": 1.256, + "theoretical_loss": 3.390024559033167, + "tokens_seen": 2565865472 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017993179538615848, + "loss": 1.2064, + "theoretical_loss": 3.39001774606757, + "tokens_seen": 2565931008 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017991574724172518, + "loss": 1.2455, + "theoretical_loss": 3.3900109333247004, + "tokens_seen": 2565996544 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017989969909729188, + "loss": 1.159, + "theoretical_loss": 3.3900041208045457, + "tokens_seen": 2566062080 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017988365095285858, + "loss": 1.304, + "theoretical_loss": 3.3899973085070925, + "tokens_seen": 2566127616 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001798676028084253, + "loss": 1.2834, + "theoretical_loss": 3.389990496432328, + "tokens_seen": 2566193152 + }, + { + "epoch": 0.78, + "learning_rate": 0.000179851554663992, + "loss": 1.2513, + "theoretical_loss": 3.3899836845802396, + "tokens_seen": 2566258688 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001798355065195587, + "loss": 1.2488, + "theoretical_loss": 3.3899768729508133, + "tokens_seen": 2566324224 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001798194583751254, + "loss": 1.2606, + "theoretical_loss": 3.3899700615440373, + "tokens_seen": 2566389760 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017980341023069208, + "loss": 1.0601, + "theoretical_loss": 3.389963250359898, + "tokens_seen": 2566455296 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017978736208625878, + "loss": 1.181, + "theoretical_loss": 3.3899564393983823, + "tokens_seen": 2566520832 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017977131394182548, + "loss": 1.1751, + "theoretical_loss": 3.389949628659478, + "tokens_seen": 2566586368 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017975526579739218, + "loss": 1.2943, + "theoretical_loss": 3.389942818143171, + "tokens_seen": 2566651904 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017973921765295888, + "loss": 1.2742, + "theoretical_loss": 3.3899360078494496, + "tokens_seen": 2566717440 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017972316950852558, + "loss": 1.193, + "theoretical_loss": 3.3899291977783, + "tokens_seen": 2566782976 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017970712136409228, + "loss": 1.2308, + "theoretical_loss": 3.389922387929709, + "tokens_seen": 2566848512 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017969107321965898, + "loss": 1.2759, + "theoretical_loss": 3.3899155783036647, + "tokens_seen": 2566914048 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017967502507522569, + "loss": 1.2766, + "theoretical_loss": 3.3899087689001535, + "tokens_seen": 2566979584 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017965897693079239, + "loss": 1.331, + "theoretical_loss": 3.3899019597191624, + "tokens_seen": 2567045120 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001796429287863591, + "loss": 1.2048, + "theoretical_loss": 3.3898951507606787, + "tokens_seen": 2567110656 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001796268806419258, + "loss": 1.2453, + "theoretical_loss": 3.389888342024689, + "tokens_seen": 2567176192 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001796108324974925, + "loss": 1.1761, + "theoretical_loss": 3.389881533511181, + "tokens_seen": 2567241728 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001795947843530592, + "loss": 1.2994, + "theoretical_loss": 3.3898747252201415, + "tokens_seen": 2567307264 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001795787362086259, + "loss": 1.1707, + "theoretical_loss": 3.389867917151557, + "tokens_seen": 2567372800 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001795626880641926, + "loss": 1.1435, + "theoretical_loss": 3.389861109305415, + "tokens_seen": 2567438336 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001795466399197593, + "loss": 1.2181, + "theoretical_loss": 3.3898543016817033, + "tokens_seen": 2567503872 + }, + { + "epoch": 0.78, + "learning_rate": 0.000179530591775326, + "loss": 1.206, + "theoretical_loss": 3.3898474942804073, + "tokens_seen": 2567569408 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001795145436308927, + "loss": 1.2691, + "theoretical_loss": 3.3898406871015156, + "tokens_seen": 2567634944 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001794984954864594, + "loss": 1.2661, + "theoretical_loss": 3.3898338801450145, + "tokens_seen": 2567700480 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001794824473420261, + "loss": 1.2389, + "theoretical_loss": 3.389827073410891, + "tokens_seen": 2567766016 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001794663991975928, + "loss": 1.1681, + "theoretical_loss": 3.3898202668991324, + "tokens_seen": 2567831552 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001794503510531595, + "loss": 1.1655, + "theoretical_loss": 3.389813460609726, + "tokens_seen": 2567897088 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001794343029087262, + "loss": 1.315, + "theoretical_loss": 3.3898066545426584, + "tokens_seen": 2567962624 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017941825476429287, + "loss": 1.1877, + "theoretical_loss": 3.389799848697917, + "tokens_seen": 2568028160 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017940220661985957, + "loss": 1.1842, + "theoretical_loss": 3.389793043075488, + "tokens_seen": 2568093696 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017938615847542627, + "loss": 1.2478, + "theoretical_loss": 3.3897862376753602, + "tokens_seen": 2568159232 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017937011033099297, + "loss": 1.2411, + "theoretical_loss": 3.389779432497519, + "tokens_seen": 2568224768 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017935406218655967, + "loss": 1.2754, + "theoretical_loss": 3.3897726275419524, + "tokens_seen": 2568290304 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017933801404212637, + "loss": 1.1493, + "theoretical_loss": 3.3897658228086467, + "tokens_seen": 2568355840 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017932196589769307, + "loss": 1.1881, + "theoretical_loss": 3.38975901829759, + "tokens_seen": 2568421376 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017930591775325977, + "loss": 1.2138, + "theoretical_loss": 3.3897522140087686, + "tokens_seen": 2568486912 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017928986960882647, + "loss": 1.1994, + "theoretical_loss": 3.3897454099421696, + "tokens_seen": 2568552448 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001792738214643932, + "loss": 1.1672, + "theoretical_loss": 3.3897386060977803, + "tokens_seen": 2568617984 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001792577733199599, + "loss": 1.2991, + "theoretical_loss": 3.389731802475588, + "tokens_seen": 2568683520 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001792417251755266, + "loss": 1.3044, + "theoretical_loss": 3.3897249990755793, + "tokens_seen": 2568749056 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001792256770310933, + "loss": 1.2355, + "theoretical_loss": 3.3897181958977414, + "tokens_seen": 2568814592 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017920962888666, + "loss": 1.1855, + "theoretical_loss": 3.3897113929420617, + "tokens_seen": 2568880128 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001791935807422267, + "loss": 1.2088, + "theoretical_loss": 3.3897045902085265, + "tokens_seen": 2568945664 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 858559, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2997121810913086, + "objective/train/theoretical_loss": 3.389697787697124, + "objective/train/tokens_used": 2589471200, + "theoretical_loss": 3.389697787697124, + "tokens_seen": 2569011200 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001791775325977934, + "loss": 1.2811, + "theoretical_loss": 3.389697787697124, + "tokens_seen": 2569011200 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001791614844533601, + "loss": 1.247, + "theoretical_loss": 3.3896909854078405, + "tokens_seen": 2569076736 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001791454363089268, + "loss": 1.2906, + "theoretical_loss": 3.3896841833406635, + "tokens_seen": 2569142272 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001791293881644935, + "loss": 1.1754, + "theoretical_loss": 3.3896773814955794, + "tokens_seen": 2569207808 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017911334002006018, + "loss": 1.2316, + "theoretical_loss": 3.3896705798725764, + "tokens_seen": 2569273344 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017909729187562688, + "loss": 1.2195, + "theoretical_loss": 3.38966377847164, + "tokens_seen": 2569338880 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017908124373119358, + "loss": 1.1101, + "theoretical_loss": 3.389656977292759, + "tokens_seen": 2569404416 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017906519558676028, + "loss": 1.2782, + "theoretical_loss": 3.3896501763359193, + "tokens_seen": 2569469952 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017904914744232698, + "loss": 1.238, + "theoretical_loss": 3.389643375601109, + "tokens_seen": 2569535488 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017903309929789368, + "loss": 1.1112, + "theoretical_loss": 3.3896365750883137, + "tokens_seen": 2569601024 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017901705115346038, + "loss": 1.2564, + "theoretical_loss": 3.389629774797522, + "tokens_seen": 2569666560 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017900100300902709, + "loss": 1.2645, + "theoretical_loss": 3.38962297472872, + "tokens_seen": 2569732096 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017898495486459379, + "loss": 1.2615, + "theoretical_loss": 3.389616174881895, + "tokens_seen": 2569797632 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001789689067201605, + "loss": 1.2755, + "theoretical_loss": 3.3896093752570344, + "tokens_seen": 2569863168 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001789528585757272, + "loss": 1.2705, + "theoretical_loss": 3.3896025758541253, + "tokens_seen": 2569928704 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001789368104312939, + "loss": 1.2235, + "theoretical_loss": 3.389595776673155, + "tokens_seen": 2569994240 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001789207622868606, + "loss": 1.1948, + "theoretical_loss": 3.3895889777141095, + "tokens_seen": 2570059776 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001789047141424273, + "loss": 1.1355, + "theoretical_loss": 3.389582178976977, + "tokens_seen": 2570125312 + }, + { + "epoch": 0.78, + "learning_rate": 0.000178888665997994, + "loss": 1.237, + "theoretical_loss": 3.389575380461744, + "tokens_seen": 2570190848 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001788726178535607, + "loss": 1.2993, + "theoretical_loss": 3.389568582168398, + "tokens_seen": 2570256384 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001788565697091274, + "loss": 1.1414, + "theoretical_loss": 3.389561784096926, + "tokens_seen": 2570321920 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001788405215646941, + "loss": 1.2056, + "theoretical_loss": 3.389554986247315, + "tokens_seen": 2570387456 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001788244734202608, + "loss": 1.1311, + "theoretical_loss": 3.389548188619552, + "tokens_seen": 2570452992 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001788084252758275, + "loss": 1.223, + "theoretical_loss": 3.3895413912136245, + "tokens_seen": 2570518528 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001787923771313942, + "loss": 1.1989, + "theoretical_loss": 3.389534594029519, + "tokens_seen": 2570584064 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001787763289869609, + "loss": 1.2353, + "theoretical_loss": 3.389527797067223, + "tokens_seen": 2570649600 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001787602808425276, + "loss": 1.2517, + "theoretical_loss": 3.3895210003267238, + "tokens_seen": 2570715136 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001787442326980943, + "loss": 1.1913, + "theoretical_loss": 3.389514203808008, + "tokens_seen": 2570780672 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017872818455366097, + "loss": 1.1575, + "theoretical_loss": 3.389507407511063, + "tokens_seen": 2570846208 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017871213640922767, + "loss": 1.2418, + "theoretical_loss": 3.389500611435876, + "tokens_seen": 2570911744 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017869608826479437, + "loss": 1.2093, + "theoretical_loss": 3.389493815582434, + "tokens_seen": 2570977280 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017868004012036107, + "loss": 1.2663, + "theoretical_loss": 3.389487019950724, + "tokens_seen": 2571042816 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001786639919759278, + "loss": 1.2734, + "theoretical_loss": 3.3894802245407334, + "tokens_seen": 2571108352 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001786479438314945, + "loss": 1.1743, + "theoretical_loss": 3.389473429352449, + "tokens_seen": 2571173888 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001786318956870612, + "loss": 1.1462, + "theoretical_loss": 3.3894666343858577, + "tokens_seen": 2571239424 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001786158475426279, + "loss": 1.1905, + "theoretical_loss": 3.3894598396409474, + "tokens_seen": 2571304960 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001785997993981946, + "loss": 1.1478, + "theoretical_loss": 3.3894530451177047, + "tokens_seen": 2571370496 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001785837512537613, + "loss": 1.1724, + "theoretical_loss": 3.3894462508161167, + "tokens_seen": 2571436032 + }, + { + "epoch": 0.78, + "learning_rate": 0.000178567703109328, + "loss": 1.2492, + "theoretical_loss": 3.389439456736171, + "tokens_seen": 2571501568 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001785516549648947, + "loss": 1.1418, + "theoretical_loss": 3.389432662877854, + "tokens_seen": 2571567104 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001785356068204614, + "loss": 1.2501, + "theoretical_loss": 3.3894258692411534, + "tokens_seen": 2571632640 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001785195586760281, + "loss": 1.2435, + "theoretical_loss": 3.389419075826056, + "tokens_seen": 2571698176 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001785035105315948, + "loss": 1.1872, + "theoretical_loss": 3.3894122826325486, + "tokens_seen": 2571763712 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001784874623871615, + "loss": 1.1577, + "theoretical_loss": 3.389405489660619, + "tokens_seen": 2571829248 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001784714142427282, + "loss": 1.218, + "theoretical_loss": 3.3893986969102543, + "tokens_seen": 2571894784 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001784553660982949, + "loss": 1.2067, + "theoretical_loss": 3.3893919043814416, + "tokens_seen": 2571960320 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001784393179538616, + "loss": 1.2239, + "theoretical_loss": 3.3893851120741676, + "tokens_seen": 2572025856 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001784232698094283, + "loss": 1.1898, + "theoretical_loss": 3.3893783199884195, + "tokens_seen": 2572091392 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017840722166499498, + "loss": 1.2695, + "theoretical_loss": 3.389371528124184, + "tokens_seen": 2572156928 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017839117352056168, + "loss": 1.306, + "theoretical_loss": 3.3893647364814496, + "tokens_seen": 2572222464 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 859589, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2575843334197998, + "objective/train/theoretical_loss": 3.3893579450602025, + "objective/train/tokens_used": 2592748000, + "theoretical_loss": 3.3893579450602025, + "tokens_seen": 2572288000 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017837512537612838, + "loss": 1.2073, + "theoretical_loss": 3.3893579450602025, + "tokens_seen": 2572288000 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017835907723169508, + "loss": 1.1739, + "theoretical_loss": 3.38935115386043, + "tokens_seen": 2572353536 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017834302908726178, + "loss": 1.2198, + "theoretical_loss": 3.3893443628821194, + "tokens_seen": 2572419072 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017832698094282848, + "loss": 1.1979, + "theoretical_loss": 3.3893375721252577, + "tokens_seen": 2572484608 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017831093279839519, + "loss": 1.1908, + "theoretical_loss": 3.3893307815898313, + "tokens_seen": 2572550144 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017829488465396189, + "loss": 1.2547, + "theoretical_loss": 3.3893239912758286, + "tokens_seen": 2572615680 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001782788365095286, + "loss": 1.3042, + "theoretical_loss": 3.389317201183236, + "tokens_seen": 2572681216 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001782627883650953, + "loss": 1.1483, + "theoretical_loss": 3.389310411312041, + "tokens_seen": 2572746752 + }, + { + "epoch": 0.78, + "learning_rate": 0.000178246740220662, + "loss": 1.2285, + "theoretical_loss": 3.38930362166223, + "tokens_seen": 2572812288 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001782306920762287, + "loss": 1.11, + "theoretical_loss": 3.3892968322337915, + "tokens_seen": 2572877824 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001782146439317954, + "loss": 1.1701, + "theoretical_loss": 3.3892900430267114, + "tokens_seen": 2572943360 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001781985957873621, + "loss": 1.2147, + "theoretical_loss": 3.3892832540409774, + "tokens_seen": 2573008896 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001781825476429288, + "loss": 1.2397, + "theoretical_loss": 3.389276465276576, + "tokens_seen": 2573074432 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001781664994984955, + "loss": 1.215, + "theoretical_loss": 3.3892696767334956, + "tokens_seen": 2573139968 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001781504513540622, + "loss": 1.298, + "theoretical_loss": 3.3892628884117224, + "tokens_seen": 2573205504 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001781344032096289, + "loss": 1.1692, + "theoretical_loss": 3.389256100311244, + "tokens_seen": 2573271040 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001781183550651956, + "loss": 1.1638, + "theoretical_loss": 3.389249312432047, + "tokens_seen": 2573336576 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001781023069207623, + "loss": 1.2133, + "theoretical_loss": 3.3892425247741187, + "tokens_seen": 2573402112 + }, + { + "epoch": 0.78, + "learning_rate": 0.000178086258776329, + "loss": 1.1287, + "theoretical_loss": 3.389235737337447, + "tokens_seen": 2573467648 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001780702106318957, + "loss": 1.2206, + "theoretical_loss": 3.389228950122018, + "tokens_seen": 2573533184 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001780541624874624, + "loss": 1.2474, + "theoretical_loss": 3.38922216312782, + "tokens_seen": 2573598720 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001780381143430291, + "loss": 1.3237, + "theoretical_loss": 3.3892153763548385, + "tokens_seen": 2573664256 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001780220661985958, + "loss": 1.2068, + "theoretical_loss": 3.3892085898030624, + "tokens_seen": 2573729792 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001780060180541625, + "loss": 1.2293, + "theoretical_loss": 3.389201803472478, + "tokens_seen": 2573795328 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001779899699097292, + "loss": 1.2125, + "theoretical_loss": 3.3891950173630723, + "tokens_seen": 2573860864 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001779739217652959, + "loss": 1.2039, + "theoretical_loss": 3.389188231474833, + "tokens_seen": 2573926400 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001779578736208626, + "loss": 1.1849, + "theoretical_loss": 3.3891814458077465, + "tokens_seen": 2573991936 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001779418254764293, + "loss": 1.2409, + "theoretical_loss": 3.389174660361801, + "tokens_seen": 2574057472 + }, + { + "epoch": 0.78, + "learning_rate": 0.000177925777331996, + "loss": 1.1624, + "theoretical_loss": 3.389167875136983, + "tokens_seen": 2574123008 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001779097291875627, + "loss": 1.2137, + "theoretical_loss": 3.3891610901332796, + "tokens_seen": 2574188544 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001778936810431294, + "loss": 1.2594, + "theoretical_loss": 3.389154305350678, + "tokens_seen": 2574254080 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001778776328986961, + "loss": 1.1259, + "theoretical_loss": 3.389147520789166, + "tokens_seen": 2574319616 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001778615847542628, + "loss": 1.1884, + "theoretical_loss": 3.38914073644873, + "tokens_seen": 2574385152 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001778455366098295, + "loss": 1.282, + "theoretical_loss": 3.3891339523293573, + "tokens_seen": 2574450688 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001778294884653962, + "loss": 1.2009, + "theoretical_loss": 3.3891271684310356, + "tokens_seen": 2574516224 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001778134403209629, + "loss": 1.2479, + "theoretical_loss": 3.389120384753751, + "tokens_seen": 2574581760 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001777973921765296, + "loss": 1.1603, + "theoretical_loss": 3.389113601297492, + "tokens_seen": 2574647296 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001777813440320963, + "loss": 1.2148, + "theoretical_loss": 3.389106818062245, + "tokens_seen": 2574712832 + }, + { + "epoch": 0.78, + "learning_rate": 0.000177765295887663, + "loss": 1.295, + "theoretical_loss": 3.3891000350479974, + "tokens_seen": 2574778368 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001777492477432297, + "loss": 1.1549, + "theoretical_loss": 3.389093252254736, + "tokens_seen": 2574843904 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001777331995987964, + "loss": 1.2427, + "theoretical_loss": 3.3890864696824483, + "tokens_seen": 2574909440 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001777171514543631, + "loss": 1.2204, + "theoretical_loss": 3.3890796873311215, + "tokens_seen": 2574974976 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017770110330992978, + "loss": 1.2795, + "theoretical_loss": 3.389072905200743, + "tokens_seen": 2575040512 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017768505516549648, + "loss": 1.2987, + "theoretical_loss": 3.3890661232912995, + "tokens_seen": 2575106048 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017766900702106318, + "loss": 1.1839, + "theoretical_loss": 3.389059341602778, + "tokens_seen": 2575171584 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017765295887662988, + "loss": 1.3078, + "theoretical_loss": 3.3890525601351666, + "tokens_seen": 2575237120 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017763691073219658, + "loss": 1.1757, + "theoretical_loss": 3.389045778888452, + "tokens_seen": 2575302656 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017762086258776329, + "loss": 1.2358, + "theoretical_loss": 3.3890389978626208, + "tokens_seen": 2575368192 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017760481444332999, + "loss": 1.1783, + "theoretical_loss": 3.3890322170576606, + "tokens_seen": 2575433728 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001775887662988967, + "loss": 1.2251, + "theoretical_loss": 3.389025436473559, + "tokens_seen": 2575499264 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 860863, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1932480335235596, + "objective/train/theoretical_loss": 3.389018656110303, + "objective/train/tokens_used": 2596024800, + "theoretical_loss": 3.389018656110303, + "tokens_seen": 2575564800 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001775727181544634, + "loss": 1.241, + "theoretical_loss": 3.389018656110303, + "tokens_seen": 2575564800 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001775566700100301, + "loss": 1.2484, + "theoretical_loss": 3.3890118759678796, + "tokens_seen": 2575630336 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001775406218655968, + "loss": 1.2103, + "theoretical_loss": 3.389005096046276, + "tokens_seen": 2575695872 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001775245737211635, + "loss": 1.3035, + "theoretical_loss": 3.388998316345479, + "tokens_seen": 2575761408 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001775085255767302, + "loss": 1.2884, + "theoretical_loss": 3.388991536865477, + "tokens_seen": 2575826944 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001774924774322969, + "loss": 1.1775, + "theoretical_loss": 3.3889847576062557, + "tokens_seen": 2575892480 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017747642928786362, + "loss": 1.3476, + "theoretical_loss": 3.388977978567804, + "tokens_seen": 2575958016 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017746038114343032, + "loss": 1.3121, + "theoretical_loss": 3.3889711997501073, + "tokens_seen": 2576023552 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017744433299899702, + "loss": 1.1326, + "theoretical_loss": 3.3889644211531538, + "tokens_seen": 2576089088 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017742828485456372, + "loss": 1.139, + "theoretical_loss": 3.388957642776931, + "tokens_seen": 2576154624 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017741223671013042, + "loss": 1.2352, + "theoretical_loss": 3.3889508646214246, + "tokens_seen": 2576220160 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001773961885656971, + "loss": 1.1567, + "theoretical_loss": 3.388944086686623, + "tokens_seen": 2576285696 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001773801404212638, + "loss": 1.1914, + "theoretical_loss": 3.3889373089725137, + "tokens_seen": 2576351232 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001773640922768305, + "loss": 1.3076, + "theoretical_loss": 3.388930531479083, + "tokens_seen": 2576416768 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001773480441323972, + "loss": 1.2488, + "theoretical_loss": 3.388923754206319, + "tokens_seen": 2576482304 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001773319959879639, + "loss": 1.153, + "theoretical_loss": 3.3889169771542083, + "tokens_seen": 2576547840 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001773159478435306, + "loss": 1.1727, + "theoretical_loss": 3.388910200322738, + "tokens_seen": 2576613376 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001772998996990973, + "loss": 1.1959, + "theoretical_loss": 3.388903423711896, + "tokens_seen": 2576678912 + }, + { + "epoch": 0.78, + "learning_rate": 0.000177283851554664, + "loss": 1.3102, + "theoretical_loss": 3.388896647321668, + "tokens_seen": 2576744448 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001772678034102307, + "loss": 1.2785, + "theoretical_loss": 3.3888898711520428, + "tokens_seen": 2576809984 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001772517552657974, + "loss": 1.1868, + "theoretical_loss": 3.388883095203007, + "tokens_seen": 2576875520 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001772357071213641, + "loss": 1.2016, + "theoretical_loss": 3.388876319474548, + "tokens_seen": 2576941056 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001772196589769308, + "loss": 1.1861, + "theoretical_loss": 3.3888695439666527, + "tokens_seen": 2577006592 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001772036108324975, + "loss": 1.1948, + "theoretical_loss": 3.3888627686793082, + "tokens_seen": 2577072128 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001771875626880642, + "loss": 1.2177, + "theoretical_loss": 3.3888559936125024, + "tokens_seen": 2577137664 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001771715145436309, + "loss": 1.2739, + "theoretical_loss": 3.388849218766222, + "tokens_seen": 2577203200 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001771554663991976, + "loss": 1.2581, + "theoretical_loss": 3.388842444140454, + "tokens_seen": 2577268736 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001771394182547643, + "loss": 1.2876, + "theoretical_loss": 3.3888356697351867, + "tokens_seen": 2577334272 + }, + { + "epoch": 0.78, + "learning_rate": 0.000177123370110331, + "loss": 1.1863, + "theoretical_loss": 3.3888288955504056, + "tokens_seen": 2577399808 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001771073219658977, + "loss": 1.1972, + "theoretical_loss": 3.3888221215860996, + "tokens_seen": 2577465344 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001770912738214644, + "loss": 1.1687, + "theoretical_loss": 3.3888153478422547, + "tokens_seen": 2577530880 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001770752256770311, + "loss": 1.2341, + "theoretical_loss": 3.3888085743188587, + "tokens_seen": 2577596416 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001770591775325978, + "loss": 1.0885, + "theoretical_loss": 3.388801801015899, + "tokens_seen": 2577661952 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001770431293881645, + "loss": 1.1525, + "theoretical_loss": 3.3887950279333623, + "tokens_seen": 2577727488 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001770270812437312, + "loss": 1.201, + "theoretical_loss": 3.3887882550712356, + "tokens_seen": 2577793024 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017701103309929788, + "loss": 1.3, + "theoretical_loss": 3.3887814824295073, + "tokens_seen": 2577858560 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017699498495486458, + "loss": 1.111, + "theoretical_loss": 3.3887747100081635, + "tokens_seen": 2577924096 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017697893681043128, + "loss": 1.2109, + "theoretical_loss": 3.388767937807192, + "tokens_seen": 2577989632 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017696288866599798, + "loss": 1.2526, + "theoretical_loss": 3.3887611658265797, + "tokens_seen": 2578055168 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017694684052156469, + "loss": 1.2274, + "theoretical_loss": 3.3887543940663143, + "tokens_seen": 2578120704 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017693079237713139, + "loss": 1.2911, + "theoretical_loss": 3.3887476225263824, + "tokens_seen": 2578186240 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017691474423269809, + "loss": 1.3194, + "theoretical_loss": 3.388740851206772, + "tokens_seen": 2578251776 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001768986960882648, + "loss": 1.0683, + "theoretical_loss": 3.3887340801074695, + "tokens_seen": 2578317312 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001768826479438315, + "loss": 1.2085, + "theoretical_loss": 3.388727309228462, + "tokens_seen": 2578382848 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017686659979939822, + "loss": 1.2281, + "theoretical_loss": 3.3887205385697383, + "tokens_seen": 2578448384 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017685055165496492, + "loss": 1.2107, + "theoretical_loss": 3.388713768131284, + "tokens_seen": 2578513920 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017683450351053162, + "loss": 1.2917, + "theoretical_loss": 3.3887069979130873, + "tokens_seen": 2578579456 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017681845536609832, + "loss": 1.2247, + "theoretical_loss": 3.388700227915135, + "tokens_seen": 2578644992 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017680240722166502, + "loss": 1.2004, + "theoretical_loss": 3.3886934581374137, + "tokens_seen": 2578710528 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017678635907723172, + "loss": 1.2369, + "theoretical_loss": 3.388686688579912, + "tokens_seen": 2578776064 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 862089, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3208363056182861, + "objective/train/theoretical_loss": 3.388679919242616, + "objective/train/tokens_used": 2599301600, + "theoretical_loss": 3.388679919242616, + "tokens_seen": 2578841600 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017677031093279842, + "loss": 1.2881, + "theoretical_loss": 3.388679919242616, + "tokens_seen": 2578841600 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017675426278836512, + "loss": 1.1887, + "theoretical_loss": 3.3886731501255136, + "tokens_seen": 2578907136 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017673821464393182, + "loss": 1.2279, + "theoretical_loss": 3.3886663812285924, + "tokens_seen": 2578972672 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017672216649949852, + "loss": 1.2466, + "theoretical_loss": 3.3886596125518382, + "tokens_seen": 2579038208 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017670611835506522, + "loss": 1.2431, + "theoretical_loss": 3.38865284409524, + "tokens_seen": 2579103744 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001766900702106319, + "loss": 1.1764, + "theoretical_loss": 3.3886460758587837, + "tokens_seen": 2579169280 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001766740220661986, + "loss": 1.3048, + "theoretical_loss": 3.388639307842457, + "tokens_seen": 2579234816 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001766579739217653, + "loss": 1.2055, + "theoretical_loss": 3.3886325400462476, + "tokens_seen": 2579300352 + }, + { + "epoch": 0.78, + "learning_rate": 0.000176641925777332, + "loss": 1.1889, + "theoretical_loss": 3.388625772470142, + "tokens_seen": 2579365888 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001766258776328987, + "loss": 1.2055, + "theoretical_loss": 3.3886190051141276, + "tokens_seen": 2579431424 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001766098294884654, + "loss": 1.2845, + "theoretical_loss": 3.388612237978192, + "tokens_seen": 2579496960 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001765937813440321, + "loss": 1.1668, + "theoretical_loss": 3.3886054710623226, + "tokens_seen": 2579562496 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001765777331995988, + "loss": 1.3261, + "theoretical_loss": 3.3885987043665056, + "tokens_seen": 2579628032 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001765616850551655, + "loss": 1.2671, + "theoretical_loss": 3.3885919378907294, + "tokens_seen": 2579693568 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001765456369107322, + "loss": 1.2551, + "theoretical_loss": 3.3885851716349813, + "tokens_seen": 2579759104 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001765295887662989, + "loss": 1.2194, + "theoretical_loss": 3.3885784055992474, + "tokens_seen": 2579824640 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001765135406218656, + "loss": 1.1647, + "theoretical_loss": 3.388571639783516, + "tokens_seen": 2579890176 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001764974924774323, + "loss": 1.2953, + "theoretical_loss": 3.3885648741877743, + "tokens_seen": 2579955712 + }, + { + "epoch": 0.78, + "learning_rate": 0.000176481444332999, + "loss": 1.2153, + "theoretical_loss": 3.388558108812009, + "tokens_seen": 2580021248 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001764653961885657, + "loss": 1.2077, + "theoretical_loss": 3.388551343656207, + "tokens_seen": 2580086784 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001764493480441324, + "loss": 1.242, + "theoretical_loss": 3.3885445787203574, + "tokens_seen": 2580152320 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001764332998996991, + "loss": 1.1567, + "theoretical_loss": 3.3885378140044455, + "tokens_seen": 2580217856 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001764172517552658, + "loss": 1.2589, + "theoretical_loss": 3.3885310495084595, + "tokens_seen": 2580283392 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001764012036108325, + "loss": 1.2567, + "theoretical_loss": 3.3885242852323865, + "tokens_seen": 2580348928 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001763851554663992, + "loss": 1.2486, + "theoretical_loss": 3.3885175211762135, + "tokens_seen": 2580414464 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001763691073219659, + "loss": 1.1785, + "theoretical_loss": 3.3885107573399282, + "tokens_seen": 2580480000 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001763530591775326, + "loss": 1.257, + "theoretical_loss": 3.3885039937235177, + "tokens_seen": 2580545536 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001763370110330993, + "loss": 1.2573, + "theoretical_loss": 3.388497230326969, + "tokens_seen": 2580611072 + }, + { + "epoch": 0.78, + "learning_rate": 0.000176320962888666, + "loss": 1.2117, + "theoretical_loss": 3.38849046715027, + "tokens_seen": 2580676608 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017630491474423268, + "loss": 1.2809, + "theoretical_loss": 3.3884837041934075, + "tokens_seen": 2580742144 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017628886659979938, + "loss": 1.2693, + "theoretical_loss": 3.388476941456369, + "tokens_seen": 2580807680 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001762728184553661, + "loss": 1.19, + "theoretical_loss": 3.3884701789391416, + "tokens_seen": 2580873216 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001762567703109328, + "loss": 1.293, + "theoretical_loss": 3.3884634166417125, + "tokens_seen": 2580938752 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001762407221664995, + "loss": 1.2969, + "theoretical_loss": 3.388456654564069, + "tokens_seen": 2581004288 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017622467402206621, + "loss": 1.2026, + "theoretical_loss": 3.3884498927061983, + "tokens_seen": 2581069824 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017620862587763291, + "loss": 1.232, + "theoretical_loss": 3.3884431310680885, + "tokens_seen": 2581135360 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017619257773319961, + "loss": 1.2468, + "theoretical_loss": 3.3884363696497255, + "tokens_seen": 2581200896 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017617652958876632, + "loss": 1.2059, + "theoretical_loss": 3.388429608451098, + "tokens_seen": 2581266432 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017616048144433302, + "loss": 1.2822, + "theoretical_loss": 3.3884228474721922, + "tokens_seen": 2581331968 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017614443329989972, + "loss": 1.2626, + "theoretical_loss": 3.3884160867129953, + "tokens_seen": 2581397504 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017612838515546642, + "loss": 1.3318, + "theoretical_loss": 3.388409326173496, + "tokens_seen": 2581463040 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017611233701103312, + "loss": 1.2436, + "theoretical_loss": 3.38840256585368, + "tokens_seen": 2581528576 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017609628886659982, + "loss": 1.3282, + "theoretical_loss": 3.3883958057535355, + "tokens_seen": 2581594112 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017608024072216652, + "loss": 1.2166, + "theoretical_loss": 3.3883890458730495, + "tokens_seen": 2581659648 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017606419257773322, + "loss": 1.3145, + "theoretical_loss": 3.3883822862122086, + "tokens_seen": 2581725184 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017604814443329992, + "loss": 1.2569, + "theoretical_loss": 3.388375526771002, + "tokens_seen": 2581790720 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017603209628886662, + "loss": 1.1943, + "theoretical_loss": 3.388368767549415, + "tokens_seen": 2581856256 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017601604814443332, + "loss": 1.2167, + "theoretical_loss": 3.3883620085474355, + "tokens_seen": 2581921792 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017600000000000002, + "loss": 1.2696, + "theoretical_loss": 3.388355249765051, + "tokens_seen": 2581987328 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001759839518555667, + "loss": 1.2522, + "theoretical_loss": 3.3883484912022492, + "tokens_seen": 2582052864 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 863113, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0789929628372192, + "objective/train/theoretical_loss": 3.3883417328590166, + "objective/train/tokens_used": 2602578400, + "theoretical_loss": 3.3883417328590166, + "tokens_seen": 2582118400 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001759679037111334, + "loss": 1.1272, + "theoretical_loss": 3.3883417328590166, + "tokens_seen": 2582118400 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001759518555667001, + "loss": 1.2293, + "theoretical_loss": 3.388334974735341, + "tokens_seen": 2582183936 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001759358074222668, + "loss": 1.146, + "theoretical_loss": 3.388328216831209, + "tokens_seen": 2582249472 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001759197592778335, + "loss": 1.1692, + "theoretical_loss": 3.388321459146609, + "tokens_seen": 2582315008 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001759037111334002, + "loss": 1.1112, + "theoretical_loss": 3.3883147016815274, + "tokens_seen": 2582380544 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001758876629889669, + "loss": 1.2691, + "theoretical_loss": 3.388307944435952, + "tokens_seen": 2582446080 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001758716148445336, + "loss": 1.1495, + "theoretical_loss": 3.3883011874098696, + "tokens_seen": 2582511616 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001758555667001003, + "loss": 1.1528, + "theoretical_loss": 3.3882944306032683, + "tokens_seen": 2582577152 + }, + { + "epoch": 0.78, + "learning_rate": 0.000175839518555667, + "loss": 1.2616, + "theoretical_loss": 3.3882876740161345, + "tokens_seen": 2582642688 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001758234704112337, + "loss": 1.2362, + "theoretical_loss": 3.3882809176484563, + "tokens_seen": 2582708224 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001758074222668004, + "loss": 1.2295, + "theoretical_loss": 3.3882741615002203, + "tokens_seen": 2582773760 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001757913741223671, + "loss": 1.2835, + "theoretical_loss": 3.388267405571414, + "tokens_seen": 2582839296 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001757753259779338, + "loss": 1.3089, + "theoretical_loss": 3.3882606498620254, + "tokens_seen": 2582904832 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001757592778335005, + "loss": 1.2554, + "theoretical_loss": 3.3882538943720406, + "tokens_seen": 2582970368 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001757432296890672, + "loss": 1.3243, + "theoretical_loss": 3.388247139101448, + "tokens_seen": 2583035904 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001757271815446339, + "loss": 1.1819, + "theoretical_loss": 3.388240384050234, + "tokens_seen": 2583101440 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001757111334002006, + "loss": 1.1495, + "theoretical_loss": 3.3882336292183868, + "tokens_seen": 2583166976 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001756950852557673, + "loss": 1.2114, + "theoretical_loss": 3.388226874605893, + "tokens_seen": 2583232512 + }, + { + "epoch": 0.78, + "learning_rate": 0.000175679037111334, + "loss": 1.2126, + "theoretical_loss": 3.38822012021274, + "tokens_seen": 2583298048 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001756629889669007, + "loss": 1.2051, + "theoretical_loss": 3.388213366038916, + "tokens_seen": 2583363584 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001756469408224674, + "loss": 1.2365, + "theoretical_loss": 3.388206612084407, + "tokens_seen": 2583429120 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001756308926780341, + "loss": 1.1507, + "theoretical_loss": 3.3881998583492012, + "tokens_seen": 2583494656 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001756148445336008, + "loss": 1.1995, + "theoretical_loss": 3.3881931048332854, + "tokens_seen": 2583560192 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001755987963891675, + "loss": 1.1943, + "theoretical_loss": 3.3881863515366475, + "tokens_seen": 2583625728 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001755827482447342, + "loss": 1.2827, + "theoretical_loss": 3.388179598459274, + "tokens_seen": 2583691264 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001755667001003009, + "loss": 1.2566, + "theoretical_loss": 3.3881728456011535, + "tokens_seen": 2583756800 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001755506519558676, + "loss": 1.3004, + "theoretical_loss": 3.3881660929622717, + "tokens_seen": 2583822336 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017553460381143431, + "loss": 1.2872, + "theoretical_loss": 3.388159340542617, + "tokens_seen": 2583887872 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017551855566700101, + "loss": 1.2414, + "theoretical_loss": 3.3881525883421766, + "tokens_seen": 2583953408 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017550250752256772, + "loss": 1.2828, + "theoretical_loss": 3.3881458363609376, + "tokens_seen": 2584018944 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017548645937813442, + "loss": 1.1387, + "theoretical_loss": 3.3881390845988877, + "tokens_seen": 2584084480 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017547041123370112, + "loss": 1.2826, + "theoretical_loss": 3.3881323330560136, + "tokens_seen": 2584150016 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017545436308926782, + "loss": 1.3971, + "theoretical_loss": 3.388125581732303, + "tokens_seen": 2584215552 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017543831494483452, + "loss": 1.3271, + "theoretical_loss": 3.3881188306277434, + "tokens_seen": 2584281088 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017542226680040122, + "loss": 1.2073, + "theoretical_loss": 3.3881120797423216, + "tokens_seen": 2584346624 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017540621865596792, + "loss": 1.1866, + "theoretical_loss": 3.3881053290760255, + "tokens_seen": 2584412160 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017539017051153462, + "loss": 1.2537, + "theoretical_loss": 3.388098578628842, + "tokens_seen": 2584477696 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017537412236710132, + "loss": 1.1758, + "theoretical_loss": 3.388091828400759, + "tokens_seen": 2584543232 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017535807422266802, + "loss": 1.2253, + "theoretical_loss": 3.3880850783917627, + "tokens_seen": 2584608768 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017534202607823472, + "loss": 1.2839, + "theoretical_loss": 3.388078328601842, + "tokens_seen": 2584674304 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017532597793380142, + "loss": 1.175, + "theoretical_loss": 3.388071579030983, + "tokens_seen": 2584739840 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017530992978936812, + "loss": 1.3016, + "theoretical_loss": 3.3880648296791733, + "tokens_seen": 2584805376 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017529388164493482, + "loss": 1.2856, + "theoretical_loss": 3.3880580805464007, + "tokens_seen": 2584870912 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001752778335005015, + "loss": 1.0882, + "theoretical_loss": 3.388051331632652, + "tokens_seen": 2584936448 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001752617853560682, + "loss": 1.2157, + "theoretical_loss": 3.388044582937915, + "tokens_seen": 2585001984 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001752457372116349, + "loss": 1.1479, + "theoretical_loss": 3.388037834462177, + "tokens_seen": 2585067520 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001752296890672016, + "loss": 1.2277, + "theoretical_loss": 3.388031086205425, + "tokens_seen": 2585133056 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001752136409227683, + "loss": 1.1536, + "theoretical_loss": 3.388024338167646, + "tokens_seen": 2585198592 + }, + { + "epoch": 0.78, + "learning_rate": 0.000175197592778335, + "loss": 1.137, + "theoretical_loss": 3.388017590348828, + "tokens_seen": 2585264128 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001751815446339017, + "loss": 1.2411, + "theoretical_loss": 3.3880108427489586, + "tokens_seen": 2585329664 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 864353, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1530442237854004, + "objective/train/theoretical_loss": 3.3880040953680246, + "objective/train/tokens_used": 2605855200, + "theoretical_loss": 3.3880040953680246, + "tokens_seen": 2585395200 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001751654964894684, + "loss": 1.146, + "theoretical_loss": 3.3880040953680246, + "tokens_seen": 2585395200 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001751494483450351, + "loss": 1.2946, + "theoretical_loss": 3.3879973482060137, + "tokens_seen": 2585460736 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001751334002006018, + "loss": 1.1847, + "theoretical_loss": 3.3879906012629126, + "tokens_seen": 2585526272 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001751173520561685, + "loss": 1.2037, + "theoretical_loss": 3.387983854538709, + "tokens_seen": 2585591808 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001751013039117352, + "loss": 1.232, + "theoretical_loss": 3.3879771080333905, + "tokens_seen": 2585657344 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001750852557673019, + "loss": 1.2301, + "theoretical_loss": 3.3879703617469445, + "tokens_seen": 2585722880 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017506920762286863, + "loss": 1.1916, + "theoretical_loss": 3.3879636156793578, + "tokens_seen": 2585788416 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017505315947843533, + "loss": 1.1821, + "theoretical_loss": 3.387956869830618, + "tokens_seen": 2585853952 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017503711133400203, + "loss": 1.2764, + "theoretical_loss": 3.387950124200713, + "tokens_seen": 2585919488 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017502106318956873, + "loss": 1.1935, + "theoretical_loss": 3.3879433787896294, + "tokens_seen": 2585985024 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017500501504513543, + "loss": 1.2317, + "theoretical_loss": 3.387936633597355, + "tokens_seen": 2586050560 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017498896690070214, + "loss": 1.2129, + "theoretical_loss": 3.387929888623877, + "tokens_seen": 2586116096 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001749729187562688, + "loss": 1.2843, + "theoretical_loss": 3.387923143869182, + "tokens_seen": 2586181632 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001749568706118355, + "loss": 1.3108, + "theoretical_loss": 3.387916399333259, + "tokens_seen": 2586247168 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001749408224674022, + "loss": 1.0919, + "theoretical_loss": 3.3879096550160943, + "tokens_seen": 2586312704 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001749247743229689, + "loss": 1.2668, + "theoretical_loss": 3.3879029109176755, + "tokens_seen": 2586378240 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001749087261785356, + "loss": 1.2245, + "theoretical_loss": 3.3878961670379897, + "tokens_seen": 2586443776 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001748926780341023, + "loss": 1.2419, + "theoretical_loss": 3.3878894233770245, + "tokens_seen": 2586509312 + }, + { + "epoch": 0.78, + "learning_rate": 0.000174876629889669, + "loss": 1.2655, + "theoretical_loss": 3.3878826799347674, + "tokens_seen": 2586574848 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001748605817452357, + "loss": 1.1806, + "theoretical_loss": 3.3878759367112057, + "tokens_seen": 2586640384 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017484453360080241, + "loss": 1.1519, + "theoretical_loss": 3.3878691937063263, + "tokens_seen": 2586705920 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017482848545636911, + "loss": 1.1578, + "theoretical_loss": 3.387862450920117, + "tokens_seen": 2586771456 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017481243731193582, + "loss": 1.1833, + "theoretical_loss": 3.387855708352565, + "tokens_seen": 2586836992 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017479638916750252, + "loss": 1.2842, + "theoretical_loss": 3.3878489660036584, + "tokens_seen": 2586902528 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017478034102306922, + "loss": 1.2204, + "theoretical_loss": 3.3878422238733834, + "tokens_seen": 2586968064 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017476429287863592, + "loss": 1.307, + "theoretical_loss": 3.3878354819617282, + "tokens_seen": 2587033600 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017474824473420262, + "loss": 1.2557, + "theoretical_loss": 3.3878287402686795, + "tokens_seen": 2587099136 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017473219658976932, + "loss": 1.2095, + "theoretical_loss": 3.3878219987942253, + "tokens_seen": 2587164672 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017471614844533602, + "loss": 1.258, + "theoretical_loss": 3.3878152575383527, + "tokens_seen": 2587230208 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017470010030090272, + "loss": 1.329, + "theoretical_loss": 3.387808516501049, + "tokens_seen": 2587295744 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017468405215646942, + "loss": 1.2254, + "theoretical_loss": 3.387801775682302, + "tokens_seen": 2587361280 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017466800401203612, + "loss": 1.2582, + "theoretical_loss": 3.3877950350820987, + "tokens_seen": 2587426816 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017465195586760282, + "loss": 1.3002, + "theoretical_loss": 3.3877882947004263, + "tokens_seen": 2587492352 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017463590772316952, + "loss": 1.2101, + "theoretical_loss": 3.387781554537273, + "tokens_seen": 2587557888 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017461985957873622, + "loss": 1.3312, + "theoretical_loss": 3.3877748145926248, + "tokens_seen": 2587623424 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017460381143430292, + "loss": 1.1435, + "theoretical_loss": 3.3877680748664707, + "tokens_seen": 2587688960 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001745877632898696, + "loss": 1.2259, + "theoretical_loss": 3.387761335358797, + "tokens_seen": 2587754496 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001745717151454363, + "loss": 1.2851, + "theoretical_loss": 3.387754596069591, + "tokens_seen": 2587820032 + }, + { + "epoch": 0.78, + "learning_rate": 0.000174555667001003, + "loss": 1.2391, + "theoretical_loss": 3.387747856998841, + "tokens_seen": 2587885568 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001745396188565697, + "loss": 1.12, + "theoretical_loss": 3.3877411181465336, + "tokens_seen": 2587951104 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001745235707121364, + "loss": 1.2151, + "theoretical_loss": 3.387734379512656, + "tokens_seen": 2588016640 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001745075225677031, + "loss": 1.2787, + "theoretical_loss": 3.3877276410971966, + "tokens_seen": 2588082176 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001744914744232698, + "loss": 1.2199, + "theoretical_loss": 3.387720902900142, + "tokens_seen": 2588147712 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001744754262788365, + "loss": 1.1933, + "theoretical_loss": 3.3877141649214795, + "tokens_seen": 2588213248 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017445937813440323, + "loss": 1.2146, + "theoretical_loss": 3.387707427161197, + "tokens_seen": 2588278784 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017444332998996993, + "loss": 1.2288, + "theoretical_loss": 3.387700689619282, + "tokens_seen": 2588344320 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017442728184553663, + "loss": 1.3002, + "theoretical_loss": 3.387693952295721, + "tokens_seen": 2588409856 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017441123370110333, + "loss": 1.1622, + "theoretical_loss": 3.3876872151905024, + "tokens_seen": 2588475392 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017439518555667003, + "loss": 1.2999, + "theoretical_loss": 3.387680478303613, + "tokens_seen": 2588540928 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017437913741223673, + "loss": 1.1932, + "theoretical_loss": 3.387673741635041, + "tokens_seen": 2588606464 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 865338, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2663666009902954, + "objective/train/theoretical_loss": 3.387667005184772, + "objective/train/tokens_used": 2609132000, + "theoretical_loss": 3.387667005184772, + "tokens_seen": 2588672000 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017436308926780343, + "loss": 1.2422, + "theoretical_loss": 3.387667005184772, + "tokens_seen": 2588672000 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017434704112337013, + "loss": 1.2062, + "theoretical_loss": 3.3876602689527955, + "tokens_seen": 2588737536 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017433099297893683, + "loss": 1.2252, + "theoretical_loss": 3.3876535329390975, + "tokens_seen": 2588803072 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017431494483450353, + "loss": 1.2304, + "theoretical_loss": 3.387646797143666, + "tokens_seen": 2588868608 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017429889669007024, + "loss": 1.2406, + "theoretical_loss": 3.387640061566488, + "tokens_seen": 2588934144 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017428284854563694, + "loss": 1.21, + "theoretical_loss": 3.3876333262075518, + "tokens_seen": 2588999680 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001742668004012036, + "loss": 1.2748, + "theoretical_loss": 3.3876265910668435, + "tokens_seen": 2589065216 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001742507522567703, + "loss": 1.2559, + "theoretical_loss": 3.387619856144352, + "tokens_seen": 2589130752 + }, + { + "epoch": 0.78, + "learning_rate": 0.000174234704112337, + "loss": 1.2898, + "theoretical_loss": 3.387613121440063, + "tokens_seen": 2589196288 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001742186559679037, + "loss": 1.3076, + "theoretical_loss": 3.3876063869539648, + "tokens_seen": 2589261824 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001742026078234704, + "loss": 1.1977, + "theoretical_loss": 3.3875996526860455, + "tokens_seen": 2589327360 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001741865596790371, + "loss": 1.2087, + "theoretical_loss": 3.387592918636291, + "tokens_seen": 2589392896 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017417051153460381, + "loss": 1.2353, + "theoretical_loss": 3.3875861848046904, + "tokens_seen": 2589458432 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017415446339017051, + "loss": 1.213, + "theoretical_loss": 3.38757945119123, + "tokens_seen": 2589523968 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017413841524573721, + "loss": 1.2511, + "theoretical_loss": 3.387572717795897, + "tokens_seen": 2589589504 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017412236710130392, + "loss": 1.2558, + "theoretical_loss": 3.3875659846186794, + "tokens_seen": 2589655040 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017410631895687062, + "loss": 1.1977, + "theoretical_loss": 3.3875592516595647, + "tokens_seen": 2589720576 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017409027081243732, + "loss": 1.1953, + "theoretical_loss": 3.38755251891854, + "tokens_seen": 2589786112 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017407422266800402, + "loss": 1.3006, + "theoretical_loss": 3.3875457863955925, + "tokens_seen": 2589851648 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017405817452357072, + "loss": 1.2918, + "theoretical_loss": 3.3875390540907104, + "tokens_seen": 2589917184 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017404212637913742, + "loss": 1.2164, + "theoretical_loss": 3.3875323220038807, + "tokens_seen": 2589982720 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017402607823470412, + "loss": 1.1891, + "theoretical_loss": 3.3875255901350902, + "tokens_seen": 2590048256 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017401003009027082, + "loss": 1.2296, + "theoretical_loss": 3.387518858484327, + "tokens_seen": 2590113792 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017399398194583752, + "loss": 1.1705, + "theoretical_loss": 3.387512127051579, + "tokens_seen": 2590179328 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017397793380140422, + "loss": 1.1944, + "theoretical_loss": 3.3875053958368326, + "tokens_seen": 2590244864 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017396188565697092, + "loss": 1.2131, + "theoretical_loss": 3.387498664840076, + "tokens_seen": 2590310400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017394583751253762, + "loss": 1.2224, + "theoretical_loss": 3.3874919340612957, + "tokens_seen": 2590375936 + }, + { + "epoch": 0.78, + "learning_rate": 0.00017392978936810432, + "loss": 1.2181, + "theoretical_loss": 3.3874852035004803, + "tokens_seen": 2590441472 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017391374122367102, + "loss": 1.2253, + "theoretical_loss": 3.387478473157616, + "tokens_seen": 2590507008 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017389769307923772, + "loss": 1.2033, + "theoretical_loss": 3.387471743032691, + "tokens_seen": 2590572544 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001738816449348044, + "loss": 1.2208, + "theoretical_loss": 3.387465013125693, + "tokens_seen": 2590638080 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017386559679037113, + "loss": 1.1451, + "theoretical_loss": 3.387458283436609, + "tokens_seen": 2590703616 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017384954864593783, + "loss": 1.2017, + "theoretical_loss": 3.3874515539654264, + "tokens_seen": 2590769152 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017383350050150453, + "loss": 1.2549, + "theoretical_loss": 3.3874448247121327, + "tokens_seen": 2590834688 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017381745235707123, + "loss": 1.1609, + "theoretical_loss": 3.387438095676715, + "tokens_seen": 2590900224 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017380140421263793, + "loss": 1.2214, + "theoretical_loss": 3.387431366859161, + "tokens_seen": 2590965760 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017378535606820463, + "loss": 1.2006, + "theoretical_loss": 3.387424638259459, + "tokens_seen": 2591031296 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017376930792377133, + "loss": 1.2083, + "theoretical_loss": 3.3874179098775947, + "tokens_seen": 2591096832 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017375325977933803, + "loss": 1.3134, + "theoretical_loss": 3.3874111817135573, + "tokens_seen": 2591162368 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017373721163490473, + "loss": 1.2511, + "theoretical_loss": 3.3874044537673327, + "tokens_seen": 2591227904 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017372116349047143, + "loss": 1.2761, + "theoretical_loss": 3.3873977260389094, + "tokens_seen": 2591293440 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017370511534603813, + "loss": 1.1932, + "theoretical_loss": 3.387390998528274, + "tokens_seen": 2591358976 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017368906720160483, + "loss": 1.2793, + "theoretical_loss": 3.3873842712354154, + "tokens_seen": 2591424512 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017367301905717153, + "loss": 1.2113, + "theoretical_loss": 3.3873775441603193, + "tokens_seen": 2591490048 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017365697091273823, + "loss": 1.3129, + "theoretical_loss": 3.3873708173029744, + "tokens_seen": 2591555584 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017364092276830493, + "loss": 1.1931, + "theoretical_loss": 3.3873640906633673, + "tokens_seen": 2591621120 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017362487462387164, + "loss": 1.2458, + "theoretical_loss": 3.387357364241486, + "tokens_seen": 2591686656 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017360882647943834, + "loss": 1.235, + "theoretical_loss": 3.3873506380373177, + "tokens_seen": 2591752192 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017359277833500504, + "loss": 1.2025, + "theoretical_loss": 3.38734391205085, + "tokens_seen": 2591817728 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017357673019057174, + "loss": 1.2485, + "theoretical_loss": 3.3873371862820703, + "tokens_seen": 2591883264 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 866617, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.222945213317871, + "objective/train/theoretical_loss": 3.3873304607309658, + "objective/train/tokens_used": 2612408800, + "theoretical_loss": 3.3873304607309658, + "tokens_seen": 2591948800 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001735606820461384, + "loss": 1.2603, + "theoretical_loss": 3.3873304607309658, + "tokens_seen": 2591948800 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001735446339017051, + "loss": 1.2435, + "theoretical_loss": 3.387323735397524, + "tokens_seen": 2592014336 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001735285857572718, + "loss": 1.3102, + "theoretical_loss": 3.387317010281733, + "tokens_seen": 2592079872 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001735125376128385, + "loss": 1.2392, + "theoretical_loss": 3.3873102853835793, + "tokens_seen": 2592145408 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001734964894684052, + "loss": 1.2833, + "theoretical_loss": 3.3873035607030513, + "tokens_seen": 2592210944 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017348044132397191, + "loss": 1.1937, + "theoretical_loss": 3.3872968362401354, + "tokens_seen": 2592276480 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017346439317953861, + "loss": 1.2002, + "theoretical_loss": 3.38729011199482, + "tokens_seen": 2592342016 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017344834503510532, + "loss": 1.2345, + "theoretical_loss": 3.3872833879670923, + "tokens_seen": 2592407552 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017343229689067202, + "loss": 1.2141, + "theoretical_loss": 3.3872766641569396, + "tokens_seen": 2592473088 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017341624874623872, + "loss": 1.123, + "theoretical_loss": 3.387269940564349, + "tokens_seen": 2592538624 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017340020060180542, + "loss": 1.2176, + "theoretical_loss": 3.3872632171893087, + "tokens_seen": 2592604160 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017338415245737212, + "loss": 1.1603, + "theoretical_loss": 3.387256494031806, + "tokens_seen": 2592669696 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017336810431293882, + "loss": 1.2691, + "theoretical_loss": 3.387249771091828, + "tokens_seen": 2592735232 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017335205616850552, + "loss": 1.2367, + "theoretical_loss": 3.387243048369362, + "tokens_seen": 2592800768 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017333600802407222, + "loss": 1.1419, + "theoretical_loss": 3.3872363258643965, + "tokens_seen": 2592866304 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017331995987963892, + "loss": 1.2271, + "theoretical_loss": 3.387229603576918, + "tokens_seen": 2592931840 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017330391173520562, + "loss": 1.1657, + "theoretical_loss": 3.387222881506914, + "tokens_seen": 2592997376 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017328786359077232, + "loss": 1.2565, + "theoretical_loss": 3.3872161596543724, + "tokens_seen": 2593062912 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017327181544633905, + "loss": 1.1884, + "theoretical_loss": 3.3872094380192808, + "tokens_seen": 2593128448 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017325576730190572, + "loss": 1.2188, + "theoretical_loss": 3.3872027166016263, + "tokens_seen": 2593193984 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017323971915747242, + "loss": 1.1616, + "theoretical_loss": 3.387195995401396, + "tokens_seen": 2593259520 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017322367101303912, + "loss": 1.3137, + "theoretical_loss": 3.387189274418578, + "tokens_seen": 2593325056 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017320762286860582, + "loss": 1.1413, + "theoretical_loss": 3.38718255365316, + "tokens_seen": 2593390592 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017319157472417253, + "loss": 1.1535, + "theoretical_loss": 3.3871758331051285, + "tokens_seen": 2593456128 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017317552657973923, + "loss": 1.2339, + "theoretical_loss": 3.3871691127744716, + "tokens_seen": 2593521664 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017315947843530593, + "loss": 1.2153, + "theoretical_loss": 3.387162392661177, + "tokens_seen": 2593587200 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017314343029087263, + "loss": 1.1137, + "theoretical_loss": 3.387155672765232, + "tokens_seen": 2593652736 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017312738214643933, + "loss": 1.223, + "theoretical_loss": 3.3871489530866232, + "tokens_seen": 2593718272 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017311133400200603, + "loss": 1.2706, + "theoretical_loss": 3.3871422336253394, + "tokens_seen": 2593783808 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017309528585757273, + "loss": 1.2182, + "theoretical_loss": 3.3871355143813675, + "tokens_seen": 2593849344 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017309528585757273, + "loss": 1.1527, + "theoretical_loss": 3.387128795354695, + "tokens_seen": 2593914880 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017307923771313943, + "loss": 1.2436, + "theoretical_loss": 3.3871220765453094, + "tokens_seen": 2593980416 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017306318956870613, + "loss": 1.2908, + "theoretical_loss": 3.3871153579531983, + "tokens_seen": 2594045952 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017304714142427283, + "loss": 1.4248, + "theoretical_loss": 3.3871086395783485, + "tokens_seen": 2594111488 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017303109327983953, + "loss": 1.2007, + "theoretical_loss": 3.3871019214207485, + "tokens_seen": 2594177024 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017301504513540623, + "loss": 1.2614, + "theoretical_loss": 3.3870952034803854, + "tokens_seen": 2594242560 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017299899699097293, + "loss": 1.2362, + "theoretical_loss": 3.3870884857572463, + "tokens_seen": 2594308096 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017298294884653963, + "loss": 1.2607, + "theoretical_loss": 3.3870817682513192, + "tokens_seen": 2594373632 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017296690070210633, + "loss": 1.1894, + "theoretical_loss": 3.3870750509625918, + "tokens_seen": 2594439168 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017295085255767303, + "loss": 1.1638, + "theoretical_loss": 3.3870683338910506, + "tokens_seen": 2594504704 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017293480441323974, + "loss": 1.212, + "theoretical_loss": 3.387061617036684, + "tokens_seen": 2594570240 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017291875626880644, + "loss": 1.2823, + "theoretical_loss": 3.3870549003994785, + "tokens_seen": 2594635776 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017290270812437314, + "loss": 1.1624, + "theoretical_loss": 3.387048183979423, + "tokens_seen": 2594701312 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017288665997993984, + "loss": 1.1834, + "theoretical_loss": 3.387041467776504, + "tokens_seen": 2594766848 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001728706118355065, + "loss": 1.309, + "theoretical_loss": 3.3870347517907096, + "tokens_seen": 2594832384 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001728545636910732, + "loss": 1.2446, + "theoretical_loss": 3.3870280360220266, + "tokens_seen": 2594897920 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001728385155466399, + "loss": 1.255, + "theoretical_loss": 3.387021320470443, + "tokens_seen": 2594963456 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001728224674022066, + "loss": 1.3284, + "theoretical_loss": 3.387014605135946, + "tokens_seen": 2595028992 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001728064192577733, + "loss": 1.2686, + "theoretical_loss": 3.3870078900185234, + "tokens_seen": 2595094528 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017279037111334001, + "loss": 1.2053, + "theoretical_loss": 3.3870011751181623, + "tokens_seen": 2595160064 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 867500, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2935428619384766, + "objective/train/theoretical_loss": 3.3869944604348508, + "objective/train/tokens_used": 2615685600, + "theoretical_loss": 3.3869944604348508, + "tokens_seen": 2595225600 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017277432296890671, + "loss": 1.2201, + "theoretical_loss": 3.3869944604348508, + "tokens_seen": 2595225600 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017275827482447342, + "loss": 1.3862, + "theoretical_loss": 3.3869877459685758, + "tokens_seen": 2595291136 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017274222668004012, + "loss": 1.2416, + "theoretical_loss": 3.3869810317193254, + "tokens_seen": 2595356672 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017272617853560682, + "loss": 1.1229, + "theoretical_loss": 3.3869743176870863, + "tokens_seen": 2595422208 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017271013039117352, + "loss": 1.2682, + "theoretical_loss": 3.386967603871847, + "tokens_seen": 2595487744 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017269408224674022, + "loss": 1.1596, + "theoretical_loss": 3.386960890273594, + "tokens_seen": 2595553280 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017267803410230692, + "loss": 1.3222, + "theoretical_loss": 3.3869541768923153, + "tokens_seen": 2595618816 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017266198595787365, + "loss": 1.16, + "theoretical_loss": 3.386947463727999, + "tokens_seen": 2595684352 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017264593781344035, + "loss": 1.2224, + "theoretical_loss": 3.3869407507806315, + "tokens_seen": 2595749888 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017262988966900705, + "loss": 1.3072, + "theoretical_loss": 3.386934038050201, + "tokens_seen": 2595815424 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017261384152457375, + "loss": 1.1697, + "theoretical_loss": 3.3869273255366945, + "tokens_seen": 2595880960 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017259779338014045, + "loss": 1.1951, + "theoretical_loss": 3.3869206132401004, + "tokens_seen": 2595946496 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017258174523570715, + "loss": 1.2609, + "theoretical_loss": 3.3869139011604052, + "tokens_seen": 2596012032 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017256569709127385, + "loss": 1.3575, + "theoretical_loss": 3.386907189297597, + "tokens_seen": 2596077568 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017254964894684052, + "loss": 1.3009, + "theoretical_loss": 3.3869004776516634, + "tokens_seen": 2596143104 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017253360080240722, + "loss": 1.2669, + "theoretical_loss": 3.3868937662225918, + "tokens_seen": 2596208640 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017251755265797392, + "loss": 1.3154, + "theoretical_loss": 3.3868870550103694, + "tokens_seen": 2596274176 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017250150451354063, + "loss": 1.2121, + "theoretical_loss": 3.3868803440149837, + "tokens_seen": 2596339712 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017248545636910733, + "loss": 1.2473, + "theoretical_loss": 3.386873633236423, + "tokens_seen": 2596405248 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017246940822467403, + "loss": 1.3068, + "theoretical_loss": 3.386866922674674, + "tokens_seen": 2596470784 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017245336008024073, + "loss": 1.25, + "theoretical_loss": 3.3868602123297245, + "tokens_seen": 2596536320 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017243731193580743, + "loss": 1.2173, + "theoretical_loss": 3.3868535022015624, + "tokens_seen": 2596601856 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017242126379137413, + "loss": 1.2819, + "theoretical_loss": 3.3868467922901746, + "tokens_seen": 2596667392 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017240521564694083, + "loss": 1.2966, + "theoretical_loss": 3.386840082595549, + "tokens_seen": 2596732928 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017238916750250753, + "loss": 1.2212, + "theoretical_loss": 3.386833373117673, + "tokens_seen": 2596798464 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017237311935807423, + "loss": 1.2407, + "theoretical_loss": 3.386826663856534, + "tokens_seen": 2596864000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017235707121364093, + "loss": 1.2206, + "theoretical_loss": 3.3868199548121196, + "tokens_seen": 2596929536 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017234102306920763, + "loss": 1.2696, + "theoretical_loss": 3.386813245984418, + "tokens_seen": 2596995072 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017232497492477433, + "loss": 1.2916, + "theoretical_loss": 3.386806537373415, + "tokens_seen": 2597060608 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017230892678034103, + "loss": 1.3016, + "theoretical_loss": 3.3867998289791004, + "tokens_seen": 2597126144 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017229287863590773, + "loss": 1.251, + "theoretical_loss": 3.3867931208014603, + "tokens_seen": 2597191680 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017227683049147443, + "loss": 1.1992, + "theoretical_loss": 3.3867864128404825, + "tokens_seen": 2597257216 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017226078234704113, + "loss": 1.1278, + "theoretical_loss": 3.386779705096154, + "tokens_seen": 2597322752 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017224473420260784, + "loss": 1.1856, + "theoretical_loss": 3.386772997568464, + "tokens_seen": 2597388288 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017222868605817454, + "loss": 1.3132, + "theoretical_loss": 3.3867662902573983, + "tokens_seen": 2597453824 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017221263791374124, + "loss": 1.3362, + "theoretical_loss": 3.386759583162945, + "tokens_seen": 2597519360 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017219658976930794, + "loss": 1.2204, + "theoretical_loss": 3.386752876285092, + "tokens_seen": 2597584896 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017218054162487464, + "loss": 1.3016, + "theoretical_loss": 3.3867461696238266, + "tokens_seen": 2597650432 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001721644934804413, + "loss": 1.1612, + "theoretical_loss": 3.3867394631791363, + "tokens_seen": 2597715968 + }, + { + "epoch": 0.79, + "learning_rate": 0.000172148445336008, + "loss": 1.2632, + "theoretical_loss": 3.3867327569510084, + "tokens_seen": 2597781504 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001721323971915747, + "loss": 1.3071, + "theoretical_loss": 3.386726050939431, + "tokens_seen": 2597847040 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001721163490471414, + "loss": 1.1998, + "theoretical_loss": 3.386719345144391, + "tokens_seen": 2597912576 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017210030090270811, + "loss": 1.1001, + "theoretical_loss": 3.386712639565877, + "tokens_seen": 2597978112 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017208425275827481, + "loss": 1.1156, + "theoretical_loss": 3.3867059342038752, + "tokens_seen": 2598043648 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017206820461384154, + "loss": 1.1023, + "theoretical_loss": 3.3866992290583737, + "tokens_seen": 2598109184 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017205215646940824, + "loss": 1.225, + "theoretical_loss": 3.3866925241293604, + "tokens_seen": 2598174720 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017203610832497494, + "loss": 1.1384, + "theoretical_loss": 3.386685819416823, + "tokens_seen": 2598240256 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017202006018054164, + "loss": 1.1971, + "theoretical_loss": 3.3866791149207476, + "tokens_seen": 2598305792 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017200401203610835, + "loss": 1.2702, + "theoretical_loss": 3.3866724106411237, + "tokens_seen": 2598371328 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017198796389167505, + "loss": 1.2759, + "theoretical_loss": 3.3866657065779373, + "tokens_seen": 2598436864 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 868777, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1186769008636475, + "objective/train/theoretical_loss": 3.386659002731177, + "objective/train/tokens_used": 2618962400, + "theoretical_loss": 3.386659002731177, + "tokens_seen": 2598502400 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017197191574724175, + "loss": 1.2016, + "theoretical_loss": 3.386659002731177, + "tokens_seen": 2598502400 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017195586760280845, + "loss": 1.142, + "theoretical_loss": 3.38665229910083, + "tokens_seen": 2598567936 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017193981945837515, + "loss": 1.3139, + "theoretical_loss": 3.3866455956868835, + "tokens_seen": 2598633472 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017192377131394185, + "loss": 1.2091, + "theoretical_loss": 3.3866388924893256, + "tokens_seen": 2598699008 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017190772316950855, + "loss": 1.0186, + "theoretical_loss": 3.386632189508144, + "tokens_seen": 2598764544 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017189167502507525, + "loss": 1.2268, + "theoretical_loss": 3.386625486743325, + "tokens_seen": 2598830080 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017187562688064195, + "loss": 1.1567, + "theoretical_loss": 3.386618784194858, + "tokens_seen": 2598895616 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017185957873620865, + "loss": 1.2143, + "theoretical_loss": 3.3866120818627286, + "tokens_seen": 2598961152 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017184353059177532, + "loss": 1.1921, + "theoretical_loss": 3.3866053797469258, + "tokens_seen": 2599026688 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017182748244734202, + "loss": 1.2387, + "theoretical_loss": 3.386598677847437, + "tokens_seen": 2599092224 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017181143430290873, + "loss": 1.1933, + "theoretical_loss": 3.386591976164249, + "tokens_seen": 2599157760 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017179538615847543, + "loss": 1.2019, + "theoretical_loss": 3.3865852746973504, + "tokens_seen": 2599223296 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017177933801404213, + "loss": 1.1098, + "theoretical_loss": 3.386578573446728, + "tokens_seen": 2599288832 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017176328986960883, + "loss": 1.2159, + "theoretical_loss": 3.3865718724123695, + "tokens_seen": 2599354368 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017174724172517553, + "loss": 1.229, + "theoretical_loss": 3.3865651715942624, + "tokens_seen": 2599419904 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017173119358074223, + "loss": 1.2518, + "theoretical_loss": 3.386558470992395, + "tokens_seen": 2599485440 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017171514543630893, + "loss": 1.2574, + "theoretical_loss": 3.3865517706067534, + "tokens_seen": 2599550976 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017169909729187563, + "loss": 1.2573, + "theoretical_loss": 3.386545070437327, + "tokens_seen": 2599616512 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017168304914744233, + "loss": 1.2931, + "theoretical_loss": 3.386538370484102, + "tokens_seen": 2599682048 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017166700100300903, + "loss": 1.2482, + "theoretical_loss": 3.3865316707470665, + "tokens_seen": 2599747584 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017165095285857573, + "loss": 1.2237, + "theoretical_loss": 3.3865249712262075, + "tokens_seen": 2599813120 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017163490471414243, + "loss": 1.292, + "theoretical_loss": 3.386518271921514, + "tokens_seen": 2599878656 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017161885656970913, + "loss": 1.2264, + "theoretical_loss": 3.3865115728329718, + "tokens_seen": 2599944192 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017160280842527583, + "loss": 1.2392, + "theoretical_loss": 3.38650487396057, + "tokens_seen": 2600009728 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017158676028084253, + "loss": 1.222, + "theoretical_loss": 3.386498175304295, + "tokens_seen": 2600075264 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017157071213640924, + "loss": 1.0486, + "theoretical_loss": 3.386491476864135, + "tokens_seen": 2600140800 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017155466399197594, + "loss": 1.254, + "theoretical_loss": 3.3864847786400776, + "tokens_seen": 2600206336 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017153861584754264, + "loss": 1.1932, + "theoretical_loss": 3.38647808063211, + "tokens_seen": 2600271872 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017152256770310934, + "loss": 1.3205, + "theoretical_loss": 3.38647138284022, + "tokens_seen": 2600337408 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017150651955867604, + "loss": 1.2231, + "theoretical_loss": 3.3864646852643956, + "tokens_seen": 2600402944 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017149047141424274, + "loss": 1.2535, + "theoretical_loss": 3.386457987904624, + "tokens_seen": 2600468480 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017147442326980944, + "loss": 1.1954, + "theoretical_loss": 3.3864512907608924, + "tokens_seen": 2600534016 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017145837512537614, + "loss": 1.2332, + "theoretical_loss": 3.386444593833189, + "tokens_seen": 2600599552 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017144232698094284, + "loss": 1.2781, + "theoretical_loss": 3.386437897121501, + "tokens_seen": 2600665088 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017142627883650954, + "loss": 1.2763, + "theoretical_loss": 3.3864312006258164, + "tokens_seen": 2600730624 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017141023069207624, + "loss": 1.0783, + "theoretical_loss": 3.386424504346122, + "tokens_seen": 2600796160 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017139418254764294, + "loss": 1.2935, + "theoretical_loss": 3.386417808282406, + "tokens_seen": 2600861696 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017137813440320964, + "loss": 1.2007, + "theoretical_loss": 3.3864111124346565, + "tokens_seen": 2600927232 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017136208625877634, + "loss": 1.2695, + "theoretical_loss": 3.3864044168028604, + "tokens_seen": 2600992768 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017134603811434304, + "loss": 1.2083, + "theoretical_loss": 3.3863977213870053, + "tokens_seen": 2601058304 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017132998996990974, + "loss": 1.2228, + "theoretical_loss": 3.386391026187079, + "tokens_seen": 2601123840 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017131394182547645, + "loss": 1.1878, + "theoretical_loss": 3.3863843312030686, + "tokens_seen": 2601189376 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017129789368104315, + "loss": 1.2929, + "theoretical_loss": 3.386377636434962, + "tokens_seen": 2601254912 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017128184553660985, + "loss": 1.2214, + "theoretical_loss": 3.3863709418827472, + "tokens_seen": 2601320448 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017126579739217655, + "loss": 1.2248, + "theoretical_loss": 3.3863642475464117, + "tokens_seen": 2601385984 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017124974924774325, + "loss": 1.1397, + "theoretical_loss": 3.3863575534259427, + "tokens_seen": 2601451520 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017123370110330995, + "loss": 1.2759, + "theoretical_loss": 3.3863508595213276, + "tokens_seen": 2601517056 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017121765295887665, + "loss": 1.2822, + "theoretical_loss": 3.386344165832555, + "tokens_seen": 2601582592 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017120160481444335, + "loss": 1.2372, + "theoretical_loss": 3.3863374723596116, + "tokens_seen": 2601648128 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017118555667001005, + "loss": 1.1435, + "theoretical_loss": 3.3863307791024853, + "tokens_seen": 2601713664 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 869720, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.024566650390625, + "objective/train/theoretical_loss": 3.3863240860611636, + "objective/train/tokens_used": 2622239200, + "theoretical_loss": 3.3863240860611636, + "tokens_seen": 2601779200 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017116950852557675, + "loss": 1.1325, + "theoretical_loss": 3.3863240860611636, + "tokens_seen": 2601779200 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017115346038114342, + "loss": 1.1851, + "theoretical_loss": 3.3863173932356343, + "tokens_seen": 2601844736 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017113741223671013, + "loss": 1.2064, + "theoretical_loss": 3.3863107006258852, + "tokens_seen": 2601910272 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017112136409227683, + "loss": 1.2823, + "theoretical_loss": 3.386304008231903, + "tokens_seen": 2601975808 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017110531594784353, + "loss": 1.2141, + "theoretical_loss": 3.3862973160536765, + "tokens_seen": 2602041344 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017108926780341023, + "loss": 1.1668, + "theoretical_loss": 3.3862906240911927, + "tokens_seen": 2602106880 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017107321965897693, + "loss": 1.17, + "theoretical_loss": 3.386283932344439, + "tokens_seen": 2602172416 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017105717151454363, + "loss": 1.1668, + "theoretical_loss": 3.3862772408134036, + "tokens_seen": 2602237952 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017104112337011033, + "loss": 1.2318, + "theoretical_loss": 3.386270549498074, + "tokens_seen": 2602303488 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017102507522567703, + "loss": 1.2274, + "theoretical_loss": 3.386263858398437, + "tokens_seen": 2602369024 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017100902708124373, + "loss": 1.1591, + "theoretical_loss": 3.386257167514481, + "tokens_seen": 2602434560 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017099297893681043, + "loss": 1.1516, + "theoretical_loss": 3.386250476846193, + "tokens_seen": 2602500096 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017097693079237713, + "loss": 1.1849, + "theoretical_loss": 3.3862437863935617, + "tokens_seen": 2602565632 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017096088264794383, + "loss": 1.2019, + "theoretical_loss": 3.3862370961565738, + "tokens_seen": 2602631168 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017094483450351053, + "loss": 1.3373, + "theoretical_loss": 3.3862304061352173, + "tokens_seen": 2602696704 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017092878635907723, + "loss": 1.2018, + "theoretical_loss": 3.38622371632948, + "tokens_seen": 2602762240 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017091273821464393, + "loss": 1.238, + "theoretical_loss": 3.3862170267393488, + "tokens_seen": 2602827776 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017089669007021063, + "loss": 1.1757, + "theoretical_loss": 3.386210337364812, + "tokens_seen": 2602893312 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017088064192577734, + "loss": 1.2018, + "theoretical_loss": 3.3862036482058566, + "tokens_seen": 2602958848 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017086459378134406, + "loss": 1.256, + "theoretical_loss": 3.3861969592624708, + "tokens_seen": 2603024384 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017084854563691076, + "loss": 1.2102, + "theoretical_loss": 3.386190270534642, + "tokens_seen": 2603089920 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017083249749247744, + "loss": 1.3176, + "theoretical_loss": 3.3861835820223583, + "tokens_seen": 2603155456 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017081644934804414, + "loss": 1.2257, + "theoretical_loss": 3.3861768937256063, + "tokens_seen": 2603220992 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017080040120361084, + "loss": 1.2117, + "theoretical_loss": 3.3861702056443743, + "tokens_seen": 2603286528 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017078435305917754, + "loss": 1.2162, + "theoretical_loss": 3.38616351777865, + "tokens_seen": 2603352064 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017076830491474424, + "loss": 1.2384, + "theoretical_loss": 3.386156830128421, + "tokens_seen": 2603417600 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017075225677031094, + "loss": 1.1341, + "theoretical_loss": 3.3861501426936744, + "tokens_seen": 2603483136 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017073620862587764, + "loss": 1.1867, + "theoretical_loss": 3.3861434554743983, + "tokens_seen": 2603548672 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017072016048144434, + "loss": 1.2525, + "theoretical_loss": 3.3861367684705805, + "tokens_seen": 2603614208 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017070411233701104, + "loss": 1.2272, + "theoretical_loss": 3.386130081682208, + "tokens_seen": 2603679744 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017068806419257774, + "loss": 1.1098, + "theoretical_loss": 3.3861233951092693, + "tokens_seen": 2603745280 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017067201604814444, + "loss": 1.241, + "theoretical_loss": 3.386116708751752, + "tokens_seen": 2603810816 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017065596790371114, + "loss": 1.2437, + "theoretical_loss": 3.3861100226096426, + "tokens_seen": 2603876352 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017063991975927784, + "loss": 1.2221, + "theoretical_loss": 3.3861033366829294, + "tokens_seen": 2603941888 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017062387161484455, + "loss": 1.3139, + "theoretical_loss": 3.3860966509716004, + "tokens_seen": 2604007424 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017060782347041125, + "loss": 1.2106, + "theoretical_loss": 3.3860899654756427, + "tokens_seen": 2604072960 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017059177532597795, + "loss": 1.2153, + "theoretical_loss": 3.3860832801950447, + "tokens_seen": 2604138496 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017057572718154465, + "loss": 1.1526, + "theoretical_loss": 3.386076595129793, + "tokens_seen": 2604204032 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017055967903711135, + "loss": 1.3029, + "theoretical_loss": 3.386069910279876, + "tokens_seen": 2604269568 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017054363089267805, + "loss": 1.1406, + "theoretical_loss": 3.386063225645281, + "tokens_seen": 2604335104 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017052758274824475, + "loss": 1.2035, + "theoretical_loss": 3.3860565412259955, + "tokens_seen": 2604400640 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017051153460381145, + "loss": 1.1899, + "theoretical_loss": 3.386049857022008, + "tokens_seen": 2604466176 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017049548645937815, + "loss": 1.2937, + "theoretical_loss": 3.386043173033305, + "tokens_seen": 2604531712 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017047943831494485, + "loss": 1.2311, + "theoretical_loss": 3.386036489259875, + "tokens_seen": 2604597248 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017046339017051155, + "loss": 1.2336, + "theoretical_loss": 3.3860298057017055, + "tokens_seen": 2604662784 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017044734202607823, + "loss": 1.2547, + "theoretical_loss": 3.3860231223587838, + "tokens_seen": 2604728320 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017043129388164493, + "loss": 1.1884, + "theoretical_loss": 3.386016439231098, + "tokens_seen": 2604793856 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017041524573721163, + "loss": 1.233, + "theoretical_loss": 3.386009756318635, + "tokens_seen": 2604859392 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017039919759277833, + "loss": 1.2576, + "theoretical_loss": 3.3860030736213833, + "tokens_seen": 2604924928 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017038314944834503, + "loss": 1.2657, + "theoretical_loss": 3.3859963911393303, + "tokens_seen": 2604990464 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 870967, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2595572471618652, + "objective/train/theoretical_loss": 3.3859897088724633, + "objective/train/tokens_used": 2625516000, + "theoretical_loss": 3.3859897088724633, + "tokens_seen": 2605056000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017036710130391173, + "loss": 1.1543, + "theoretical_loss": 3.3859897088724633, + "tokens_seen": 2605056000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017035105315947843, + "loss": 1.2483, + "theoretical_loss": 3.3859830268207705, + "tokens_seen": 2605121536 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017033500501504513, + "loss": 1.1958, + "theoretical_loss": 3.3859763449842393, + "tokens_seen": 2605187072 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017031895687061183, + "loss": 1.1906, + "theoretical_loss": 3.3859696633628573, + "tokens_seen": 2605252608 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017030290872617853, + "loss": 1.3546, + "theoretical_loss": 3.3859629819566117, + "tokens_seen": 2605318144 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017028686058174523, + "loss": 1.2204, + "theoretical_loss": 3.3859563007654914, + "tokens_seen": 2605383680 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017027081243731193, + "loss": 1.2398, + "theoretical_loss": 3.385949619789483, + "tokens_seen": 2605449216 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017025476429287866, + "loss": 1.255, + "theoretical_loss": 3.3859429390285745, + "tokens_seen": 2605514752 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017023871614844536, + "loss": 1.2037, + "theoretical_loss": 3.3859362584827535, + "tokens_seen": 2605580288 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017022266800401206, + "loss": 1.2931, + "theoretical_loss": 3.385929578152008, + "tokens_seen": 2605645824 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017020661985957876, + "loss": 1.2856, + "theoretical_loss": 3.385922898036325, + "tokens_seen": 2605711360 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017019057171514546, + "loss": 1.2999, + "theoretical_loss": 3.385916218135693, + "tokens_seen": 2605776896 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017017452357071216, + "loss": 1.2362, + "theoretical_loss": 3.385909538450099, + "tokens_seen": 2605842432 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017015847542627886, + "loss": 1.2724, + "theoretical_loss": 3.385902858979531, + "tokens_seen": 2605907968 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017014242728184556, + "loss": 1.3852, + "theoretical_loss": 3.3858961797239764, + "tokens_seen": 2605973504 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017012637913741224, + "loss": 1.1625, + "theoretical_loss": 3.3858895006834233, + "tokens_seen": 2606039040 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017011033099297894, + "loss": 1.2253, + "theoretical_loss": 3.3858828218578587, + "tokens_seen": 2606104576 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017009428284854564, + "loss": 1.3155, + "theoretical_loss": 3.385876143247271, + "tokens_seen": 2606170112 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017007823470411234, + "loss": 1.3072, + "theoretical_loss": 3.385869464851648, + "tokens_seen": 2606235648 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017006218655967904, + "loss": 1.1748, + "theoretical_loss": 3.3858627866709763, + "tokens_seen": 2606301184 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017004613841524574, + "loss": 1.2022, + "theoretical_loss": 3.3858561087052443, + "tokens_seen": 2606366720 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017003009027081244, + "loss": 1.3129, + "theoretical_loss": 3.3858494309544396, + "tokens_seen": 2606432256 + }, + { + "epoch": 0.79, + "learning_rate": 0.00017001404212637914, + "loss": 1.2566, + "theoretical_loss": 3.38584275341855, + "tokens_seen": 2606497792 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016999799398194584, + "loss": 1.1721, + "theoretical_loss": 3.385836076097563, + "tokens_seen": 2606563328 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016998194583751254, + "loss": 1.2103, + "theoretical_loss": 3.3858293989914663, + "tokens_seen": 2606628864 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016996589769307924, + "loss": 1.2351, + "theoretical_loss": 3.3858227221002477, + "tokens_seen": 2606694400 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016994984954864594, + "loss": 1.2447, + "theoretical_loss": 3.385816045423895, + "tokens_seen": 2606759936 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016993380140421265, + "loss": 1.2751, + "theoretical_loss": 3.3858093689623954, + "tokens_seen": 2606825472 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016991775325977935, + "loss": 1.1853, + "theoretical_loss": 3.3858026927157368, + "tokens_seen": 2606891008 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016990170511534605, + "loss": 1.1921, + "theoretical_loss": 3.385796016683907, + "tokens_seen": 2606956544 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016988565697091275, + "loss": 1.2133, + "theoretical_loss": 3.3857893408668938, + "tokens_seen": 2607022080 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016986960882647945, + "loss": 1.1407, + "theoretical_loss": 3.385782665264685, + "tokens_seen": 2607087616 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016985356068204615, + "loss": 1.3111, + "theoretical_loss": 3.3857759898772675, + "tokens_seen": 2607153152 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016983751253761285, + "loss": 1.1399, + "theoretical_loss": 3.3857693147046297, + "tokens_seen": 2607218688 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016982146439317955, + "loss": 1.292, + "theoretical_loss": 3.3857626397467593, + "tokens_seen": 2607284224 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016980541624874625, + "loss": 1.2486, + "theoretical_loss": 3.3857559650036433, + "tokens_seen": 2607349760 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016978936810431295, + "loss": 1.2484, + "theoretical_loss": 3.38574929047527, + "tokens_seen": 2607415296 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016977331995987965, + "loss": 1.2124, + "theoretical_loss": 3.385742616161627, + "tokens_seen": 2607480832 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016975727181544635, + "loss": 1.3056, + "theoretical_loss": 3.3857359420627025, + "tokens_seen": 2607546368 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016974122367101303, + "loss": 1.18, + "theoretical_loss": 3.3857292681784834, + "tokens_seen": 2607611904 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016972517552657973, + "loss": 1.1419, + "theoretical_loss": 3.3857225945089575, + "tokens_seen": 2607677440 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016970912738214643, + "loss": 1.2472, + "theoretical_loss": 3.385715921054113, + "tokens_seen": 2607742976 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016969307923771313, + "loss": 1.1902, + "theoretical_loss": 3.385709247813937, + "tokens_seen": 2607808512 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016967703109327983, + "loss": 1.2354, + "theoretical_loss": 3.385702574788417, + "tokens_seen": 2607874048 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016966098294884656, + "loss": 1.0969, + "theoretical_loss": 3.3856959019775417, + "tokens_seen": 2607939584 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016964493480441326, + "loss": 1.2418, + "theoretical_loss": 3.385689229381298, + "tokens_seen": 2608005120 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016962888665997996, + "loss": 1.2278, + "theoretical_loss": 3.3856825569996745, + "tokens_seen": 2608070656 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016961283851554666, + "loss": 1.1736, + "theoretical_loss": 3.385675884832658, + "tokens_seen": 2608136192 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016959679037111336, + "loss": 1.2711, + "theoretical_loss": 3.385669212880236, + "tokens_seen": 2608201728 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016958074222668006, + "loss": 1.228, + "theoretical_loss": 3.385662541142397, + "tokens_seen": 2608267264 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 872150, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2419285774230957, + "objective/train/theoretical_loss": 3.3856558696191286, + "objective/train/tokens_used": 2628792800, + "theoretical_loss": 3.3856558696191286, + "tokens_seen": 2608332800 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016956469408224676, + "loss": 1.1865, + "theoretical_loss": 3.3856558696191286, + "tokens_seen": 2608332800 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016954864593781346, + "loss": 1.2419, + "theoretical_loss": 3.3856491983104178, + "tokens_seen": 2608398336 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016953259779338016, + "loss": 1.3424, + "theoretical_loss": 3.385642527216253, + "tokens_seen": 2608463872 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016951654964894686, + "loss": 1.1517, + "theoretical_loss": 3.385635856336622, + "tokens_seen": 2608529408 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016950050150451356, + "loss": 1.3027, + "theoretical_loss": 3.385629185671512, + "tokens_seen": 2608594944 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016948445336008026, + "loss": 1.2372, + "theoretical_loss": 3.385622515220911, + "tokens_seen": 2608660480 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016946840521564696, + "loss": 1.2367, + "theoretical_loss": 3.3856158449848066, + "tokens_seen": 2608726016 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016945235707121366, + "loss": 1.2451, + "theoretical_loss": 3.3856091749631867, + "tokens_seen": 2608791552 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016943630892678034, + "loss": 1.1902, + "theoretical_loss": 3.3856025051560388, + "tokens_seen": 2608857088 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016942026078234704, + "loss": 1.3121, + "theoretical_loss": 3.385595835563351, + "tokens_seen": 2608922624 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016940421263791374, + "loss": 1.2943, + "theoretical_loss": 3.38558916618511, + "tokens_seen": 2608988160 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016938816449348044, + "loss": 1.3143, + "theoretical_loss": 3.385582497021305, + "tokens_seen": 2609053696 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016937211634904714, + "loss": 1.1525, + "theoretical_loss": 3.3855758280719224, + "tokens_seen": 2609119232 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016935606820461384, + "loss": 1.1841, + "theoretical_loss": 3.385569159336951, + "tokens_seen": 2609184768 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016934002006018054, + "loss": 1.1959, + "theoretical_loss": 3.385562490816378, + "tokens_seen": 2609250304 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016932397191574724, + "loss": 1.2736, + "theoretical_loss": 3.3855558225101907, + "tokens_seen": 2609315840 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016930792377131394, + "loss": 1.2516, + "theoretical_loss": 3.3855491544183773, + "tokens_seen": 2609381376 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016929187562688064, + "loss": 1.1596, + "theoretical_loss": 3.3855424865409254, + "tokens_seen": 2609446912 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016927582748244734, + "loss": 1.2844, + "theoretical_loss": 3.3855358188778233, + "tokens_seen": 2609512448 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016925977933801405, + "loss": 1.1782, + "theoretical_loss": 3.3855291514290577, + "tokens_seen": 2609577984 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016924373119358075, + "loss": 1.2169, + "theoretical_loss": 3.385522484194617, + "tokens_seen": 2609643520 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016922768304914745, + "loss": 1.176, + "theoretical_loss": 3.385515817174489, + "tokens_seen": 2609709056 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016921163490471415, + "loss": 1.232, + "theoretical_loss": 3.385509150368661, + "tokens_seen": 2609774592 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016919558676028085, + "loss": 1.2884, + "theoretical_loss": 3.385502483777121, + "tokens_seen": 2609840128 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016917953861584755, + "loss": 1.0545, + "theoretical_loss": 3.385495817399857, + "tokens_seen": 2609905664 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016916349047141425, + "loss": 1.277, + "theoretical_loss": 3.3854891512368557, + "tokens_seen": 2609971200 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016914744232698095, + "loss": 1.2624, + "theoretical_loss": 3.385482485288106, + "tokens_seen": 2610036736 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016913139418254765, + "loss": 1.2994, + "theoretical_loss": 3.385475819553595, + "tokens_seen": 2610102272 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016911534603811435, + "loss": 1.2652, + "theoretical_loss": 3.3854691540333106, + "tokens_seen": 2610167808 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016909929789368105, + "loss": 1.2449, + "theoretical_loss": 3.3854624887272404, + "tokens_seen": 2610233344 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016908324974924775, + "loss": 1.2374, + "theoretical_loss": 3.3854558236353727, + "tokens_seen": 2610298880 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016906720160481445, + "loss": 1.2267, + "theoretical_loss": 3.3854491587576945, + "tokens_seen": 2610364416 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016905115346038115, + "loss": 1.2022, + "theoretical_loss": 3.385442494094194, + "tokens_seen": 2610429952 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016903510531594785, + "loss": 1.1957, + "theoretical_loss": 3.385435829644859, + "tokens_seen": 2610495488 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016901905717151455, + "loss": 1.3015, + "theoretical_loss": 3.3854291654096764, + "tokens_seen": 2610561024 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016900300902708126, + "loss": 1.3187, + "theoretical_loss": 3.385422501388635, + "tokens_seen": 2610626560 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016898696088264796, + "loss": 1.2453, + "theoretical_loss": 3.3854158375817223, + "tokens_seen": 2610692096 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016897091273821466, + "loss": 1.0755, + "theoretical_loss": 3.3854091739889256, + "tokens_seen": 2610757632 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016895486459378136, + "loss": 1.2581, + "theoretical_loss": 3.3854025106102332, + "tokens_seen": 2610823168 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016893881644934806, + "loss": 1.2159, + "theoretical_loss": 3.3853958474456323, + "tokens_seen": 2610888704 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016892276830491476, + "loss": 1.2971, + "theoretical_loss": 3.3853891844951107, + "tokens_seen": 2610954240 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016890672016048146, + "loss": 1.2983, + "theoretical_loss": 3.3853825217586566, + "tokens_seen": 2611019776 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016889067201604816, + "loss": 1.2521, + "theoretical_loss": 3.385375859236258, + "tokens_seen": 2611085312 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016887462387161486, + "loss": 1.2069, + "theoretical_loss": 3.3853691969279014, + "tokens_seen": 2611150848 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016885857572718156, + "loss": 1.2595, + "theoretical_loss": 3.385362534833576, + "tokens_seen": 2611216384 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016884252758274826, + "loss": 1.2228, + "theoretical_loss": 3.3853558729532685, + "tokens_seen": 2611281920 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016882647943831496, + "loss": 1.1859, + "theoretical_loss": 3.3853492112869668, + "tokens_seen": 2611347456 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016881043129388166, + "loss": 1.3231, + "theoretical_loss": 3.385342549834659, + "tokens_seen": 2611412992 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016879438314944836, + "loss": 1.189, + "theoretical_loss": 3.385335888596333, + "tokens_seen": 2611478528 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016877833500501506, + "loss": 1.3082, + "theoretical_loss": 3.385329227571976, + "tokens_seen": 2611544064 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 873055, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1384434700012207, + "objective/train/theoretical_loss": 3.3853225667615763, + "objective/train/tokens_used": 2632069600, + "theoretical_loss": 3.3853225667615763, + "tokens_seen": 2611609600 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016876228686058176, + "loss": 1.215, + "theoretical_loss": 3.3853225667615763, + "tokens_seen": 2611609600 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016874623871614847, + "loss": 1.2868, + "theoretical_loss": 3.385315906165121, + "tokens_seen": 2611675136 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016873019057171514, + "loss": 1.2178, + "theoretical_loss": 3.3853092457825986, + "tokens_seen": 2611740672 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016871414242728184, + "loss": 1.288, + "theoretical_loss": 3.385302585613996, + "tokens_seen": 2611806208 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016869809428284854, + "loss": 1.2948, + "theoretical_loss": 3.3852959256593023, + "tokens_seen": 2611871744 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016868204613841524, + "loss": 1.2146, + "theoretical_loss": 3.3852892659185043, + "tokens_seen": 2611937280 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016866599799398194, + "loss": 1.26, + "theoretical_loss": 3.3852826063915895, + "tokens_seen": 2612002816 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016864994984954864, + "loss": 1.2703, + "theoretical_loss": 3.3852759470785463, + "tokens_seen": 2612068352 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016863390170511534, + "loss": 1.3293, + "theoretical_loss": 3.385269287979362, + "tokens_seen": 2612133888 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016861785356068204, + "loss": 1.153, + "theoretical_loss": 3.385262629094025, + "tokens_seen": 2612199424 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016860180541624874, + "loss": 1.0797, + "theoretical_loss": 3.3852559704225227, + "tokens_seen": 2612264960 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016858575727181544, + "loss": 1.1908, + "theoretical_loss": 3.3852493119648424, + "tokens_seen": 2612330496 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016856970912738215, + "loss": 1.2822, + "theoretical_loss": 3.3852426537209723, + "tokens_seen": 2612396032 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016855366098294885, + "loss": 1.2198, + "theoretical_loss": 3.3852359956909006, + "tokens_seen": 2612461568 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016853761283851555, + "loss": 1.2771, + "theoretical_loss": 3.3852293378746143, + "tokens_seen": 2612527104 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016852156469408225, + "loss": 1.2405, + "theoretical_loss": 3.385222680272102, + "tokens_seen": 2612592640 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016850551654964895, + "loss": 1.2426, + "theoretical_loss": 3.385216022883351, + "tokens_seen": 2612658176 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016848946840521565, + "loss": 1.1061, + "theoretical_loss": 3.3852093657083486, + "tokens_seen": 2612723712 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016847342026078235, + "loss": 1.2411, + "theoretical_loss": 3.3852027087470833, + "tokens_seen": 2612789248 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016845737211634908, + "loss": 1.3229, + "theoretical_loss": 3.3851960519995425, + "tokens_seen": 2612854784 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016844132397191578, + "loss": 1.3325, + "theoretical_loss": 3.385189395465714, + "tokens_seen": 2612920320 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016842527582748248, + "loss": 1.2822, + "theoretical_loss": 3.385182739145586, + "tokens_seen": 2612985856 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016840922768304915, + "loss": 1.284, + "theoretical_loss": 3.3851760830391457, + "tokens_seen": 2613051392 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016839317953861585, + "loss": 1.1827, + "theoretical_loss": 3.3851694271463817, + "tokens_seen": 2613116928 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016837713139418255, + "loss": 1.3158, + "theoretical_loss": 3.3851627714672805, + "tokens_seen": 2613182464 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016836108324974925, + "loss": 1.2324, + "theoretical_loss": 3.385156116001831, + "tokens_seen": 2613248000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016834503510531595, + "loss": 1.2266, + "theoretical_loss": 3.3851494607500205, + "tokens_seen": 2613313536 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016832898696088265, + "loss": 1.178, + "theoretical_loss": 3.385142805711837, + "tokens_seen": 2613379072 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016831293881644936, + "loss": 1.1424, + "theoretical_loss": 3.385136150887268, + "tokens_seen": 2613444608 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016829689067201606, + "loss": 1.2393, + "theoretical_loss": 3.385129496276302, + "tokens_seen": 2613510144 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016828084252758276, + "loss": 1.1723, + "theoretical_loss": 3.3851228418789256, + "tokens_seen": 2613575680 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016826479438314946, + "loss": 1.2367, + "theoretical_loss": 3.3851161876951275, + "tokens_seen": 2613641216 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016824874623871616, + "loss": 1.3545, + "theoretical_loss": 3.3851095337248953, + "tokens_seen": 2613706752 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016823269809428286, + "loss": 1.1597, + "theoretical_loss": 3.3851028799682163, + "tokens_seen": 2613772288 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016821664994984956, + "loss": 1.2352, + "theoretical_loss": 3.385096226425079, + "tokens_seen": 2613837824 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016820060180541626, + "loss": 1.1962, + "theoretical_loss": 3.385089573095471, + "tokens_seen": 2613903360 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016818455366098296, + "loss": 1.1827, + "theoretical_loss": 3.38508291997938, + "tokens_seen": 2613968896 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016816850551654966, + "loss": 1.1724, + "theoretical_loss": 3.3850762670767933, + "tokens_seen": 2614034432 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016815245737211636, + "loss": 1.3049, + "theoretical_loss": 3.3850696143876995, + "tokens_seen": 2614099968 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016813640922768306, + "loss": 1.2785, + "theoretical_loss": 3.3850629619120864, + "tokens_seen": 2614165504 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016812036108324976, + "loss": 1.2684, + "theoretical_loss": 3.385056309649941, + "tokens_seen": 2614231040 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016810431293881646, + "loss": 1.2728, + "theoretical_loss": 3.385049657601252, + "tokens_seen": 2614296576 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016808826479438316, + "loss": 1.1985, + "theoretical_loss": 3.3850430057660064, + "tokens_seen": 2614362112 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016807221664994986, + "loss": 1.2302, + "theoretical_loss": 3.3850363541441926, + "tokens_seen": 2614427648 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016805616850551657, + "loss": 1.2629, + "theoretical_loss": 3.3850297027357983, + "tokens_seen": 2614493184 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016804012036108327, + "loss": 1.2341, + "theoretical_loss": 3.385023051540811, + "tokens_seen": 2614558720 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016802407221664994, + "loss": 1.3219, + "theoretical_loss": 3.3850164005592185, + "tokens_seen": 2614624256 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016800802407221664, + "loss": 1.2591, + "theoretical_loss": 3.385009749791009, + "tokens_seen": 2614689792 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016799197592778334, + "loss": 1.2642, + "theoretical_loss": 3.38500309923617, + "tokens_seen": 2614755328 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016797592778335004, + "loss": 1.2432, + "theoretical_loss": 3.3849964488946895, + "tokens_seen": 2614820864 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 874224, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2787208557128906, + "objective/train/theoretical_loss": 3.384989798766555, + "objective/train/tokens_used": 2635346400, + "theoretical_loss": 3.384989798766555, + "tokens_seen": 2614886400 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016795987963891674, + "loss": 1.2356, + "theoretical_loss": 3.384989798766555, + "tokens_seen": 2614886400 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016794383149448344, + "loss": 1.2377, + "theoretical_loss": 3.384983148851755, + "tokens_seen": 2614951936 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016792778335005014, + "loss": 1.2531, + "theoretical_loss": 3.3849764991502767, + "tokens_seen": 2615017472 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016791173520561684, + "loss": 1.2361, + "theoretical_loss": 3.3849698496621077, + "tokens_seen": 2615083008 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016789568706118354, + "loss": 1.1571, + "theoretical_loss": 3.3849632003872365, + "tokens_seen": 2615148544 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016787963891675025, + "loss": 1.1759, + "theoretical_loss": 3.3849565513256503, + "tokens_seen": 2615214080 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016786359077231697, + "loss": 1.2038, + "theoretical_loss": 3.384949902477337, + "tokens_seen": 2615279616 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016784754262788367, + "loss": 1.2945, + "theoretical_loss": 3.3849432538422852, + "tokens_seen": 2615345152 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016783149448345037, + "loss": 1.1668, + "theoretical_loss": 3.384936605420482, + "tokens_seen": 2615410688 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016781544633901708, + "loss": 1.1838, + "theoretical_loss": 3.3849299572119147, + "tokens_seen": 2615476224 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016779939819458378, + "loss": 1.3118, + "theoretical_loss": 3.3849233092165725, + "tokens_seen": 2615541760 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016778335005015048, + "loss": 1.2835, + "theoretical_loss": 3.384916661434442, + "tokens_seen": 2615607296 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016776730190571718, + "loss": 1.3089, + "theoretical_loss": 3.384910013865512, + "tokens_seen": 2615672832 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016775125376128388, + "loss": 1.2713, + "theoretical_loss": 3.384903366509769, + "tokens_seen": 2615738368 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016773520561685058, + "loss": 1.2629, + "theoretical_loss": 3.384896719367202, + "tokens_seen": 2615803904 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016771915747241728, + "loss": 1.3956, + "theoretical_loss": 3.384890072437799, + "tokens_seen": 2615869440 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016770310932798395, + "loss": 1.2264, + "theoretical_loss": 3.3848834257215463, + "tokens_seen": 2615934976 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016768706118355065, + "loss": 1.2027, + "theoretical_loss": 3.3848767792184336, + "tokens_seen": 2616000512 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016767101303911735, + "loss": 1.2908, + "theoretical_loss": 3.384870132928447, + "tokens_seen": 2616066048 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016765496489468405, + "loss": 1.2497, + "theoretical_loss": 3.384863486851576, + "tokens_seen": 2616131584 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016763891675025075, + "loss": 1.2624, + "theoretical_loss": 3.384856840987807, + "tokens_seen": 2616197120 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016762286860581746, + "loss": 1.0917, + "theoretical_loss": 3.384850195337129, + "tokens_seen": 2616262656 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016760682046138416, + "loss": 1.2987, + "theoretical_loss": 3.3848435498995286, + "tokens_seen": 2616328192 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016759077231695086, + "loss": 1.1984, + "theoretical_loss": 3.3848369046749944, + "tokens_seen": 2616393728 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016757472417251756, + "loss": 1.2002, + "theoretical_loss": 3.3848302596635143, + "tokens_seen": 2616459264 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016755867602808426, + "loss": 1.2797, + "theoretical_loss": 3.3848236148650757, + "tokens_seen": 2616524800 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016754262788365096, + "loss": 1.2631, + "theoretical_loss": 3.3848169702796667, + "tokens_seen": 2616590336 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016752657973921766, + "loss": 1.2049, + "theoretical_loss": 3.3848103259072753, + "tokens_seen": 2616655872 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016751053159478436, + "loss": 1.2776, + "theoretical_loss": 3.384803681747889, + "tokens_seen": 2616721408 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016749448345035106, + "loss": 1.3258, + "theoretical_loss": 3.384797037801496, + "tokens_seen": 2616786944 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016747843530591776, + "loss": 1.276, + "theoretical_loss": 3.3847903940680837, + "tokens_seen": 2616852480 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016746238716148446, + "loss": 1.1697, + "theoretical_loss": 3.38478375054764, + "tokens_seen": 2616918016 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016744633901705116, + "loss": 1.2393, + "theoretical_loss": 3.3847771072401533, + "tokens_seen": 2616983552 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016743029087261786, + "loss": 1.3329, + "theoretical_loss": 3.384770464145611, + "tokens_seen": 2617049088 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016741424272818456, + "loss": 1.2222, + "theoretical_loss": 3.3847638212640008, + "tokens_seen": 2617114624 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016739819458375126, + "loss": 1.2212, + "theoretical_loss": 3.3847571785953106, + "tokens_seen": 2617180160 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016738214643931797, + "loss": 1.2713, + "theoretical_loss": 3.3847505361395287, + "tokens_seen": 2617245696 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016736609829488467, + "loss": 1.2496, + "theoretical_loss": 3.3847438938966423, + "tokens_seen": 2617311232 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016735005015045137, + "loss": 1.1258, + "theoretical_loss": 3.38473725186664, + "tokens_seen": 2617376768 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016733400200601807, + "loss": 1.3258, + "theoretical_loss": 3.384730610049509, + "tokens_seen": 2617442304 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016731795386158474, + "loss": 1.2922, + "theoretical_loss": 3.384723968445237, + "tokens_seen": 2617507840 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016730190571715144, + "loss": 1.1316, + "theoretical_loss": 3.3847173270538127, + "tokens_seen": 2617573376 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016728585757271814, + "loss": 1.3189, + "theoretical_loss": 3.384710685875223, + "tokens_seen": 2617638912 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016726980942828484, + "loss": 1.3079, + "theoretical_loss": 3.3847040449094563, + "tokens_seen": 2617704448 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016725376128385157, + "loss": 1.2491, + "theoretical_loss": 3.3846974041565003, + "tokens_seen": 2617769984 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016723771313941827, + "loss": 1.2274, + "theoretical_loss": 3.3846907636163435, + "tokens_seen": 2617835520 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016722166499498497, + "loss": 1.26, + "theoretical_loss": 3.3846841232889724, + "tokens_seen": 2617901056 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016720561685055167, + "loss": 1.3491, + "theoretical_loss": 3.384677483174376, + "tokens_seen": 2617966592 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016718956870611837, + "loss": 1.2777, + "theoretical_loss": 3.384670843272542, + "tokens_seen": 2618032128 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016717352056168507, + "loss": 1.2068, + "theoretical_loss": 3.3846642035834575, + "tokens_seen": 2618097664 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 875113, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3086519241333008, + "objective/train/theoretical_loss": 3.384657564107111, + "objective/train/tokens_used": 2638623200, + "theoretical_loss": 3.384657564107111, + "tokens_seen": 2618163200 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016715747241725177, + "loss": 1.2691, + "theoretical_loss": 3.384657564107111, + "tokens_seen": 2618163200 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016714142427281847, + "loss": 1.2629, + "theoretical_loss": 3.3846509248434904, + "tokens_seen": 2618228736 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016712537612838518, + "loss": 1.1998, + "theoretical_loss": 3.3846442857925836, + "tokens_seen": 2618294272 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016710932798395188, + "loss": 1.3553, + "theoretical_loss": 3.384637646954378, + "tokens_seen": 2618359808 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016709327983951858, + "loss": 1.2328, + "theoretical_loss": 3.3846310083288618, + "tokens_seen": 2618425344 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016707723169508528, + "loss": 1.1874, + "theoretical_loss": 3.384624369916023, + "tokens_seen": 2618490880 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016706118355065198, + "loss": 1.2423, + "theoretical_loss": 3.384617731715849, + "tokens_seen": 2618556416 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016704513540621868, + "loss": 1.2853, + "theoretical_loss": 3.384611093728328, + "tokens_seen": 2618621952 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016702908726178538, + "loss": 1.042, + "theoretical_loss": 3.384604455953448, + "tokens_seen": 2618687488 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016701303911735205, + "loss": 1.2206, + "theoretical_loss": 3.384597818391196, + "tokens_seen": 2618753024 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016699699097291875, + "loss": 1.2023, + "theoretical_loss": 3.3845911810415608, + "tokens_seen": 2618818560 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016698094282848545, + "loss": 1.2691, + "theoretical_loss": 3.38458454390453, + "tokens_seen": 2618884096 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016696489468405215, + "loss": 1.3746, + "theoretical_loss": 3.3845779069800916, + "tokens_seen": 2618949632 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016694884653961886, + "loss": 1.2727, + "theoretical_loss": 3.3845712702682333, + "tokens_seen": 2619015168 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016693279839518556, + "loss": 1.1702, + "theoretical_loss": 3.3845646337689432, + "tokens_seen": 2619080704 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016691675025075226, + "loss": 1.2712, + "theoretical_loss": 3.384557997482209, + "tokens_seen": 2619146240 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016690070210631896, + "loss": 1.3124, + "theoretical_loss": 3.3845513614080183, + "tokens_seen": 2619211776 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016688465396188566, + "loss": 1.3187, + "theoretical_loss": 3.384544725546359, + "tokens_seen": 2619277312 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016686860581745236, + "loss": 1.1336, + "theoretical_loss": 3.3845380898972195, + "tokens_seen": 2619342848 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016685255767301906, + "loss": 1.2219, + "theoretical_loss": 3.3845314544605873, + "tokens_seen": 2619408384 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016683650952858576, + "loss": 1.2748, + "theoretical_loss": 3.3845248192364505, + "tokens_seen": 2619473920 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016682046138415246, + "loss": 1.2611, + "theoretical_loss": 3.3845181842247967, + "tokens_seen": 2619539456 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016680441323971916, + "loss": 1.39, + "theoretical_loss": 3.384511549425614, + "tokens_seen": 2619604992 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016678836509528586, + "loss": 1.1171, + "theoretical_loss": 3.38450491483889, + "tokens_seen": 2619670528 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016677231695085256, + "loss": 1.178, + "theoretical_loss": 3.384498280464613, + "tokens_seen": 2619736064 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016675626880641926, + "loss": 1.3206, + "theoretical_loss": 3.3844916463027706, + "tokens_seen": 2619801600 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016674022066198596, + "loss": 1.1987, + "theoretical_loss": 3.384485012353351, + "tokens_seen": 2619867136 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016672417251755266, + "loss": 1.1255, + "theoretical_loss": 3.3844783786163415, + "tokens_seen": 2619932672 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016670812437311936, + "loss": 1.2536, + "theoretical_loss": 3.38447174509173, + "tokens_seen": 2619998208 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016669207622868607, + "loss": 1.2508, + "theoretical_loss": 3.3844651117795053, + "tokens_seen": 2620063744 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016667602808425277, + "loss": 1.2442, + "theoretical_loss": 3.3844584786796545, + "tokens_seen": 2620129280 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016665997993981947, + "loss": 1.2818, + "theoretical_loss": 3.3844518457921655, + "tokens_seen": 2620194816 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016664393179538617, + "loss": 1.4008, + "theoretical_loss": 3.3844452131170266, + "tokens_seen": 2620260352 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016662788365095287, + "loss": 1.2616, + "theoretical_loss": 3.3844385806542254, + "tokens_seen": 2620325888 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016661183550651957, + "loss": 1.2534, + "theoretical_loss": 3.38443194840375, + "tokens_seen": 2620391424 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016659578736208627, + "loss": 1.2473, + "theoretical_loss": 3.3844253163655877, + "tokens_seen": 2620456960 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016657973921765297, + "loss": 1.1846, + "theoretical_loss": 3.3844186845397273, + "tokens_seen": 2620522496 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016656369107321967, + "loss": 1.2849, + "theoretical_loss": 3.384412052926156, + "tokens_seen": 2620588032 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016654764292878637, + "loss": 1.1846, + "theoretical_loss": 3.3844054215248622, + "tokens_seen": 2620653568 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016653159478435307, + "loss": 1.2914, + "theoretical_loss": 3.3843987903358332, + "tokens_seen": 2620719104 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016651554663991977, + "loss": 1.2257, + "theoretical_loss": 3.384392159359057, + "tokens_seen": 2620784640 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016649949849548647, + "loss": 1.2509, + "theoretical_loss": 3.3843855285945224, + "tokens_seen": 2620850176 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016648345035105317, + "loss": 1.1446, + "theoretical_loss": 3.384378898042216, + "tokens_seen": 2620915712 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016646740220661987, + "loss": 1.1978, + "theoretical_loss": 3.3843722677021266, + "tokens_seen": 2620981248 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016645135406218657, + "loss": 1.3028, + "theoretical_loss": 3.3843656375742417, + "tokens_seen": 2621046784 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016643530591775328, + "loss": 1.2063, + "theoretical_loss": 3.3843590076585492, + "tokens_seen": 2621112320 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016641925777331998, + "loss": 1.2767, + "theoretical_loss": 3.3843523779550377, + "tokens_seen": 2621177856 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016640320962888668, + "loss": 1.1978, + "theoretical_loss": 3.3843457484636943, + "tokens_seen": 2621243392 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016638716148445338, + "loss": 1.3784, + "theoretical_loss": 3.384339119184507, + "tokens_seen": 2621308928 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016637111334002008, + "loss": 1.1895, + "theoretical_loss": 3.3843324901174636, + "tokens_seen": 2621374464 + }, + { + "debugging/Compilability": 0.0, + "debugging/distinct-1-grams": 0.646233227762004, + "debugging/entropy-1-grams": 4.872327732208451, + "debugging/length": 579.1111111111111, + "debugging/num_segments": 9, + "debugging/score": 0.0698136403192916, + "debugging/score_std": 0.015129296720156065, + "epoch": 0.79, + "objective/train/docs_used": 876316, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4646857976913452, + "objective/train/theoretical_loss": 3.384325861262553, + "objective/train/tokens_used": 2641900000, + "theoretical_loss": 3.384325861262553, + "tokens_seen": 2621440000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016635506519558678, + "loss": 1.2757, + "theoretical_loss": 3.384325861262553, + "tokens_seen": 2621440000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016633901705115348, + "loss": 1.1777, + "theoretical_loss": 3.3843192326197618, + "tokens_seen": 2621505536 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016632296890672018, + "loss": 1.1128, + "theoretical_loss": 3.3843126041890783, + "tokens_seen": 2621571072 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016630692076228685, + "loss": 1.1557, + "theoretical_loss": 3.384305975970491, + "tokens_seen": 2621636608 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016629087261785355, + "loss": 1.4819, + "theoretical_loss": 3.384299347963987, + "tokens_seen": 2621702144 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016627482447342025, + "loss": 1.2207, + "theoretical_loss": 3.384292720169555, + "tokens_seen": 2621767680 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016625877632898696, + "loss": 1.1934, + "theoretical_loss": 3.3842860925871823, + "tokens_seen": 2621833216 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016624272818455366, + "loss": 1.3128, + "theoretical_loss": 3.3842794652168573, + "tokens_seen": 2621898752 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016622668004012036, + "loss": 1.2801, + "theoretical_loss": 3.3842728380585676, + "tokens_seen": 2621964288 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016621063189568706, + "loss": 1.3624, + "theoretical_loss": 3.3842662111123007, + "tokens_seen": 2622029824 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016619458375125376, + "loss": 1.2669, + "theoretical_loss": 3.3842595843780456, + "tokens_seen": 2622095360 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016617853560682046, + "loss": 1.1789, + "theoretical_loss": 3.3842529578557894, + "tokens_seen": 2622160896 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016616248746238716, + "loss": 1.2743, + "theoretical_loss": 3.38424633154552, + "tokens_seen": 2622226432 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016614643931795386, + "loss": 1.3366, + "theoretical_loss": 3.3842397054472255, + "tokens_seen": 2622291968 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016613039117352056, + "loss": 1.2898, + "theoretical_loss": 3.384233079560894, + "tokens_seen": 2622357504 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016611434302908726, + "loss": 1.211, + "theoretical_loss": 3.3842264538865137, + "tokens_seen": 2622423040 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016609829488465396, + "loss": 1.2775, + "theoretical_loss": 3.3842198284240714, + "tokens_seen": 2622488576 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016608224674022066, + "loss": 1.1832, + "theoretical_loss": 3.384213203173556, + "tokens_seen": 2622554112 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016606619859578736, + "loss": 1.3084, + "theoretical_loss": 3.3842065781349557, + "tokens_seen": 2622619648 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001660501504513541, + "loss": 1.1484, + "theoretical_loss": 3.3841999533082574, + "tokens_seen": 2622685184 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001660341023069208, + "loss": 1.155, + "theoretical_loss": 3.384193328693449, + "tokens_seen": 2622750720 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001660180541624875, + "loss": 1.2484, + "theoretical_loss": 3.38418670429052, + "tokens_seen": 2622816256 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001660020060180542, + "loss": 1.2522, + "theoretical_loss": 3.3841800800994566, + "tokens_seen": 2622881792 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016598595787362087, + "loss": 1.2965, + "theoretical_loss": 3.384173456120248, + "tokens_seen": 2622947328 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016596990972918757, + "loss": 1.1983, + "theoretical_loss": 3.384166832352881, + "tokens_seen": 2623012864 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016595386158475427, + "loss": 1.2284, + "theoretical_loss": 3.384160208797344, + "tokens_seen": 2623078400 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016593781344032097, + "loss": 1.3588, + "theoretical_loss": 3.384153585453625, + "tokens_seen": 2623143936 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016592176529588767, + "loss": 1.2518, + "theoretical_loss": 3.3841469623217124, + "tokens_seen": 2623209472 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016590571715145437, + "loss": 1.3169, + "theoretical_loss": 3.3841403394015934, + "tokens_seen": 2623275008 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016588966900702107, + "loss": 1.1193, + "theoretical_loss": 3.384133716693256, + "tokens_seen": 2623340544 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016587362086258777, + "loss": 1.3204, + "theoretical_loss": 3.3841270941966886, + "tokens_seen": 2623406080 + }, + { + "epoch": 0.79, + "learning_rate": 0.00016585757271815447, + "loss": 1.306, + "theoretical_loss": 3.3841204719118787, + "tokens_seen": 2623471616 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016584152457372117, + "loss": 1.1898, + "theoretical_loss": 3.3841138498388146, + "tokens_seen": 2623537152 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016582547642928787, + "loss": 1.2685, + "theoretical_loss": 3.384107227977484, + "tokens_seen": 2623602688 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016580942828485457, + "loss": 1.2553, + "theoretical_loss": 3.384100606327875, + "tokens_seen": 2623668224 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016579338014042127, + "loss": 1.2423, + "theoretical_loss": 3.3840939848899754, + "tokens_seen": 2623733760 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016577733199598797, + "loss": 1.151, + "theoretical_loss": 3.3840873636637734, + "tokens_seen": 2623799296 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016576128385155468, + "loss": 1.1821, + "theoretical_loss": 3.3840807426492563, + "tokens_seen": 2623864832 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016574523570712138, + "loss": 1.1377, + "theoretical_loss": 3.384074121846413, + "tokens_seen": 2623930368 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016572918756268808, + "loss": 1.2848, + "theoretical_loss": 3.38406750125523, + "tokens_seen": 2623995904 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016571313941825478, + "loss": 1.3457, + "theoretical_loss": 3.384060880875697, + "tokens_seen": 2624061440 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016569709127382148, + "loss": 1.2018, + "theoretical_loss": 3.384054260707801, + "tokens_seen": 2624126976 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016568104312938818, + "loss": 1.2165, + "theoretical_loss": 3.38404764075153, + "tokens_seen": 2624192512 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016566499498495488, + "loss": 1.3442, + "theoretical_loss": 3.384041021006872, + "tokens_seen": 2624258048 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016564894684052158, + "loss": 1.2412, + "theoretical_loss": 3.384034401473815, + "tokens_seen": 2624323584 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016563289869608828, + "loss": 1.2687, + "theoretical_loss": 3.384027782152347, + "tokens_seen": 2624389120 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016561685055165498, + "loss": 1.2427, + "theoretical_loss": 3.3840211630424557, + "tokens_seen": 2624454656 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016560080240722165, + "loss": 1.4015, + "theoretical_loss": 3.3840145441441294, + "tokens_seen": 2624520192 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016558475426278835, + "loss": 1.3053, + "theoretical_loss": 3.3840079254573556, + "tokens_seen": 2624585728 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016556870611835506, + "loss": 1.1565, + "theoretical_loss": 3.384001306982123, + "tokens_seen": 2624651264 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 877275, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.162557601928711, + "objective/train/theoretical_loss": 3.3839946887184187, + "objective/train/tokens_used": 2645176800, + "theoretical_loss": 3.3839946887184187, + "tokens_seen": 2624716800 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016555265797392176, + "loss": 1.2469, + "theoretical_loss": 3.3839946887184187, + "tokens_seen": 2624716800 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016553660982948846, + "loss": 1.224, + "theoretical_loss": 3.383988070666231, + "tokens_seen": 2624782336 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016552056168505516, + "loss": 1.3207, + "theoretical_loss": 3.383981452825548, + "tokens_seen": 2624847872 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016550451354062186, + "loss": 1.3044, + "theoretical_loss": 3.3839748351963577, + "tokens_seen": 2624913408 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016548846539618856, + "loss": 1.2721, + "theoretical_loss": 3.383968217778648, + "tokens_seen": 2624978944 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016547241725175526, + "loss": 1.3217, + "theoretical_loss": 3.3839616005724067, + "tokens_seen": 2625044480 + }, + { + "epoch": 0.8, + "learning_rate": 0.000165456369107322, + "loss": 1.1243, + "theoretical_loss": 3.383954983577622, + "tokens_seen": 2625110016 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001654403209628887, + "loss": 1.1605, + "theoretical_loss": 3.3839483667942813, + "tokens_seen": 2625175552 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001654242728184554, + "loss": 1.1798, + "theoretical_loss": 3.3839417502223736, + "tokens_seen": 2625241088 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001654082246740221, + "loss": 1.1099, + "theoretical_loss": 3.383935133861886, + "tokens_seen": 2625306624 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001653921765295888, + "loss": 1.269, + "theoretical_loss": 3.383928517712807, + "tokens_seen": 2625372160 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001653761283851555, + "loss": 1.2308, + "theoretical_loss": 3.383921901775124, + "tokens_seen": 2625437696 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001653600802407222, + "loss": 1.2238, + "theoretical_loss": 3.3839152860488246, + "tokens_seen": 2625503232 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001653440320962889, + "loss": 1.1166, + "theoretical_loss": 3.3839086705338985, + "tokens_seen": 2625568768 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001653279839518556, + "loss": 1.0857, + "theoretical_loss": 3.3839020552303323, + "tokens_seen": 2625634304 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001653119358074223, + "loss": 1.151, + "theoretical_loss": 3.3838954401381143, + "tokens_seen": 2625699840 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016529588766298897, + "loss": 1.2354, + "theoretical_loss": 3.383888825257232, + "tokens_seen": 2625765376 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016527983951855567, + "loss": 1.2418, + "theoretical_loss": 3.3838822105876742, + "tokens_seen": 2625830912 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016526379137412237, + "loss": 1.2158, + "theoretical_loss": 3.3838755961294282, + "tokens_seen": 2625896448 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016524774322968907, + "loss": 1.1846, + "theoretical_loss": 3.3838689818824825, + "tokens_seen": 2625961984 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016523169508525577, + "loss": 1.3278, + "theoretical_loss": 3.383862367846825, + "tokens_seen": 2626027520 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016521564694082247, + "loss": 1.314, + "theoretical_loss": 3.383855754022443, + "tokens_seen": 2626093056 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016519959879638917, + "loss": 1.2619, + "theoretical_loss": 3.3838491404093256, + "tokens_seen": 2626158592 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016518355065195587, + "loss": 1.2109, + "theoretical_loss": 3.3838425270074604, + "tokens_seen": 2626224128 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016516750250752257, + "loss": 1.2691, + "theoretical_loss": 3.3838359138168346, + "tokens_seen": 2626289664 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016515145436308927, + "loss": 1.2424, + "theoretical_loss": 3.3838293008374367, + "tokens_seen": 2626355200 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016513540621865597, + "loss": 1.2495, + "theoretical_loss": 3.3838226880692552, + "tokens_seen": 2626420736 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016511935807422267, + "loss": 1.1317, + "theoretical_loss": 3.3838160755122773, + "tokens_seen": 2626486272 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016510330992978937, + "loss": 1.2657, + "theoretical_loss": 3.3838094631664912, + "tokens_seen": 2626551808 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016508726178535607, + "loss": 1.1781, + "theoretical_loss": 3.383802851031885, + "tokens_seen": 2626617344 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016507121364092278, + "loss": 1.1867, + "theoretical_loss": 3.383796239108447, + "tokens_seen": 2626682880 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016505516549648948, + "loss": 1.2721, + "theoretical_loss": 3.3837896273961645, + "tokens_seen": 2626748416 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016503911735205618, + "loss": 1.2663, + "theoretical_loss": 3.383783015895026, + "tokens_seen": 2626813952 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016502306920762288, + "loss": 1.3369, + "theoretical_loss": 3.3837764046050194, + "tokens_seen": 2626879488 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016500702106318958, + "loss": 1.1585, + "theoretical_loss": 3.3837697935261324, + "tokens_seen": 2626945024 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016499097291875628, + "loss": 1.2613, + "theoretical_loss": 3.3837631826583534, + "tokens_seen": 2627010560 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016497492477432298, + "loss": 1.2894, + "theoretical_loss": 3.3837565720016705, + "tokens_seen": 2627076096 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016495887662988968, + "loss": 1.2048, + "theoretical_loss": 3.383749961556071, + "tokens_seen": 2627141632 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016494282848545638, + "loss": 1.374, + "theoretical_loss": 3.3837433513215434, + "tokens_seen": 2627207168 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016492678034102308, + "loss": 1.2906, + "theoretical_loss": 3.3837367412980757, + "tokens_seen": 2627272704 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016491073219658975, + "loss": 1.2061, + "theoretical_loss": 3.3837301314856556, + "tokens_seen": 2627338240 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016489468405215646, + "loss": 1.2009, + "theoretical_loss": 3.383723521884271, + "tokens_seen": 2627403776 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016487863590772316, + "loss": 1.2099, + "theoretical_loss": 3.383716912493911, + "tokens_seen": 2627469312 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016486258776328988, + "loss": 1.2554, + "theoretical_loss": 3.383710303314562, + "tokens_seen": 2627534848 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016484653961885658, + "loss": 1.2591, + "theoretical_loss": 3.3837036943462135, + "tokens_seen": 2627600384 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016483049147442328, + "loss": 1.2632, + "theoretical_loss": 3.383697085588852, + "tokens_seen": 2627665920 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016481444332998999, + "loss": 1.1343, + "theoretical_loss": 3.383690477042467, + "tokens_seen": 2627731456 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016479839518555669, + "loss": 1.2316, + "theoretical_loss": 3.3836838687070454, + "tokens_seen": 2627796992 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001647823470411234, + "loss": 1.1782, + "theoretical_loss": 3.383677260582576, + "tokens_seen": 2627862528 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001647662988966901, + "loss": 1.2531, + "theoretical_loss": 3.3836706526690463, + "tokens_seen": 2627928064 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 878547, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2687060832977295, + "objective/train/theoretical_loss": 3.383664044966444, + "objective/train/tokens_used": 2648453600, + "theoretical_loss": 3.383664044966444, + "tokens_seen": 2627993600 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001647502507522568, + "loss": 1.2507, + "theoretical_loss": 3.383664044966444, + "tokens_seen": 2627993600 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001647342026078235, + "loss": 1.2081, + "theoretical_loss": 3.3836574374747577, + "tokens_seen": 2628059136 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001647181544633902, + "loss": 1.1698, + "theoretical_loss": 3.383650830193975, + "tokens_seen": 2628124672 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001647021063189569, + "loss": 1.2341, + "theoretical_loss": 3.383644223124085, + "tokens_seen": 2628190208 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001646860581745236, + "loss": 1.1599, + "theoretical_loss": 3.383637616265074, + "tokens_seen": 2628255744 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001646700100300903, + "loss": 1.265, + "theoretical_loss": 3.383631009616931, + "tokens_seen": 2628321280 + }, + { + "epoch": 0.8, + "learning_rate": 0.000164653961885657, + "loss": 1.3057, + "theoretical_loss": 3.3836244031796445, + "tokens_seen": 2628386816 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001646379137412237, + "loss": 1.3214, + "theoretical_loss": 3.3836177969532013, + "tokens_seen": 2628452352 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001646218655967904, + "loss": 1.2545, + "theoretical_loss": 3.38361119093759, + "tokens_seen": 2628517888 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001646058174523571, + "loss": 1.2502, + "theoretical_loss": 3.3836045851327987, + "tokens_seen": 2628583424 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016458976930792377, + "loss": 1.2575, + "theoretical_loss": 3.3835979795388154, + "tokens_seen": 2628648960 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016457372116349047, + "loss": 1.3566, + "theoretical_loss": 3.383591374155628, + "tokens_seen": 2628714496 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016455767301905717, + "loss": 1.3206, + "theoretical_loss": 3.3835847689832246, + "tokens_seen": 2628780032 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016454162487462387, + "loss": 1.2305, + "theoretical_loss": 3.3835781640215927, + "tokens_seen": 2628845568 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016452557673019057, + "loss": 1.2521, + "theoretical_loss": 3.383571559270721, + "tokens_seen": 2628911104 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016450952858575727, + "loss": 1.2798, + "theoretical_loss": 3.383564954730598, + "tokens_seen": 2628976640 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016449348044132397, + "loss": 1.1763, + "theoretical_loss": 3.3835583504012106, + "tokens_seen": 2629042176 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016447743229689067, + "loss": 1.2138, + "theoretical_loss": 3.383551746282547, + "tokens_seen": 2629107712 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016446138415245737, + "loss": 1.2015, + "theoretical_loss": 3.383545142374596, + "tokens_seen": 2629173248 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016444533600802407, + "loss": 1.1857, + "theoretical_loss": 3.383538538677345, + "tokens_seen": 2629238784 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016442928786359077, + "loss": 1.2884, + "theoretical_loss": 3.3835319351907818, + "tokens_seen": 2629304320 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016441323971915747, + "loss": 1.2393, + "theoretical_loss": 3.3835253319148952, + "tokens_seen": 2629369856 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016439719157472417, + "loss": 1.1943, + "theoretical_loss": 3.3835187288496726, + "tokens_seen": 2629435392 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016438114343029088, + "loss": 1.2023, + "theoretical_loss": 3.383512125995102, + "tokens_seen": 2629500928 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016436509528585758, + "loss": 1.2522, + "theoretical_loss": 3.3835055233511717, + "tokens_seen": 2629566464 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016434904714142428, + "loss": 1.2429, + "theoretical_loss": 3.3834989209178703, + "tokens_seen": 2629632000 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016433299899699098, + "loss": 1.1629, + "theoretical_loss": 3.3834923186951844, + "tokens_seen": 2629697536 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016431695085255768, + "loss": 1.2284, + "theoretical_loss": 3.3834857166831034, + "tokens_seen": 2629763072 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016430090270812438, + "loss": 1.2257, + "theoretical_loss": 3.383479114881615, + "tokens_seen": 2629828608 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016428485456369108, + "loss": 1.23, + "theoretical_loss": 3.383472513290706, + "tokens_seen": 2629894144 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016426880641925778, + "loss": 1.2238, + "theoretical_loss": 3.3834659119103665, + "tokens_seen": 2629959680 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016425275827482448, + "loss": 1.1593, + "theoretical_loss": 3.383459310740583, + "tokens_seen": 2630025216 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016423671013039118, + "loss": 1.2652, + "theoretical_loss": 3.383452709781344, + "tokens_seen": 2630090752 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016422066198595788, + "loss": 1.35, + "theoretical_loss": 3.3834461090326378, + "tokens_seen": 2630156288 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016420461384152458, + "loss": 1.2541, + "theoretical_loss": 3.383439508494452, + "tokens_seen": 2630221824 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016418856569709128, + "loss": 1.1271, + "theoretical_loss": 3.383432908166775, + "tokens_seen": 2630287360 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016417251755265798, + "loss": 1.1563, + "theoretical_loss": 3.3834263080495948, + "tokens_seen": 2630352896 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016415646940822468, + "loss": 1.2519, + "theoretical_loss": 3.383419708142899, + "tokens_seen": 2630418432 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016414042126379138, + "loss": 1.3375, + "theoretical_loss": 3.383413108446676, + "tokens_seen": 2630483968 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016412437311935809, + "loss": 1.2535, + "theoretical_loss": 3.383406508960914, + "tokens_seen": 2630549504 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016410832497492479, + "loss": 1.2638, + "theoretical_loss": 3.383399909685601, + "tokens_seen": 2630615040 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001640922768304915, + "loss": 1.192, + "theoretical_loss": 3.3833933106207246, + "tokens_seen": 2630680576 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001640762286860582, + "loss": 1.3046, + "theoretical_loss": 3.3833867117662733, + "tokens_seen": 2630746112 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001640601805416249, + "loss": 1.2568, + "theoretical_loss": 3.383380113122235, + "tokens_seen": 2630811648 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001640441323971916, + "loss": 1.2829, + "theoretical_loss": 3.383373514688598, + "tokens_seen": 2630877184 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001640280842527583, + "loss": 1.2495, + "theoretical_loss": 3.38336691646535, + "tokens_seen": 2630942720 + }, + { + "epoch": 0.8, + "learning_rate": 0.000164012036108325, + "loss": 1.1632, + "theoretical_loss": 3.3833603184524788, + "tokens_seen": 2631008256 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001639959879638917, + "loss": 1.3089, + "theoretical_loss": 3.383353720649973, + "tokens_seen": 2631073792 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001639799398194584, + "loss": 1.3096, + "theoretical_loss": 3.3833471230578205, + "tokens_seen": 2631139328 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001639638916750251, + "loss": 1.2454, + "theoretical_loss": 3.38334052567601, + "tokens_seen": 2631204864 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 879542, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.327216386795044, + "objective/train/theoretical_loss": 3.383333928504528, + "objective/train/tokens_used": 2651730400, + "theoretical_loss": 3.383333928504528, + "tokens_seen": 2631270400 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001639478435305918, + "loss": 1.4171, + "theoretical_loss": 3.383333928504528, + "tokens_seen": 2631270400 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001639317953861585, + "loss": 1.1969, + "theoretical_loss": 3.3833273315433634, + "tokens_seen": 2631335936 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001639157472417252, + "loss": 1.2897, + "theoretical_loss": 3.3833207347925045, + "tokens_seen": 2631401472 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001638996990972919, + "loss": 1.2808, + "theoretical_loss": 3.383314138251939, + "tokens_seen": 2631467008 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016388365095285857, + "loss": 1.2278, + "theoretical_loss": 3.383307541921656, + "tokens_seen": 2631532544 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016386760280842527, + "loss": 1.2461, + "theoretical_loss": 3.3833009458016416, + "tokens_seen": 2631598080 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016385155466399197, + "loss": 1.2604, + "theoretical_loss": 3.3832943498918855, + "tokens_seen": 2631663616 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016383550651955867, + "loss": 1.1665, + "theoretical_loss": 3.383287754192375, + "tokens_seen": 2631729152 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016381945837512537, + "loss": 1.2691, + "theoretical_loss": 3.383281158703098, + "tokens_seen": 2631794688 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016380341023069207, + "loss": 1.2527, + "theoretical_loss": 3.3832745634240435, + "tokens_seen": 2631860224 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016378736208625877, + "loss": 1.2151, + "theoretical_loss": 3.3832679683551987, + "tokens_seen": 2631925760 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016377131394182547, + "loss": 1.2778, + "theoretical_loss": 3.383261373496552, + "tokens_seen": 2631991296 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016375526579739217, + "loss": 1.2913, + "theoretical_loss": 3.3832547788480913, + "tokens_seen": 2632056832 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016373921765295887, + "loss": 1.2097, + "theoretical_loss": 3.383248184409805, + "tokens_seen": 2632122368 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016372316950852557, + "loss": 1.1613, + "theoretical_loss": 3.383241590181681, + "tokens_seen": 2632187904 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016370712136409227, + "loss": 1.1103, + "theoretical_loss": 3.383234996163707, + "tokens_seen": 2632253440 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016369107321965898, + "loss": 1.2234, + "theoretical_loss": 3.3832284023558716, + "tokens_seen": 2632318976 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016367502507522568, + "loss": 1.2667, + "theoretical_loss": 3.3832218087581625, + "tokens_seen": 2632384512 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001636589769307924, + "loss": 1.2318, + "theoretical_loss": 3.3832152153705684, + "tokens_seen": 2632450048 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001636429287863591, + "loss": 1.2098, + "theoretical_loss": 3.3832086221930764, + "tokens_seen": 2632515584 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001636268806419258, + "loss": 1.3017, + "theoretical_loss": 3.383202029225675, + "tokens_seen": 2632581120 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001636108324974925, + "loss": 1.2621, + "theoretical_loss": 3.3831954364683527, + "tokens_seen": 2632646656 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001635947843530592, + "loss": 1.3021, + "theoretical_loss": 3.383188843921097, + "tokens_seen": 2632712192 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016357873620862588, + "loss": 1.2984, + "theoretical_loss": 3.3831822515838965, + "tokens_seen": 2632777728 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016356268806419258, + "loss": 1.3514, + "theoretical_loss": 3.383175659456739, + "tokens_seen": 2632843264 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016354663991975928, + "loss": 1.2489, + "theoretical_loss": 3.383169067539612, + "tokens_seen": 2632908800 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016353059177532598, + "loss": 1.476, + "theoretical_loss": 3.383162475832505, + "tokens_seen": 2632974336 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016351454363089268, + "loss": 1.1989, + "theoretical_loss": 3.383155884335405, + "tokens_seen": 2633039872 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016349849548645938, + "loss": 1.1911, + "theoretical_loss": 3.3831492930483, + "tokens_seen": 2633105408 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016348244734202608, + "loss": 1.1765, + "theoretical_loss": 3.3831427019711784, + "tokens_seen": 2633170944 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016346639919759278, + "loss": 1.2953, + "theoretical_loss": 3.3831361111040286, + "tokens_seen": 2633236480 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016345035105315949, + "loss": 1.1401, + "theoretical_loss": 3.383129520446838, + "tokens_seen": 2633302016 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016343430290872619, + "loss": 1.3037, + "theoretical_loss": 3.3831229299995953, + "tokens_seen": 2633367552 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016341825476429289, + "loss": 1.1595, + "theoretical_loss": 3.3831163397622883, + "tokens_seen": 2633433088 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001634022066198596, + "loss": 1.179, + "theoretical_loss": 3.383109749734905, + "tokens_seen": 2633498624 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001633861584754263, + "loss": 1.3342, + "theoretical_loss": 3.3831031599174337, + "tokens_seen": 2633564160 + }, + { + "epoch": 0.8, + "learning_rate": 0.000163370110330993, + "loss": 1.2452, + "theoretical_loss": 3.3830965703098625, + "tokens_seen": 2633629696 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001633540621865597, + "loss": 1.2448, + "theoretical_loss": 3.3830899809121795, + "tokens_seen": 2633695232 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001633380140421264, + "loss": 1.2722, + "theoretical_loss": 3.3830833917243726, + "tokens_seen": 2633760768 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001633219658976931, + "loss": 1.2356, + "theoretical_loss": 3.38307680274643, + "tokens_seen": 2633826304 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001633059177532598, + "loss": 1.3422, + "theoretical_loss": 3.38307021397834, + "tokens_seen": 2633891840 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001632898696088265, + "loss": 1.2373, + "theoretical_loss": 3.38306362542009, + "tokens_seen": 2633957376 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001632738214643932, + "loss": 1.1736, + "theoretical_loss": 3.3830570370716684, + "tokens_seen": 2634022912 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001632577733199599, + "loss": 1.2057, + "theoretical_loss": 3.383050448933064, + "tokens_seen": 2634088448 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001632417251755266, + "loss": 1.1326, + "theoretical_loss": 3.383043861004264, + "tokens_seen": 2634153984 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001632256770310933, + "loss": 1.253, + "theoretical_loss": 3.3830372732852574, + "tokens_seen": 2634219520 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016320962888666, + "loss": 1.2414, + "theoretical_loss": 3.3830306857760317, + "tokens_seen": 2634285056 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016319358074222667, + "loss": 1.2201, + "theoretical_loss": 3.3830240984765747, + "tokens_seen": 2634350592 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016317753259779337, + "loss": 1.4769, + "theoretical_loss": 3.383017511386875, + "tokens_seen": 2634416128 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016316148445336007, + "loss": 1.1934, + "theoretical_loss": 3.3830109245069204, + "tokens_seen": 2634481664 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 880840, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4140818119049072, + "objective/train/theoretical_loss": 3.3830043378366996, + "objective/train/tokens_used": 2655007200, + "theoretical_loss": 3.3830043378366996, + "tokens_seen": 2634547200 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016314543630892677, + "loss": 1.2645, + "theoretical_loss": 3.3830043378366996, + "tokens_seen": 2634547200 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016312938816449347, + "loss": 1.2554, + "theoretical_loss": 3.3829977513762, + "tokens_seen": 2634612736 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016311334002006017, + "loss": 1.276, + "theoretical_loss": 3.38299116512541, + "tokens_seen": 2634678272 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016309729187562687, + "loss": 1.2897, + "theoretical_loss": 3.382984579084318, + "tokens_seen": 2634743808 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016308124373119357, + "loss": 1.1849, + "theoretical_loss": 3.3829779932529114, + "tokens_seen": 2634809344 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016306519558676027, + "loss": 1.2132, + "theoretical_loss": 3.382971407631179, + "tokens_seen": 2634874880 + }, + { + "epoch": 0.8, + "learning_rate": 0.000163049147442327, + "loss": 1.2982, + "theoretical_loss": 3.382964822219108, + "tokens_seen": 2634940416 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001630330992978937, + "loss": 1.2924, + "theoretical_loss": 3.382958237016688, + "tokens_seen": 2635005952 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001630170511534604, + "loss": 1.2351, + "theoretical_loss": 3.382951652023906, + "tokens_seen": 2635071488 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001630010030090271, + "loss": 1.2314, + "theoretical_loss": 3.38294506724075, + "tokens_seen": 2635137024 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001629849548645938, + "loss": 1.2226, + "theoretical_loss": 3.3829384826672086, + "tokens_seen": 2635202560 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001629689067201605, + "loss": 1.4151, + "theoretical_loss": 3.38293189830327, + "tokens_seen": 2635268096 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001629528585757272, + "loss": 1.1746, + "theoretical_loss": 3.382925314148922, + "tokens_seen": 2635333632 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001629368104312939, + "loss": 1.2166, + "theoretical_loss": 3.382918730204153, + "tokens_seen": 2635399168 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001629207622868606, + "loss": 1.1974, + "theoretical_loss": 3.3829121464689504, + "tokens_seen": 2635464704 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001629047141424273, + "loss": 1.2154, + "theoretical_loss": 3.3829055629433036, + "tokens_seen": 2635530240 + }, + { + "epoch": 0.8, + "learning_rate": 0.000162888665997994, + "loss": 1.37, + "theoretical_loss": 3.3828989796271993, + "tokens_seen": 2635595776 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016287261785356068, + "loss": 1.1677, + "theoretical_loss": 3.3828923965206266, + "tokens_seen": 2635661312 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016285656970912738, + "loss": 1.2831, + "theoretical_loss": 3.3828858136235733, + "tokens_seen": 2635726848 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016284052156469408, + "loss": 1.2254, + "theoretical_loss": 3.3828792309360276, + "tokens_seen": 2635792384 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016282447342026078, + "loss": 1.1988, + "theoretical_loss": 3.3828726484579774, + "tokens_seen": 2635857920 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016280842527582748, + "loss": 1.3002, + "theoretical_loss": 3.382866066189411, + "tokens_seen": 2635923456 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016279237713139418, + "loss": 1.2311, + "theoretical_loss": 3.3828594841303166, + "tokens_seen": 2635988992 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016277632898696088, + "loss": 1.2643, + "theoretical_loss": 3.3828529022806824, + "tokens_seen": 2636054528 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016276028084252759, + "loss": 1.2499, + "theoretical_loss": 3.382846320640496, + "tokens_seen": 2636120064 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016274423269809429, + "loss": 1.2136, + "theoretical_loss": 3.382839739209746, + "tokens_seen": 2636185600 + }, + { + "epoch": 0.8, + "learning_rate": 0.000162728184553661, + "loss": 1.2333, + "theoretical_loss": 3.3828331579884203, + "tokens_seen": 2636251136 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001627121364092277, + "loss": 1.1442, + "theoretical_loss": 3.3828265769765076, + "tokens_seen": 2636316672 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001626960882647944, + "loss": 1.2347, + "theoretical_loss": 3.3828199961739953, + "tokens_seen": 2636382208 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001626800401203611, + "loss": 1.2268, + "theoretical_loss": 3.3828134155808716, + "tokens_seen": 2636447744 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001626639919759278, + "loss": 1.3039, + "theoretical_loss": 3.382806835197125, + "tokens_seen": 2636513280 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001626479438314945, + "loss": 1.2361, + "theoretical_loss": 3.382800255022744, + "tokens_seen": 2636578816 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001626318956870612, + "loss": 1.2594, + "theoretical_loss": 3.382793675057716, + "tokens_seen": 2636644352 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001626158475426279, + "loss": 1.3217, + "theoretical_loss": 3.382787095302029, + "tokens_seen": 2636709888 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001625997993981946, + "loss": 1.1948, + "theoretical_loss": 3.3827805157556714, + "tokens_seen": 2636775424 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001625837512537613, + "loss": 1.2311, + "theoretical_loss": 3.3827739364186318, + "tokens_seen": 2636840960 + }, + { + "epoch": 0.8, + "learning_rate": 0.000162567703109328, + "loss": 1.264, + "theoretical_loss": 3.3827673572908976, + "tokens_seen": 2636906496 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001625516549648947, + "loss": 1.2626, + "theoretical_loss": 3.3827607783724574, + "tokens_seen": 2636972032 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001625356068204614, + "loss": 1.2667, + "theoretical_loss": 3.3827541996632995, + "tokens_seen": 2637037568 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001625195586760281, + "loss": 1.2373, + "theoretical_loss": 3.3827476211634115, + "tokens_seen": 2637103104 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001625035105315948, + "loss": 1.3033, + "theoretical_loss": 3.382741042872782, + "tokens_seen": 2637168640 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016248746238716147, + "loss": 1.1612, + "theoretical_loss": 3.3827344647913993, + "tokens_seen": 2637234176 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016247141424272817, + "loss": 1.4153, + "theoretical_loss": 3.3827278869192505, + "tokens_seen": 2637299712 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001624553660982949, + "loss": 1.2658, + "theoretical_loss": 3.382721309256325, + "tokens_seen": 2637365248 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001624393179538616, + "loss": 1.1774, + "theoretical_loss": 3.38271473180261, + "tokens_seen": 2637430784 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001624232698094283, + "loss": 1.3479, + "theoretical_loss": 3.3827081545580944, + "tokens_seen": 2637496320 + }, + { + "epoch": 0.8, + "learning_rate": 0.000162407221664995, + "loss": 1.2358, + "theoretical_loss": 3.382701577522766, + "tokens_seen": 2637561856 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001623911735205617, + "loss": 1.388, + "theoretical_loss": 3.3826950006966126, + "tokens_seen": 2637627392 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001623751253761284, + "loss": 1.3124, + "theoretical_loss": 3.382688424079623, + "tokens_seen": 2637692928 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001623590772316951, + "loss": 1.3643, + "theoretical_loss": 3.382681847671785, + "tokens_seen": 2637758464 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 881912, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2314331531524658, + "objective/train/theoretical_loss": 3.382675271473087, + "objective/train/tokens_used": 2658284000, + "theoretical_loss": 3.382675271473087, + "tokens_seen": 2637824000 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001623430290872618, + "loss": 1.2611, + "theoretical_loss": 3.382675271473087, + "tokens_seen": 2637824000 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001623269809428285, + "loss": 1.2343, + "theoretical_loss": 3.382668695483517, + "tokens_seen": 2637889536 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001623109327983952, + "loss": 1.2205, + "theoretical_loss": 3.3826621197030624, + "tokens_seen": 2637955072 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001622948846539619, + "loss": 1.1403, + "theoretical_loss": 3.3826555441317128, + "tokens_seen": 2638020608 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001622788365095286, + "loss": 1.1898, + "theoretical_loss": 3.3826489687694554, + "tokens_seen": 2638086144 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001622627883650953, + "loss": 1.2792, + "theoretical_loss": 3.382642393616279, + "tokens_seen": 2638151680 + }, + { + "epoch": 0.8, + "learning_rate": 0.000162246740220662, + "loss": 1.2384, + "theoretical_loss": 3.3826358186721706, + "tokens_seen": 2638217216 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001622306920762287, + "loss": 1.2648, + "theoretical_loss": 3.3826292439371195, + "tokens_seen": 2638282752 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001622146439317954, + "loss": 1.2618, + "theoretical_loss": 3.3826226694111132, + "tokens_seen": 2638348288 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001621985957873621, + "loss": 1.3526, + "theoretical_loss": 3.38261609509414, + "tokens_seen": 2638413824 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001621825476429288, + "loss": 1.2672, + "theoretical_loss": 3.382609520986189, + "tokens_seen": 2638479360 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016216649949849548, + "loss": 1.1992, + "theoretical_loss": 3.382602947087247, + "tokens_seen": 2638544896 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016215045135406218, + "loss": 1.2421, + "theoretical_loss": 3.382596373397303, + "tokens_seen": 2638610432 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016213440320962888, + "loss": 1.3485, + "theoretical_loss": 3.382589799916344, + "tokens_seen": 2638675968 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016211835506519558, + "loss": 1.1863, + "theoretical_loss": 3.38258322664436, + "tokens_seen": 2638741504 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016210230692076228, + "loss": 1.2969, + "theoretical_loss": 3.3825766535813377, + "tokens_seen": 2638807040 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016208625877632898, + "loss": 1.3751, + "theoretical_loss": 3.382570080727266, + "tokens_seen": 2638872576 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016207021063189569, + "loss": 1.149, + "theoretical_loss": 3.382563508082133, + "tokens_seen": 2638938112 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016205416248746239, + "loss": 1.3637, + "theoretical_loss": 3.3825569356459266, + "tokens_seen": 2639003648 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001620381143430291, + "loss": 1.211, + "theoretical_loss": 3.382550363418635, + "tokens_seen": 2639069184 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001620220661985958, + "loss": 1.2961, + "theoretical_loss": 3.3825437914002467, + "tokens_seen": 2639134720 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001620060180541625, + "loss": 1.3135, + "theoretical_loss": 3.3825372195907493, + "tokens_seen": 2639200256 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001619899699097292, + "loss": 1.2297, + "theoretical_loss": 3.3825306479901314, + "tokens_seen": 2639265792 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001619739217652959, + "loss": 1.2903, + "theoretical_loss": 3.382524076598381, + "tokens_seen": 2639331328 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001619578736208626, + "loss": 1.2297, + "theoretical_loss": 3.3825175054154863, + "tokens_seen": 2639396864 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001619418254764293, + "loss": 1.2882, + "theoretical_loss": 3.382510934441436, + "tokens_seen": 2639462400 + }, + { + "epoch": 0.8, + "learning_rate": 0.000161925777331996, + "loss": 1.2311, + "theoretical_loss": 3.3825043636762175, + "tokens_seen": 2639527936 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001619097291875627, + "loss": 1.2086, + "theoretical_loss": 3.382497793119819, + "tokens_seen": 2639593472 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001618936810431294, + "loss": 1.2067, + "theoretical_loss": 3.382491222772229, + "tokens_seen": 2639659008 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001618776328986961, + "loss": 1.3184, + "theoretical_loss": 3.382484652633436, + "tokens_seen": 2639724544 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001618615847542628, + "loss": 1.215, + "theoretical_loss": 3.382478082703428, + "tokens_seen": 2639790080 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001618455366098295, + "loss": 1.1551, + "theoretical_loss": 3.3824715129821925, + "tokens_seen": 2639855616 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001618294884653962, + "loss": 1.3447, + "theoretical_loss": 3.3824649434697185, + "tokens_seen": 2639921152 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001618134403209629, + "loss": 1.3053, + "theoretical_loss": 3.382458374165994, + "tokens_seen": 2639986688 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001617973921765296, + "loss": 1.1419, + "theoretical_loss": 3.382451805071007, + "tokens_seen": 2640052224 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001617813440320963, + "loss": 1.2038, + "theoretical_loss": 3.3824452361847452, + "tokens_seen": 2640117760 + }, + { + "epoch": 0.8, + "learning_rate": 0.000161765295887663, + "loss": 1.1231, + "theoretical_loss": 3.382438667507198, + "tokens_seen": 2640183296 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001617492477432297, + "loss": 1.1916, + "theoretical_loss": 3.3824320990383523, + "tokens_seen": 2640248832 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001617331995987964, + "loss": 1.2029, + "theoretical_loss": 3.3824255307781974, + "tokens_seen": 2640314368 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001617171514543631, + "loss": 1.1804, + "theoretical_loss": 3.382418962726721, + "tokens_seen": 2640379904 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001617011033099298, + "loss": 1.2152, + "theoretical_loss": 3.382412394883911, + "tokens_seen": 2640445440 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001616850551654965, + "loss": 1.2344, + "theoretical_loss": 3.382405827249756, + "tokens_seen": 2640510976 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001616690070210632, + "loss": 1.2236, + "theoretical_loss": 3.3823992598242443, + "tokens_seen": 2640576512 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001616529588766299, + "loss": 1.2015, + "theoretical_loss": 3.3823926926073637, + "tokens_seen": 2640642048 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001616369107321966, + "loss": 1.2029, + "theoretical_loss": 3.382386125599102, + "tokens_seen": 2640707584 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001616208625877633, + "loss": 1.1842, + "theoretical_loss": 3.3823795587994487, + "tokens_seen": 2640773120 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016160481444333, + "loss": 1.4271, + "theoretical_loss": 3.382372992208391, + "tokens_seen": 2640838656 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001615887662988967, + "loss": 1.2951, + "theoretical_loss": 3.382366425825918, + "tokens_seen": 2640904192 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001615727181544634, + "loss": 1.4406, + "theoretical_loss": 3.3823598596520164, + "tokens_seen": 2640969728 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001615566700100301, + "loss": 1.1799, + "theoretical_loss": 3.3823532936866756, + "tokens_seen": 2641035264 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 882790, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.229627013206482, + "objective/train/theoretical_loss": 3.3823467279298836, + "objective/train/tokens_used": 2661560800, + "theoretical_loss": 3.3823467279298836, + "tokens_seen": 2641100800 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001615406218655968, + "loss": 1.2329, + "theoretical_loss": 3.3823467279298836, + "tokens_seen": 2641100800 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001615245737211635, + "loss": 1.2648, + "theoretical_loss": 3.382340162381628, + "tokens_seen": 2641166336 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001615085255767302, + "loss": 1.229, + "theoretical_loss": 3.3823335970418977, + "tokens_seen": 2641231872 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001614924774322969, + "loss": 1.3268, + "theoretical_loss": 3.3823270319106804, + "tokens_seen": 2641297408 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016147642928786358, + "loss": 1.2337, + "theoretical_loss": 3.3823204669879647, + "tokens_seen": 2641362944 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016146038114343028, + "loss": 1.1202, + "theoretical_loss": 3.382313902273739, + "tokens_seen": 2641428480 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016144433299899698, + "loss": 1.3051, + "theoretical_loss": 3.382307337767991, + "tokens_seen": 2641494016 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016142828485456368, + "loss": 1.3052, + "theoretical_loss": 3.382300773470709, + "tokens_seen": 2641559552 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016141223671013038, + "loss": 1.1932, + "theoretical_loss": 3.3822942093818815, + "tokens_seen": 2641625088 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016139618856569708, + "loss": 1.3194, + "theoretical_loss": 3.382287645501496, + "tokens_seen": 2641690624 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016138014042126379, + "loss": 1.1744, + "theoretical_loss": 3.3822810818295417, + "tokens_seen": 2641756160 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016136409227683049, + "loss": 1.3228, + "theoretical_loss": 3.3822745183660063, + "tokens_seen": 2641821696 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001613480441323972, + "loss": 1.4672, + "theoretical_loss": 3.3822679551108776, + "tokens_seen": 2641887232 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001613319959879639, + "loss": 1.2198, + "theoretical_loss": 3.3822613920641444, + "tokens_seen": 2641952768 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001613159478435306, + "loss": 1.1613, + "theoretical_loss": 3.3822548292257952, + "tokens_seen": 2642018304 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001612998996990973, + "loss": 1.2391, + "theoretical_loss": 3.3822482665958176, + "tokens_seen": 2642083840 + }, + { + "epoch": 0.8, + "learning_rate": 0.000161283851554664, + "loss": 1.2255, + "theoretical_loss": 3.3822417041742, + "tokens_seen": 2642149376 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001612678034102307, + "loss": 1.2354, + "theoretical_loss": 3.3822351419609302, + "tokens_seen": 2642214912 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016125175526579742, + "loss": 1.2043, + "theoretical_loss": 3.3822285799559975, + "tokens_seen": 2642280448 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016123570712136412, + "loss": 1.2457, + "theoretical_loss": 3.3822220181593887, + "tokens_seen": 2642345984 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016121965897693082, + "loss": 1.2553, + "theoretical_loss": 3.3822154565710933, + "tokens_seen": 2642411520 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016120361083249752, + "loss": 1.2815, + "theoretical_loss": 3.3822088951910985, + "tokens_seen": 2642477056 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016118756268806422, + "loss": 1.3324, + "theoretical_loss": 3.3822023340193934, + "tokens_seen": 2642542592 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016117151454363092, + "loss": 1.2736, + "theoretical_loss": 3.3821957730559657, + "tokens_seen": 2642608128 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001611554663991976, + "loss": 1.1897, + "theoretical_loss": 3.382189212300804, + "tokens_seen": 2642673664 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001611394182547643, + "loss": 1.3507, + "theoretical_loss": 3.3821826517538955, + "tokens_seen": 2642739200 + }, + { + "epoch": 0.8, + "learning_rate": 0.000161123370110331, + "loss": 1.1152, + "theoretical_loss": 3.3821760914152303, + "tokens_seen": 2642804736 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001611073219658977, + "loss": 1.2872, + "theoretical_loss": 3.382169531284795, + "tokens_seen": 2642870272 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001610912738214644, + "loss": 1.248, + "theoretical_loss": 3.382162971362578, + "tokens_seen": 2642935808 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001610752256770311, + "loss": 1.1307, + "theoretical_loss": 3.3821564116485683, + "tokens_seen": 2643001344 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001610591775325978, + "loss": 1.2059, + "theoretical_loss": 3.3821498521427538, + "tokens_seen": 2643066880 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001610431293881645, + "loss": 1.3026, + "theoretical_loss": 3.3821432928451225, + "tokens_seen": 2643132416 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001610270812437312, + "loss": 1.1708, + "theoretical_loss": 3.3821367337556625, + "tokens_seen": 2643197952 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001610110330992979, + "loss": 1.204, + "theoretical_loss": 3.3821301748743626, + "tokens_seen": 2643263488 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001609949849548646, + "loss": 1.2531, + "theoretical_loss": 3.382123616201211, + "tokens_seen": 2643329024 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001609789368104313, + "loss": 1.2112, + "theoretical_loss": 3.382117057736195, + "tokens_seen": 2643394560 + }, + { + "epoch": 0.8, + "learning_rate": 0.000160962888665998, + "loss": 1.1932, + "theoretical_loss": 3.382110499479304, + "tokens_seen": 2643460096 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001609468405215647, + "loss": 1.2288, + "theoretical_loss": 3.3821039414305254, + "tokens_seen": 2643525632 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001609307923771314, + "loss": 1.2695, + "theoretical_loss": 3.3820973835898482, + "tokens_seen": 2643591168 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001609147442326981, + "loss": 1.1986, + "theoretical_loss": 3.38209082595726, + "tokens_seen": 2643656704 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001608986960882648, + "loss": 1.2231, + "theoretical_loss": 3.3820842685327492, + "tokens_seen": 2643722240 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001608826479438315, + "loss": 1.1745, + "theoretical_loss": 3.3820777113163043, + "tokens_seen": 2643787776 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001608665997993982, + "loss": 1.1469, + "theoretical_loss": 3.382071154307913, + "tokens_seen": 2643853312 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001608505516549649, + "loss": 1.2005, + "theoretical_loss": 3.3820645975075645, + "tokens_seen": 2643918848 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001608345035105316, + "loss": 1.1153, + "theoretical_loss": 3.3820580409152456, + "tokens_seen": 2643984384 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001608184553660983, + "loss": 1.2525, + "theoretical_loss": 3.382051484530946, + "tokens_seen": 2644049920 + }, + { + "epoch": 0.8, + "learning_rate": 0.000160802407221665, + "loss": 1.2165, + "theoretical_loss": 3.382044928354653, + "tokens_seen": 2644115456 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001607863590772317, + "loss": 1.2043, + "theoretical_loss": 3.382038372386355, + "tokens_seen": 2644180992 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016077031093279838, + "loss": 1.1146, + "theoretical_loss": 3.382031816626041, + "tokens_seen": 2644246528 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016075426278836508, + "loss": 1.1391, + "theoretical_loss": 3.3820252610736983, + "tokens_seen": 2644312064 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 883914, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0941942930221558, + "objective/train/theoretical_loss": 3.3820187057293154, + "objective/train/tokens_used": 2664837600, + "theoretical_loss": 3.3820187057293154, + "tokens_seen": 2644377600 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016073821464393178, + "loss": 1.119, + "theoretical_loss": 3.3820187057293154, + "tokens_seen": 2644377600 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016072216649949848, + "loss": 1.3514, + "theoretical_loss": 3.382012150592881, + "tokens_seen": 2644443136 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016070611835506519, + "loss": 1.2238, + "theoretical_loss": 3.3820055956643826, + "tokens_seen": 2644508672 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016069007021063189, + "loss": 1.2456, + "theoretical_loss": 3.3819990409438088, + "tokens_seen": 2644574208 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016067402206619859, + "loss": 1.2422, + "theoretical_loss": 3.381992486431148, + "tokens_seen": 2644639744 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016065797392176531, + "loss": 1.2061, + "theoretical_loss": 3.3819859321263888, + "tokens_seen": 2644705280 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016064192577733201, + "loss": 1.1601, + "theoretical_loss": 3.381979378029518, + "tokens_seen": 2644770816 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016062587763289872, + "loss": 1.2694, + "theoretical_loss": 3.381972824140526, + "tokens_seen": 2644836352 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016060982948846542, + "loss": 1.2985, + "theoretical_loss": 3.381966270459399, + "tokens_seen": 2644901888 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016059378134403212, + "loss": 1.2316, + "theoretical_loss": 3.381959716986127, + "tokens_seen": 2644967424 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016057773319959882, + "loss": 1.1716, + "theoretical_loss": 3.3819531637206968, + "tokens_seen": 2645032960 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016056168505516552, + "loss": 1.1592, + "theoretical_loss": 3.381946610663098, + "tokens_seen": 2645098496 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016054563691073222, + "loss": 1.1403, + "theoretical_loss": 3.3819400578133174, + "tokens_seen": 2645164032 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016052958876629892, + "loss": 1.3048, + "theoretical_loss": 3.3819335051713444, + "tokens_seen": 2645229568 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016051354062186562, + "loss": 1.2095, + "theoretical_loss": 3.3819269527371665, + "tokens_seen": 2645295104 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016049749247743232, + "loss": 1.1824, + "theoretical_loss": 3.381920400510773, + "tokens_seen": 2645360640 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016048144433299902, + "loss": 1.2064, + "theoretical_loss": 3.381913848492151, + "tokens_seen": 2645426176 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016046539618856572, + "loss": 1.1316, + "theoretical_loss": 3.3819072966812893, + "tokens_seen": 2645491712 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001604493480441324, + "loss": 1.1623, + "theoretical_loss": 3.381900745078176, + "tokens_seen": 2645557248 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001604332998996991, + "loss": 1.1171, + "theoretical_loss": 3.3818941936827995, + "tokens_seen": 2645622784 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001604172517552658, + "loss": 1.148, + "theoretical_loss": 3.3818876424951485, + "tokens_seen": 2645688320 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001604012036108325, + "loss": 1.2724, + "theoretical_loss": 3.381881091515211, + "tokens_seen": 2645753856 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001603851554663992, + "loss": 1.2139, + "theoretical_loss": 3.3818745407429742, + "tokens_seen": 2645819392 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001603691073219659, + "loss": 1.1898, + "theoretical_loss": 3.381867990178428, + "tokens_seen": 2645884928 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001603530591775326, + "loss": 1.1317, + "theoretical_loss": 3.38186143982156, + "tokens_seen": 2645950464 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001603370110330993, + "loss": 1.3181, + "theoretical_loss": 3.3818548896723577, + "tokens_seen": 2646016000 + }, + { + "epoch": 0.8, + "learning_rate": 0.000160320962888666, + "loss": 1.1821, + "theoretical_loss": 3.381848339730811, + "tokens_seen": 2646081536 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001603049147442327, + "loss": 1.1976, + "theoretical_loss": 3.3818417899969067, + "tokens_seen": 2646147072 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001602888665997994, + "loss": 1.159, + "theoretical_loss": 3.3818352404706333, + "tokens_seen": 2646212608 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001602728184553661, + "loss": 1.226, + "theoretical_loss": 3.3818286911519797, + "tokens_seen": 2646278144 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001602567703109328, + "loss": 1.3473, + "theoretical_loss": 3.3818221420409342, + "tokens_seen": 2646343680 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001602407221664995, + "loss": 1.1656, + "theoretical_loss": 3.3818155931374845, + "tokens_seen": 2646409216 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001602246740220662, + "loss": 1.3664, + "theoretical_loss": 3.3818090444416193, + "tokens_seen": 2646474752 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001602086258776329, + "loss": 1.2457, + "theoretical_loss": 3.381802495953327, + "tokens_seen": 2646540288 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001601925777331996, + "loss": 1.2018, + "theoretical_loss": 3.381795947672595, + "tokens_seen": 2646605824 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001601765295887663, + "loss": 1.1907, + "theoretical_loss": 3.3817893995994126, + "tokens_seen": 2646671360 + }, + { + "epoch": 0.8, + "learning_rate": 0.000160160481444333, + "loss": 1.2387, + "theoretical_loss": 3.3817828517337674, + "tokens_seen": 2646736896 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001601444332998997, + "loss": 1.2369, + "theoretical_loss": 3.381776304075648, + "tokens_seen": 2646802432 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001601283851554664, + "loss": 1.2266, + "theoretical_loss": 3.381769756625043, + "tokens_seen": 2646867968 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001601123370110331, + "loss": 1.2105, + "theoretical_loss": 3.38176320938194, + "tokens_seen": 2646933504 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001600962888665998, + "loss": 1.1897, + "theoretical_loss": 3.3817566623463278, + "tokens_seen": 2646999040 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001600802407221665, + "loss": 1.1628, + "theoretical_loss": 3.3817501155181944, + "tokens_seen": 2647064576 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016006419257773318, + "loss": 1.2635, + "theoretical_loss": 3.381743568897528, + "tokens_seen": 2647130112 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001600481444332999, + "loss": 1.3658, + "theoretical_loss": 3.3817370224843173, + "tokens_seen": 2647195648 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001600320962888666, + "loss": 1.2415, + "theoretical_loss": 3.38173047627855, + "tokens_seen": 2647261184 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001600160481444333, + "loss": 1.2845, + "theoretical_loss": 3.3817239302802156, + "tokens_seen": 2647326720 + }, + { + "epoch": 0.8, + "learning_rate": 0.00016, + "loss": 1.2727, + "theoretical_loss": 3.381717384489301, + "tokens_seen": 2647392256 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015998395185556671, + "loss": 1.3802, + "theoretical_loss": 3.3817108389057946, + "tokens_seen": 2647457792 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015996790371113341, + "loss": 1.2506, + "theoretical_loss": 3.381704293529686, + "tokens_seen": 2647523328 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015995185556670011, + "loss": 1.2756, + "theoretical_loss": 3.381697748360962, + "tokens_seen": 2647588864 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 884848, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1985678672790527, + "objective/train/theoretical_loss": 3.381691203399612, + "objective/train/tokens_used": 2668114400, + "theoretical_loss": 3.381691203399612, + "tokens_seen": 2647654400 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015993580742226682, + "loss": 1.1797, + "theoretical_loss": 3.381691203399612, + "tokens_seen": 2647654400 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015991975927783352, + "loss": 1.2751, + "theoretical_loss": 3.3816846586456233, + "tokens_seen": 2647719936 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015990371113340022, + "loss": 1.1986, + "theoretical_loss": 3.3816781140989853, + "tokens_seen": 2647785472 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015988766298896692, + "loss": 1.3165, + "theoretical_loss": 3.3816715697596855, + "tokens_seen": 2647851008 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015987161484453362, + "loss": 1.233, + "theoretical_loss": 3.3816650256277123, + "tokens_seen": 2647916544 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015985556670010032, + "loss": 1.2638, + "theoretical_loss": 3.381658481703054, + "tokens_seen": 2647982080 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015983951855566702, + "loss": 1.2441, + "theoretical_loss": 3.3816519379856995, + "tokens_seen": 2648047616 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015982347041123372, + "loss": 1.2365, + "theoretical_loss": 3.3816453944756364, + "tokens_seen": 2648113152 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015980742226680042, + "loss": 1.3608, + "theoretical_loss": 3.3816388511728532, + "tokens_seen": 2648178688 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015979137412236712, + "loss": 1.2437, + "theoretical_loss": 3.381632308077338, + "tokens_seen": 2648244224 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015977532597793382, + "loss": 1.1211, + "theoretical_loss": 3.3816257651890798, + "tokens_seen": 2648309760 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015975927783350052, + "loss": 1.2903, + "theoretical_loss": 3.381619222508066, + "tokens_seen": 2648375296 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001597432296890672, + "loss": 1.1333, + "theoretical_loss": 3.381612680034286, + "tokens_seen": 2648440832 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001597271815446339, + "loss": 1.258, + "theoretical_loss": 3.3816061377677267, + "tokens_seen": 2648506368 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001597111334002006, + "loss": 1.4137, + "theoretical_loss": 3.3815995957083773, + "tokens_seen": 2648571904 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001596950852557673, + "loss": 1.2163, + "theoretical_loss": 3.381593053856226, + "tokens_seen": 2648637440 + }, + { + "epoch": 0.8, + "learning_rate": 0.000159679037111334, + "loss": 1.1701, + "theoretical_loss": 3.381586512211262, + "tokens_seen": 2648702976 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001596629889669007, + "loss": 1.3169, + "theoretical_loss": 3.3815799707734717, + "tokens_seen": 2648768512 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001596469408224674, + "loss": 1.1762, + "theoretical_loss": 3.381573429542845, + "tokens_seen": 2648834048 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001596308926780341, + "loss": 1.2784, + "theoretical_loss": 3.381566888519369, + "tokens_seen": 2648899584 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001596148445336008, + "loss": 1.2996, + "theoretical_loss": 3.381560347703033, + "tokens_seen": 2648965120 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001595987963891675, + "loss": 1.1512, + "theoretical_loss": 3.381553807093825, + "tokens_seen": 2649030656 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001595827482447342, + "loss": 1.1335, + "theoretical_loss": 3.3815472666917334, + "tokens_seen": 2649096192 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001595667001003009, + "loss": 1.2408, + "theoretical_loss": 3.381540726496746, + "tokens_seen": 2649161728 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001595506519558676, + "loss": 1.2297, + "theoretical_loss": 3.381534186508852, + "tokens_seen": 2649227264 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001595346038114343, + "loss": 1.2126, + "theoretical_loss": 3.381527646728039, + "tokens_seen": 2649292800 + }, + { + "epoch": 0.8, + "learning_rate": 0.000159518555667001, + "loss": 1.2164, + "theoretical_loss": 3.3815211071542954, + "tokens_seen": 2649358336 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001595025075225677, + "loss": 1.293, + "theoretical_loss": 3.38151456778761, + "tokens_seen": 2649423872 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001594864593781344, + "loss": 1.213, + "theoretical_loss": 3.3815080286279704, + "tokens_seen": 2649489408 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001594704112337011, + "loss": 1.2852, + "theoretical_loss": 3.3815014896753657, + "tokens_seen": 2649554944 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015945436308926783, + "loss": 1.1606, + "theoretical_loss": 3.381494950929784, + "tokens_seen": 2649620480 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001594383149448345, + "loss": 1.2241, + "theoretical_loss": 3.381488412391213, + "tokens_seen": 2649686016 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001594222668004012, + "loss": 1.2315, + "theoretical_loss": 3.3814818740596415, + "tokens_seen": 2649751552 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001594062186559679, + "loss": 1.314, + "theoretical_loss": 3.3814753359350584, + "tokens_seen": 2649817088 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001593901705115346, + "loss": 1.1586, + "theoretical_loss": 3.3814687980174507, + "tokens_seen": 2649882624 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001593741223671013, + "loss": 1.2715, + "theoretical_loss": 3.381462260306808, + "tokens_seen": 2649948160 + }, + { + "epoch": 0.8, + "learning_rate": 0.000159358074222668, + "loss": 1.25, + "theoretical_loss": 3.381455722803118, + "tokens_seen": 2650013696 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001593420260782347, + "loss": 1.1796, + "theoretical_loss": 3.3814491855063693, + "tokens_seen": 2650079232 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001593259779338014, + "loss": 1.1439, + "theoretical_loss": 3.3814426484165496, + "tokens_seen": 2650144768 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001593099297893681, + "loss": 1.1577, + "theoretical_loss": 3.381436111533648, + "tokens_seen": 2650210304 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015929388164493481, + "loss": 1.207, + "theoretical_loss": 3.381429574857653, + "tokens_seen": 2650275840 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015927783350050151, + "loss": 1.4096, + "theoretical_loss": 3.3814230383885517, + "tokens_seen": 2650341376 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015926178535606822, + "loss": 1.2271, + "theoretical_loss": 3.381416502126334, + "tokens_seen": 2650406912 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015924573721163492, + "loss": 1.3151, + "theoretical_loss": 3.381409966070987, + "tokens_seen": 2650472448 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015922968906720162, + "loss": 1.2721, + "theoretical_loss": 3.381403430222499, + "tokens_seen": 2650537984 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015921364092276832, + "loss": 1.2351, + "theoretical_loss": 3.3813968945808597, + "tokens_seen": 2650603520 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015919759277833502, + "loss": 1.1892, + "theoretical_loss": 3.3813903591460566, + "tokens_seen": 2650669056 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015918154463390172, + "loss": 1.2642, + "theoretical_loss": 3.3813838239180773, + "tokens_seen": 2650734592 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015916549648946842, + "loss": 1.2219, + "theoretical_loss": 3.3813772888969114, + "tokens_seen": 2650800128 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015914944834503512, + "loss": 1.2253, + "theoretical_loss": 3.381370754082546, + "tokens_seen": 2650865664 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 886131, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4121907949447632, + "objective/train/theoretical_loss": 3.381364219474971, + "objective/train/tokens_used": 2671391200, + "theoretical_loss": 3.381364219474971, + "tokens_seen": 2650931200 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015913340020060182, + "loss": 1.3389, + "theoretical_loss": 3.381364219474971, + "tokens_seen": 2650931200 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015911735205616852, + "loss": 1.1667, + "theoretical_loss": 3.3813576850741733, + "tokens_seen": 2650996736 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015910130391173522, + "loss": 1.2626, + "theoretical_loss": 3.381351150880142, + "tokens_seen": 2651062272 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015908525576730192, + "loss": 1.2653, + "theoretical_loss": 3.3813446168928656, + "tokens_seen": 2651127808 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015906920762286862, + "loss": 1.2116, + "theoretical_loss": 3.3813380831123316, + "tokens_seen": 2651193344 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001590531594784353, + "loss": 1.271, + "theoretical_loss": 3.381331549538529, + "tokens_seen": 2651258880 + }, + { + "epoch": 0.8, + "learning_rate": 0.000159037111334002, + "loss": 1.2065, + "theoretical_loss": 3.3813250161714463, + "tokens_seen": 2651324416 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001590210631895687, + "loss": 1.2004, + "theoretical_loss": 3.3813184830110714, + "tokens_seen": 2651389952 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001590050150451354, + "loss": 1.3927, + "theoretical_loss": 3.381311950057393, + "tokens_seen": 2651455488 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001589889669007021, + "loss": 1.1661, + "theoretical_loss": 3.381305417310399, + "tokens_seen": 2651521024 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001589729187562688, + "loss": 1.1828, + "theoretical_loss": 3.3812988847700782, + "tokens_seen": 2651586560 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001589568706118355, + "loss": 1.1083, + "theoretical_loss": 3.381292352436419, + "tokens_seen": 2651652096 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001589408224674022, + "loss": 1.2342, + "theoretical_loss": 3.381285820309409, + "tokens_seen": 2651717632 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001589247743229689, + "loss": 1.2398, + "theoretical_loss": 3.3812792883890372, + "tokens_seen": 2651783168 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001589087261785356, + "loss": 1.3167, + "theoretical_loss": 3.381272756675292, + "tokens_seen": 2651848704 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001588926780341023, + "loss": 1.2657, + "theoretical_loss": 3.3812662251681616, + "tokens_seen": 2651914240 + }, + { + "epoch": 0.8, + "learning_rate": 0.000158876629889669, + "loss": 1.3092, + "theoretical_loss": 3.381259693867634, + "tokens_seen": 2651979776 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001588605817452357, + "loss": 1.3331, + "theoretical_loss": 3.3812531627736986, + "tokens_seen": 2652045312 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015884453360080243, + "loss": 1.2387, + "theoretical_loss": 3.3812466318863423, + "tokens_seen": 2652110848 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015882848545636913, + "loss": 1.1226, + "theoretical_loss": 3.381240101205555, + "tokens_seen": 2652176384 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015881243731193583, + "loss": 1.2531, + "theoretical_loss": 3.381233570731324, + "tokens_seen": 2652241920 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015879638916750253, + "loss": 1.2154, + "theoretical_loss": 3.3812270404636378, + "tokens_seen": 2652307456 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015878034102306923, + "loss": 1.1878, + "theoretical_loss": 3.381220510402485, + "tokens_seen": 2652372992 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015876429287863593, + "loss": 1.1858, + "theoretical_loss": 3.381213980547854, + "tokens_seen": 2652438528 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015874824473420264, + "loss": 1.2569, + "theoretical_loss": 3.381207450899733, + "tokens_seen": 2652504064 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001587321965897693, + "loss": 1.2299, + "theoretical_loss": 3.3812009214581105, + "tokens_seen": 2652569600 + }, + { + "epoch": 0.8, + "learning_rate": 0.000158716148445336, + "loss": 1.1513, + "theoretical_loss": 3.381194392222975, + "tokens_seen": 2652635136 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001587001003009027, + "loss": 1.215, + "theoretical_loss": 3.381187863194314, + "tokens_seen": 2652700672 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001586840521564694, + "loss": 1.2106, + "theoretical_loss": 3.3811813343721173, + "tokens_seen": 2652766208 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001586680040120361, + "loss": 1.2455, + "theoretical_loss": 3.381174805756372, + "tokens_seen": 2652831744 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001586519558676028, + "loss": 1.072, + "theoretical_loss": 3.381168277347067, + "tokens_seen": 2652897280 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001586359077231695, + "loss": 1.1801, + "theoretical_loss": 3.3811617491441908, + "tokens_seen": 2652962816 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001586198595787362, + "loss": 1.2209, + "theoretical_loss": 3.381155221147732, + "tokens_seen": 2653028352 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015860381143430291, + "loss": 1.2475, + "theoretical_loss": 3.381148693357678, + "tokens_seen": 2653093888 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015858776328986961, + "loss": 1.2259, + "theoretical_loss": 3.3811421657740177, + "tokens_seen": 2653159424 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015857171514543632, + "loss": 1.2024, + "theoretical_loss": 3.38113563839674, + "tokens_seen": 2653224960 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015855566700100302, + "loss": 1.3482, + "theoretical_loss": 3.3811291112258326, + "tokens_seen": 2653290496 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015853961885656972, + "loss": 1.2263, + "theoretical_loss": 3.381122584261284, + "tokens_seen": 2653356032 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015852357071213642, + "loss": 1.1604, + "theoretical_loss": 3.381116057503083, + "tokens_seen": 2653421568 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015850752256770312, + "loss": 1.2799, + "theoretical_loss": 3.3811095309512176, + "tokens_seen": 2653487104 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015849147442326982, + "loss": 1.1357, + "theoretical_loss": 3.3811030046056763, + "tokens_seen": 2653552640 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015847542627883652, + "loss": 1.3403, + "theoretical_loss": 3.3810964784664472, + "tokens_seen": 2653618176 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015845937813440322, + "loss": 1.19, + "theoretical_loss": 3.381089952533519, + "tokens_seen": 2653683712 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015844332998996992, + "loss": 1.2744, + "theoretical_loss": 3.38108342680688, + "tokens_seen": 2653749248 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015842728184553662, + "loss": 1.2263, + "theoretical_loss": 3.3810769012865185, + "tokens_seen": 2653814784 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015841123370110332, + "loss": 1.2238, + "theoretical_loss": 3.381070375972423, + "tokens_seen": 2653880320 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015839518555667002, + "loss": 1.3144, + "theoretical_loss": 3.381063850864582, + "tokens_seen": 2653945856 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015837913741223672, + "loss": 1.1931, + "theoretical_loss": 3.381057325962984, + "tokens_seen": 2654011392 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015836308926780342, + "loss": 1.2151, + "theoretical_loss": 3.3810508012676164, + "tokens_seen": 2654076928 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001583470411233701, + "loss": 1.2274, + "theoretical_loss": 3.3810442767784687, + "tokens_seen": 2654142464 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 886983, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1632782220840454, + "objective/train/theoretical_loss": 3.381037752495529, + "objective/train/tokens_used": 2674668000, + "theoretical_loss": 3.381037752495529, + "tokens_seen": 2654208000 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001583309929789368, + "loss": 1.2298, + "theoretical_loss": 3.381037752495529, + "tokens_seen": 2654208000 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001583149448345035, + "loss": 1.314, + "theoretical_loss": 3.3810312284187853, + "tokens_seen": 2654273536 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001582988966900702, + "loss": 1.2958, + "theoretical_loss": 3.381024704548227, + "tokens_seen": 2654339072 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001582828485456369, + "loss": 1.222, + "theoretical_loss": 3.3810181808838413, + "tokens_seen": 2654404608 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001582668004012036, + "loss": 1.1561, + "theoretical_loss": 3.3810116574256166, + "tokens_seen": 2654470144 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015825075225677033, + "loss": 1.2745, + "theoretical_loss": 3.3810051341735425, + "tokens_seen": 2654535680 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015823470411233703, + "loss": 1.2136, + "theoretical_loss": 3.3809986111276062, + "tokens_seen": 2654601216 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015821865596790373, + "loss": 1.2162, + "theoretical_loss": 3.380992088287797, + "tokens_seen": 2654666752 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015820260782347043, + "loss": 1.2214, + "theoretical_loss": 3.3809855656541026, + "tokens_seen": 2654732288 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015818655967903713, + "loss": 1.2916, + "theoretical_loss": 3.3809790432265117, + "tokens_seen": 2654797824 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015817051153460383, + "loss": 1.3434, + "theoretical_loss": 3.3809725210050128, + "tokens_seen": 2654863360 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015815446339017053, + "loss": 1.2185, + "theoretical_loss": 3.380965998989594, + "tokens_seen": 2654928896 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015813841524573723, + "loss": 1.1793, + "theoretical_loss": 3.380959477180244, + "tokens_seen": 2654994432 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015812236710130393, + "loss": 1.153, + "theoretical_loss": 3.380952955576951, + "tokens_seen": 2655059968 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015810631895687063, + "loss": 1.3021, + "theoretical_loss": 3.3809464341797035, + "tokens_seen": 2655125504 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015809027081243733, + "loss": 1.245, + "theoretical_loss": 3.38093991298849, + "tokens_seen": 2655191040 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015807422266800404, + "loss": 1.2207, + "theoretical_loss": 3.3809333920032985, + "tokens_seen": 2655256576 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015805817452357074, + "loss": 1.0925, + "theoretical_loss": 3.380926871224118, + "tokens_seen": 2655322112 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015804212637913744, + "loss": 1.1614, + "theoretical_loss": 3.380920350650936, + "tokens_seen": 2655387648 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001580260782347041, + "loss": 1.2341, + "theoretical_loss": 3.380913830283742, + "tokens_seen": 2655453184 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001580100300902708, + "loss": 1.1506, + "theoretical_loss": 3.3809073101225238, + "tokens_seen": 2655518720 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001579939819458375, + "loss": 1.1611, + "theoretical_loss": 3.3809007901672703, + "tokens_seen": 2655584256 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001579779338014042, + "loss": 1.1651, + "theoretical_loss": 3.380894270417969, + "tokens_seen": 2655649792 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001579618856569709, + "loss": 1.1234, + "theoretical_loss": 3.380887750874609, + "tokens_seen": 2655715328 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001579458375125376, + "loss": 1.3845, + "theoretical_loss": 3.380881231537179, + "tokens_seen": 2655780864 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015792978936810431, + "loss": 1.2013, + "theoretical_loss": 3.3808747124056664, + "tokens_seen": 2655846400 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015791374122367101, + "loss": 1.3091, + "theoretical_loss": 3.3808681934800604, + "tokens_seen": 2655911936 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015789769307923771, + "loss": 1.202, + "theoretical_loss": 3.3808616747603493, + "tokens_seen": 2655977472 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015788164493480442, + "loss": 1.2241, + "theoretical_loss": 3.380855156246521, + "tokens_seen": 2656043008 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015786559679037112, + "loss": 1.2966, + "theoretical_loss": 3.380848637938565, + "tokens_seen": 2656108544 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015784954864593782, + "loss": 1.233, + "theoretical_loss": 3.3808421198364687, + "tokens_seen": 2656174080 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015783350050150452, + "loss": 1.187, + "theoretical_loss": 3.380835601940221, + "tokens_seen": 2656239616 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015781745235707122, + "loss": 1.2244, + "theoretical_loss": 3.38082908424981, + "tokens_seen": 2656305152 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015780140421263792, + "loss": 1.2922, + "theoretical_loss": 3.3808225667652243, + "tokens_seen": 2656370688 + }, + { + "epoch": 0.8, + "learning_rate": 0.00015778535606820462, + "loss": 1.249, + "theoretical_loss": 3.3808160494864525, + "tokens_seen": 2656436224 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015776930792377132, + "loss": 1.2248, + "theoretical_loss": 3.380809532413483, + "tokens_seen": 2656501760 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015775325977933802, + "loss": 1.3127, + "theoretical_loss": 3.380803015546304, + "tokens_seen": 2656567296 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015773721163490472, + "loss": 1.1884, + "theoretical_loss": 3.380796498884904, + "tokens_seen": 2656632832 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015772116349047142, + "loss": 1.3342, + "theoretical_loss": 3.3807899824292713, + "tokens_seen": 2656698368 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015770511534603812, + "loss": 1.3992, + "theoretical_loss": 3.3807834661793947, + "tokens_seen": 2656763904 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015768906720160482, + "loss": 1.0844, + "theoretical_loss": 3.380776950135262, + "tokens_seen": 2656829440 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015767301905717152, + "loss": 1.3298, + "theoretical_loss": 3.3807704342968625, + "tokens_seen": 2656894976 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015765697091273822, + "loss": 1.2294, + "theoretical_loss": 3.3807639186641842, + "tokens_seen": 2656960512 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015764092276830493, + "loss": 1.3352, + "theoretical_loss": 3.380757403237215, + "tokens_seen": 2657026048 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015762487462387163, + "loss": 1.2571, + "theoretical_loss": 3.3807508880159443, + "tokens_seen": 2657091584 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015760882647943833, + "loss": 1.1846, + "theoretical_loss": 3.38074437300036, + "tokens_seen": 2657157120 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015759277833500503, + "loss": 1.2217, + "theoretical_loss": 3.38073785819045, + "tokens_seen": 2657222656 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015757673019057173, + "loss": 1.2818, + "theoretical_loss": 3.380731343586204, + "tokens_seen": 2657288192 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015756068204613843, + "loss": 1.1785, + "theoretical_loss": 3.3807248291876095, + "tokens_seen": 2657353728 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015754463390170513, + "loss": 1.2147, + "theoretical_loss": 3.380718314994655, + "tokens_seen": 2657419264 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 888207, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1872183084487915, + "objective/train/theoretical_loss": 3.3807118010073296, + "objective/train/tokens_used": 2677944800, + "theoretical_loss": 3.3807118010073296, + "tokens_seen": 2657484800 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015752858575727183, + "loss": 1.1925, + "theoretical_loss": 3.3807118010073296, + "tokens_seen": 2657484800 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015751253761283853, + "loss": 1.2441, + "theoretical_loss": 3.380705287225621, + "tokens_seen": 2657550336 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015749648946840523, + "loss": 1.3148, + "theoretical_loss": 3.380698773649518, + "tokens_seen": 2657615872 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015748044132397193, + "loss": 1.3287, + "theoretical_loss": 3.380692260279009, + "tokens_seen": 2657681408 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015746439317953863, + "loss": 1.3123, + "theoretical_loss": 3.380685747114082, + "tokens_seen": 2657746944 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015744834503510533, + "loss": 1.2265, + "theoretical_loss": 3.380679234154726, + "tokens_seen": 2657812480 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015743229689067203, + "loss": 1.2941, + "theoretical_loss": 3.3806727214009293, + "tokens_seen": 2657878016 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015741624874623873, + "loss": 1.1624, + "theoretical_loss": 3.3806662088526807, + "tokens_seen": 2657943552 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015740020060180543, + "loss": 1.2331, + "theoretical_loss": 3.3806596965099676, + "tokens_seen": 2658009088 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015738415245737214, + "loss": 1.3538, + "theoretical_loss": 3.38065318437278, + "tokens_seen": 2658074624 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015736810431293884, + "loss": 1.264, + "theoretical_loss": 3.3806466724411046, + "tokens_seen": 2658140160 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015735205616850554, + "loss": 1.2426, + "theoretical_loss": 3.380640160714931, + "tokens_seen": 2658205696 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001573360080240722, + "loss": 1.2145, + "theoretical_loss": 3.3806336491942472, + "tokens_seen": 2658271232 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001573199598796389, + "loss": 1.2978, + "theoretical_loss": 3.380627137879042, + "tokens_seen": 2658336768 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001573039117352056, + "loss": 1.263, + "theoretical_loss": 3.3806206267693035, + "tokens_seen": 2658402304 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001572878635907723, + "loss": 1.1519, + "theoretical_loss": 3.3806141158650207, + "tokens_seen": 2658467840 + }, + { + "epoch": 0.81, + "learning_rate": 0.000157271815446339, + "loss": 1.2643, + "theoretical_loss": 3.3806076051661815, + "tokens_seen": 2658533376 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001572557673019057, + "loss": 1.1913, + "theoretical_loss": 3.380601094672774, + "tokens_seen": 2658598912 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015723971915747241, + "loss": 1.1897, + "theoretical_loss": 3.380594584384788, + "tokens_seen": 2658664448 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015722367101303911, + "loss": 1.2105, + "theoretical_loss": 3.3805880743022105, + "tokens_seen": 2658729984 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015720762286860582, + "loss": 1.2213, + "theoretical_loss": 3.3805815644250305, + "tokens_seen": 2658795520 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015719157472417252, + "loss": 1.2542, + "theoretical_loss": 3.380575054753237, + "tokens_seen": 2658861056 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015717552657973922, + "loss": 1.0891, + "theoretical_loss": 3.3805685452868177, + "tokens_seen": 2658926592 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015715947843530592, + "loss": 1.2212, + "theoretical_loss": 3.3805620360257613, + "tokens_seen": 2658992128 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015714343029087262, + "loss": 1.1773, + "theoretical_loss": 3.3805555269700567, + "tokens_seen": 2659057664 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015712738214643932, + "loss": 1.3241, + "theoretical_loss": 3.3805490181196918, + "tokens_seen": 2659123200 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015711133400200602, + "loss": 1.3084, + "theoretical_loss": 3.380542509474655, + "tokens_seen": 2659188736 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015709528585757272, + "loss": 1.2789, + "theoretical_loss": 3.380536001034935, + "tokens_seen": 2659254272 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015707923771313942, + "loss": 1.2295, + "theoretical_loss": 3.3805294928005205, + "tokens_seen": 2659319808 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015706318956870612, + "loss": 1.23, + "theoretical_loss": 3.3805229847713996, + "tokens_seen": 2659385344 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015704714142427285, + "loss": 1.2793, + "theoretical_loss": 3.380516476947561, + "tokens_seen": 2659450880 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015703109327983955, + "loss": 1.2441, + "theoretical_loss": 3.380509969328993, + "tokens_seen": 2659516416 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015701504513540622, + "loss": 1.2659, + "theoretical_loss": 3.380503461915684, + "tokens_seen": 2659581952 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015699899699097292, + "loss": 1.3101, + "theoretical_loss": 3.3804969547076227, + "tokens_seen": 2659647488 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015698294884653962, + "loss": 1.1916, + "theoretical_loss": 3.3804904477047977, + "tokens_seen": 2659713024 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015696690070210632, + "loss": 1.3001, + "theoretical_loss": 3.3804839409071965, + "tokens_seen": 2659778560 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015695085255767303, + "loss": 1.2377, + "theoretical_loss": 3.380477434314809, + "tokens_seen": 2659844096 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015693480441323973, + "loss": 1.1234, + "theoretical_loss": 3.3804709279276226, + "tokens_seen": 2659909632 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015691875626880643, + "loss": 1.1787, + "theoretical_loss": 3.3804644217456263, + "tokens_seen": 2659975168 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015690270812437313, + "loss": 1.2557, + "theoretical_loss": 3.3804579157688086, + "tokens_seen": 2660040704 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015688665997993983, + "loss": 1.1251, + "theoretical_loss": 3.3804514099971574, + "tokens_seen": 2660106240 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015687061183550653, + "loss": 1.252, + "theoretical_loss": 3.3804449044306617, + "tokens_seen": 2660171776 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015685456369107323, + "loss": 1.2592, + "theoretical_loss": 3.38043839906931, + "tokens_seen": 2660237312 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015683851554663993, + "loss": 1.2311, + "theoretical_loss": 3.3804318939130904, + "tokens_seen": 2660302848 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015682246740220663, + "loss": 1.3326, + "theoretical_loss": 3.3804253889619917, + "tokens_seen": 2660368384 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015680641925777333, + "loss": 1.2764, + "theoretical_loss": 3.3804188842160023, + "tokens_seen": 2660433920 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015679037111334003, + "loss": 1.2763, + "theoretical_loss": 3.3804123796751107, + "tokens_seen": 2660499456 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015677432296890673, + "loss": 1.292, + "theoretical_loss": 3.3804058753393056, + "tokens_seen": 2660564992 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015675827482447343, + "loss": 1.3022, + "theoretical_loss": 3.3803993712085747, + "tokens_seen": 2660630528 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015674222668004013, + "loss": 1.1676, + "theoretical_loss": 3.3803928672829073, + "tokens_seen": 2660696064 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 889219, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.11279296875, + "objective/train/theoretical_loss": 3.3803863635622915, + "objective/train/tokens_used": 2681221600, + "theoretical_loss": 3.3803863635622915, + "tokens_seen": 2660761600 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015672617853560683, + "loss": 1.1967, + "theoretical_loss": 3.3803863635622915, + "tokens_seen": 2660761600 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015671013039117353, + "loss": 1.2085, + "theoretical_loss": 3.380379860046716, + "tokens_seen": 2660827136 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015669408224674024, + "loss": 1.1446, + "theoretical_loss": 3.380373356736169, + "tokens_seen": 2660892672 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015667803410230694, + "loss": 1.2615, + "theoretical_loss": 3.380366853630639, + "tokens_seen": 2660958208 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015666198595787364, + "loss": 1.1681, + "theoretical_loss": 3.3803603507301148, + "tokens_seen": 2661023744 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015664593781344034, + "loss": 1.3255, + "theoretical_loss": 3.380353848034585, + "tokens_seen": 2661089280 + }, + { + "epoch": 0.81, + "learning_rate": 0.000156629889669007, + "loss": 1.2353, + "theoretical_loss": 3.380347345544038, + "tokens_seen": 2661154816 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001566138415245737, + "loss": 1.2204, + "theoretical_loss": 3.380340843258461, + "tokens_seen": 2661220352 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001565977933801404, + "loss": 1.1534, + "theoretical_loss": 3.3803343411778446, + "tokens_seen": 2661285888 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001565817452357071, + "loss": 1.1934, + "theoretical_loss": 3.380327839302176, + "tokens_seen": 2661351424 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001565656970912738, + "loss": 1.2339, + "theoretical_loss": 3.380321337631444, + "tokens_seen": 2661416960 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015654964894684051, + "loss": 1.2893, + "theoretical_loss": 3.380314836165637, + "tokens_seen": 2661482496 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015653360080240721, + "loss": 1.3037, + "theoretical_loss": 3.380308334904744, + "tokens_seen": 2661548032 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015651755265797392, + "loss": 1.207, + "theoretical_loss": 3.3803018338487525, + "tokens_seen": 2661613568 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015650150451354062, + "loss": 1.1996, + "theoretical_loss": 3.3802953329976515, + "tokens_seen": 2661679104 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015648545636910732, + "loss": 1.3018, + "theoretical_loss": 3.3802888323514297, + "tokens_seen": 2661744640 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015646940822467402, + "loss": 1.254, + "theoretical_loss": 3.3802823319100757, + "tokens_seen": 2661810176 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015645336008024074, + "loss": 1.3045, + "theoretical_loss": 3.3802758316735777, + "tokens_seen": 2661875712 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015643731193580745, + "loss": 1.2802, + "theoretical_loss": 3.3802693316419243, + "tokens_seen": 2661941248 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015642126379137415, + "loss": 1.2073, + "theoretical_loss": 3.380262831815104, + "tokens_seen": 2662006784 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015640521564694085, + "loss": 1.2365, + "theoretical_loss": 3.380256332193105, + "tokens_seen": 2662072320 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015638916750250755, + "loss": 1.2131, + "theoretical_loss": 3.380249832775916, + "tokens_seen": 2662137856 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015637311935807425, + "loss": 1.2265, + "theoretical_loss": 3.3802433335635262, + "tokens_seen": 2662203392 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015635707121364095, + "loss": 1.2266, + "theoretical_loss": 3.380236834555923, + "tokens_seen": 2662268928 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015634102306920765, + "loss": 1.1879, + "theoretical_loss": 3.380230335753095, + "tokens_seen": 2662334464 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015632497492477435, + "loss": 1.1657, + "theoretical_loss": 3.3802238371550315, + "tokens_seen": 2662400000 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015630892678034102, + "loss": 1.0773, + "theoretical_loss": 3.380217338761721, + "tokens_seen": 2662465536 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015629287863590772, + "loss": 1.1577, + "theoretical_loss": 3.3802108405731515, + "tokens_seen": 2662531072 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015627683049147442, + "loss": 1.23, + "theoretical_loss": 3.380204342589311, + "tokens_seen": 2662596608 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015626078234704113, + "loss": 1.1529, + "theoretical_loss": 3.3801978448101893, + "tokens_seen": 2662662144 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015624473420260783, + "loss": 1.2747, + "theoretical_loss": 3.380191347235774, + "tokens_seen": 2662727680 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015622868605817453, + "loss": 1.2709, + "theoretical_loss": 3.3801848498660534, + "tokens_seen": 2662793216 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015621263791374123, + "loss": 1.2495, + "theoretical_loss": 3.380178352701017, + "tokens_seen": 2662858752 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015619658976930793, + "loss": 1.2399, + "theoretical_loss": 3.380171855740653, + "tokens_seen": 2662924288 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015618054162487463, + "loss": 1.2125, + "theoretical_loss": 3.380165358984949, + "tokens_seen": 2662989824 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015616449348044133, + "loss": 1.2133, + "theoretical_loss": 3.3801588624338947, + "tokens_seen": 2663055360 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015614844533600803, + "loss": 1.1868, + "theoretical_loss": 3.380152366087478, + "tokens_seen": 2663120896 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015613239719157473, + "loss": 1.2454, + "theoretical_loss": 3.3801458699456877, + "tokens_seen": 2663186432 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015611634904714143, + "loss": 1.1642, + "theoretical_loss": 3.380139374008512, + "tokens_seen": 2663251968 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015610030090270813, + "loss": 1.2891, + "theoretical_loss": 3.3801328782759397, + "tokens_seen": 2663317504 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015608425275827483, + "loss": 1.2276, + "theoretical_loss": 3.3801263827479593, + "tokens_seen": 2663383040 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015606820461384153, + "loss": 1.1094, + "theoretical_loss": 3.380119887424559, + "tokens_seen": 2663448576 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015605215646940823, + "loss": 1.2366, + "theoretical_loss": 3.3801133923057276, + "tokens_seen": 2663514112 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015603610832497493, + "loss": 1.2212, + "theoretical_loss": 3.3801068973914536, + "tokens_seen": 2663579648 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015602006018054163, + "loss": 1.178, + "theoretical_loss": 3.380100402681726, + "tokens_seen": 2663645184 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015600401203610834, + "loss": 1.1995, + "theoretical_loss": 3.3800939081765318, + "tokens_seen": 2663710720 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015598796389167504, + "loss": 1.2264, + "theoretical_loss": 3.380087413875861, + "tokens_seen": 2663776256 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015597191574724174, + "loss": 1.1542, + "theoretical_loss": 3.380080919779702, + "tokens_seen": 2663841792 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015595586760280844, + "loss": 1.2035, + "theoretical_loss": 3.380074425888043, + "tokens_seen": 2663907328 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015593981945837514, + "loss": 1.1709, + "theoretical_loss": 3.3800679322008724, + "tokens_seen": 2663972864 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 890477, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.275221586227417, + "objective/train/theoretical_loss": 3.3800614387181787, + "objective/train/tokens_used": 2684498400, + "theoretical_loss": 3.3800614387181787, + "tokens_seen": 2664038400 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001559237713139418, + "loss": 1.3066, + "theoretical_loss": 3.3800614387181787, + "tokens_seen": 2664038400 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001559077231695085, + "loss": 1.1935, + "theoretical_loss": 3.380054945439951, + "tokens_seen": 2664103936 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001558916750250752, + "loss": 1.216, + "theoretical_loss": 3.3800484523661773, + "tokens_seen": 2664169472 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001558756268806419, + "loss": 1.2213, + "theoretical_loss": 3.3800419594968463, + "tokens_seen": 2664235008 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015585957873620861, + "loss": 1.233, + "theoretical_loss": 3.3800354668319463, + "tokens_seen": 2664300544 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015584353059177534, + "loss": 1.3028, + "theoretical_loss": 3.380028974371466, + "tokens_seen": 2664366080 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015582748244734204, + "loss": 1.1978, + "theoretical_loss": 3.380022482115394, + "tokens_seen": 2664431616 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015581143430290874, + "loss": 1.2696, + "theoretical_loss": 3.3800159900637192, + "tokens_seen": 2664497152 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015579538615847544, + "loss": 1.1582, + "theoretical_loss": 3.3800094982164293, + "tokens_seen": 2664562688 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015577933801404214, + "loss": 1.202, + "theoretical_loss": 3.3800030065735136, + "tokens_seen": 2664628224 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015576328986960885, + "loss": 1.2514, + "theoretical_loss": 3.3799965151349602, + "tokens_seen": 2664693760 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015574724172517555, + "loss": 1.1653, + "theoretical_loss": 3.379990023900758, + "tokens_seen": 2664759296 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015573119358074225, + "loss": 1.2343, + "theoretical_loss": 3.379983532870895, + "tokens_seen": 2664824832 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015571514543630895, + "loss": 1.193, + "theoretical_loss": 3.3799770420453603, + "tokens_seen": 2664890368 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015569909729187565, + "loss": 1.2159, + "theoretical_loss": 3.379970551424142, + "tokens_seen": 2664955904 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015568304914744235, + "loss": 1.1681, + "theoretical_loss": 3.379964061007229, + "tokens_seen": 2665021440 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015566700100300905, + "loss": 1.277, + "theoretical_loss": 3.3799575707946095, + "tokens_seen": 2665086976 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015565095285857575, + "loss": 1.1957, + "theoretical_loss": 3.379951080786272, + "tokens_seen": 2665152512 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015563490471414245, + "loss": 1.1705, + "theoretical_loss": 3.379944590982206, + "tokens_seen": 2665218048 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015561885656970912, + "loss": 1.1386, + "theoretical_loss": 3.3799381013823986, + "tokens_seen": 2665283584 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015560280842527582, + "loss": 1.2904, + "theoretical_loss": 3.37993161198684, + "tokens_seen": 2665349120 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015558676028084252, + "loss": 1.3412, + "theoretical_loss": 3.379925122795517, + "tokens_seen": 2665414656 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015557071213640923, + "loss": 1.209, + "theoretical_loss": 3.379918633808419, + "tokens_seen": 2665480192 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015555466399197593, + "loss": 1.2938, + "theoretical_loss": 3.379912145025535, + "tokens_seen": 2665545728 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015553861584754263, + "loss": 1.2487, + "theoretical_loss": 3.379905656446853, + "tokens_seen": 2665611264 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015552256770310933, + "loss": 1.1899, + "theoretical_loss": 3.3798991680723613, + "tokens_seen": 2665676800 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015550651955867603, + "loss": 1.1217, + "theoretical_loss": 3.3798926799020492, + "tokens_seen": 2665742336 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015549047141424273, + "loss": 1.0708, + "theoretical_loss": 3.3798861919359044, + "tokens_seen": 2665807872 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015547442326980943, + "loss": 1.2029, + "theoretical_loss": 3.3798797041739164, + "tokens_seen": 2665873408 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015545837512537613, + "loss": 1.169, + "theoretical_loss": 3.379873216616073, + "tokens_seen": 2665938944 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015544232698094283, + "loss": 1.0759, + "theoretical_loss": 3.3798667292623628, + "tokens_seen": 2666004480 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015542627883650953, + "loss": 1.1736, + "theoretical_loss": 3.379860242112775, + "tokens_seen": 2666070016 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015541023069207623, + "loss": 1.1981, + "theoretical_loss": 3.3798537551672974, + "tokens_seen": 2666135552 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015539418254764293, + "loss": 1.158, + "theoretical_loss": 3.379847268425919, + "tokens_seen": 2666201088 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015537813440320963, + "loss": 1.1047, + "theoretical_loss": 3.3798407818886282, + "tokens_seen": 2666266624 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015536208625877633, + "loss": 1.3161, + "theoretical_loss": 3.379834295555414, + "tokens_seen": 2666332160 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015534603811434303, + "loss": 1.0764, + "theoretical_loss": 3.3798278094262644, + "tokens_seen": 2666397696 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015532998996990974, + "loss": 1.0623, + "theoretical_loss": 3.379821323501168, + "tokens_seen": 2666463232 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015531394182547644, + "loss": 1.2105, + "theoretical_loss": 3.3798148377801134, + "tokens_seen": 2666528768 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015529789368104314, + "loss": 1.1374, + "theoretical_loss": 3.3798083522630895, + "tokens_seen": 2666594304 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015528184553660984, + "loss": 1.1795, + "theoretical_loss": 3.3798018669500847, + "tokens_seen": 2666659840 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015526579739217654, + "loss": 1.1188, + "theoretical_loss": 3.3797953818410877, + "tokens_seen": 2666725376 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015524974924774324, + "loss": 1.0718, + "theoretical_loss": 3.3797888969360868, + "tokens_seen": 2666790912 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015523370110330994, + "loss": 1.1619, + "theoretical_loss": 3.37978241223507, + "tokens_seen": 2666856448 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015521765295887664, + "loss": 1.1134, + "theoretical_loss": 3.3797759277380273, + "tokens_seen": 2666921984 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015520160481444334, + "loss": 1.2259, + "theoretical_loss": 3.379769443444946, + "tokens_seen": 2666987520 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015518555667001004, + "loss": 1.1773, + "theoretical_loss": 3.3797629593558156, + "tokens_seen": 2667053056 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015516950852557674, + "loss": 1.255, + "theoretical_loss": 3.379756475470624, + "tokens_seen": 2667118592 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015515346038114344, + "loss": 1.2235, + "theoretical_loss": 3.3797499917893603, + "tokens_seen": 2667184128 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015513741223671014, + "loss": 1.1821, + "theoretical_loss": 3.3797435083120124, + "tokens_seen": 2667249664 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 891201, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.19282066822052, + "objective/train/theoretical_loss": 3.37973702503857, + "objective/train/tokens_used": 2687775200, + "theoretical_loss": 3.37973702503857, + "tokens_seen": 2667315200 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015512136409227684, + "loss": 1.2293, + "theoretical_loss": 3.37973702503857, + "tokens_seen": 2667315200 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015510531594784354, + "loss": 1.1679, + "theoretical_loss": 3.3797305419690202, + "tokens_seen": 2667380736 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015508926780341024, + "loss": 1.2255, + "theoretical_loss": 3.3797240591033524, + "tokens_seen": 2667446272 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015507321965897695, + "loss": 1.1528, + "theoretical_loss": 3.3797175764415552, + "tokens_seen": 2667511808 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015505717151454365, + "loss": 1.2658, + "theoretical_loss": 3.3797110939836172, + "tokens_seen": 2667577344 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015504112337011035, + "loss": 1.1814, + "theoretical_loss": 3.379704611729527, + "tokens_seen": 2667642880 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015502507522567705, + "loss": 1.2194, + "theoretical_loss": 3.379698129679273, + "tokens_seen": 2667708416 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015500902708124375, + "loss": 1.3306, + "theoretical_loss": 3.3796916478328436, + "tokens_seen": 2667773952 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015499297893681045, + "loss": 1.2095, + "theoretical_loss": 3.3796851661902276, + "tokens_seen": 2667839488 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015497693079237715, + "loss": 1.181, + "theoretical_loss": 3.379678684751414, + "tokens_seen": 2667905024 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015496088264794385, + "loss": 1.2691, + "theoretical_loss": 3.3796722035163906, + "tokens_seen": 2667970560 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015494483450351055, + "loss": 1.1532, + "theoretical_loss": 3.3796657224851465, + "tokens_seen": 2668036096 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015492878635907725, + "loss": 1.2039, + "theoretical_loss": 3.37965924165767, + "tokens_seen": 2668101632 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015491273821464392, + "loss": 1.1483, + "theoretical_loss": 3.37965276103395, + "tokens_seen": 2668167168 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015489669007021063, + "loss": 1.2751, + "theoretical_loss": 3.3796462806139753, + "tokens_seen": 2668232704 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015488064192577733, + "loss": 1.2917, + "theoretical_loss": 3.379639800397734, + "tokens_seen": 2668298240 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015486459378134403, + "loss": 1.2762, + "theoretical_loss": 3.3796333203852145, + "tokens_seen": 2668363776 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015484854563691073, + "loss": 1.2056, + "theoretical_loss": 3.379626840576406, + "tokens_seen": 2668429312 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015483249749247743, + "loss": 1.3026, + "theoretical_loss": 3.3796203609712965, + "tokens_seen": 2668494848 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015481644934804413, + "loss": 1.239, + "theoretical_loss": 3.379613881569875, + "tokens_seen": 2668560384 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015480040120361083, + "loss": 1.167, + "theoretical_loss": 3.3796074023721303, + "tokens_seen": 2668625920 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015478435305917753, + "loss": 1.223, + "theoretical_loss": 3.3796009233780504, + "tokens_seen": 2668691456 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015476830491474423, + "loss": 1.1719, + "theoretical_loss": 3.3795944445876245, + "tokens_seen": 2668756992 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015475225677031093, + "loss": 1.2175, + "theoretical_loss": 3.3795879660008405, + "tokens_seen": 2668822528 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015473620862587763, + "loss": 1.1507, + "theoretical_loss": 3.3795814876176875, + "tokens_seen": 2668888064 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015472016048144433, + "loss": 1.1482, + "theoretical_loss": 3.3795750094381543, + "tokens_seen": 2668953600 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015470411233701103, + "loss": 1.1665, + "theoretical_loss": 3.379568531462229, + "tokens_seen": 2669019136 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015468806419257773, + "loss": 1.162, + "theoretical_loss": 3.3795620536899, + "tokens_seen": 2669084672 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015467201604814443, + "loss": 1.1339, + "theoretical_loss": 3.379555576121157, + "tokens_seen": 2669150208 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015465596790371113, + "loss": 1.1465, + "theoretical_loss": 3.3795490987559873, + "tokens_seen": 2669215744 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015463991975927786, + "loss": 1.2398, + "theoretical_loss": 3.3795426215943802, + "tokens_seen": 2669281280 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015462387161484456, + "loss": 1.201, + "theoretical_loss": 3.3795361446363246, + "tokens_seen": 2669346816 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015460782347041126, + "loss": 1.1449, + "theoretical_loss": 3.3795296678818083, + "tokens_seen": 2669412352 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015459177532597794, + "loss": 1.1556, + "theoretical_loss": 3.3795231913308204, + "tokens_seen": 2669477888 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015457572718154464, + "loss": 1.2738, + "theoretical_loss": 3.3795167149833496, + "tokens_seen": 2669543424 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015455967903711134, + "loss": 1.2362, + "theoretical_loss": 3.3795102388393845, + "tokens_seen": 2669608960 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015454363089267804, + "loss": 1.19, + "theoretical_loss": 3.379503762898913, + "tokens_seen": 2669674496 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015452758274824474, + "loss": 1.1053, + "theoretical_loss": 3.3794972871619247, + "tokens_seen": 2669740032 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015451153460381144, + "loss": 1.2464, + "theoretical_loss": 3.3794908116284077, + "tokens_seen": 2669805568 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015449548645937814, + "loss": 1.2592, + "theoretical_loss": 3.3794843362983507, + "tokens_seen": 2669871104 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015447943831494484, + "loss": 1.1979, + "theoretical_loss": 3.379477861171742, + "tokens_seen": 2669936640 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015446339017051154, + "loss": 1.1063, + "theoretical_loss": 3.3794713862485706, + "tokens_seen": 2670002176 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015444734202607824, + "loss": 1.2655, + "theoretical_loss": 3.3794649115288253, + "tokens_seen": 2670067712 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015443129388164494, + "loss": 1.1903, + "theoretical_loss": 3.3794584370124943, + "tokens_seen": 2670133248 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015441524573721164, + "loss": 1.2484, + "theoretical_loss": 3.379451962699566, + "tokens_seen": 2670198784 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015439919759277834, + "loss": 1.1642, + "theoretical_loss": 3.37944548859003, + "tokens_seen": 2670264320 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015438314944834505, + "loss": 1.1146, + "theoretical_loss": 3.3794390146838738, + "tokens_seen": 2670329856 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015436710130391175, + "loss": 1.2388, + "theoretical_loss": 3.3794325409810866, + "tokens_seen": 2670395392 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015435105315947845, + "loss": 1.1763, + "theoretical_loss": 3.379426067481657, + "tokens_seen": 2670460928 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015433500501504515, + "loss": 1.1507, + "theoretical_loss": 3.3794195941855736, + "tokens_seen": 2670526464 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 892379, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2288014888763428, + "objective/train/theoretical_loss": 3.379413121092825, + "objective/train/tokens_used": 2691052000, + "theoretical_loss": 3.379413121092825, + "tokens_seen": 2670592000 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015431895687061185, + "loss": 1.225, + "theoretical_loss": 3.379413121092825, + "tokens_seen": 2670592000 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015430290872617855, + "loss": 1.2734, + "theoretical_loss": 3.3794066482033998, + "tokens_seen": 2670657536 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015428686058174525, + "loss": 1.2153, + "theoretical_loss": 3.3794001755172864, + "tokens_seen": 2670723072 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015427081243731195, + "loss": 1.2558, + "theoretical_loss": 3.379393703034474, + "tokens_seen": 2670788608 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015425476429287865, + "loss": 1.1225, + "theoretical_loss": 3.3793872307549506, + "tokens_seen": 2670854144 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015423871614844535, + "loss": 1.1985, + "theoretical_loss": 3.3793807586787055, + "tokens_seen": 2670919680 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015422266800401205, + "loss": 1.1707, + "theoretical_loss": 3.3793742868057266, + "tokens_seen": 2670985216 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015420661985957873, + "loss": 1.1624, + "theoretical_loss": 3.379367815136003, + "tokens_seen": 2671050752 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015419057171514543, + "loss": 1.1952, + "theoretical_loss": 3.379361343669523, + "tokens_seen": 2671116288 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015417452357071213, + "loss": 1.0599, + "theoretical_loss": 3.3793548724062754, + "tokens_seen": 2671181824 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015415847542627883, + "loss": 1.1838, + "theoretical_loss": 3.379348401346249, + "tokens_seen": 2671247360 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015414242728184553, + "loss": 1.2044, + "theoretical_loss": 3.3793419304894323, + "tokens_seen": 2671312896 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015412637913741223, + "loss": 1.12, + "theoretical_loss": 3.379335459835814, + "tokens_seen": 2671378432 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015411033099297893, + "loss": 1.1006, + "theoretical_loss": 3.3793289893853826, + "tokens_seen": 2671443968 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015409428284854563, + "loss": 1.0955, + "theoretical_loss": 3.3793225191381264, + "tokens_seen": 2671509504 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015407823470411233, + "loss": 1.2458, + "theoretical_loss": 3.379316049094035, + "tokens_seen": 2671575040 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015406218655967903, + "loss": 1.2876, + "theoretical_loss": 3.3793095792530963, + "tokens_seen": 2671640576 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015404613841524576, + "loss": 1.2895, + "theoretical_loss": 3.379303109615299, + "tokens_seen": 2671706112 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015403009027081246, + "loss": 1.1726, + "theoretical_loss": 3.3792966401806317, + "tokens_seen": 2671771648 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015401404212637916, + "loss": 1.2931, + "theoretical_loss": 3.3792901709490835, + "tokens_seen": 2671837184 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015399799398194586, + "loss": 1.1903, + "theoretical_loss": 3.3792837019206425, + "tokens_seen": 2671902720 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015398194583751256, + "loss": 1.1551, + "theoretical_loss": 3.3792772330952974, + "tokens_seen": 2671968256 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015396589769307926, + "loss": 1.1705, + "theoretical_loss": 3.3792707644730373, + "tokens_seen": 2672033792 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015394984954864596, + "loss": 1.2323, + "theoretical_loss": 3.3792642960538504, + "tokens_seen": 2672099328 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015393380140421266, + "loss": 1.2683, + "theoretical_loss": 3.3792578278377254, + "tokens_seen": 2672164864 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015391775325977936, + "loss": 1.1352, + "theoretical_loss": 3.3792513598246514, + "tokens_seen": 2672230400 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015390170511534606, + "loss": 1.2956, + "theoretical_loss": 3.3792448920146168, + "tokens_seen": 2672295936 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015388565697091274, + "loss": 1.1847, + "theoretical_loss": 3.3792384244076095, + "tokens_seen": 2672361472 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015386960882647944, + "loss": 1.0937, + "theoretical_loss": 3.3792319570036193, + "tokens_seen": 2672427008 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015385356068204614, + "loss": 1.2357, + "theoretical_loss": 3.379225489802634, + "tokens_seen": 2672492544 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015383751253761284, + "loss": 1.2187, + "theoretical_loss": 3.3792190228046426, + "tokens_seen": 2672558080 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015382146439317954, + "loss": 1.1373, + "theoretical_loss": 3.3792125560096338, + "tokens_seen": 2672623616 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015380541624874624, + "loss": 1.2192, + "theoretical_loss": 3.3792060894175964, + "tokens_seen": 2672689152 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015378936810431294, + "loss": 1.2581, + "theoretical_loss": 3.3791996230285184, + "tokens_seen": 2672754688 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015377331995987964, + "loss": 1.1284, + "theoretical_loss": 3.379193156842389, + "tokens_seen": 2672820224 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015375727181544634, + "loss": 1.3526, + "theoretical_loss": 3.379186690859197, + "tokens_seen": 2672885760 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015374122367101304, + "loss": 1.2868, + "theoretical_loss": 3.3791802250789305, + "tokens_seen": 2672951296 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015372517552657974, + "loss": 1.2517, + "theoretical_loss": 3.3791737595015787, + "tokens_seen": 2673016832 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015370912738214644, + "loss": 1.1981, + "theoretical_loss": 3.37916729412713, + "tokens_seen": 2673082368 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015369307923771315, + "loss": 1.2017, + "theoretical_loss": 3.3791608289555732, + "tokens_seen": 2673147904 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015367703109327985, + "loss": 1.2535, + "theoretical_loss": 3.3791543639868964, + "tokens_seen": 2673213440 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015366098294884655, + "loss": 1.208, + "theoretical_loss": 3.379147899221089, + "tokens_seen": 2673278976 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015364493480441325, + "loss": 1.2222, + "theoretical_loss": 3.3791414346581394, + "tokens_seen": 2673344512 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015362888665997995, + "loss": 1.2519, + "theoretical_loss": 3.379134970298036, + "tokens_seen": 2673410048 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015361283851554665, + "loss": 1.1038, + "theoretical_loss": 3.3791285061407677, + "tokens_seen": 2673475584 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015359679037111335, + "loss": 1.1408, + "theoretical_loss": 3.379122042186323, + "tokens_seen": 2673541120 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015358074222668005, + "loss": 1.1482, + "theoretical_loss": 3.379115578434691, + "tokens_seen": 2673606656 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015356469408224675, + "loss": 1.2248, + "theoretical_loss": 3.3791091148858596, + "tokens_seen": 2673672192 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015354864593781345, + "loss": 1.1657, + "theoretical_loss": 3.3791026515398186, + "tokens_seen": 2673737728 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015353259779338015, + "loss": 1.1042, + "theoretical_loss": 3.379096188396556, + "tokens_seen": 2673803264 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 893294, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1958637237548828, + "objective/train/theoretical_loss": 3.3790897254560597, + "objective/train/tokens_used": 2694328800, + "theoretical_loss": 3.3790897254560597, + "tokens_seen": 2673868800 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015351654964894685, + "loss": 1.1671, + "theoretical_loss": 3.3790897254560597, + "tokens_seen": 2673868800 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015350050150451353, + "loss": 1.2255, + "theoretical_loss": 3.3790832627183196, + "tokens_seen": 2673934336 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015348445336008023, + "loss": 1.2032, + "theoretical_loss": 3.379076800183324, + "tokens_seen": 2673999872 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015346840521564693, + "loss": 1.2888, + "theoretical_loss": 3.3790703378510614, + "tokens_seen": 2674065408 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015345235707121366, + "loss": 1.1497, + "theoretical_loss": 3.3790638757215206, + "tokens_seen": 2674130944 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015343630892678036, + "loss": 1.1298, + "theoretical_loss": 3.37905741379469, + "tokens_seen": 2674196480 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015342026078234706, + "loss": 1.1677, + "theoretical_loss": 3.3790509520705587, + "tokens_seen": 2674262016 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015340421263791376, + "loss": 1.2314, + "theoretical_loss": 3.379044490549115, + "tokens_seen": 2674327552 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015338816449348046, + "loss": 1.2294, + "theoretical_loss": 3.379038029230348, + "tokens_seen": 2674393088 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015337211634904716, + "loss": 1.1105, + "theoretical_loss": 3.3790315681142458, + "tokens_seen": 2674458624 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015335606820461386, + "loss": 1.2035, + "theoretical_loss": 3.3790251072007975, + "tokens_seen": 2674524160 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015334002006018056, + "loss": 1.2035, + "theoretical_loss": 3.3790186464899916, + "tokens_seen": 2674589696 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015332397191574726, + "loss": 1.1129, + "theoretical_loss": 3.379012185981817, + "tokens_seen": 2674655232 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015330792377131396, + "loss": 1.1388, + "theoretical_loss": 3.3790057256762625, + "tokens_seen": 2674720768 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015329187562688066, + "loss": 1.2496, + "theoretical_loss": 3.378999265573316, + "tokens_seen": 2674786304 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015327582748244736, + "loss": 1.0915, + "theoretical_loss": 3.3789928056729672, + "tokens_seen": 2674851840 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015325977933801406, + "loss": 1.2064, + "theoretical_loss": 3.378986345975204, + "tokens_seen": 2674917376 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015324373119358076, + "loss": 1.1312, + "theoretical_loss": 3.3789798864800153, + "tokens_seen": 2674982912 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015322768304914746, + "loss": 1.2138, + "theoretical_loss": 3.37897342718739, + "tokens_seen": 2675048448 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015321163490471416, + "loss": 1.2561, + "theoretical_loss": 3.3789669680973167, + "tokens_seen": 2675113984 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015319558676028084, + "loss": 1.1382, + "theoretical_loss": 3.378960509209784, + "tokens_seen": 2675179520 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015317953861584754, + "loss": 1.1922, + "theoretical_loss": 3.37895405052478, + "tokens_seen": 2675245056 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015316349047141424, + "loss": 1.1832, + "theoretical_loss": 3.378947592042295, + "tokens_seen": 2675310592 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015314744232698094, + "loss": 1.2474, + "theoretical_loss": 3.378941133762316, + "tokens_seen": 2675376128 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015313139418254764, + "loss": 1.1666, + "theoretical_loss": 3.3789346756848326, + "tokens_seen": 2675441664 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015311534603811434, + "loss": 1.1679, + "theoretical_loss": 3.3789282178098334, + "tokens_seen": 2675507200 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015309929789368104, + "loss": 1.1847, + "theoretical_loss": 3.378921760137307, + "tokens_seen": 2675572736 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015308324974924774, + "loss": 1.2007, + "theoretical_loss": 3.3789153026672416, + "tokens_seen": 2675638272 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015306720160481444, + "loss": 1.1813, + "theoretical_loss": 3.3789088453996268, + "tokens_seen": 2675703808 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015305115346038114, + "loss": 1.1668, + "theoretical_loss": 3.3789023883344504, + "tokens_seen": 2675769344 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015303510531594784, + "loss": 1.1257, + "theoretical_loss": 3.378895931471702, + "tokens_seen": 2675834880 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015301905717151455, + "loss": 1.1044, + "theoretical_loss": 3.3788894748113694, + "tokens_seen": 2675900416 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015300300902708125, + "loss": 1.1715, + "theoretical_loss": 3.378883018353442, + "tokens_seen": 2675965952 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015298696088264795, + "loss": 1.2333, + "theoretical_loss": 3.378876562097908, + "tokens_seen": 2676031488 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015297091273821465, + "loss": 1.1813, + "theoretical_loss": 3.378870106044757, + "tokens_seen": 2676097024 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015295486459378135, + "loss": 1.1264, + "theoretical_loss": 3.3788636501939764, + "tokens_seen": 2676162560 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015293881644934805, + "loss": 1.1886, + "theoretical_loss": 3.3788571945455557, + "tokens_seen": 2676228096 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015292276830491475, + "loss": 1.2531, + "theoretical_loss": 3.3788507390994833, + "tokens_seen": 2676293632 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015290672016048145, + "loss": 1.1192, + "theoretical_loss": 3.378844283855748, + "tokens_seen": 2676359168 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015289067201604815, + "loss": 1.1889, + "theoretical_loss": 3.378837828814339, + "tokens_seen": 2676424704 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015287462387161485, + "loss": 1.0671, + "theoretical_loss": 3.378831373975244, + "tokens_seen": 2676490240 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015285857572718155, + "loss": 1.3054, + "theoretical_loss": 3.3788249193384523, + "tokens_seen": 2676555776 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015284252758274825, + "loss": 1.2227, + "theoretical_loss": 3.378818464903953, + "tokens_seen": 2676621312 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015282647943831495, + "loss": 1.2266, + "theoretical_loss": 3.3788120106717336, + "tokens_seen": 2676686848 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015281043129388165, + "loss": 1.1294, + "theoretical_loss": 3.378805556641784, + "tokens_seen": 2676752384 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015279438314944835, + "loss": 1.2001, + "theoretical_loss": 3.3787991028140927, + "tokens_seen": 2676817920 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015277833500501505, + "loss": 1.154, + "theoretical_loss": 3.378792649188648, + "tokens_seen": 2676883456 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015276228686058176, + "loss": 1.2067, + "theoretical_loss": 3.378786195765439, + "tokens_seen": 2676948992 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015274623871614846, + "loss": 1.2272, + "theoretical_loss": 3.378779742544454, + "tokens_seen": 2677014528 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015273019057171516, + "loss": 1.1669, + "theoretical_loss": 3.378773289525682, + "tokens_seen": 2677080064 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 894532, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0622397661209106, + "objective/train/theoretical_loss": 3.3787668367091115, + "objective/train/tokens_used": 2697605600, + "theoretical_loss": 3.3787668367091115, + "tokens_seen": 2677145600 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015271414242728186, + "loss": 1.1195, + "theoretical_loss": 3.3787668367091115, + "tokens_seen": 2677145600 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015269809428284856, + "loss": 1.1758, + "theoretical_loss": 3.3787603840947313, + "tokens_seen": 2677211136 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015268204613841526, + "loss": 1.139, + "theoretical_loss": 3.37875393168253, + "tokens_seen": 2677276672 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015266599799398196, + "loss": 1.2394, + "theoretical_loss": 3.378747479472497, + "tokens_seen": 2677342208 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015264994984954866, + "loss": 1.2118, + "theoretical_loss": 3.37874102746462, + "tokens_seen": 2677407744 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015263390170511536, + "loss": 1.2179, + "theoretical_loss": 3.3787345756588887, + "tokens_seen": 2677473280 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015261785356068206, + "loss": 1.1907, + "theoretical_loss": 3.378728124055291, + "tokens_seen": 2677538816 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015260180541624876, + "loss": 1.2283, + "theoretical_loss": 3.378721672653816, + "tokens_seen": 2677604352 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015258575727181546, + "loss": 1.1382, + "theoretical_loss": 3.3787152214544527, + "tokens_seen": 2677669888 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015256970912738216, + "loss": 1.1672, + "theoretical_loss": 3.378708770457189, + "tokens_seen": 2677735424 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015255366098294886, + "loss": 1.2635, + "theoretical_loss": 3.3787023196620147, + "tokens_seen": 2677800960 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015253761283851556, + "loss": 1.1509, + "theoretical_loss": 3.3786958690689177, + "tokens_seen": 2677866496 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015252156469408226, + "loss": 1.0911, + "theoretical_loss": 3.3786894186778866, + "tokens_seen": 2677932032 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015250551654964897, + "loss": 1.1986, + "theoretical_loss": 3.378682968488911, + "tokens_seen": 2677997568 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015248946840521564, + "loss": 1.1815, + "theoretical_loss": 3.3786765185019787, + "tokens_seen": 2678063104 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015247342026078234, + "loss": 1.1053, + "theoretical_loss": 3.3786700687170788, + "tokens_seen": 2678128640 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015245737211634904, + "loss": 1.1625, + "theoretical_loss": 3.3786636191342003, + "tokens_seen": 2678194176 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015244132397191574, + "loss": 1.0731, + "theoretical_loss": 3.378657169753332, + "tokens_seen": 2678259712 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015242527582748244, + "loss": 1.1755, + "theoretical_loss": 3.378650720574462, + "tokens_seen": 2678325248 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015240922768304914, + "loss": 1.2146, + "theoretical_loss": 3.3786442715975795, + "tokens_seen": 2678390784 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015239317953861584, + "loss": 1.2985, + "theoretical_loss": 3.378637822822673, + "tokens_seen": 2678456320 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015237713139418254, + "loss": 1.1111, + "theoretical_loss": 3.378631374249731, + "tokens_seen": 2678521856 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015236108324974924, + "loss": 1.2523, + "theoretical_loss": 3.3786249258787433, + "tokens_seen": 2678587392 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015234503510531594, + "loss": 1.2252, + "theoretical_loss": 3.3786184777096975, + "tokens_seen": 2678652928 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015232898696088265, + "loss": 1.1574, + "theoretical_loss": 3.3786120297425826, + "tokens_seen": 2678718464 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015231293881644935, + "loss": 1.2518, + "theoretical_loss": 3.3786055819773875, + "tokens_seen": 2678784000 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015229689067201605, + "loss": 1.2442, + "theoretical_loss": 3.378599134414101, + "tokens_seen": 2678849536 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015228084252758275, + "loss": 1.1811, + "theoretical_loss": 3.378592687052712, + "tokens_seen": 2678915072 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015226479438314945, + "loss": 1.0941, + "theoretical_loss": 3.3785862398932087, + "tokens_seen": 2678980608 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015224874623871618, + "loss": 1.2741, + "theoretical_loss": 3.37857979293558, + "tokens_seen": 2679046144 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015223269809428288, + "loss": 1.129, + "theoretical_loss": 3.3785733461798153, + "tokens_seen": 2679111680 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015221664994984958, + "loss": 1.211, + "theoretical_loss": 3.3785668996259024, + "tokens_seen": 2679177216 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015220060180541628, + "loss": 1.1888, + "theoretical_loss": 3.3785604532738303, + "tokens_seen": 2679242752 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015218455366098298, + "loss": 1.2175, + "theoretical_loss": 3.378554007123588, + "tokens_seen": 2679308288 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015216850551654965, + "loss": 1.1521, + "theoretical_loss": 3.3785475611751643, + "tokens_seen": 2679373824 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015215245737211635, + "loss": 1.1798, + "theoretical_loss": 3.378541115428548, + "tokens_seen": 2679439360 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015213640922768305, + "loss": 1.2244, + "theoretical_loss": 3.3785346698837273, + "tokens_seen": 2679504896 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015212036108324975, + "loss": 1.1259, + "theoretical_loss": 3.378528224540691, + "tokens_seen": 2679570432 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015210431293881645, + "loss": 1.2075, + "theoretical_loss": 3.3785217793994287, + "tokens_seen": 2679635968 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015208826479438315, + "loss": 1.2615, + "theoretical_loss": 3.3785153344599284, + "tokens_seen": 2679701504 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015207221664994986, + "loss": 1.3073, + "theoretical_loss": 3.3785088897221787, + "tokens_seen": 2679767040 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015205616850551656, + "loss": 1.276, + "theoretical_loss": 3.378502445186169, + "tokens_seen": 2679832576 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015204012036108326, + "loss": 1.1028, + "theoretical_loss": 3.3784960008518876, + "tokens_seen": 2679898112 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015202407221664996, + "loss": 1.1543, + "theoretical_loss": 3.378489556719323, + "tokens_seen": 2679963648 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015200802407221666, + "loss": 1.0659, + "theoretical_loss": 3.3784831127884654, + "tokens_seen": 2680029184 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015199197592778336, + "loss": 1.2011, + "theoretical_loss": 3.3784766690593013, + "tokens_seen": 2680094720 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015197592778335006, + "loss": 1.1619, + "theoretical_loss": 3.3784702255318213, + "tokens_seen": 2680160256 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015195987963891676, + "loss": 1.3099, + "theoretical_loss": 3.3784637822060137, + "tokens_seen": 2680225792 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015194383149448346, + "loss": 1.1626, + "theoretical_loss": 3.3784573390818666, + "tokens_seen": 2680291328 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015192778335005016, + "loss": 1.0881, + "theoretical_loss": 3.3784508961593693, + "tokens_seen": 2680356864 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 895474, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1890168190002441, + "objective/train/theoretical_loss": 3.3784444534385103, + "objective/train/tokens_used": 2700882400, + "theoretical_loss": 3.3784444534385103, + "tokens_seen": 2680422400 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015191173520561686, + "loss": 1.2701, + "theoretical_loss": 3.3784444534385103, + "tokens_seen": 2680422400 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015189568706118356, + "loss": 1.1488, + "theoretical_loss": 3.378438010919279, + "tokens_seen": 2680487936 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015187963891675026, + "loss": 1.1861, + "theoretical_loss": 3.3784315686016635, + "tokens_seen": 2680553472 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015186359077231696, + "loss": 1.2713, + "theoretical_loss": 3.378425126485652, + "tokens_seen": 2680619008 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015184754262788366, + "loss": 1.1398, + "theoretical_loss": 3.378418684571235, + "tokens_seen": 2680684544 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015183149448345037, + "loss": 1.2229, + "theoretical_loss": 3.3784122428584, + "tokens_seen": 2680750080 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015181544633901707, + "loss": 1.1243, + "theoretical_loss": 3.378405801347136, + "tokens_seen": 2680815616 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015179939819458377, + "loss": 1.2315, + "theoretical_loss": 3.378399360037432, + "tokens_seen": 2680881152 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015178335005015044, + "loss": 1.2723, + "theoretical_loss": 3.378392918929276, + "tokens_seen": 2680946688 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015176730190571714, + "loss": 1.2152, + "theoretical_loss": 3.378386478022658, + "tokens_seen": 2681012224 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015175125376128384, + "loss": 1.1618, + "theoretical_loss": 3.3783800373175654, + "tokens_seen": 2681077760 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015173520561685054, + "loss": 1.1644, + "theoretical_loss": 3.3783735968139883, + "tokens_seen": 2681143296 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015171915747241724, + "loss": 1.1162, + "theoretical_loss": 3.3783671565119144, + "tokens_seen": 2681208832 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015170310932798394, + "loss": 1.1233, + "theoretical_loss": 3.3783607164113336, + "tokens_seen": 2681274368 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015168706118355064, + "loss": 1.2275, + "theoretical_loss": 3.3783542765122334, + "tokens_seen": 2681339904 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015167101303911734, + "loss": 1.2182, + "theoretical_loss": 3.378347836814603, + "tokens_seen": 2681405440 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015165496489468404, + "loss": 1.1264, + "theoretical_loss": 3.3783413973184317, + "tokens_seen": 2681470976 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015163891675025077, + "loss": 1.2179, + "theoretical_loss": 3.378334958023708, + "tokens_seen": 2681536512 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015162286860581747, + "loss": 1.1818, + "theoretical_loss": 3.37832851893042, + "tokens_seen": 2681602048 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015160682046138417, + "loss": 1.0645, + "theoretical_loss": 3.378322080038558, + "tokens_seen": 2681667584 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015159077231695087, + "loss": 1.181, + "theoretical_loss": 3.378315641348109, + "tokens_seen": 2681733120 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015157472417251758, + "loss": 1.1715, + "theoretical_loss": 3.378309202859063, + "tokens_seen": 2681798656 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015155867602808428, + "loss": 1.225, + "theoretical_loss": 3.3783027645714085, + "tokens_seen": 2681864192 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015154262788365098, + "loss": 1.0807, + "theoretical_loss": 3.378296326485134, + "tokens_seen": 2681929728 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015152657973921768, + "loss": 1.155, + "theoretical_loss": 3.378289888600228, + "tokens_seen": 2681995264 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015151053159478438, + "loss": 1.2065, + "theoretical_loss": 3.3782834509166806, + "tokens_seen": 2682060800 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015149448345035108, + "loss": 1.215, + "theoretical_loss": 3.378277013434479, + "tokens_seen": 2682126336 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015147843530591775, + "loss": 1.0857, + "theoretical_loss": 3.378270576153613, + "tokens_seen": 2682191872 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015146238716148445, + "loss": 1.1188, + "theoretical_loss": 3.378264139074071, + "tokens_seen": 2682257408 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015144633901705115, + "loss": 1.1953, + "theoretical_loss": 3.378257702195842, + "tokens_seen": 2682322944 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015143029087261785, + "loss": 1.1391, + "theoretical_loss": 3.3782512655189145, + "tokens_seen": 2682388480 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015141424272818455, + "loss": 1.1573, + "theoretical_loss": 3.3782448290432776, + "tokens_seen": 2682454016 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015139819458375126, + "loss": 1.1204, + "theoretical_loss": 3.3782383927689197, + "tokens_seen": 2682519552 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015138214643931796, + "loss": 1.202, + "theoretical_loss": 3.37823195669583, + "tokens_seen": 2682585088 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015136609829488466, + "loss": 1.193, + "theoretical_loss": 3.378225520823997, + "tokens_seen": 2682650624 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015135005015045136, + "loss": 1.277, + "theoretical_loss": 3.3782190851534097, + "tokens_seen": 2682716160 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015133400200601806, + "loss": 1.1559, + "theoretical_loss": 3.3782126496840568, + "tokens_seen": 2682781696 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015131795386158476, + "loss": 1.1737, + "theoretical_loss": 3.378206214415927, + "tokens_seen": 2682847232 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015130190571715146, + "loss": 1.1538, + "theoretical_loss": 3.378199779349009, + "tokens_seen": 2682912768 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015128585757271816, + "loss": 1.2794, + "theoretical_loss": 3.378193344483292, + "tokens_seen": 2682978304 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015126980942828486, + "loss": 1.2486, + "theoretical_loss": 3.3781869098187647, + "tokens_seen": 2683043840 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015125376128385156, + "loss": 1.1823, + "theoretical_loss": 3.3781804753554154, + "tokens_seen": 2683109376 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015123771313941826, + "loss": 1.2869, + "theoretical_loss": 3.3781740410932333, + "tokens_seen": 2683174912 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015122166499498496, + "loss": 1.2124, + "theoretical_loss": 3.378167607032207, + "tokens_seen": 2683240448 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015120561685055166, + "loss": 1.1797, + "theoretical_loss": 3.378161173172326, + "tokens_seen": 2683305984 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015118956870611836, + "loss": 1.1283, + "theoretical_loss": 3.3781547395135783, + "tokens_seen": 2683371520 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015117352056168506, + "loss": 1.1802, + "theoretical_loss": 3.378148306055953, + "tokens_seen": 2683437056 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015115747241725176, + "loss": 1.2079, + "theoretical_loss": 3.3781418727994383, + "tokens_seen": 2683502592 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015114142427281847, + "loss": 1.2493, + "theoretical_loss": 3.378135439744024, + "tokens_seen": 2683568128 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015112537612838517, + "loss": 1.2005, + "theoretical_loss": 3.378129006889699, + "tokens_seen": 2683633664 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 896709, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2746180295944214, + "objective/train/theoretical_loss": 3.3781225742364507, + "objective/train/tokens_used": 2704159200, + "theoretical_loss": 3.3781225742364507, + "tokens_seen": 2683699200 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015110932798395187, + "loss": 1.1525, + "theoretical_loss": 3.3781225742364507, + "tokens_seen": 2683699200 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015109327983951854, + "loss": 1.2235, + "theoretical_loss": 3.378116141784269, + "tokens_seen": 2683764736 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015107723169508524, + "loss": 1.2567, + "theoretical_loss": 3.3781097095331427, + "tokens_seen": 2683830272 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015106118355065194, + "loss": 1.1953, + "theoretical_loss": 3.37810327748306, + "tokens_seen": 2683895808 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015104513540621867, + "loss": 1.1573, + "theoretical_loss": 3.3780968456340106, + "tokens_seen": 2683961344 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015102908726178537, + "loss": 1.198, + "theoretical_loss": 3.378090413985982, + "tokens_seen": 2684026880 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015101303911735207, + "loss": 1.2556, + "theoretical_loss": 3.3780839825389646, + "tokens_seen": 2684092416 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015099699097291877, + "loss": 1.0749, + "theoretical_loss": 3.378077551292946, + "tokens_seen": 2684157952 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015098094282848547, + "loss": 1.1716, + "theoretical_loss": 3.3780711202479154, + "tokens_seen": 2684223488 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015096489468405217, + "loss": 1.2793, + "theoretical_loss": 3.3780646894038617, + "tokens_seen": 2684289024 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015094884653961887, + "loss": 1.1219, + "theoretical_loss": 3.378058258760774, + "tokens_seen": 2684354560 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015093279839518557, + "loss": 1.2208, + "theoretical_loss": 3.37805182831864, + "tokens_seen": 2684420096 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015091675025075227, + "loss": 1.2486, + "theoretical_loss": 3.37804539807745, + "tokens_seen": 2684485632 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015090070210631897, + "loss": 1.1086, + "theoretical_loss": 3.378038968037192, + "tokens_seen": 2684551168 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015088465396188568, + "loss": 1.1932, + "theoretical_loss": 3.3780325381978544, + "tokens_seen": 2684616704 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015086860581745238, + "loss": 1.1518, + "theoretical_loss": 3.3780261085594265, + "tokens_seen": 2684682240 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015085255767301908, + "loss": 1.3016, + "theoretical_loss": 3.3780196791218975, + "tokens_seen": 2684747776 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015083650952858578, + "loss": 1.1794, + "theoretical_loss": 3.3780132498852558, + "tokens_seen": 2684813312 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015082046138415248, + "loss": 1.106, + "theoretical_loss": 3.3780068208494907, + "tokens_seen": 2684878848 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015080441323971918, + "loss": 1.201, + "theoretical_loss": 3.37800039201459, + "tokens_seen": 2684944384 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015078836509528588, + "loss": 1.2188, + "theoretical_loss": 3.3779939633805434, + "tokens_seen": 2685009920 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015077231695085255, + "loss": 1.1496, + "theoretical_loss": 3.377987534947339, + "tokens_seen": 2685075456 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015075626880641925, + "loss": 1.2836, + "theoretical_loss": 3.3779811067149668, + "tokens_seen": 2685140992 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015074022066198595, + "loss": 1.2908, + "theoretical_loss": 3.3779746786834144, + "tokens_seen": 2685206528 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015072417251755265, + "loss": 1.1808, + "theoretical_loss": 3.377968250852671, + "tokens_seen": 2685272064 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015070812437311936, + "loss": 1.2704, + "theoretical_loss": 3.3779618232227255, + "tokens_seen": 2685337600 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015069207622868606, + "loss": 1.2257, + "theoretical_loss": 3.3779553957935673, + "tokens_seen": 2685403136 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015067602808425276, + "loss": 1.2448, + "theoretical_loss": 3.3779489685651845, + "tokens_seen": 2685468672 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015065997993981946, + "loss": 1.1891, + "theoretical_loss": 3.377942541537566, + "tokens_seen": 2685534208 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015064393179538616, + "loss": 1.183, + "theoretical_loss": 3.3779361147107005, + "tokens_seen": 2685599744 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015062788365095286, + "loss": 1.1149, + "theoretical_loss": 3.3779296880845777, + "tokens_seen": 2685665280 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015061183550651956, + "loss": 1.2306, + "theoretical_loss": 3.3779232616591854, + "tokens_seen": 2685730816 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015059578736208626, + "loss": 1.3616, + "theoretical_loss": 3.377916835434513, + "tokens_seen": 2685796352 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015057973921765296, + "loss": 1.3026, + "theoretical_loss": 3.377910409410549, + "tokens_seen": 2685861888 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015056369107321966, + "loss": 1.1863, + "theoretical_loss": 3.3779039835872826, + "tokens_seen": 2685927424 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015054764292878636, + "loss": 1.0323, + "theoretical_loss": 3.3778975579647024, + "tokens_seen": 2685992960 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015053159478435306, + "loss": 1.1435, + "theoretical_loss": 3.377891132542797, + "tokens_seen": 2686058496 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015051554663991976, + "loss": 1.1575, + "theoretical_loss": 3.377884707321556, + "tokens_seen": 2686124032 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015049949849548646, + "loss": 1.257, + "theoretical_loss": 3.3778782823009674, + "tokens_seen": 2686189568 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015048345035105316, + "loss": 1.2069, + "theoretical_loss": 3.3778718574810207, + "tokens_seen": 2686255104 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015046740220661986, + "loss": 1.2016, + "theoretical_loss": 3.3778654328617046, + "tokens_seen": 2686320640 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015045135406218657, + "loss": 1.1937, + "theoretical_loss": 3.377859008443007, + "tokens_seen": 2686386176 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015043530591775327, + "loss": 1.1897, + "theoretical_loss": 3.377852584224918, + "tokens_seen": 2686451712 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015041925777331997, + "loss": 1.1147, + "theoretical_loss": 3.3778461602074263, + "tokens_seen": 2686517248 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015040320962888667, + "loss": 1.1743, + "theoretical_loss": 3.37783973639052, + "tokens_seen": 2686582784 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015038716148445337, + "loss": 1.2473, + "theoretical_loss": 3.3778333127741886, + "tokens_seen": 2686648320 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015037111334002007, + "loss": 1.2981, + "theoretical_loss": 3.3778268893584205, + "tokens_seen": 2686713856 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015035506519558677, + "loss": 1.2995, + "theoretical_loss": 3.3778204661432047, + "tokens_seen": 2686779392 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015033901705115347, + "loss": 1.1811, + "theoretical_loss": 3.37781404312853, + "tokens_seen": 2686844928 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015032296890672017, + "loss": 1.2448, + "theoretical_loss": 3.3778076203143854, + "tokens_seen": 2686910464 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 897611, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2123100757598877, + "objective/train/theoretical_loss": 3.37780119770076, + "objective/train/tokens_used": 2707436000, + "theoretical_loss": 3.37780119770076, + "tokens_seen": 2686976000 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015030692076228687, + "loss": 1.1904, + "theoretical_loss": 3.37780119770076, + "tokens_seen": 2686976000 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015029087261785357, + "loss": 1.1497, + "theoretical_loss": 3.3777947752876423, + "tokens_seen": 2687041536 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015027482447342027, + "loss": 1.178, + "theoretical_loss": 3.3777883530750206, + "tokens_seen": 2687107072 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015025877632898697, + "loss": 1.1245, + "theoretical_loss": 3.377781931062885, + "tokens_seen": 2687172608 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015024272818455367, + "loss": 1.308, + "theoretical_loss": 3.377775509251223, + "tokens_seen": 2687238144 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015022668004012037, + "loss": 1.2342, + "theoretical_loss": 3.377769087640025, + "tokens_seen": 2687303680 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015021063189568707, + "loss": 1.2575, + "theoretical_loss": 3.3777626662292786, + "tokens_seen": 2687369216 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015019458375125378, + "loss": 1.1849, + "theoretical_loss": 3.377756245018973, + "tokens_seen": 2687434752 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015017853560682048, + "loss": 1.1438, + "theoretical_loss": 3.377749824009097, + "tokens_seen": 2687500288 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015016248746238718, + "loss": 1.2122, + "theoretical_loss": 3.3777434031996396, + "tokens_seen": 2687565824 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015014643931795388, + "loss": 1.1562, + "theoretical_loss": 3.37773698259059, + "tokens_seen": 2687631360 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015013039117352058, + "loss": 1.1824, + "theoretical_loss": 3.3777305621819362, + "tokens_seen": 2687696896 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015011434302908728, + "loss": 1.228, + "theoretical_loss": 3.3777241419736677, + "tokens_seen": 2687762432 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015009829488465398, + "loss": 1.2264, + "theoretical_loss": 3.377717721965773, + "tokens_seen": 2687827968 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015008224674022068, + "loss": 1.18, + "theoretical_loss": 3.3777113021582417, + "tokens_seen": 2687893504 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015006619859578735, + "loss": 1.1753, + "theoretical_loss": 3.3777048825510616, + "tokens_seen": 2687959040 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015005015045135405, + "loss": 1.0971, + "theoretical_loss": 3.377698463144222, + "tokens_seen": 2688024576 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015003410230692075, + "loss": 1.1481, + "theoretical_loss": 3.3776920439377127, + "tokens_seen": 2688090112 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015001805416248746, + "loss": 1.1975, + "theoretical_loss": 3.377685624931521, + "tokens_seen": 2688155648 + }, + { + "epoch": 0.81, + "learning_rate": 0.00015000200601805416, + "loss": 1.1237, + "theoretical_loss": 3.377679206125636, + "tokens_seen": 2688221184 + }, + { + "epoch": 0.81, + "learning_rate": 0.00014998595787362086, + "loss": 1.1474, + "theoretical_loss": 3.377672787520048, + "tokens_seen": 2688286720 + }, + { + "epoch": 0.81, + "learning_rate": 0.00014996990972918756, + "loss": 1.1087, + "theoretical_loss": 3.3776663691147446, + "tokens_seen": 2688352256 + }, + { + "epoch": 0.81, + "learning_rate": 0.00014995386158475426, + "loss": 1.112, + "theoretical_loss": 3.377659950909715, + "tokens_seen": 2688417792 + }, + { + "epoch": 0.81, + "learning_rate": 0.00014993781344032096, + "loss": 1.2853, + "theoretical_loss": 3.377653532904948, + "tokens_seen": 2688483328 + }, + { + "epoch": 0.81, + "learning_rate": 0.00014992176529588766, + "loss": 1.2343, + "theoretical_loss": 3.3776471151004324, + "tokens_seen": 2688548864 + }, + { + "epoch": 0.81, + "learning_rate": 0.00014990571715145436, + "loss": 1.2115, + "theoretical_loss": 3.3776406974961573, + "tokens_seen": 2688614400 + }, + { + "epoch": 0.81, + "learning_rate": 0.00014988966900702106, + "loss": 1.3188, + "theoretical_loss": 3.3776342800921113, + "tokens_seen": 2688679936 + }, + { + "epoch": 0.81, + "learning_rate": 0.00014987362086258776, + "loss": 1.303, + "theoretical_loss": 3.377627862888284, + "tokens_seen": 2688745472 + }, + { + "epoch": 0.81, + "learning_rate": 0.00014985757271815446, + "loss": 1.2393, + "theoretical_loss": 3.377621445884663, + "tokens_seen": 2688811008 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001498415245737212, + "loss": 1.2312, + "theoretical_loss": 3.3776150290812383, + "tokens_seen": 2688876544 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001498254764292879, + "loss": 1.2601, + "theoretical_loss": 3.377608612477998, + "tokens_seen": 2688942080 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001498094282848546, + "loss": 1.2374, + "theoretical_loss": 3.377602196074932, + "tokens_seen": 2689007616 + }, + { + "epoch": 0.81, + "learning_rate": 0.0001497933801404213, + "loss": 1.3204, + "theoretical_loss": 3.3775957798720277, + "tokens_seen": 2689073152 + }, + { + "epoch": 0.81, + "learning_rate": 0.000149777331995988, + "loss": 1.2637, + "theoretical_loss": 3.377589363869275, + "tokens_seen": 2689138688 + }, + { + "epoch": 0.81, + "learning_rate": 0.00014976128385155467, + "loss": 1.1575, + "theoretical_loss": 3.377582948066663, + "tokens_seen": 2689204224 + }, + { + "epoch": 0.81, + "learning_rate": 0.00014974523570712137, + "loss": 1.2405, + "theoretical_loss": 3.3775765324641793, + "tokens_seen": 2689269760 + }, + { + "epoch": 0.81, + "learning_rate": 0.00014972918756268807, + "loss": 1.1301, + "theoretical_loss": 3.3775701170618144, + "tokens_seen": 2689335296 + }, + { + "epoch": 0.81, + "learning_rate": 0.00014971313941825477, + "loss": 1.0555, + "theoretical_loss": 3.377563701859556, + "tokens_seen": 2689400832 + }, + { + "epoch": 0.81, + "learning_rate": 0.00014969709127382147, + "loss": 1.2499, + "theoretical_loss": 3.3775572868573938, + "tokens_seen": 2689466368 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014968104312938817, + "loss": 1.1278, + "theoretical_loss": 3.377550872055316, + "tokens_seen": 2689531904 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014966499498495487, + "loss": 1.1656, + "theoretical_loss": 3.3775444574533116, + "tokens_seen": 2689597440 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014964894684052157, + "loss": 1.2786, + "theoretical_loss": 3.37753804305137, + "tokens_seen": 2689662976 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014963289869608827, + "loss": 1.2958, + "theoretical_loss": 3.3775316288494794, + "tokens_seen": 2689728512 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014961685055165497, + "loss": 1.1895, + "theoretical_loss": 3.377525214847629, + "tokens_seen": 2689794048 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014960080240722167, + "loss": 1.1692, + "theoretical_loss": 3.3775188010458077, + "tokens_seen": 2689859584 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014958475426278837, + "loss": 1.1739, + "theoretical_loss": 3.3775123874440047, + "tokens_seen": 2689925120 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014956870611835507, + "loss": 1.2225, + "theoretical_loss": 3.3775059740422084, + "tokens_seen": 2689990656 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014955265797392177, + "loss": 1.1577, + "theoretical_loss": 3.377499560840408, + "tokens_seen": 2690056192 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014953660982948847, + "loss": 1.1732, + "theoretical_loss": 3.3774931478385923, + "tokens_seen": 2690121728 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014952056168505518, + "loss": 1.2189, + "theoretical_loss": 3.37748673503675, + "tokens_seen": 2690187264 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 898800, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1520065069198608, + "objective/train/theoretical_loss": 3.37748032243487, + "objective/train/tokens_used": 2710712800, + "theoretical_loss": 3.37748032243487, + "tokens_seen": 2690252800 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014950451354062188, + "loss": 1.3682, + "theoretical_loss": 3.37748032243487, + "tokens_seen": 2690252800 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014948846539618858, + "loss": 1.2475, + "theoretical_loss": 3.3774739100329416, + "tokens_seen": 2690318336 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014947241725175528, + "loss": 1.2714, + "theoretical_loss": 3.377467497830953, + "tokens_seen": 2690383872 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014945636910732198, + "loss": 1.1274, + "theoretical_loss": 3.377461085828894, + "tokens_seen": 2690449408 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014944032096288868, + "loss": 1.286, + "theoretical_loss": 3.377454674026753, + "tokens_seen": 2690514944 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014942427281845538, + "loss": 1.2902, + "theoretical_loss": 3.377448262424519, + "tokens_seen": 2690580480 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014940822467402208, + "loss": 1.2034, + "theoretical_loss": 3.3774418510221804, + "tokens_seen": 2690646016 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014939217652958878, + "loss": 1.2155, + "theoretical_loss": 3.3774354398197266, + "tokens_seen": 2690711552 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014937612838515545, + "loss": 1.16, + "theoretical_loss": 3.3774290288171467, + "tokens_seen": 2690777088 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014936008024072215, + "loss": 1.1729, + "theoretical_loss": 3.377422618014429, + "tokens_seen": 2690842624 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014934403209628885, + "loss": 1.2256, + "theoretical_loss": 3.377416207411563, + "tokens_seen": 2690908160 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014932798395185556, + "loss": 1.1849, + "theoretical_loss": 3.377409797008537, + "tokens_seen": 2690973696 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014931193580742226, + "loss": 1.2887, + "theoretical_loss": 3.377403386805341, + "tokens_seen": 2691039232 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014929588766298896, + "loss": 1.1997, + "theoretical_loss": 3.377396976801962, + "tokens_seen": 2691104768 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014927983951855566, + "loss": 1.3001, + "theoretical_loss": 3.3773905669983906, + "tokens_seen": 2691170304 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014926379137412236, + "loss": 1.1779, + "theoretical_loss": 3.3773841573946153, + "tokens_seen": 2691235840 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014924774322968909, + "loss": 1.2922, + "theoretical_loss": 3.3773777479906246, + "tokens_seen": 2691301376 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001492316950852558, + "loss": 1.1935, + "theoretical_loss": 3.377371338786408, + "tokens_seen": 2691366912 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001492156469408225, + "loss": 1.171, + "theoretical_loss": 3.3773649297819537, + "tokens_seen": 2691432448 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001491995987963892, + "loss": 1.3049, + "theoretical_loss": 3.377358520977251, + "tokens_seen": 2691497984 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001491835506519559, + "loss": 1.3338, + "theoretical_loss": 3.3773521123722894, + "tokens_seen": 2691563520 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001491675025075226, + "loss": 1.115, + "theoretical_loss": 3.3773457039670562, + "tokens_seen": 2691629056 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001491514543630893, + "loss": 1.185, + "theoretical_loss": 3.377339295761542, + "tokens_seen": 2691694592 + }, + { + "epoch": 0.82, + "learning_rate": 0.000149135406218656, + "loss": 1.2223, + "theoretical_loss": 3.3773328877557347, + "tokens_seen": 2691760128 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001491193580742227, + "loss": 1.1319, + "theoretical_loss": 3.377326479949624, + "tokens_seen": 2691825664 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001491033099297894, + "loss": 1.139, + "theoretical_loss": 3.377320072343198, + "tokens_seen": 2691891200 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001490872617853561, + "loss": 1.1671, + "theoretical_loss": 3.3773136649364464, + "tokens_seen": 2691956736 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001490712136409228, + "loss": 1.2245, + "theoretical_loss": 3.377307257729357, + "tokens_seen": 2692022272 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014905516549648947, + "loss": 1.1311, + "theoretical_loss": 3.3773008507219195, + "tokens_seen": 2692087808 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014903911735205617, + "loss": 1.3018, + "theoretical_loss": 3.3772944439141233, + "tokens_seen": 2692153344 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014902306920762287, + "loss": 1.1431, + "theoretical_loss": 3.377288037305956, + "tokens_seen": 2692218880 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014900702106318957, + "loss": 1.1343, + "theoretical_loss": 3.377281630897408, + "tokens_seen": 2692284416 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014899097291875627, + "loss": 1.2423, + "theoretical_loss": 3.377275224688467, + "tokens_seen": 2692349952 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014897492477432297, + "loss": 1.2483, + "theoretical_loss": 3.3772688186791227, + "tokens_seen": 2692415488 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014895887662988967, + "loss": 1.1319, + "theoretical_loss": 3.3772624128693636, + "tokens_seen": 2692481024 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014894282848545637, + "loss": 1.1721, + "theoretical_loss": 3.3772560072591786, + "tokens_seen": 2692546560 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014892678034102307, + "loss": 1.2776, + "theoretical_loss": 3.377249601848557, + "tokens_seen": 2692612096 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014891073219658977, + "loss": 1.2492, + "theoretical_loss": 3.3772431966374876, + "tokens_seen": 2692677632 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014889468405215647, + "loss": 1.3014, + "theoretical_loss": 3.377236791625959, + "tokens_seen": 2692743168 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014887863590772317, + "loss": 1.1706, + "theoretical_loss": 3.3772303868139604, + "tokens_seen": 2692808704 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014886258776328987, + "loss": 1.2104, + "theoretical_loss": 3.377223982201481, + "tokens_seen": 2692874240 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014884653961885657, + "loss": 1.2153, + "theoretical_loss": 3.377217577788509, + "tokens_seen": 2692939776 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014883049147442328, + "loss": 1.2991, + "theoretical_loss": 3.377211173575034, + "tokens_seen": 2693005312 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014881444332998998, + "loss": 1.2906, + "theoretical_loss": 3.377204769561044, + "tokens_seen": 2693070848 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014879839518555668, + "loss": 1.2006, + "theoretical_loss": 3.3771983657465294, + "tokens_seen": 2693136384 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014878234704112338, + "loss": 1.3719, + "theoretical_loss": 3.3771919621314783, + "tokens_seen": 2693201920 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014876629889669008, + "loss": 1.2002, + "theoretical_loss": 3.3771855587158797, + "tokens_seen": 2693267456 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014875025075225678, + "loss": 1.1287, + "theoretical_loss": 3.3771791554997224, + "tokens_seen": 2693332992 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014873420260782348, + "loss": 1.2506, + "theoretical_loss": 3.377172752482995, + "tokens_seen": 2693398528 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014871815446339018, + "loss": 1.3165, + "theoretical_loss": 3.3771663496656874, + "tokens_seen": 2693464064 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 899825, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1790976524353027, + "objective/train/theoretical_loss": 3.377159947047788, + "objective/train/tokens_used": 2713989600, + "theoretical_loss": 3.377159947047788, + "tokens_seen": 2693529600 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014870210631895688, + "loss": 1.1677, + "theoretical_loss": 3.377159947047788, + "tokens_seen": 2693529600 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014868605817452358, + "loss": 1.1929, + "theoretical_loss": 3.3771535446292855, + "tokens_seen": 2693595136 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014867001003009025, + "loss": 1.2951, + "theoretical_loss": 3.3771471424101693, + "tokens_seen": 2693660672 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014865396188565696, + "loss": 1.1206, + "theoretical_loss": 3.377140740390428, + "tokens_seen": 2693726208 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014863791374122368, + "loss": 1.1631, + "theoretical_loss": 3.377134338570051, + "tokens_seen": 2693791744 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014862186559679038, + "loss": 1.1869, + "theoretical_loss": 3.3771279369490266, + "tokens_seen": 2693857280 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014860581745235708, + "loss": 1.1891, + "theoretical_loss": 3.377121535527344, + "tokens_seen": 2693922816 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014858976930792378, + "loss": 1.2594, + "theoretical_loss": 3.3771151343049923, + "tokens_seen": 2693988352 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014857372116349049, + "loss": 1.1632, + "theoretical_loss": 3.3771087332819603, + "tokens_seen": 2694053888 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014855767301905719, + "loss": 1.1427, + "theoretical_loss": 3.377102332458237, + "tokens_seen": 2694119424 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001485416248746239, + "loss": 1.1802, + "theoretical_loss": 3.3770959318338116, + "tokens_seen": 2694184960 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001485255767301906, + "loss": 1.2225, + "theoretical_loss": 3.377089531408673, + "tokens_seen": 2694250496 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001485095285857573, + "loss": 1.0897, + "theoretical_loss": 3.3770831311828093, + "tokens_seen": 2694316032 + }, + { + "epoch": 0.82, + "learning_rate": 0.000148493480441324, + "loss": 1.1857, + "theoretical_loss": 3.3770767311562104, + "tokens_seen": 2694381568 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001484774322968907, + "loss": 1.1358, + "theoretical_loss": 3.377070331328865, + "tokens_seen": 2694447104 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001484613841524574, + "loss": 1.1814, + "theoretical_loss": 3.3770639317007616, + "tokens_seen": 2694512640 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001484453360080241, + "loss": 1.1853, + "theoretical_loss": 3.37705753227189, + "tokens_seen": 2694578176 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001484292878635908, + "loss": 1.2033, + "theoretical_loss": 3.3770511330422384, + "tokens_seen": 2694643712 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001484132397191575, + "loss": 1.151, + "theoretical_loss": 3.3770447340117964, + "tokens_seen": 2694709248 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001483971915747242, + "loss": 1.2817, + "theoretical_loss": 3.3770383351805524, + "tokens_seen": 2694774784 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001483811434302909, + "loss": 1.1677, + "theoretical_loss": 3.3770319365484953, + "tokens_seen": 2694840320 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001483650952858576, + "loss": 1.186, + "theoretical_loss": 3.3770255381156145, + "tokens_seen": 2694905856 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014834904714142427, + "loss": 1.2618, + "theoretical_loss": 3.3770191398818987, + "tokens_seen": 2694971392 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014833299899699097, + "loss": 1.3476, + "theoretical_loss": 3.377012741847337, + "tokens_seen": 2695036928 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014831695085255767, + "loss": 1.2116, + "theoretical_loss": 3.3770063440119182, + "tokens_seen": 2695102464 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014830090270812437, + "loss": 1.2895, + "theoretical_loss": 3.3769999463756317, + "tokens_seen": 2695168000 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014828485456369107, + "loss": 1.2786, + "theoretical_loss": 3.376993548938466, + "tokens_seen": 2695233536 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014826880641925777, + "loss": 1.1877, + "theoretical_loss": 3.3769871517004098, + "tokens_seen": 2695299072 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014825275827482447, + "loss": 1.1886, + "theoretical_loss": 3.3769807546614525, + "tokens_seen": 2695364608 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014823671013039117, + "loss": 1.1761, + "theoretical_loss": 3.376974357821583, + "tokens_seen": 2695430144 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014822066198595787, + "loss": 1.2708, + "theoretical_loss": 3.3769679611807906, + "tokens_seen": 2695495680 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014820461384152457, + "loss": 1.2022, + "theoretical_loss": 3.3769615647390636, + "tokens_seen": 2695561216 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014818856569709127, + "loss": 1.1766, + "theoretical_loss": 3.376955168496391, + "tokens_seen": 2695626752 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014817251755265797, + "loss": 1.2457, + "theoretical_loss": 3.3769487724527627, + "tokens_seen": 2695692288 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014815646940822467, + "loss": 1.0995, + "theoretical_loss": 3.3769423766081665, + "tokens_seen": 2695757824 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014814042126379138, + "loss": 1.1479, + "theoretical_loss": 3.3769359809625925, + "tokens_seen": 2695823360 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014812437311935808, + "loss": 1.2598, + "theoretical_loss": 3.376929585516028, + "tokens_seen": 2695888896 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014810832497492478, + "loss": 1.1956, + "theoretical_loss": 3.3769231902684638, + "tokens_seen": 2695954432 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014809227683049148, + "loss": 1.2576, + "theoretical_loss": 3.376916795219888, + "tokens_seen": 2696019968 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014807622868605818, + "loss": 1.0836, + "theoretical_loss": 3.37691040037029, + "tokens_seen": 2696085504 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014806018054162488, + "loss": 1.3227, + "theoretical_loss": 3.3769040057196578, + "tokens_seen": 2696151040 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014804413239719158, + "loss": 1.2559, + "theoretical_loss": 3.376897611267981, + "tokens_seen": 2696216576 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014802808425275828, + "loss": 1.2302, + "theoretical_loss": 3.376891217015249, + "tokens_seen": 2696282112 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014801203610832498, + "loss": 1.2075, + "theoretical_loss": 3.37688482296145, + "tokens_seen": 2696347648 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014799598796389168, + "loss": 1.2854, + "theoretical_loss": 3.376878429106574, + "tokens_seen": 2696413184 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014797993981945838, + "loss": 1.3126, + "theoretical_loss": 3.3768720354506083, + "tokens_seen": 2696478720 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014796389167502508, + "loss": 1.309, + "theoretical_loss": 3.3768656419935437, + "tokens_seen": 2696544256 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014794784353059178, + "loss": 1.1647, + "theoretical_loss": 3.376859248735368, + "tokens_seen": 2696609792 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014793179538615848, + "loss": 1.1737, + "theoretical_loss": 3.376852855676071, + "tokens_seen": 2696675328 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014791574724172518, + "loss": 1.1981, + "theoretical_loss": 3.3768464628156405, + "tokens_seen": 2696740864 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 901033, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1442465782165527, + "objective/train/theoretical_loss": 3.3768400701540666, + "objective/train/tokens_used": 2717266400, + "theoretical_loss": 3.3768400701540666, + "tokens_seen": 2696806400 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014789969909729188, + "loss": 1.1773, + "theoretical_loss": 3.3768400701540666, + "tokens_seen": 2696806400 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014788365095285859, + "loss": 1.2814, + "theoretical_loss": 3.3768336776913377, + "tokens_seen": 2696871936 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014786760280842529, + "loss": 1.1843, + "theoretical_loss": 3.376827285427443, + "tokens_seen": 2696937472 + }, + { + "epoch": 0.82, + "learning_rate": 0.000147851554663992, + "loss": 1.2544, + "theoretical_loss": 3.376820893362372, + "tokens_seen": 2697003008 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001478355065195587, + "loss": 1.2214, + "theoretical_loss": 3.3768145014961126, + "tokens_seen": 2697068544 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001478194583751254, + "loss": 1.1183, + "theoretical_loss": 3.376808109828654, + "tokens_seen": 2697134080 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001478034102306921, + "loss": 1.1205, + "theoretical_loss": 3.376801718359986, + "tokens_seen": 2697199616 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001477873620862588, + "loss": 1.2301, + "theoretical_loss": 3.3767953270900977, + "tokens_seen": 2697265152 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001477713139418255, + "loss": 1.2236, + "theoretical_loss": 3.3767889360189765, + "tokens_seen": 2697330688 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001477552657973922, + "loss": 1.2134, + "theoretical_loss": 3.3767825451466127, + "tokens_seen": 2697396224 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001477392176529589, + "loss": 1.1231, + "theoretical_loss": 3.376776154472995, + "tokens_seen": 2697461760 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001477231695085256, + "loss": 1.28, + "theoretical_loss": 3.3767697639981122, + "tokens_seen": 2697527296 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001477071213640923, + "loss": 1.2311, + "theoretical_loss": 3.376763373721954, + "tokens_seen": 2697592832 + }, + { + "epoch": 0.82, + "learning_rate": 0.000147691073219659, + "loss": 1.2909, + "theoretical_loss": 3.3767569836445084, + "tokens_seen": 2697658368 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001476750250752257, + "loss": 1.2006, + "theoretical_loss": 3.376750593765765, + "tokens_seen": 2697723904 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014765897693079237, + "loss": 1.2348, + "theoretical_loss": 3.3767442040857123, + "tokens_seen": 2697789440 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014764292878635907, + "loss": 1.2337, + "theoretical_loss": 3.37673781460434, + "tokens_seen": 2697854976 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014762688064192577, + "loss": 1.2314, + "theoretical_loss": 3.3767314253216365, + "tokens_seen": 2697920512 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014761083249749247, + "loss": 1.1852, + "theoretical_loss": 3.376725036237591, + "tokens_seen": 2697986048 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014759478435305917, + "loss": 1.2262, + "theoretical_loss": 3.3767186473521926, + "tokens_seen": 2698051584 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014757873620862587, + "loss": 1.1895, + "theoretical_loss": 3.37671225866543, + "tokens_seen": 2698117120 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014756268806419257, + "loss": 1.3109, + "theoretical_loss": 3.3767058701772927, + "tokens_seen": 2698182656 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014754663991975927, + "loss": 1.1185, + "theoretical_loss": 3.3766994818877696, + "tokens_seen": 2698248192 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014753059177532597, + "loss": 1.1702, + "theoretical_loss": 3.376693093796849, + "tokens_seen": 2698313728 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014751454363089267, + "loss": 1.1331, + "theoretical_loss": 3.376686705904521, + "tokens_seen": 2698379264 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014749849548645937, + "loss": 1.2294, + "theoretical_loss": 3.3766803182107736, + "tokens_seen": 2698444800 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014748244734202607, + "loss": 1.2295, + "theoretical_loss": 3.3766739307155964, + "tokens_seen": 2698510336 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014746639919759278, + "loss": 1.2463, + "theoretical_loss": 3.376667543418978, + "tokens_seen": 2698575872 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014745035105315948, + "loss": 1.2622, + "theoretical_loss": 3.376661156320908, + "tokens_seen": 2698641408 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001474343029087262, + "loss": 1.188, + "theoretical_loss": 3.376654769421375, + "tokens_seen": 2698706944 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001474182547642929, + "loss": 1.1798, + "theoretical_loss": 3.3766483827203677, + "tokens_seen": 2698772480 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001474022066198596, + "loss": 1.135, + "theoretical_loss": 3.3766419962178755, + "tokens_seen": 2698838016 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001473861584754263, + "loss": 1.1863, + "theoretical_loss": 3.3766356099138877, + "tokens_seen": 2698903552 + }, + { + "epoch": 0.82, + "learning_rate": 0.000147370110330993, + "loss": 1.2338, + "theoretical_loss": 3.376629223808393, + "tokens_seen": 2698969088 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001473540621865597, + "loss": 1.1696, + "theoretical_loss": 3.37662283790138, + "tokens_seen": 2699034624 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014733801404212638, + "loss": 1.1597, + "theoretical_loss": 3.376616452192838, + "tokens_seen": 2699100160 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014732196589769308, + "loss": 1.2298, + "theoretical_loss": 3.376610066682757, + "tokens_seen": 2699165696 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014730591775325978, + "loss": 1.1301, + "theoretical_loss": 3.376603681371124, + "tokens_seen": 2699231232 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014728986960882648, + "loss": 1.2549, + "theoretical_loss": 3.37659729625793, + "tokens_seen": 2699296768 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014727382146439318, + "loss": 1.1659, + "theoretical_loss": 3.376590911343163, + "tokens_seen": 2699362304 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014725777331995988, + "loss": 1.287, + "theoretical_loss": 3.376584526626812, + "tokens_seen": 2699427840 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014724172517552658, + "loss": 1.1751, + "theoretical_loss": 3.3765781421088663, + "tokens_seen": 2699493376 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014722567703109328, + "loss": 1.2545, + "theoretical_loss": 3.3765717577893146, + "tokens_seen": 2699558912 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014720962888665999, + "loss": 1.1548, + "theoretical_loss": 3.3765653736681465, + "tokens_seen": 2699624448 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014719358074222669, + "loss": 1.2399, + "theoretical_loss": 3.3765589897453507, + "tokens_seen": 2699689984 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014717753259779339, + "loss": 1.2209, + "theoretical_loss": 3.3765526060209154, + "tokens_seen": 2699755520 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001471614844533601, + "loss": 1.2227, + "theoretical_loss": 3.3765462224948313, + "tokens_seen": 2699821056 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001471454363089268, + "loss": 1.2119, + "theoretical_loss": 3.376539839167086, + "tokens_seen": 2699886592 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001471293881644935, + "loss": 1.1787, + "theoretical_loss": 3.376533456037669, + "tokens_seen": 2699952128 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001471133400200602, + "loss": 1.271, + "theoretical_loss": 3.37652707310657, + "tokens_seen": 2700017664 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 901956, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2134134769439697, + "objective/train/theoretical_loss": 3.3765206903737766, + "objective/train/tokens_used": 2720543200, + "theoretical_loss": 3.3765206903737766, + "tokens_seen": 2700083200 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001470972918756269, + "loss": 1.2121, + "theoretical_loss": 3.3765206903737766, + "tokens_seen": 2700083200 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001470812437311936, + "loss": 1.1235, + "theoretical_loss": 3.376514307839279, + "tokens_seen": 2700148736 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001470651955867603, + "loss": 1.2354, + "theoretical_loss": 3.376507925503066, + "tokens_seen": 2700214272 + }, + { + "epoch": 0.82, + "learning_rate": 0.000147049147442327, + "loss": 1.1802, + "theoretical_loss": 3.376501543365126, + "tokens_seen": 2700279808 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001470330992978937, + "loss": 1.2, + "theoretical_loss": 3.376495161425449, + "tokens_seen": 2700345344 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001470170511534604, + "loss": 1.2474, + "theoretical_loss": 3.376488779684023, + "tokens_seen": 2700410880 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001470010030090271, + "loss": 1.3481, + "theoretical_loss": 3.376482398140838, + "tokens_seen": 2700476416 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001469849548645938, + "loss": 1.1148, + "theoretical_loss": 3.3764760167958823, + "tokens_seen": 2700541952 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001469689067201605, + "loss": 1.2286, + "theoretical_loss": 3.376469635649145, + "tokens_seen": 2700607488 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014695285857572717, + "loss": 1.1517, + "theoretical_loss": 3.376463254700616, + "tokens_seen": 2700673024 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014693681043129387, + "loss": 1.1896, + "theoretical_loss": 3.3764568739502834, + "tokens_seen": 2700738560 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014692076228686057, + "loss": 1.2199, + "theoretical_loss": 3.3764504933981363, + "tokens_seen": 2700804096 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014690471414242727, + "loss": 1.2212, + "theoretical_loss": 3.3764441130441645, + "tokens_seen": 2700869632 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014688866599799397, + "loss": 1.2791, + "theoretical_loss": 3.376437732888356, + "tokens_seen": 2700935168 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014687261785356067, + "loss": 1.2161, + "theoretical_loss": 3.3764313529307004, + "tokens_seen": 2701000704 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014685656970912737, + "loss": 1.198, + "theoretical_loss": 3.376424973171187, + "tokens_seen": 2701066240 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001468405215646941, + "loss": 1.2325, + "theoretical_loss": 3.376418593609804, + "tokens_seen": 2701131776 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001468244734202608, + "loss": 1.1997, + "theoretical_loss": 3.376412214246541, + "tokens_seen": 2701197312 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001468084252758275, + "loss": 1.1545, + "theoretical_loss": 3.3764058350813873, + "tokens_seen": 2701262848 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001467923771313942, + "loss": 1.2069, + "theoretical_loss": 3.376399456114332, + "tokens_seen": 2701328384 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001467763289869609, + "loss": 1.1894, + "theoretical_loss": 3.376393077345363, + "tokens_seen": 2701393920 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001467602808425276, + "loss": 1.1307, + "theoretical_loss": 3.3763866987744704, + "tokens_seen": 2701459456 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001467442326980943, + "loss": 1.1599, + "theoretical_loss": 3.376380320401643, + "tokens_seen": 2701524992 + }, + { + "epoch": 0.82, + "learning_rate": 0.000146728184553661, + "loss": 1.2243, + "theoretical_loss": 3.37637394222687, + "tokens_seen": 2701590528 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001467121364092277, + "loss": 1.0544, + "theoretical_loss": 3.3763675642501396, + "tokens_seen": 2701656064 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001466960882647944, + "loss": 1.2468, + "theoretical_loss": 3.376361186471442, + "tokens_seen": 2701721600 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001466800401203611, + "loss": 1.1888, + "theoretical_loss": 3.376354808890766, + "tokens_seen": 2701787136 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001466639919759278, + "loss": 1.2155, + "theoretical_loss": 3.3763484315080996, + "tokens_seen": 2701852672 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001466479438314945, + "loss": 1.2525, + "theoretical_loss": 3.3763420543234335, + "tokens_seen": 2701918208 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014663189568706118, + "loss": 1.3456, + "theoretical_loss": 3.376335677336755, + "tokens_seen": 2701983744 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014661584754262788, + "loss": 1.2171, + "theoretical_loss": 3.376329300548055, + "tokens_seen": 2702049280 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014659979939819458, + "loss": 1.2724, + "theoretical_loss": 3.376322923957321, + "tokens_seen": 2702114816 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014658375125376128, + "loss": 1.22, + "theoretical_loss": 3.3763165475645427, + "tokens_seen": 2702180352 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014656770310932798, + "loss": 1.2231, + "theoretical_loss": 3.3763101713697092, + "tokens_seen": 2702245888 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014655165496489468, + "loss": 1.2175, + "theoretical_loss": 3.3763037953728094, + "tokens_seen": 2702311424 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014653560682046138, + "loss": 1.1715, + "theoretical_loss": 3.3762974195738327, + "tokens_seen": 2702376960 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014651955867602809, + "loss": 1.3133, + "theoretical_loss": 3.376291043972768, + "tokens_seen": 2702442496 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014650351053159479, + "loss": 1.1885, + "theoretical_loss": 3.3762846685696033, + "tokens_seen": 2702508032 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001464874623871615, + "loss": 1.193, + "theoretical_loss": 3.3762782933643294, + "tokens_seen": 2702573568 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001464714142427282, + "loss": 1.318, + "theoretical_loss": 3.3762719183569345, + "tokens_seen": 2702639104 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001464553660982949, + "loss": 1.1201, + "theoretical_loss": 3.376265543547407, + "tokens_seen": 2702704640 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001464393179538616, + "loss": 1.2442, + "theoretical_loss": 3.3762591689357375, + "tokens_seen": 2702770176 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001464232698094283, + "loss": 1.1896, + "theoretical_loss": 3.376252794521914, + "tokens_seen": 2702835712 + }, + { + "epoch": 0.82, + "learning_rate": 0.000146407221664995, + "loss": 1.2706, + "theoretical_loss": 3.3762464203059257, + "tokens_seen": 2702901248 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001463911735205617, + "loss": 1.1855, + "theoretical_loss": 3.3762400462877618, + "tokens_seen": 2702966784 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001463751253761284, + "loss": 1.2744, + "theoretical_loss": 3.376233672467411, + "tokens_seen": 2703032320 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001463590772316951, + "loss": 1.2324, + "theoretical_loss": 3.376227298844863, + "tokens_seen": 2703097856 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001463430290872618, + "loss": 1.2685, + "theoretical_loss": 3.3762209254201068, + "tokens_seen": 2703163392 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001463269809428285, + "loss": 1.2532, + "theoretical_loss": 3.376214552193131, + "tokens_seen": 2703228928 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001463109327983952, + "loss": 1.2321, + "theoretical_loss": 3.3762081791639247, + "tokens_seen": 2703294464 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 903184, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.220799207687378, + "objective/train/theoretical_loss": 3.3762018063324772, + "objective/train/tokens_used": 2723820000, + "theoretical_loss": 3.3762018063324772, + "tokens_seen": 2703360000 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001462948846539619, + "loss": 1.2105, + "theoretical_loss": 3.3762018063324772, + "tokens_seen": 2703360000 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001462788365095286, + "loss": 1.2015, + "theoretical_loss": 3.3761954336987774, + "tokens_seen": 2703425536 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001462627883650953, + "loss": 1.1908, + "theoretical_loss": 3.376189061262815, + "tokens_seen": 2703491072 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014624674022066197, + "loss": 1.2339, + "theoretical_loss": 3.3761826890245783, + "tokens_seen": 2703556608 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001462306920762287, + "loss": 1.2763, + "theoretical_loss": 3.3761763169840564, + "tokens_seen": 2703622144 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001462146439317954, + "loss": 1.1473, + "theoretical_loss": 3.376169945141239, + "tokens_seen": 2703687680 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001461985957873621, + "loss": 1.1725, + "theoretical_loss": 3.3761635734961146, + "tokens_seen": 2703753216 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001461825476429288, + "loss": 1.1953, + "theoretical_loss": 3.376157202048672, + "tokens_seen": 2703818752 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001461664994984955, + "loss": 1.2334, + "theoretical_loss": 3.3761508307989017, + "tokens_seen": 2703884288 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001461504513540622, + "loss": 1.2803, + "theoretical_loss": 3.376144459746791, + "tokens_seen": 2703949824 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001461344032096289, + "loss": 1.1246, + "theoretical_loss": 3.37613808889233, + "tokens_seen": 2704015360 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001461183550651956, + "loss": 1.2484, + "theoretical_loss": 3.3761317182355075, + "tokens_seen": 2704080896 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001461023069207623, + "loss": 1.26, + "theoretical_loss": 3.3761253477763127, + "tokens_seen": 2704146432 + }, + { + "epoch": 0.82, + "learning_rate": 0.000146086258776329, + "loss": 1.1473, + "theoretical_loss": 3.376118977514735, + "tokens_seen": 2704211968 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001460702106318957, + "loss": 1.1471, + "theoretical_loss": 3.3761126074507626, + "tokens_seen": 2704277504 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001460541624874624, + "loss": 1.1512, + "theoretical_loss": 3.3761062375843855, + "tokens_seen": 2704343040 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001460381143430291, + "loss": 1.0744, + "theoretical_loss": 3.3760998679155922, + "tokens_seen": 2704408576 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001460220661985958, + "loss": 1.0917, + "theoretical_loss": 3.376093498444372, + "tokens_seen": 2704474112 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001460060180541625, + "loss": 1.1551, + "theoretical_loss": 3.3760871291707137, + "tokens_seen": 2704539648 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001459899699097292, + "loss": 1.2083, + "theoretical_loss": 3.376080760094607, + "tokens_seen": 2704605184 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001459739217652959, + "loss": 1.2046, + "theoretical_loss": 3.37607439121604, + "tokens_seen": 2704670720 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001459578736208626, + "loss": 1.2046, + "theoretical_loss": 3.3760680225350033, + "tokens_seen": 2704736256 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001459418254764293, + "loss": 1.1088, + "theoretical_loss": 3.376061654051484, + "tokens_seen": 2704801792 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014592577733199598, + "loss": 1.1922, + "theoretical_loss": 3.376055285765473, + "tokens_seen": 2704867328 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014590972918756268, + "loss": 1.2315, + "theoretical_loss": 3.3760489176769584, + "tokens_seen": 2704932864 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014589368104312938, + "loss": 1.1315, + "theoretical_loss": 3.3760425497859297, + "tokens_seen": 2704998400 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014587763289869608, + "loss": 1.2228, + "theoretical_loss": 3.376036182092376, + "tokens_seen": 2705063936 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014586158475426278, + "loss": 1.2729, + "theoretical_loss": 3.376029814596286, + "tokens_seen": 2705129472 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014584553660982948, + "loss": 1.2648, + "theoretical_loss": 3.376023447297649, + "tokens_seen": 2705195008 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014582948846539619, + "loss": 1.2273, + "theoretical_loss": 3.3760170801964544, + "tokens_seen": 2705260544 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014581344032096289, + "loss": 1.243, + "theoretical_loss": 3.3760107132926906, + "tokens_seen": 2705326080 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001457973921765296, + "loss": 1.2009, + "theoretical_loss": 3.376004346586347, + "tokens_seen": 2705391616 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001457813440320963, + "loss": 1.2481, + "theoretical_loss": 3.3759979800774134, + "tokens_seen": 2705457152 + }, + { + "epoch": 0.82, + "learning_rate": 0.000145765295887663, + "loss": 1.3151, + "theoretical_loss": 3.375991613765878, + "tokens_seen": 2705522688 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001457492477432297, + "loss": 1.2354, + "theoretical_loss": 3.3759852476517302, + "tokens_seen": 2705588224 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001457331995987964, + "loss": 1.2416, + "theoretical_loss": 3.375978881734959, + "tokens_seen": 2705653760 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001457171514543631, + "loss": 1.1716, + "theoretical_loss": 3.375972516015554, + "tokens_seen": 2705719296 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001457011033099298, + "loss": 1.2311, + "theoretical_loss": 3.375966150493504, + "tokens_seen": 2705784832 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001456850551654965, + "loss": 1.2116, + "theoretical_loss": 3.3759597851687975, + "tokens_seen": 2705850368 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001456690070210632, + "loss": 1.2547, + "theoretical_loss": 3.375953420041424, + "tokens_seen": 2705915904 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001456529588766299, + "loss": 1.237, + "theoretical_loss": 3.375947055111373, + "tokens_seen": 2705981440 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014563691073219662, + "loss": 1.2411, + "theoretical_loss": 3.3759406903786333, + "tokens_seen": 2706046976 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001456208625877633, + "loss": 1.2124, + "theoretical_loss": 3.3759343258431938, + "tokens_seen": 2706112512 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014560481444333, + "loss": 1.2567, + "theoretical_loss": 3.3759279615050444, + "tokens_seen": 2706178048 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001455887662988967, + "loss": 1.2434, + "theoretical_loss": 3.375921597364173, + "tokens_seen": 2706243584 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001455727181544634, + "loss": 1.2045, + "theoretical_loss": 3.3759152334205695, + "tokens_seen": 2706309120 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001455566700100301, + "loss": 1.2242, + "theoretical_loss": 3.375908869674223, + "tokens_seen": 2706374656 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001455406218655968, + "loss": 1.188, + "theoretical_loss": 3.375902506125122, + "tokens_seen": 2706440192 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001455245737211635, + "loss": 1.2495, + "theoretical_loss": 3.375896142773257, + "tokens_seen": 2706505728 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001455085255767302, + "loss": 1.142, + "theoretical_loss": 3.3758897796186154, + "tokens_seen": 2706571264 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 904383, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2287869453430176, + "objective/train/theoretical_loss": 3.3758834166611873, + "objective/train/tokens_used": 2727096800, + "theoretical_loss": 3.3758834166611873, + "tokens_seen": 2706636800 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001454924774322969, + "loss": 1.2131, + "theoretical_loss": 3.3758834166611873, + "tokens_seen": 2706636800 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001454764292878636, + "loss": 1.3476, + "theoretical_loss": 3.375877053900962, + "tokens_seen": 2706702336 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001454603811434303, + "loss": 1.2256, + "theoretical_loss": 3.3758706913379277, + "tokens_seen": 2706767872 + }, + { + "epoch": 0.82, + "learning_rate": 0.000145444332998997, + "loss": 1.1851, + "theoretical_loss": 3.3758643289720744, + "tokens_seen": 2706833408 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001454282848545637, + "loss": 1.2474, + "theoretical_loss": 3.37585796680339, + "tokens_seen": 2706898944 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001454122367101304, + "loss": 1.2671, + "theoretical_loss": 3.3758516048318654, + "tokens_seen": 2706964480 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001453961885656971, + "loss": 1.1632, + "theoretical_loss": 3.375845243057489, + "tokens_seen": 2707030016 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001453801404212638, + "loss": 1.2829, + "theoretical_loss": 3.3758388814802487, + "tokens_seen": 2707095552 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001453640922768305, + "loss": 1.2249, + "theoretical_loss": 3.3758325201001353, + "tokens_seen": 2707161088 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001453480441323972, + "loss": 1.2123, + "theoretical_loss": 3.3758261589171368, + "tokens_seen": 2707226624 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001453319959879639, + "loss": 1.2101, + "theoretical_loss": 3.375819797931243, + "tokens_seen": 2707292160 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001453159478435306, + "loss": 1.1227, + "theoretical_loss": 3.375813437142443, + "tokens_seen": 2707357696 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001452998996990973, + "loss": 1.1785, + "theoretical_loss": 3.3758070765507253, + "tokens_seen": 2707423232 + }, + { + "epoch": 0.82, + "learning_rate": 0.000145283851554664, + "loss": 1.1134, + "theoretical_loss": 3.3758007161560797, + "tokens_seen": 2707488768 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001452678034102307, + "loss": 1.2483, + "theoretical_loss": 3.3757943559584946, + "tokens_seen": 2707554304 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001452517552657974, + "loss": 1.1672, + "theoretical_loss": 3.3757879959579604, + "tokens_seen": 2707619840 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014523570712136408, + "loss": 1.2603, + "theoretical_loss": 3.375781636154465, + "tokens_seen": 2707685376 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014521965897693078, + "loss": 1.1926, + "theoretical_loss": 3.375775276547998, + "tokens_seen": 2707750912 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014520361083249748, + "loss": 1.1351, + "theoretical_loss": 3.375768917138548, + "tokens_seen": 2707816448 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014518756268806418, + "loss": 1.2192, + "theoretical_loss": 3.375762557926105, + "tokens_seen": 2707881984 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014517151454363088, + "loss": 1.1713, + "theoretical_loss": 3.3757561989106573, + "tokens_seen": 2707947520 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014515546639919759, + "loss": 1.1273, + "theoretical_loss": 3.375749840092195, + "tokens_seen": 2708013056 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014513941825476429, + "loss": 1.1799, + "theoretical_loss": 3.3757434814707064, + "tokens_seen": 2708078592 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014512337011033099, + "loss": 1.1575, + "theoretical_loss": 3.3757371230461812, + "tokens_seen": 2708144128 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001451073219658977, + "loss": 1.2103, + "theoretical_loss": 3.3757307648186075, + "tokens_seen": 2708209664 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001450912738214644, + "loss": 1.272, + "theoretical_loss": 3.3757244067879757, + "tokens_seen": 2708275200 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001450752256770311, + "loss": 1.1923, + "theoretical_loss": 3.3757180489542744, + "tokens_seen": 2708340736 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001450591775325978, + "loss": 1.1509, + "theoretical_loss": 3.375711691317493, + "tokens_seen": 2708406272 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014504312938816452, + "loss": 1.277, + "theoretical_loss": 3.3757053338776197, + "tokens_seen": 2708471808 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014502708124373122, + "loss": 1.1333, + "theoretical_loss": 3.375698976634645, + "tokens_seen": 2708537344 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014501103309929792, + "loss": 1.2564, + "theoretical_loss": 3.375692619588557, + "tokens_seen": 2708602880 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014499498495486462, + "loss": 1.26, + "theoretical_loss": 3.375686262739345, + "tokens_seen": 2708668416 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014497893681043132, + "loss": 1.2141, + "theoretical_loss": 3.3756799060869986, + "tokens_seen": 2708733952 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014496288866599802, + "loss": 1.245, + "theoretical_loss": 3.3756735496315065, + "tokens_seen": 2708799488 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014494684052156472, + "loss": 1.24, + "theoretical_loss": 3.375667193372858, + "tokens_seen": 2708865024 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014493079237713142, + "loss": 1.3615, + "theoretical_loss": 3.3756608373110426, + "tokens_seen": 2708930560 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001449147442326981, + "loss": 1.2667, + "theoretical_loss": 3.375654481446049, + "tokens_seen": 2708996096 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001448986960882648, + "loss": 1.1629, + "theoretical_loss": 3.3756481257778663, + "tokens_seen": 2709061632 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001448826479438315, + "loss": 1.2104, + "theoretical_loss": 3.375641770306484, + "tokens_seen": 2709127168 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001448665997993982, + "loss": 1.1743, + "theoretical_loss": 3.3756354150318906, + "tokens_seen": 2709192704 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001448505516549649, + "loss": 1.1987, + "theoretical_loss": 3.3756290599540764, + "tokens_seen": 2709258240 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001448345035105316, + "loss": 1.1681, + "theoretical_loss": 3.375622705073029, + "tokens_seen": 2709323776 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001448184553660983, + "loss": 1.2328, + "theoretical_loss": 3.375616350388739, + "tokens_seen": 2709389312 + }, + { + "epoch": 0.82, + "learning_rate": 0.000144802407221665, + "loss": 1.2572, + "theoretical_loss": 3.3756099959011943, + "tokens_seen": 2709454848 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001447863590772317, + "loss": 1.2341, + "theoretical_loss": 3.375603641610385, + "tokens_seen": 2709520384 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001447703109327984, + "loss": 1.2338, + "theoretical_loss": 3.3755972875163, + "tokens_seen": 2709585920 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001447542627883651, + "loss": 1.2019, + "theoretical_loss": 3.3755909336189287, + "tokens_seen": 2709651456 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001447382146439318, + "loss": 1.233, + "theoretical_loss": 3.3755845799182596, + "tokens_seen": 2709716992 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001447221664994985, + "loss": 1.2556, + "theoretical_loss": 3.375578226414282, + "tokens_seen": 2709782528 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001447061183550652, + "loss": 1.2559, + "theoretical_loss": 3.3755718731069853, + "tokens_seen": 2709848064 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 905460, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1922107934951782, + "objective/train/theoretical_loss": 3.375565519996359, + "objective/train/tokens_used": 2730373600, + "theoretical_loss": 3.375565519996359, + "tokens_seen": 2709913600 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001446900702106319, + "loss": 1.2342, + "theoretical_loss": 3.375565519996359, + "tokens_seen": 2709913600 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001446740220661986, + "loss": 1.1639, + "theoretical_loss": 3.375559167082391, + "tokens_seen": 2709979136 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001446579739217653, + "loss": 1.2397, + "theoretical_loss": 3.375552814365072, + "tokens_seen": 2710044672 + }, + { + "epoch": 0.82, + "learning_rate": 0.000144641925777332, + "loss": 1.2332, + "theoretical_loss": 3.3755464618443902, + "tokens_seen": 2710110208 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001446258776328987, + "loss": 1.2171, + "theoretical_loss": 3.375540109520335, + "tokens_seen": 2710175744 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001446098294884654, + "loss": 1.1351, + "theoretical_loss": 3.375533757392896, + "tokens_seen": 2710241280 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001445937813440321, + "loss": 1.1717, + "theoretical_loss": 3.3755274054620616, + "tokens_seen": 2710306816 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001445777331995988, + "loss": 1.256, + "theoretical_loss": 3.3755210537278213, + "tokens_seen": 2710372352 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001445616850551655, + "loss": 1.2615, + "theoretical_loss": 3.375514702190164, + "tokens_seen": 2710437888 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001445456369107322, + "loss": 1.2746, + "theoretical_loss": 3.3755083508490795, + "tokens_seen": 2710503424 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014452958876629888, + "loss": 1.2604, + "theoretical_loss": 3.3755019997045568, + "tokens_seen": 2710568960 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014451354062186558, + "loss": 1.1411, + "theoretical_loss": 3.3754956487565844, + "tokens_seen": 2710634496 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014449749247743228, + "loss": 1.2971, + "theoretical_loss": 3.375489298005152, + "tokens_seen": 2710700032 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014448144433299898, + "loss": 1.2541, + "theoretical_loss": 3.375482947450249, + "tokens_seen": 2710765568 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014446539618856569, + "loss": 1.3069, + "theoretical_loss": 3.375476597091864, + "tokens_seen": 2710831104 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014444934804413239, + "loss": 1.2792, + "theoretical_loss": 3.3754702469299867, + "tokens_seen": 2710896640 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014443329989969911, + "loss": 1.2295, + "theoretical_loss": 3.3754638969646056, + "tokens_seen": 2710962176 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014441725175526581, + "loss": 1.2096, + "theoretical_loss": 3.37545754719571, + "tokens_seen": 2711027712 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014440120361083251, + "loss": 1.1891, + "theoretical_loss": 3.37545119762329, + "tokens_seen": 2711093248 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014438515546639922, + "loss": 1.1774, + "theoretical_loss": 3.3754448482473336, + "tokens_seen": 2711158784 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014436910732196592, + "loss": 1.2528, + "theoretical_loss": 3.375438499067831, + "tokens_seen": 2711224320 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014435305917753262, + "loss": 1.1862, + "theoretical_loss": 3.3754321500847704, + "tokens_seen": 2711289856 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014433701103309932, + "loss": 1.2045, + "theoretical_loss": 3.3754258012981415, + "tokens_seen": 2711355392 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014432096288866602, + "loss": 1.286, + "theoretical_loss": 3.3754194527079333, + "tokens_seen": 2711420928 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014430491474423272, + "loss": 1.186, + "theoretical_loss": 3.3754131043141355, + "tokens_seen": 2711486464 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014428886659979942, + "loss": 1.2809, + "theoretical_loss": 3.3754067561167367, + "tokens_seen": 2711552000 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014427281845536612, + "loss": 1.2937, + "theoretical_loss": 3.375400408115726, + "tokens_seen": 2711617536 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014425677031093282, + "loss": 1.2801, + "theoretical_loss": 3.375394060311093, + "tokens_seen": 2711683072 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014424072216649952, + "loss": 1.2251, + "theoretical_loss": 3.3753877127028264, + "tokens_seen": 2711748608 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014422467402206622, + "loss": 1.2388, + "theoretical_loss": 3.3753813652909157, + "tokens_seen": 2711814144 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001442086258776329, + "loss": 1.1219, + "theoretical_loss": 3.3753750180753506, + "tokens_seen": 2711879680 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001441925777331996, + "loss": 1.1416, + "theoretical_loss": 3.375368671056119, + "tokens_seen": 2711945216 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001441765295887663, + "loss": 1.2096, + "theoretical_loss": 3.3753623242332114, + "tokens_seen": 2712010752 + }, + { + "epoch": 0.82, + "learning_rate": 0.000144160481444333, + "loss": 1.2015, + "theoretical_loss": 3.375355977606616, + "tokens_seen": 2712076288 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001441444332998997, + "loss": 1.277, + "theoretical_loss": 3.3753496311763227, + "tokens_seen": 2712141824 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001441283851554664, + "loss": 1.265, + "theoretical_loss": 3.37534328494232, + "tokens_seen": 2712207360 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001441123370110331, + "loss": 1.275, + "theoretical_loss": 3.3753369389045975, + "tokens_seen": 2712272896 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001440962888665998, + "loss": 1.2574, + "theoretical_loss": 3.3753305930631443, + "tokens_seen": 2712338432 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001440802407221665, + "loss": 1.2189, + "theoretical_loss": 3.37532424741795, + "tokens_seen": 2712403968 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001440641925777332, + "loss": 1.1739, + "theoretical_loss": 3.3753179019690034, + "tokens_seen": 2712469504 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001440481444332999, + "loss": 1.308, + "theoretical_loss": 3.3753115567162935, + "tokens_seen": 2712535040 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001440320962888666, + "loss": 1.2771, + "theoretical_loss": 3.3753052116598097, + "tokens_seen": 2712600576 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001440160481444333, + "loss": 1.2494, + "theoretical_loss": 3.375298866799541, + "tokens_seen": 2712666112 + }, + { + "epoch": 0.82, + "learning_rate": 0.000144, + "loss": 1.2352, + "theoretical_loss": 3.375292522135477, + "tokens_seen": 2712731648 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001439839518555667, + "loss": 1.2628, + "theoretical_loss": 3.3752861776676064, + "tokens_seen": 2712797184 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001439679037111334, + "loss": 1.1266, + "theoretical_loss": 3.375279833395919, + "tokens_seen": 2712862720 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001439518555667001, + "loss": 1.1627, + "theoretical_loss": 3.3752734893204037, + "tokens_seen": 2712928256 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001439358074222668, + "loss": 1.3255, + "theoretical_loss": 3.3752671454410494, + "tokens_seen": 2712993792 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001439197592778335, + "loss": 1.2037, + "theoretical_loss": 3.3752608017578454, + "tokens_seen": 2713059328 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001439037111334002, + "loss": 1.2839, + "theoretical_loss": 3.3752544582707813, + "tokens_seen": 2713124864 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 906795, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2472453117370605, + "objective/train/theoretical_loss": 3.375248114979846, + "objective/train/tokens_used": 2733650400, + "theoretical_loss": 3.375248114979846, + "tokens_seen": 2713190400 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001438876629889669, + "loss": 1.2112, + "theoretical_loss": 3.375248114979846, + "tokens_seen": 2713190400 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001438716148445336, + "loss": 1.2517, + "theoretical_loss": 3.3752417718850287, + "tokens_seen": 2713255936 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001438555667001003, + "loss": 1.2653, + "theoretical_loss": 3.3752354289863185, + "tokens_seen": 2713321472 + }, + { + "epoch": 0.82, + "learning_rate": 0.000143839518555667, + "loss": 1.1545, + "theoretical_loss": 3.375229086283705, + "tokens_seen": 2713387008 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001438234704112337, + "loss": 1.2086, + "theoretical_loss": 3.375222743777177, + "tokens_seen": 2713452544 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001438074222668004, + "loss": 1.2552, + "theoretical_loss": 3.375216401466724, + "tokens_seen": 2713518080 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001437913741223671, + "loss": 1.3292, + "theoretical_loss": 3.375210059352335, + "tokens_seen": 2713583616 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001437753259779338, + "loss": 1.2501, + "theoretical_loss": 3.3752037174339993, + "tokens_seen": 2713649152 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001437592778335005, + "loss": 1.2796, + "theoretical_loss": 3.3751973757117057, + "tokens_seen": 2713714688 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014374322968906721, + "loss": 1.2262, + "theoretical_loss": 3.3751910341854443, + "tokens_seen": 2713780224 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014372718154463391, + "loss": 1.2908, + "theoretical_loss": 3.3751846928552034, + "tokens_seen": 2713845760 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014371113340020062, + "loss": 1.2201, + "theoretical_loss": 3.3751783517209724, + "tokens_seen": 2713911296 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014369508525576732, + "loss": 1.3446, + "theoretical_loss": 3.375172010782741, + "tokens_seen": 2713976832 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014367903711133402, + "loss": 1.2206, + "theoretical_loss": 3.375165670040498, + "tokens_seen": 2714042368 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014366298896690072, + "loss": 1.2442, + "theoretical_loss": 3.375159329494233, + "tokens_seen": 2714107904 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014364694082246742, + "loss": 1.1604, + "theoretical_loss": 3.3751529891439347, + "tokens_seen": 2714173440 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014363089267803412, + "loss": 1.2236, + "theoretical_loss": 3.3751466489895927, + "tokens_seen": 2714238976 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014361484453360082, + "loss": 1.1936, + "theoretical_loss": 3.3751403090311958, + "tokens_seen": 2714304512 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014359879638916752, + "loss": 1.3272, + "theoretical_loss": 3.375133969268733, + "tokens_seen": 2714370048 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014358274824473422, + "loss": 1.2553, + "theoretical_loss": 3.375127629702195, + "tokens_seen": 2714435584 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014356670010030092, + "loss": 1.1575, + "theoretical_loss": 3.375121290331569, + "tokens_seen": 2714501120 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014355065195586762, + "loss": 1.2163, + "theoretical_loss": 3.3751149511568457, + "tokens_seen": 2714566656 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014353460381143432, + "loss": 1.1598, + "theoretical_loss": 3.375108612178014, + "tokens_seen": 2714632192 + }, + { + "epoch": 0.82, + "learning_rate": 0.000143518555667001, + "loss": 1.0906, + "theoretical_loss": 3.3751022733950626, + "tokens_seen": 2714697728 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001435025075225677, + "loss": 1.2749, + "theoretical_loss": 3.375095934807981, + "tokens_seen": 2714763264 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001434864593781344, + "loss": 1.2055, + "theoretical_loss": 3.3750895964167587, + "tokens_seen": 2714828800 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001434704112337011, + "loss": 1.2605, + "theoretical_loss": 3.3750832582213843, + "tokens_seen": 2714894336 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001434543630892678, + "loss": 1.2409, + "theoretical_loss": 3.375076920221848, + "tokens_seen": 2714959872 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001434383149448345, + "loss": 1.1991, + "theoretical_loss": 3.3750705824181377, + "tokens_seen": 2715025408 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001434222668004012, + "loss": 1.3619, + "theoretical_loss": 3.375064244810244, + "tokens_seen": 2715090944 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001434062186559679, + "loss": 1.2864, + "theoretical_loss": 3.3750579073981553, + "tokens_seen": 2715156480 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001433901705115346, + "loss": 1.1987, + "theoretical_loss": 3.375051570181861, + "tokens_seen": 2715222016 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001433741223671013, + "loss": 1.1843, + "theoretical_loss": 3.3750452331613503, + "tokens_seen": 2715287552 + }, + { + "epoch": 0.82, + "learning_rate": 0.000143358074222668, + "loss": 1.1453, + "theoretical_loss": 3.3750388963366125, + "tokens_seen": 2715353088 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001433420260782347, + "loss": 1.2546, + "theoretical_loss": 3.3750325597076367, + "tokens_seen": 2715418624 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001433259779338014, + "loss": 1.2627, + "theoretical_loss": 3.3750262232744124, + "tokens_seen": 2715484160 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001433099297893681, + "loss": 1.1666, + "theoretical_loss": 3.3750198870369283, + "tokens_seen": 2715549696 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001432938816449348, + "loss": 1.2002, + "theoretical_loss": 3.3750135509951744, + "tokens_seen": 2715615232 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001432778335005015, + "loss": 1.2426, + "theoretical_loss": 3.375007215149139, + "tokens_seen": 2715680768 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001432617853560682, + "loss": 1.2157, + "theoretical_loss": 3.375000879498812, + "tokens_seen": 2715746304 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001432457372116349, + "loss": 1.2915, + "theoretical_loss": 3.3749945440441826, + "tokens_seen": 2715811840 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014322968906720163, + "loss": 1.2045, + "theoretical_loss": 3.37498820878524, + "tokens_seen": 2715877376 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014321364092276833, + "loss": 1.2827, + "theoretical_loss": 3.374981873721973, + "tokens_seen": 2715942912 + }, + { + "epoch": 0.82, + "learning_rate": 0.000143197592778335, + "loss": 1.3091, + "theoretical_loss": 3.3749755388543714, + "tokens_seen": 2716008448 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001431815446339017, + "loss": 1.1271, + "theoretical_loss": 3.3749692041824244, + "tokens_seen": 2716073984 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001431654964894684, + "loss": 1.1383, + "theoretical_loss": 3.3749628697061205, + "tokens_seen": 2716139520 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001431494483450351, + "loss": 1.1515, + "theoretical_loss": 3.37495653542545, + "tokens_seen": 2716205056 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001431334002006018, + "loss": 1.2619, + "theoretical_loss": 3.3749502013404014, + "tokens_seen": 2716270592 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001431173520561685, + "loss": 1.1197, + "theoretical_loss": 3.374943867450964, + "tokens_seen": 2716336128 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001431013039117352, + "loss": 1.2479, + "theoretical_loss": 3.3749375337571275, + "tokens_seen": 2716401664 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 907855, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3315622806549072, + "objective/train/theoretical_loss": 3.3749312002588803, + "objective/train/tokens_used": 2736927200, + "theoretical_loss": 3.3749312002588803, + "tokens_seen": 2716467200 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001430852557673019, + "loss": 1.2978, + "theoretical_loss": 3.3749312002588803, + "tokens_seen": 2716467200 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001430692076228686, + "loss": 1.2571, + "theoretical_loss": 3.3749248669562126, + "tokens_seen": 2716532736 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014305315947843531, + "loss": 1.1892, + "theoretical_loss": 3.374918533849113, + "tokens_seen": 2716598272 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014303711133400201, + "loss": 1.2361, + "theoretical_loss": 3.374912200937571, + "tokens_seen": 2716663808 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014302106318956872, + "loss": 1.1687, + "theoretical_loss": 3.374905868221576, + "tokens_seen": 2716729344 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014300501504513542, + "loss": 1.263, + "theoretical_loss": 3.3748995357011173, + "tokens_seen": 2716794880 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014298896690070212, + "loss": 1.0694, + "theoretical_loss": 3.374893203376183, + "tokens_seen": 2716860416 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014297291875626882, + "loss": 1.1777, + "theoretical_loss": 3.374886871246764, + "tokens_seen": 2716925952 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014295687061183552, + "loss": 1.2548, + "theoretical_loss": 3.3748805393128487, + "tokens_seen": 2716991488 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014294082246740222, + "loss": 1.1854, + "theoretical_loss": 3.3748742075744262, + "tokens_seen": 2717057024 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014292477432296892, + "loss": 1.1876, + "theoretical_loss": 3.374867876031486, + "tokens_seen": 2717122560 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014290872617853562, + "loss": 1.3031, + "theoretical_loss": 3.3748615446840176, + "tokens_seen": 2717188096 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014289267803410232, + "loss": 1.1702, + "theoretical_loss": 3.3748552135320096, + "tokens_seen": 2717253632 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014287662988966902, + "loss": 1.1474, + "theoretical_loss": 3.374848882575452, + "tokens_seen": 2717319168 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014286058174523572, + "loss": 1.1917, + "theoretical_loss": 3.3748425518143335, + "tokens_seen": 2717384704 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014284453360080242, + "loss": 1.3467, + "theoretical_loss": 3.3748362212486436, + "tokens_seen": 2717450240 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014282848545636912, + "loss": 1.2748, + "theoretical_loss": 3.374829890878371, + "tokens_seen": 2717515776 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001428124373119358, + "loss": 1.2036, + "theoretical_loss": 3.3748235607035064, + "tokens_seen": 2717581312 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001427963891675025, + "loss": 1.3219, + "theoretical_loss": 3.3748172307240374, + "tokens_seen": 2717646848 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001427803410230692, + "loss": 1.2265, + "theoretical_loss": 3.3748109009399543, + "tokens_seen": 2717712384 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001427642928786359, + "loss": 1.2163, + "theoretical_loss": 3.374804571351246, + "tokens_seen": 2717777920 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001427482447342026, + "loss": 1.1316, + "theoretical_loss": 3.374798241957902, + "tokens_seen": 2717843456 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001427321965897693, + "loss": 1.275, + "theoretical_loss": 3.374791912759911, + "tokens_seen": 2717908992 + }, + { + "epoch": 0.82, + "learning_rate": 0.000142716148445336, + "loss": 1.1392, + "theoretical_loss": 3.3747855837572622, + "tokens_seen": 2717974528 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001427001003009027, + "loss": 1.2398, + "theoretical_loss": 3.374779254949946, + "tokens_seen": 2718040064 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001426840521564694, + "loss": 1.183, + "theoretical_loss": 3.3747729263379505, + "tokens_seen": 2718105600 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001426680040120361, + "loss": 1.2898, + "theoretical_loss": 3.3747665979212655, + "tokens_seen": 2718171136 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001426519558676028, + "loss": 1.2998, + "theoretical_loss": 3.3747602696998804, + "tokens_seen": 2718236672 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014263590772316953, + "loss": 1.2156, + "theoretical_loss": 3.374753941673784, + "tokens_seen": 2718302208 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014261985957873623, + "loss": 1.1264, + "theoretical_loss": 3.3747476138429655, + "tokens_seen": 2718367744 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014260381143430293, + "loss": 1.2146, + "theoretical_loss": 3.374741286207415, + "tokens_seen": 2718433280 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014258776328986963, + "loss": 1.2049, + "theoretical_loss": 3.374734958767121, + "tokens_seen": 2718498816 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014257171514543633, + "loss": 1.1217, + "theoretical_loss": 3.3747286315220726, + "tokens_seen": 2718564352 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014255566700100303, + "loss": 1.17, + "theoretical_loss": 3.37472230447226, + "tokens_seen": 2718629888 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014253961885656973, + "loss": 1.2355, + "theoretical_loss": 3.3747159776176714, + "tokens_seen": 2718695424 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014252357071213643, + "loss": 1.219, + "theoretical_loss": 3.3747096509582972, + "tokens_seen": 2718760960 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014250752256770314, + "loss": 1.1983, + "theoretical_loss": 3.3747033244941256, + "tokens_seen": 2718826496 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001424914744232698, + "loss": 1.2644, + "theoretical_loss": 3.3746969982251467, + "tokens_seen": 2718892032 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001424754262788365, + "loss": 1.2298, + "theoretical_loss": 3.374690672151349, + "tokens_seen": 2718957568 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001424593781344032, + "loss": 1.236, + "theoretical_loss": 3.3746843462727227, + "tokens_seen": 2719023104 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001424433299899699, + "loss": 1.2318, + "theoretical_loss": 3.374678020589256, + "tokens_seen": 2719088640 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001424272818455366, + "loss": 1.1647, + "theoretical_loss": 3.374671695100939, + "tokens_seen": 2719154176 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001424112337011033, + "loss": 1.2393, + "theoretical_loss": 3.3746653698077607, + "tokens_seen": 2719219712 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014239518555667, + "loss": 1.2586, + "theoretical_loss": 3.3746590447097105, + "tokens_seen": 2719285248 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001423791374122367, + "loss": 1.1062, + "theoretical_loss": 3.374652719806777, + "tokens_seen": 2719350784 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014236308926780341, + "loss": 1.288, + "theoretical_loss": 3.3746463950989507, + "tokens_seen": 2719416320 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014234704112337011, + "loss": 1.2452, + "theoretical_loss": 3.37464007058622, + "tokens_seen": 2719481856 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014233099297893682, + "loss": 1.2687, + "theoretical_loss": 3.3746337462685743, + "tokens_seen": 2719547392 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014231494483450352, + "loss": 1.1595, + "theoretical_loss": 3.3746274221460033, + "tokens_seen": 2719612928 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014229889669007022, + "loss": 1.2258, + "theoretical_loss": 3.374621098218496, + "tokens_seen": 2719678464 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 908995, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2070744037628174, + "objective/train/theoretical_loss": 3.374614774486041, + "objective/train/tokens_used": 2740204000, + "theoretical_loss": 3.374614774486041, + "tokens_seen": 2719744000 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014228284854563692, + "loss": 1.1909, + "theoretical_loss": 3.374614774486041, + "tokens_seen": 2719744000 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014226680040120362, + "loss": 1.3464, + "theoretical_loss": 3.374608450948629, + "tokens_seen": 2719809536 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014225075225677032, + "loss": 1.1953, + "theoretical_loss": 3.3746021276062477, + "tokens_seen": 2719875072 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014223470411233702, + "loss": 1.2162, + "theoretical_loss": 3.3745958044588877, + "tokens_seen": 2719940608 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014221865596790372, + "loss": 1.3007, + "theoretical_loss": 3.374589481506538, + "tokens_seen": 2720006144 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014220260782347042, + "loss": 1.2435, + "theoretical_loss": 3.3745831587491875, + "tokens_seen": 2720071680 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014218655967903712, + "loss": 1.2302, + "theoretical_loss": 3.3745768361868254, + "tokens_seen": 2720137216 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014217051153460382, + "loss": 1.1189, + "theoretical_loss": 3.3745705138194415, + "tokens_seen": 2720202752 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014215446339017052, + "loss": 1.2254, + "theoretical_loss": 3.374564191647025, + "tokens_seen": 2720268288 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014213841524573722, + "loss": 1.0979, + "theoretical_loss": 3.3745578696695646, + "tokens_seen": 2720333824 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014212236710130392, + "loss": 1.1545, + "theoretical_loss": 3.3745515478870503, + "tokens_seen": 2720399360 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001421063189568706, + "loss": 1.3077, + "theoretical_loss": 3.374545226299471, + "tokens_seen": 2720464896 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001420902708124373, + "loss": 1.1252, + "theoretical_loss": 3.3745389049068164, + "tokens_seen": 2720530432 + }, + { + "epoch": 0.82, + "learning_rate": 0.000142074222668004, + "loss": 1.1799, + "theoretical_loss": 3.374532583709075, + "tokens_seen": 2720595968 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001420581745235707, + "loss": 1.2328, + "theoretical_loss": 3.374526262706237, + "tokens_seen": 2720661504 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001420421263791374, + "loss": 1.1849, + "theoretical_loss": 3.374519941898291, + "tokens_seen": 2720727040 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014202607823470413, + "loss": 1.2031, + "theoretical_loss": 3.374513621285227, + "tokens_seen": 2720792576 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014201003009027083, + "loss": 1.1638, + "theoretical_loss": 3.374507300867034, + "tokens_seen": 2720858112 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014199398194583753, + "loss": 1.2097, + "theoretical_loss": 3.3745009806437007, + "tokens_seen": 2720923648 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014197793380140423, + "loss": 1.1972, + "theoretical_loss": 3.374494660615217, + "tokens_seen": 2720989184 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014196188565697093, + "loss": 1.0361, + "theoretical_loss": 3.3744883407815722, + "tokens_seen": 2721054720 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014194583751253763, + "loss": 1.2158, + "theoretical_loss": 3.3744820211427555, + "tokens_seen": 2721120256 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014192978936810433, + "loss": 1.2791, + "theoretical_loss": 3.374475701698756, + "tokens_seen": 2721185792 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014191374122367103, + "loss": 1.2395, + "theoretical_loss": 3.3744693824495635, + "tokens_seen": 2721251328 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014189769307923773, + "loss": 1.2538, + "theoretical_loss": 3.374463063395167, + "tokens_seen": 2721316864 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014188164493480443, + "loss": 1.2695, + "theoretical_loss": 3.3744567445355553, + "tokens_seen": 2721382400 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014186559679037113, + "loss": 1.1695, + "theoretical_loss": 3.3744504258707186, + "tokens_seen": 2721447936 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014184954864593783, + "loss": 1.3267, + "theoretical_loss": 3.3744441074006457, + "tokens_seen": 2721513472 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014183350050150454, + "loss": 1.2021, + "theoretical_loss": 3.3744377891253263, + "tokens_seen": 2721579008 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014181745235707124, + "loss": 1.1853, + "theoretical_loss": 3.3744314710447494, + "tokens_seen": 2721644544 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001418014042126379, + "loss": 1.1981, + "theoretical_loss": 3.374425153158904, + "tokens_seen": 2721710080 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001417853560682046, + "loss": 1.2537, + "theoretical_loss": 3.3744188354677798, + "tokens_seen": 2721775616 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001417693079237713, + "loss": 1.1665, + "theoretical_loss": 3.374412517971366, + "tokens_seen": 2721841152 + }, + { + "epoch": 0.82, + "learning_rate": 0.000141753259779338, + "loss": 1.212, + "theoretical_loss": 3.3744062006696525, + "tokens_seen": 2721906688 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001417372116349047, + "loss": 1.2229, + "theoretical_loss": 3.3743998835626274, + "tokens_seen": 2721972224 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001417211634904714, + "loss": 1.0674, + "theoretical_loss": 3.3743935666502813, + "tokens_seen": 2722037760 + }, + { + "epoch": 0.82, + "learning_rate": 0.0001417051153460381, + "loss": 1.1796, + "theoretical_loss": 3.3743872499326026, + "tokens_seen": 2722103296 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014168906720160481, + "loss": 1.2306, + "theoretical_loss": 3.374380933409581, + "tokens_seen": 2722168832 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014167301905717151, + "loss": 1.1956, + "theoretical_loss": 3.3743746170812057, + "tokens_seen": 2722234368 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014165697091273821, + "loss": 1.3298, + "theoretical_loss": 3.374368300947466, + "tokens_seen": 2722299904 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014164092276830492, + "loss": 1.1574, + "theoretical_loss": 3.3743619850083513, + "tokens_seen": 2722365440 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014162487462387162, + "loss": 1.2866, + "theoretical_loss": 3.374355669263851, + "tokens_seen": 2722430976 + }, + { + "epoch": 0.82, + "learning_rate": 0.00014160882647943832, + "loss": 1.2627, + "theoretical_loss": 3.374349353713954, + "tokens_seen": 2722496512 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014159277833500502, + "loss": 1.2113, + "theoretical_loss": 3.37434303835865, + "tokens_seen": 2722562048 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014157673019057172, + "loss": 1.1756, + "theoretical_loss": 3.3743367231979287, + "tokens_seen": 2722627584 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014156068204613842, + "loss": 1.2057, + "theoretical_loss": 3.3743304082317787, + "tokens_seen": 2722693120 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014154463390170512, + "loss": 1.1728, + "theoretical_loss": 3.3743240934601895, + "tokens_seen": 2722758656 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014152858575727182, + "loss": 1.1886, + "theoretical_loss": 3.3743177788831504, + "tokens_seen": 2722824192 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014151253761283852, + "loss": 1.2609, + "theoretical_loss": 3.374311464500651, + "tokens_seen": 2722889728 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014149648946840522, + "loss": 1.256, + "theoretical_loss": 3.37430515031268, + "tokens_seen": 2722955264 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 909957, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2262158393859863, + "objective/train/theoretical_loss": 3.374298836319228, + "objective/train/tokens_used": 2743480800, + "theoretical_loss": 3.374298836319228, + "tokens_seen": 2723020800 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014148044132397192, + "loss": 1.1927, + "theoretical_loss": 3.374298836319228, + "tokens_seen": 2723020800 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014146439317953862, + "loss": 1.1792, + "theoretical_loss": 3.374292522520283, + "tokens_seen": 2723086336 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014144834503510532, + "loss": 1.2476, + "theoretical_loss": 3.374286208915835, + "tokens_seen": 2723151872 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014143229689067202, + "loss": 1.2541, + "theoretical_loss": 3.3742798955058735, + "tokens_seen": 2723217408 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014141624874623872, + "loss": 1.2123, + "theoretical_loss": 3.374273582290387, + "tokens_seen": 2723282944 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014140020060180543, + "loss": 1.2602, + "theoretical_loss": 3.3742672692693656, + "tokens_seen": 2723348480 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014138415245737213, + "loss": 1.1833, + "theoretical_loss": 3.374260956442798, + "tokens_seen": 2723414016 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014136810431293883, + "loss": 1.1004, + "theoretical_loss": 3.3742546438106737, + "tokens_seen": 2723479552 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014135205616850553, + "loss": 1.0793, + "theoretical_loss": 3.374248331372983, + "tokens_seen": 2723545088 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014133600802407223, + "loss": 1.2406, + "theoretical_loss": 3.374242019129714, + "tokens_seen": 2723610624 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014131995987963893, + "loss": 1.2582, + "theoretical_loss": 3.3742357070808566, + "tokens_seen": 2723676160 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014130391173520563, + "loss": 1.2541, + "theoretical_loss": 3.3742293952263998, + "tokens_seen": 2723741696 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014128786359077233, + "loss": 1.2162, + "theoretical_loss": 3.3742230835663336, + "tokens_seen": 2723807232 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014127181544633903, + "loss": 1.1881, + "theoretical_loss": 3.3742167721006466, + "tokens_seen": 2723872768 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014125576730190573, + "loss": 1.2811, + "theoretical_loss": 3.3742104608293286, + "tokens_seen": 2723938304 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014123971915747243, + "loss": 1.2981, + "theoretical_loss": 3.3742041497523685, + "tokens_seen": 2724003840 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014122367101303913, + "loss": 1.326, + "theoretical_loss": 3.374197838869756, + "tokens_seen": 2724069376 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014120762286860583, + "loss": 1.2633, + "theoretical_loss": 3.3741915281814805, + "tokens_seen": 2724134912 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014119157472417253, + "loss": 1.1787, + "theoretical_loss": 3.374185217687531, + "tokens_seen": 2724200448 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014117552657973923, + "loss": 1.2743, + "theoretical_loss": 3.374178907387897, + "tokens_seen": 2724265984 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014115947843530593, + "loss": 1.204, + "theoretical_loss": 3.374172597282568, + "tokens_seen": 2724331520 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014114343029087264, + "loss": 1.2286, + "theoretical_loss": 3.3741662873715335, + "tokens_seen": 2724397056 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014112738214643934, + "loss": 1.0814, + "theoretical_loss": 3.374159977654782, + "tokens_seen": 2724462592 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014111133400200604, + "loss": 1.2512, + "theoretical_loss": 3.374153668132304, + "tokens_seen": 2724528128 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001410952858575727, + "loss": 1.0742, + "theoretical_loss": 3.3741473588040876, + "tokens_seen": 2724593664 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001410792377131394, + "loss": 1.2063, + "theoretical_loss": 3.374141049670123, + "tokens_seen": 2724659200 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001410631895687061, + "loss": 1.2532, + "theoretical_loss": 3.3741347407303994, + "tokens_seen": 2724724736 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001410471414242728, + "loss": 1.1572, + "theoretical_loss": 3.3741284319849063, + "tokens_seen": 2724790272 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001410310932798395, + "loss": 1.2116, + "theoretical_loss": 3.3741221234336325, + "tokens_seen": 2724855808 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001410150451354062, + "loss": 1.1668, + "theoretical_loss": 3.3741158150765678, + "tokens_seen": 2724921344 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014099899699097291, + "loss": 1.2198, + "theoretical_loss": 3.3741095069137015, + "tokens_seen": 2724986880 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014098294884653961, + "loss": 1.1779, + "theoretical_loss": 3.374103198945023, + "tokens_seen": 2725052416 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014096690070210632, + "loss": 1.2207, + "theoretical_loss": 3.3740968911705216, + "tokens_seen": 2725117952 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014095085255767302, + "loss": 1.1575, + "theoretical_loss": 3.374090583590186, + "tokens_seen": 2725183488 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014093480441323972, + "loss": 1.051, + "theoretical_loss": 3.374084276204007, + "tokens_seen": 2725249024 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014091875626880642, + "loss": 1.0896, + "theoretical_loss": 3.3740779690119727, + "tokens_seen": 2725314560 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014090270812437312, + "loss": 1.3227, + "theoretical_loss": 3.3740716620140727, + "tokens_seen": 2725380096 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014088665997993982, + "loss": 1.2132, + "theoretical_loss": 3.3740653552102966, + "tokens_seen": 2725445632 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014087061183550652, + "loss": 1.248, + "theoretical_loss": 3.3740590486006337, + "tokens_seen": 2725511168 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014085456369107322, + "loss": 1.1791, + "theoretical_loss": 3.3740527421850732, + "tokens_seen": 2725576704 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014083851554663995, + "loss": 1.2365, + "theoretical_loss": 3.374046435963605, + "tokens_seen": 2725642240 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014082246740220665, + "loss": 1.1785, + "theoretical_loss": 3.3740401299362177, + "tokens_seen": 2725707776 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014080641925777335, + "loss": 1.1872, + "theoretical_loss": 3.3740338241029013, + "tokens_seen": 2725773312 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014079037111334005, + "loss": 1.2564, + "theoretical_loss": 3.374027518463645, + "tokens_seen": 2725838848 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014077432296890672, + "loss": 1.164, + "theoretical_loss": 3.3740212130184375, + "tokens_seen": 2725904384 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014075827482447342, + "loss": 1.2657, + "theoretical_loss": 3.3740149077672688, + "tokens_seen": 2725969920 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014074222668004012, + "loss": 1.171, + "theoretical_loss": 3.3740086027101284, + "tokens_seen": 2726035456 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014072617853560682, + "loss": 1.2294, + "theoretical_loss": 3.3740022978470052, + "tokens_seen": 2726100992 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014071013039117353, + "loss": 1.2561, + "theoretical_loss": 3.373995993177889, + "tokens_seen": 2726166528 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014069408224674023, + "loss": 1.1513, + "theoretical_loss": 3.373989688702769, + "tokens_seen": 2726232064 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 911186, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1067233085632324, + "objective/train/theoretical_loss": 3.373983384421634, + "objective/train/tokens_used": 2746757600, + "theoretical_loss": 3.373983384421634, + "tokens_seen": 2726297600 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014067803410230693, + "loss": 1.1108, + "theoretical_loss": 3.373983384421634, + "tokens_seen": 2726297600 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014066198595787363, + "loss": 1.1537, + "theoretical_loss": 3.3739770803344746, + "tokens_seen": 2726363136 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014064593781344033, + "loss": 1.2324, + "theoretical_loss": 3.373970776441279, + "tokens_seen": 2726428672 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014062988966900703, + "loss": 1.233, + "theoretical_loss": 3.3739644727420375, + "tokens_seen": 2726494208 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014061384152457373, + "loss": 1.1643, + "theoretical_loss": 3.3739581692367384, + "tokens_seen": 2726559744 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014059779338014043, + "loss": 1.0751, + "theoretical_loss": 3.373951865925372, + "tokens_seen": 2726625280 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014058174523570713, + "loss": 1.1636, + "theoretical_loss": 3.3739455628079273, + "tokens_seen": 2726690816 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014056569709127383, + "loss": 1.1942, + "theoretical_loss": 3.373939259884394, + "tokens_seen": 2726756352 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014054964894684053, + "loss": 1.2364, + "theoretical_loss": 3.3739329571547607, + "tokens_seen": 2726821888 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014053360080240723, + "loss": 1.1413, + "theoretical_loss": 3.3739266546190176, + "tokens_seen": 2726887424 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014051755265797393, + "loss": 1.2609, + "theoretical_loss": 3.373920352277154, + "tokens_seen": 2726952960 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014050150451354063, + "loss": 1.2031, + "theoretical_loss": 3.3739140501291587, + "tokens_seen": 2727018496 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014048545636910733, + "loss": 1.2364, + "theoretical_loss": 3.3739077481750215, + "tokens_seen": 2727084032 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014046940822467403, + "loss": 1.2, + "theoretical_loss": 3.3739014464147314, + "tokens_seen": 2727149568 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014045336008024074, + "loss": 1.2077, + "theoretical_loss": 3.3738951448482783, + "tokens_seen": 2727215104 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014043731193580744, + "loss": 1.2428, + "theoretical_loss": 3.373888843475651, + "tokens_seen": 2727280640 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014042126379137414, + "loss": 1.2566, + "theoretical_loss": 3.37388254229684, + "tokens_seen": 2727346176 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014040521564694084, + "loss": 1.1487, + "theoretical_loss": 3.3738762413118333, + "tokens_seen": 2727411712 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001403891675025075, + "loss": 1.1404, + "theoretical_loss": 3.373869940520621, + "tokens_seen": 2727477248 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001403731193580742, + "loss": 1.1355, + "theoretical_loss": 3.3738636399231923, + "tokens_seen": 2727542784 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001403570712136409, + "loss": 1.1552, + "theoretical_loss": 3.3738573395195366, + "tokens_seen": 2727608320 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001403410230692076, + "loss": 1.2976, + "theoretical_loss": 3.3738510393096437, + "tokens_seen": 2727673856 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001403249749247743, + "loss": 1.2602, + "theoretical_loss": 3.373844739293502, + "tokens_seen": 2727739392 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014030892678034101, + "loss": 1.1669, + "theoretical_loss": 3.3738384394711023, + "tokens_seen": 2727804928 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014029287863590771, + "loss": 1.1875, + "theoretical_loss": 3.373832139842433, + "tokens_seen": 2727870464 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014027683049147442, + "loss": 1.1745, + "theoretical_loss": 3.3738258404074832, + "tokens_seen": 2727936000 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014026078234704112, + "loss": 1.1811, + "theoretical_loss": 3.373819541166243, + "tokens_seen": 2728001536 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014024473420260782, + "loss": 1.1759, + "theoretical_loss": 3.3738132421187017, + "tokens_seen": 2728067072 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014022868605817454, + "loss": 1.1833, + "theoretical_loss": 3.3738069432648485, + "tokens_seen": 2728132608 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014021263791374124, + "loss": 1.2749, + "theoretical_loss": 3.373800644604673, + "tokens_seen": 2728198144 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014019658976930795, + "loss": 1.113, + "theoretical_loss": 3.373794346138164, + "tokens_seen": 2728263680 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014018054162487465, + "loss": 1.2257, + "theoretical_loss": 3.3737880478653115, + "tokens_seen": 2728329216 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014016449348044135, + "loss": 1.1441, + "theoretical_loss": 3.373781749786105, + "tokens_seen": 2728394752 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014014844533600805, + "loss": 1.1979, + "theoretical_loss": 3.373775451900533, + "tokens_seen": 2728460288 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014013239719157475, + "loss": 1.2315, + "theoretical_loss": 3.3737691542085866, + "tokens_seen": 2728525824 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014011634904714145, + "loss": 1.2431, + "theoretical_loss": 3.373762856710253, + "tokens_seen": 2728591360 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014010030090270815, + "loss": 1.1844, + "theoretical_loss": 3.3737565594055234, + "tokens_seen": 2728656896 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014008425275827485, + "loss": 1.135, + "theoretical_loss": 3.3737502622943865, + "tokens_seen": 2728722432 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014006820461384152, + "loss": 1.2451, + "theoretical_loss": 3.3737439653768315, + "tokens_seen": 2728787968 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014005215646940822, + "loss": 1.2361, + "theoretical_loss": 3.373737668652848, + "tokens_seen": 2728853504 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014003610832497492, + "loss": 1.2556, + "theoretical_loss": 3.3737313721224256, + "tokens_seen": 2728919040 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014002006018054163, + "loss": 1.2375, + "theoretical_loss": 3.3737250757855533, + "tokens_seen": 2728984576 + }, + { + "epoch": 0.83, + "learning_rate": 0.00014000401203610833, + "loss": 1.1357, + "theoretical_loss": 3.3737187796422208, + "tokens_seen": 2729050112 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013998796389167503, + "loss": 1.192, + "theoretical_loss": 3.373712483692417, + "tokens_seen": 2729115648 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013997191574724173, + "loss": 1.2318, + "theoretical_loss": 3.3737061879361327, + "tokens_seen": 2729181184 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013995586760280843, + "loss": 1.1588, + "theoretical_loss": 3.3736998923733554, + "tokens_seen": 2729246720 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013993981945837513, + "loss": 1.1777, + "theoretical_loss": 3.373693597004076, + "tokens_seen": 2729312256 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013992377131394183, + "loss": 1.0935, + "theoretical_loss": 3.373687301828283, + "tokens_seen": 2729377792 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013990772316950853, + "loss": 1.2744, + "theoretical_loss": 3.3736810068459664, + "tokens_seen": 2729443328 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013989167502507523, + "loss": 1.2158, + "theoretical_loss": 3.3736747120571153, + "tokens_seen": 2729508864 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 912132, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2257804870605469, + "objective/train/theoretical_loss": 3.373668417461719, + "objective/train/tokens_used": 2750034400, + "theoretical_loss": 3.373668417461719, + "tokens_seen": 2729574400 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013987562688064193, + "loss": 1.2399, + "theoretical_loss": 3.373668417461719, + "tokens_seen": 2729574400 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013985957873620863, + "loss": 1.1171, + "theoretical_loss": 3.373662123059767, + "tokens_seen": 2729639936 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013984353059177533, + "loss": 1.2116, + "theoretical_loss": 3.3736558288512493, + "tokens_seen": 2729705472 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013982748244734203, + "loss": 1.0749, + "theoretical_loss": 3.3736495348361544, + "tokens_seen": 2729771008 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013981143430290873, + "loss": 1.1191, + "theoretical_loss": 3.3736432410144723, + "tokens_seen": 2729836544 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013979538615847543, + "loss": 1.2184, + "theoretical_loss": 3.373636947386192, + "tokens_seen": 2729902080 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013977933801404214, + "loss": 1.1526, + "theoretical_loss": 3.3736306539513032, + "tokens_seen": 2729967616 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013976328986960884, + "loss": 1.0687, + "theoretical_loss": 3.3736243607097953, + "tokens_seen": 2730033152 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013974724172517554, + "loss": 1.2978, + "theoretical_loss": 3.3736180676616576, + "tokens_seen": 2730098688 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013973119358074224, + "loss": 1.2107, + "theoretical_loss": 3.3736117748068795, + "tokens_seen": 2730164224 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013971514543630894, + "loss": 1.2452, + "theoretical_loss": 3.373605482145451, + "tokens_seen": 2730229760 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013969909729187564, + "loss": 1.1337, + "theoretical_loss": 3.3735991896773605, + "tokens_seen": 2730295296 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001396830491474423, + "loss": 1.1504, + "theoretical_loss": 3.3735928974025984, + "tokens_seen": 2730360832 + }, + { + "epoch": 0.83, + "learning_rate": 0.000139667001003009, + "loss": 1.2948, + "theoretical_loss": 3.373586605321153, + "tokens_seen": 2730426368 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001396509528585757, + "loss": 1.2068, + "theoretical_loss": 3.373580313433015, + "tokens_seen": 2730491904 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013963490471414244, + "loss": 1.3106, + "theoretical_loss": 3.373574021738173, + "tokens_seen": 2730557440 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013961885656970914, + "loss": 1.1683, + "theoretical_loss": 3.3735677302366165, + "tokens_seen": 2730622976 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013960280842527584, + "loss": 1.2699, + "theoretical_loss": 3.373561438928335, + "tokens_seen": 2730688512 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013958676028084254, + "loss": 1.1567, + "theoretical_loss": 3.3735551478133186, + "tokens_seen": 2730754048 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013957071213640924, + "loss": 1.1382, + "theoretical_loss": 3.3735488568915555, + "tokens_seen": 2730819584 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013955466399197594, + "loss": 1.2062, + "theoretical_loss": 3.373542566163036, + "tokens_seen": 2730885120 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013953861584754264, + "loss": 1.1911, + "theoretical_loss": 3.373536275627749, + "tokens_seen": 2730950656 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013952256770310935, + "loss": 1.235, + "theoretical_loss": 3.373529985285684, + "tokens_seen": 2731016192 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013950651955867605, + "loss": 1.1755, + "theoretical_loss": 3.3735236951368313, + "tokens_seen": 2731081728 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013949047141424275, + "loss": 1.2418, + "theoretical_loss": 3.373517405181179, + "tokens_seen": 2731147264 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013947442326980945, + "loss": 1.2014, + "theoretical_loss": 3.373511115418718, + "tokens_seen": 2731212800 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013945837512537615, + "loss": 1.1424, + "theoretical_loss": 3.373504825849436, + "tokens_seen": 2731278336 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013944232698094285, + "loss": 1.3236, + "theoretical_loss": 3.3734985364733237, + "tokens_seen": 2731343872 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013942627883650955, + "loss": 1.2273, + "theoretical_loss": 3.37349224729037, + "tokens_seen": 2731409408 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013941023069207625, + "loss": 1.1997, + "theoretical_loss": 3.3734859583005647, + "tokens_seen": 2731474944 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013939418254764295, + "loss": 1.3265, + "theoretical_loss": 3.373479669503897, + "tokens_seen": 2731540480 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013937813440320962, + "loss": 1.1184, + "theoretical_loss": 3.3734733809003563, + "tokens_seen": 2731606016 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013936208625877632, + "loss": 1.1139, + "theoretical_loss": 3.373467092489932, + "tokens_seen": 2731671552 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013934603811434303, + "loss": 1.1184, + "theoretical_loss": 3.3734608042726135, + "tokens_seen": 2731737088 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013932998996990973, + "loss": 1.1829, + "theoretical_loss": 3.373454516248391, + "tokens_seen": 2731802624 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013931394182547643, + "loss": 1.2398, + "theoretical_loss": 3.373448228417253, + "tokens_seen": 2731868160 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013929789368104313, + "loss": 1.0984, + "theoretical_loss": 3.373441940779189, + "tokens_seen": 2731933696 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013928184553660983, + "loss": 1.1755, + "theoretical_loss": 3.373435653334189, + "tokens_seen": 2731999232 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013926579739217653, + "loss": 1.247, + "theoretical_loss": 3.373429366082242, + "tokens_seen": 2732064768 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013924974924774323, + "loss": 1.1993, + "theoretical_loss": 3.3734230790233375, + "tokens_seen": 2732130304 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013923370110330993, + "loss": 1.2426, + "theoretical_loss": 3.373416792157465, + "tokens_seen": 2732195840 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013921765295887663, + "loss": 1.2193, + "theoretical_loss": 3.3734105054846144, + "tokens_seen": 2732261376 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013920160481444333, + "loss": 1.179, + "theoretical_loss": 3.373404219004774, + "tokens_seen": 2732326912 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013918555667001003, + "loss": 1.2333, + "theoretical_loss": 3.3733979327179346, + "tokens_seen": 2732392448 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013916950852557673, + "loss": 1.1017, + "theoretical_loss": 3.3733916466240847, + "tokens_seen": 2732457984 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013915346038114343, + "loss": 1.2382, + "theoretical_loss": 3.3733853607232143, + "tokens_seen": 2732523520 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013913741223671013, + "loss": 1.24, + "theoretical_loss": 3.373379075015312, + "tokens_seen": 2732589056 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013912136409227683, + "loss": 1.3008, + "theoretical_loss": 3.3733727895003685, + "tokens_seen": 2732654592 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013910531594784353, + "loss": 1.2727, + "theoretical_loss": 3.3733665041783722, + "tokens_seen": 2732720128 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013908926780341024, + "loss": 1.2318, + "theoretical_loss": 3.3733602190493133, + "tokens_seen": 2732785664 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 913267, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2152161598205566, + "objective/train/theoretical_loss": 3.3733539341131804, + "objective/train/tokens_used": 2753311200, + "theoretical_loss": 3.3733539341131804, + "tokens_seen": 2732851200 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013907321965897694, + "loss": 1.2295, + "theoretical_loss": 3.3733539341131804, + "tokens_seen": 2732851200 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013905717151454364, + "loss": 1.2686, + "theoretical_loss": 3.373347649369964, + "tokens_seen": 2732916736 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013904112337011034, + "loss": 1.167, + "theoretical_loss": 3.3733413648196526, + "tokens_seen": 2732982272 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013902507522567704, + "loss": 1.1461, + "theoretical_loss": 3.373335080462236, + "tokens_seen": 2733047808 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013900902708124374, + "loss": 1.2819, + "theoretical_loss": 3.373328796297704, + "tokens_seen": 2733113344 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013899297893681044, + "loss": 1.1816, + "theoretical_loss": 3.3733225123260455, + "tokens_seen": 2733178880 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013897693079237714, + "loss": 1.1807, + "theoretical_loss": 3.3733162285472504, + "tokens_seen": 2733244416 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013896088264794384, + "loss": 1.0659, + "theoretical_loss": 3.3733099449613078, + "tokens_seen": 2733309952 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013894483450351054, + "loss": 1.2013, + "theoretical_loss": 3.3733036615682073, + "tokens_seen": 2733375488 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013892878635907724, + "loss": 1.2048, + "theoretical_loss": 3.3732973783679387, + "tokens_seen": 2733441024 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013891273821464394, + "loss": 1.2105, + "theoretical_loss": 3.3732910953604907, + "tokens_seen": 2733506560 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013889669007021064, + "loss": 1.1578, + "theoretical_loss": 3.3732848125458537, + "tokens_seen": 2733572096 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013888064192577734, + "loss": 1.2006, + "theoretical_loss": 3.3732785299240162, + "tokens_seen": 2733637632 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013886459378134404, + "loss": 1.1753, + "theoretical_loss": 3.3732722474949686, + "tokens_seen": 2733703168 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013884854563691074, + "loss": 1.2187, + "theoretical_loss": 3.373265965258699, + "tokens_seen": 2733768704 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013883249749247745, + "loss": 1.2245, + "theoretical_loss": 3.3732596832151986, + "tokens_seen": 2733834240 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013881644934804415, + "loss": 1.1245, + "theoretical_loss": 3.373253401364456, + "tokens_seen": 2733899776 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013880040120361085, + "loss": 1.2565, + "theoretical_loss": 3.3732471197064604, + "tokens_seen": 2733965312 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013878435305917755, + "loss": 1.2291, + "theoretical_loss": 3.373240838241202, + "tokens_seen": 2734030848 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013876830491474425, + "loss": 1.1065, + "theoretical_loss": 3.373234556968669, + "tokens_seen": 2734096384 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013875225677031095, + "loss": 1.2785, + "theoretical_loss": 3.373228275888852, + "tokens_seen": 2734161920 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013873620862587765, + "loss": 1.2075, + "theoretical_loss": 3.3732219950017406, + "tokens_seen": 2734227456 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013872016048144435, + "loss": 1.2966, + "theoretical_loss": 3.373215714307323, + "tokens_seen": 2734292992 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013870411233701105, + "loss": 1.2858, + "theoretical_loss": 3.3732094338055902, + "tokens_seen": 2734358528 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013868806419257775, + "loss": 1.2102, + "theoretical_loss": 3.3732031534965308, + "tokens_seen": 2734424064 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013867201604814442, + "loss": 1.2256, + "theoretical_loss": 3.373196873380134, + "tokens_seen": 2734489600 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013865596790371113, + "loss": 1.194, + "theoretical_loss": 3.3731905934563904, + "tokens_seen": 2734555136 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013863991975927783, + "loss": 1.2122, + "theoretical_loss": 3.373184313725288, + "tokens_seen": 2734620672 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013862387161484453, + "loss": 1.1693, + "theoretical_loss": 3.373178034186818, + "tokens_seen": 2734686208 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013860782347041123, + "loss": 1.2369, + "theoretical_loss": 3.373171754840968, + "tokens_seen": 2734751744 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013859177532597793, + "loss": 1.1712, + "theoretical_loss": 3.373165475687729, + "tokens_seen": 2734817280 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013857572718154463, + "loss": 1.0811, + "theoretical_loss": 3.3731591967270895, + "tokens_seen": 2734882816 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013855967903711133, + "loss": 1.2031, + "theoretical_loss": 3.3731529179590396, + "tokens_seen": 2734948352 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013854363089267803, + "loss": 1.1991, + "theoretical_loss": 3.3731466393835685, + "tokens_seen": 2735013888 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013852758274824473, + "loss": 1.1942, + "theoretical_loss": 3.3731403610006656, + "tokens_seen": 2735079424 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013851153460381143, + "loss": 1.1446, + "theoretical_loss": 3.3731340828103207, + "tokens_seen": 2735144960 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013849548645937813, + "loss": 1.2137, + "theoretical_loss": 3.373127804812523, + "tokens_seen": 2735210496 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013847943831494483, + "loss": 1.2595, + "theoretical_loss": 3.373121527007262, + "tokens_seen": 2735276032 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013846339017051153, + "loss": 1.2583, + "theoretical_loss": 3.373115249394527, + "tokens_seen": 2735341568 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013844734202607823, + "loss": 1.1829, + "theoretical_loss": 3.3731089719743084, + "tokens_seen": 2735407104 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013843129388164496, + "loss": 1.1623, + "theoretical_loss": 3.3731026947465943, + "tokens_seen": 2735472640 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013841524573721166, + "loss": 1.21, + "theoretical_loss": 3.373096417711375, + "tokens_seen": 2735538176 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013839919759277836, + "loss": 1.1355, + "theoretical_loss": 3.3730901408686402, + "tokens_seen": 2735603712 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013838314944834506, + "loss": 1.2026, + "theoretical_loss": 3.373083864218379, + "tokens_seen": 2735669248 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013836710130391176, + "loss": 1.1946, + "theoretical_loss": 3.3730775877605805, + "tokens_seen": 2735734784 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013835105315947844, + "loss": 1.134, + "theoretical_loss": 3.3730713114952353, + "tokens_seen": 2735800320 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013833500501504514, + "loss": 1.2552, + "theoretical_loss": 3.3730650354223317, + "tokens_seen": 2735865856 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013831895687061184, + "loss": 1.2614, + "theoretical_loss": 3.37305875954186, + "tokens_seen": 2735931392 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013830290872617854, + "loss": 1.3023, + "theoretical_loss": 3.3730524838538094, + "tokens_seen": 2735996928 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013828686058174524, + "loss": 1.2148, + "theoretical_loss": 3.3730462083581694, + "tokens_seen": 2736062464 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 914246, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1928385496139526, + "objective/train/theoretical_loss": 3.373039933054929, + "objective/train/tokens_used": 2756588000, + "theoretical_loss": 3.373039933054929, + "tokens_seen": 2736128000 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013827081243731194, + "loss": 1.206, + "theoretical_loss": 3.373039933054929, + "tokens_seen": 2736128000 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013825476429287864, + "loss": 1.1643, + "theoretical_loss": 3.3730336579440787, + "tokens_seen": 2736193536 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013823871614844534, + "loss": 1.278, + "theoretical_loss": 3.3730273830256072, + "tokens_seen": 2736259072 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013822266800401204, + "loss": 1.2389, + "theoretical_loss": 3.3730211082995045, + "tokens_seen": 2736324608 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013820661985957874, + "loss": 1.1869, + "theoretical_loss": 3.37301483376576, + "tokens_seen": 2736390144 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013819057171514544, + "loss": 1.1536, + "theoretical_loss": 3.373008559424363, + "tokens_seen": 2736455680 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013817452357071214, + "loss": 1.1557, + "theoretical_loss": 3.3730022852753025, + "tokens_seen": 2736521216 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013815847542627884, + "loss": 1.2354, + "theoretical_loss": 3.372996011318569, + "tokens_seen": 2736586752 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013814242728184555, + "loss": 1.151, + "theoretical_loss": 3.3729897375541515, + "tokens_seen": 2736652288 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013812637913741225, + "loss": 1.2369, + "theoretical_loss": 3.3729834639820395, + "tokens_seen": 2736717824 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013811033099297895, + "loss": 1.216, + "theoretical_loss": 3.372977190602223, + "tokens_seen": 2736783360 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013809428284854565, + "loss": 1.2391, + "theoretical_loss": 3.3729709174146905, + "tokens_seen": 2736848896 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013807823470411235, + "loss": 1.2359, + "theoretical_loss": 3.372964644419432, + "tokens_seen": 2736914432 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013806218655967905, + "loss": 1.2573, + "theoretical_loss": 3.3729583716164377, + "tokens_seen": 2736979968 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013804613841524575, + "loss": 1.2328, + "theoretical_loss": 3.372952099005696, + "tokens_seen": 2737045504 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013803009027081245, + "loss": 1.1541, + "theoretical_loss": 3.372945826587197, + "tokens_seen": 2737111040 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013801404212637915, + "loss": 1.1593, + "theoretical_loss": 3.37293955436093, + "tokens_seen": 2737176576 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013799799398194585, + "loss": 1.2003, + "theoretical_loss": 3.3729332823268847, + "tokens_seen": 2737242112 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013798194583751255, + "loss": 1.1904, + "theoretical_loss": 3.3729270104850504, + "tokens_seen": 2737307648 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013796589769307923, + "loss": 1.1932, + "theoretical_loss": 3.3729207388354165, + "tokens_seen": 2737373184 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013794984954864593, + "loss": 1.2391, + "theoretical_loss": 3.372914467377973, + "tokens_seen": 2737438720 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013793380140421263, + "loss": 1.1446, + "theoretical_loss": 3.372908196112709, + "tokens_seen": 2737504256 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013791775325977933, + "loss": 1.2671, + "theoretical_loss": 3.372901925039614, + "tokens_seen": 2737569792 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013790170511534603, + "loss": 1.1809, + "theoretical_loss": 3.3728956541586776, + "tokens_seen": 2737635328 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013788565697091273, + "loss": 1.0654, + "theoretical_loss": 3.37288938346989, + "tokens_seen": 2737700864 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013786960882647943, + "loss": 1.1786, + "theoretical_loss": 3.3728831129732395, + "tokens_seen": 2737766400 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013785356068204613, + "loss": 1.2103, + "theoretical_loss": 3.372876842668716, + "tokens_seen": 2737831936 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013783751253761286, + "loss": 1.1522, + "theoretical_loss": 3.3728705725563097, + "tokens_seen": 2737897472 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013782146439317956, + "loss": 1.2702, + "theoretical_loss": 3.3728643026360094, + "tokens_seen": 2737963008 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013780541624874626, + "loss": 1.1934, + "theoretical_loss": 3.3728580329078044, + "tokens_seen": 2738028544 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013778936810431296, + "loss": 1.2006, + "theoretical_loss": 3.3728517633716852, + "tokens_seen": 2738094080 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013777331995987966, + "loss": 1.1417, + "theoretical_loss": 3.3728454940276404, + "tokens_seen": 2738159616 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013775727181544636, + "loss": 1.1429, + "theoretical_loss": 3.3728392248756602, + "tokens_seen": 2738225152 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013774122367101306, + "loss": 1.227, + "theoretical_loss": 3.3728329559157335, + "tokens_seen": 2738290688 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013772517552657976, + "loss": 1.3685, + "theoretical_loss": 3.3728266871478505, + "tokens_seen": 2738356224 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013770912738214646, + "loss": 1.2093, + "theoretical_loss": 3.372820418572, + "tokens_seen": 2738421760 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013769307923771316, + "loss": 1.1843, + "theoretical_loss": 3.372814150188172, + "tokens_seen": 2738487296 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013767703109327986, + "loss": 1.1555, + "theoretical_loss": 3.372807881996356, + "tokens_seen": 2738552832 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013766098294884654, + "loss": 1.1697, + "theoretical_loss": 3.3728016139965407, + "tokens_seen": 2738618368 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013764493480441324, + "loss": 1.1298, + "theoretical_loss": 3.372795346188717, + "tokens_seen": 2738683904 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013762888665997994, + "loss": 1.1202, + "theoretical_loss": 3.3727890785728736, + "tokens_seen": 2738749440 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013761283851554664, + "loss": 1.2367, + "theoretical_loss": 3.3727828111490004, + "tokens_seen": 2738814976 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013759679037111334, + "loss": 1.3065, + "theoretical_loss": 3.3727765439170865, + "tokens_seen": 2738880512 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013758074222668004, + "loss": 1.2391, + "theoretical_loss": 3.3727702768771213, + "tokens_seen": 2738946048 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013756469408224674, + "loss": 1.1912, + "theoretical_loss": 3.372764010029095, + "tokens_seen": 2739011584 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013754864593781344, + "loss": 1.188, + "theoretical_loss": 3.372757743372997, + "tokens_seen": 2739077120 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013753259779338014, + "loss": 1.0811, + "theoretical_loss": 3.3727514769088165, + "tokens_seen": 2739142656 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013751654964894684, + "loss": 1.1538, + "theoretical_loss": 3.372745210636543, + "tokens_seen": 2739208192 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013750050150451354, + "loss": 1.1964, + "theoretical_loss": 3.372738944556166, + "tokens_seen": 2739273728 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013748445336008024, + "loss": 1.2324, + "theoretical_loss": 3.3727326786676755, + "tokens_seen": 2739339264 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 915536, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1874685287475586, + "objective/train/theoretical_loss": 3.372726412971061, + "objective/train/tokens_used": 2759864800, + "theoretical_loss": 3.372726412971061, + "tokens_seen": 2739404800 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013746840521564695, + "loss": 1.2205, + "theoretical_loss": 3.372726412971061, + "tokens_seen": 2739404800 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013745235707121365, + "loss": 1.2892, + "theoretical_loss": 3.3727201474663113, + "tokens_seen": 2739470336 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013743630892678035, + "loss": 1.2027, + "theoretical_loss": 3.3727138821534166, + "tokens_seen": 2739535872 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013742026078234705, + "loss": 1.2004, + "theoretical_loss": 3.372707617032366, + "tokens_seen": 2739601408 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013740421263791375, + "loss": 1.1917, + "theoretical_loss": 3.3727013521031495, + "tokens_seen": 2739666944 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013738816449348045, + "loss": 1.3048, + "theoretical_loss": 3.3726950873657566, + "tokens_seen": 2739732480 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013737211634904715, + "loss": 1.2499, + "theoretical_loss": 3.3726888228201766, + "tokens_seen": 2739798016 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013735606820461385, + "loss": 1.1609, + "theoretical_loss": 3.372682558466399, + "tokens_seen": 2739863552 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013734002006018055, + "loss": 1.1952, + "theoretical_loss": 3.372676294304414, + "tokens_seen": 2739929088 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013732397191574725, + "loss": 1.1728, + "theoretical_loss": 3.3726700303342096, + "tokens_seen": 2739994624 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013730792377131395, + "loss": 1.2209, + "theoretical_loss": 3.372663766555777, + "tokens_seen": 2740060160 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013729187562688065, + "loss": 1.0875, + "theoretical_loss": 3.372657502969105, + "tokens_seen": 2740125696 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013727582748244733, + "loss": 1.1403, + "theoretical_loss": 3.372651239574183, + "tokens_seen": 2740191232 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013725977933801403, + "loss": 1.2145, + "theoretical_loss": 3.3726449763710007, + "tokens_seen": 2740256768 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013724373119358073, + "loss": 1.229, + "theoretical_loss": 3.3726387133595477, + "tokens_seen": 2740322304 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013722768304914745, + "loss": 1.2423, + "theoretical_loss": 3.3726324505398138, + "tokens_seen": 2740387840 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013721163490471416, + "loss": 1.1087, + "theoretical_loss": 3.3726261879117887, + "tokens_seen": 2740453376 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013719558676028086, + "loss": 1.3439, + "theoretical_loss": 3.372619925475461, + "tokens_seen": 2740518912 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013717953861584756, + "loss": 1.1614, + "theoretical_loss": 3.372613663230821, + "tokens_seen": 2740584448 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013716349047141426, + "loss": 1.2006, + "theoretical_loss": 3.3726074011778575, + "tokens_seen": 2740649984 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013714744232698096, + "loss": 1.2119, + "theoretical_loss": 3.372601139316561, + "tokens_seen": 2740715520 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013713139418254766, + "loss": 1.1229, + "theoretical_loss": 3.372594877646921, + "tokens_seen": 2740781056 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013711534603811436, + "loss": 1.1805, + "theoretical_loss": 3.372588616168926, + "tokens_seen": 2740846592 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013709929789368106, + "loss": 1.14, + "theoretical_loss": 3.3725823548825664, + "tokens_seen": 2740912128 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013708324974924776, + "loss": 1.2108, + "theoretical_loss": 3.3725760937878317, + "tokens_seen": 2740977664 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013706720160481446, + "loss": 1.2381, + "theoretical_loss": 3.372569832884712, + "tokens_seen": 2741043200 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013705115346038116, + "loss": 1.2696, + "theoretical_loss": 3.372563572173195, + "tokens_seen": 2741108736 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013703510531594786, + "loss": 1.18, + "theoretical_loss": 3.372557311653272, + "tokens_seen": 2741174272 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013701905717151456, + "loss": 1.1191, + "theoretical_loss": 3.3725510513249324, + "tokens_seen": 2741239808 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013700300902708126, + "loss": 1.1854, + "theoretical_loss": 3.372544791188165, + "tokens_seen": 2741305344 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013698696088264796, + "loss": 1.2801, + "theoretical_loss": 3.3725385312429594, + "tokens_seen": 2741370880 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013697091273821466, + "loss": 1.2088, + "theoretical_loss": 3.3725322714893062, + "tokens_seen": 2741436416 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013695486459378134, + "loss": 1.1817, + "theoretical_loss": 3.372526011927194, + "tokens_seen": 2741501952 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013693881644934804, + "loss": 1.2425, + "theoretical_loss": 3.372519752556612, + "tokens_seen": 2741567488 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013692276830491474, + "loss": 1.2716, + "theoretical_loss": 3.3725134933775514, + "tokens_seen": 2741633024 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013690672016048144, + "loss": 1.1663, + "theoretical_loss": 3.3725072343900004, + "tokens_seen": 2741698560 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013689067201604814, + "loss": 1.2813, + "theoretical_loss": 3.3725009755939483, + "tokens_seen": 2741764096 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013687462387161484, + "loss": 1.2141, + "theoretical_loss": 3.372494716989386, + "tokens_seen": 2741829632 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013685857572718154, + "loss": 1.2124, + "theoretical_loss": 3.372488458576302, + "tokens_seen": 2741895168 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013684252758274824, + "loss": 1.1407, + "theoretical_loss": 3.3724822003546864, + "tokens_seen": 2741960704 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013682647943831494, + "loss": 1.2642, + "theoretical_loss": 3.3724759423245283, + "tokens_seen": 2742026240 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013681043129388164, + "loss": 1.2202, + "theoretical_loss": 3.3724696844858175, + "tokens_seen": 2742091776 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013679438314944834, + "loss": 1.1538, + "theoretical_loss": 3.372463426838544, + "tokens_seen": 2742157312 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013677833500501505, + "loss": 1.2057, + "theoretical_loss": 3.3724571693826966, + "tokens_seen": 2742222848 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013676228686058175, + "loss": 1.1706, + "theoretical_loss": 3.372450912118265, + "tokens_seen": 2742288384 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013674623871614845, + "loss": 1.1729, + "theoretical_loss": 3.3724446550452396, + "tokens_seen": 2742353920 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013673019057171515, + "loss": 1.1613, + "theoretical_loss": 3.372438398163609, + "tokens_seen": 2742419456 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013671414242728185, + "loss": 1.1303, + "theoretical_loss": 3.372432141473363, + "tokens_seen": 2742484992 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013669809428284855, + "loss": 1.1263, + "theoretical_loss": 3.3724258849744917, + "tokens_seen": 2742550528 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013668204613841525, + "loss": 1.3351, + "theoretical_loss": 3.372419628666984, + "tokens_seen": 2742616064 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 916520, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1723819971084595, + "objective/train/theoretical_loss": 3.37241337255083, + "objective/train/tokens_used": 2763141600, + "theoretical_loss": 3.37241337255083, + "tokens_seen": 2742681600 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013666599799398195, + "loss": 1.1749, + "theoretical_loss": 3.37241337255083, + "tokens_seen": 2742681600 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013664994984954865, + "loss": 1.2596, + "theoretical_loss": 3.372407116626019, + "tokens_seen": 2742747136 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013663390170511535, + "loss": 1.1698, + "theoretical_loss": 3.3724008608925407, + "tokens_seen": 2742812672 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013661785356068205, + "loss": 1.2483, + "theoretical_loss": 3.3723946053503844, + "tokens_seen": 2742878208 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013660180541624875, + "loss": 1.1712, + "theoretical_loss": 3.37238834999954, + "tokens_seen": 2742943744 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013658575727181545, + "loss": 1.2577, + "theoretical_loss": 3.3723820948399967, + "tokens_seen": 2743009280 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013656970912738215, + "loss": 1.1905, + "theoretical_loss": 3.3723758398717445, + "tokens_seen": 2743074816 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013655366098294885, + "loss": 1.202, + "theoretical_loss": 3.3723695850947726, + "tokens_seen": 2743140352 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013653761283851555, + "loss": 1.2944, + "theoretical_loss": 3.372363330509071, + "tokens_seen": 2743205888 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013652156469408226, + "loss": 1.2341, + "theoretical_loss": 3.3723570761146293, + "tokens_seen": 2743271424 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013650551654964896, + "loss": 1.2815, + "theoretical_loss": 3.3723508219114366, + "tokens_seen": 2743336960 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013648946840521566, + "loss": 1.1729, + "theoretical_loss": 3.3723445678994826, + "tokens_seen": 2743402496 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013647342026078236, + "loss": 1.2789, + "theoretical_loss": 3.3723383140787573, + "tokens_seen": 2743468032 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013645737211634906, + "loss": 1.2405, + "theoretical_loss": 3.3723320604492497, + "tokens_seen": 2743533568 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013644132397191576, + "loss": 1.1805, + "theoretical_loss": 3.37232580701095, + "tokens_seen": 2743599104 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013642527582748246, + "loss": 1.1742, + "theoretical_loss": 3.3723195537638473, + "tokens_seen": 2743664640 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013640922768304916, + "loss": 1.2148, + "theoretical_loss": 3.3723133007079316, + "tokens_seen": 2743730176 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013639317953861586, + "loss": 1.2291, + "theoretical_loss": 3.372307047843192, + "tokens_seen": 2743795712 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013637713139418256, + "loss": 1.1784, + "theoretical_loss": 3.3723007951696182, + "tokens_seen": 2743861248 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013636108324974926, + "loss": 1.2197, + "theoretical_loss": 3.3722945426872, + "tokens_seen": 2743926784 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013634503510531596, + "loss": 1.2471, + "theoretical_loss": 3.372288290395927, + "tokens_seen": 2743992320 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013632898696088266, + "loss": 1.2667, + "theoretical_loss": 3.372282038295789, + "tokens_seen": 2744057856 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013631293881644936, + "loss": 1.1896, + "theoretical_loss": 3.372275786386775, + "tokens_seen": 2744123392 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013629689067201606, + "loss": 1.1476, + "theoretical_loss": 3.372269534668875, + "tokens_seen": 2744188928 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013628084252758276, + "loss": 1.2562, + "theoretical_loss": 3.3722632831420785, + "tokens_seen": 2744254464 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013626479438314947, + "loss": 1.1797, + "theoretical_loss": 3.372257031806375, + "tokens_seen": 2744320000 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013624874623871614, + "loss": 1.2092, + "theoretical_loss": 3.372250780661754, + "tokens_seen": 2744385536 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013623269809428284, + "loss": 1.2172, + "theoretical_loss": 3.372244529708206, + "tokens_seen": 2744451072 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013621664994984954, + "loss": 1.1277, + "theoretical_loss": 3.3722382789457193, + "tokens_seen": 2744516608 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013620060180541624, + "loss": 1.1888, + "theoretical_loss": 3.372232028374284, + "tokens_seen": 2744582144 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013618455366098294, + "loss": 1.2041, + "theoretical_loss": 3.37222577799389, + "tokens_seen": 2744647680 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013616850551654964, + "loss": 1.239, + "theoretical_loss": 3.3722195278045266, + "tokens_seen": 2744713216 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013615245737211634, + "loss": 1.186, + "theoretical_loss": 3.3722132778061837, + "tokens_seen": 2744778752 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013613640922768304, + "loss": 1.229, + "theoretical_loss": 3.3722070279988507, + "tokens_seen": 2744844288 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013612036108324974, + "loss": 1.1709, + "theoretical_loss": 3.372200778382517, + "tokens_seen": 2744909824 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013610431293881644, + "loss": 1.1623, + "theoretical_loss": 3.3721945289571718, + "tokens_seen": 2744975360 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013608826479438315, + "loss": 1.1314, + "theoretical_loss": 3.372188279722806, + "tokens_seen": 2745040896 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013607221664994985, + "loss": 1.1322, + "theoretical_loss": 3.3721820306794084, + "tokens_seen": 2745106432 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013605616850551655, + "loss": 1.067, + "theoretical_loss": 3.3721757818269684, + "tokens_seen": 2745171968 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013604012036108325, + "loss": 1.2299, + "theoretical_loss": 3.372169533165476, + "tokens_seen": 2745237504 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013602407221664998, + "loss": 1.1877, + "theoretical_loss": 3.372163284694921, + "tokens_seen": 2745303040 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013600802407221668, + "loss": 1.091, + "theoretical_loss": 3.3721570364152926, + "tokens_seen": 2745368576 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013599197592778338, + "loss": 1.1586, + "theoretical_loss": 3.3721507883265804, + "tokens_seen": 2745434112 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013597592778335008, + "loss": 1.2279, + "theoretical_loss": 3.372144540428774, + "tokens_seen": 2745499648 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013595987963891678, + "loss": 1.2029, + "theoretical_loss": 3.3721382927218637, + "tokens_seen": 2745565184 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013594383149448345, + "loss": 1.2885, + "theoretical_loss": 3.372132045205838, + "tokens_seen": 2745630720 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013592778335005015, + "loss": 1.3137, + "theoretical_loss": 3.3721257978806873, + "tokens_seen": 2745696256 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013591173520561685, + "loss": 1.1258, + "theoretical_loss": 3.372119550746401, + "tokens_seen": 2745761792 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013589568706118355, + "loss": 1.2802, + "theoretical_loss": 3.3721133038029687, + "tokens_seen": 2745827328 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013587963891675025, + "loss": 1.2046, + "theoretical_loss": 3.37210705705038, + "tokens_seen": 2745892864 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 917749, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2041547298431396, + "objective/train/theoretical_loss": 3.3721008104886243, + "objective/train/tokens_used": 2766418400, + "theoretical_loss": 3.3721008104886243, + "tokens_seen": 2745958400 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013586359077231695, + "loss": 1.239, + "theoretical_loss": 3.3721008104886243, + "tokens_seen": 2745958400 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013584754262788365, + "loss": 1.1621, + "theoretical_loss": 3.372094564117692, + "tokens_seen": 2746023936 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013583149448345036, + "loss": 1.1812, + "theoretical_loss": 3.372088317937571, + "tokens_seen": 2746089472 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013581544633901706, + "loss": 1.1459, + "theoretical_loss": 3.3720820719482534, + "tokens_seen": 2746155008 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013579939819458376, + "loss": 1.2313, + "theoretical_loss": 3.372075826149727, + "tokens_seen": 2746220544 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013578335005015046, + "loss": 1.2098, + "theoretical_loss": 3.3720695805419814, + "tokens_seen": 2746286080 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013576730190571716, + "loss": 1.1986, + "theoretical_loss": 3.3720633351250076, + "tokens_seen": 2746351616 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013575125376128386, + "loss": 1.2325, + "theoretical_loss": 3.3720570898987936, + "tokens_seen": 2746417152 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013573520561685056, + "loss": 1.1785, + "theoretical_loss": 3.37205084486333, + "tokens_seen": 2746482688 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013571915747241726, + "loss": 1.0955, + "theoretical_loss": 3.372044600018606, + "tokens_seen": 2746548224 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013570310932798396, + "loss": 1.2693, + "theoretical_loss": 3.3720383553646123, + "tokens_seen": 2746613760 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013568706118355066, + "loss": 1.2324, + "theoretical_loss": 3.372032110901337, + "tokens_seen": 2746679296 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013567101303911736, + "loss": 1.1979, + "theoretical_loss": 3.3720258666287704, + "tokens_seen": 2746744832 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013565496489468406, + "loss": 1.1721, + "theoretical_loss": 3.372019622546902, + "tokens_seen": 2746810368 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013563891675025076, + "loss": 1.2116, + "theoretical_loss": 3.372013378655722, + "tokens_seen": 2746875904 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013562286860581746, + "loss": 1.3021, + "theoretical_loss": 3.3720071349552185, + "tokens_seen": 2746941440 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013560682046138416, + "loss": 1.2013, + "theoretical_loss": 3.372000891445383, + "tokens_seen": 2747006976 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013559077231695087, + "loss": 1.2323, + "theoretical_loss": 3.371994648126204, + "tokens_seen": 2747072512 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013557472417251757, + "loss": 1.2744, + "theoretical_loss": 3.371988404997672, + "tokens_seen": 2747138048 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013555867602808424, + "loss": 1.1581, + "theoretical_loss": 3.3719821620597754, + "tokens_seen": 2747203584 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013554262788365094, + "loss": 1.1814, + "theoretical_loss": 3.3719759193125047, + "tokens_seen": 2747269120 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013552657973921764, + "loss": 1.1935, + "theoretical_loss": 3.3719696767558496, + "tokens_seen": 2747334656 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013551053159478434, + "loss": 1.1466, + "theoretical_loss": 3.371963434389799, + "tokens_seen": 2747400192 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013549448345035104, + "loss": 1.2106, + "theoretical_loss": 3.371957192214343, + "tokens_seen": 2747465728 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013547843530591774, + "loss": 1.2047, + "theoretical_loss": 3.3719509502294716, + "tokens_seen": 2747531264 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013546238716148444, + "loss": 1.2291, + "theoretical_loss": 3.3719447084351737, + "tokens_seen": 2747596800 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013544633901705114, + "loss": 1.1876, + "theoretical_loss": 3.37193846683144, + "tokens_seen": 2747662336 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013543029087261787, + "loss": 1.1989, + "theoretical_loss": 3.3719322254182584, + "tokens_seen": 2747727872 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013541424272818457, + "loss": 1.1429, + "theoretical_loss": 3.37192598419562, + "tokens_seen": 2747793408 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013539819458375127, + "loss": 1.1293, + "theoretical_loss": 3.3719197431635144, + "tokens_seen": 2747858944 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013538214643931797, + "loss": 1.2256, + "theoretical_loss": 3.371913502321931, + "tokens_seen": 2747924480 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013536609829488467, + "loss": 1.2787, + "theoretical_loss": 3.3719072616708585, + "tokens_seen": 2747990016 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013535005015045137, + "loss": 1.1638, + "theoretical_loss": 3.371901021210288, + "tokens_seen": 2748055552 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013533400200601808, + "loss": 1.1983, + "theoretical_loss": 3.371894780940208, + "tokens_seen": 2748121088 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013531795386158478, + "loss": 1.2306, + "theoretical_loss": 3.371888540860609, + "tokens_seen": 2748186624 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013530190571715148, + "loss": 1.1641, + "theoretical_loss": 3.37188230097148, + "tokens_seen": 2748252160 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013528585757271818, + "loss": 1.3056, + "theoretical_loss": 3.3718760612728107, + "tokens_seen": 2748317696 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013526980942828488, + "loss": 1.1736, + "theoretical_loss": 3.3718698217645913, + "tokens_seen": 2748383232 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013525376128385158, + "loss": 1.1988, + "theoretical_loss": 3.371863582446811, + "tokens_seen": 2748448768 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013523771313941825, + "loss": 1.3181, + "theoretical_loss": 3.37185734331946, + "tokens_seen": 2748514304 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013522166499498495, + "loss": 1.2981, + "theoretical_loss": 3.371851104382527, + "tokens_seen": 2748579840 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013520561685055165, + "loss": 1.2689, + "theoretical_loss": 3.371844865636002, + "tokens_seen": 2748645376 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013518956870611835, + "loss": 1.216, + "theoretical_loss": 3.3718386270798746, + "tokens_seen": 2748710912 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013517352056168505, + "loss": 1.2427, + "theoretical_loss": 3.3718323887141355, + "tokens_seen": 2748776448 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013515747241725176, + "loss": 1.2129, + "theoretical_loss": 3.371826150538773, + "tokens_seen": 2748841984 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013514142427281846, + "loss": 1.1509, + "theoretical_loss": 3.371819912553777, + "tokens_seen": 2748907520 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013512537612838516, + "loss": 1.273, + "theoretical_loss": 3.371813674759138, + "tokens_seen": 2748973056 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013510932798395186, + "loss": 1.1762, + "theoretical_loss": 3.3718074371548443, + "tokens_seen": 2749038592 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013509327983951856, + "loss": 1.1972, + "theoretical_loss": 3.371801199740887, + "tokens_seen": 2749104128 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013507723169508526, + "loss": 1.229, + "theoretical_loss": 3.3717949625172547, + "tokens_seen": 2749169664 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 918912, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1920430660247803, + "objective/train/theoretical_loss": 3.3717887254839374, + "objective/train/tokens_used": 2769695200, + "theoretical_loss": 3.3717887254839374, + "tokens_seen": 2749235200 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013506118355065196, + "loss": 1.1951, + "theoretical_loss": 3.3717887254839374, + "tokens_seen": 2749235200 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013504513540621866, + "loss": 1.1636, + "theoretical_loss": 3.3717824886409247, + "tokens_seen": 2749300736 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013502908726178536, + "loss": 1.1956, + "theoretical_loss": 3.371776251988207, + "tokens_seen": 2749366272 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013501303911735206, + "loss": 1.214, + "theoretical_loss": 3.3717700155257724, + "tokens_seen": 2749431808 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013499699097291876, + "loss": 1.2102, + "theoretical_loss": 3.371763779253612, + "tokens_seen": 2749497344 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013498094282848546, + "loss": 1.1956, + "theoretical_loss": 3.3717575431717144, + "tokens_seen": 2749562880 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013496489468405216, + "loss": 1.2128, + "theoretical_loss": 3.37175130728007, + "tokens_seen": 2749628416 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013494884653961886, + "loss": 1.2326, + "theoretical_loss": 3.3717450715786685, + "tokens_seen": 2749693952 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013493279839518556, + "loss": 1.2243, + "theoretical_loss": 3.371738836067499, + "tokens_seen": 2749759488 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013491675025075226, + "loss": 1.1887, + "theoretical_loss": 3.3717326007465513, + "tokens_seen": 2749825024 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013490070210631897, + "loss": 1.2744, + "theoretical_loss": 3.3717263656158156, + "tokens_seen": 2749890560 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013488465396188567, + "loss": 1.3104, + "theoretical_loss": 3.371720130675281, + "tokens_seen": 2749956096 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013486860581745237, + "loss": 1.2011, + "theoretical_loss": 3.3717138959249375, + "tokens_seen": 2750021632 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013485255767301904, + "loss": 1.2059, + "theoretical_loss": 3.3717076613647743, + "tokens_seen": 2750087168 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013483650952858574, + "loss": 1.2761, + "theoretical_loss": 3.3717014269947816, + "tokens_seen": 2750152704 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013482046138415247, + "loss": 1.1598, + "theoretical_loss": 3.371695192814949, + "tokens_seen": 2750218240 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013480441323971917, + "loss": 1.2348, + "theoretical_loss": 3.3716889588252656, + "tokens_seen": 2750283776 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013478836509528587, + "loss": 1.3102, + "theoretical_loss": 3.371682725025722, + "tokens_seen": 2750349312 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013477231695085257, + "loss": 1.1449, + "theoretical_loss": 3.3716764914163067, + "tokens_seen": 2750414848 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013475626880641927, + "loss": 1.1573, + "theoretical_loss": 3.3716702579970104, + "tokens_seen": 2750480384 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013474022066198597, + "loss": 1.1647, + "theoretical_loss": 3.3716640247678225, + "tokens_seen": 2750545920 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013472417251755267, + "loss": 1.2041, + "theoretical_loss": 3.3716577917287323, + "tokens_seen": 2750611456 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013470812437311937, + "loss": 1.2207, + "theoretical_loss": 3.37165155887973, + "tokens_seen": 2750676992 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013469207622868607, + "loss": 1.2137, + "theoretical_loss": 3.3716453262208046, + "tokens_seen": 2750742528 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013467602808425277, + "loss": 1.2133, + "theoretical_loss": 3.3716390937519467, + "tokens_seen": 2750808064 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013465997993981947, + "loss": 1.0833, + "theoretical_loss": 3.3716328614731452, + "tokens_seen": 2750873600 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013464393179538618, + "loss": 1.1543, + "theoretical_loss": 3.37162662938439, + "tokens_seen": 2750939136 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013462788365095288, + "loss": 1.2169, + "theoretical_loss": 3.3716203974856707, + "tokens_seen": 2751004672 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013461183550651958, + "loss": 1.2404, + "theoretical_loss": 3.3716141657769776, + "tokens_seen": 2751070208 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013459578736208628, + "loss": 1.2146, + "theoretical_loss": 3.3716079342582996, + "tokens_seen": 2751135744 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013457973921765298, + "loss": 1.235, + "theoretical_loss": 3.3716017029296266, + "tokens_seen": 2751201280 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013456369107321968, + "loss": 1.0339, + "theoretical_loss": 3.371595471790948, + "tokens_seen": 2751266816 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013454764292878638, + "loss": 1.3238, + "theoretical_loss": 3.3715892408422543, + "tokens_seen": 2751332352 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013453159478435305, + "loss": 1.1583, + "theoretical_loss": 3.3715830100835347, + "tokens_seen": 2751397888 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013451554663991975, + "loss": 1.2337, + "theoretical_loss": 3.3715767795147786, + "tokens_seen": 2751463424 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013449949849548645, + "loss": 1.1953, + "theoretical_loss": 3.3715705491359764, + "tokens_seen": 2751528960 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013448345035105315, + "loss": 1.2389, + "theoretical_loss": 3.371564318947117, + "tokens_seen": 2751594496 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013446740220661986, + "loss": 1.1672, + "theoretical_loss": 3.3715580889481904, + "tokens_seen": 2751660032 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013445135406218656, + "loss": 1.272, + "theoretical_loss": 3.3715518591391866, + "tokens_seen": 2751725568 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013443530591775326, + "loss": 1.2003, + "theoretical_loss": 3.371545629520095, + "tokens_seen": 2751791104 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013441925777331996, + "loss": 1.12, + "theoretical_loss": 3.371539400090905, + "tokens_seen": 2751856640 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013440320962888666, + "loss": 1.2311, + "theoretical_loss": 3.371533170851607, + "tokens_seen": 2751922176 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013438716148445336, + "loss": 1.2581, + "theoretical_loss": 3.37152694180219, + "tokens_seen": 2751987712 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013437111334002006, + "loss": 1.2599, + "theoretical_loss": 3.371520712942644, + "tokens_seen": 2752053248 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013435506519558676, + "loss": 1.1705, + "theoretical_loss": 3.3715144842729585, + "tokens_seen": 2752118784 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013433901705115346, + "loss": 1.317, + "theoretical_loss": 3.3715082557931235, + "tokens_seen": 2752184320 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013432296890672016, + "loss": 1.1233, + "theoretical_loss": 3.3715020275031287, + "tokens_seen": 2752249856 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013430692076228686, + "loss": 1.2343, + "theoretical_loss": 3.3714957994029633, + "tokens_seen": 2752315392 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013429087261785356, + "loss": 1.1871, + "theoretical_loss": 3.3714895714926176, + "tokens_seen": 2752380928 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013427482447342026, + "loss": 1.2123, + "theoretical_loss": 3.371483343772081, + "tokens_seen": 2752446464 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 919874, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2211925983428955, + "objective/train/theoretical_loss": 3.3714771162413433, + "objective/train/tokens_used": 2772972000, + "theoretical_loss": 3.3714771162413433, + "tokens_seen": 2752512000 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013425877632898696, + "loss": 1.2464, + "theoretical_loss": 3.3714771162413433, + "tokens_seen": 2752512000 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013424272818455366, + "loss": 1.2255, + "theoretical_loss": 3.3714708889003937, + "tokens_seen": 2752577536 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013422668004012036, + "loss": 1.2319, + "theoretical_loss": 3.371464661749223, + "tokens_seen": 2752643072 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013421063189568707, + "loss": 1.2474, + "theoretical_loss": 3.3714584347878196, + "tokens_seen": 2752708608 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013419458375125377, + "loss": 1.1775, + "theoretical_loss": 3.371452208016174, + "tokens_seen": 2752774144 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013417853560682047, + "loss": 1.1834, + "theoretical_loss": 3.371445981434276, + "tokens_seen": 2752839680 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013416248746238717, + "loss": 1.1982, + "theoretical_loss": 3.3714397550421147, + "tokens_seen": 2752905216 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013414643931795387, + "loss": 1.2032, + "theoretical_loss": 3.37143352883968, + "tokens_seen": 2752970752 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013413039117352057, + "loss": 1.1368, + "theoretical_loss": 3.3714273028269623, + "tokens_seen": 2753036288 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013411434302908727, + "loss": 1.2415, + "theoretical_loss": 3.37142107700395, + "tokens_seen": 2753101824 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013409829488465397, + "loss": 1.2207, + "theoretical_loss": 3.371414851370634, + "tokens_seen": 2753167360 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013408224674022067, + "loss": 1.262, + "theoretical_loss": 3.3714086259270033, + "tokens_seen": 2753232896 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013406619859578737, + "loss": 1.2269, + "theoretical_loss": 3.3714024006730483, + "tokens_seen": 2753298432 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013405015045135407, + "loss": 1.1935, + "theoretical_loss": 3.3713961756087576, + "tokens_seen": 2753363968 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013403410230692077, + "loss": 1.1659, + "theoretical_loss": 3.3713899507341223, + "tokens_seen": 2753429504 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013401805416248747, + "loss": 1.1691, + "theoretical_loss": 3.371383726049131, + "tokens_seen": 2753495040 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013400200601805417, + "loss": 1.1909, + "theoretical_loss": 3.3713775015537735, + "tokens_seen": 2753560576 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013398595787362087, + "loss": 1.2253, + "theoretical_loss": 3.37137127724804, + "tokens_seen": 2753626112 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013396990972918757, + "loss": 1.3036, + "theoretical_loss": 3.3713650531319197, + "tokens_seen": 2753691648 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013395386158475428, + "loss": 1.3517, + "theoretical_loss": 3.371358829205403, + "tokens_seen": 2753757184 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013393781344032098, + "loss": 1.155, + "theoretical_loss": 3.371352605468479, + "tokens_seen": 2753822720 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013392176529588768, + "loss": 1.1987, + "theoretical_loss": 3.371346381921138, + "tokens_seen": 2753888256 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013390571715145438, + "loss": 1.3025, + "theoretical_loss": 3.3713401585633687, + "tokens_seen": 2753953792 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013388966900702108, + "loss": 1.1881, + "theoretical_loss": 3.371333935395162, + "tokens_seen": 2754019328 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013387362086258778, + "loss": 1.1925, + "theoretical_loss": 3.3713277124165066, + "tokens_seen": 2754084864 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013385757271815448, + "loss": 1.1825, + "theoretical_loss": 3.371321489627393, + "tokens_seen": 2754150400 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013384152457372115, + "loss": 1.135, + "theoretical_loss": 3.3713152670278106, + "tokens_seen": 2754215936 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013382547642928785, + "loss": 1.3062, + "theoretical_loss": 3.371309044617749, + "tokens_seen": 2754281472 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013380942828485455, + "loss": 1.2512, + "theoretical_loss": 3.3713028223971984, + "tokens_seen": 2754347008 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013379338014042125, + "loss": 1.2047, + "theoretical_loss": 3.3712966003661475, + "tokens_seen": 2754412544 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013377733199598796, + "loss": 1.24, + "theoretical_loss": 3.371290378524587, + "tokens_seen": 2754478080 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013376128385155466, + "loss": 1.1802, + "theoretical_loss": 3.3712841568725067, + "tokens_seen": 2754543616 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013374523570712136, + "loss": 1.2714, + "theoretical_loss": 3.3712779354098954, + "tokens_seen": 2754609152 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013372918756268806, + "loss": 1.1748, + "theoretical_loss": 3.3712717141367436, + "tokens_seen": 2754674688 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013371313941825476, + "loss": 1.2408, + "theoretical_loss": 3.3712654930530404, + "tokens_seen": 2754740224 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013369709127382146, + "loss": 1.1141, + "theoretical_loss": 3.371259272158776, + "tokens_seen": 2754805760 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013368104312938816, + "loss": 1.2164, + "theoretical_loss": 3.3712530514539405, + "tokens_seen": 2754871296 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013366499498495486, + "loss": 1.1565, + "theoretical_loss": 3.3712468309385226, + "tokens_seen": 2754936832 + }, + { + "epoch": 0.83, + "learning_rate": 0.00013364894684052156, + "loss": 1.1714, + "theoretical_loss": 3.371240610612513, + "tokens_seen": 2755002368 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001336328986960883, + "loss": 1.1898, + "theoretical_loss": 3.371234390475901, + "tokens_seen": 2755067904 + }, + { + "epoch": 0.83, + "learning_rate": 0.000133616850551655, + "loss": 1.187, + "theoretical_loss": 3.371228170528676, + "tokens_seen": 2755133440 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001336008024072217, + "loss": 1.1858, + "theoretical_loss": 3.371221950770828, + "tokens_seen": 2755198976 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001335847542627884, + "loss": 1.2342, + "theoretical_loss": 3.371215731202347, + "tokens_seen": 2755264512 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001335687061183551, + "loss": 1.2026, + "theoretical_loss": 3.3712095118232224, + "tokens_seen": 2755330048 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001335526579739218, + "loss": 1.2184, + "theoretical_loss": 3.371203292633444, + "tokens_seen": 2755395584 + }, + { + "epoch": 0.83, + "learning_rate": 0.0001335366098294885, + "loss": 1.2284, + "theoretical_loss": 3.3711970736330015, + "tokens_seen": 2755461120 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013352056168505517, + "loss": 1.2129, + "theoretical_loss": 3.3711908548218847, + "tokens_seen": 2755526656 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013350451354062187, + "loss": 1.2647, + "theoretical_loss": 3.371184636200084, + "tokens_seen": 2755592192 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013348846539618857, + "loss": 1.0326, + "theoretical_loss": 3.371178417767588, + "tokens_seen": 2755657728 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013347241725175527, + "loss": 1.1824, + "theoretical_loss": 3.371172199524387, + "tokens_seen": 2755723264 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 921031, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1628398895263672, + "objective/train/theoretical_loss": 3.37116598147047, + "objective/train/tokens_used": 2776248800, + "theoretical_loss": 3.37116598147047, + "tokens_seen": 2755788800 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013345636910732197, + "loss": 1.1921, + "theoretical_loss": 3.37116598147047, + "tokens_seen": 2755788800 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013344032096288867, + "loss": 1.2765, + "theoretical_loss": 3.3711597636058284, + "tokens_seen": 2755854336 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013342427281845537, + "loss": 1.2893, + "theoretical_loss": 3.3711535459304502, + "tokens_seen": 2755919872 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013340822467402207, + "loss": 1.251, + "theoretical_loss": 3.371147328444326, + "tokens_seen": 2755985408 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013339217652958877, + "loss": 1.2413, + "theoretical_loss": 3.3711411111474456, + "tokens_seen": 2756050944 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013337612838515547, + "loss": 1.2709, + "theoretical_loss": 3.3711348940397987, + "tokens_seen": 2756116480 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013336008024072217, + "loss": 1.2751, + "theoretical_loss": 3.3711286771213747, + "tokens_seen": 2756182016 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013334403209628887, + "loss": 1.3046, + "theoretical_loss": 3.3711224603921632, + "tokens_seen": 2756247552 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013332798395185557, + "loss": 1.2615, + "theoretical_loss": 3.3711162438521542, + "tokens_seen": 2756313088 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013331193580742227, + "loss": 1.2169, + "theoretical_loss": 3.371110027501338, + "tokens_seen": 2756378624 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013329588766298897, + "loss": 1.2263, + "theoretical_loss": 3.371103811339704, + "tokens_seen": 2756444160 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013327983951855568, + "loss": 1.1832, + "theoretical_loss": 3.3710975953672415, + "tokens_seen": 2756509696 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013326379137412238, + "loss": 1.3173, + "theoretical_loss": 3.37109137958394, + "tokens_seen": 2756575232 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013324774322968908, + "loss": 1.1588, + "theoretical_loss": 3.3710851639897905, + "tokens_seen": 2756640768 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013323169508525578, + "loss": 1.2463, + "theoretical_loss": 3.371078948584782, + "tokens_seen": 2756706304 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013321564694082248, + "loss": 1.1906, + "theoretical_loss": 3.371072733368904, + "tokens_seen": 2756771840 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013319959879638918, + "loss": 1.1687, + "theoretical_loss": 3.371066518342147, + "tokens_seen": 2756837376 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013318355065195588, + "loss": 1.2007, + "theoretical_loss": 3.3710603035044997, + "tokens_seen": 2756902912 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013316750250752258, + "loss": 1.2768, + "theoretical_loss": 3.371054088855953, + "tokens_seen": 2756968448 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013315145436308928, + "loss": 1.3704, + "theoretical_loss": 3.3710478743964956, + "tokens_seen": 2757033984 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013313540621865595, + "loss": 1.1709, + "theoretical_loss": 3.371041660126118, + "tokens_seen": 2757099520 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013311935807422265, + "loss": 1.2199, + "theoretical_loss": 3.3710354460448095, + "tokens_seen": 2757165056 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013310330992978936, + "loss": 1.2243, + "theoretical_loss": 3.3710292321525603, + "tokens_seen": 2757230592 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013308726178535606, + "loss": 1.162, + "theoretical_loss": 3.3710230184493595, + "tokens_seen": 2757296128 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013307121364092276, + "loss": 1.2104, + "theoretical_loss": 3.3710168049351976, + "tokens_seen": 2757361664 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013305516549648946, + "loss": 1.1265, + "theoretical_loss": 3.371010591610064, + "tokens_seen": 2757427200 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013303911735205616, + "loss": 1.2853, + "theoretical_loss": 3.3710043784739483, + "tokens_seen": 2757492736 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013302306920762289, + "loss": 1.2649, + "theoretical_loss": 3.3709981655268404, + "tokens_seen": 2757558272 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013300702106318959, + "loss": 1.2869, + "theoretical_loss": 3.3709919527687306, + "tokens_seen": 2757623808 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001329909729187563, + "loss": 1.2748, + "theoretical_loss": 3.3709857401996075, + "tokens_seen": 2757689344 + }, + { + "epoch": 0.84, + "learning_rate": 0.000132974924774323, + "loss": 1.1141, + "theoretical_loss": 3.3709795278194616, + "tokens_seen": 2757754880 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001329588766298897, + "loss": 1.1851, + "theoretical_loss": 3.370973315628283, + "tokens_seen": 2757820416 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001329428284854564, + "loss": 1.2029, + "theoretical_loss": 3.370967103626061, + "tokens_seen": 2757885952 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001329267803410231, + "loss": 1.0943, + "theoretical_loss": 3.3709608918127847, + "tokens_seen": 2757951488 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001329107321965898, + "loss": 1.2149, + "theoretical_loss": 3.370954680188445, + "tokens_seen": 2758017024 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001328946840521565, + "loss": 1.1917, + "theoretical_loss": 3.3709484687530313, + "tokens_seen": 2758082560 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001328786359077232, + "loss": 1.206, + "theoretical_loss": 3.370942257506533, + "tokens_seen": 2758148096 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001328625877632899, + "loss": 1.2656, + "theoretical_loss": 3.37093604644894, + "tokens_seen": 2758213632 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001328465396188566, + "loss": 1.2502, + "theoretical_loss": 3.370929835580243, + "tokens_seen": 2758279168 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001328304914744233, + "loss": 1.2151, + "theoretical_loss": 3.3709236249004304, + "tokens_seen": 2758344704 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013281444332998997, + "loss": 1.245, + "theoretical_loss": 3.370917414409493, + "tokens_seen": 2758410240 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013279839518555667, + "loss": 1.2213, + "theoretical_loss": 3.3709112041074194, + "tokens_seen": 2758475776 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013278234704112337, + "loss": 1.2285, + "theoretical_loss": 3.3709049939942006, + "tokens_seen": 2758541312 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013276629889669007, + "loss": 1.2227, + "theoretical_loss": 3.3708987840698255, + "tokens_seen": 2758606848 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013275025075225677, + "loss": 1.2365, + "theoretical_loss": 3.3708925743342846, + "tokens_seen": 2758672384 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013273420260782347, + "loss": 1.1997, + "theoretical_loss": 3.3708863647875673, + "tokens_seen": 2758737920 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013271815446339017, + "loss": 1.296, + "theoretical_loss": 3.370880155429663, + "tokens_seen": 2758803456 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013270210631895687, + "loss": 1.1504, + "theoretical_loss": 3.370873946260562, + "tokens_seen": 2758868992 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013268605817452357, + "loss": 1.2075, + "theoretical_loss": 3.3708677372802542, + "tokens_seen": 2758934528 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013267001003009027, + "loss": 1.2237, + "theoretical_loss": 3.370861528488729, + "tokens_seen": 2759000064 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 921894, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1541354656219482, + "objective/train/theoretical_loss": 3.370855319885976, + "objective/train/tokens_used": 2779525600, + "theoretical_loss": 3.370855319885976, + "tokens_seen": 2759065600 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013265396188565697, + "loss": 1.1514, + "theoretical_loss": 3.370855319885976, + "tokens_seen": 2759065600 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013263791374122367, + "loss": 1.1879, + "theoretical_loss": 3.370849111471985, + "tokens_seen": 2759131136 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013262186559679037, + "loss": 1.26, + "theoretical_loss": 3.370842903246747, + "tokens_seen": 2759196672 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013260581745235707, + "loss": 1.1586, + "theoretical_loss": 3.37083669521025, + "tokens_seen": 2759262208 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013258976930792378, + "loss": 1.1833, + "theoretical_loss": 3.370830487362485, + "tokens_seen": 2759327744 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013257372116349048, + "loss": 1.2935, + "theoretical_loss": 3.370824279703441, + "tokens_seen": 2759393280 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013255767301905718, + "loss": 1.2013, + "theoretical_loss": 3.3708180722331083, + "tokens_seen": 2759458816 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013254162487462388, + "loss": 1.2403, + "theoretical_loss": 3.3708118649514764, + "tokens_seen": 2759524352 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013252557673019058, + "loss": 1.2208, + "theoretical_loss": 3.3708056578585355, + "tokens_seen": 2759589888 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013250952858575728, + "loss": 1.2808, + "theoretical_loss": 3.370799450954275, + "tokens_seen": 2759655424 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013249348044132398, + "loss": 1.1123, + "theoretical_loss": 3.370793244238685, + "tokens_seen": 2759720960 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013247743229689068, + "loss": 1.2021, + "theoretical_loss": 3.3707870377117546, + "tokens_seen": 2759786496 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013246138415245738, + "loss": 1.2405, + "theoretical_loss": 3.3707808313734744, + "tokens_seen": 2759852032 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013244533600802408, + "loss": 1.1562, + "theoretical_loss": 3.3707746252238335, + "tokens_seen": 2759917568 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013242928786359078, + "loss": 1.196, + "theoretical_loss": 3.3707684192628227, + "tokens_seen": 2759983104 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013241323971915748, + "loss": 1.1779, + "theoretical_loss": 3.3707622134904303, + "tokens_seen": 2760048640 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013239719157472418, + "loss": 1.1875, + "theoretical_loss": 3.3707560079066474, + "tokens_seen": 2760114176 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013238114343029088, + "loss": 1.2066, + "theoretical_loss": 3.370749802511463, + "tokens_seen": 2760179712 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013236509528585758, + "loss": 1.1973, + "theoretical_loss": 3.3707435973048674, + "tokens_seen": 2760245248 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013234904714142428, + "loss": 1.2003, + "theoretical_loss": 3.3707373922868498, + "tokens_seen": 2760310784 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013233299899699099, + "loss": 1.2164, + "theoretical_loss": 3.370731187457401, + "tokens_seen": 2760376320 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013231695085255769, + "loss": 1.1876, + "theoretical_loss": 3.3707249828165096, + "tokens_seen": 2760441856 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001323009027081244, + "loss": 1.2366, + "theoretical_loss": 3.370718778364166, + "tokens_seen": 2760507392 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001322848545636911, + "loss": 1.2531, + "theoretical_loss": 3.3707125741003603, + "tokens_seen": 2760572928 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001322688064192578, + "loss": 1.1253, + "theoretical_loss": 3.3707063700250814, + "tokens_seen": 2760638464 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001322527582748245, + "loss": 1.1826, + "theoretical_loss": 3.3707001661383202, + "tokens_seen": 2760704000 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001322367101303912, + "loss": 1.1795, + "theoretical_loss": 3.3706939624400656, + "tokens_seen": 2760769536 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001322206619859579, + "loss": 1.1817, + "theoretical_loss": 3.3706877589303077, + "tokens_seen": 2760835072 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001322046138415246, + "loss": 1.1527, + "theoretical_loss": 3.3706815556090364, + "tokens_seen": 2760900608 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001321885656970913, + "loss": 1.2558, + "theoretical_loss": 3.3706753524762414, + "tokens_seen": 2760966144 + }, + { + "epoch": 0.84, + "learning_rate": 0.000132172517552658, + "loss": 1.2851, + "theoretical_loss": 3.3706691495319125, + "tokens_seen": 2761031680 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001321564694082247, + "loss": 1.2009, + "theoretical_loss": 3.3706629467760396, + "tokens_seen": 2761097216 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001321404212637914, + "loss": 1.2496, + "theoretical_loss": 3.3706567442086124, + "tokens_seen": 2761162752 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001321243731193581, + "loss": 1.1776, + "theoretical_loss": 3.3706505418296206, + "tokens_seen": 2761228288 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013210832497492477, + "loss": 1.2123, + "theoretical_loss": 3.3706443396390546, + "tokens_seen": 2761293824 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013209227683049147, + "loss": 1.2248, + "theoretical_loss": 3.3706381376369032, + "tokens_seen": 2761359360 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013207622868605817, + "loss": 1.2213, + "theoretical_loss": 3.370631935823157, + "tokens_seen": 2761424896 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013206018054162487, + "loss": 1.2216, + "theoretical_loss": 3.3706257341978056, + "tokens_seen": 2761490432 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013204413239719157, + "loss": 1.2389, + "theoretical_loss": 3.3706195327608386, + "tokens_seen": 2761555968 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013202808425275827, + "loss": 1.2101, + "theoretical_loss": 3.3706133315122457, + "tokens_seen": 2761621504 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013201203610832497, + "loss": 1.1573, + "theoretical_loss": 3.3706071304520178, + "tokens_seen": 2761687040 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013199598796389167, + "loss": 1.2682, + "theoretical_loss": 3.370600929580143, + "tokens_seen": 2761752576 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013197993981945837, + "loss": 1.2127, + "theoretical_loss": 3.3705947288966125, + "tokens_seen": 2761818112 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013196389167502507, + "loss": 1.2063, + "theoretical_loss": 3.3705885284014157, + "tokens_seen": 2761883648 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013194784353059177, + "loss": 1.1223, + "theoretical_loss": 3.3705823280945415, + "tokens_seen": 2761949184 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013193179538615847, + "loss": 1.3583, + "theoretical_loss": 3.370576127975981, + "tokens_seen": 2762014720 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013191574724172517, + "loss": 1.3131, + "theoretical_loss": 3.370569928045724, + "tokens_seen": 2762080256 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013189969909729188, + "loss": 1.1781, + "theoretical_loss": 3.3705637283037593, + "tokens_seen": 2762145792 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013188365095285858, + "loss": 1.25, + "theoretical_loss": 3.3705575287500773, + "tokens_seen": 2762211328 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013186760280842528, + "loss": 1.2548, + "theoretical_loss": 3.370551329384668, + "tokens_seen": 2762276864 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 923138, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2166162729263306, + "objective/train/theoretical_loss": 3.370545130207521, + "objective/train/tokens_used": 2782802400, + "theoretical_loss": 3.370545130207521, + "tokens_seen": 2762342400 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013185155466399198, + "loss": 1.2282, + "theoretical_loss": 3.370545130207521, + "tokens_seen": 2762342400 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013183550651955868, + "loss": 1.2431, + "theoretical_loss": 3.3705389312186256, + "tokens_seen": 2762407936 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001318194583751254, + "loss": 1.1439, + "theoretical_loss": 3.370532732417973, + "tokens_seen": 2762473472 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013180341023069208, + "loss": 1.1546, + "theoretical_loss": 3.3705265338055512, + "tokens_seen": 2762539008 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013178736208625878, + "loss": 1.2931, + "theoretical_loss": 3.3705203353813515, + "tokens_seen": 2762604544 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013177131394182548, + "loss": 1.1872, + "theoretical_loss": 3.370514137145363, + "tokens_seen": 2762670080 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013175526579739218, + "loss": 1.2442, + "theoretical_loss": 3.3705079390975756, + "tokens_seen": 2762735616 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013173921765295888, + "loss": 1.2499, + "theoretical_loss": 3.3705017412379794, + "tokens_seen": 2762801152 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013172316950852558, + "loss": 1.2363, + "theoretical_loss": 3.3704955435665642, + "tokens_seen": 2762866688 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013170712136409228, + "loss": 1.2567, + "theoretical_loss": 3.3704893460833194, + "tokens_seen": 2762932224 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013169107321965898, + "loss": 1.2725, + "theoretical_loss": 3.370483148788235, + "tokens_seen": 2762997760 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013167502507522568, + "loss": 1.2067, + "theoretical_loss": 3.3704769516813013, + "tokens_seen": 2763063296 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013165897693079239, + "loss": 1.2191, + "theoretical_loss": 3.3704707547625077, + "tokens_seen": 2763128832 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013164292878635909, + "loss": 1.1749, + "theoretical_loss": 3.370464558031844, + "tokens_seen": 2763194368 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013162688064192579, + "loss": 1.2404, + "theoretical_loss": 3.3704583614893, + "tokens_seen": 2763259904 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001316108324974925, + "loss": 1.2029, + "theoretical_loss": 3.3704521651348656, + "tokens_seen": 2763325440 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001315947843530592, + "loss": 1.1922, + "theoretical_loss": 3.3704459689685304, + "tokens_seen": 2763390976 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001315787362086259, + "loss": 1.2628, + "theoretical_loss": 3.370439772990285, + "tokens_seen": 2763456512 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001315626880641926, + "loss": 1.2406, + "theoretical_loss": 3.3704335772001186, + "tokens_seen": 2763522048 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001315466399197593, + "loss": 1.2072, + "theoretical_loss": 3.370427381598021, + "tokens_seen": 2763587584 + }, + { + "epoch": 0.84, + "learning_rate": 0.000131530591775326, + "loss": 1.1319, + "theoretical_loss": 3.3704211861839823, + "tokens_seen": 2763653120 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001315145436308927, + "loss": 1.2526, + "theoretical_loss": 3.370414990957992, + "tokens_seen": 2763718656 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001314984954864594, + "loss": 1.1072, + "theoretical_loss": 3.37040879592004, + "tokens_seen": 2763784192 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001314824473420261, + "loss": 1.1258, + "theoretical_loss": 3.3704026010701167, + "tokens_seen": 2763849728 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001314663991975928, + "loss": 1.158, + "theoretical_loss": 3.370396406408211, + "tokens_seen": 2763915264 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001314503510531595, + "loss": 1.1822, + "theoretical_loss": 3.3703902119343137, + "tokens_seen": 2763980800 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001314343029087262, + "loss": 1.2164, + "theoretical_loss": 3.370384017648414, + "tokens_seen": 2764046336 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013141825476429287, + "loss": 1.2522, + "theoretical_loss": 3.370377823550502, + "tokens_seen": 2764111872 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013140220661985957, + "loss": 1.1937, + "theoretical_loss": 3.3703716296405672, + "tokens_seen": 2764177408 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013138615847542627, + "loss": 1.0612, + "theoretical_loss": 3.3703654359186, + "tokens_seen": 2764242944 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013137011033099297, + "loss": 1.2551, + "theoretical_loss": 3.37035924238459, + "tokens_seen": 2764308480 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013135406218655967, + "loss": 1.2368, + "theoretical_loss": 3.3703530490385267, + "tokens_seen": 2764374016 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013133801404212637, + "loss": 1.1429, + "theoretical_loss": 3.3703468558804, + "tokens_seen": 2764439552 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013132196589769307, + "loss": 1.2269, + "theoretical_loss": 3.3703406629102, + "tokens_seen": 2764505088 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013130591775325977, + "loss": 1.198, + "theoretical_loss": 3.370334470127917, + "tokens_seen": 2764570624 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013128986960882647, + "loss": 1.1366, + "theoretical_loss": 3.37032827753354, + "tokens_seen": 2764636160 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013127382146439317, + "loss": 1.2264, + "theoretical_loss": 3.3703220851270594, + "tokens_seen": 2764701696 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013125777331995987, + "loss": 1.2027, + "theoretical_loss": 3.3703158929084647, + "tokens_seen": 2764767232 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013124172517552657, + "loss": 1.1575, + "theoretical_loss": 3.370309700877746, + "tokens_seen": 2764832768 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001312256770310933, + "loss": 1.2208, + "theoretical_loss": 3.3703035090348927, + "tokens_seen": 2764898304 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013120962888666, + "loss": 1.2909, + "theoretical_loss": 3.3702973173798956, + "tokens_seen": 2764963840 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001311935807422267, + "loss": 1.195, + "theoretical_loss": 3.3702911259127433, + "tokens_seen": 2765029376 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001311775325977934, + "loss": 1.1828, + "theoretical_loss": 3.3702849346334265, + "tokens_seen": 2765094912 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001311614844533601, + "loss": 1.1543, + "theoretical_loss": 3.3702787435419346, + "tokens_seen": 2765160448 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001311454363089268, + "loss": 1.2071, + "theoretical_loss": 3.3702725526382578, + "tokens_seen": 2765225984 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001311293881644935, + "loss": 1.1957, + "theoretical_loss": 3.370266361922386, + "tokens_seen": 2765291520 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001311133400200602, + "loss": 1.1464, + "theoretical_loss": 3.3702601713943086, + "tokens_seen": 2765357056 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013109729187562688, + "loss": 1.0962, + "theoretical_loss": 3.3702539810540157, + "tokens_seen": 2765422592 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013108124373119358, + "loss": 1.1762, + "theoretical_loss": 3.370247790901497, + "tokens_seen": 2765488128 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013106519558676028, + "loss": 1.1895, + "theoretical_loss": 3.370241600936743, + "tokens_seen": 2765553664 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 924051, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2223986387252808, + "objective/train/theoretical_loss": 3.3702354111597432, + "objective/train/tokens_used": 2786079200, + "theoretical_loss": 3.3702354111597432, + "tokens_seen": 2765619200 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013104914744232698, + "loss": 1.2109, + "theoretical_loss": 3.3702354111597432, + "tokens_seen": 2765619200 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013103309929789368, + "loss": 1.262, + "theoretical_loss": 3.370229221570487, + "tokens_seen": 2765684736 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013101705115346038, + "loss": 1.109, + "theoretical_loss": 3.3702230321689646, + "tokens_seen": 2765750272 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013100100300902708, + "loss": 1.2318, + "theoretical_loss": 3.370216842955166, + "tokens_seen": 2765815808 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013098495486459378, + "loss": 1.2676, + "theoretical_loss": 3.370210653929081, + "tokens_seen": 2765881344 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013096890672016049, + "loss": 1.1739, + "theoretical_loss": 3.3702044650906995, + "tokens_seen": 2765946880 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013095285857572719, + "loss": 1.2275, + "theoretical_loss": 3.3701982764400107, + "tokens_seen": 2766012416 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001309368104312939, + "loss": 1.2441, + "theoretical_loss": 3.3701920879770055, + "tokens_seen": 2766077952 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001309207622868606, + "loss": 1.2056, + "theoretical_loss": 3.370185899701673, + "tokens_seen": 2766143488 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001309047141424273, + "loss": 1.0927, + "theoretical_loss": 3.370179711614003, + "tokens_seen": 2766209024 + }, + { + "epoch": 0.84, + "learning_rate": 0.000130888665997994, + "loss": 1.1049, + "theoretical_loss": 3.3701735237139863, + "tokens_seen": 2766274560 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001308726178535607, + "loss": 1.1625, + "theoretical_loss": 3.370167336001612, + "tokens_seen": 2766340096 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001308565697091274, + "loss": 1.195, + "theoretical_loss": 3.3701611484768694, + "tokens_seen": 2766405632 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001308405215646941, + "loss": 1.2202, + "theoretical_loss": 3.37015496113975, + "tokens_seen": 2766471168 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001308244734202608, + "loss": 1.1451, + "theoretical_loss": 3.370148773990242, + "tokens_seen": 2766536704 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001308084252758275, + "loss": 1.1372, + "theoretical_loss": 3.3701425870283366, + "tokens_seen": 2766602240 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001307923771313942, + "loss": 1.279, + "theoretical_loss": 3.3701364002540233, + "tokens_seen": 2766667776 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001307763289869609, + "loss": 1.1673, + "theoretical_loss": 3.370130213667291, + "tokens_seen": 2766733312 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001307602808425276, + "loss": 1.1529, + "theoretical_loss": 3.3701240272681305, + "tokens_seen": 2766798848 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001307442326980943, + "loss": 1.2229, + "theoretical_loss": 3.370117841056532, + "tokens_seen": 2766864384 + }, + { + "epoch": 0.84, + "learning_rate": 0.000130728184553661, + "loss": 1.1708, + "theoretical_loss": 3.3701116550324843, + "tokens_seen": 2766929920 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013071213640922767, + "loss": 1.224, + "theoretical_loss": 3.370105469195978, + "tokens_seen": 2766995456 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013069608826479437, + "loss": 1.2212, + "theoretical_loss": 3.370099283547003, + "tokens_seen": 2767060992 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013068004012036107, + "loss": 1.2708, + "theoretical_loss": 3.370093098085549, + "tokens_seen": 2767126528 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013066399197592777, + "loss": 1.1726, + "theoretical_loss": 3.3700869128116056, + "tokens_seen": 2767192064 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013064794383149447, + "loss": 1.2189, + "theoretical_loss": 3.370080727725163, + "tokens_seen": 2767257600 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013063189568706117, + "loss": 1.2723, + "theoretical_loss": 3.370074542826211, + "tokens_seen": 2767323136 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001306158475426279, + "loss": 1.2601, + "theoretical_loss": 3.3700683581147395, + "tokens_seen": 2767388672 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001305997993981946, + "loss": 1.2144, + "theoretical_loss": 3.370062173590738, + "tokens_seen": 2767454208 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001305837512537613, + "loss": 1.1923, + "theoretical_loss": 3.3700559892541975, + "tokens_seen": 2767519744 + }, + { + "epoch": 0.84, + "learning_rate": 0.000130567703109328, + "loss": 1.1379, + "theoretical_loss": 3.3700498051051064, + "tokens_seen": 2767585280 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001305516549648947, + "loss": 1.1877, + "theoretical_loss": 3.3700436211434557, + "tokens_seen": 2767650816 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001305356068204614, + "loss": 1.1656, + "theoretical_loss": 3.3700374373692346, + "tokens_seen": 2767716352 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001305195586760281, + "loss": 1.2144, + "theoretical_loss": 3.3700312537824333, + "tokens_seen": 2767781888 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001305035105315948, + "loss": 1.2448, + "theoretical_loss": 3.3700250703830417, + "tokens_seen": 2767847424 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001304874623871615, + "loss": 1.2347, + "theoretical_loss": 3.37001888717105, + "tokens_seen": 2767912960 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001304714142427282, + "loss": 1.214, + "theoretical_loss": 3.370012704146447, + "tokens_seen": 2767978496 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001304553660982949, + "loss": 1.3647, + "theoretical_loss": 3.370006521309224, + "tokens_seen": 2768044032 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001304393179538616, + "loss": 1.1815, + "theoretical_loss": 3.3700003386593695, + "tokens_seen": 2768109568 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001304232698094283, + "loss": 1.2529, + "theoretical_loss": 3.3699941561968743, + "tokens_seen": 2768175104 + }, + { + "epoch": 0.84, + "learning_rate": 0.000130407221664995, + "loss": 1.2408, + "theoretical_loss": 3.369987973921728, + "tokens_seen": 2768240640 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013039117352056168, + "loss": 1.1721, + "theoretical_loss": 3.369981791833921, + "tokens_seen": 2768306176 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013037512537612838, + "loss": 1.1794, + "theoretical_loss": 3.3699756099334426, + "tokens_seen": 2768371712 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013035907723169508, + "loss": 1.1999, + "theoretical_loss": 3.3699694282202826, + "tokens_seen": 2768437248 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013034302908726178, + "loss": 1.209, + "theoretical_loss": 3.369963246694431, + "tokens_seen": 2768502784 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013032698094282848, + "loss": 1.1785, + "theoretical_loss": 3.3699570653558775, + "tokens_seen": 2768568320 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013031093279839518, + "loss": 1.2527, + "theoretical_loss": 3.369950884204613, + "tokens_seen": 2768633856 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013029488465396188, + "loss": 1.2196, + "theoretical_loss": 3.369944703240626, + "tokens_seen": 2768699392 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013027883650952859, + "loss": 1.245, + "theoretical_loss": 3.3699385224639076, + "tokens_seen": 2768764928 + }, + { + "epoch": 0.84, + "learning_rate": 0.00013026278836509529, + "loss": 1.1845, + "theoretical_loss": 3.369932341874447, + "tokens_seen": 2768830464 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 925248, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0558922290802002, + "objective/train/theoretical_loss": 3.3699261614722342, + "objective/train/tokens_used": 2789356000, + "theoretical_loss": 3.3699261614722342, + "tokens_seen": 2768896000 + }, + { + "epoch": 0.84, + "learning_rate": 0.000130246740220662, + "loss": 1.0931, + "theoretical_loss": 3.3699261614722342, + "tokens_seen": 2768896000 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001302306920762287, + "loss": 1.2221, + "theoretical_loss": 3.3699199812572593, + "tokens_seen": 2768961536 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001302146439317954, + "loss": 1.2005, + "theoretical_loss": 3.3699138012295116, + "tokens_seen": 2769027072 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001301985957873621, + "loss": 1.1833, + "theoretical_loss": 3.369907621388982, + "tokens_seen": 2769092608 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001301825476429288, + "loss": 1.2116, + "theoretical_loss": 3.3699014417356596, + "tokens_seen": 2769158144 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001301664994984955, + "loss": 1.2722, + "theoretical_loss": 3.3698952622695346, + "tokens_seen": 2769223680 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001301504513540622, + "loss": 1.3422, + "theoretical_loss": 3.3698890829905963, + "tokens_seen": 2769289216 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001301344032096289, + "loss": 1.1782, + "theoretical_loss": 3.369882903898836, + "tokens_seen": 2769354752 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001301183550651956, + "loss": 1.2702, + "theoretical_loss": 3.369876724994242, + "tokens_seen": 2769420288 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001301023069207623, + "loss": 1.2308, + "theoretical_loss": 3.3698705462768057, + "tokens_seen": 2769485824 + }, + { + "epoch": 0.84, + "learning_rate": 0.000130086258776329, + "loss": 1.239, + "theoretical_loss": 3.3698643677465157, + "tokens_seen": 2769551360 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001300702106318957, + "loss": 1.1645, + "theoretical_loss": 3.3698581894033626, + "tokens_seen": 2769616896 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001300541624874624, + "loss": 1.1782, + "theoretical_loss": 3.3698520112473362, + "tokens_seen": 2769682432 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001300381143430291, + "loss": 1.146, + "theoretical_loss": 3.3698458332784265, + "tokens_seen": 2769747968 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001300220661985958, + "loss": 1.1918, + "theoretical_loss": 3.369839655496623, + "tokens_seen": 2769813504 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001300060180541625, + "loss": 1.1777, + "theoretical_loss": 3.369833477901916, + "tokens_seen": 2769879040 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001299899699097292, + "loss": 1.2226, + "theoretical_loss": 3.369827300494295, + "tokens_seen": 2769944576 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001299739217652959, + "loss": 1.1621, + "theoretical_loss": 3.3698211232737503, + "tokens_seen": 2770010112 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001299578736208626, + "loss": 1.2683, + "theoretical_loss": 3.369814946240272, + "tokens_seen": 2770075648 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001299418254764293, + "loss": 1.1332, + "theoretical_loss": 3.3698087693938494, + "tokens_seen": 2770141184 + }, + { + "epoch": 0.84, + "learning_rate": 0.000129925777331996, + "loss": 1.2179, + "theoretical_loss": 3.369802592734473, + "tokens_seen": 2770206720 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001299097291875627, + "loss": 1.1869, + "theoretical_loss": 3.3697964162621323, + "tokens_seen": 2770272256 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001298936810431294, + "loss": 1.1398, + "theoretical_loss": 3.369790239976817, + "tokens_seen": 2770337792 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001298776328986961, + "loss": 1.275, + "theoretical_loss": 3.3697840638785177, + "tokens_seen": 2770403328 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001298615847542628, + "loss": 1.2449, + "theoretical_loss": 3.3697778879672238, + "tokens_seen": 2770468864 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001298455366098295, + "loss": 1.1191, + "theoretical_loss": 3.3697717122429256, + "tokens_seen": 2770534400 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001298294884653962, + "loss": 1.2479, + "theoretical_loss": 3.3697655367056125, + "tokens_seen": 2770599936 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001298134403209629, + "loss": 1.2257, + "theoretical_loss": 3.3697593613552748, + "tokens_seen": 2770665472 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001297973921765296, + "loss": 1.1237, + "theoretical_loss": 3.3697531861919026, + "tokens_seen": 2770731008 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001297813440320963, + "loss": 1.2416, + "theoretical_loss": 3.3697470112154853, + "tokens_seen": 2770796544 + }, + { + "epoch": 0.84, + "learning_rate": 0.000129765295887663, + "loss": 1.2345, + "theoretical_loss": 3.369740836426013, + "tokens_seen": 2770862080 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001297492477432297, + "loss": 1.1201, + "theoretical_loss": 3.369734661823476, + "tokens_seen": 2770927616 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001297331995987964, + "loss": 1.2218, + "theoretical_loss": 3.3697284874078637, + "tokens_seen": 2770993152 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001297171514543631, + "loss": 1.1769, + "theoretical_loss": 3.369722313179166, + "tokens_seen": 2771058688 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012970110330992978, + "loss": 1.1774, + "theoretical_loss": 3.369716139137373, + "tokens_seen": 2771124224 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012968505516549648, + "loss": 1.2139, + "theoretical_loss": 3.369709965282475, + "tokens_seen": 2771189760 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012966900702106318, + "loss": 1.1154, + "theoretical_loss": 3.3697037916144614, + "tokens_seen": 2771255296 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012965295887662988, + "loss": 1.163, + "theoretical_loss": 3.3696976181333227, + "tokens_seen": 2771320832 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012963691073219658, + "loss": 1.2276, + "theoretical_loss": 3.369691444839048, + "tokens_seen": 2771386368 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012962086258776328, + "loss": 1.1947, + "theoretical_loss": 3.369685271731628, + "tokens_seen": 2771451904 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012960481444332998, + "loss": 1.1199, + "theoretical_loss": 3.369679098811052, + "tokens_seen": 2771517440 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012958876629889669, + "loss": 1.2232, + "theoretical_loss": 3.36967292607731, + "tokens_seen": 2771582976 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012957271815446339, + "loss": 1.1479, + "theoretical_loss": 3.3696667535303924, + "tokens_seen": 2771648512 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001295566700100301, + "loss": 1.1618, + "theoretical_loss": 3.369660581170289, + "tokens_seen": 2771714048 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001295406218655968, + "loss": 1.1388, + "theoretical_loss": 3.3696544089969893, + "tokens_seen": 2771779584 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001295245737211635, + "loss": 1.1167, + "theoretical_loss": 3.369648237010484, + "tokens_seen": 2771845120 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001295085255767302, + "loss": 1.0824, + "theoretical_loss": 3.369642065210762, + "tokens_seen": 2771910656 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001294924774322969, + "loss": 1.0583, + "theoretical_loss": 3.369635893597814, + "tokens_seen": 2771976192 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001294764292878636, + "loss": 1.1226, + "theoretical_loss": 3.36962972217163, + "tokens_seen": 2772041728 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001294603811434303, + "loss": 1.2281, + "theoretical_loss": 3.3696235509321992, + "tokens_seen": 2772107264 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 926012, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1974785327911377, + "objective/train/theoretical_loss": 3.369617379879512, + "objective/train/tokens_used": 2792632800, + "theoretical_loss": 3.369617379879512, + "tokens_seen": 2772172800 + }, + { + "epoch": 0.84, + "learning_rate": 0.000129444332998997, + "loss": 1.1606, + "theoretical_loss": 3.369617379879512, + "tokens_seen": 2772172800 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012942828485456372, + "loss": 1.0849, + "theoretical_loss": 3.369611209013559, + "tokens_seen": 2772238336 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012941223671013042, + "loss": 1.1816, + "theoretical_loss": 3.3696050383343286, + "tokens_seen": 2772303872 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012939618856569712, + "loss": 1.1687, + "theoretical_loss": 3.369598867841812, + "tokens_seen": 2772369408 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001293801404212638, + "loss": 1.1195, + "theoretical_loss": 3.3695926975359987, + "tokens_seen": 2772434944 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001293640922768305, + "loss": 1.1283, + "theoretical_loss": 3.3695865274168786, + "tokens_seen": 2772500480 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001293480441323972, + "loss": 1.1884, + "theoretical_loss": 3.369580357484442, + "tokens_seen": 2772566016 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001293319959879639, + "loss": 1.2292, + "theoretical_loss": 3.369574187738678, + "tokens_seen": 2772631552 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001293159478435306, + "loss": 1.1999, + "theoretical_loss": 3.3695680181795775, + "tokens_seen": 2772697088 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001292998996990973, + "loss": 1.1783, + "theoretical_loss": 3.36956184880713, + "tokens_seen": 2772762624 + }, + { + "epoch": 0.84, + "learning_rate": 0.000129283851554664, + "loss": 1.2477, + "theoretical_loss": 3.3695556796213255, + "tokens_seen": 2772828160 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001292678034102307, + "loss": 1.1856, + "theoretical_loss": 3.3695495106221536, + "tokens_seen": 2772893696 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001292517552657974, + "loss": 1.1888, + "theoretical_loss": 3.369543341809605, + "tokens_seen": 2772959232 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001292357071213641, + "loss": 1.2304, + "theoretical_loss": 3.3695371731836685, + "tokens_seen": 2773024768 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001292196589769308, + "loss": 1.2049, + "theoretical_loss": 3.3695310047443354, + "tokens_seen": 2773090304 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001292036108324975, + "loss": 1.0873, + "theoretical_loss": 3.369524836491595, + "tokens_seen": 2773155840 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001291875626880642, + "loss": 1.1492, + "theoretical_loss": 3.369518668425437, + "tokens_seen": 2773221376 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001291715145436309, + "loss": 1.2283, + "theoretical_loss": 3.3695125005458513, + "tokens_seen": 2773286912 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001291554663991976, + "loss": 1.2555, + "theoretical_loss": 3.3695063328528283, + "tokens_seen": 2773352448 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001291394182547643, + "loss": 1.1744, + "theoretical_loss": 3.369500165346358, + "tokens_seen": 2773417984 + }, + { + "epoch": 0.84, + "learning_rate": 0.000129123370110331, + "loss": 1.0687, + "theoretical_loss": 3.36949399802643, + "tokens_seen": 2773483520 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001291073219658977, + "loss": 1.2866, + "theoretical_loss": 3.3694878308930347, + "tokens_seen": 2773549056 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001290912738214644, + "loss": 1.1051, + "theoretical_loss": 3.369481663946161, + "tokens_seen": 2773614592 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001290752256770311, + "loss": 1.1207, + "theoretical_loss": 3.3694754971858, + "tokens_seen": 2773680128 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001290591775325978, + "loss": 1.1413, + "theoretical_loss": 3.3694693306119414, + "tokens_seen": 2773745664 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001290431293881645, + "loss": 1.1756, + "theoretical_loss": 3.369463164224575, + "tokens_seen": 2773811200 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001290270812437312, + "loss": 1.2523, + "theoretical_loss": 3.3694569980236904, + "tokens_seen": 2773876736 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001290110330992979, + "loss": 1.2012, + "theoretical_loss": 3.369450832009278, + "tokens_seen": 2773942272 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012899498495486458, + "loss": 1.1962, + "theoretical_loss": 3.3694446661813275, + "tokens_seen": 2774007808 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012897893681043128, + "loss": 1.1731, + "theoretical_loss": 3.3694385005398293, + "tokens_seen": 2774073344 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012896288866599798, + "loss": 1.1642, + "theoretical_loss": 3.369432335084773, + "tokens_seen": 2774138880 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012894684052156468, + "loss": 1.2754, + "theoretical_loss": 3.3694261698161485, + "tokens_seen": 2774204416 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012893079237713138, + "loss": 1.1822, + "theoretical_loss": 3.369420004733946, + "tokens_seen": 2774269952 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012891474423269809, + "loss": 1.3089, + "theoretical_loss": 3.3694138398381552, + "tokens_seen": 2774335488 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012889869608826479, + "loss": 1.1418, + "theoretical_loss": 3.369407675128766, + "tokens_seen": 2774401024 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012888264794383149, + "loss": 1.3639, + "theoretical_loss": 3.369401510605769, + "tokens_seen": 2774466560 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001288665997993982, + "loss": 1.1765, + "theoretical_loss": 3.3693953462691537, + "tokens_seen": 2774532096 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001288505516549649, + "loss": 1.063, + "theoretical_loss": 3.36938918211891, + "tokens_seen": 2774597632 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001288345035105316, + "loss": 1.2164, + "theoretical_loss": 3.3693830181550277, + "tokens_seen": 2774663168 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012881845536609832, + "loss": 1.1729, + "theoretical_loss": 3.3693768543774976, + "tokens_seen": 2774728704 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012880240722166502, + "loss": 1.27, + "theoretical_loss": 3.369370690786308, + "tokens_seen": 2774794240 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012878635907723172, + "loss": 1.121, + "theoretical_loss": 3.369364527381451, + "tokens_seen": 2774859776 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012877031093279842, + "loss": 1.1677, + "theoretical_loss": 3.369358364162915, + "tokens_seen": 2774925312 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012875426278836512, + "loss": 1.2726, + "theoretical_loss": 3.369352201130691, + "tokens_seen": 2774990848 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012873821464393182, + "loss": 1.18, + "theoretical_loss": 3.3693460382847675, + "tokens_seen": 2775056384 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012872216649949852, + "loss": 1.2372, + "theoretical_loss": 3.369339875625136, + "tokens_seen": 2775121920 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012870611835506522, + "loss": 1.1957, + "theoretical_loss": 3.369333713151786, + "tokens_seen": 2775187456 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012869007021063192, + "loss": 1.2199, + "theoretical_loss": 3.3693275508647074, + "tokens_seen": 2775252992 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001286740220661986, + "loss": 1.1291, + "theoretical_loss": 3.3693213887638898, + "tokens_seen": 2775318528 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001286579739217653, + "loss": 1.0958, + "theoretical_loss": 3.3693152268493236, + "tokens_seen": 2775384064 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 927320, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.250281572341919, + "objective/train/theoretical_loss": 3.3693090651209987, + "objective/train/tokens_used": 2795909600, + "theoretical_loss": 3.3693090651209987, + "tokens_seen": 2775449600 + }, + { + "epoch": 0.84, + "learning_rate": 0.000128641925777332, + "loss": 1.1458, + "theoretical_loss": 3.3693090651209987, + "tokens_seen": 2775449600 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001286258776328987, + "loss": 1.2266, + "theoretical_loss": 3.369302903578905, + "tokens_seen": 2775515136 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001286098294884654, + "loss": 1.1491, + "theoretical_loss": 3.3692967422230327, + "tokens_seen": 2775580672 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001285937813440321, + "loss": 1.1151, + "theoretical_loss": 3.369290581053371, + "tokens_seen": 2775646208 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001285777331995988, + "loss": 1.1938, + "theoretical_loss": 3.369284420069911, + "tokens_seen": 2775711744 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001285616850551655, + "loss": 1.1595, + "theoretical_loss": 3.369278259272642, + "tokens_seen": 2775777280 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001285456369107322, + "loss": 1.2009, + "theoretical_loss": 3.369272098661554, + "tokens_seen": 2775842816 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001285295887662989, + "loss": 1.1629, + "theoretical_loss": 3.3692659382366377, + "tokens_seen": 2775908352 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001285135406218656, + "loss": 1.1492, + "theoretical_loss": 3.3692597779978817, + "tokens_seen": 2775973888 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001284974924774323, + "loss": 1.2032, + "theoretical_loss": 3.3692536179452772, + "tokens_seen": 2776039424 + }, + { + "epoch": 0.84, + "learning_rate": 0.000128481444332999, + "loss": 1.2697, + "theoretical_loss": 3.3692474580788137, + "tokens_seen": 2776104960 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001284653961885657, + "loss": 1.2111, + "theoretical_loss": 3.369241298398481, + "tokens_seen": 2776170496 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001284493480441324, + "loss": 1.2791, + "theoretical_loss": 3.3692351389042696, + "tokens_seen": 2776236032 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001284332998996991, + "loss": 1.0851, + "theoretical_loss": 3.369228979596169, + "tokens_seen": 2776301568 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001284172517552658, + "loss": 1.1815, + "theoretical_loss": 3.369222820474169, + "tokens_seen": 2776367104 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001284012036108325, + "loss": 1.2147, + "theoretical_loss": 3.369216661538261, + "tokens_seen": 2776432640 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001283851554663992, + "loss": 1.2406, + "theoretical_loss": 3.369210502788433, + "tokens_seen": 2776498176 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001283691073219659, + "loss": 1.1856, + "theoretical_loss": 3.369204344224676, + "tokens_seen": 2776563712 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001283530591775326, + "loss": 1.1686, + "theoretical_loss": 3.3691981858469804, + "tokens_seen": 2776629248 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001283370110330993, + "loss": 1.2107, + "theoretical_loss": 3.3691920276553353, + "tokens_seen": 2776694784 + }, + { + "epoch": 0.84, + "learning_rate": 0.000128320962888666, + "loss": 1.2074, + "theoretical_loss": 3.369185869649731, + "tokens_seen": 2776760320 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001283049147442327, + "loss": 1.2239, + "theoretical_loss": 3.3691797118301583, + "tokens_seen": 2776825856 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012828886659979938, + "loss": 1.1366, + "theoretical_loss": 3.3691735541966055, + "tokens_seen": 2776891392 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012827281845536608, + "loss": 1.1498, + "theoretical_loss": 3.3691673967490643, + "tokens_seen": 2776956928 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012825677031093278, + "loss": 1.233, + "theoretical_loss": 3.369161239487523, + "tokens_seen": 2777022464 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012824072216649948, + "loss": 1.072, + "theoretical_loss": 3.3691550824119734, + "tokens_seen": 2777088000 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001282246740220662, + "loss": 1.203, + "theoretical_loss": 3.3691489255224045, + "tokens_seen": 2777153536 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001282086258776329, + "loss": 1.1504, + "theoretical_loss": 3.3691427688188065, + "tokens_seen": 2777219072 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012819257773319961, + "loss": 1.1919, + "theoretical_loss": 3.369136612301169, + "tokens_seen": 2777284608 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012817652958876631, + "loss": 1.0836, + "theoretical_loss": 3.369130455969483, + "tokens_seen": 2777350144 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012816048144433301, + "loss": 1.1506, + "theoretical_loss": 3.3691242998237367, + "tokens_seen": 2777415680 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012814443329989972, + "loss": 1.2049, + "theoretical_loss": 3.369118143863922, + "tokens_seen": 2777481216 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012812838515546642, + "loss": 1.1804, + "theoretical_loss": 3.3691119880900278, + "tokens_seen": 2777546752 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012811233701103312, + "loss": 1.2957, + "theoretical_loss": 3.3691058325020444, + "tokens_seen": 2777612288 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012809628886659982, + "loss": 1.1919, + "theoretical_loss": 3.3690996770999617, + "tokens_seen": 2777677824 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012808024072216652, + "loss": 1.1966, + "theoretical_loss": 3.3690935218837703, + "tokens_seen": 2777743360 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012806419257773322, + "loss": 1.1866, + "theoretical_loss": 3.369087366853459, + "tokens_seen": 2777808896 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012804814443329992, + "loss": 1.2658, + "theoretical_loss": 3.369081212009019, + "tokens_seen": 2777874432 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012803209628886662, + "loss": 1.1131, + "theoretical_loss": 3.3690750573504396, + "tokens_seen": 2777939968 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012801604814443332, + "loss": 1.1786, + "theoretical_loss": 3.369068902877711, + "tokens_seen": 2778005504 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012800000000000002, + "loss": 1.2318, + "theoretical_loss": 3.3690627485908236, + "tokens_seen": 2778071040 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001279839518555667, + "loss": 1.1955, + "theoretical_loss": 3.369056594489767, + "tokens_seen": 2778136576 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001279679037111334, + "loss": 1.2536, + "theoretical_loss": 3.3690504405745307, + "tokens_seen": 2778202112 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001279518555667001, + "loss": 1.1624, + "theoretical_loss": 3.3690442868451056, + "tokens_seen": 2778267648 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001279358074222668, + "loss": 1.204, + "theoretical_loss": 3.3690381333014807, + "tokens_seen": 2778333184 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001279197592778335, + "loss": 1.1345, + "theoretical_loss": 3.3690319799436472, + "tokens_seen": 2778398720 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001279037111334002, + "loss": 1.1492, + "theoretical_loss": 3.369025826771595, + "tokens_seen": 2778464256 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001278876629889669, + "loss": 1.1843, + "theoretical_loss": 3.3690196737853126, + "tokens_seen": 2778529792 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001278716148445336, + "loss": 1.2389, + "theoretical_loss": 3.369013520984792, + "tokens_seen": 2778595328 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001278555667001003, + "loss": 1.2449, + "theoretical_loss": 3.369007368370022, + "tokens_seen": 2778660864 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 928171, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1773011684417725, + "objective/train/theoretical_loss": 3.3690012159409926, + "objective/train/tokens_used": 2799186400, + "theoretical_loss": 3.3690012159409926, + "tokens_seen": 2778726400 + }, + { + "epoch": 0.84, + "learning_rate": 0.000127839518555667, + "loss": 1.2601, + "theoretical_loss": 3.3690012159409926, + "tokens_seen": 2778726400 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001278234704112337, + "loss": 1.239, + "theoretical_loss": 3.368995063697694, + "tokens_seen": 2778791936 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001278074222668004, + "loss": 1.2062, + "theoretical_loss": 3.368988911640117, + "tokens_seen": 2778857472 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001277913741223671, + "loss": 1.1803, + "theoretical_loss": 3.3689827597682505, + "tokens_seen": 2778923008 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001277753259779338, + "loss": 1.1666, + "theoretical_loss": 3.368976608082085, + "tokens_seen": 2778988544 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001277592778335005, + "loss": 1.2239, + "theoretical_loss": 3.3689704565816103, + "tokens_seen": 2779054080 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001277432296890672, + "loss": 1.1529, + "theoretical_loss": 3.368964305266817, + "tokens_seen": 2779119616 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001277271815446339, + "loss": 1.2097, + "theoretical_loss": 3.3689581541376943, + "tokens_seen": 2779185152 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001277111334002006, + "loss": 1.3381, + "theoretical_loss": 3.3689520031942326, + "tokens_seen": 2779250688 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001276950852557673, + "loss": 1.1709, + "theoretical_loss": 3.368945852436422, + "tokens_seen": 2779316224 + }, + { + "epoch": 0.84, + "learning_rate": 0.000127679037111334, + "loss": 1.1251, + "theoretical_loss": 3.3689397018642526, + "tokens_seen": 2779381760 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001276629889669007, + "loss": 1.2638, + "theoretical_loss": 3.3689335514777143, + "tokens_seen": 2779447296 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001276469408224674, + "loss": 1.2271, + "theoretical_loss": 3.368927401276797, + "tokens_seen": 2779512832 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001276308926780341, + "loss": 1.2677, + "theoretical_loss": 3.3689212512614906, + "tokens_seen": 2779578368 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001276148445336008, + "loss": 1.1139, + "theoretical_loss": 3.3689151014317855, + "tokens_seen": 2779643904 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001275987963891675, + "loss": 1.1919, + "theoretical_loss": 3.3689089517876716, + "tokens_seen": 2779709440 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001275827482447342, + "loss": 1.2331, + "theoretical_loss": 3.3689028023291385, + "tokens_seen": 2779774976 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001275667001003009, + "loss": 1.1897, + "theoretical_loss": 3.368896653056177, + "tokens_seen": 2779840512 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001275506519558676, + "loss": 1.2366, + "theoretical_loss": 3.368890503968777, + "tokens_seen": 2779906048 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001275346038114343, + "loss": 1.2699, + "theoretical_loss": 3.3688843550669274, + "tokens_seen": 2779971584 + }, + { + "epoch": 0.84, + "learning_rate": 0.000127518555667001, + "loss": 1.1119, + "theoretical_loss": 3.3688782063506197, + "tokens_seen": 2780037120 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012750250752256771, + "loss": 1.1898, + "theoretical_loss": 3.3688720578198437, + "tokens_seen": 2780102656 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012748645937813441, + "loss": 1.1372, + "theoretical_loss": 3.368865909474588, + "tokens_seen": 2780168192 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012747041123370112, + "loss": 1.1877, + "theoretical_loss": 3.3688597613148445, + "tokens_seen": 2780233728 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012745436308926782, + "loss": 1.2146, + "theoretical_loss": 3.368853613340602, + "tokens_seen": 2780299264 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012743831494483452, + "loss": 1.1025, + "theoretical_loss": 3.368847465551851, + "tokens_seen": 2780364800 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012742226680040122, + "loss": 1.1771, + "theoretical_loss": 3.3688413179485814, + "tokens_seen": 2780430336 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012740621865596792, + "loss": 1.1725, + "theoretical_loss": 3.3688351705307835, + "tokens_seen": 2780495872 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012739017051153462, + "loss": 1.1327, + "theoretical_loss": 3.368829023298447, + "tokens_seen": 2780561408 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012737412236710132, + "loss": 1.166, + "theoretical_loss": 3.368822876251562, + "tokens_seen": 2780626944 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012735807422266802, + "loss": 1.277, + "theoretical_loss": 3.368816729390119, + "tokens_seen": 2780692480 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012734202607823472, + "loss": 1.1048, + "theoretical_loss": 3.368810582714107, + "tokens_seen": 2780758016 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012732597793380142, + "loss": 1.1528, + "theoretical_loss": 3.3688044362235168, + "tokens_seen": 2780823552 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012730992978936812, + "loss": 1.2066, + "theoretical_loss": 3.3687982899183386, + "tokens_seen": 2780889088 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012729388164493482, + "loss": 1.1485, + "theoretical_loss": 3.368792143798562, + "tokens_seen": 2780954624 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001272778335005015, + "loss": 1.1638, + "theoretical_loss": 3.368785997864177, + "tokens_seen": 2781020160 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001272617853560682, + "loss": 1.1515, + "theoretical_loss": 3.368779852115174, + "tokens_seen": 2781085696 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001272457372116349, + "loss": 1.2734, + "theoretical_loss": 3.368773706551543, + "tokens_seen": 2781151232 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001272296890672016, + "loss": 1.2366, + "theoretical_loss": 3.3687675611732737, + "tokens_seen": 2781216768 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001272136409227683, + "loss": 1.0824, + "theoretical_loss": 3.3687614159803565, + "tokens_seen": 2781282304 + }, + { + "epoch": 0.84, + "learning_rate": 0.000127197592778335, + "loss": 1.1399, + "theoretical_loss": 3.3687552709727813, + "tokens_seen": 2781347840 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001271815446339017, + "loss": 1.2259, + "theoretical_loss": 3.368749126150538, + "tokens_seen": 2781413376 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001271654964894684, + "loss": 1.2042, + "theoretical_loss": 3.3687429815136167, + "tokens_seen": 2781478912 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001271494483450351, + "loss": 1.249, + "theoretical_loss": 3.3687368370620074, + "tokens_seen": 2781544448 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001271334002006018, + "loss": 1.084, + "theoretical_loss": 3.3687306927957006, + "tokens_seen": 2781609984 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001271173520561685, + "loss": 1.2328, + "theoretical_loss": 3.3687245487146855, + "tokens_seen": 2781675520 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001271013039117352, + "loss": 1.2381, + "theoretical_loss": 3.368718404818953, + "tokens_seen": 2781741056 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001270852557673019, + "loss": 1.2415, + "theoretical_loss": 3.368712261108493, + "tokens_seen": 2781806592 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001270692076228686, + "loss": 1.1738, + "theoretical_loss": 3.368706117583295, + "tokens_seen": 2781872128 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001270531594784353, + "loss": 1.1966, + "theoretical_loss": 3.3686999742433494, + "tokens_seen": 2781937664 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 929391, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1108996868133545, + "objective/train/theoretical_loss": 3.3686938310886463, + "objective/train/tokens_used": 2802463200, + "theoretical_loss": 3.3686938310886463, + "tokens_seen": 2782003200 + }, + { + "epoch": 0.84, + "learning_rate": 0.000127037111334002, + "loss": 1.1141, + "theoretical_loss": 3.3686938310886463, + "tokens_seen": 2782003200 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012702106318956873, + "loss": 1.2654, + "theoretical_loss": 3.368687688119176, + "tokens_seen": 2782068736 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012700501504513543, + "loss": 1.1816, + "theoretical_loss": 3.368681545334928, + "tokens_seen": 2782134272 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012698896690070213, + "loss": 1.2527, + "theoretical_loss": 3.3686754027358923, + "tokens_seen": 2782199808 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012697291875626883, + "loss": 1.2276, + "theoretical_loss": 3.3686692603220596, + "tokens_seen": 2782265344 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001269568706118355, + "loss": 1.1918, + "theoretical_loss": 3.3686631180934197, + "tokens_seen": 2782330880 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001269408224674022, + "loss": 1.1519, + "theoretical_loss": 3.3686569760499623, + "tokens_seen": 2782396416 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001269247743229689, + "loss": 1.1778, + "theoretical_loss": 3.368650834191678, + "tokens_seen": 2782461952 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001269087261785356, + "loss": 1.1816, + "theoretical_loss": 3.3686446925185565, + "tokens_seen": 2782527488 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001268926780341023, + "loss": 1.1884, + "theoretical_loss": 3.3686385510305876, + "tokens_seen": 2782593024 + }, + { + "epoch": 0.84, + "learning_rate": 0.000126876629889669, + "loss": 1.2085, + "theoretical_loss": 3.368632409727762, + "tokens_seen": 2782658560 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001268605817452357, + "loss": 1.2539, + "theoretical_loss": 3.368626268610069, + "tokens_seen": 2782724096 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001268445336008024, + "loss": 1.1844, + "theoretical_loss": 3.3686201276774996, + "tokens_seen": 2782789632 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001268284854563691, + "loss": 1.2085, + "theoretical_loss": 3.3686139869300433, + "tokens_seen": 2782855168 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012681243731193581, + "loss": 1.1441, + "theoretical_loss": 3.3686078463676905, + "tokens_seen": 2782920704 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012679638916750251, + "loss": 1.1088, + "theoretical_loss": 3.3686017059904305, + "tokens_seen": 2782986240 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012678034102306922, + "loss": 1.2605, + "theoretical_loss": 3.368595565798254, + "tokens_seen": 2783051776 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012676429287863592, + "loss": 1.1642, + "theoretical_loss": 3.368589425791151, + "tokens_seen": 2783117312 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012674824473420262, + "loss": 1.253, + "theoretical_loss": 3.3685832859691116, + "tokens_seen": 2783182848 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012673219658976932, + "loss": 1.1683, + "theoretical_loss": 3.3685771463321252, + "tokens_seen": 2783248384 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012671614844533602, + "loss": 1.1853, + "theoretical_loss": 3.368571006880183, + "tokens_seen": 2783313920 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012670010030090272, + "loss": 1.2157, + "theoretical_loss": 3.368564867613274, + "tokens_seen": 2783379456 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012668405215646942, + "loss": 1.2221, + "theoretical_loss": 3.368558728531389, + "tokens_seen": 2783444992 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012666800401203612, + "loss": 1.1719, + "theoretical_loss": 3.368552589634518, + "tokens_seen": 2783510528 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012665195586760282, + "loss": 1.2454, + "theoretical_loss": 3.3685464509226506, + "tokens_seen": 2783576064 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012663590772316952, + "loss": 1.2469, + "theoretical_loss": 3.368540312395777, + "tokens_seen": 2783641600 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012661985957873622, + "loss": 1.1819, + "theoretical_loss": 3.3685341740538877, + "tokens_seen": 2783707136 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012660381143430292, + "loss": 1.1683, + "theoretical_loss": 3.3685280358969725, + "tokens_seen": 2783772672 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012658776328986962, + "loss": 1.2116, + "theoretical_loss": 3.368521897925021, + "tokens_seen": 2783838208 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001265717151454363, + "loss": 1.3417, + "theoretical_loss": 3.368515760138024, + "tokens_seen": 2783903744 + }, + { + "epoch": 0.84, + "learning_rate": 0.000126555667001003, + "loss": 1.1924, + "theoretical_loss": 3.3685096225359716, + "tokens_seen": 2783969280 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001265396188565697, + "loss": 1.183, + "theoretical_loss": 3.368503485118853, + "tokens_seen": 2784034816 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001265235707121364, + "loss": 1.1974, + "theoretical_loss": 3.3684973478866596, + "tokens_seen": 2784100352 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001265075225677031, + "loss": 1.2107, + "theoretical_loss": 3.3684912108393803, + "tokens_seen": 2784165888 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001264914744232698, + "loss": 1.3215, + "theoretical_loss": 3.3684850739770056, + "tokens_seen": 2784231424 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001264754262788365, + "loss": 1.2008, + "theoretical_loss": 3.3684789372995256, + "tokens_seen": 2784296960 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001264593781344032, + "loss": 1.2453, + "theoretical_loss": 3.3684728008069302, + "tokens_seen": 2784362496 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001264433299899699, + "loss": 1.2259, + "theoretical_loss": 3.36846666449921, + "tokens_seen": 2784428032 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001264272818455366, + "loss": 1.085, + "theoretical_loss": 3.3684605283763545, + "tokens_seen": 2784493568 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012641123370110333, + "loss": 1.2587, + "theoretical_loss": 3.3684543924383537, + "tokens_seen": 2784559104 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012639518555667003, + "loss": 1.2257, + "theoretical_loss": 3.3684482566851983, + "tokens_seen": 2784624640 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012637913741223673, + "loss": 1.1231, + "theoretical_loss": 3.3684421211168782, + "tokens_seen": 2784690176 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012636308926780343, + "loss": 1.1889, + "theoretical_loss": 3.368435985733383, + "tokens_seen": 2784755712 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012634704112337013, + "loss": 1.1612, + "theoretical_loss": 3.3684298505347035, + "tokens_seen": 2784821248 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012633099297893683, + "loss": 1.1851, + "theoretical_loss": 3.368423715520829, + "tokens_seen": 2784886784 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012631494483450353, + "loss": 1.1095, + "theoretical_loss": 3.3684175806917502, + "tokens_seen": 2784952320 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012629889669007023, + "loss": 1.1736, + "theoretical_loss": 3.368411446047457, + "tokens_seen": 2785017856 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012628284854563694, + "loss": 1.1422, + "theoretical_loss": 3.3684053115879395, + "tokens_seen": 2785083392 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012626680040120364, + "loss": 1.147, + "theoretical_loss": 3.3683991773131874, + "tokens_seen": 2785148928 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001262507522567703, + "loss": 1.216, + "theoretical_loss": 3.3683930432231914, + "tokens_seen": 2785214464 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 930237, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2340691089630127, + "objective/train/theoretical_loss": 3.3683869093179415, + "objective/train/tokens_used": 2805740000, + "theoretical_loss": 3.3683869093179415, + "tokens_seen": 2785280000 + }, + { + "epoch": 0.84, + "learning_rate": 0.000126234704112337, + "loss": 1.1842, + "theoretical_loss": 3.3683869093179415, + "tokens_seen": 2785280000 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001262186559679037, + "loss": 1.1955, + "theoretical_loss": 3.3683807755974273, + "tokens_seen": 2785345536 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001262026078234704, + "loss": 1.2286, + "theoretical_loss": 3.3683746420616396, + "tokens_seen": 2785411072 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001261865596790371, + "loss": 1.155, + "theoretical_loss": 3.3683685087105677, + "tokens_seen": 2785476608 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001261705115346038, + "loss": 1.083, + "theoretical_loss": 3.3683623755442023, + "tokens_seen": 2785542144 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001261544633901705, + "loss": 1.0882, + "theoretical_loss": 3.368356242562533, + "tokens_seen": 2785607680 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012613841524573721, + "loss": 1.1105, + "theoretical_loss": 3.3683501097655504, + "tokens_seen": 2785673216 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012612236710130391, + "loss": 1.1453, + "theoretical_loss": 3.368343977153244, + "tokens_seen": 2785738752 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012610631895687061, + "loss": 1.2239, + "theoretical_loss": 3.368337844725605, + "tokens_seen": 2785804288 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012609027081243732, + "loss": 1.1719, + "theoretical_loss": 3.368331712482622, + "tokens_seen": 2785869824 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012607422266800402, + "loss": 1.0351, + "theoretical_loss": 3.3683255804242864, + "tokens_seen": 2785935360 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012605817452357072, + "loss": 1.2638, + "theoretical_loss": 3.3683194485505874, + "tokens_seen": 2786000896 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012604212637913742, + "loss": 1.1321, + "theoretical_loss": 3.3683133168615154, + "tokens_seen": 2786066432 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012602607823470412, + "loss": 1.1886, + "theoretical_loss": 3.368307185357061, + "tokens_seen": 2786131968 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012601003009027082, + "loss": 1.214, + "theoretical_loss": 3.368301054037213, + "tokens_seen": 2786197504 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012599398194583752, + "loss": 1.2646, + "theoretical_loss": 3.3682949229019634, + "tokens_seen": 2786263040 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012597793380140422, + "loss": 1.1825, + "theoretical_loss": 3.3682887919513007, + "tokens_seen": 2786328576 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012596188565697092, + "loss": 1.1355, + "theoretical_loss": 3.3682826611852152, + "tokens_seen": 2786394112 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012594583751253762, + "loss": 1.1229, + "theoretical_loss": 3.3682765306036977, + "tokens_seen": 2786459648 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012592978936810432, + "loss": 1.2568, + "theoretical_loss": 3.368270400206738, + "tokens_seen": 2786525184 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012591374122367102, + "loss": 1.1566, + "theoretical_loss": 3.3682642699943264, + "tokens_seen": 2786590720 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012589769307923772, + "loss": 1.1551, + "theoretical_loss": 3.368258139966452, + "tokens_seen": 2786656256 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012588164493480442, + "loss": 1.15, + "theoretical_loss": 3.3682520101231064, + "tokens_seen": 2786721792 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001258655967903711, + "loss": 1.1823, + "theoretical_loss": 3.3682458804642783, + "tokens_seen": 2786787328 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001258495486459378, + "loss": 1.2894, + "theoretical_loss": 3.368239750989959, + "tokens_seen": 2786852864 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001258335005015045, + "loss": 1.1366, + "theoretical_loss": 3.368233621700138, + "tokens_seen": 2786918400 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012581745235707123, + "loss": 1.0673, + "theoretical_loss": 3.3682274925948055, + "tokens_seen": 2786983936 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012580140421263793, + "loss": 1.2273, + "theoretical_loss": 3.3682213636739515, + "tokens_seen": 2787049472 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012578535606820463, + "loss": 1.2, + "theoretical_loss": 3.368215234937566, + "tokens_seen": 2787115008 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012576930792377133, + "loss": 1.1064, + "theoretical_loss": 3.3682091063856396, + "tokens_seen": 2787180544 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012575325977933803, + "loss": 1.1163, + "theoretical_loss": 3.368202978018162, + "tokens_seen": 2787246080 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012573721163490473, + "loss": 1.1267, + "theoretical_loss": 3.3681968498351234, + "tokens_seen": 2787311616 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012572116349047143, + "loss": 1.1512, + "theoretical_loss": 3.3681907218365144, + "tokens_seen": 2787377152 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012570511534603813, + "loss": 1.2173, + "theoretical_loss": 3.368184594022324, + "tokens_seen": 2787442688 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012568906720160483, + "loss": 1.1784, + "theoretical_loss": 3.368178466392543, + "tokens_seen": 2787508224 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012567301905717153, + "loss": 1.2082, + "theoretical_loss": 3.3681723389471623, + "tokens_seen": 2787573760 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012565697091273823, + "loss": 1.1866, + "theoretical_loss": 3.3681662116861704, + "tokens_seen": 2787639296 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012564092276830493, + "loss": 1.2322, + "theoretical_loss": 3.3681600846095585, + "tokens_seen": 2787704832 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012562487462387163, + "loss": 1.0845, + "theoretical_loss": 3.3681539577173165, + "tokens_seen": 2787770368 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012560882647943833, + "loss": 1.2176, + "theoretical_loss": 3.368147831009434, + "tokens_seen": 2787835904 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012559277833500504, + "loss": 1.1646, + "theoretical_loss": 3.368141704485902, + "tokens_seen": 2787901440 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012557673019057174, + "loss": 1.2929, + "theoretical_loss": 3.3681355781467106, + "tokens_seen": 2787966976 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001255606820461384, + "loss": 1.1984, + "theoretical_loss": 3.368129451991849, + "tokens_seen": 2788032512 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001255446339017051, + "loss": 1.1338, + "theoretical_loss": 3.368123326021308, + "tokens_seen": 2788098048 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001255285857572718, + "loss": 1.1202, + "theoretical_loss": 3.3681172002350777, + "tokens_seen": 2788163584 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001255125376128385, + "loss": 1.0521, + "theoretical_loss": 3.368111074633148, + "tokens_seen": 2788229120 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001254964894684052, + "loss": 1.1253, + "theoretical_loss": 3.3681049492155086, + "tokens_seen": 2788294656 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001254804413239719, + "loss": 1.1351, + "theoretical_loss": 3.3680988239821508, + "tokens_seen": 2788360192 + }, + { + "epoch": 0.84, + "learning_rate": 0.0001254643931795386, + "loss": 1.1616, + "theoretical_loss": 3.3680926989330637, + "tokens_seen": 2788425728 + }, + { + "epoch": 0.84, + "learning_rate": 0.00012544834503510531, + "loss": 1.1693, + "theoretical_loss": 3.368086574068238, + "tokens_seen": 2788491264 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 931418, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0683073997497559, + "objective/train/theoretical_loss": 3.3680804493876635, + "objective/train/tokens_used": 2809016800, + "theoretical_loss": 3.3680804493876635, + "tokens_seen": 2788556800 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012543229689067201, + "loss": 1.0662, + "theoretical_loss": 3.3680804493876635, + "tokens_seen": 2788556800 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012541624874623872, + "loss": 1.2064, + "theoretical_loss": 3.3680743248913303, + "tokens_seen": 2788622336 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012540020060180542, + "loss": 1.2023, + "theoretical_loss": 3.368068200579229, + "tokens_seen": 2788687872 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012538415245737212, + "loss": 1.2818, + "theoretical_loss": 3.3680620764513494, + "tokens_seen": 2788753408 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012536810431293882, + "loss": 1.1508, + "theoretical_loss": 3.368055952507681, + "tokens_seen": 2788818944 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012535205616850552, + "loss": 1.1967, + "theoretical_loss": 3.368049828748215, + "tokens_seen": 2788884480 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012533600802407222, + "loss": 1.2573, + "theoretical_loss": 3.3680437051729415, + "tokens_seen": 2788950016 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012531995987963892, + "loss": 1.1599, + "theoretical_loss": 3.3680375817818495, + "tokens_seen": 2789015552 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012530391173520562, + "loss": 1.2191, + "theoretical_loss": 3.3680314585749302, + "tokens_seen": 2789081088 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012528786359077232, + "loss": 1.1599, + "theoretical_loss": 3.3680253355521734, + "tokens_seen": 2789146624 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012527181544633902, + "loss": 1.194, + "theoretical_loss": 3.3680192127135693, + "tokens_seen": 2789212160 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012525576730190572, + "loss": 1.2177, + "theoretical_loss": 3.3680130900591077, + "tokens_seen": 2789277696 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012523971915747242, + "loss": 1.0088, + "theoretical_loss": 3.368006967588779, + "tokens_seen": 2789343232 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012522367101303912, + "loss": 1.1821, + "theoretical_loss": 3.368000845302573, + "tokens_seen": 2789408768 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012520762286860582, + "loss": 1.1239, + "theoretical_loss": 3.367994723200481, + "tokens_seen": 2789474304 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012519157472417252, + "loss": 1.1483, + "theoretical_loss": 3.367988601282492, + "tokens_seen": 2789539840 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012517552657973922, + "loss": 1.2026, + "theoretical_loss": 3.367982479548596, + "tokens_seen": 2789605376 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012515947843530593, + "loss": 1.2203, + "theoretical_loss": 3.367976357998784, + "tokens_seen": 2789670912 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012514343029087263, + "loss": 1.1928, + "theoretical_loss": 3.3679702366330457, + "tokens_seen": 2789736448 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012512738214643933, + "loss": 1.2546, + "theoretical_loss": 3.367964115451371, + "tokens_seen": 2789801984 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012511133400200603, + "loss": 1.2199, + "theoretical_loss": 3.3679579944537505, + "tokens_seen": 2789867520 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012509528585757273, + "loss": 1.1671, + "theoretical_loss": 3.367951873640174, + "tokens_seen": 2789933056 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012507923771313943, + "loss": 1.1867, + "theoretical_loss": 3.3679457530106323, + "tokens_seen": 2789998592 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012506318956870613, + "loss": 1.2169, + "theoretical_loss": 3.3679396325651147, + "tokens_seen": 2790064128 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012504714142427283, + "loss": 1.2241, + "theoretical_loss": 3.3679335123036114, + "tokens_seen": 2790129664 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012503109327983953, + "loss": 1.175, + "theoretical_loss": 3.367927392226113, + "tokens_seen": 2790195200 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012501504513540623, + "loss": 1.1042, + "theoretical_loss": 3.3679212723326097, + "tokens_seen": 2790260736 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012499899699097293, + "loss": 1.1577, + "theoretical_loss": 3.367915152623091, + "tokens_seen": 2790326272 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012498294884653963, + "loss": 1.2101, + "theoretical_loss": 3.367909033097548, + "tokens_seen": 2790391808 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012496690070210633, + "loss": 1.1542, + "theoretical_loss": 3.3679029137559695, + "tokens_seen": 2790457344 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012495085255767303, + "loss": 1.139, + "theoretical_loss": 3.367896794598347, + "tokens_seen": 2790522880 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012493480441323973, + "loss": 1.0988, + "theoretical_loss": 3.36789067562467, + "tokens_seen": 2790588416 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012491875626880643, + "loss": 1.25, + "theoretical_loss": 3.367884556834929, + "tokens_seen": 2790653952 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012490270812437314, + "loss": 1.3033, + "theoretical_loss": 3.3678784382291136, + "tokens_seen": 2790719488 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012488665997993984, + "loss": 1.1705, + "theoretical_loss": 3.3678723198072147, + "tokens_seen": 2790785024 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012487061183550654, + "loss": 1.1915, + "theoretical_loss": 3.3678662015692216, + "tokens_seen": 2790850560 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001248545636910732, + "loss": 1.1594, + "theoretical_loss": 3.367860083515125, + "tokens_seen": 2790916096 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001248385155466399, + "loss": 1.2502, + "theoretical_loss": 3.3678539656449145, + "tokens_seen": 2790981632 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001248224674022066, + "loss": 1.2531, + "theoretical_loss": 3.3678478479585814, + "tokens_seen": 2791047168 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001248064192577733, + "loss": 1.1106, + "theoretical_loss": 3.3678417304561146, + "tokens_seen": 2791112704 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012479037111334, + "loss": 1.1386, + "theoretical_loss": 3.367835613137505, + "tokens_seen": 2791178240 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001247743229689067, + "loss": 1.1725, + "theoretical_loss": 3.3678294960027424, + "tokens_seen": 2791243776 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012475827482447341, + "loss": 1.1991, + "theoretical_loss": 3.3678233790518175, + "tokens_seen": 2791309312 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012474222668004011, + "loss": 1.193, + "theoretical_loss": 3.36781726228472, + "tokens_seen": 2791374848 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012472617853560682, + "loss": 1.1723, + "theoretical_loss": 3.36781114570144, + "tokens_seen": 2791440384 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012471013039117352, + "loss": 1.131, + "theoretical_loss": 3.3678050293019677, + "tokens_seen": 2791505920 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012469408224674022, + "loss": 1.1733, + "theoretical_loss": 3.3677989130862933, + "tokens_seen": 2791571456 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012467803410230692, + "loss": 1.1426, + "theoretical_loss": 3.367792797054407, + "tokens_seen": 2791636992 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012466198595787362, + "loss": 1.2101, + "theoretical_loss": 3.367786681206299, + "tokens_seen": 2791702528 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012464593781344032, + "loss": 1.2162, + "theoretical_loss": 3.3677805655419597, + "tokens_seen": 2791768064 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 932411, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1167442798614502, + "objective/train/theoretical_loss": 3.3677744500613787, + "objective/train/tokens_used": 2812293600, + "theoretical_loss": 3.3677744500613787, + "tokens_seen": 2791833600 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012462988966900702, + "loss": 1.1715, + "theoretical_loss": 3.3677744500613787, + "tokens_seen": 2791833600 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012461384152457375, + "loss": 1.1382, + "theoretical_loss": 3.3677683347645466, + "tokens_seen": 2791899136 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012459779338014045, + "loss": 1.0548, + "theoretical_loss": 3.3677622196514534, + "tokens_seen": 2791964672 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012458174523570715, + "loss": 1.1316, + "theoretical_loss": 3.3677561047220896, + "tokens_seen": 2792030208 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012456569709127385, + "loss": 1.1697, + "theoretical_loss": 3.3677499899764447, + "tokens_seen": 2792095744 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012454964894684055, + "loss": 1.2404, + "theoretical_loss": 3.3677438754145093, + "tokens_seen": 2792161280 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012453360080240722, + "loss": 1.1189, + "theoretical_loss": 3.3677377610362735, + "tokens_seen": 2792226816 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012451755265797392, + "loss": 1.2557, + "theoretical_loss": 3.3677316468417278, + "tokens_seen": 2792292352 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012450150451354062, + "loss": 1.1538, + "theoretical_loss": 3.3677255328308613, + "tokens_seen": 2792357888 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012448545636910732, + "loss": 1.2386, + "theoretical_loss": 3.3677194190036657, + "tokens_seen": 2792423424 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012446940822467403, + "loss": 1.1844, + "theoretical_loss": 3.3677133053601302, + "tokens_seen": 2792488960 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012445336008024073, + "loss": 1.0803, + "theoretical_loss": 3.367707191900245, + "tokens_seen": 2792554496 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012443731193580743, + "loss": 1.1798, + "theoretical_loss": 3.367701078624, + "tokens_seen": 2792620032 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012442126379137413, + "loss": 1.239, + "theoretical_loss": 3.3676949655313866, + "tokens_seen": 2792685568 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012440521564694083, + "loss": 1.2191, + "theoretical_loss": 3.367688852622394, + "tokens_seen": 2792751104 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012438916750250753, + "loss": 1.2481, + "theoretical_loss": 3.3676827398970124, + "tokens_seen": 2792816640 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012437311935807423, + "loss": 1.2427, + "theoretical_loss": 3.367676627355232, + "tokens_seen": 2792882176 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012435707121364093, + "loss": 1.1228, + "theoretical_loss": 3.367670514997043, + "tokens_seen": 2792947712 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012434102306920763, + "loss": 1.1673, + "theoretical_loss": 3.3676644028224363, + "tokens_seen": 2793013248 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012432497492477433, + "loss": 1.1998, + "theoretical_loss": 3.3676582908314012, + "tokens_seen": 2793078784 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012430892678034103, + "loss": 1.2349, + "theoretical_loss": 3.3676521790239278, + "tokens_seen": 2793144320 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012429287863590773, + "loss": 1.2819, + "theoretical_loss": 3.367646067400007, + "tokens_seen": 2793209856 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012427683049147443, + "loss": 1.2255, + "theoretical_loss": 3.3676399559596284, + "tokens_seen": 2793275392 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012426078234704113, + "loss": 1.2386, + "theoretical_loss": 3.367633844702783, + "tokens_seen": 2793340928 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012424473420260783, + "loss": 1.1125, + "theoretical_loss": 3.36762773362946, + "tokens_seen": 2793406464 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012422868605817453, + "loss": 1.1159, + "theoretical_loss": 3.36762162273965, + "tokens_seen": 2793472000 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012421263791374124, + "loss": 1.1725, + "theoretical_loss": 3.3676155120333426, + "tokens_seen": 2793537536 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012419658976930794, + "loss": 1.1414, + "theoretical_loss": 3.3676094015105287, + "tokens_seen": 2793603072 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012418054162487464, + "loss": 1.2152, + "theoretical_loss": 3.3676032911711986, + "tokens_seen": 2793668608 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012416449348044134, + "loss": 1.1705, + "theoretical_loss": 3.3675971810153422, + "tokens_seen": 2793734144 + }, + { + "epoch": 0.85, + "learning_rate": 0.000124148445336008, + "loss": 1.1624, + "theoretical_loss": 3.3675910710429497, + "tokens_seen": 2793799680 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001241323971915747, + "loss": 1.2063, + "theoretical_loss": 3.3675849612540114, + "tokens_seen": 2793865216 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001241163490471414, + "loss": 1.2725, + "theoretical_loss": 3.367578851648517, + "tokens_seen": 2793930752 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001241003009027081, + "loss": 1.1894, + "theoretical_loss": 3.367572742226457, + "tokens_seen": 2793996288 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001240842527582748, + "loss": 1.1411, + "theoretical_loss": 3.367566632987822, + "tokens_seen": 2794061824 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012406820461384151, + "loss": 1.1519, + "theoretical_loss": 3.367560523932602, + "tokens_seen": 2794127360 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012405215646940821, + "loss": 1.1292, + "theoretical_loss": 3.367554415060787, + "tokens_seen": 2794192896 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012403610832497492, + "loss": 1.2327, + "theoretical_loss": 3.367548306372367, + "tokens_seen": 2794258432 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012402006018054164, + "loss": 1.1502, + "theoretical_loss": 3.367542197867332, + "tokens_seen": 2794323968 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012400401203610834, + "loss": 1.1493, + "theoretical_loss": 3.367536089545673, + "tokens_seen": 2794389504 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012398796389167504, + "loss": 1.186, + "theoretical_loss": 3.3675299814073796, + "tokens_seen": 2794455040 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012397191574724175, + "loss": 1.1473, + "theoretical_loss": 3.3675238734524426, + "tokens_seen": 2794520576 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012395586760280845, + "loss": 1.1381, + "theoretical_loss": 3.3675177656808515, + "tokens_seen": 2794586112 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012393981945837515, + "loss": 1.1569, + "theoretical_loss": 3.367511658092597, + "tokens_seen": 2794651648 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012392377131394185, + "loss": 1.1757, + "theoretical_loss": 3.3675055506876688, + "tokens_seen": 2794717184 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012390772316950855, + "loss": 1.197, + "theoretical_loss": 3.3674994434660577, + "tokens_seen": 2794782720 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012389167502507525, + "loss": 1.2128, + "theoretical_loss": 3.3674933364277533, + "tokens_seen": 2794848256 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012387562688064195, + "loss": 1.2162, + "theoretical_loss": 3.3674872295727463, + "tokens_seen": 2794913792 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012385957873620865, + "loss": 1.2068, + "theoretical_loss": 3.3674811229010264, + "tokens_seen": 2794979328 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012384353059177532, + "loss": 1.2336, + "theoretical_loss": 3.367475016412584, + "tokens_seen": 2795044864 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 933612, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0824105739593506, + "objective/train/theoretical_loss": 3.36746891010741, + "objective/train/tokens_used": 2815570400, + "theoretical_loss": 3.36746891010741, + "tokens_seen": 2795110400 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012382748244734202, + "loss": 1.0395, + "theoretical_loss": 3.36746891010741, + "tokens_seen": 2795110400 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012381143430290872, + "loss": 1.2287, + "theoretical_loss": 3.3674628039854935, + "tokens_seen": 2795175936 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012379538615847542, + "loss": 1.1621, + "theoretical_loss": 3.3674566980468255, + "tokens_seen": 2795241472 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012377933801404213, + "loss": 1.2189, + "theoretical_loss": 3.3674505922913958, + "tokens_seen": 2795307008 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012376328986960883, + "loss": 1.1413, + "theoretical_loss": 3.3674444867191946, + "tokens_seen": 2795372544 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012374724172517553, + "loss": 1.1328, + "theoretical_loss": 3.3674383813302122, + "tokens_seen": 2795438080 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012373119358074223, + "loss": 1.1673, + "theoretical_loss": 3.367432276124439, + "tokens_seen": 2795503616 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012371514543630893, + "loss": 1.1426, + "theoretical_loss": 3.3674261711018647, + "tokens_seen": 2795569152 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012369909729187563, + "loss": 1.1249, + "theoretical_loss": 3.36742006626248, + "tokens_seen": 2795634688 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012368304914744233, + "loss": 1.0945, + "theoretical_loss": 3.367413961606275, + "tokens_seen": 2795700224 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012366700100300903, + "loss": 1.2224, + "theoretical_loss": 3.36740785713324, + "tokens_seen": 2795765760 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012365095285857573, + "loss": 1.177, + "theoretical_loss": 3.3674017528433646, + "tokens_seen": 2795831296 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012363490471414243, + "loss": 1.1426, + "theoretical_loss": 3.36739564873664, + "tokens_seen": 2795896832 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012361885656970913, + "loss": 1.3023, + "theoretical_loss": 3.3673895448130553, + "tokens_seen": 2795962368 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012360280842527583, + "loss": 1.3377, + "theoretical_loss": 3.367383441072602, + "tokens_seen": 2796027904 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012358676028084253, + "loss": 1.2005, + "theoretical_loss": 3.3673773375152694, + "tokens_seen": 2796093440 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012357071213640923, + "loss": 1.1262, + "theoretical_loss": 3.3673712341410473, + "tokens_seen": 2796158976 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012355466399197593, + "loss": 1.254, + "theoretical_loss": 3.3673651309499273, + "tokens_seen": 2796224512 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012353861584754264, + "loss": 1.1214, + "theoretical_loss": 3.3673590279418986, + "tokens_seen": 2796290048 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012352256770310934, + "loss": 1.2629, + "theoretical_loss": 3.367352925116952, + "tokens_seen": 2796355584 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012350651955867604, + "loss": 1.2227, + "theoretical_loss": 3.367346822475077, + "tokens_seen": 2796421120 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012349047141424274, + "loss": 1.1337, + "theoretical_loss": 3.367340720016264, + "tokens_seen": 2796486656 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012347442326980944, + "loss": 1.2083, + "theoretical_loss": 3.3673346177405037, + "tokens_seen": 2796552192 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001234583751253761, + "loss": 1.1712, + "theoretical_loss": 3.367328515647786, + "tokens_seen": 2796617728 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001234423269809428, + "loss": 1.1809, + "theoretical_loss": 3.3673224137381013, + "tokens_seen": 2796683264 + }, + { + "epoch": 0.85, + "learning_rate": 0.0001234262788365095, + "loss": 1.1335, + "theoretical_loss": 3.3673163120114395, + "tokens_seen": 2796748800 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012341023069207624, + "loss": 1.1351, + "theoretical_loss": 3.367310210467791, + "tokens_seen": 2796814336 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012339418254764294, + "loss": 1.2009, + "theoretical_loss": 3.3673041091071463, + "tokens_seen": 2796879872 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012337813440320964, + "loss": 1.2132, + "theoretical_loss": 3.367298007929495, + "tokens_seen": 2796945408 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012336208625877634, + "loss": 1.1312, + "theoretical_loss": 3.367291906934828, + "tokens_seen": 2797010944 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012334603811434304, + "loss": 1.2703, + "theoretical_loss": 3.367285806123135, + "tokens_seen": 2797076480 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012332998996990974, + "loss": 1.1772, + "theoretical_loss": 3.367279705494407, + "tokens_seen": 2797142016 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012331394182547644, + "loss": 1.3169, + "theoretical_loss": 3.3672736050486325, + "tokens_seen": 2797207552 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012329789368104314, + "loss": 1.2802, + "theoretical_loss": 3.367267504785804, + "tokens_seen": 2797273088 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012328184553660985, + "loss": 1.1935, + "theoretical_loss": 3.3672614047059097, + "tokens_seen": 2797338624 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012326579739217655, + "loss": 1.2313, + "theoretical_loss": 3.3672553048089413, + "tokens_seen": 2797404160 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012324974924774325, + "loss": 1.1623, + "theoretical_loss": 3.3672492050948883, + "tokens_seen": 2797469696 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012323370110330995, + "loss": 1.2157, + "theoretical_loss": 3.367243105563741, + "tokens_seen": 2797535232 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012321765295887665, + "loss": 1.4317, + "theoretical_loss": 3.36723700621549, + "tokens_seen": 2797600768 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012320160481444335, + "loss": 1.2917, + "theoretical_loss": 3.3672309070501254, + "tokens_seen": 2797666304 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012318555667001005, + "loss": 1.2183, + "theoretical_loss": 3.367224808067637, + "tokens_seen": 2797731840 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012316950852557675, + "loss": 1.3096, + "theoretical_loss": 3.3672187092680153, + "tokens_seen": 2797797376 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012315346038114345, + "loss": 1.317, + "theoretical_loss": 3.3672126106512508, + "tokens_seen": 2797862912 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012313741223671012, + "loss": 1.2093, + "theoretical_loss": 3.367206512217333, + "tokens_seen": 2797928448 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012312136409227682, + "loss": 1.3241, + "theoretical_loss": 3.3672004139662532, + "tokens_seen": 2797993984 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012310531594784353, + "loss": 1.282, + "theoretical_loss": 3.3671943158980007, + "tokens_seen": 2798059520 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012308926780341023, + "loss": 1.1147, + "theoretical_loss": 3.3671882180125663, + "tokens_seen": 2798125056 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012307321965897693, + "loss": 1.4463, + "theoretical_loss": 3.36718212030994, + "tokens_seen": 2798190592 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012305717151454363, + "loss": 1.3231, + "theoretical_loss": 3.3671760227901117, + "tokens_seen": 2798256128 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012304112337011033, + "loss": 1.3708, + "theoretical_loss": 3.367169925453073, + "tokens_seen": 2798321664 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 934391, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.097245454788208, + "objective/train/theoretical_loss": 3.3671638282988123, + "objective/train/tokens_used": 2818847200, + "theoretical_loss": 3.3671638282988123, + "tokens_seen": 2798387200 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012302507522567703, + "loss": 1.1464, + "theoretical_loss": 3.3671638282988123, + "tokens_seen": 2798387200 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012300902708124373, + "loss": 1.2234, + "theoretical_loss": 3.3671577313273207, + "tokens_seen": 2798452736 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012299297893681043, + "loss": 1.272, + "theoretical_loss": 3.3671516345385886, + "tokens_seen": 2798518272 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012297693079237713, + "loss": 1.1837, + "theoretical_loss": 3.3671455379326063, + "tokens_seen": 2798583808 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012296088264794383, + "loss": 1.212, + "theoretical_loss": 3.3671394415093636, + "tokens_seen": 2798649344 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012294483450351053, + "loss": 1.3865, + "theoretical_loss": 3.367133345268851, + "tokens_seen": 2798714880 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012292878635907723, + "loss": 1.1523, + "theoretical_loss": 3.367127249211059, + "tokens_seen": 2798780416 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012291273821464393, + "loss": 1.2531, + "theoretical_loss": 3.367121153335977, + "tokens_seen": 2798845952 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012289669007021063, + "loss": 1.2008, + "theoretical_loss": 3.367115057643596, + "tokens_seen": 2798911488 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012288064192577733, + "loss": 1.2713, + "theoretical_loss": 3.3671089621339063, + "tokens_seen": 2798977024 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012286459378134403, + "loss": 1.295, + "theoretical_loss": 3.367102866806898, + "tokens_seen": 2799042560 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012284854563691074, + "loss": 1.3439, + "theoretical_loss": 3.3670967716625606, + "tokens_seen": 2799108096 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012283249749247744, + "loss": 1.2665, + "theoretical_loss": 3.3670906767008857, + "tokens_seen": 2799173632 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012281644934804414, + "loss": 1.216, + "theoretical_loss": 3.3670845819218624, + "tokens_seen": 2799239168 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012280040120361084, + "loss": 1.0779, + "theoretical_loss": 3.3670784873254815, + "tokens_seen": 2799304704 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012278435305917754, + "loss": 1.3123, + "theoretical_loss": 3.367072392911733, + "tokens_seen": 2799370240 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012276830491474424, + "loss": 1.1763, + "theoretical_loss": 3.3670662986806077, + "tokens_seen": 2799435776 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012275225677031094, + "loss": 1.2139, + "theoretical_loss": 3.367060204632095, + "tokens_seen": 2799501312 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012273620862587764, + "loss": 1.2605, + "theoretical_loss": 3.367054110766186, + "tokens_seen": 2799566848 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012272016048144434, + "loss": 1.2618, + "theoretical_loss": 3.3670480170828703, + "tokens_seen": 2799632384 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012270411233701104, + "loss": 1.1217, + "theoretical_loss": 3.3670419235821383, + "tokens_seen": 2799697920 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012268806419257774, + "loss": 1.3126, + "theoretical_loss": 3.3670358302639807, + "tokens_seen": 2799763456 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012267201604814444, + "loss": 1.2197, + "theoretical_loss": 3.3670297371283873, + "tokens_seen": 2799828992 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012265596790371114, + "loss": 1.3364, + "theoretical_loss": 3.3670236441753483, + "tokens_seen": 2799894528 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012263991975927784, + "loss": 1.2072, + "theoretical_loss": 3.3670175514048544, + "tokens_seen": 2799960064 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012262387161484454, + "loss": 1.2624, + "theoretical_loss": 3.3670114588168953, + "tokens_seen": 2800025600 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012260782347041124, + "loss": 1.1742, + "theoretical_loss": 3.367005366411462, + "tokens_seen": 2800091136 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012259177532597795, + "loss": 1.1781, + "theoretical_loss": 3.3669992741885437, + "tokens_seen": 2800156672 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012257572718154465, + "loss": 1.2667, + "theoretical_loss": 3.3669931821481316, + "tokens_seen": 2800222208 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012255967903711135, + "loss": 1.2684, + "theoretical_loss": 3.3669870902902157, + "tokens_seen": 2800287744 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012254363089267805, + "loss": 1.2029, + "theoretical_loss": 3.3669809986147863, + "tokens_seen": 2800353280 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012252758274824475, + "loss": 1.1622, + "theoretical_loss": 3.3669749071218336, + "tokens_seen": 2800418816 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012251153460381145, + "loss": 1.2955, + "theoretical_loss": 3.3669688158113473, + "tokens_seen": 2800484352 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012249548645937815, + "loss": 1.3033, + "theoretical_loss": 3.3669627246833187, + "tokens_seen": 2800549888 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012247943831494485, + "loss": 1.1464, + "theoretical_loss": 3.3669566337377375, + "tokens_seen": 2800615424 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012246339017051155, + "loss": 1.2999, + "theoretical_loss": 3.366950542974594, + "tokens_seen": 2800680960 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012244734202607825, + "loss": 1.1927, + "theoretical_loss": 3.366944452393878, + "tokens_seen": 2800746496 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012243129388164492, + "loss": 1.2763, + "theoretical_loss": 3.366938361995581, + "tokens_seen": 2800812032 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012241524573721163, + "loss": 1.3264, + "theoretical_loss": 3.366932271779692, + "tokens_seen": 2800877568 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012239919759277833, + "loss": 1.2353, + "theoretical_loss": 3.366926181746202, + "tokens_seen": 2800943104 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012238314944834503, + "loss": 1.2104, + "theoretical_loss": 3.3669200918951008, + "tokens_seen": 2801008640 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012236710130391173, + "loss": 1.106, + "theoretical_loss": 3.3669140022263795, + "tokens_seen": 2801074176 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012235105315947843, + "loss": 1.1065, + "theoretical_loss": 3.3669079127400274, + "tokens_seen": 2801139712 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012233500501504513, + "loss": 1.2755, + "theoretical_loss": 3.3669018234360353, + "tokens_seen": 2801205248 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012231895687061183, + "loss": 1.1064, + "theoretical_loss": 3.366895734314393, + "tokens_seen": 2801270784 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012230290872617853, + "loss": 1.2016, + "theoretical_loss": 3.3668896453750916, + "tokens_seen": 2801336320 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012228686058174523, + "loss": 1.2652, + "theoretical_loss": 3.3668835566181206, + "tokens_seen": 2801401856 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012227081243731193, + "loss": 1.1714, + "theoretical_loss": 3.3668774680434708, + "tokens_seen": 2801467392 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012225476429287863, + "loss": 1.2005, + "theoretical_loss": 3.366871379651132, + "tokens_seen": 2801532928 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012223871614844533, + "loss": 1.1543, + "theoretical_loss": 3.366865291441095, + "tokens_seen": 2801598464 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 935571, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1240062713623047, + "objective/train/theoretical_loss": 3.3668592034133495, + "objective/train/tokens_used": 2822124000, + "theoretical_loss": 3.3668592034133495, + "tokens_seen": 2801664000 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012222266800401203, + "loss": 1.1036, + "theoretical_loss": 3.3668592034133495, + "tokens_seen": 2801664000 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012220661985957876, + "loss": 1.2562, + "theoretical_loss": 3.3668531155678862, + "tokens_seen": 2801729536 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012219057171514546, + "loss": 1.2514, + "theoretical_loss": 3.3668470279046954, + "tokens_seen": 2801795072 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012217452357071216, + "loss": 1.1901, + "theoretical_loss": 3.366840940423767, + "tokens_seen": 2801860608 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012215847542627886, + "loss": 1.2241, + "theoretical_loss": 3.366834853125092, + "tokens_seen": 2801926144 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012214242728184556, + "loss": 1.1715, + "theoretical_loss": 3.3668287660086595, + "tokens_seen": 2801991680 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012212637913741224, + "loss": 1.2062, + "theoretical_loss": 3.3668226790744606, + "tokens_seen": 2802057216 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012211033099297894, + "loss": 1.3523, + "theoretical_loss": 3.3668165923224858, + "tokens_seen": 2802122752 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012209428284854564, + "loss": 1.0217, + "theoretical_loss": 3.366810505752725, + "tokens_seen": 2802188288 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012207823470411234, + "loss": 1.2159, + "theoretical_loss": 3.3668044193651685, + "tokens_seen": 2802253824 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012206218655967905, + "loss": 1.2121, + "theoretical_loss": 3.3667983331598066, + "tokens_seen": 2802319360 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012204613841524575, + "loss": 1.2034, + "theoretical_loss": 3.3667922471366296, + "tokens_seen": 2802384896 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012203009027081244, + "loss": 1.2189, + "theoretical_loss": 3.366786161295628, + "tokens_seen": 2802450432 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012201404212637914, + "loss": 1.321, + "theoretical_loss": 3.3667800756367914, + "tokens_seen": 2802515968 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012199799398194584, + "loss": 1.2087, + "theoretical_loss": 3.3667739901601106, + "tokens_seen": 2802581504 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012198194583751254, + "loss": 1.2273, + "theoretical_loss": 3.366767904865576, + "tokens_seen": 2802647040 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012196589769307924, + "loss": 1.2694, + "theoretical_loss": 3.3667618197531777, + "tokens_seen": 2802712576 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012194984954864594, + "loss": 1.2626, + "theoretical_loss": 3.366755734822906, + "tokens_seen": 2802778112 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012193380140421264, + "loss": 1.1716, + "theoretical_loss": 3.366749650074752, + "tokens_seen": 2802843648 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012191775325977934, + "loss": 1.1565, + "theoretical_loss": 3.366743565508704, + "tokens_seen": 2802909184 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012190170511534605, + "loss": 1.2575, + "theoretical_loss": 3.366737481124754, + "tokens_seen": 2802974720 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012188565697091275, + "loss": 1.1452, + "theoretical_loss": 3.3667313969228916, + "tokens_seen": 2803040256 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012186960882647945, + "loss": 1.2925, + "theoretical_loss": 3.3667253129031076, + "tokens_seen": 2803105792 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012185356068204615, + "loss": 1.2241, + "theoretical_loss": 3.366719229065392, + "tokens_seen": 2803171328 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012183751253761283, + "loss": 1.2599, + "theoretical_loss": 3.3667131454097348, + "tokens_seen": 2803236864 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012182146439317954, + "loss": 1.1684, + "theoretical_loss": 3.366707061936127, + "tokens_seen": 2803302400 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012180541624874624, + "loss": 1.2785, + "theoretical_loss": 3.366700978644558, + "tokens_seen": 2803367936 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012178936810431294, + "loss": 1.1427, + "theoretical_loss": 3.366694895535019, + "tokens_seen": 2803433472 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012177331995987964, + "loss": 1.1712, + "theoretical_loss": 3.3666888126074994, + "tokens_seen": 2803499008 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012175727181544634, + "loss": 1.1145, + "theoretical_loss": 3.3666827298619904, + "tokens_seen": 2803564544 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012174122367101304, + "loss": 1.192, + "theoretical_loss": 3.3666766472984815, + "tokens_seen": 2803630080 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012172517552657974, + "loss": 1.2147, + "theoretical_loss": 3.3666705649169635, + "tokens_seen": 2803695616 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012170912738214644, + "loss": 1.2226, + "theoretical_loss": 3.3666644827174266, + "tokens_seen": 2803761152 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012169307923771314, + "loss": 1.2001, + "theoretical_loss": 3.366658400699861, + "tokens_seen": 2803826688 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012167703109327984, + "loss": 1.21, + "theoretical_loss": 3.366652318864257, + "tokens_seen": 2803892224 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012166098294884654, + "loss": 1.1955, + "theoretical_loss": 3.366646237210605, + "tokens_seen": 2803957760 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012164493480441323, + "loss": 1.1195, + "theoretical_loss": 3.3666401557388954, + "tokens_seen": 2804023296 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012162888665997993, + "loss": 1.1424, + "theoretical_loss": 3.3666340744491183, + "tokens_seen": 2804088832 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012161283851554666, + "loss": 1.3405, + "theoretical_loss": 3.366627993341264, + "tokens_seen": 2804154368 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012159679037111336, + "loss": 1.2765, + "theoretical_loss": 3.366621912415323, + "tokens_seen": 2804219904 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012158074222668006, + "loss": 1.2024, + "theoretical_loss": 3.366615831671286, + "tokens_seen": 2804285440 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012156469408224676, + "loss": 1.2015, + "theoretical_loss": 3.366609751109142, + "tokens_seen": 2804350976 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012154864593781346, + "loss": 1.1815, + "theoretical_loss": 3.3666036707288827, + "tokens_seen": 2804416512 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012153259779338016, + "loss": 1.1839, + "theoretical_loss": 3.3665975905304975, + "tokens_seen": 2804482048 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012151654964894685, + "loss": 1.2589, + "theoretical_loss": 3.366591510513977, + "tokens_seen": 2804547584 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012150050150451355, + "loss": 1.2431, + "theoretical_loss": 3.3665854306793115, + "tokens_seen": 2804613120 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012148445336008025, + "loss": 1.225, + "theoretical_loss": 3.3665793510264916, + "tokens_seen": 2804678656 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012146840521564695, + "loss": 1.1958, + "theoretical_loss": 3.3665732715555072, + "tokens_seen": 2804744192 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012145235707121365, + "loss": 1.2094, + "theoretical_loss": 3.366567192266349, + "tokens_seen": 2804809728 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012143630892678035, + "loss": 1.3676, + "theoretical_loss": 3.366561113159007, + "tokens_seen": 2804875264 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 936523, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2646677494049072, + "objective/train/theoretical_loss": 3.366555034233471, + "objective/train/tokens_used": 2825400800, + "theoretical_loss": 3.366555034233471, + "tokens_seen": 2804940800 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012142026078234705, + "loss": 1.2098, + "theoretical_loss": 3.366555034233471, + "tokens_seen": 2804940800 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012140421263791375, + "loss": 1.2011, + "theoretical_loss": 3.3665489554897325, + "tokens_seen": 2805006336 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012138816449348045, + "loss": 1.3104, + "theoretical_loss": 3.3665428769277814, + "tokens_seen": 2805071872 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012137211634904715, + "loss": 1.1562, + "theoretical_loss": 3.3665367985476076, + "tokens_seen": 2805137408 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012135606820461385, + "loss": 1.1817, + "theoretical_loss": 3.3665307203492016, + "tokens_seen": 2805202944 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012134002006018055, + "loss": 1.2262, + "theoretical_loss": 3.3665246423325543, + "tokens_seen": 2805268480 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012132397191574724, + "loss": 1.1219, + "theoretical_loss": 3.366518564497655, + "tokens_seen": 2805334016 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012130792377131394, + "loss": 1.1304, + "theoretical_loss": 3.3665124868444947, + "tokens_seen": 2805399552 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012129187562688064, + "loss": 1.2332, + "theoretical_loss": 3.3665064093730637, + "tokens_seen": 2805465088 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012127582748244734, + "loss": 1.1477, + "theoretical_loss": 3.3665003320833518, + "tokens_seen": 2805530624 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012125977933801404, + "loss": 1.1466, + "theoretical_loss": 3.3664942549753496, + "tokens_seen": 2805596160 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012124373119358074, + "loss": 1.1465, + "theoretical_loss": 3.366488178049048, + "tokens_seen": 2805661696 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012122768304914745, + "loss": 1.1096, + "theoretical_loss": 3.366482101304437, + "tokens_seen": 2805727232 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012121163490471415, + "loss": 1.1742, + "theoretical_loss": 3.366476024741506, + "tokens_seen": 2805792768 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012119558676028085, + "loss": 1.2881, + "theoretical_loss": 3.3664699483602467, + "tokens_seen": 2805858304 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012117953861584755, + "loss": 1.1503, + "theoretical_loss": 3.3664638721606486, + "tokens_seen": 2805923840 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012116349047141425, + "loss": 1.1201, + "theoretical_loss": 3.366457796142702, + "tokens_seen": 2805989376 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012114744232698095, + "loss": 1.1653, + "theoretical_loss": 3.3664517203063977, + "tokens_seen": 2806054912 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012113139418254764, + "loss": 1.1846, + "theoretical_loss": 3.366445644651726, + "tokens_seen": 2806120448 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012111534603811434, + "loss": 1.1688, + "theoretical_loss": 3.3664395691786764, + "tokens_seen": 2806185984 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012109929789368104, + "loss": 1.1335, + "theoretical_loss": 3.3664334938872407, + "tokens_seen": 2806251520 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012108324974924774, + "loss": 1.1806, + "theoretical_loss": 3.3664274187774077, + "tokens_seen": 2806317056 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012106720160481444, + "loss": 1.1461, + "theoretical_loss": 3.3664213438491686, + "tokens_seen": 2806382592 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012105115346038114, + "loss": 1.2657, + "theoretical_loss": 3.3664152691025135, + "tokens_seen": 2806448128 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012103510531594784, + "loss": 1.2031, + "theoretical_loss": 3.3664091945374333, + "tokens_seen": 2806513664 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012101905717151457, + "loss": 1.2881, + "theoretical_loss": 3.366403120153917, + "tokens_seen": 2806579200 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012100300902708125, + "loss": 1.219, + "theoretical_loss": 3.366397045951956, + "tokens_seen": 2806644736 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012098696088264795, + "loss": 1.1853, + "theoretical_loss": 3.366390971931541, + "tokens_seen": 2806710272 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012097091273821466, + "loss": 1.1941, + "theoretical_loss": 3.366384898092661, + "tokens_seen": 2806775808 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012095486459378136, + "loss": 1.1762, + "theoretical_loss": 3.3663788244353072, + "tokens_seen": 2806841344 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012093881644934806, + "loss": 1.2538, + "theoretical_loss": 3.36637275095947, + "tokens_seen": 2806906880 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012092276830491476, + "loss": 1.2104, + "theoretical_loss": 3.3663666776651393, + "tokens_seen": 2806972416 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012090672016048146, + "loss": 1.1912, + "theoretical_loss": 3.366360604552306, + "tokens_seen": 2807037952 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012089067201604816, + "loss": 1.1654, + "theoretical_loss": 3.3663545316209595, + "tokens_seen": 2807103488 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012087462387161486, + "loss": 1.1406, + "theoretical_loss": 3.3663484588710912, + "tokens_seen": 2807169024 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012085857572718156, + "loss": 1.1948, + "theoretical_loss": 3.3663423863026907, + "tokens_seen": 2807234560 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012084252758274826, + "loss": 1.1925, + "theoretical_loss": 3.366336313915749, + "tokens_seen": 2807300096 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012082647943831496, + "loss": 1.1874, + "theoretical_loss": 3.366330241710256, + "tokens_seen": 2807365632 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012081043129388165, + "loss": 1.1526, + "theoretical_loss": 3.3663241696862016, + "tokens_seen": 2807431168 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012079438314944835, + "loss": 1.144, + "theoretical_loss": 3.3663180978435774, + "tokens_seen": 2807496704 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012077833500501505, + "loss": 1.1811, + "theoretical_loss": 3.3663120261823725, + "tokens_seen": 2807562240 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012076228686058175, + "loss": 1.3104, + "theoretical_loss": 3.3663059547025775, + "tokens_seen": 2807627776 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012074623871614845, + "loss": 1.2595, + "theoretical_loss": 3.3662998834041833, + "tokens_seen": 2807693312 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012073019057171515, + "loss": 1.3085, + "theoretical_loss": 3.36629381228718, + "tokens_seen": 2807758848 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012071414242728185, + "loss": 1.2142, + "theoretical_loss": 3.3662877413515577, + "tokens_seen": 2807824384 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012069809428284855, + "loss": 1.144, + "theoretical_loss": 3.3662816705973073, + "tokens_seen": 2807889920 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012068204613841525, + "loss": 1.1574, + "theoretical_loss": 3.3662756000244185, + "tokens_seen": 2807955456 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012066599799398195, + "loss": 1.1742, + "theoretical_loss": 3.366269529632882, + "tokens_seen": 2808020992 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012064994984954865, + "loss": 1.1758, + "theoretical_loss": 3.366263459422688, + "tokens_seen": 2808086528 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012063390170511536, + "loss": 1.1605, + "theoretical_loss": 3.3662573893938266, + "tokens_seen": 2808152064 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 937690, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2693142890930176, + "objective/train/theoretical_loss": 3.366251319546289, + "objective/train/tokens_used": 2828677600, + "theoretical_loss": 3.366251319546289, + "tokens_seen": 2808217600 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012061785356068204, + "loss": 1.2099, + "theoretical_loss": 3.366251319546289, + "tokens_seen": 2808217600 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012060180541624874, + "loss": 1.2386, + "theoretical_loss": 3.366245249880065, + "tokens_seen": 2808283136 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012058575727181544, + "loss": 1.1913, + "theoretical_loss": 3.3662391803951444, + "tokens_seen": 2808348672 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012056970912738214, + "loss": 1.1883, + "theoretical_loss": 3.3662331110915185, + "tokens_seen": 2808414208 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012055366098294884, + "loss": 1.2456, + "theoretical_loss": 3.3662270419691773, + "tokens_seen": 2808479744 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012053761283851555, + "loss": 1.2101, + "theoretical_loss": 3.366220973028111, + "tokens_seen": 2808545280 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012052156469408225, + "loss": 1.2556, + "theoretical_loss": 3.3662149042683103, + "tokens_seen": 2808610816 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012050551654964895, + "loss": 1.0976, + "theoretical_loss": 3.3662088356897657, + "tokens_seen": 2808676352 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012048946840521565, + "loss": 1.0534, + "theoretical_loss": 3.3662027672924664, + "tokens_seen": 2808741888 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012047342026078235, + "loss": 1.1552, + "theoretical_loss": 3.3661966990764043, + "tokens_seen": 2808807424 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012045737211634905, + "loss": 1.2465, + "theoretical_loss": 3.366190631041569, + "tokens_seen": 2808872960 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012044132397191575, + "loss": 1.2294, + "theoretical_loss": 3.36618456318795, + "tokens_seen": 2808938496 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012042527582748244, + "loss": 1.1644, + "theoretical_loss": 3.3661784955155394, + "tokens_seen": 2809004032 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012040922768304916, + "loss": 1.1963, + "theoretical_loss": 3.366172428024326, + "tokens_seen": 2809069568 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012039317953861586, + "loss": 1.1859, + "theoretical_loss": 3.3661663607143013, + "tokens_seen": 2809135104 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012037713139418257, + "loss": 1.1619, + "theoretical_loss": 3.3661602935854553, + "tokens_seen": 2809200640 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012036108324974927, + "loss": 1.2613, + "theoretical_loss": 3.366154226637778, + "tokens_seen": 2809266176 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012034503510531597, + "loss": 1.2603, + "theoretical_loss": 3.3661481598712606, + "tokens_seen": 2809331712 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012032898696088267, + "loss": 1.4623, + "theoretical_loss": 3.3661420932858928, + "tokens_seen": 2809397248 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012031293881644935, + "loss": 1.3322, + "theoretical_loss": 3.366136026881665, + "tokens_seen": 2809462784 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012029689067201605, + "loss": 1.2753, + "theoretical_loss": 3.3661299606585673, + "tokens_seen": 2809528320 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012028084252758276, + "loss": 1.158, + "theoretical_loss": 3.3661238946165906, + "tokens_seen": 2809593856 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012026479438314946, + "loss": 1.293, + "theoretical_loss": 3.3661178287557254, + "tokens_seen": 2809659392 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012024874623871616, + "loss": 1.24, + "theoretical_loss": 3.3661117630759616, + "tokens_seen": 2809724928 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012023269809428286, + "loss": 1.1844, + "theoretical_loss": 3.3661056975772894, + "tokens_seen": 2809790464 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012021664994984956, + "loss": 1.1573, + "theoretical_loss": 3.3660996322597, + "tokens_seen": 2809856000 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012020060180541626, + "loss": 1.2276, + "theoretical_loss": 3.366093567123183, + "tokens_seen": 2809921536 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012018455366098296, + "loss": 1.1968, + "theoretical_loss": 3.366087502167729, + "tokens_seen": 2809987072 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012016850551654966, + "loss": 1.2528, + "theoretical_loss": 3.366081437393328, + "tokens_seen": 2810052608 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012015245737211636, + "loss": 1.1064, + "theoretical_loss": 3.3660753727999717, + "tokens_seen": 2810118144 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012013640922768306, + "loss": 1.1956, + "theoretical_loss": 3.366069308387649, + "tokens_seen": 2810183680 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012012036108324975, + "loss": 1.1992, + "theoretical_loss": 3.3660632441563507, + "tokens_seen": 2810249216 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012010431293881645, + "loss": 1.2845, + "theoretical_loss": 3.366057180106068, + "tokens_seen": 2810314752 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012008826479438315, + "loss": 1.205, + "theoretical_loss": 3.36605111623679, + "tokens_seen": 2810380288 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012007221664994985, + "loss": 1.2454, + "theoretical_loss": 3.3660450525485075, + "tokens_seen": 2810445824 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012005616850551655, + "loss": 1.1671, + "theoretical_loss": 3.3660389890412117, + "tokens_seen": 2810511360 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012004012036108325, + "loss": 1.2502, + "theoretical_loss": 3.366032925714892, + "tokens_seen": 2810576896 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012002407221664995, + "loss": 1.1788, + "theoretical_loss": 3.3660268625695386, + "tokens_seen": 2810642432 + }, + { + "epoch": 0.85, + "learning_rate": 0.00012000802407221665, + "loss": 1.2257, + "theoretical_loss": 3.3660207996051428, + "tokens_seen": 2810707968 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011999197592778335, + "loss": 1.1455, + "theoretical_loss": 3.3660147368216946, + "tokens_seen": 2810773504 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011997592778335005, + "loss": 1.202, + "theoretical_loss": 3.3660086742191844, + "tokens_seen": 2810839040 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011995987963891675, + "loss": 1.1597, + "theoretical_loss": 3.3660026117976027, + "tokens_seen": 2810904576 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011994383149448346, + "loss": 1.2788, + "theoretical_loss": 3.365996549556939, + "tokens_seen": 2810970112 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011992778335005014, + "loss": 1.2916, + "theoretical_loss": 3.365990487497185, + "tokens_seen": 2811035648 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011991173520561684, + "loss": 1.2348, + "theoretical_loss": 3.36598442561833, + "tokens_seen": 2811101184 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011989568706118354, + "loss": 1.115, + "theoretical_loss": 3.365978363920365, + "tokens_seen": 2811166720 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011987963891675024, + "loss": 1.2472, + "theoretical_loss": 3.365972302403281, + "tokens_seen": 2811232256 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011986359077231694, + "loss": 1.1688, + "theoretical_loss": 3.3659662410670665, + "tokens_seen": 2811297792 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011984754262788365, + "loss": 1.2574, + "theoretical_loss": 3.3659601799117134, + "tokens_seen": 2811363328 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011983149448345035, + "loss": 1.1592, + "theoretical_loss": 3.365954118937212, + "tokens_seen": 2811428864 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 938558, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2163076400756836, + "objective/train/theoretical_loss": 3.365948058143552, + "objective/train/tokens_used": 2831954400, + "theoretical_loss": 3.365948058143552, + "tokens_seen": 2811494400 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011981544633901707, + "loss": 1.2344, + "theoretical_loss": 3.365948058143552, + "tokens_seen": 2811494400 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011979939819458376, + "loss": 1.2406, + "theoretical_loss": 3.365941997530724, + "tokens_seen": 2811559936 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011978335005015046, + "loss": 1.2351, + "theoretical_loss": 3.365935937098719, + "tokens_seen": 2811625472 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011976730190571716, + "loss": 1.2447, + "theoretical_loss": 3.365929876847527, + "tokens_seen": 2811691008 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011975125376128386, + "loss": 1.1833, + "theoretical_loss": 3.365923816777138, + "tokens_seen": 2811756544 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011973520561685056, + "loss": 1.1611, + "theoretical_loss": 3.365917756887543, + "tokens_seen": 2811822080 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011971915747241726, + "loss": 1.2462, + "theoretical_loss": 3.365911697178732, + "tokens_seen": 2811887616 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011970310932798396, + "loss": 1.1839, + "theoretical_loss": 3.3659056376506955, + "tokens_seen": 2811953152 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011968706118355067, + "loss": 1.2456, + "theoretical_loss": 3.365899578303424, + "tokens_seen": 2812018688 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011967101303911737, + "loss": 1.1086, + "theoretical_loss": 3.365893519136908, + "tokens_seen": 2812084224 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011965496489468407, + "loss": 1.2684, + "theoretical_loss": 3.3658874601511375, + "tokens_seen": 2812149760 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011963891675025077, + "loss": 1.2457, + "theoretical_loss": 3.365881401346103, + "tokens_seen": 2812215296 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011962286860581747, + "loss": 1.216, + "theoretical_loss": 3.365875342721795, + "tokens_seen": 2812280832 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011960682046138416, + "loss": 1.2024, + "theoretical_loss": 3.3658692842782036, + "tokens_seen": 2812346368 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011959077231695086, + "loss": 1.3079, + "theoretical_loss": 3.3658632260153203, + "tokens_seen": 2812411904 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011957472417251756, + "loss": 1.1679, + "theoretical_loss": 3.365857167933134, + "tokens_seen": 2812477440 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011955867602808426, + "loss": 1.0897, + "theoretical_loss": 3.3658511100316364, + "tokens_seen": 2812542976 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011954262788365096, + "loss": 1.2207, + "theoretical_loss": 3.3658450523108168, + "tokens_seen": 2812608512 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011952657973921766, + "loss": 1.2187, + "theoretical_loss": 3.365838994770666, + "tokens_seen": 2812674048 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011951053159478436, + "loss": 1.1792, + "theoretical_loss": 3.365832937411175, + "tokens_seen": 2812739584 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011949448345035106, + "loss": 1.1828, + "theoretical_loss": 3.3658268802323335, + "tokens_seen": 2812805120 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011947843530591776, + "loss": 1.2791, + "theoretical_loss": 3.365820823234132, + "tokens_seen": 2812870656 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011946238716148446, + "loss": 1.1937, + "theoretical_loss": 3.3658147664165607, + "tokens_seen": 2812936192 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011944633901705116, + "loss": 1.1383, + "theoretical_loss": 3.365808709779611, + "tokens_seen": 2813001728 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011943029087261786, + "loss": 1.1574, + "theoretical_loss": 3.3658026533232723, + "tokens_seen": 2813067264 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011941424272818455, + "loss": 1.1149, + "theoretical_loss": 3.3657965970475354, + "tokens_seen": 2813132800 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011939819458375125, + "loss": 1.2396, + "theoretical_loss": 3.3657905409523905, + "tokens_seen": 2813198336 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011938214643931795, + "loss": 1.2965, + "theoretical_loss": 3.3657844850378282, + "tokens_seen": 2813263872 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011936609829488465, + "loss": 1.1905, + "theoretical_loss": 3.365778429303839, + "tokens_seen": 2813329408 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011935005015045135, + "loss": 1.2068, + "theoretical_loss": 3.3657723737504126, + "tokens_seen": 2813394944 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011933400200601805, + "loss": 1.1766, + "theoretical_loss": 3.3657663183775406, + "tokens_seen": 2813460480 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011931795386158475, + "loss": 1.2202, + "theoretical_loss": 3.3657602631852126, + "tokens_seen": 2813526016 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011930190571715145, + "loss": 1.2233, + "theoretical_loss": 3.365754208173419, + "tokens_seen": 2813591552 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011928585757271815, + "loss": 1.1447, + "theoretical_loss": 3.3657481533421505, + "tokens_seen": 2813657088 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011926980942828485, + "loss": 1.1716, + "theoretical_loss": 3.3657420986913973, + "tokens_seen": 2813722624 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011925376128385156, + "loss": 1.1205, + "theoretical_loss": 3.3657360442211504, + "tokens_seen": 2813788160 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011923771313941826, + "loss": 1.2439, + "theoretical_loss": 3.3657299899313995, + "tokens_seen": 2813853696 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011922166499498494, + "loss": 1.1634, + "theoretical_loss": 3.3657239358221354, + "tokens_seen": 2813919232 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011920561685055167, + "loss": 1.2597, + "theoretical_loss": 3.365717881893348, + "tokens_seen": 2813984768 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011918956870611837, + "loss": 1.2026, + "theoretical_loss": 3.3657118281450282, + "tokens_seen": 2814050304 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011917352056168507, + "loss": 1.2558, + "theoretical_loss": 3.3657057745771666, + "tokens_seen": 2814115840 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011915747241725177, + "loss": 1.2515, + "theoretical_loss": 3.365699721189753, + "tokens_seen": 2814181376 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011914142427281847, + "loss": 1.1529, + "theoretical_loss": 3.3656936679827782, + "tokens_seen": 2814246912 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011912537612838517, + "loss": 1.2118, + "theoretical_loss": 3.365687614956233, + "tokens_seen": 2814312448 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011910932798395187, + "loss": 1.2588, + "theoretical_loss": 3.365681562110107, + "tokens_seen": 2814377984 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011909327983951856, + "loss": 1.1693, + "theoretical_loss": 3.365675509444391, + "tokens_seen": 2814443520 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011907723169508526, + "loss": 1.075, + "theoretical_loss": 3.3656694569590755, + "tokens_seen": 2814509056 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011906118355065196, + "loss": 1.1684, + "theoretical_loss": 3.3656634046541507, + "tokens_seen": 2814574592 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011904513540621866, + "loss": 1.2277, + "theoretical_loss": 3.3656573525296074, + "tokens_seen": 2814640128 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011902908726178536, + "loss": 1.3156, + "theoretical_loss": 3.365651300585436, + "tokens_seen": 2814705664 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 939733, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.073119878768921, + "objective/train/theoretical_loss": 3.365645248821626, + "objective/train/tokens_used": 2835231200, + "theoretical_loss": 3.365645248821626, + "tokens_seen": 2814771200 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011901303911735206, + "loss": 1.0649, + "theoretical_loss": 3.365645248821626, + "tokens_seen": 2814771200 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011899699097291877, + "loss": 1.2539, + "theoretical_loss": 3.3656391972381696, + "tokens_seen": 2814836736 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011898094282848547, + "loss": 1.1403, + "theoretical_loss": 3.3656331458350555, + "tokens_seen": 2814902272 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011896489468405217, + "loss": 1.2528, + "theoretical_loss": 3.365627094612275, + "tokens_seen": 2814967808 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011894884653961887, + "loss": 1.3, + "theoretical_loss": 3.365621043569818, + "tokens_seen": 2815033344 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011893279839518557, + "loss": 1.1999, + "theoretical_loss": 3.3656149927076755, + "tokens_seen": 2815098880 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011891675025075227, + "loss": 1.2921, + "theoretical_loss": 3.3656089420258377, + "tokens_seen": 2815164416 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011890070210631896, + "loss": 1.1776, + "theoretical_loss": 3.3656028915242953, + "tokens_seen": 2815229952 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011888465396188566, + "loss": 1.1377, + "theoretical_loss": 3.365596841203038, + "tokens_seen": 2815295488 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011886860581745236, + "loss": 1.2006, + "theoretical_loss": 3.365590791062057, + "tokens_seen": 2815361024 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011885255767301906, + "loss": 1.1573, + "theoretical_loss": 3.365584741101342, + "tokens_seen": 2815426560 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011883650952858576, + "loss": 1.167, + "theoretical_loss": 3.3655786913208843, + "tokens_seen": 2815492096 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011882046138415246, + "loss": 1.2064, + "theoretical_loss": 3.3655726417206737, + "tokens_seen": 2815557632 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011880441323971916, + "loss": 1.2203, + "theoretical_loss": 3.365566592300701, + "tokens_seen": 2815623168 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011878836509528586, + "loss": 1.2241, + "theoretical_loss": 3.365560543060956, + "tokens_seen": 2815688704 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011877231695085256, + "loss": 1.2115, + "theoretical_loss": 3.3655544940014304, + "tokens_seen": 2815754240 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011875626880641926, + "loss": 1.2811, + "theoretical_loss": 3.365548445122113, + "tokens_seen": 2815819776 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011874022066198596, + "loss": 1.1879, + "theoretical_loss": 3.3655423964229954, + "tokens_seen": 2815885312 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011872417251755266, + "loss": 1.1704, + "theoretical_loss": 3.365536347904068, + "tokens_seen": 2815950848 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011870812437311935, + "loss": 1.1638, + "theoretical_loss": 3.3655302995653207, + "tokens_seen": 2816016384 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011869207622868605, + "loss": 1.2676, + "theoretical_loss": 3.3655242514067436, + "tokens_seen": 2816081920 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011867602808425275, + "loss": 1.2554, + "theoretical_loss": 3.3655182034283286, + "tokens_seen": 2816147456 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011865997993981945, + "loss": 1.3026, + "theoretical_loss": 3.3655121556300647, + "tokens_seen": 2816212992 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011864393179538615, + "loss": 1.2599, + "theoretical_loss": 3.3655061080119433, + "tokens_seen": 2816278528 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011862788365095285, + "loss": 1.2127, + "theoretical_loss": 3.3655000605739542, + "tokens_seen": 2816344064 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011861183550651958, + "loss": 1.2861, + "theoretical_loss": 3.365494013316088, + "tokens_seen": 2816409600 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011859578736208628, + "loss": 1.2101, + "theoretical_loss": 3.3654879662383355, + "tokens_seen": 2816475136 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011857973921765297, + "loss": 1.2423, + "theoretical_loss": 3.3654819193406866, + "tokens_seen": 2816540672 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011856369107321967, + "loss": 1.1212, + "theoretical_loss": 3.3654758726231324, + "tokens_seen": 2816606208 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011854764292878637, + "loss": 1.2161, + "theoretical_loss": 3.3654698260856626, + "tokens_seen": 2816671744 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011853159478435307, + "loss": 1.1652, + "theoretical_loss": 3.3654637797282683, + "tokens_seen": 2816737280 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011851554663991977, + "loss": 1.233, + "theoretical_loss": 3.3654577335509392, + "tokens_seen": 2816802816 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011849949849548647, + "loss": 1.2067, + "theoretical_loss": 3.3654516875536666, + "tokens_seen": 2816868352 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011848345035105317, + "loss": 1.1894, + "theoretical_loss": 3.3654456417364407, + "tokens_seen": 2816933888 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011846740220661987, + "loss": 1.1493, + "theoretical_loss": 3.3654395960992516, + "tokens_seen": 2816999424 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011845135406218657, + "loss": 1.1541, + "theoretical_loss": 3.36543355064209, + "tokens_seen": 2817064960 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011843530591775327, + "loss": 1.1981, + "theoretical_loss": 3.3654275053649463, + "tokens_seen": 2817130496 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011841925777331997, + "loss": 1.3229, + "theoretical_loss": 3.365421460267811, + "tokens_seen": 2817196032 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011840320962888668, + "loss": 1.2062, + "theoretical_loss": 3.3654154153506743, + "tokens_seen": 2817261568 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011838716148445336, + "loss": 1.1036, + "theoretical_loss": 3.365409370613527, + "tokens_seen": 2817327104 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011837111334002006, + "loss": 1.1652, + "theoretical_loss": 3.3654033260563594, + "tokens_seen": 2817392640 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011835506519558676, + "loss": 1.2087, + "theoretical_loss": 3.3653972816791624, + "tokens_seen": 2817458176 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011833901705115346, + "loss": 1.1299, + "theoretical_loss": 3.3653912374819255, + "tokens_seen": 2817523712 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011832296890672017, + "loss": 1.3145, + "theoretical_loss": 3.3653851934646397, + "tokens_seen": 2817589248 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011830692076228687, + "loss": 1.2394, + "theoretical_loss": 3.365379149627296, + "tokens_seen": 2817654784 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011829087261785357, + "loss": 1.1571, + "theoretical_loss": 3.3653731059698835, + "tokens_seen": 2817720320 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011827482447342027, + "loss": 1.2837, + "theoretical_loss": 3.365367062492394, + "tokens_seen": 2817785856 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011825877632898697, + "loss": 1.1511, + "theoretical_loss": 3.3653610191948173, + "tokens_seen": 2817851392 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011824272818455367, + "loss": 1.2772, + "theoretical_loss": 3.365354976077144, + "tokens_seen": 2817916928 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011822668004012037, + "loss": 1.2289, + "theoretical_loss": 3.3653489331393645, + "tokens_seen": 2817982464 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 940663, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1507428884506226, + "objective/train/theoretical_loss": 3.3653428903814695, + "objective/train/tokens_used": 2838508000, + "theoretical_loss": 3.3653428903814695, + "tokens_seen": 2818048000 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011821063189568707, + "loss": 1.1458, + "theoretical_loss": 3.3653428903814695, + "tokens_seen": 2818048000 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011819458375125376, + "loss": 1.2715, + "theoretical_loss": 3.365336847803449, + "tokens_seen": 2818113536 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011817853560682046, + "loss": 1.2784, + "theoretical_loss": 3.3653308054052937, + "tokens_seen": 2818179072 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011816248746238716, + "loss": 1.1996, + "theoretical_loss": 3.3653247631869943, + "tokens_seen": 2818244608 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011814643931795386, + "loss": 1.1682, + "theoretical_loss": 3.365318721148541, + "tokens_seen": 2818310144 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011813039117352056, + "loss": 1.1733, + "theoretical_loss": 3.365312679289924, + "tokens_seen": 2818375680 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011811434302908726, + "loss": 1.1494, + "theoretical_loss": 3.365306637611135, + "tokens_seen": 2818441216 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011809829488465396, + "loss": 1.2774, + "theoretical_loss": 3.3653005961121627, + "tokens_seen": 2818506752 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011808224674022066, + "loss": 1.313, + "theoretical_loss": 3.3652945547929987, + "tokens_seen": 2818572288 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011806619859578736, + "loss": 1.2368, + "theoretical_loss": 3.365288513653633, + "tokens_seen": 2818637824 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011805015045135406, + "loss": 1.2341, + "theoretical_loss": 3.3652824726940564, + "tokens_seen": 2818703360 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011803410230692076, + "loss": 1.1698, + "theoretical_loss": 3.3652764319142596, + "tokens_seen": 2818768896 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011801805416248746, + "loss": 1.2902, + "theoretical_loss": 3.3652703913142323, + "tokens_seen": 2818834432 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011800200601805418, + "loss": 1.2286, + "theoretical_loss": 3.3652643508939653, + "tokens_seen": 2818899968 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011798595787362088, + "loss": 1.266, + "theoretical_loss": 3.3652583106534495, + "tokens_seen": 2818965504 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011796990972918758, + "loss": 1.135, + "theoretical_loss": 3.3652522705926744, + "tokens_seen": 2819031040 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011795386158475428, + "loss": 1.2195, + "theoretical_loss": 3.3652462307116315, + "tokens_seen": 2819096576 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011793781344032098, + "loss": 1.2464, + "theoretical_loss": 3.3652401910103107, + "tokens_seen": 2819162112 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011792176529588768, + "loss": 1.1833, + "theoretical_loss": 3.365234151488703, + "tokens_seen": 2819227648 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011790571715145438, + "loss": 1.1251, + "theoretical_loss": 3.365228112146798, + "tokens_seen": 2819293184 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011788966900702107, + "loss": 1.1903, + "theoretical_loss": 3.3652220729845874, + "tokens_seen": 2819358720 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011787362086258777, + "loss": 1.2903, + "theoretical_loss": 3.36521603400206, + "tokens_seen": 2819424256 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011785757271815447, + "loss": 1.2684, + "theoretical_loss": 3.365209995199208, + "tokens_seen": 2819489792 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011784152457372117, + "loss": 1.3005, + "theoretical_loss": 3.365203956576021, + "tokens_seen": 2819555328 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011782547642928787, + "loss": 1.1662, + "theoretical_loss": 3.3651979181324894, + "tokens_seen": 2819620864 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011780942828485457, + "loss": 1.1789, + "theoretical_loss": 3.365191879868604, + "tokens_seen": 2819686400 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011779338014042127, + "loss": 1.1872, + "theoretical_loss": 3.365185841784355, + "tokens_seen": 2819751936 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011777733199598797, + "loss": 1.0916, + "theoretical_loss": 3.365179803879733, + "tokens_seen": 2819817472 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011776128385155467, + "loss": 1.1884, + "theoretical_loss": 3.3651737661547285, + "tokens_seen": 2819883008 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011774523570712137, + "loss": 1.2931, + "theoretical_loss": 3.3651677286093324, + "tokens_seen": 2819948544 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011772918756268808, + "loss": 1.1118, + "theoretical_loss": 3.3651616912435345, + "tokens_seen": 2820014080 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011771313941825478, + "loss": 1.1885, + "theoretical_loss": 3.3651556540573258, + "tokens_seen": 2820079616 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011769709127382146, + "loss": 1.1928, + "theoretical_loss": 3.365149617050696, + "tokens_seen": 2820145152 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011768104312938816, + "loss": 1.2791, + "theoretical_loss": 3.365143580223637, + "tokens_seen": 2820210688 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011766499498495486, + "loss": 1.1237, + "theoretical_loss": 3.3651375435761377, + "tokens_seen": 2820276224 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011764894684052156, + "loss": 1.2496, + "theoretical_loss": 3.365131507108189, + "tokens_seen": 2820341760 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011763289869608827, + "loss": 1.3651, + "theoretical_loss": 3.3651254708197826, + "tokens_seen": 2820407296 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011761685055165497, + "loss": 1.1978, + "theoretical_loss": 3.3651194347109077, + "tokens_seen": 2820472832 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011760080240722167, + "loss": 1.1929, + "theoretical_loss": 3.3651133987815554, + "tokens_seen": 2820538368 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011758475426278837, + "loss": 1.1647, + "theoretical_loss": 3.3651073630317154, + "tokens_seen": 2820603904 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011756870611835507, + "loss": 1.2478, + "theoretical_loss": 3.365101327461379, + "tokens_seen": 2820669440 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011755265797392177, + "loss": 1.2153, + "theoretical_loss": 3.365095292070537, + "tokens_seen": 2820734976 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011753660982948847, + "loss": 1.3021, + "theoretical_loss": 3.3650892568591786, + "tokens_seen": 2820800512 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011752056168505517, + "loss": 1.2864, + "theoretical_loss": 3.3650832218272955, + "tokens_seen": 2820866048 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011750451354062186, + "loss": 1.2474, + "theoretical_loss": 3.3650771869748772, + "tokens_seen": 2820931584 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011748846539618856, + "loss": 1.1941, + "theoretical_loss": 3.3650711523019154, + "tokens_seen": 2820997120 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011747241725175526, + "loss": 1.1688, + "theoretical_loss": 3.3650651178083995, + "tokens_seen": 2821062656 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011745636910732196, + "loss": 1.2495, + "theoretical_loss": 3.3650590834943204, + "tokens_seen": 2821128192 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011744032096288866, + "loss": 1.2143, + "theoretical_loss": 3.365053049359669, + "tokens_seen": 2821193728 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011742427281845536, + "loss": 1.1829, + "theoretical_loss": 3.3650470154044347, + "tokens_seen": 2821259264 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 941927, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.190575361251831, + "objective/train/theoretical_loss": 3.365040981628609, + "objective/train/tokens_used": 2841784800, + "theoretical_loss": 3.365040981628609, + "tokens_seen": 2821324800 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011740822467402209, + "loss": 1.2109, + "theoretical_loss": 3.365040981628609, + "tokens_seen": 2821324800 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011739217652958879, + "loss": 1.1258, + "theoretical_loss": 3.365034948032182, + "tokens_seen": 2821390336 + }, + { + "epoch": 0.85, + "learning_rate": 0.00011737612838515548, + "loss": 1.1843, + "theoretical_loss": 3.365028914615145, + "tokens_seen": 2821455872 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011736008024072218, + "loss": 1.1796, + "theoretical_loss": 3.365022881377487, + "tokens_seen": 2821521408 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011734403209628888, + "loss": 1.1335, + "theoretical_loss": 3.3650168483191996, + "tokens_seen": 2821586944 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011732798395185558, + "loss": 1.169, + "theoretical_loss": 3.365010815440273, + "tokens_seen": 2821652480 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011731193580742228, + "loss": 1.1348, + "theoretical_loss": 3.365004782740698, + "tokens_seen": 2821718016 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011729588766298898, + "loss": 1.1396, + "theoretical_loss": 3.3649987502204644, + "tokens_seen": 2821783552 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011727983951855568, + "loss": 1.218, + "theoretical_loss": 3.364992717879563, + "tokens_seen": 2821849088 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011726379137412238, + "loss": 1.1721, + "theoretical_loss": 3.3649866857179846, + "tokens_seen": 2821914624 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011724774322968908, + "loss": 1.201, + "theoretical_loss": 3.3649806537357194, + "tokens_seen": 2821980160 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011723169508525578, + "loss": 1.2074, + "theoretical_loss": 3.3649746219327583, + "tokens_seen": 2822045696 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011721564694082248, + "loss": 1.2577, + "theoretical_loss": 3.3649685903090916, + "tokens_seen": 2822111232 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011719959879638918, + "loss": 1.2535, + "theoretical_loss": 3.3649625588647094, + "tokens_seen": 2822176768 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011718355065195587, + "loss": 1.1379, + "theoretical_loss": 3.3649565275996025, + "tokens_seen": 2822242304 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011716750250752257, + "loss": 1.1741, + "theoretical_loss": 3.3649504965137615, + "tokens_seen": 2822307840 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011715145436308927, + "loss": 1.165, + "theoretical_loss": 3.364944465607177, + "tokens_seen": 2822373376 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011713540621865597, + "loss": 1.2645, + "theoretical_loss": 3.3649384348798392, + "tokens_seen": 2822438912 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011711935807422267, + "loss": 1.1266, + "theoretical_loss": 3.3649324043317392, + "tokens_seen": 2822504448 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011710330992978937, + "loss": 1.2776, + "theoretical_loss": 3.3649263739628665, + "tokens_seen": 2822569984 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011708726178535607, + "loss": 1.1387, + "theoretical_loss": 3.3649203437732127, + "tokens_seen": 2822635520 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011707121364092277, + "loss": 1.1406, + "theoretical_loss": 3.3649143137627675, + "tokens_seen": 2822701056 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011705516549648947, + "loss": 1.2242, + "theoretical_loss": 3.3649082839315216, + "tokens_seen": 2822766592 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011703911735205618, + "loss": 1.2172, + "theoretical_loss": 3.364902254279466, + "tokens_seen": 2822832128 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011702306920762288, + "loss": 1.3431, + "theoretical_loss": 3.364896224806591, + "tokens_seen": 2822897664 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011700702106318958, + "loss": 1.2457, + "theoretical_loss": 3.3648901955128867, + "tokens_seen": 2822963200 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011699097291875626, + "loss": 1.1831, + "theoretical_loss": 3.364884166398344, + "tokens_seen": 2823028736 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011697492477432296, + "loss": 1.1306, + "theoretical_loss": 3.364878137462953, + "tokens_seen": 2823094272 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011695887662988966, + "loss": 1.1945, + "theoretical_loss": 3.364872108706705, + "tokens_seen": 2823159808 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011694282848545637, + "loss": 1.0889, + "theoretical_loss": 3.3648660801295893, + "tokens_seen": 2823225344 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011692678034102307, + "loss": 1.2155, + "theoretical_loss": 3.364860051731598, + "tokens_seen": 2823290880 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011691073219658977, + "loss": 1.1789, + "theoretical_loss": 3.3648540235127204, + "tokens_seen": 2823356416 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011689468405215647, + "loss": 1.21, + "theoretical_loss": 3.3648479954729473, + "tokens_seen": 2823421952 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011687863590772317, + "loss": 1.2112, + "theoretical_loss": 3.3648419676122696, + "tokens_seen": 2823487488 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011686258776328987, + "loss": 1.2079, + "theoretical_loss": 3.3648359399306775, + "tokens_seen": 2823553024 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011684653961885657, + "loss": 1.2171, + "theoretical_loss": 3.3648299124281618, + "tokens_seen": 2823618560 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011683049147442327, + "loss": 1.1725, + "theoretical_loss": 3.364823885104712, + "tokens_seen": 2823684096 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011681444332998998, + "loss": 1.2079, + "theoretical_loss": 3.36481785796032, + "tokens_seen": 2823749632 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011679839518555668, + "loss": 1.1612, + "theoretical_loss": 3.364811830994976, + "tokens_seen": 2823815168 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011678234704112339, + "loss": 1.2371, + "theoretical_loss": 3.36480580420867, + "tokens_seen": 2823880704 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011676629889669009, + "loss": 1.1947, + "theoretical_loss": 3.3647997776013927, + "tokens_seen": 2823946240 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011675025075225679, + "loss": 1.0589, + "theoretical_loss": 3.3647937511731345, + "tokens_seen": 2824011776 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011673420260782349, + "loss": 1.2328, + "theoretical_loss": 3.3647877249238864, + "tokens_seen": 2824077312 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011671815446339019, + "loss": 1.226, + "theoretical_loss": 3.3647816988536388, + "tokens_seen": 2824142848 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011670210631895689, + "loss": 1.2789, + "theoretical_loss": 3.3647756729623817, + "tokens_seen": 2824208384 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011668605817452359, + "loss": 1.2528, + "theoretical_loss": 3.3647696472501067, + "tokens_seen": 2824273920 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011667001003009028, + "loss": 1.1754, + "theoretical_loss": 3.3647636217168033, + "tokens_seen": 2824339456 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011665396188565698, + "loss": 1.2573, + "theoretical_loss": 3.3647575963624625, + "tokens_seen": 2824404992 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011663791374122368, + "loss": 1.3048, + "theoretical_loss": 3.3647515711870746, + "tokens_seen": 2824470528 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011662186559679038, + "loss": 1.1656, + "theoretical_loss": 3.36474554619063, + "tokens_seen": 2824536064 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 943148, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.281390905380249, + "objective/train/theoretical_loss": 3.36473952137312, + "objective/train/tokens_used": 2845061600, + "theoretical_loss": 3.36473952137312, + "tokens_seen": 2824601600 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011660581745235708, + "loss": 1.2658, + "theoretical_loss": 3.36473952137312, + "tokens_seen": 2824601600 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011658976930792378, + "loss": 1.2548, + "theoretical_loss": 3.3647334967345346, + "tokens_seen": 2824667136 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011657372116349048, + "loss": 1.1725, + "theoretical_loss": 3.3647274722748644, + "tokens_seen": 2824732672 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011655767301905718, + "loss": 1.1693, + "theoretical_loss": 3.3647214479940994, + "tokens_seen": 2824798208 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011654162487462388, + "loss": 1.3404, + "theoretical_loss": 3.364715423892231, + "tokens_seen": 2824863744 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011652557673019058, + "loss": 1.1218, + "theoretical_loss": 3.3647093999692497, + "tokens_seen": 2824929280 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011650952858575728, + "loss": 1.343, + "theoretical_loss": 3.3647033762251453, + "tokens_seen": 2824994816 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011649348044132398, + "loss": 1.1681, + "theoretical_loss": 3.364697352659909, + "tokens_seen": 2825060352 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011647743229689067, + "loss": 1.23, + "theoretical_loss": 3.3646913292735308, + "tokens_seen": 2825125888 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011646138415245737, + "loss": 1.2021, + "theoretical_loss": 3.364685306066002, + "tokens_seen": 2825191424 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011644533600802407, + "loss": 1.281, + "theoretical_loss": 3.3646792830373125, + "tokens_seen": 2825256960 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011642928786359077, + "loss": 1.1973, + "theoretical_loss": 3.3646732601874527, + "tokens_seen": 2825322496 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011641323971915747, + "loss": 1.2437, + "theoretical_loss": 3.3646672375164135, + "tokens_seen": 2825388032 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011639719157472417, + "loss": 1.2279, + "theoretical_loss": 3.364661215024186, + "tokens_seen": 2825453568 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011638114343029087, + "loss": 1.208, + "theoretical_loss": 3.3646551927107593, + "tokens_seen": 2825519104 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011636509528585757, + "loss": 1.2424, + "theoretical_loss": 3.3646491705761257, + "tokens_seen": 2825584640 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011634904714142428, + "loss": 1.3056, + "theoretical_loss": 3.364643148620274, + "tokens_seen": 2825650176 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011633299899699098, + "loss": 1.1312, + "theoretical_loss": 3.364637126843196, + "tokens_seen": 2825715712 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011631695085255768, + "loss": 1.1733, + "theoretical_loss": 3.364631105244882, + "tokens_seen": 2825781248 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011630090270812438, + "loss": 1.1974, + "theoretical_loss": 3.364625083825322, + "tokens_seen": 2825846784 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011628485456369106, + "loss": 1.2392, + "theoretical_loss": 3.364619062584507, + "tokens_seen": 2825912320 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011626880641925777, + "loss": 1.2942, + "theoretical_loss": 3.3646130415224276, + "tokens_seen": 2825977856 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011625275827482447, + "loss": 1.2328, + "theoretical_loss": 3.3646070206390744, + "tokens_seen": 2826043392 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011623671013039117, + "loss": 1.2027, + "theoretical_loss": 3.3646009999344377, + "tokens_seen": 2826108928 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011622066198595787, + "loss": 1.2518, + "theoretical_loss": 3.364594979408508, + "tokens_seen": 2826174464 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001162046138415246, + "loss": 1.1542, + "theoretical_loss": 3.364588959061276, + "tokens_seen": 2826240000 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001161885656970913, + "loss": 1.2489, + "theoretical_loss": 3.364582938892732, + "tokens_seen": 2826305536 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011617251755265798, + "loss": 1.3086, + "theoretical_loss": 3.364576918902867, + "tokens_seen": 2826371072 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011615646940822468, + "loss": 1.1888, + "theoretical_loss": 3.364570899091672, + "tokens_seen": 2826436608 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011614042126379138, + "loss": 1.3329, + "theoretical_loss": 3.364564879459136, + "tokens_seen": 2826502144 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011612437311935808, + "loss": 1.1353, + "theoretical_loss": 3.3645588600052507, + "tokens_seen": 2826567680 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011610832497492478, + "loss": 1.164, + "theoretical_loss": 3.3645528407300063, + "tokens_seen": 2826633216 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011609227683049149, + "loss": 1.3083, + "theoretical_loss": 3.3645468216333936, + "tokens_seen": 2826698752 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011607622868605819, + "loss": 1.2252, + "theoretical_loss": 3.364540802715403, + "tokens_seen": 2826764288 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011606018054162489, + "loss": 1.3175, + "theoretical_loss": 3.3645347839760253, + "tokens_seen": 2826829824 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011604413239719159, + "loss": 1.2764, + "theoretical_loss": 3.3645287654152503, + "tokens_seen": 2826895360 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011602808425275829, + "loss": 1.1956, + "theoretical_loss": 3.3645227470330696, + "tokens_seen": 2826960896 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011601203610832499, + "loss": 1.186, + "theoretical_loss": 3.3645167288294733, + "tokens_seen": 2827026432 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011599598796389169, + "loss": 1.1216, + "theoretical_loss": 3.3645107108044514, + "tokens_seen": 2827091968 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011597993981945838, + "loss": 1.2104, + "theoretical_loss": 3.364504692957995, + "tokens_seen": 2827157504 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011596389167502508, + "loss": 1.1491, + "theoretical_loss": 3.364498675290095, + "tokens_seen": 2827223040 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011594784353059178, + "loss": 1.1685, + "theoretical_loss": 3.364492657800742, + "tokens_seen": 2827288576 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011593179538615848, + "loss": 1.1789, + "theoretical_loss": 3.3644866404899254, + "tokens_seen": 2827354112 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011591574724172518, + "loss": 1.2877, + "theoretical_loss": 3.364480623357637, + "tokens_seen": 2827419648 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011589969909729188, + "loss": 1.1911, + "theoretical_loss": 3.3644746064038666, + "tokens_seen": 2827485184 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011588365095285858, + "loss": 1.2638, + "theoretical_loss": 3.3644685896286055, + "tokens_seen": 2827550720 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011586760280842528, + "loss": 1.2728, + "theoretical_loss": 3.3644625730318434, + "tokens_seen": 2827616256 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011585155466399198, + "loss": 1.161, + "theoretical_loss": 3.3644565566135713, + "tokens_seen": 2827681792 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011583550651955868, + "loss": 1.2416, + "theoretical_loss": 3.36445054037378, + "tokens_seen": 2827747328 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011581945837512538, + "loss": 1.2548, + "theoretical_loss": 3.36444452431246, + "tokens_seen": 2827812864 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 944125, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2729814052581787, + "objective/train/theoretical_loss": 3.3644385084296013, + "objective/train/tokens_used": 2848338400, + "theoretical_loss": 3.3644385084296013, + "tokens_seen": 2827878400 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011580341023069208, + "loss": 1.2778, + "theoretical_loss": 3.3644385084296013, + "tokens_seen": 2827878400 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011578736208625877, + "loss": 1.1444, + "theoretical_loss": 3.364432492725195, + "tokens_seen": 2827943936 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011577131394182547, + "loss": 1.1977, + "theoretical_loss": 3.3644264771992316, + "tokens_seen": 2828009472 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011575526579739217, + "loss": 1.2721, + "theoretical_loss": 3.3644204618517017, + "tokens_seen": 2828075008 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011573921765295887, + "loss": 1.2646, + "theoretical_loss": 3.364414446682596, + "tokens_seen": 2828140544 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011572316950852557, + "loss": 1.3138, + "theoretical_loss": 3.3644084316919045, + "tokens_seen": 2828206080 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011570712136409227, + "loss": 1.207, + "theoretical_loss": 3.364402416879618, + "tokens_seen": 2828271616 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011569107321965897, + "loss": 1.1388, + "theoretical_loss": 3.3643964022457276, + "tokens_seen": 2828337152 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011567502507522567, + "loss": 1.3198, + "theoretical_loss": 3.3643903877902233, + "tokens_seen": 2828402688 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011565897693079238, + "loss": 1.1947, + "theoretical_loss": 3.364384373513096, + "tokens_seen": 2828468224 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011564292878635908, + "loss": 1.1302, + "theoretical_loss": 3.364378359414336, + "tokens_seen": 2828533760 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011562688064192578, + "loss": 1.2306, + "theoretical_loss": 3.3643723454939343, + "tokens_seen": 2828599296 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011561083249749249, + "loss": 1.2746, + "theoretical_loss": 3.3643663317518806, + "tokens_seen": 2828664832 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011559478435305919, + "loss": 1.3339, + "theoretical_loss": 3.3643603181881665, + "tokens_seen": 2828730368 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011557873620862589, + "loss": 1.2186, + "theoretical_loss": 3.364354304802782, + "tokens_seen": 2828795904 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011556268806419259, + "loss": 1.1549, + "theoretical_loss": 3.364348291595718, + "tokens_seen": 2828861440 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001155466399197593, + "loss": 1.2354, + "theoretical_loss": 3.3643422785669648, + "tokens_seen": 2828926976 + }, + { + "epoch": 0.86, + "learning_rate": 0.000115530591775326, + "loss": 1.3079, + "theoretical_loss": 3.3643362657165135, + "tokens_seen": 2828992512 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001155145436308927, + "loss": 1.366, + "theoretical_loss": 3.3643302530443537, + "tokens_seen": 2829058048 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001154984954864594, + "loss": 1.1065, + "theoretical_loss": 3.364324240550477, + "tokens_seen": 2829123584 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001154824473420261, + "loss": 1.1644, + "theoretical_loss": 3.3643182282348736, + "tokens_seen": 2829189120 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011546639919759278, + "loss": 1.1819, + "theoretical_loss": 3.3643122160975336, + "tokens_seen": 2829254656 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011545035105315948, + "loss": 1.2343, + "theoretical_loss": 3.3643062041384484, + "tokens_seen": 2829320192 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011543430290872618, + "loss": 1.1704, + "theoretical_loss": 3.364300192357608, + "tokens_seen": 2829385728 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011541825476429289, + "loss": 1.2167, + "theoretical_loss": 3.364294180755003, + "tokens_seen": 2829451264 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011540220661985959, + "loss": 1.2862, + "theoretical_loss": 3.364288169330625, + "tokens_seen": 2829516800 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011538615847542629, + "loss": 1.2031, + "theoretical_loss": 3.364282158084463, + "tokens_seen": 2829582336 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011537011033099299, + "loss": 1.2854, + "theoretical_loss": 3.3642761470165086, + "tokens_seen": 2829647872 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011535406218655969, + "loss": 1.2259, + "theoretical_loss": 3.364270136126752, + "tokens_seen": 2829713408 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011533801404212639, + "loss": 1.2247, + "theoretical_loss": 3.364264125415184, + "tokens_seen": 2829778944 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011532196589769309, + "loss": 1.249, + "theoretical_loss": 3.364258114881795, + "tokens_seen": 2829844480 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011530591775325979, + "loss": 1.2539, + "theoretical_loss": 3.3642521045265763, + "tokens_seen": 2829910016 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011528986960882649, + "loss": 1.2902, + "theoretical_loss": 3.364246094349517, + "tokens_seen": 2829975552 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011527382146439318, + "loss": 1.2276, + "theoretical_loss": 3.3642400843506097, + "tokens_seen": 2830041088 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011525777331995988, + "loss": 1.174, + "theoretical_loss": 3.3642340745298434, + "tokens_seen": 2830106624 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011524172517552658, + "loss": 1.1979, + "theoretical_loss": 3.364228064887209, + "tokens_seen": 2830172160 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011522567703109328, + "loss": 1.2986, + "theoretical_loss": 3.3642220554226974, + "tokens_seen": 2830237696 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011520962888665998, + "loss": 1.2347, + "theoretical_loss": 3.364216046136299, + "tokens_seen": 2830303232 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011519358074222668, + "loss": 1.1541, + "theoretical_loss": 3.3642100370280046, + "tokens_seen": 2830368768 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011517753259779338, + "loss": 1.296, + "theoretical_loss": 3.364204028097805, + "tokens_seen": 2830434304 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011516148445336008, + "loss": 1.1933, + "theoretical_loss": 3.36419801934569, + "tokens_seen": 2830499840 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011514543630892678, + "loss": 1.2369, + "theoretical_loss": 3.364192010771651, + "tokens_seen": 2830565376 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011512938816449348, + "loss": 1.1152, + "theoretical_loss": 3.364186002375678, + "tokens_seen": 2830630912 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011511334002006018, + "loss": 1.3077, + "theoretical_loss": 3.3641799941577624, + "tokens_seen": 2830696448 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011509729187562688, + "loss": 1.1251, + "theoretical_loss": 3.3641739861178936, + "tokens_seen": 2830761984 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011508124373119357, + "loss": 1.2611, + "theoretical_loss": 3.3641679782560634, + "tokens_seen": 2830827520 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011506519558676027, + "loss": 1.2967, + "theoretical_loss": 3.364161970572262, + "tokens_seen": 2830893056 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011504914744232697, + "loss": 1.1578, + "theoretical_loss": 3.3641559630664792, + "tokens_seen": 2830958592 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011503309929789367, + "loss": 1.2061, + "theoretical_loss": 3.3641499557387067, + "tokens_seen": 2831024128 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011501705115346037, + "loss": 1.1956, + "theoretical_loss": 3.364143948588935, + "tokens_seen": 2831089664 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 945384, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3815860748291016, + "objective/train/theoretical_loss": 3.364137941617154, + "objective/train/tokens_used": 2851615200, + "theoretical_loss": 3.364137941617154, + "tokens_seen": 2831155200 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001150010030090271, + "loss": 1.4128, + "theoretical_loss": 3.364137941617154, + "tokens_seen": 2831155200 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001149849548645938, + "loss": 1.2622, + "theoretical_loss": 3.3641319348233547, + "tokens_seen": 2831220736 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001149689067201605, + "loss": 1.2697, + "theoretical_loss": 3.3641259282075278, + "tokens_seen": 2831286272 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011495285857572719, + "loss": 1.1535, + "theoretical_loss": 3.364119921769664, + "tokens_seen": 2831351808 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011493681043129389, + "loss": 1.325, + "theoretical_loss": 3.3641139155097535, + "tokens_seen": 2831417344 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011492076228686059, + "loss": 1.2315, + "theoretical_loss": 3.364107909427787, + "tokens_seen": 2831482880 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011490471414242729, + "loss": 1.2116, + "theoretical_loss": 3.3641019035237556, + "tokens_seen": 2831548416 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011488866599799399, + "loss": 1.2031, + "theoretical_loss": 3.3640958977976494, + "tokens_seen": 2831613952 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011487261785356069, + "loss": 1.1853, + "theoretical_loss": 3.364089892249459, + "tokens_seen": 2831679488 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001148565697091274, + "loss": 1.2135, + "theoretical_loss": 3.364083886879176, + "tokens_seen": 2831745024 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001148405215646941, + "loss": 1.184, + "theoretical_loss": 3.3640778816867893, + "tokens_seen": 2831810560 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001148244734202608, + "loss": 1.1457, + "theoretical_loss": 3.364071876672291, + "tokens_seen": 2831876096 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001148084252758275, + "loss": 1.1592, + "theoretical_loss": 3.364065871835671, + "tokens_seen": 2831941632 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001147923771313942, + "loss": 1.1393, + "theoretical_loss": 3.3640598671769197, + "tokens_seen": 2832007168 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001147763289869609, + "loss": 1.2891, + "theoretical_loss": 3.364053862696028, + "tokens_seen": 2832072704 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011476028084252758, + "loss": 1.2281, + "theoretical_loss": 3.364047858392987, + "tokens_seen": 2832138240 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011474423269809428, + "loss": 1.1438, + "theoretical_loss": 3.3640418542677866, + "tokens_seen": 2832203776 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011472818455366099, + "loss": 1.2122, + "theoretical_loss": 3.364035850320418, + "tokens_seen": 2832269312 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011471213640922769, + "loss": 1.1835, + "theoretical_loss": 3.3640298465508716, + "tokens_seen": 2832334848 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011469608826479439, + "loss": 1.2983, + "theoretical_loss": 3.3640238429591376, + "tokens_seen": 2832400384 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011468004012036109, + "loss": 1.2127, + "theoretical_loss": 3.364017839545207, + "tokens_seen": 2832465920 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011466399197592779, + "loss": 1.2272, + "theoretical_loss": 3.3640118363090705, + "tokens_seen": 2832531456 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011464794383149449, + "loss": 1.2441, + "theoretical_loss": 3.364005833250719, + "tokens_seen": 2832596992 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011463189568706119, + "loss": 1.3533, + "theoretical_loss": 3.3639998303701417, + "tokens_seen": 2832662528 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011461584754262789, + "loss": 1.1247, + "theoretical_loss": 3.363993827667331, + "tokens_seen": 2832728064 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011459979939819459, + "loss": 1.302, + "theoretical_loss": 3.3639878251422766, + "tokens_seen": 2832793600 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011458375125376129, + "loss": 1.1623, + "theoretical_loss": 3.3639818227949694, + "tokens_seen": 2832859136 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011456770310932798, + "loss": 1.325, + "theoretical_loss": 3.3639758206253996, + "tokens_seen": 2832924672 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011455165496489468, + "loss": 1.3475, + "theoretical_loss": 3.3639698186335583, + "tokens_seen": 2832990208 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011453560682046138, + "loss": 1.39, + "theoretical_loss": 3.363963816819436, + "tokens_seen": 2833055744 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011451955867602808, + "loss": 1.2899, + "theoretical_loss": 3.3639578151830234, + "tokens_seen": 2833121280 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011450351053159478, + "loss": 1.1182, + "theoretical_loss": 3.363951813724311, + "tokens_seen": 2833186816 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011448746238716148, + "loss": 1.237, + "theoretical_loss": 3.3639458124432897, + "tokens_seen": 2833252352 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011447141424272818, + "loss": 1.2392, + "theoretical_loss": 3.3639398113399492, + "tokens_seen": 2833317888 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011445536609829488, + "loss": 1.2179, + "theoretical_loss": 3.3639338104142817, + "tokens_seen": 2833383424 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011443931795386158, + "loss": 1.1949, + "theoretical_loss": 3.363927809666276, + "tokens_seen": 2833448960 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011442326980942828, + "loss": 1.2797, + "theoretical_loss": 3.3639218090959244, + "tokens_seen": 2833514496 + }, + { + "epoch": 0.86, + "learning_rate": 0.000114407221664995, + "loss": 1.1263, + "theoretical_loss": 3.363915808703217, + "tokens_seen": 2833580032 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001143911735205617, + "loss": 1.2222, + "theoretical_loss": 3.3639098084881436, + "tokens_seen": 2833645568 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001143751253761284, + "loss": 1.2188, + "theoretical_loss": 3.3639038084506954, + "tokens_seen": 2833711104 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001143590772316951, + "loss": 1.2189, + "theoretical_loss": 3.3638978085908637, + "tokens_seen": 2833776640 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001143430290872618, + "loss": 1.1908, + "theoretical_loss": 3.363891808908638, + "tokens_seen": 2833842176 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001143269809428285, + "loss": 1.1734, + "theoretical_loss": 3.36388580940401, + "tokens_seen": 2833907712 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001143109327983952, + "loss": 1.1505, + "theoretical_loss": 3.3638798100769693, + "tokens_seen": 2833973248 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001142948846539619, + "loss": 1.2328, + "theoretical_loss": 3.3638738109275073, + "tokens_seen": 2834038784 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001142788365095286, + "loss": 1.3212, + "theoretical_loss": 3.3638678119556147, + "tokens_seen": 2834104320 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011426278836509529, + "loss": 1.2807, + "theoretical_loss": 3.3638618131612814, + "tokens_seen": 2834169856 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011424674022066199, + "loss": 1.0879, + "theoretical_loss": 3.3638558145444986, + "tokens_seen": 2834235392 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011423069207622869, + "loss": 1.3351, + "theoretical_loss": 3.3638498161052564, + "tokens_seen": 2834300928 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011421464393179539, + "loss": 1.2371, + "theoretical_loss": 3.3638438178435464, + "tokens_seen": 2834366464 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 946280, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2681708335876465, + "objective/train/theoretical_loss": 3.3638378197593584, + "objective/train/tokens_used": 2854892000, + "theoretical_loss": 3.3638378197593584, + "tokens_seen": 2834432000 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011419859578736209, + "loss": 1.1154, + "theoretical_loss": 3.3638378197593584, + "tokens_seen": 2834432000 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001141825476429288, + "loss": 1.2201, + "theoretical_loss": 3.3638318218526835, + "tokens_seen": 2834497536 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001141664994984955, + "loss": 1.2239, + "theoretical_loss": 3.363825824123512, + "tokens_seen": 2834563072 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001141504513540622, + "loss": 1.2647, + "theoretical_loss": 3.363819826571835, + "tokens_seen": 2834628608 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001141344032096289, + "loss": 1.2845, + "theoretical_loss": 3.3638138291976425, + "tokens_seen": 2834694144 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001141183550651956, + "loss": 1.1121, + "theoretical_loss": 3.363807832000926, + "tokens_seen": 2834759680 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001141023069207623, + "loss": 1.1918, + "theoretical_loss": 3.3638018349816754, + "tokens_seen": 2834825216 + }, + { + "epoch": 0.86, + "learning_rate": 0.000114086258776329, + "loss": 1.2767, + "theoretical_loss": 3.363795838139881, + "tokens_seen": 2834890752 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011407021063189568, + "loss": 1.1714, + "theoretical_loss": 3.3637898414755347, + "tokens_seen": 2834956288 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011405416248746238, + "loss": 1.1539, + "theoretical_loss": 3.3637838449886264, + "tokens_seen": 2835021824 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011403811434302909, + "loss": 1.1646, + "theoretical_loss": 3.3637778486791468, + "tokens_seen": 2835087360 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011402206619859579, + "loss": 1.3264, + "theoretical_loss": 3.3637718525470865, + "tokens_seen": 2835152896 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011400601805416249, + "loss": 1.3448, + "theoretical_loss": 3.3637658565924364, + "tokens_seen": 2835218432 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011398996990972919, + "loss": 1.3098, + "theoretical_loss": 3.363759860815187, + "tokens_seen": 2835283968 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011397392176529589, + "loss": 1.1783, + "theoretical_loss": 3.3637538652153287, + "tokens_seen": 2835349504 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011395787362086259, + "loss": 1.1315, + "theoretical_loss": 3.3637478697928525, + "tokens_seen": 2835415040 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011394182547642929, + "loss": 1.1573, + "theoretical_loss": 3.363741874547749, + "tokens_seen": 2835480576 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011392577733199599, + "loss": 1.2614, + "theoretical_loss": 3.363735879480009, + "tokens_seen": 2835546112 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011390972918756269, + "loss": 1.175, + "theoretical_loss": 3.363729884589622, + "tokens_seen": 2835611648 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011389368104312939, + "loss": 1.2286, + "theoretical_loss": 3.3637238898765807, + "tokens_seen": 2835677184 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011387763289869608, + "loss": 1.2793, + "theoretical_loss": 3.363717895340874, + "tokens_seen": 2835742720 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011386158475426278, + "loss": 1.1028, + "theoretical_loss": 3.363711900982494, + "tokens_seen": 2835808256 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011384553660982948, + "loss": 1.2135, + "theoretical_loss": 3.36370590680143, + "tokens_seen": 2835873792 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011382948846539618, + "loss": 1.2457, + "theoretical_loss": 3.363699912797673, + "tokens_seen": 2835939328 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011381344032096291, + "loss": 1.1666, + "theoretical_loss": 3.3636939189712143, + "tokens_seen": 2836004864 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011379739217652961, + "loss": 1.1562, + "theoretical_loss": 3.3636879253220435, + "tokens_seen": 2836070400 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011378134403209631, + "loss": 1.2147, + "theoretical_loss": 3.3636819318501527, + "tokens_seen": 2836135936 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011376529588766301, + "loss": 1.2053, + "theoretical_loss": 3.3636759385555317, + "tokens_seen": 2836201472 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001137492477432297, + "loss": 1.253, + "theoretical_loss": 3.3636699454381707, + "tokens_seen": 2836267008 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001137331995987964, + "loss": 1.2219, + "theoretical_loss": 3.363663952498061, + "tokens_seen": 2836332544 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001137171514543631, + "loss": 1.2242, + "theoretical_loss": 3.3636579597351934, + "tokens_seen": 2836398080 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001137011033099298, + "loss": 1.2528, + "theoretical_loss": 3.363651967149558, + "tokens_seen": 2836463616 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001136850551654965, + "loss": 1.1369, + "theoretical_loss": 3.3636459747411465, + "tokens_seen": 2836529152 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001136690070210632, + "loss": 1.2149, + "theoretical_loss": 3.363639982509948, + "tokens_seen": 2836594688 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001136529588766299, + "loss": 1.2158, + "theoretical_loss": 3.3636339904559542, + "tokens_seen": 2836660224 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001136369107321966, + "loss": 1.2803, + "theoretical_loss": 3.363627998579156, + "tokens_seen": 2836725760 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001136208625877633, + "loss": 1.1337, + "theoretical_loss": 3.363622006879543, + "tokens_seen": 2836791296 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011360481444333, + "loss": 1.2159, + "theoretical_loss": 3.363616015357107, + "tokens_seen": 2836856832 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001135887662988967, + "loss": 1.2368, + "theoretical_loss": 3.3636100240118383, + "tokens_seen": 2836922368 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001135727181544634, + "loss": 1.1453, + "theoretical_loss": 3.363604032843727, + "tokens_seen": 2836987904 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011355667001003009, + "loss": 1.2525, + "theoretical_loss": 3.3635980418527645, + "tokens_seen": 2837053440 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011354062186559679, + "loss": 1.1672, + "theoretical_loss": 3.3635920510389408, + "tokens_seen": 2837118976 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011352457372116349, + "loss": 1.2712, + "theoretical_loss": 3.3635860604022474, + "tokens_seen": 2837184512 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011350852557673019, + "loss": 1.1624, + "theoretical_loss": 3.363580069942674, + "tokens_seen": 2837250048 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001134924774322969, + "loss": 1.2733, + "theoretical_loss": 3.363574079660212, + "tokens_seen": 2837315584 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001134764292878636, + "loss": 1.2684, + "theoretical_loss": 3.3635680895548523, + "tokens_seen": 2837381120 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001134603811434303, + "loss": 1.0994, + "theoretical_loss": 3.3635620996265847, + "tokens_seen": 2837446656 + }, + { + "epoch": 0.86, + "learning_rate": 0.000113444332998997, + "loss": 1.2313, + "theoretical_loss": 3.3635561098754003, + "tokens_seen": 2837512192 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001134282848545637, + "loss": 1.1804, + "theoretical_loss": 3.36355012030129, + "tokens_seen": 2837577728 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001134122367101304, + "loss": 1.3148, + "theoretical_loss": 3.363544130904244, + "tokens_seen": 2837643264 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 947431, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2450013160705566, + "objective/train/theoretical_loss": 3.363538141684254, + "objective/train/tokens_used": 2858168800, + "theoretical_loss": 3.363538141684254, + "tokens_seen": 2837708800 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001133961885656971, + "loss": 1.2529, + "theoretical_loss": 3.363538141684254, + "tokens_seen": 2837708800 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001133801404212638, + "loss": 1.2631, + "theoretical_loss": 3.3635321526413087, + "tokens_seen": 2837774336 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011336409227683049, + "loss": 1.1456, + "theoretical_loss": 3.3635261637754006, + "tokens_seen": 2837839872 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011334804413239719, + "loss": 1.1687, + "theoretical_loss": 3.36352017508652, + "tokens_seen": 2837905408 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011333199598796389, + "loss": 1.1096, + "theoretical_loss": 3.3635141865746574, + "tokens_seen": 2837970944 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011331594784353059, + "loss": 1.2365, + "theoretical_loss": 3.3635081982398027, + "tokens_seen": 2838036480 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011329989969909729, + "loss": 1.2329, + "theoretical_loss": 3.363502210081948, + "tokens_seen": 2838102016 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011328385155466399, + "loss": 1.294, + "theoretical_loss": 3.3634962221010833, + "tokens_seen": 2838167552 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011326780341023069, + "loss": 1.1863, + "theoretical_loss": 3.363490234297199, + "tokens_seen": 2838233088 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011325175526579739, + "loss": 1.2443, + "theoretical_loss": 3.363484246670286, + "tokens_seen": 2838298624 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011323570712136409, + "loss": 1.2156, + "theoretical_loss": 3.363478259220335, + "tokens_seen": 2838364160 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011321965897693079, + "loss": 1.1072, + "theoretical_loss": 3.3634722719473373, + "tokens_seen": 2838429696 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001132036108324975, + "loss": 1.1554, + "theoretical_loss": 3.363466284851283, + "tokens_seen": 2838495232 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001131875626880642, + "loss": 1.24, + "theoretical_loss": 3.363460297932162, + "tokens_seen": 2838560768 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001131715145436309, + "loss": 1.1252, + "theoretical_loss": 3.363454311189966, + "tokens_seen": 2838626304 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011315546639919761, + "loss": 1.1285, + "theoretical_loss": 3.363448324624686, + "tokens_seen": 2838691840 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011313941825476431, + "loss": 1.1099, + "theoretical_loss": 3.363442338236312, + "tokens_seen": 2838757376 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011312337011033101, + "loss": 1.3177, + "theoretical_loss": 3.3634363520248343, + "tokens_seen": 2838822912 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011310732196589771, + "loss": 1.1703, + "theoretical_loss": 3.3634303659902445, + "tokens_seen": 2838888448 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011309127382146441, + "loss": 1.2472, + "theoretical_loss": 3.363424380132533, + "tokens_seen": 2838953984 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011307522567703111, + "loss": 1.2531, + "theoretical_loss": 3.3634183944516907, + "tokens_seen": 2839019520 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011305917753259781, + "loss": 1.2639, + "theoretical_loss": 3.3634124089477075, + "tokens_seen": 2839085056 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001130431293881645, + "loss": 1.2918, + "theoretical_loss": 3.363406423620575, + "tokens_seen": 2839150592 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001130270812437312, + "loss": 1.2697, + "theoretical_loss": 3.363400438470283, + "tokens_seen": 2839216128 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001130110330992979, + "loss": 1.2765, + "theoretical_loss": 3.363394453496823, + "tokens_seen": 2839281664 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001129949849548646, + "loss": 1.1928, + "theoretical_loss": 3.3633884687001854, + "tokens_seen": 2839347200 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001129789368104313, + "loss": 1.1009, + "theoretical_loss": 3.363382484080361, + "tokens_seen": 2839412736 + }, + { + "epoch": 0.86, + "learning_rate": 0.000112962888665998, + "loss": 1.2627, + "theoretical_loss": 3.36337649963734, + "tokens_seen": 2839478272 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001129468405215647, + "loss": 1.2915, + "theoretical_loss": 3.3633705153711135, + "tokens_seen": 2839543808 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001129307923771314, + "loss": 1.1921, + "theoretical_loss": 3.3633645312816727, + "tokens_seen": 2839609344 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001129147442326981, + "loss": 1.2176, + "theoretical_loss": 3.3633585473690073, + "tokens_seen": 2839674880 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001128986960882648, + "loss": 1.182, + "theoretical_loss": 3.3633525636331085, + "tokens_seen": 2839740416 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001128826479438315, + "loss": 1.1418, + "theoretical_loss": 3.363346580073967, + "tokens_seen": 2839805952 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001128665997993982, + "loss": 1.2227, + "theoretical_loss": 3.3633405966915735, + "tokens_seen": 2839871488 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011285055165496489, + "loss": 1.1269, + "theoretical_loss": 3.3633346134859186, + "tokens_seen": 2839937024 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011283450351053159, + "loss": 1.2001, + "theoretical_loss": 3.3633286304569934, + "tokens_seen": 2840002560 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011281845536609829, + "loss": 1.1559, + "theoretical_loss": 3.3633226476047877, + "tokens_seen": 2840068096 + }, + { + "epoch": 0.86, + "learning_rate": 0.000112802407221665, + "loss": 1.1834, + "theoretical_loss": 3.363316664929293, + "tokens_seen": 2840133632 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001127863590772317, + "loss": 1.3363, + "theoretical_loss": 3.3633106824305, + "tokens_seen": 2840199168 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001127703109327984, + "loss": 1.1392, + "theoretical_loss": 3.363304700108399, + "tokens_seen": 2840264704 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001127542627883651, + "loss": 1.2123, + "theoretical_loss": 3.363298717962981, + "tokens_seen": 2840330240 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001127382146439318, + "loss": 1.1259, + "theoretical_loss": 3.363292735994236, + "tokens_seen": 2840395776 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001127221664994985, + "loss": 1.2158, + "theoretical_loss": 3.363286754202156, + "tokens_seen": 2840461312 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001127061183550652, + "loss": 1.1247, + "theoretical_loss": 3.3632807725867306, + "tokens_seen": 2840526848 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001126900702106319, + "loss": 1.3665, + "theoretical_loss": 3.3632747911479512, + "tokens_seen": 2840592384 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001126740220661986, + "loss": 1.131, + "theoretical_loss": 3.363268809885808, + "tokens_seen": 2840657920 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011265797392176529, + "loss": 1.1926, + "theoretical_loss": 3.3632628288002917, + "tokens_seen": 2840723456 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011264192577733199, + "loss": 1.15, + "theoretical_loss": 3.3632568478913933, + "tokens_seen": 2840788992 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011262587763289869, + "loss": 1.2549, + "theoretical_loss": 3.363250867159104, + "tokens_seen": 2840854528 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011260982948846541, + "loss": 1.1999, + "theoretical_loss": 3.3632448866034137, + "tokens_seen": 2840920064 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 948338, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1716320514678955, + "objective/train/theoretical_loss": 3.363238906224313, + "objective/train/tokens_used": 2861445600, + "theoretical_loss": 3.363238906224313, + "tokens_seen": 2840985600 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011259378134403212, + "loss": 1.1862, + "theoretical_loss": 3.363238906224313, + "tokens_seen": 2840985600 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011257773319959882, + "loss": 1.0916, + "theoretical_loss": 3.363232926021793, + "tokens_seen": 2841051136 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011256168505516552, + "loss": 1.1357, + "theoretical_loss": 3.363226945995845, + "tokens_seen": 2841116672 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001125456369107322, + "loss": 1.1739, + "theoretical_loss": 3.3632209661464585, + "tokens_seen": 2841182208 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001125295887662989, + "loss": 1.2913, + "theoretical_loss": 3.363214986473625, + "tokens_seen": 2841247744 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001125135406218656, + "loss": 1.2061, + "theoretical_loss": 3.363209006977335, + "tokens_seen": 2841313280 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001124974924774323, + "loss": 1.3355, + "theoretical_loss": 3.3632030276575793, + "tokens_seen": 2841378816 + }, + { + "epoch": 0.86, + "learning_rate": 0.000112481444332999, + "loss": 1.2226, + "theoretical_loss": 3.3631970485143485, + "tokens_seen": 2841444352 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011246539618856571, + "loss": 1.2313, + "theoretical_loss": 3.3631910695476335, + "tokens_seen": 2841509888 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011244934804413241, + "loss": 1.1943, + "theoretical_loss": 3.3631850907574248, + "tokens_seen": 2841575424 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011243329989969911, + "loss": 1.1516, + "theoretical_loss": 3.363179112143713, + "tokens_seen": 2841640960 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011241725175526581, + "loss": 1.1707, + "theoretical_loss": 3.3631731337064896, + "tokens_seen": 2841706496 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011240120361083251, + "loss": 1.2441, + "theoretical_loss": 3.3631671554457445, + "tokens_seen": 2841772032 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011238515546639921, + "loss": 1.2358, + "theoretical_loss": 3.3631611773614685, + "tokens_seen": 2841837568 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011236910732196591, + "loss": 1.2143, + "theoretical_loss": 3.3631551994536526, + "tokens_seen": 2841903104 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001123530591775326, + "loss": 1.2642, + "theoretical_loss": 3.3631492217222876, + "tokens_seen": 2841968640 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001123370110330993, + "loss": 1.2758, + "theoretical_loss": 3.3631432441673637, + "tokens_seen": 2842034176 + }, + { + "epoch": 0.86, + "learning_rate": 0.000112320962888666, + "loss": 1.2067, + "theoretical_loss": 3.363137266788872, + "tokens_seen": 2842099712 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001123049147442327, + "loss": 1.1831, + "theoretical_loss": 3.3631312895868035, + "tokens_seen": 2842165248 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001122888665997994, + "loss": 1.2661, + "theoretical_loss": 3.3631253125611487, + "tokens_seen": 2842230784 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001122728184553661, + "loss": 1.2244, + "theoretical_loss": 3.3631193357118976, + "tokens_seen": 2842296320 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001122567703109328, + "loss": 1.1793, + "theoretical_loss": 3.3631133590390423, + "tokens_seen": 2842361856 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001122407221664995, + "loss": 1.3616, + "theoretical_loss": 3.363107382542572, + "tokens_seen": 2842427392 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001122246740220662, + "loss": 1.2943, + "theoretical_loss": 3.363101406222479, + "tokens_seen": 2842492928 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001122086258776329, + "loss": 1.1433, + "theoretical_loss": 3.363095430078753, + "tokens_seen": 2842558464 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001121925777331996, + "loss": 1.2497, + "theoretical_loss": 3.363089454111385, + "tokens_seen": 2842624000 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001121765295887663, + "loss": 1.2242, + "theoretical_loss": 3.3630834783203656, + "tokens_seen": 2842689536 + }, + { + "epoch": 0.86, + "learning_rate": 0.000112160481444333, + "loss": 1.2123, + "theoretical_loss": 3.3630775027056856, + "tokens_seen": 2842755072 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011214443329989969, + "loss": 1.16, + "theoretical_loss": 3.363071527267336, + "tokens_seen": 2842820608 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001121283851554664, + "loss": 1.191, + "theoretical_loss": 3.363065552005307, + "tokens_seen": 2842886144 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001121123370110331, + "loss": 1.3131, + "theoretical_loss": 3.36305957691959, + "tokens_seen": 2842951680 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001120962888665998, + "loss": 1.2122, + "theoretical_loss": 3.363053602010175, + "tokens_seen": 2843017216 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001120802407221665, + "loss": 1.2222, + "theoretical_loss": 3.3630476272770533, + "tokens_seen": 2843082752 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001120641925777332, + "loss": 1.262, + "theoretical_loss": 3.3630416527202156, + "tokens_seen": 2843148288 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001120481444332999, + "loss": 1.2033, + "theoretical_loss": 3.363035678339652, + "tokens_seen": 2843213824 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001120320962888666, + "loss": 1.0719, + "theoretical_loss": 3.363029704135354, + "tokens_seen": 2843279360 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001120160481444333, + "loss": 1.1874, + "theoretical_loss": 3.363023730107312, + "tokens_seen": 2843344896 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011200000000000001, + "loss": 1.3588, + "theoretical_loss": 3.363017756255517, + "tokens_seen": 2843410432 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011198395185556671, + "loss": 1.0793, + "theoretical_loss": 3.3630117825799593, + "tokens_seen": 2843475968 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011196790371113341, + "loss": 1.1338, + "theoretical_loss": 3.36300580908063, + "tokens_seen": 2843541504 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011195185556670011, + "loss": 1.19, + "theoretical_loss": 3.3629998357575195, + "tokens_seen": 2843607040 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011193580742226681, + "loss": 1.2192, + "theoretical_loss": 3.362993862610619, + "tokens_seen": 2843672576 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011191975927783352, + "loss": 1.226, + "theoretical_loss": 3.362987889639919, + "tokens_seen": 2843738112 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011190371113340022, + "loss": 1.231, + "theoretical_loss": 3.36298191684541, + "tokens_seen": 2843803648 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011188766298896692, + "loss": 1.2579, + "theoretical_loss": 3.362975944227083, + "tokens_seen": 2843869184 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011187161484453362, + "loss": 1.3352, + "theoretical_loss": 3.3629699717849286, + "tokens_seen": 2843934720 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011185556670010032, + "loss": 1.2378, + "theoretical_loss": 3.362963999518938, + "tokens_seen": 2844000256 + }, + { + "epoch": 0.86, + "learning_rate": 0.000111839518555667, + "loss": 1.2603, + "theoretical_loss": 3.3629580274291015, + "tokens_seen": 2844065792 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001118234704112337, + "loss": 1.1868, + "theoretical_loss": 3.36295205551541, + "tokens_seen": 2844131328 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001118074222668004, + "loss": 1.1595, + "theoretical_loss": 3.362946083777854, + "tokens_seen": 2844196864 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 949535, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2085310220718384, + "objective/train/theoretical_loss": 3.3629401122164246, + "objective/train/tokens_used": 2864722400, + "theoretical_loss": 3.3629401122164246, + "tokens_seen": 2844262400 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001117913741223671, + "loss": 1.1655, + "theoretical_loss": 3.3629401122164246, + "tokens_seen": 2844262400 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011177532597793381, + "loss": 1.1428, + "theoretical_loss": 3.362934140831112, + "tokens_seen": 2844327936 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011175927783350051, + "loss": 1.1165, + "theoretical_loss": 3.3629281696219078, + "tokens_seen": 2844393472 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011174322968906721, + "loss": 1.2209, + "theoretical_loss": 3.362922198588802, + "tokens_seen": 2844459008 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011172718154463391, + "loss": 1.1598, + "theoretical_loss": 3.3629162277317857, + "tokens_seen": 2844524544 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011171113340020061, + "loss": 1.2121, + "theoretical_loss": 3.36291025705085, + "tokens_seen": 2844590080 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011169508525576731, + "loss": 1.2265, + "theoretical_loss": 3.3629042865459846, + "tokens_seen": 2844655616 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011167903711133401, + "loss": 1.2526, + "theoretical_loss": 3.3628983162171813, + "tokens_seen": 2844721152 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011166298896690071, + "loss": 1.2426, + "theoretical_loss": 3.36289234606443, + "tokens_seen": 2844786688 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001116469408224674, + "loss": 1.1317, + "theoretical_loss": 3.3628863760877223, + "tokens_seen": 2844852224 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001116308926780341, + "loss": 1.2767, + "theoretical_loss": 3.3628804062870485, + "tokens_seen": 2844917760 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001116148445336008, + "loss": 1.2928, + "theoretical_loss": 3.3628744366623993, + "tokens_seen": 2844983296 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001115987963891675, + "loss": 1.2485, + "theoretical_loss": 3.3628684672137656, + "tokens_seen": 2845048832 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001115827482447342, + "loss": 1.22, + "theoretical_loss": 3.362862497941138, + "tokens_seen": 2845114368 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001115667001003009, + "loss": 1.2072, + "theoretical_loss": 3.3628565288445076, + "tokens_seen": 2845179904 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001115506519558676, + "loss": 1.174, + "theoretical_loss": 3.3628505599238645, + "tokens_seen": 2845245440 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001115346038114343, + "loss": 1.1935, + "theoretical_loss": 3.3628445911792006, + "tokens_seen": 2845310976 + }, + { + "epoch": 0.86, + "learning_rate": 0.000111518555667001, + "loss": 1.2369, + "theoretical_loss": 3.3628386226105054, + "tokens_seen": 2845376512 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001115025075225677, + "loss": 1.1443, + "theoretical_loss": 3.3628326542177702, + "tokens_seen": 2845442048 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001114864593781344, + "loss": 1.2526, + "theoretical_loss": 3.362826686000986, + "tokens_seen": 2845507584 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001114704112337011, + "loss": 1.1714, + "theoretical_loss": 3.362820717960143, + "tokens_seen": 2845573120 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011145436308926779, + "loss": 1.1515, + "theoretical_loss": 3.3628147500952323, + "tokens_seen": 2845638656 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001114383149448345, + "loss": 1.2063, + "theoretical_loss": 3.362808782406245, + "tokens_seen": 2845704192 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001114222668004012, + "loss": 1.3416, + "theoretical_loss": 3.3628028148931715, + "tokens_seen": 2845769728 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011140621865596792, + "loss": 1.2247, + "theoretical_loss": 3.362796847556002, + "tokens_seen": 2845835264 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011139017051153462, + "loss": 1.2075, + "theoretical_loss": 3.3627908803947286, + "tokens_seen": 2845900800 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011137412236710132, + "loss": 1.2251, + "theoretical_loss": 3.3627849134093406, + "tokens_seen": 2845966336 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011135807422266802, + "loss": 1.132, + "theoretical_loss": 3.36277894659983, + "tokens_seen": 2846031872 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011134202607823472, + "loss": 1.1873, + "theoretical_loss": 3.362772979966187, + "tokens_seen": 2846097408 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011132597793380141, + "loss": 1.1815, + "theoretical_loss": 3.3627670135084022, + "tokens_seen": 2846162944 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011130992978936811, + "loss": 1.2274, + "theoretical_loss": 3.362761047226466, + "tokens_seen": 2846228480 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011129388164493481, + "loss": 1.1375, + "theoretical_loss": 3.362755081120371, + "tokens_seen": 2846294016 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011127783350050151, + "loss": 1.2266, + "theoretical_loss": 3.362749115190106, + "tokens_seen": 2846359552 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011126178535606821, + "loss": 1.2918, + "theoretical_loss": 3.362743149435662, + "tokens_seen": 2846425088 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011124573721163491, + "loss": 1.2898, + "theoretical_loss": 3.362737183857031, + "tokens_seen": 2846490624 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011122968906720162, + "loss": 1.252, + "theoretical_loss": 3.362731218454203, + "tokens_seen": 2846556160 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011121364092276832, + "loss": 1.1722, + "theoretical_loss": 3.3627252532271683, + "tokens_seen": 2846621696 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011119759277833502, + "loss": 1.1554, + "theoretical_loss": 3.3627192881759185, + "tokens_seen": 2846687232 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011118154463390172, + "loss": 1.2001, + "theoretical_loss": 3.3627133233004436, + "tokens_seen": 2846752768 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011116549648946842, + "loss": 1.1255, + "theoretical_loss": 3.362707358600735, + "tokens_seen": 2846818304 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011114944834503512, + "loss": 1.2074, + "theoretical_loss": 3.362701394076784, + "tokens_seen": 2846883840 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001111334002006018, + "loss": 1.2179, + "theoretical_loss": 3.36269542972858, + "tokens_seen": 2846949376 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001111173520561685, + "loss": 1.1911, + "theoretical_loss": 3.362689465556114, + "tokens_seen": 2847014912 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001111013039117352, + "loss": 1.2527, + "theoretical_loss": 3.362683501559378, + "tokens_seen": 2847080448 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011108525576730191, + "loss": 1.1669, + "theoretical_loss": 3.362677537738362, + "tokens_seen": 2847145984 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011106920762286861, + "loss": 1.3615, + "theoretical_loss": 3.3626715740930564, + "tokens_seen": 2847211520 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011105315947843531, + "loss": 1.1462, + "theoretical_loss": 3.362665610623452, + "tokens_seen": 2847277056 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011103711133400201, + "loss": 1.3122, + "theoretical_loss": 3.3626596473295405, + "tokens_seen": 2847342592 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011102106318956871, + "loss": 1.1521, + "theoretical_loss": 3.362653684211312, + "tokens_seen": 2847408128 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011100501504513541, + "loss": 1.208, + "theoretical_loss": 3.3626477212687576, + "tokens_seen": 2847473664 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 950465, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.247605323791504, + "objective/train/theoretical_loss": 3.3626417585018675, + "objective/train/tokens_used": 2867999200, + "theoretical_loss": 3.3626417585018675, + "tokens_seen": 2847539200 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011098896690070211, + "loss": 1.2359, + "theoretical_loss": 3.3626417585018675, + "tokens_seen": 2847539200 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011097291875626881, + "loss": 1.1346, + "theoretical_loss": 3.3626357959106326, + "tokens_seen": 2847604736 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011095687061183551, + "loss": 1.3134, + "theoretical_loss": 3.3626298334950446, + "tokens_seen": 2847670272 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001109408224674022, + "loss": 1.1568, + "theoretical_loss": 3.362623871255093, + "tokens_seen": 2847735808 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001109247743229689, + "loss": 1.2939, + "theoretical_loss": 3.3626179091907695, + "tokens_seen": 2847801344 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001109087261785356, + "loss": 1.1799, + "theoretical_loss": 3.3626119473020646, + "tokens_seen": 2847866880 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001108926780341023, + "loss": 1.1629, + "theoretical_loss": 3.362605985588969, + "tokens_seen": 2847932416 + }, + { + "epoch": 0.86, + "learning_rate": 0.000110876629889669, + "loss": 1.3356, + "theoretical_loss": 3.3626000240514737, + "tokens_seen": 2847997952 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001108605817452357, + "loss": 1.212, + "theoretical_loss": 3.362594062689569, + "tokens_seen": 2848063488 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001108445336008024, + "loss": 1.2545, + "theoretical_loss": 3.3625881015032464, + "tokens_seen": 2848129024 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001108284854563691, + "loss": 1.34, + "theoretical_loss": 3.362582140492496, + "tokens_seen": 2848194560 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001108124373119358, + "loss": 1.2823, + "theoretical_loss": 3.3625761796573093, + "tokens_seen": 2848260096 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011079638916750252, + "loss": 1.1708, + "theoretical_loss": 3.3625702189976767, + "tokens_seen": 2848325632 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011078034102306922, + "loss": 1.1588, + "theoretical_loss": 3.3625642585135886, + "tokens_seen": 2848391168 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011076429287863592, + "loss": 1.3088, + "theoretical_loss": 3.362558298205036, + "tokens_seen": 2848456704 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011074824473420262, + "loss": 1.185, + "theoretical_loss": 3.3625523380720104, + "tokens_seen": 2848522240 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011073219658976932, + "loss": 1.108, + "theoretical_loss": 3.362546378114502, + "tokens_seen": 2848587776 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011071614844533602, + "loss": 1.297, + "theoretical_loss": 3.3625404183325016, + "tokens_seen": 2848653312 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011070010030090272, + "loss": 1.133, + "theoretical_loss": 3.362534458726, + "tokens_seen": 2848718848 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011068405215646942, + "loss": 1.2025, + "theoretical_loss": 3.3625284992949878, + "tokens_seen": 2848784384 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011066800401203612, + "loss": 1.2353, + "theoretical_loss": 3.3625225400394565, + "tokens_seen": 2848849920 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011065195586760282, + "loss": 1.1352, + "theoretical_loss": 3.3625165809593964, + "tokens_seen": 2848915456 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011063590772316953, + "loss": 1.3802, + "theoretical_loss": 3.3625106220547982, + "tokens_seen": 2848980992 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011061985957873621, + "loss": 1.1928, + "theoretical_loss": 3.3625046633256526, + "tokens_seen": 2849046528 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011060381143430291, + "loss": 1.2753, + "theoretical_loss": 3.362498704771951, + "tokens_seen": 2849112064 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011058776328986961, + "loss": 1.2393, + "theoretical_loss": 3.3624927463936833, + "tokens_seen": 2849177600 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011057171514543631, + "loss": 1.2222, + "theoretical_loss": 3.3624867881908416, + "tokens_seen": 2849243136 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011055566700100301, + "loss": 1.1903, + "theoretical_loss": 3.3624808301634155, + "tokens_seen": 2849308672 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011053961885656972, + "loss": 1.1575, + "theoretical_loss": 3.362474872311396, + "tokens_seen": 2849374208 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011052357071213642, + "loss": 1.1878, + "theoretical_loss": 3.3624689146347744, + "tokens_seen": 2849439744 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011050752256770312, + "loss": 1.2721, + "theoretical_loss": 3.362462957133541, + "tokens_seen": 2849505280 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011049147442326982, + "loss": 1.1613, + "theoretical_loss": 3.3624569998076876, + "tokens_seen": 2849570816 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011047542627883652, + "loss": 1.2568, + "theoretical_loss": 3.3624510426572036, + "tokens_seen": 2849636352 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011045937813440322, + "loss": 1.2899, + "theoretical_loss": 3.3624450856820807, + "tokens_seen": 2849701888 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011044332998996992, + "loss": 1.2019, + "theoretical_loss": 3.362439128882309, + "tokens_seen": 2849767424 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001104272818455366, + "loss": 1.2292, + "theoretical_loss": 3.36243317225788, + "tokens_seen": 2849832960 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011041123370110331, + "loss": 1.2182, + "theoretical_loss": 3.3624272158087845, + "tokens_seen": 2849898496 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011039518555667001, + "loss": 1.1623, + "theoretical_loss": 3.362421259535013, + "tokens_seen": 2849964032 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011037913741223671, + "loss": 1.1332, + "theoretical_loss": 3.3624153034365563, + "tokens_seen": 2850029568 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011036308926780341, + "loss": 1.1731, + "theoretical_loss": 3.362409347513405, + "tokens_seen": 2850095104 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011034704112337011, + "loss": 1.1947, + "theoretical_loss": 3.3624033917655507, + "tokens_seen": 2850160640 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011033099297893681, + "loss": 1.1956, + "theoretical_loss": 3.3623974361929836, + "tokens_seen": 2850226176 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011031494483450351, + "loss": 1.1585, + "theoretical_loss": 3.3623914807956945, + "tokens_seen": 2850291712 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011029889669007021, + "loss": 1.3261, + "theoretical_loss": 3.362385525573674, + "tokens_seen": 2850357248 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011028284854563691, + "loss": 1.2657, + "theoretical_loss": 3.362379570526914, + "tokens_seen": 2850422784 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011026680040120361, + "loss": 1.1978, + "theoretical_loss": 3.362373615655404, + "tokens_seen": 2850488320 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011025075225677031, + "loss": 1.205, + "theoretical_loss": 3.3623676609591353, + "tokens_seen": 2850553856 + }, + { + "epoch": 0.86, + "learning_rate": 0.000110234704112337, + "loss": 1.1943, + "theoretical_loss": 3.362361706438099, + "tokens_seen": 2850619392 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001102186559679037, + "loss": 1.1938, + "theoretical_loss": 3.3623557520922858, + "tokens_seen": 2850684928 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011020260782347043, + "loss": 1.1868, + "theoretical_loss": 3.3623497979216865, + "tokens_seen": 2850750464 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 951737, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1213226318359375, + "objective/train/theoretical_loss": 3.3623438439262916, + "objective/train/tokens_used": 2871276000, + "theoretical_loss": 3.3623438439262916, + "tokens_seen": 2850816000 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011018655967903713, + "loss": 1.1799, + "theoretical_loss": 3.3623438439262916, + "tokens_seen": 2850816000 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011017051153460383, + "loss": 1.1972, + "theoretical_loss": 3.362337890106092, + "tokens_seen": 2850881536 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011015446339017053, + "loss": 1.1317, + "theoretical_loss": 3.362331936461079, + "tokens_seen": 2850947072 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011013841524573723, + "loss": 1.251, + "theoretical_loss": 3.362325982991243, + "tokens_seen": 2851012608 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011012236710130392, + "loss": 1.2316, + "theoretical_loss": 3.362320029696575, + "tokens_seen": 2851078144 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011010631895687062, + "loss": 1.1499, + "theoretical_loss": 3.362314076577065, + "tokens_seen": 2851143680 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011009027081243732, + "loss": 1.2626, + "theoretical_loss": 3.3623081236327055, + "tokens_seen": 2851209216 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011007422266800402, + "loss": 1.1801, + "theoretical_loss": 3.362302170863486, + "tokens_seen": 2851274752 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011005817452357072, + "loss": 1.2166, + "theoretical_loss": 3.362296218269398, + "tokens_seen": 2851340288 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011004212637913742, + "loss": 1.226, + "theoretical_loss": 3.3622902658504312, + "tokens_seen": 2851405824 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011002607823470412, + "loss": 1.1337, + "theoretical_loss": 3.362284313606578, + "tokens_seen": 2851471360 + }, + { + "epoch": 0.86, + "learning_rate": 0.00011001003009027082, + "loss": 1.2055, + "theoretical_loss": 3.362278361537828, + "tokens_seen": 2851536896 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010999398194583752, + "loss": 1.2368, + "theoretical_loss": 3.3622724096441727, + "tokens_seen": 2851602432 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010997793380140422, + "loss": 1.1548, + "theoretical_loss": 3.3622664579256023, + "tokens_seen": 2851667968 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010996188565697092, + "loss": 1.1506, + "theoretical_loss": 3.3622605063821087, + "tokens_seen": 2851733504 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010994583751253763, + "loss": 1.1397, + "theoretical_loss": 3.3622545550136818, + "tokens_seen": 2851799040 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010992978936810431, + "loss": 1.2475, + "theoretical_loss": 3.3622486038203125, + "tokens_seen": 2851864576 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010991374122367101, + "loss": 1.1979, + "theoretical_loss": 3.3622426528019917, + "tokens_seen": 2851930112 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010989769307923771, + "loss": 1.3204, + "theoretical_loss": 3.362236701958711, + "tokens_seen": 2851995648 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010988164493480441, + "loss": 1.2811, + "theoretical_loss": 3.36223075129046, + "tokens_seen": 2852061184 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010986559679037111, + "loss": 1.1692, + "theoretical_loss": 3.36222480079723, + "tokens_seen": 2852126720 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010984954864593782, + "loss": 1.165, + "theoretical_loss": 3.3622188504790125, + "tokens_seen": 2852192256 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010983350050150452, + "loss": 1.1992, + "theoretical_loss": 3.3622129003357974, + "tokens_seen": 2852257792 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010981745235707122, + "loss": 1.2399, + "theoretical_loss": 3.3622069503675758, + "tokens_seen": 2852323328 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010980140421263792, + "loss": 1.137, + "theoretical_loss": 3.3622010005743386, + "tokens_seen": 2852388864 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010978535606820462, + "loss": 1.1429, + "theoretical_loss": 3.362195050956077, + "tokens_seen": 2852454400 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010976930792377132, + "loss": 1.1085, + "theoretical_loss": 3.362189101512781, + "tokens_seen": 2852519936 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010975325977933802, + "loss": 1.2353, + "theoretical_loss": 3.3621831522444428, + "tokens_seen": 2852585472 + }, + { + "epoch": 0.86, + "learning_rate": 0.0001097372116349047, + "loss": 1.1459, + "theoretical_loss": 3.3621772031510515, + "tokens_seen": 2852651008 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010972116349047141, + "loss": 1.2076, + "theoretical_loss": 3.3621712542325994, + "tokens_seen": 2852716544 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010970511534603811, + "loss": 1.1964, + "theoretical_loss": 3.3621653054890763, + "tokens_seen": 2852782080 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010968906720160481, + "loss": 1.2388, + "theoretical_loss": 3.3621593569204737, + "tokens_seen": 2852847616 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010967301905717151, + "loss": 1.2018, + "theoretical_loss": 3.3621534085267823, + "tokens_seen": 2852913152 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010965697091273821, + "loss": 1.2636, + "theoretical_loss": 3.362147460307993, + "tokens_seen": 2852978688 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010964092276830491, + "loss": 1.2309, + "theoretical_loss": 3.362141512264096, + "tokens_seen": 2853044224 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010962487462387161, + "loss": 1.0764, + "theoretical_loss": 3.362135564395083, + "tokens_seen": 2853109760 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010960882647943833, + "loss": 1.1675, + "theoretical_loss": 3.362129616700944, + "tokens_seen": 2853175296 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010959277833500503, + "loss": 1.2419, + "theoretical_loss": 3.362123669181671, + "tokens_seen": 2853240832 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010957673019057173, + "loss": 1.2889, + "theoretical_loss": 3.3621177218372535, + "tokens_seen": 2853306368 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010956068204613843, + "loss": 1.1638, + "theoretical_loss": 3.3621117746676834, + "tokens_seen": 2853371904 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010954463390170513, + "loss": 1.2538, + "theoretical_loss": 3.362105827672951, + "tokens_seen": 2853437440 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010952858575727183, + "loss": 1.253, + "theoretical_loss": 3.3620998808530476, + "tokens_seen": 2853502976 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010951253761283853, + "loss": 1.2517, + "theoretical_loss": 3.362093934207963, + "tokens_seen": 2853568512 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010949648946840523, + "loss": 1.1599, + "theoretical_loss": 3.3620879877376897, + "tokens_seen": 2853634048 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010948044132397193, + "loss": 1.2539, + "theoretical_loss": 3.3620820414422172, + "tokens_seen": 2853699584 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010946439317953863, + "loss": 1.2963, + "theoretical_loss": 3.362076095321537, + "tokens_seen": 2853765120 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010944834503510533, + "loss": 1.1418, + "theoretical_loss": 3.3620701493756395, + "tokens_seen": 2853830656 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010943229689067203, + "loss": 1.128, + "theoretical_loss": 3.3620642036045156, + "tokens_seen": 2853896192 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010941624874623872, + "loss": 1.2343, + "theoretical_loss": 3.362058258008157, + "tokens_seen": 2853961728 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010940020060180542, + "loss": 1.2031, + "theoretical_loss": 3.3620523125865533, + "tokens_seen": 2854027264 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 952559, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2233026027679443, + "objective/train/theoretical_loss": 3.362046367339696, + "objective/train/tokens_used": 2874552800, + "theoretical_loss": 3.362046367339696, + "tokens_seen": 2854092800 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010938415245737212, + "loss": 1.1567, + "theoretical_loss": 3.362046367339696, + "tokens_seen": 2854092800 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010936810431293882, + "loss": 1.2462, + "theoretical_loss": 3.362040422267576, + "tokens_seen": 2854158336 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010935205616850552, + "loss": 1.2518, + "theoretical_loss": 3.362034477370184, + "tokens_seen": 2854223872 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010933600802407222, + "loss": 1.1862, + "theoretical_loss": 3.3620285326475106, + "tokens_seen": 2854289408 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010931995987963892, + "loss": 1.2336, + "theoretical_loss": 3.3620225880995473, + "tokens_seen": 2854354944 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010930391173520562, + "loss": 1.2244, + "theoretical_loss": 3.3620166437262844, + "tokens_seen": 2854420480 + }, + { + "epoch": 0.86, + "learning_rate": 0.00010928786359077232, + "loss": 1.1794, + "theoretical_loss": 3.3620106995277133, + "tokens_seen": 2854486016 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010927181544633902, + "loss": 1.2483, + "theoretical_loss": 3.362004755503824, + "tokens_seen": 2854551552 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010925576730190573, + "loss": 1.1846, + "theoretical_loss": 3.3619988116546082, + "tokens_seen": 2854617088 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010923971915747243, + "loss": 1.2637, + "theoretical_loss": 3.361992867980056, + "tokens_seen": 2854682624 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010922367101303911, + "loss": 1.263, + "theoretical_loss": 3.361986924480159, + "tokens_seen": 2854748160 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010920762286860581, + "loss": 1.185, + "theoretical_loss": 3.361980981154908, + "tokens_seen": 2854813696 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010919157472417251, + "loss": 1.1834, + "theoretical_loss": 3.361975038004293, + "tokens_seen": 2854879232 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010917552657973922, + "loss": 1.1469, + "theoretical_loss": 3.361969095028306, + "tokens_seen": 2854944768 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010915947843530592, + "loss": 1.2319, + "theoretical_loss": 3.361963152226937, + "tokens_seen": 2855010304 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010914343029087262, + "loss": 1.272, + "theoretical_loss": 3.3619572096001775, + "tokens_seen": 2855075840 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010912738214643932, + "loss": 1.224, + "theoretical_loss": 3.361951267148018, + "tokens_seen": 2855141376 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010911133400200602, + "loss": 1.1056, + "theoretical_loss": 3.3619453248704487, + "tokens_seen": 2855206912 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010909528585757272, + "loss": 1.3261, + "theoretical_loss": 3.3619393827674617, + "tokens_seen": 2855272448 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010907923771313942, + "loss": 1.175, + "theoretical_loss": 3.361933440839047, + "tokens_seen": 2855337984 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010906318956870612, + "loss": 1.2139, + "theoretical_loss": 3.3619274990851964, + "tokens_seen": 2855403520 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010904714142427282, + "loss": 1.2383, + "theoretical_loss": 3.3619215575059, + "tokens_seen": 2855469056 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010903109327983951, + "loss": 1.1582, + "theoretical_loss": 3.3619156161011485, + "tokens_seen": 2855534592 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010901504513540621, + "loss": 1.1907, + "theoretical_loss": 3.361909674870933, + "tokens_seen": 2855600128 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010899899699097294, + "loss": 1.25, + "theoretical_loss": 3.361903733815245, + "tokens_seen": 2855665664 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010898294884653964, + "loss": 1.3063, + "theoretical_loss": 3.3618977929340743, + "tokens_seen": 2855731200 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010896690070210634, + "loss": 1.2143, + "theoretical_loss": 3.361891852227412, + "tokens_seen": 2855796736 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010895085255767304, + "loss": 1.2946, + "theoretical_loss": 3.36188591169525, + "tokens_seen": 2855862272 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010893480441323974, + "loss": 1.1477, + "theoretical_loss": 3.3618799713375784, + "tokens_seen": 2855927808 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010891875626880644, + "loss": 1.1459, + "theoretical_loss": 3.3618740311543878, + "tokens_seen": 2855993344 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010890270812437313, + "loss": 1.1384, + "theoretical_loss": 3.3618680911456695, + "tokens_seen": 2856058880 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010888665997993983, + "loss": 1.1389, + "theoretical_loss": 3.361862151311414, + "tokens_seen": 2856124416 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010887061183550653, + "loss": 1.3121, + "theoretical_loss": 3.3618562116516126, + "tokens_seen": 2856189952 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010885456369107323, + "loss": 1.2022, + "theoretical_loss": 3.361850272166256, + "tokens_seen": 2856255488 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010883851554663993, + "loss": 1.2465, + "theoretical_loss": 3.361844332855335, + "tokens_seen": 2856321024 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010882246740220663, + "loss": 1.2301, + "theoretical_loss": 3.3618383937188407, + "tokens_seen": 2856386560 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010880641925777333, + "loss": 1.1512, + "theoretical_loss": 3.3618324547567635, + "tokens_seen": 2856452096 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010879037111334003, + "loss": 1.2173, + "theoretical_loss": 3.3618265159690948, + "tokens_seen": 2856517632 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010877432296890673, + "loss": 1.1861, + "theoretical_loss": 3.361820577355825, + "tokens_seen": 2856583168 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010875827482447343, + "loss": 1.2032, + "theoretical_loss": 3.361814638916946, + "tokens_seen": 2856648704 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010874222668004013, + "loss": 1.1935, + "theoretical_loss": 3.3618087006524475, + "tokens_seen": 2856714240 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010872617853560683, + "loss": 1.1764, + "theoretical_loss": 3.361802762562321, + "tokens_seen": 2856779776 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010871013039117352, + "loss": 1.2768, + "theoretical_loss": 3.361796824646557, + "tokens_seen": 2856845312 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010869408224674022, + "loss": 1.1918, + "theoretical_loss": 3.3617908869051463, + "tokens_seen": 2856910848 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010867803410230692, + "loss": 1.2362, + "theoretical_loss": 3.3617849493380803, + "tokens_seen": 2856976384 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010866198595787362, + "loss": 1.2688, + "theoretical_loss": 3.3617790119453494, + "tokens_seen": 2857041920 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010864593781344032, + "loss": 1.2525, + "theoretical_loss": 3.361773074726945, + "tokens_seen": 2857107456 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010862988966900702, + "loss": 1.2077, + "theoretical_loss": 3.3617671376828575, + "tokens_seen": 2857172992 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010861384152457372, + "loss": 1.2232, + "theoretical_loss": 3.361761200813078, + "tokens_seen": 2857238528 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010859779338014042, + "loss": 1.2355, + "theoretical_loss": 3.3617552641175976, + "tokens_seen": 2857304064 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 953809, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0995903015136719, + "objective/train/theoretical_loss": 3.3617493275964065, + "objective/train/tokens_used": 2877829600, + "theoretical_loss": 3.3617493275964065, + "tokens_seen": 2857369600 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010858174523570713, + "loss": 1.1362, + "theoretical_loss": 3.3617493275964065, + "tokens_seen": 2857369600 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010856569709127383, + "loss": 1.1394, + "theoretical_loss": 3.3617433912494965, + "tokens_seen": 2857435136 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010854964894684053, + "loss": 1.2726, + "theoretical_loss": 3.3617374550768577, + "tokens_seen": 2857500672 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010853360080240723, + "loss": 1.2953, + "theoretical_loss": 3.3617315190784813, + "tokens_seen": 2857566208 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010851755265797391, + "loss": 1.2416, + "theoretical_loss": 3.3617255832543584, + "tokens_seen": 2857631744 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010850150451354061, + "loss": 1.2888, + "theoretical_loss": 3.3617196476044797, + "tokens_seen": 2857697280 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010848545636910732, + "loss": 1.2538, + "theoretical_loss": 3.361713712128836, + "tokens_seen": 2857762816 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010846940822467402, + "loss": 1.1816, + "theoretical_loss": 3.3617077768274184, + "tokens_seen": 2857828352 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010845336008024072, + "loss": 1.1453, + "theoretical_loss": 3.361701841700217, + "tokens_seen": 2857893888 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010843731193580742, + "loss": 1.1555, + "theoretical_loss": 3.361695906747224, + "tokens_seen": 2857959424 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010842126379137412, + "loss": 1.2446, + "theoretical_loss": 3.3616899719684294, + "tokens_seen": 2858024960 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010840521564694083, + "loss": 1.3167, + "theoretical_loss": 3.3616840373638244, + "tokens_seen": 2858090496 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010838916750250753, + "loss": 1.2616, + "theoretical_loss": 3.3616781029334, + "tokens_seen": 2858156032 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010837311935807423, + "loss": 1.3196, + "theoretical_loss": 3.361672168677147, + "tokens_seen": 2858221568 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010835707121364093, + "loss": 1.2905, + "theoretical_loss": 3.361666234595056, + "tokens_seen": 2858287104 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010834102306920763, + "loss": 1.2577, + "theoretical_loss": 3.3616603006871175, + "tokens_seen": 2858352640 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010832497492477434, + "loss": 1.286, + "theoretical_loss": 3.361654366953324, + "tokens_seen": 2858418176 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010830892678034104, + "loss": 1.1616, + "theoretical_loss": 3.3616484333936647, + "tokens_seen": 2858483712 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010829287863590774, + "loss": 1.2592, + "theoretical_loss": 3.3616425000081316, + "tokens_seen": 2858549248 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010827683049147444, + "loss": 1.1989, + "theoretical_loss": 3.361636566796715, + "tokens_seen": 2858614784 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010826078234704114, + "loss": 1.182, + "theoretical_loss": 3.361630633759406, + "tokens_seen": 2858680320 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010824473420260784, + "loss": 1.231, + "theoretical_loss": 3.3616247008961957, + "tokens_seen": 2858745856 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010822868605817454, + "loss": 1.1719, + "theoretical_loss": 3.3616187682070744, + "tokens_seen": 2858811392 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010821263791374123, + "loss": 1.1443, + "theoretical_loss": 3.361612835692034, + "tokens_seen": 2858876928 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010819658976930793, + "loss": 1.2421, + "theoretical_loss": 3.3616069033510643, + "tokens_seen": 2858942464 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010818054162487463, + "loss": 1.1763, + "theoretical_loss": 3.361600971184157, + "tokens_seen": 2859008000 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010816449348044133, + "loss": 1.1478, + "theoretical_loss": 3.361595039191302, + "tokens_seen": 2859073536 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010814844533600803, + "loss": 1.1394, + "theoretical_loss": 3.3615891073724917, + "tokens_seen": 2859139072 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010813239719157473, + "loss": 1.1737, + "theoretical_loss": 3.361583175727716, + "tokens_seen": 2859204608 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010811634904714143, + "loss": 1.1478, + "theoretical_loss": 3.3615772442569662, + "tokens_seen": 2859270144 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010810030090270813, + "loss": 1.2233, + "theoretical_loss": 3.3615713129602325, + "tokens_seen": 2859335680 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010808425275827483, + "loss": 1.228, + "theoretical_loss": 3.3615653818375066, + "tokens_seen": 2859401216 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010806820461384153, + "loss": 1.2941, + "theoretical_loss": 3.3615594508887794, + "tokens_seen": 2859466752 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010805215646940823, + "loss": 1.0583, + "theoretical_loss": 3.361553520114041, + "tokens_seen": 2859532288 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010803610832497493, + "loss": 1.2505, + "theoretical_loss": 3.3615475895132834, + "tokens_seen": 2859597824 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010802006018054162, + "loss": 1.284, + "theoretical_loss": 3.3615416590864964, + "tokens_seen": 2859663360 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010800401203610832, + "loss": 1.2652, + "theoretical_loss": 3.361535728833672, + "tokens_seen": 2859728896 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010798796389167502, + "loss": 1.2045, + "theoretical_loss": 3.3615297987548, + "tokens_seen": 2859794432 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010797191574724172, + "loss": 1.2659, + "theoretical_loss": 3.3615238688498725, + "tokens_seen": 2859859968 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010795586760280842, + "loss": 1.2444, + "theoretical_loss": 3.3615179391188796, + "tokens_seen": 2859925504 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010793981945837512, + "loss": 1.3102, + "theoretical_loss": 3.3615120095618125, + "tokens_seen": 2859991040 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010792377131394182, + "loss": 1.2931, + "theoretical_loss": 3.361506080178662, + "tokens_seen": 2860056576 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010790772316950852, + "loss": 1.2036, + "theoretical_loss": 3.3615001509694187, + "tokens_seen": 2860122112 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010789167502507523, + "loss": 1.1744, + "theoretical_loss": 3.361494221934074, + "tokens_seen": 2860187648 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010787562688064193, + "loss": 1.2217, + "theoretical_loss": 3.361488293072619, + "tokens_seen": 2860253184 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010785957873620863, + "loss": 1.2136, + "theoretical_loss": 3.361482364385044, + "tokens_seen": 2860318720 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010784353059177533, + "loss": 1.2632, + "theoretical_loss": 3.3614764358713405, + "tokens_seen": 2860384256 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010782748244734201, + "loss": 1.2384, + "theoretical_loss": 3.361470507531499, + "tokens_seen": 2860449792 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010781143430290871, + "loss": 1.2034, + "theoretical_loss": 3.36146457936551, + "tokens_seen": 2860515328 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010779538615847544, + "loss": 1.2539, + "theoretical_loss": 3.3614586513733657, + "tokens_seen": 2860580864 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 954765, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2228093147277832, + "objective/train/theoretical_loss": 3.3614527235550558, + "objective/train/tokens_used": 2881106400, + "theoretical_loss": 3.3614527235550558, + "tokens_seen": 2860646400 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010777933801404214, + "loss": 1.2618, + "theoretical_loss": 3.3614527235550558, + "tokens_seen": 2860646400 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010776328986960884, + "loss": 1.3365, + "theoretical_loss": 3.361446795910572, + "tokens_seen": 2860711936 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010774724172517554, + "loss": 1.1734, + "theoretical_loss": 3.361440868439905, + "tokens_seen": 2860777472 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010773119358074225, + "loss": 1.2552, + "theoretical_loss": 3.3614349411430453, + "tokens_seen": 2860843008 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010771514543630895, + "loss": 1.1501, + "theoretical_loss": 3.3614290140199845, + "tokens_seen": 2860908544 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010769909729187563, + "loss": 1.2431, + "theoretical_loss": 3.3614230870707127, + "tokens_seen": 2860974080 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010768304914744233, + "loss": 1.2867, + "theoretical_loss": 3.361417160295222, + "tokens_seen": 2861039616 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010766700100300903, + "loss": 1.2534, + "theoretical_loss": 3.3614112336935023, + "tokens_seen": 2861105152 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010765095285857573, + "loss": 1.2211, + "theoretical_loss": 3.3614053072655445, + "tokens_seen": 2861170688 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010763490471414244, + "loss": 1.1467, + "theoretical_loss": 3.3613993810113403, + "tokens_seen": 2861236224 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010761885656970914, + "loss": 1.3225, + "theoretical_loss": 3.36139345493088, + "tokens_seen": 2861301760 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010760280842527584, + "loss": 1.1612, + "theoretical_loss": 3.361387529024155, + "tokens_seen": 2861367296 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010758676028084254, + "loss": 1.2052, + "theoretical_loss": 3.3613816032911554, + "tokens_seen": 2861432832 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010757071213640924, + "loss": 1.32, + "theoretical_loss": 3.361375677731873, + "tokens_seen": 2861498368 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010755466399197594, + "loss": 1.172, + "theoretical_loss": 3.361369752346299, + "tokens_seen": 2861563904 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010753861584754264, + "loss": 1.2965, + "theoretical_loss": 3.361363827134423, + "tokens_seen": 2861629440 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010752256770310934, + "loss": 1.2326, + "theoretical_loss": 3.361357902096237, + "tokens_seen": 2861694976 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010750651955867603, + "loss": 1.229, + "theoretical_loss": 3.361351977231732, + "tokens_seen": 2861760512 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010749047141424273, + "loss": 1.2247, + "theoretical_loss": 3.361346052540898, + "tokens_seen": 2861826048 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010747442326980943, + "loss": 1.2118, + "theoretical_loss": 3.3613401280237265, + "tokens_seen": 2861891584 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010745837512537613, + "loss": 1.269, + "theoretical_loss": 3.3613342036802085, + "tokens_seen": 2861957120 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010744232698094283, + "loss": 1.236, + "theoretical_loss": 3.3613282795103347, + "tokens_seen": 2862022656 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010742627883650953, + "loss": 1.2596, + "theoretical_loss": 3.3613223555140963, + "tokens_seen": 2862088192 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010741023069207623, + "loss": 1.1455, + "theoretical_loss": 3.3613164316914843, + "tokens_seen": 2862153728 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010739418254764293, + "loss": 1.2215, + "theoretical_loss": 3.3613105080424894, + "tokens_seen": 2862219264 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010737813440320963, + "loss": 1.167, + "theoretical_loss": 3.3613045845671023, + "tokens_seen": 2862284800 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010736208625877633, + "loss": 1.2393, + "theoretical_loss": 3.3612986612653146, + "tokens_seen": 2862350336 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010734603811434303, + "loss": 1.2072, + "theoretical_loss": 3.3612927381371165, + "tokens_seen": 2862415872 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010732998996990973, + "loss": 1.2848, + "theoretical_loss": 3.3612868151824995, + "tokens_seen": 2862481408 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010731394182547642, + "loss": 1.2237, + "theoretical_loss": 3.361280892401455, + "tokens_seen": 2862546944 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010729789368104312, + "loss": 1.2154, + "theoretical_loss": 3.361274969793972, + "tokens_seen": 2862612480 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010728184553660982, + "loss": 1.2235, + "theoretical_loss": 3.3612690473600435, + "tokens_seen": 2862678016 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010726579739217652, + "loss": 1.1884, + "theoretical_loss": 3.3612631250996596, + "tokens_seen": 2862743552 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010724974924774322, + "loss": 1.183, + "theoretical_loss": 3.361257203012811, + "tokens_seen": 2862809088 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010723370110330992, + "loss": 1.1692, + "theoretical_loss": 3.3612512810994892, + "tokens_seen": 2862874624 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010721765295887662, + "loss": 1.254, + "theoretical_loss": 3.361245359359685, + "tokens_seen": 2862940160 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010720160481444335, + "loss": 1.1459, + "theoretical_loss": 3.361239437793389, + "tokens_seen": 2863005696 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010718555667001004, + "loss": 1.1993, + "theoretical_loss": 3.3612335164005924, + "tokens_seen": 2863071232 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010716950852557674, + "loss": 1.0864, + "theoretical_loss": 3.361227595181286, + "tokens_seen": 2863136768 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010715346038114344, + "loss": 1.1864, + "theoretical_loss": 3.361221674135461, + "tokens_seen": 2863202304 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010713741223671014, + "loss": 1.2719, + "theoretical_loss": 3.3612157532631084, + "tokens_seen": 2863267840 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010712136409227684, + "loss": 1.1153, + "theoretical_loss": 3.3612098325642186, + "tokens_seen": 2863333376 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010710531594784354, + "loss": 1.1457, + "theoretical_loss": 3.3612039120387833, + "tokens_seen": 2863398912 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010708926780341024, + "loss": 1.2704, + "theoretical_loss": 3.361197991686793, + "tokens_seen": 2863464448 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010707321965897694, + "loss": 1.2721, + "theoretical_loss": 3.3611920715082384, + "tokens_seen": 2863529984 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010705717151454364, + "loss": 1.1447, + "theoretical_loss": 3.361186151503111, + "tokens_seen": 2863595520 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010704112337011035, + "loss": 1.1166, + "theoretical_loss": 3.3611802316714012, + "tokens_seen": 2863661056 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010702507522567705, + "loss": 1.2548, + "theoretical_loss": 3.361174312013101, + "tokens_seen": 2863726592 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010700902708124375, + "loss": 1.2212, + "theoretical_loss": 3.3611683925282, + "tokens_seen": 2863792128 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010699297893681043, + "loss": 1.2846, + "theoretical_loss": 3.36116247321669, + "tokens_seen": 2863857664 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 955895, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1332433223724365, + "objective/train/theoretical_loss": 3.3611565540785615, + "objective/train/tokens_used": 2884383200, + "theoretical_loss": 3.3611565540785615, + "tokens_seen": 2863923200 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010697693079237713, + "loss": 1.0856, + "theoretical_loss": 3.3611565540785615, + "tokens_seen": 2863923200 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010696088264794383, + "loss": 1.1332, + "theoretical_loss": 3.361150635113806, + "tokens_seen": 2863988736 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010694483450351054, + "loss": 1.133, + "theoretical_loss": 3.3611447163224137, + "tokens_seen": 2864054272 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010692878635907724, + "loss": 1.1265, + "theoretical_loss": 3.3611387977043763, + "tokens_seen": 2864119808 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010691273821464394, + "loss": 1.2224, + "theoretical_loss": 3.361132879259684, + "tokens_seen": 2864185344 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010689669007021064, + "loss": 1.2428, + "theoretical_loss": 3.3611269609883285, + "tokens_seen": 2864250880 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010688064192577734, + "loss": 1.2372, + "theoretical_loss": 3.361121042890301, + "tokens_seen": 2864316416 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010686459378134404, + "loss": 1.168, + "theoretical_loss": 3.361115124965591, + "tokens_seen": 2864381952 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010684854563691074, + "loss": 1.2402, + "theoretical_loss": 3.361109207214191, + "tokens_seen": 2864447488 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010683249749247744, + "loss": 1.2751, + "theoretical_loss": 3.361103289636091, + "tokens_seen": 2864513024 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010681644934804414, + "loss": 1.1807, + "theoretical_loss": 3.3610973722312822, + "tokens_seen": 2864578560 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010680040120361083, + "loss": 1.242, + "theoretical_loss": 3.361091454999756, + "tokens_seen": 2864644096 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010678435305917753, + "loss": 1.2706, + "theoretical_loss": 3.361085537941503, + "tokens_seen": 2864709632 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010676830491474423, + "loss": 1.2321, + "theoretical_loss": 3.361079621056514, + "tokens_seen": 2864775168 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010675225677031093, + "loss": 1.1785, + "theoretical_loss": 3.36107370434478, + "tokens_seen": 2864840704 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010673620862587763, + "loss": 1.1786, + "theoretical_loss": 3.3610677878062925, + "tokens_seen": 2864906240 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010672016048144433, + "loss": 1.1467, + "theoretical_loss": 3.361061871441042, + "tokens_seen": 2864971776 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010670411233701103, + "loss": 1.2576, + "theoretical_loss": 3.3610559552490193, + "tokens_seen": 2865037312 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010668806419257773, + "loss": 1.2616, + "theoretical_loss": 3.3610500392302156, + "tokens_seen": 2865102848 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010667201604814443, + "loss": 1.2034, + "theoretical_loss": 3.361044123384622, + "tokens_seen": 2865168384 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010665596790371113, + "loss": 1.1781, + "theoretical_loss": 3.3610382077122294, + "tokens_seen": 2865233920 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010663991975927783, + "loss": 1.1136, + "theoretical_loss": 3.3610322922130287, + "tokens_seen": 2865299456 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010662387161484453, + "loss": 1.2474, + "theoretical_loss": 3.361026376887011, + "tokens_seen": 2865364992 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010660782347041122, + "loss": 1.2165, + "theoretical_loss": 3.3610204617341664, + "tokens_seen": 2865430528 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010659177532597795, + "loss": 1.2459, + "theoretical_loss": 3.361014546754487, + "tokens_seen": 2865496064 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010657572718154465, + "loss": 1.1216, + "theoretical_loss": 3.3610086319479637, + "tokens_seen": 2865561600 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010655967903711135, + "loss": 1.1847, + "theoretical_loss": 3.361002717314587, + "tokens_seen": 2865627136 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010654363089267805, + "loss": 1.167, + "theoretical_loss": 3.3609968028543475, + "tokens_seen": 2865692672 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010652758274824475, + "loss": 1.2198, + "theoretical_loss": 3.3609908885672373, + "tokens_seen": 2865758208 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010651153460381145, + "loss": 1.2062, + "theoretical_loss": 3.3609849744532467, + "tokens_seen": 2865823744 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010649548645937814, + "loss": 1.2227, + "theoretical_loss": 3.3609790605123666, + "tokens_seen": 2865889280 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010647943831494484, + "loss": 1.1737, + "theoretical_loss": 3.3609731467445885, + "tokens_seen": 2865954816 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010646339017051154, + "loss": 1.2153, + "theoretical_loss": 3.3609672331499025, + "tokens_seen": 2866020352 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010644734202607824, + "loss": 1.1011, + "theoretical_loss": 3.3609613197283, + "tokens_seen": 2866085888 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010643129388164494, + "loss": 1.126, + "theoretical_loss": 3.360955406479772, + "tokens_seen": 2866151424 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010641524573721164, + "loss": 1.1502, + "theoretical_loss": 3.36094949340431, + "tokens_seen": 2866216960 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010639919759277834, + "loss": 1.1618, + "theoretical_loss": 3.360943580501904, + "tokens_seen": 2866282496 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010638314944834504, + "loss": 1.2299, + "theoretical_loss": 3.360937667772546, + "tokens_seen": 2866348032 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010636710130391174, + "loss": 1.1219, + "theoretical_loss": 3.3609317552162263, + "tokens_seen": 2866413568 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010635105315947845, + "loss": 1.2045, + "theoretical_loss": 3.360925842832936, + "tokens_seen": 2866479104 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010633500501504515, + "loss": 1.1868, + "theoretical_loss": 3.360919930622666, + "tokens_seen": 2866544640 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010631895687061185, + "loss": 1.2763, + "theoretical_loss": 3.360914018585407, + "tokens_seen": 2866610176 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010630290872617853, + "loss": 1.2117, + "theoretical_loss": 3.3609081067211513, + "tokens_seen": 2866675712 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010628686058174523, + "loss": 1.181, + "theoretical_loss": 3.3609021950298885, + "tokens_seen": 2866741248 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010627081243731194, + "loss": 1.1686, + "theoretical_loss": 3.3608962835116096, + "tokens_seen": 2866806784 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010625476429287864, + "loss": 1.2376, + "theoretical_loss": 3.3608903721663066, + "tokens_seen": 2866872320 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010623871614844534, + "loss": 1.2764, + "theoretical_loss": 3.3608844609939696, + "tokens_seen": 2866937856 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010622266800401204, + "loss": 1.2386, + "theoretical_loss": 3.36087854999459, + "tokens_seen": 2867003392 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010620661985957874, + "loss": 1.1849, + "theoretical_loss": 3.360872639168159, + "tokens_seen": 2867068928 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010619057171514544, + "loss": 1.2541, + "theoretical_loss": 3.360866728514667, + "tokens_seen": 2867134464 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 956750, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 0.9791686534881592, + "objective/train/theoretical_loss": 3.360860818034105, + "objective/train/tokens_used": 2887660000, + "theoretical_loss": 3.360860818034105, + "tokens_seen": 2867200000 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010617452357071214, + "loss": 1.1585, + "theoretical_loss": 3.360860818034105, + "tokens_seen": 2867200000 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010615847542627884, + "loss": 1.1246, + "theoretical_loss": 3.3608549077264644, + "tokens_seen": 2867265536 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010614242728184554, + "loss": 1.1738, + "theoretical_loss": 3.360848997591736, + "tokens_seen": 2867331072 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010612637913741224, + "loss": 1.1393, + "theoretical_loss": 3.3608430876299114, + "tokens_seen": 2867396608 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010611033099297893, + "loss": 1.2351, + "theoretical_loss": 3.3608371778409802, + "tokens_seen": 2867462144 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010609428284854563, + "loss": 1.2135, + "theoretical_loss": 3.360831268224935, + "tokens_seen": 2867527680 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010607823470411233, + "loss": 1.0887, + "theoretical_loss": 3.3608253587817654, + "tokens_seen": 2867593216 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010606218655967903, + "loss": 1.26, + "theoretical_loss": 3.360819449511463, + "tokens_seen": 2867658752 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010604613841524573, + "loss": 1.216, + "theoretical_loss": 3.3608135404140187, + "tokens_seen": 2867724288 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010603009027081243, + "loss": 1.1378, + "theoretical_loss": 3.3608076314894237, + "tokens_seen": 2867789824 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010601404212637913, + "loss": 1.2528, + "theoretical_loss": 3.360801722737669, + "tokens_seen": 2867855360 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010599799398194586, + "loss": 1.2573, + "theoretical_loss": 3.3607958141587457, + "tokens_seen": 2867920896 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010598194583751255, + "loss": 1.2039, + "theoretical_loss": 3.360789905752644, + "tokens_seen": 2867986432 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010596589769307925, + "loss": 1.1904, + "theoretical_loss": 3.360783997519356, + "tokens_seen": 2868051968 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010594984954864595, + "loss": 1.1814, + "theoretical_loss": 3.360778089458872, + "tokens_seen": 2868117504 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010593380140421265, + "loss": 1.367, + "theoretical_loss": 3.360772181571183, + "tokens_seen": 2868183040 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010591775325977935, + "loss": 1.1009, + "theoretical_loss": 3.3607662738562802, + "tokens_seen": 2868248576 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010590170511534605, + "loss": 1.1683, + "theoretical_loss": 3.3607603663141545, + "tokens_seen": 2868314112 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010588565697091275, + "loss": 1.2071, + "theoretical_loss": 3.360754458944797, + "tokens_seen": 2868379648 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010586960882647945, + "loss": 1.1491, + "theoretical_loss": 3.3607485517481983, + "tokens_seen": 2868445184 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010585356068204615, + "loss": 1.1697, + "theoretical_loss": 3.3607426447243505, + "tokens_seen": 2868510720 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010583751253761285, + "loss": 1.1843, + "theoretical_loss": 3.360736737873243, + "tokens_seen": 2868576256 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010582146439317955, + "loss": 1.2905, + "theoretical_loss": 3.3607308311948683, + "tokens_seen": 2868641792 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010580541624874625, + "loss": 1.2447, + "theoretical_loss": 3.360724924689216, + "tokens_seen": 2868707328 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010578936810431294, + "loss": 1.1695, + "theoretical_loss": 3.3607190183562787, + "tokens_seen": 2868772864 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010577331995987964, + "loss": 1.2506, + "theoretical_loss": 3.3607131121960463, + "tokens_seen": 2868838400 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010575727181544634, + "loss": 1.2425, + "theoretical_loss": 3.36070720620851, + "tokens_seen": 2868903936 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010574122367101304, + "loss": 1.3182, + "theoretical_loss": 3.3607013003936608, + "tokens_seen": 2868969472 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010572517552657974, + "loss": 1.1731, + "theoretical_loss": 3.36069539475149, + "tokens_seen": 2869035008 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010570912738214644, + "loss": 1.2516, + "theoretical_loss": 3.360689489281988, + "tokens_seen": 2869100544 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010569307923771314, + "loss": 1.2015, + "theoretical_loss": 3.3606835839851463, + "tokens_seen": 2869166080 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010567703109327985, + "loss": 1.2028, + "theoretical_loss": 3.360677678860956, + "tokens_seen": 2869231616 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010566098294884655, + "loss": 1.1573, + "theoretical_loss": 3.3606717739094076, + "tokens_seen": 2869297152 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010564493480441325, + "loss": 1.2511, + "theoretical_loss": 3.360665869130493, + "tokens_seen": 2869362688 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010562888665997995, + "loss": 1.2098, + "theoretical_loss": 3.360659964524202, + "tokens_seen": 2869428224 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010561283851554665, + "loss": 1.2001, + "theoretical_loss": 3.3606540600905266, + "tokens_seen": 2869493760 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010559679037111333, + "loss": 1.1782, + "theoretical_loss": 3.360648155829457, + "tokens_seen": 2869559296 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010558074222668004, + "loss": 1.2326, + "theoretical_loss": 3.3606422517409853, + "tokens_seen": 2869624832 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010556469408224674, + "loss": 1.1294, + "theoretical_loss": 3.3606363478251016, + "tokens_seen": 2869690368 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010554864593781344, + "loss": 1.2161, + "theoretical_loss": 3.360630444081797, + "tokens_seen": 2869755904 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010553259779338014, + "loss": 1.101, + "theoretical_loss": 3.360624540511063, + "tokens_seen": 2869821440 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010551654964894684, + "loss": 1.1717, + "theoretical_loss": 3.36061863711289, + "tokens_seen": 2869886976 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010550050150451354, + "loss": 1.2942, + "theoretical_loss": 3.3606127338872698, + "tokens_seen": 2869952512 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010548445336008024, + "loss": 1.0947, + "theoretical_loss": 3.3606068308341923, + "tokens_seen": 2870018048 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010546840521564694, + "loss": 1.1878, + "theoretical_loss": 3.3606009279536497, + "tokens_seen": 2870083584 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010545235707121364, + "loss": 1.2353, + "theoretical_loss": 3.3605950252456323, + "tokens_seen": 2870149120 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010543630892678034, + "loss": 1.2151, + "theoretical_loss": 3.3605891227101314, + "tokens_seen": 2870214656 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010542026078234704, + "loss": 1.1845, + "theoretical_loss": 3.360583220347138, + "tokens_seen": 2870280192 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010542026078234704, + "loss": 1.0815, + "theoretical_loss": 3.3605773181566425, + "tokens_seen": 2870345728 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010540421263791376, + "loss": 1.1018, + "theoretical_loss": 3.360571416138637, + "tokens_seen": 2870411264 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 957825, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2266452312469482, + "objective/train/theoretical_loss": 3.360565514293112, + "objective/train/tokens_used": 2890936800, + "theoretical_loss": 3.360565514293112, + "tokens_seen": 2870476800 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010538816449348046, + "loss": 1.1919, + "theoretical_loss": 3.360565514293112, + "tokens_seen": 2870476800 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010537211634904716, + "loss": 1.1562, + "theoretical_loss": 3.360559612620058, + "tokens_seen": 2870542336 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010535606820461386, + "loss": 1.0581, + "theoretical_loss": 3.3605537111194668, + "tokens_seen": 2870607872 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010534002006018056, + "loss": 1.1866, + "theoretical_loss": 3.3605478097913295, + "tokens_seen": 2870673408 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010532397191574726, + "loss": 1.1931, + "theoretical_loss": 3.3605419086356365, + "tokens_seen": 2870738944 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010530792377131396, + "loss": 1.1642, + "theoretical_loss": 3.360536007652379, + "tokens_seen": 2870804480 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010529187562688066, + "loss": 1.2684, + "theoretical_loss": 3.360530106841548, + "tokens_seen": 2870870016 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010527582748244735, + "loss": 1.1372, + "theoretical_loss": 3.360524206203135, + "tokens_seen": 2870935552 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010525977933801405, + "loss": 1.2334, + "theoretical_loss": 3.3605183057371306, + "tokens_seen": 2871001088 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010524373119358075, + "loss": 1.2205, + "theoretical_loss": 3.360512405443526, + "tokens_seen": 2871066624 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010522768304914745, + "loss": 1.2612, + "theoretical_loss": 3.3605065053223115, + "tokens_seen": 2871132160 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010521163490471415, + "loss": 1.1392, + "theoretical_loss": 3.3605006053734794, + "tokens_seen": 2871197696 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010519558676028085, + "loss": 1.1868, + "theoretical_loss": 3.36049470559702, + "tokens_seen": 2871263232 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010517953861584755, + "loss": 1.2005, + "theoretical_loss": 3.360488805992924, + "tokens_seen": 2871328768 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010516349047141425, + "loss": 1.1632, + "theoretical_loss": 3.360482906561183, + "tokens_seen": 2871394304 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010514744232698095, + "loss": 1.2334, + "theoretical_loss": 3.3604770073017884, + "tokens_seen": 2871459840 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010513139418254765, + "loss": 1.1871, + "theoretical_loss": 3.36047110821473, + "tokens_seen": 2871525376 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010511534603811435, + "loss": 1.2198, + "theoretical_loss": 3.3604652093, + "tokens_seen": 2871590912 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010509929789368105, + "loss": 1.0777, + "theoretical_loss": 3.360459310557589, + "tokens_seen": 2871656448 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010508324974924774, + "loss": 1.2046, + "theoretical_loss": 3.3604534119874883, + "tokens_seen": 2871721984 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010506720160481444, + "loss": 1.1915, + "theoretical_loss": 3.360447513589688, + "tokens_seen": 2871787520 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010505115346038114, + "loss": 1.1726, + "theoretical_loss": 3.36044161536418, + "tokens_seen": 2871853056 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010503510531594784, + "loss": 1.2179, + "theoretical_loss": 3.360435717310955, + "tokens_seen": 2871918592 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010501905717151454, + "loss": 1.1148, + "theoretical_loss": 3.360429819430004, + "tokens_seen": 2871984128 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010500300902708124, + "loss": 1.1829, + "theoretical_loss": 3.3604239217213188, + "tokens_seen": 2872049664 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010498696088264795, + "loss": 1.1744, + "theoretical_loss": 3.3604180241848893, + "tokens_seen": 2872115200 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010497091273821465, + "loss": 1.189, + "theoretical_loss": 3.3604121268207074, + "tokens_seen": 2872180736 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010495486459378135, + "loss": 1.0886, + "theoretical_loss": 3.3604062296287633, + "tokens_seen": 2872246272 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010493881644934805, + "loss": 1.282, + "theoretical_loss": 3.360400332609049, + "tokens_seen": 2872311808 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010492276830491475, + "loss": 1.1447, + "theoretical_loss": 3.360394435761555, + "tokens_seen": 2872377344 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010490672016048145, + "loss": 1.1264, + "theoretical_loss": 3.360388539086273, + "tokens_seen": 2872442880 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010489067201604814, + "loss": 1.2158, + "theoretical_loss": 3.3603826425831924, + "tokens_seen": 2872508416 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010487462387161484, + "loss": 1.2095, + "theoretical_loss": 3.3603767462523058, + "tokens_seen": 2872573952 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010485857572718154, + "loss": 1.2454, + "theoretical_loss": 3.360370850093604, + "tokens_seen": 2872639488 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010484252758274824, + "loss": 1.1421, + "theoretical_loss": 3.3603649541070775, + "tokens_seen": 2872705024 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010482647943831494, + "loss": 1.2967, + "theoretical_loss": 3.3603590582927176, + "tokens_seen": 2872770560 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010481043129388164, + "loss": 1.1481, + "theoretical_loss": 3.360353162650515, + "tokens_seen": 2872836096 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010479438314944837, + "loss": 1.181, + "theoretical_loss": 3.3603472671804617, + "tokens_seen": 2872901632 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010477833500501507, + "loss": 1.233, + "theoretical_loss": 3.3603413718825483, + "tokens_seen": 2872967168 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010476228686058175, + "loss": 1.1762, + "theoretical_loss": 3.3603354767567657, + "tokens_seen": 2873032704 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010474623871614845, + "loss": 1.2096, + "theoretical_loss": 3.3603295818031045, + "tokens_seen": 2873098240 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010473019057171516, + "loss": 1.0724, + "theoretical_loss": 3.3603236870215567, + "tokens_seen": 2873163776 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010471414242728186, + "loss": 1.1427, + "theoretical_loss": 3.3603177924121126, + "tokens_seen": 2873229312 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010469809428284856, + "loss": 1.1606, + "theoretical_loss": 3.3603118979747637, + "tokens_seen": 2873294848 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010468204613841526, + "loss": 1.2006, + "theoretical_loss": 3.3603060037095007, + "tokens_seen": 2873360384 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010466599799398196, + "loss": 1.1504, + "theoretical_loss": 3.3603001096163148, + "tokens_seen": 2873425920 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010464994984954866, + "loss": 1.1591, + "theoretical_loss": 3.3602942156951974, + "tokens_seen": 2873491456 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010463390170511536, + "loss": 1.1812, + "theoretical_loss": 3.360288321946139, + "tokens_seen": 2873556992 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010461785356068206, + "loss": 1.182, + "theoretical_loss": 3.3602824283691306, + "tokens_seen": 2873622528 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010460180541624876, + "loss": 1.2512, + "theoretical_loss": 3.360276534964164, + "tokens_seen": 2873688064 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 959016, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1979799270629883, + "objective/train/theoretical_loss": 3.3602706417312294, + "objective/train/tokens_used": 2894213600, + "theoretical_loss": 3.3602706417312294, + "tokens_seen": 2873753600 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010458575727181546, + "loss": 1.1955, + "theoretical_loss": 3.3602706417312294, + "tokens_seen": 2873753600 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010456970912738215, + "loss": 1.2136, + "theoretical_loss": 3.3602647486703185, + "tokens_seen": 2873819136 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010455366098294885, + "loss": 1.166, + "theoretical_loss": 3.360258855781422, + "tokens_seen": 2873884672 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010453761283851555, + "loss": 1.1826, + "theoretical_loss": 3.3602529630645313, + "tokens_seen": 2873950208 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010452156469408225, + "loss": 1.2558, + "theoretical_loss": 3.360247070519637, + "tokens_seen": 2874015744 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010450551654964895, + "loss": 1.1911, + "theoretical_loss": 3.3602411781467305, + "tokens_seen": 2874081280 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010448946840521565, + "loss": 1.1539, + "theoretical_loss": 3.3602352859458025, + "tokens_seen": 2874146816 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010447342026078235, + "loss": 1.2185, + "theoretical_loss": 3.3602293939168444, + "tokens_seen": 2874212352 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010445737211634905, + "loss": 1.1542, + "theoretical_loss": 3.360223502059847, + "tokens_seen": 2874277888 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010444132397191575, + "loss": 1.1258, + "theoretical_loss": 3.360217610374802, + "tokens_seen": 2874343424 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010442527582748245, + "loss": 1.1287, + "theoretical_loss": 3.3602117188616996, + "tokens_seen": 2874408960 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010440922768304915, + "loss": 1.2722, + "theoretical_loss": 3.360205827520531, + "tokens_seen": 2874474496 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010439317953861586, + "loss": 1.1982, + "theoretical_loss": 3.3601999363512878, + "tokens_seen": 2874540032 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010437713139418254, + "loss": 1.1946, + "theoretical_loss": 3.3601940453539605, + "tokens_seen": 2874605568 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010436108324974924, + "loss": 1.1915, + "theoretical_loss": 3.360188154528541, + "tokens_seen": 2874671104 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010434503510531594, + "loss": 1.1349, + "theoretical_loss": 3.3601822638750196, + "tokens_seen": 2874736640 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010432898696088264, + "loss": 1.2423, + "theoretical_loss": 3.3601763733933874, + "tokens_seen": 2874802176 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010431293881644934, + "loss": 1.1079, + "theoretical_loss": 3.3601704830836354, + "tokens_seen": 2874867712 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010429689067201605, + "loss": 1.238, + "theoretical_loss": 3.3601645929457553, + "tokens_seen": 2874933248 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010428084252758275, + "loss": 1.1877, + "theoretical_loss": 3.3601587029797373, + "tokens_seen": 2874998784 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010426479438314945, + "loss": 1.1146, + "theoretical_loss": 3.3601528131855734, + "tokens_seen": 2875064320 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010424874623871615, + "loss": 1.1888, + "theoretical_loss": 3.3601469235632537, + "tokens_seen": 2875129856 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010423269809428285, + "loss": 1.2255, + "theoretical_loss": 3.36014103411277, + "tokens_seen": 2875195392 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010421664994984955, + "loss": 1.1658, + "theoretical_loss": 3.3601351448341132, + "tokens_seen": 2875260928 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010420060180541626, + "loss": 1.3076, + "theoretical_loss": 3.3601292557272746, + "tokens_seen": 2875326464 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010418455366098296, + "loss": 1.2902, + "theoretical_loss": 3.3601233667922443, + "tokens_seen": 2875392000 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010416850551654966, + "loss": 1.1582, + "theoretical_loss": 3.3601174780290144, + "tokens_seen": 2875457536 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010415245737211636, + "loss": 1.2093, + "theoretical_loss": 3.360111589437576, + "tokens_seen": 2875523072 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010413640922768307, + "loss": 1.2088, + "theoretical_loss": 3.360105701017919, + "tokens_seen": 2875588608 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010412036108324977, + "loss": 1.1839, + "theoretical_loss": 3.3600998127700357, + "tokens_seen": 2875654144 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010410431293881647, + "loss": 1.2339, + "theoretical_loss": 3.360093924693917, + "tokens_seen": 2875719680 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010408826479438317, + "loss": 1.0799, + "theoretical_loss": 3.360088036789553, + "tokens_seen": 2875785216 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010407221664994985, + "loss": 1.1365, + "theoretical_loss": 3.360082149056936, + "tokens_seen": 2875850752 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010405616850551655, + "loss": 1.1581, + "theoretical_loss": 3.360076261496056, + "tokens_seen": 2875916288 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010404012036108326, + "loss": 1.1455, + "theoretical_loss": 3.360070374106906, + "tokens_seen": 2875981824 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010402407221664996, + "loss": 1.2067, + "theoretical_loss": 3.3600644868894745, + "tokens_seen": 2876047360 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010400802407221666, + "loss": 1.211, + "theoretical_loss": 3.3600585998437538, + "tokens_seen": 2876112896 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010399197592778336, + "loss": 1.1922, + "theoretical_loss": 3.3600527129697353, + "tokens_seen": 2876178432 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010397592778335006, + "loss": 1.1291, + "theoretical_loss": 3.3600468262674097, + "tokens_seen": 2876243968 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010395987963891676, + "loss": 1.1583, + "theoretical_loss": 3.360040939736768, + "tokens_seen": 2876309504 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010394383149448346, + "loss": 1.168, + "theoretical_loss": 3.360035053377802, + "tokens_seen": 2876375040 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010392778335005016, + "loss": 1.2344, + "theoretical_loss": 3.360029167190502, + "tokens_seen": 2876440576 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010391173520561686, + "loss": 1.1376, + "theoretical_loss": 3.3600232811748585, + "tokens_seen": 2876506112 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010389568706118356, + "loss": 1.3017, + "theoretical_loss": 3.360017395330864, + "tokens_seen": 2876571648 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010387963891675025, + "loss": 1.2357, + "theoretical_loss": 3.3600115096585093, + "tokens_seen": 2876637184 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010386359077231695, + "loss": 1.1988, + "theoretical_loss": 3.3600056241577847, + "tokens_seen": 2876702720 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010384754262788365, + "loss": 1.1601, + "theoretical_loss": 3.359999738828682, + "tokens_seen": 2876768256 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010383149448345035, + "loss": 1.2118, + "theoretical_loss": 3.3599938536711917, + "tokens_seen": 2876833792 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010381544633901705, + "loss": 1.2063, + "theoretical_loss": 3.359987968685305, + "tokens_seen": 2876899328 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010379939819458375, + "loss": 1.1782, + "theoretical_loss": 3.3599820838710137, + "tokens_seen": 2876964864 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 959747, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.357374668121338, + "objective/train/theoretical_loss": 3.359976199228308, + "objective/train/tokens_used": 2897490400, + "theoretical_loss": 3.359976199228308, + "tokens_seen": 2877030400 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010378335005015045, + "loss": 1.2293, + "theoretical_loss": 3.359976199228308, + "tokens_seen": 2877030400 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010376730190571715, + "loss": 1.1938, + "theoretical_loss": 3.3599703147571796, + "tokens_seen": 2877095936 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010375125376128385, + "loss": 1.1106, + "theoretical_loss": 3.359964430457619, + "tokens_seen": 2877161472 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010373520561685055, + "loss": 1.1319, + "theoretical_loss": 3.3599585463296178, + "tokens_seen": 2877227008 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010371915747241725, + "loss": 1.1478, + "theoretical_loss": 3.359952662373167, + "tokens_seen": 2877292544 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010370310932798396, + "loss": 1.2289, + "theoretical_loss": 3.3599467785882577, + "tokens_seen": 2877358080 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010368706118355064, + "loss": 1.2074, + "theoretical_loss": 3.359940894974881, + "tokens_seen": 2877423616 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010368706118355064, + "loss": 1.2072, + "theoretical_loss": 3.3599350115330275, + "tokens_seen": 2877489152 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010367101303911734, + "loss": 1.1789, + "theoretical_loss": 3.359929128262689, + "tokens_seen": 2877554688 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010365496489468404, + "loss": 1.2446, + "theoretical_loss": 3.359923245163856, + "tokens_seen": 2877620224 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010363891675025074, + "loss": 1.1939, + "theoretical_loss": 3.35991736223652, + "tokens_seen": 2877685760 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010362286860581744, + "loss": 1.2002, + "theoretical_loss": 3.3599114794806724, + "tokens_seen": 2877751296 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010360682046138415, + "loss": 1.1366, + "theoretical_loss": 3.3599055968963034, + "tokens_seen": 2877816832 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010359077231695087, + "loss": 1.1985, + "theoretical_loss": 3.359899714483405, + "tokens_seen": 2877882368 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010357472417251757, + "loss": 1.1864, + "theoretical_loss": 3.3598938322419674, + "tokens_seen": 2877947904 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010355867602808426, + "loss": 1.1752, + "theoretical_loss": 3.3598879501719825, + "tokens_seen": 2878013440 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010354262788365096, + "loss": 1.1422, + "theoretical_loss": 3.359882068273441, + "tokens_seen": 2878078976 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010352657973921766, + "loss": 1.093, + "theoretical_loss": 3.359876186546334, + "tokens_seen": 2878144512 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010351053159478436, + "loss": 1.0306, + "theoretical_loss": 3.3598703049906526, + "tokens_seen": 2878210048 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010349448345035106, + "loss": 1.0998, + "theoretical_loss": 3.359864423606388, + "tokens_seen": 2878275584 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010347843530591776, + "loss": 1.2073, + "theoretical_loss": 3.359858542393531, + "tokens_seen": 2878341120 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010346238716148446, + "loss": 1.1204, + "theoretical_loss": 3.3598526613520736, + "tokens_seen": 2878406656 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010344633901705117, + "loss": 1.0917, + "theoretical_loss": 3.359846780482006, + "tokens_seen": 2878472192 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010343029087261787, + "loss": 1.0937, + "theoretical_loss": 3.3598408997833196, + "tokens_seen": 2878537728 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010341424272818457, + "loss": 1.1118, + "theoretical_loss": 3.359835019256005, + "tokens_seen": 2878603264 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010339819458375127, + "loss": 1.1582, + "theoretical_loss": 3.3598291389000545, + "tokens_seen": 2878668800 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010338214643931797, + "loss": 1.2878, + "theoretical_loss": 3.359823258715458, + "tokens_seen": 2878734336 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010336609829488466, + "loss": 1.2587, + "theoretical_loss": 3.3598173787022074, + "tokens_seen": 2878799872 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010335005015045136, + "loss": 1.2083, + "theoretical_loss": 3.359811498860293, + "tokens_seen": 2878865408 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010333400200601806, + "loss": 1.1838, + "theoretical_loss": 3.359805619189707, + "tokens_seen": 2878930944 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010331795386158476, + "loss": 1.1665, + "theoretical_loss": 3.3597997396904398, + "tokens_seen": 2878996480 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010330190571715146, + "loss": 1.1684, + "theoretical_loss": 3.3597938603624824, + "tokens_seen": 2879062016 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010328585757271816, + "loss": 1.0751, + "theoretical_loss": 3.3597879812058267, + "tokens_seen": 2879127552 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010326980942828486, + "loss": 1.1248, + "theoretical_loss": 3.3597821022204624, + "tokens_seen": 2879193088 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010325376128385156, + "loss": 1.1347, + "theoretical_loss": 3.359776223406382, + "tokens_seen": 2879258624 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010323771313941826, + "loss": 1.2006, + "theoretical_loss": 3.359770344763576, + "tokens_seen": 2879324160 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010322166499498496, + "loss": 1.1436, + "theoretical_loss": 3.3597644662920354, + "tokens_seen": 2879389696 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010320561685055166, + "loss": 1.2598, + "theoretical_loss": 3.3597585879917515, + "tokens_seen": 2879455232 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010318956870611836, + "loss": 1.0738, + "theoretical_loss": 3.3597527098627156, + "tokens_seen": 2879520768 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010317352056168505, + "loss": 1.1708, + "theoretical_loss": 3.3597468319049186, + "tokens_seen": 2879586304 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010315747241725175, + "loss": 1.0906, + "theoretical_loss": 3.3597409541183514, + "tokens_seen": 2879651840 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010314142427281845, + "loss": 1.0137, + "theoretical_loss": 3.3597350765030054, + "tokens_seen": 2879717376 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010312537612838515, + "loss": 1.2805, + "theoretical_loss": 3.359729199058872, + "tokens_seen": 2879782912 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010310932798395185, + "loss": 1.0843, + "theoretical_loss": 3.3597233217859417, + "tokens_seen": 2879848448 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010309327983951855, + "loss": 1.1597, + "theoretical_loss": 3.3597174446842057, + "tokens_seen": 2879913984 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010307723169508525, + "loss": 1.1723, + "theoretical_loss": 3.3597115677536555, + "tokens_seen": 2879979520 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010306118355065195, + "loss": 1.1834, + "theoretical_loss": 3.359705690994282, + "tokens_seen": 2880045056 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010304513540621865, + "loss": 1.041, + "theoretical_loss": 3.359699814406077, + "tokens_seen": 2880110592 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010302908726178535, + "loss": 1.0916, + "theoretical_loss": 3.3596939379890305, + "tokens_seen": 2880176128 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010301303911735206, + "loss": 1.1948, + "theoretical_loss": 3.3596880617431335, + "tokens_seen": 2880241664 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 960876, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1923925876617432, + "objective/train/theoretical_loss": 3.3596821856683787, + "objective/train/tokens_used": 2900767200, + "theoretical_loss": 3.3596821856683787, + "tokens_seen": 2880307200 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010299699097291877, + "loss": 1.1123, + "theoretical_loss": 3.3596821856683787, + "tokens_seen": 2880307200 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010298094282848547, + "loss": 1.2128, + "theoretical_loss": 3.3596763097647555, + "tokens_seen": 2880372736 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010296489468405217, + "loss": 1.2198, + "theoretical_loss": 3.359670434032256, + "tokens_seen": 2880438272 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010294884653961887, + "loss": 1.3133, + "theoretical_loss": 3.359664558470871, + "tokens_seen": 2880503808 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010293279839518557, + "loss": 1.2215, + "theoretical_loss": 3.3596586830805917, + "tokens_seen": 2880569344 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010291675025075227, + "loss": 1.1957, + "theoretical_loss": 3.3596528078614094, + "tokens_seen": 2880634880 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010290070210631897, + "loss": 1.1414, + "theoretical_loss": 3.359646932813315, + "tokens_seen": 2880700416 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010288465396188567, + "loss": 1.1374, + "theoretical_loss": 3.3596410579363, + "tokens_seen": 2880765952 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010286860581745237, + "loss": 1.1874, + "theoretical_loss": 3.359635183230355, + "tokens_seen": 2880831488 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010285255767301906, + "loss": 1.1498, + "theoretical_loss": 3.359629308695471, + "tokens_seen": 2880897024 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010283650952858576, + "loss": 1.1875, + "theoretical_loss": 3.3596234343316396, + "tokens_seen": 2880962560 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010282046138415246, + "loss": 1.1763, + "theoretical_loss": 3.359617560138852, + "tokens_seen": 2881028096 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010280441323971916, + "loss": 1.0839, + "theoretical_loss": 3.359611686117099, + "tokens_seen": 2881093632 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010278836509528586, + "loss": 1.2201, + "theoretical_loss": 3.3596058122663717, + "tokens_seen": 2881159168 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010277231695085257, + "loss": 1.1389, + "theoretical_loss": 3.3595999385866615, + "tokens_seen": 2881224704 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010275626880641927, + "loss": 0.9971, + "theoretical_loss": 3.3595940650779594, + "tokens_seen": 2881290240 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010274022066198597, + "loss": 1.1612, + "theoretical_loss": 3.3595881917402566, + "tokens_seen": 2881355776 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010272417251755267, + "loss": 1.2166, + "theoretical_loss": 3.359582318573544, + "tokens_seen": 2881421312 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010270812437311937, + "loss": 1.2089, + "theoretical_loss": 3.359576445577813, + "tokens_seen": 2881486848 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010269207622868607, + "loss": 1.0906, + "theoretical_loss": 3.359570572753055, + "tokens_seen": 2881552384 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010267602808425277, + "loss": 1.1374, + "theoretical_loss": 3.35956470009926, + "tokens_seen": 2881617920 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010265997993981946, + "loss": 0.9683, + "theoretical_loss": 3.3595588276164206, + "tokens_seen": 2881683456 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010264393179538616, + "loss": 1.1308, + "theoretical_loss": 3.359552955304527, + "tokens_seen": 2881748992 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010262788365095286, + "loss": 1.1474, + "theoretical_loss": 3.3595470831635708, + "tokens_seen": 2881814528 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010261183550651956, + "loss": 1.1679, + "theoretical_loss": 3.3595412111935428, + "tokens_seen": 2881880064 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010259578736208626, + "loss": 1.1409, + "theoretical_loss": 3.359535339394434, + "tokens_seen": 2881945600 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010257973921765296, + "loss": 1.1297, + "theoretical_loss": 3.3595294677662357, + "tokens_seen": 2882011136 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010256369107321966, + "loss": 1.1378, + "theoretical_loss": 3.3595235963089394, + "tokens_seen": 2882076672 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010254764292878636, + "loss": 1.1882, + "theoretical_loss": 3.359517725022536, + "tokens_seen": 2882142208 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010253159478435306, + "loss": 1.2019, + "theoretical_loss": 3.359511853907017, + "tokens_seen": 2882207744 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010251554663991976, + "loss": 1.2674, + "theoretical_loss": 3.3595059829623724, + "tokens_seen": 2882273280 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010249949849548646, + "loss": 1.3716, + "theoretical_loss": 3.3595001121885946, + "tokens_seen": 2882338816 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010248345035105316, + "loss": 1.1229, + "theoretical_loss": 3.359494241585674, + "tokens_seen": 2882404352 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010246740220661985, + "loss": 1.124, + "theoretical_loss": 3.3594883711536023, + "tokens_seen": 2882469888 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010245135406218655, + "loss": 1.2209, + "theoretical_loss": 3.35948250089237, + "tokens_seen": 2882535424 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010243530591775325, + "loss": 1.1782, + "theoretical_loss": 3.359476630801969, + "tokens_seen": 2882600960 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010241925777331995, + "loss": 1.1891, + "theoretical_loss": 3.3594707608823895, + "tokens_seen": 2882666496 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010240320962888665, + "loss": 1.1753, + "theoretical_loss": 3.3594648911336233, + "tokens_seen": 2882732032 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010238716148445338, + "loss": 1.0708, + "theoretical_loss": 3.3594590215556615, + "tokens_seen": 2882797568 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010237111334002008, + "loss": 1.1716, + "theoretical_loss": 3.359453152148495, + "tokens_seen": 2882863104 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010235506519558677, + "loss": 1.1384, + "theoretical_loss": 3.3594472829121154, + "tokens_seen": 2882928640 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010233901705115347, + "loss": 1.1542, + "theoretical_loss": 3.359441413846513, + "tokens_seen": 2882994176 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010232296890672017, + "loss": 1.1119, + "theoretical_loss": 3.35943554495168, + "tokens_seen": 2883059712 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010230692076228687, + "loss": 1.2465, + "theoretical_loss": 3.3594296762276072, + "tokens_seen": 2883125248 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010229087261785357, + "loss": 1.116, + "theoretical_loss": 3.3594238076742853, + "tokens_seen": 2883190784 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010227482447342027, + "loss": 1.1916, + "theoretical_loss": 3.359417939291706, + "tokens_seen": 2883256320 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010225877632898697, + "loss": 1.1203, + "theoretical_loss": 3.3594120710798596, + "tokens_seen": 2883321856 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010224272818455367, + "loss": 1.1085, + "theoretical_loss": 3.359406203038738, + "tokens_seen": 2883387392 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010222668004012037, + "loss": 1.109, + "theoretical_loss": 3.359400335168333, + "tokens_seen": 2883452928 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010221063189568707, + "loss": 1.0242, + "theoretical_loss": 3.359394467468634, + "tokens_seen": 2883518464 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 961829, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.284890055656433, + "objective/train/theoretical_loss": 3.3593885999396336, + "objective/train/tokens_used": 2904044000, + "theoretical_loss": 3.3593885999396336, + "tokens_seen": 2883584000 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010219458375125377, + "loss": 1.2539, + "theoretical_loss": 3.3593885999396336, + "tokens_seen": 2883584000 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010217853560682047, + "loss": 1.2065, + "theoretical_loss": 3.3593827325813224, + "tokens_seen": 2883649536 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010216248746238716, + "loss": 1.1578, + "theoretical_loss": 3.3593768653936915, + "tokens_seen": 2883715072 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010214643931795386, + "loss": 1.26, + "theoretical_loss": 3.3593709983767326, + "tokens_seen": 2883780608 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010213039117352056, + "loss": 1.0748, + "theoretical_loss": 3.359365131530436, + "tokens_seen": 2883846144 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010211434302908726, + "loss": 1.1519, + "theoretical_loss": 3.359359264854793, + "tokens_seen": 2883911680 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010209829488465396, + "loss": 1.2984, + "theoretical_loss": 3.359353398349796, + "tokens_seen": 2883977216 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010208224674022067, + "loss": 1.1458, + "theoretical_loss": 3.359347532015435, + "tokens_seen": 2884042752 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010206619859578737, + "loss": 1.1595, + "theoretical_loss": 3.3593416658517006, + "tokens_seen": 2884108288 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010205015045135407, + "loss": 1.2796, + "theoretical_loss": 3.3593357998585853, + "tokens_seen": 2884173824 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010203410230692077, + "loss": 1.032, + "theoretical_loss": 3.3593299340360794, + "tokens_seen": 2884239360 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010201805416248747, + "loss": 1.3192, + "theoretical_loss": 3.3593240683841747, + "tokens_seen": 2884304896 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010200200601805417, + "loss": 1.1531, + "theoretical_loss": 3.3593182029028616, + "tokens_seen": 2884370432 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010198595787362087, + "loss": 1.279, + "theoretical_loss": 3.359312337592132, + "tokens_seen": 2884435968 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010196990972918756, + "loss": 1.1231, + "theoretical_loss": 3.3593064724519768, + "tokens_seen": 2884501504 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010195386158475426, + "loss": 1.1557, + "theoretical_loss": 3.3593006074823872, + "tokens_seen": 2884567040 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010193781344032096, + "loss": 1.0206, + "theoretical_loss": 3.359294742683354, + "tokens_seen": 2884632576 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010192176529588766, + "loss": 1.0825, + "theoretical_loss": 3.3592888780548686, + "tokens_seen": 2884698112 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010190571715145436, + "loss": 1.1686, + "theoretical_loss": 3.3592830135969223, + "tokens_seen": 2884763648 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010188966900702106, + "loss": 1.0817, + "theoretical_loss": 3.359277149309506, + "tokens_seen": 2884829184 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010187362086258776, + "loss": 1.1812, + "theoretical_loss": 3.3592712851926114, + "tokens_seen": 2884894720 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010185757271815446, + "loss": 1.143, + "theoretical_loss": 3.359265421246229, + "tokens_seen": 2884960256 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010184152457372116, + "loss": 1.147, + "theoretical_loss": 3.3592595574703505, + "tokens_seen": 2885025792 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010182547642928786, + "loss": 1.0703, + "theoretical_loss": 3.359253693864967, + "tokens_seen": 2885091328 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010180942828485456, + "loss": 1.2512, + "theoretical_loss": 3.359247830430069, + "tokens_seen": 2885156864 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010179338014042128, + "loss": 1.2436, + "theoretical_loss": 3.3592419671656484, + "tokens_seen": 2885222400 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010177733199598798, + "loss": 1.2456, + "theoretical_loss": 3.3592361040716963, + "tokens_seen": 2885287936 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010176128385155468, + "loss": 1.1007, + "theoretical_loss": 3.359230241148204, + "tokens_seen": 2885353472 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010174523570712138, + "loss": 1.1008, + "theoretical_loss": 3.3592243783951616, + "tokens_seen": 2885419008 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010172918756268808, + "loss": 1.0814, + "theoretical_loss": 3.359218515812562, + "tokens_seen": 2885484544 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010171313941825478, + "loss": 1.1619, + "theoretical_loss": 3.3592126534003945, + "tokens_seen": 2885550080 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010169709127382148, + "loss": 1.2176, + "theoretical_loss": 3.359206791158652, + "tokens_seen": 2885615616 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010168104312938818, + "loss": 1.138, + "theoretical_loss": 3.3592009290873244, + "tokens_seen": 2885681152 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010166499498495488, + "loss": 1.2019, + "theoretical_loss": 3.3591950671864033, + "tokens_seen": 2885746688 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010164894684052157, + "loss": 1.1725, + "theoretical_loss": 3.3591892054558805, + "tokens_seen": 2885812224 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010163289869608827, + "loss": 1.1894, + "theoretical_loss": 3.3591833438957464, + "tokens_seen": 2885877760 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010161685055165497, + "loss": 1.1086, + "theoretical_loss": 3.359177482505992, + "tokens_seen": 2885943296 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010160080240722167, + "loss": 1.1681, + "theoretical_loss": 3.359171621286609, + "tokens_seen": 2886008832 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010158475426278837, + "loss": 1.1631, + "theoretical_loss": 3.359165760237589, + "tokens_seen": 2886074368 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010156870611835507, + "loss": 1.103, + "theoretical_loss": 3.359159899358922, + "tokens_seen": 2886139904 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010155265797392177, + "loss": 1.1244, + "theoretical_loss": 3.3591540386506002, + "tokens_seen": 2886205440 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010153660982948847, + "loss": 1.2388, + "theoretical_loss": 3.3591481781126147, + "tokens_seen": 2886270976 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010152056168505517, + "loss": 1.162, + "theoretical_loss": 3.359142317744956, + "tokens_seen": 2886336512 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010150451354062187, + "loss": 1.1435, + "theoretical_loss": 3.3591364575476153, + "tokens_seen": 2886402048 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010148846539618858, + "loss": 1.181, + "theoretical_loss": 3.3591305975205845, + "tokens_seen": 2886467584 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010147241725175528, + "loss": 1.1873, + "theoretical_loss": 3.359124737663855, + "tokens_seen": 2886533120 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010145636910732196, + "loss": 1.1163, + "theoretical_loss": 3.3591188779774166, + "tokens_seen": 2886598656 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010144032096288866, + "loss": 1.1394, + "theoretical_loss": 3.3591130184612616, + "tokens_seen": 2886664192 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010142427281845536, + "loss": 1.1739, + "theoretical_loss": 3.3591071591153807, + "tokens_seen": 2886729728 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010140822467402206, + "loss": 1.2532, + "theoretical_loss": 3.359101299939766, + "tokens_seen": 2886795264 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 963081, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3193817138671875, + "objective/train/theoretical_loss": 3.359095440934407, + "objective/train/tokens_used": 2907320800, + "theoretical_loss": 3.359095440934407, + "tokens_seen": 2886860800 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010139217652958877, + "loss": 1.2113, + "theoretical_loss": 3.359095440934407, + "tokens_seen": 2886860800 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010137612838515547, + "loss": 1.1604, + "theoretical_loss": 3.3590895820992963, + "tokens_seen": 2886926336 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010136008024072217, + "loss": 1.2275, + "theoretical_loss": 3.3590837234344244, + "tokens_seen": 2886991872 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010134403209628887, + "loss": 1.1334, + "theoretical_loss": 3.359077864939783, + "tokens_seen": 2887057408 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010132798395185557, + "loss": 1.1523, + "theoretical_loss": 3.359072006615363, + "tokens_seen": 2887122944 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010131193580742227, + "loss": 1.0487, + "theoretical_loss": 3.3590661484611557, + "tokens_seen": 2887188480 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010129588766298897, + "loss": 1.2449, + "theoretical_loss": 3.359060290477152, + "tokens_seen": 2887254016 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010127983951855567, + "loss": 1.1939, + "theoretical_loss": 3.359054432663344, + "tokens_seen": 2887319552 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010126379137412236, + "loss": 1.1485, + "theoretical_loss": 3.359048575019721, + "tokens_seen": 2887385088 + }, + { + "epoch": 0.87, + "learning_rate": 0.00010124774322968906, + "loss": 1.1034, + "theoretical_loss": 3.3590427175462763, + "tokens_seen": 2887450624 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010123169508525576, + "loss": 1.1324, + "theoretical_loss": 3.3590368602429996, + "tokens_seen": 2887516160 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010121564694082246, + "loss": 1.1331, + "theoretical_loss": 3.359031003109883, + "tokens_seen": 2887581696 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010119959879638919, + "loss": 1.1271, + "theoretical_loss": 3.359025146146917, + "tokens_seen": 2887647232 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010118355065195589, + "loss": 1.1674, + "theoretical_loss": 3.3590192893540936, + "tokens_seen": 2887712768 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010116750250752259, + "loss": 1.2357, + "theoretical_loss": 3.3590134327314036, + "tokens_seen": 2887778304 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010115145436308929, + "loss": 1.1953, + "theoretical_loss": 3.3590075762788376, + "tokens_seen": 2887843840 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010113540621865598, + "loss": 1.2259, + "theoretical_loss": 3.359001719996388, + "tokens_seen": 2887909376 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010111935807422268, + "loss": 1.3, + "theoretical_loss": 3.3589958638840445, + "tokens_seen": 2887974912 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010110330992978938, + "loss": 1.1808, + "theoretical_loss": 3.3589900079417996, + "tokens_seen": 2888040448 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010108726178535608, + "loss": 1.167, + "theoretical_loss": 3.3589841521696444, + "tokens_seen": 2888105984 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010107121364092278, + "loss": 1.152, + "theoretical_loss": 3.358978296567569, + "tokens_seen": 2888171520 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010105516549648948, + "loss": 1.166, + "theoretical_loss": 3.3589724411355664, + "tokens_seen": 2888237056 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010103911735205618, + "loss": 1.1236, + "theoretical_loss": 3.3589665858736257, + "tokens_seen": 2888302592 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010102306920762288, + "loss": 1.0816, + "theoretical_loss": 3.3589607307817397, + "tokens_seen": 2888368128 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010100702106318958, + "loss": 1.2072, + "theoretical_loss": 3.358954875859899, + "tokens_seen": 2888433664 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010099097291875628, + "loss": 1.1786, + "theoretical_loss": 3.358949021108095, + "tokens_seen": 2888499200 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010097492477432298, + "loss": 1.2034, + "theoretical_loss": 3.358943166526318, + "tokens_seen": 2888564736 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010095887662988968, + "loss": 1.1703, + "theoretical_loss": 3.358937312114561, + "tokens_seen": 2888630272 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010094282848545637, + "loss": 1.2116, + "theoretical_loss": 3.3589314578728136, + "tokens_seen": 2888695808 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010092678034102307, + "loss": 1.2435, + "theoretical_loss": 3.3589256038010675, + "tokens_seen": 2888761344 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010091073219658977, + "loss": 1.2237, + "theoretical_loss": 3.3589197498993144, + "tokens_seen": 2888826880 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010089468405215647, + "loss": 1.0555, + "theoretical_loss": 3.3589138961675444, + "tokens_seen": 2888892416 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010087863590772317, + "loss": 1.2625, + "theoretical_loss": 3.35890804260575, + "tokens_seen": 2888957952 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010086258776328987, + "loss": 1.2011, + "theoretical_loss": 3.3589021892139215, + "tokens_seen": 2889023488 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010084653961885657, + "loss": 1.2322, + "theoretical_loss": 3.3588963359920507, + "tokens_seen": 2889089024 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010083049147442327, + "loss": 1.1787, + "theoretical_loss": 3.3588904829401285, + "tokens_seen": 2889154560 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010081444332998997, + "loss": 1.246, + "theoretical_loss": 3.358884630058146, + "tokens_seen": 2889220096 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010079839518555668, + "loss": 1.1784, + "theoretical_loss": 3.3588787773460944, + "tokens_seen": 2889285632 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010078234704112338, + "loss": 1.2014, + "theoretical_loss": 3.3588729248039653, + "tokens_seen": 2889351168 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010076629889669008, + "loss": 1.2418, + "theoretical_loss": 3.3588670724317495, + "tokens_seen": 2889416704 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010075025075225676, + "loss": 1.1478, + "theoretical_loss": 3.3588612202294383, + "tokens_seen": 2889482240 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010073420260782346, + "loss": 1.2482, + "theoretical_loss": 3.3588553681970232, + "tokens_seen": 2889547776 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010071815446339016, + "loss": 1.1873, + "theoretical_loss": 3.3588495163344954, + "tokens_seen": 2889613312 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010070210631895687, + "loss": 1.0628, + "theoretical_loss": 3.358843664641846, + "tokens_seen": 2889678848 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010068605817452357, + "loss": 1.067, + "theoretical_loss": 3.3588378131190657, + "tokens_seen": 2889744384 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010067001003009027, + "loss": 1.189, + "theoretical_loss": 3.358831961766146, + "tokens_seen": 2889809920 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010065396188565697, + "loss": 1.1143, + "theoretical_loss": 3.358826110583079, + "tokens_seen": 2889875456 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010063791374122367, + "loss": 1.2235, + "theoretical_loss": 3.3588202595698546, + "tokens_seen": 2889940992 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010062186559679037, + "loss": 1.2695, + "theoretical_loss": 3.358814408726465, + "tokens_seen": 2890006528 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010060581745235707, + "loss": 1.3426, + "theoretical_loss": 3.3588085580529006, + "tokens_seen": 2890072064 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 963977, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1469203233718872, + "objective/train/theoretical_loss": 3.3588027075491533, + "objective/train/tokens_used": 2910597600, + "theoretical_loss": 3.3588027075491533, + "tokens_seen": 2890137600 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010058976930792378, + "loss": 1.1645, + "theoretical_loss": 3.3588027075491533, + "tokens_seen": 2890137600 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010057372116349048, + "loss": 1.2199, + "theoretical_loss": 3.3587968572152143, + "tokens_seen": 2890203136 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010055767301905718, + "loss": 1.1907, + "theoretical_loss": 3.358791007051074, + "tokens_seen": 2890268672 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010054162487462389, + "loss": 1.1582, + "theoretical_loss": 3.358785157056725, + "tokens_seen": 2890334208 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010052557673019059, + "loss": 1.3296, + "theoretical_loss": 3.3587793072321572, + "tokens_seen": 2890399744 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010050952858575729, + "loss": 1.1953, + "theoretical_loss": 3.3587734575773625, + "tokens_seen": 2890465280 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010049348044132399, + "loss": 1.216, + "theoretical_loss": 3.358767608092332, + "tokens_seen": 2890530816 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010047743229689069, + "loss": 1.1447, + "theoretical_loss": 3.3587617587770566, + "tokens_seen": 2890596352 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010046138415245739, + "loss": 1.2829, + "theoretical_loss": 3.3587559096315283, + "tokens_seen": 2890661888 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010044533600802408, + "loss": 1.1279, + "theoretical_loss": 3.358750060655738, + "tokens_seen": 2890727424 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010042928786359078, + "loss": 1.1715, + "theoretical_loss": 3.358744211849676, + "tokens_seen": 2890792960 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010041323971915748, + "loss": 1.1222, + "theoretical_loss": 3.358738363213335, + "tokens_seen": 2890858496 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010039719157472418, + "loss": 1.1222, + "theoretical_loss": 3.3587325147467055, + "tokens_seen": 2890924032 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010038114343029088, + "loss": 1.2194, + "theoretical_loss": 3.358726666449779, + "tokens_seen": 2890989568 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010036509528585758, + "loss": 1.1893, + "theoretical_loss": 3.3587208183225457, + "tokens_seen": 2891055104 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010034904714142428, + "loss": 1.2239, + "theoretical_loss": 3.358714970364998, + "tokens_seen": 2891120640 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010033299899699098, + "loss": 1.2196, + "theoretical_loss": 3.358709122577127, + "tokens_seen": 2891186176 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010031695085255768, + "loss": 1.2232, + "theoretical_loss": 3.3587032749589234, + "tokens_seen": 2891251712 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010030090270812438, + "loss": 1.2754, + "theoretical_loss": 3.3586974275103785, + "tokens_seen": 2891317248 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010028485456369108, + "loss": 1.2484, + "theoretical_loss": 3.3586915802314845, + "tokens_seen": 2891382784 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010026880641925778, + "loss": 1.2761, + "theoretical_loss": 3.3586857331222313, + "tokens_seen": 2891448320 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010025275827482447, + "loss": 1.2021, + "theoretical_loss": 3.358679886182611, + "tokens_seen": 2891513856 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010023671013039117, + "loss": 1.2829, + "theoretical_loss": 3.358674039412614, + "tokens_seen": 2891579392 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010022066198595787, + "loss": 1.1779, + "theoretical_loss": 3.3586681928122326, + "tokens_seen": 2891644928 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010020461384152457, + "loss": 1.1978, + "theoretical_loss": 3.3586623463814576, + "tokens_seen": 2891710464 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010018856569709127, + "loss": 1.184, + "theoretical_loss": 3.35865650012028, + "tokens_seen": 2891776000 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010017251755265797, + "loss": 1.1424, + "theoretical_loss": 3.3586506540286916, + "tokens_seen": 2891841536 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010015646940822467, + "loss": 1.1755, + "theoretical_loss": 3.358644808106683, + "tokens_seen": 2891907072 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010014042126379137, + "loss": 1.1801, + "theoretical_loss": 3.358638962354245, + "tokens_seen": 2891972608 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010012437311935807, + "loss": 1.2806, + "theoretical_loss": 3.3586331167713706, + "tokens_seen": 2892038144 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010010832497492478, + "loss": 1.1313, + "theoretical_loss": 3.358627271358049, + "tokens_seen": 2892103680 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010009227683049148, + "loss": 1.1835, + "theoretical_loss": 3.358621426114273, + "tokens_seen": 2892169216 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010007622868605818, + "loss": 1.153, + "theoretical_loss": 3.358615581040033, + "tokens_seen": 2892234752 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010006018054162486, + "loss": 1.0942, + "theoretical_loss": 3.35860973613532, + "tokens_seen": 2892300288 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010004413239719156, + "loss": 1.3001, + "theoretical_loss": 3.3586038914001266, + "tokens_seen": 2892365824 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010002808425275827, + "loss": 1.0866, + "theoretical_loss": 3.3585980468344427, + "tokens_seen": 2892431360 + }, + { + "epoch": 0.88, + "learning_rate": 0.00010001203610832497, + "loss": 1.1694, + "theoretical_loss": 3.35859220243826, + "tokens_seen": 2892496896 + }, + { + "epoch": 0.88, + "learning_rate": 9.999598796389168e-05, + "loss": 1.1454, + "theoretical_loss": 3.35858635821157, + "tokens_seen": 2892562432 + }, + { + "epoch": 0.88, + "learning_rate": 9.997993981945838e-05, + "loss": 1.2082, + "theoretical_loss": 3.3585805141543634, + "tokens_seen": 2892627968 + }, + { + "epoch": 0.88, + "learning_rate": 9.996389167502508e-05, + "loss": 1.2157, + "theoretical_loss": 3.358574670266632, + "tokens_seen": 2892693504 + }, + { + "epoch": 0.88, + "learning_rate": 9.994784353059178e-05, + "loss": 1.1965, + "theoretical_loss": 3.3585688265483666, + "tokens_seen": 2892759040 + }, + { + "epoch": 0.88, + "learning_rate": 9.993179538615848e-05, + "loss": 1.1482, + "theoretical_loss": 3.3585629829995587, + "tokens_seen": 2892824576 + }, + { + "epoch": 0.88, + "learning_rate": 9.991574724172518e-05, + "loss": 1.1579, + "theoretical_loss": 3.358557139620199, + "tokens_seen": 2892890112 + }, + { + "epoch": 0.88, + "learning_rate": 9.989969909729187e-05, + "loss": 1.192, + "theoretical_loss": 3.35855129641028, + "tokens_seen": 2892955648 + }, + { + "epoch": 0.88, + "learning_rate": 9.988365095285857e-05, + "loss": 1.2253, + "theoretical_loss": 3.358545453369792, + "tokens_seen": 2893021184 + }, + { + "epoch": 0.88, + "learning_rate": 9.986760280842527e-05, + "loss": 1.2297, + "theoretical_loss": 3.358539610498726, + "tokens_seen": 2893086720 + }, + { + "epoch": 0.88, + "learning_rate": 9.985155466399197e-05, + "loss": 1.1731, + "theoretical_loss": 3.3585337677970744, + "tokens_seen": 2893152256 + }, + { + "epoch": 0.88, + "learning_rate": 9.983550651955869e-05, + "loss": 1.1754, + "theoretical_loss": 3.358527925264827, + "tokens_seen": 2893217792 + }, + { + "epoch": 0.88, + "learning_rate": 9.981945837512539e-05, + "loss": 1.1473, + "theoretical_loss": 3.358522082901976, + "tokens_seen": 2893283328 + }, + { + "epoch": 0.88, + "learning_rate": 9.980341023069209e-05, + "loss": 1.2847, + "theoretical_loss": 3.3585162407085125, + "tokens_seen": 2893348864 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 965234, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1895378828048706, + "objective/train/theoretical_loss": 3.3585103986844276, + "objective/train/tokens_used": 2913874400, + "theoretical_loss": 3.3585103986844276, + "tokens_seen": 2893414400 + }, + { + "epoch": 0.88, + "learning_rate": 9.978736208625879e-05, + "loss": 1.2178, + "theoretical_loss": 3.3585103986844276, + "tokens_seen": 2893414400 + }, + { + "epoch": 0.88, + "learning_rate": 9.977131394182549e-05, + "loss": 1.1901, + "theoretical_loss": 3.358504556829713, + "tokens_seen": 2893479936 + }, + { + "epoch": 0.88, + "learning_rate": 9.975526579739219e-05, + "loss": 1.1986, + "theoretical_loss": 3.358498715144359, + "tokens_seen": 2893545472 + }, + { + "epoch": 0.88, + "learning_rate": 9.973921765295888e-05, + "loss": 1.2356, + "theoretical_loss": 3.3584928736283577, + "tokens_seen": 2893611008 + }, + { + "epoch": 0.88, + "learning_rate": 9.972316950852558e-05, + "loss": 1.1464, + "theoretical_loss": 3.3584870322817, + "tokens_seen": 2893676544 + }, + { + "epoch": 0.88, + "learning_rate": 9.970712136409228e-05, + "loss": 1.1807, + "theoretical_loss": 3.3584811911043775, + "tokens_seen": 2893742080 + }, + { + "epoch": 0.88, + "learning_rate": 9.969107321965898e-05, + "loss": 1.1803, + "theoretical_loss": 3.358475350096381, + "tokens_seen": 2893807616 + }, + { + "epoch": 0.88, + "learning_rate": 9.967502507522568e-05, + "loss": 1.1452, + "theoretical_loss": 3.358469509257702, + "tokens_seen": 2893873152 + }, + { + "epoch": 0.88, + "learning_rate": 9.965897693079238e-05, + "loss": 1.1541, + "theoretical_loss": 3.358463668588332, + "tokens_seen": 2893938688 + }, + { + "epoch": 0.88, + "learning_rate": 9.964292878635908e-05, + "loss": 1.3072, + "theoretical_loss": 3.3584578280882615, + "tokens_seen": 2894004224 + }, + { + "epoch": 0.88, + "learning_rate": 9.962688064192578e-05, + "loss": 1.1243, + "theoretical_loss": 3.3584519877574825, + "tokens_seen": 2894069760 + }, + { + "epoch": 0.88, + "learning_rate": 9.961083249749248e-05, + "loss": 1.1869, + "theoretical_loss": 3.358446147595986, + "tokens_seen": 2894135296 + }, + { + "epoch": 0.88, + "learning_rate": 9.959478435305918e-05, + "loss": 1.1467, + "theoretical_loss": 3.3584403076037628, + "tokens_seen": 2894200832 + }, + { + "epoch": 0.88, + "learning_rate": 9.957873620862588e-05, + "loss": 1.2844, + "theoretical_loss": 3.358434467780805, + "tokens_seen": 2894266368 + }, + { + "epoch": 0.88, + "learning_rate": 9.956268806419258e-05, + "loss": 1.2979, + "theoretical_loss": 3.3584286281271036, + "tokens_seen": 2894331904 + }, + { + "epoch": 0.88, + "learning_rate": 9.954663991975928e-05, + "loss": 1.3834, + "theoretical_loss": 3.35842278864265, + "tokens_seen": 2894397440 + }, + { + "epoch": 0.88, + "learning_rate": 9.953059177532598e-05, + "loss": 1.09, + "theoretical_loss": 3.3584169493274345, + "tokens_seen": 2894462976 + }, + { + "epoch": 0.88, + "learning_rate": 9.951454363089269e-05, + "loss": 1.1939, + "theoretical_loss": 3.3584111101814496, + "tokens_seen": 2894528512 + }, + { + "epoch": 0.88, + "learning_rate": 9.949849548645939e-05, + "loss": 1.2137, + "theoretical_loss": 3.3584052712046857, + "tokens_seen": 2894594048 + }, + { + "epoch": 0.88, + "learning_rate": 9.948244734202609e-05, + "loss": 1.1608, + "theoretical_loss": 3.3583994323971345, + "tokens_seen": 2894659584 + }, + { + "epoch": 0.88, + "learning_rate": 9.946639919759279e-05, + "loss": 1.1054, + "theoretical_loss": 3.3583935937587874, + "tokens_seen": 2894725120 + }, + { + "epoch": 0.88, + "learning_rate": 9.945035105315949e-05, + "loss": 1.1784, + "theoretical_loss": 3.358387755289635, + "tokens_seen": 2894790656 + }, + { + "epoch": 0.88, + "learning_rate": 9.943430290872619e-05, + "loss": 1.1964, + "theoretical_loss": 3.358381916989669, + "tokens_seen": 2894856192 + }, + { + "epoch": 0.88, + "learning_rate": 9.941825476429289e-05, + "loss": 1.2449, + "theoretical_loss": 3.358376078858881, + "tokens_seen": 2894921728 + }, + { + "epoch": 0.88, + "learning_rate": 9.940220661985959e-05, + "loss": 1.1951, + "theoretical_loss": 3.358370240897262, + "tokens_seen": 2894987264 + }, + { + "epoch": 0.88, + "learning_rate": 9.938615847542628e-05, + "loss": 1.197, + "theoretical_loss": 3.3583644031048028, + "tokens_seen": 2895052800 + }, + { + "epoch": 0.88, + "learning_rate": 9.937011033099298e-05, + "loss": 1.2553, + "theoretical_loss": 3.3583585654814954, + "tokens_seen": 2895118336 + }, + { + "epoch": 0.88, + "learning_rate": 9.935406218655968e-05, + "loss": 1.3215, + "theoretical_loss": 3.358352728027331, + "tokens_seen": 2895183872 + }, + { + "epoch": 0.88, + "learning_rate": 9.933801404212638e-05, + "loss": 1.2349, + "theoretical_loss": 3.3583468907423, + "tokens_seen": 2895249408 + }, + { + "epoch": 0.88, + "learning_rate": 9.932196589769308e-05, + "loss": 1.2138, + "theoretical_loss": 3.3583410536263942, + "tokens_seen": 2895314944 + }, + { + "epoch": 0.88, + "learning_rate": 9.930591775325978e-05, + "loss": 1.1641, + "theoretical_loss": 3.358335216679605, + "tokens_seen": 2895380480 + }, + { + "epoch": 0.88, + "learning_rate": 9.928986960882648e-05, + "loss": 1.1268, + "theoretical_loss": 3.3583293799019245, + "tokens_seen": 2895446016 + }, + { + "epoch": 0.88, + "learning_rate": 9.927382146439318e-05, + "loss": 1.2055, + "theoretical_loss": 3.3583235432933423, + "tokens_seen": 2895511552 + }, + { + "epoch": 0.88, + "learning_rate": 9.925777331995988e-05, + "loss": 1.1998, + "theoretical_loss": 3.3583177068538506, + "tokens_seen": 2895577088 + }, + { + "epoch": 0.88, + "learning_rate": 9.92417251755266e-05, + "loss": 1.1777, + "theoretical_loss": 3.3583118705834405, + "tokens_seen": 2895642624 + }, + { + "epoch": 0.88, + "learning_rate": 9.922567703109328e-05, + "loss": 1.1674, + "theoretical_loss": 3.3583060344821036, + "tokens_seen": 2895708160 + }, + { + "epoch": 0.88, + "learning_rate": 9.920962888665998e-05, + "loss": 1.1252, + "theoretical_loss": 3.3583001985498306, + "tokens_seen": 2895773696 + }, + { + "epoch": 0.88, + "learning_rate": 9.919358074222668e-05, + "loss": 1.2196, + "theoretical_loss": 3.358294362786613, + "tokens_seen": 2895839232 + }, + { + "epoch": 0.88, + "learning_rate": 9.917753259779339e-05, + "loss": 1.1816, + "theoretical_loss": 3.3582885271924425, + "tokens_seen": 2895904768 + }, + { + "epoch": 0.88, + "learning_rate": 9.916148445336009e-05, + "loss": 1.2148, + "theoretical_loss": 3.35828269176731, + "tokens_seen": 2895970304 + }, + { + "epoch": 0.88, + "learning_rate": 9.914543630892679e-05, + "loss": 1.1916, + "theoretical_loss": 3.3582768565112064, + "tokens_seen": 2896035840 + }, + { + "epoch": 0.88, + "learning_rate": 9.912938816449349e-05, + "loss": 1.2607, + "theoretical_loss": 3.358271021424124, + "tokens_seen": 2896101376 + }, + { + "epoch": 0.88, + "learning_rate": 9.911334002006019e-05, + "loss": 1.2065, + "theoretical_loss": 3.3582651865060527, + "tokens_seen": 2896166912 + }, + { + "epoch": 0.88, + "learning_rate": 9.909729187562689e-05, + "loss": 1.2418, + "theoretical_loss": 3.358259351756985, + "tokens_seen": 2896232448 + }, + { + "epoch": 0.88, + "learning_rate": 9.908124373119359e-05, + "loss": 1.2522, + "theoretical_loss": 3.358253517176912, + "tokens_seen": 2896297984 + }, + { + "epoch": 0.88, + "learning_rate": 9.906519558676029e-05, + "loss": 1.2795, + "theoretical_loss": 3.3582476827658243, + "tokens_seen": 2896363520 + }, + { + "epoch": 0.88, + "learning_rate": 9.904914744232699e-05, + "loss": 1.1551, + "theoretical_loss": 3.3582418485237135, + "tokens_seen": 2896429056 + }, + { + "epoch": 0.88, + "learning_rate": 9.903309929789368e-05, + "loss": 1.2932, + "theoretical_loss": 3.358236014450571, + "tokens_seen": 2896494592 + }, + { + "epoch": 0.88, + "learning_rate": 9.901705115346038e-05, + "loss": 1.1127, + "theoretical_loss": 3.3582301805463883, + "tokens_seen": 2896560128 + }, + { + "epoch": 0.88, + "learning_rate": 9.900100300902708e-05, + "loss": 1.203, + "theoretical_loss": 3.3582243468111566, + "tokens_seen": 2896625664 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 966107, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0807238817214966, + "objective/train/theoretical_loss": 3.358218513244867, + "objective/train/tokens_used": 2917151200, + "theoretical_loss": 3.358218513244867, + "tokens_seen": 2896691200 + }, + { + "epoch": 0.88, + "learning_rate": 9.898495486459378e-05, + "loss": 1.1857, + "theoretical_loss": 3.358218513244867, + "tokens_seen": 2896691200 + }, + { + "epoch": 0.88, + "learning_rate": 9.896890672016048e-05, + "loss": 1.1453, + "theoretical_loss": 3.3582126798475103, + "tokens_seen": 2896756736 + }, + { + "epoch": 0.88, + "learning_rate": 9.895285857572718e-05, + "loss": 1.1328, + "theoretical_loss": 3.358206846619079, + "tokens_seen": 2896822272 + }, + { + "epoch": 0.88, + "learning_rate": 9.89368104312939e-05, + "loss": 1.2444, + "theoretical_loss": 3.358201013559563, + "tokens_seen": 2896887808 + }, + { + "epoch": 0.88, + "learning_rate": 9.89207622868606e-05, + "loss": 1.1695, + "theoretical_loss": 3.3581951806689543, + "tokens_seen": 2896953344 + }, + { + "epoch": 0.88, + "learning_rate": 9.89047141424273e-05, + "loss": 1.2047, + "theoretical_loss": 3.3581893479472447, + "tokens_seen": 2897018880 + }, + { + "epoch": 0.88, + "learning_rate": 9.8888665997994e-05, + "loss": 1.2214, + "theoretical_loss": 3.3581835153944244, + "tokens_seen": 2897084416 + }, + { + "epoch": 0.88, + "learning_rate": 9.887261785356068e-05, + "loss": 1.2252, + "theoretical_loss": 3.3581776830104855, + "tokens_seen": 2897149952 + }, + { + "epoch": 0.88, + "learning_rate": 9.885656970912738e-05, + "loss": 1.2151, + "theoretical_loss": 3.358171850795419, + "tokens_seen": 2897215488 + }, + { + "epoch": 0.88, + "learning_rate": 9.884052156469408e-05, + "loss": 1.0617, + "theoretical_loss": 3.358166018749216, + "tokens_seen": 2897281024 + }, + { + "epoch": 0.88, + "learning_rate": 9.882447342026079e-05, + "loss": 1.1013, + "theoretical_loss": 3.3581601868718685, + "tokens_seen": 2897346560 + }, + { + "epoch": 0.88, + "learning_rate": 9.880842527582749e-05, + "loss": 1.1483, + "theoretical_loss": 3.358154355163367, + "tokens_seen": 2897412096 + }, + { + "epoch": 0.88, + "learning_rate": 9.879237713139419e-05, + "loss": 1.1384, + "theoretical_loss": 3.358148523623703, + "tokens_seen": 2897477632 + }, + { + "epoch": 0.88, + "learning_rate": 9.877632898696089e-05, + "loss": 1.1612, + "theoretical_loss": 3.358142692252868, + "tokens_seen": 2897543168 + }, + { + "epoch": 0.88, + "learning_rate": 9.876028084252759e-05, + "loss": 1.1982, + "theoretical_loss": 3.3581368610508533, + "tokens_seen": 2897608704 + }, + { + "epoch": 0.88, + "learning_rate": 9.874423269809429e-05, + "loss": 1.2267, + "theoretical_loss": 3.35813103001765, + "tokens_seen": 2897674240 + }, + { + "epoch": 0.88, + "learning_rate": 9.872818455366099e-05, + "loss": 1.1372, + "theoretical_loss": 3.3581251991532493, + "tokens_seen": 2897739776 + }, + { + "epoch": 0.88, + "learning_rate": 9.871213640922769e-05, + "loss": 1.2531, + "theoretical_loss": 3.358119368457643, + "tokens_seen": 2897805312 + }, + { + "epoch": 0.88, + "learning_rate": 9.869608826479439e-05, + "loss": 1.1508, + "theoretical_loss": 3.3581135379308216, + "tokens_seen": 2897870848 + }, + { + "epoch": 0.88, + "learning_rate": 9.868004012036108e-05, + "loss": 1.2178, + "theoretical_loss": 3.358107707572777, + "tokens_seen": 2897936384 + }, + { + "epoch": 0.88, + "learning_rate": 9.866399197592778e-05, + "loss": 1.1655, + "theoretical_loss": 3.3581018773835005, + "tokens_seen": 2898001920 + }, + { + "epoch": 0.88, + "learning_rate": 9.864794383149449e-05, + "loss": 1.1672, + "theoretical_loss": 3.3580960473629835, + "tokens_seen": 2898067456 + }, + { + "epoch": 0.88, + "learning_rate": 9.86318956870612e-05, + "loss": 1.1759, + "theoretical_loss": 3.358090217511217, + "tokens_seen": 2898132992 + }, + { + "epoch": 0.88, + "learning_rate": 9.86158475426279e-05, + "loss": 1.183, + "theoretical_loss": 3.3580843878281916, + "tokens_seen": 2898198528 + }, + { + "epoch": 0.88, + "learning_rate": 9.85997993981946e-05, + "loss": 1.2836, + "theoretical_loss": 3.3580785583139, + "tokens_seen": 2898264064 + }, + { + "epoch": 0.88, + "learning_rate": 9.85837512537613e-05, + "loss": 1.2438, + "theoretical_loss": 3.3580727289683328, + "tokens_seen": 2898329600 + }, + { + "epoch": 0.88, + "learning_rate": 9.8567703109328e-05, + "loss": 1.1424, + "theoretical_loss": 3.3580668997914813, + "tokens_seen": 2898395136 + }, + { + "epoch": 0.88, + "learning_rate": 9.85516549648947e-05, + "loss": 1.2531, + "theoretical_loss": 3.3580610707833367, + "tokens_seen": 2898460672 + }, + { + "epoch": 0.88, + "learning_rate": 9.853560682046138e-05, + "loss": 1.1744, + "theoretical_loss": 3.358055241943891, + "tokens_seen": 2898526208 + }, + { + "epoch": 0.88, + "learning_rate": 9.851955867602808e-05, + "loss": 1.2312, + "theoretical_loss": 3.358049413273134, + "tokens_seen": 2898591744 + }, + { + "epoch": 0.88, + "learning_rate": 9.850351053159478e-05, + "loss": 1.1281, + "theoretical_loss": 3.358043584771059, + "tokens_seen": 2898657280 + }, + { + "epoch": 0.88, + "learning_rate": 9.848746238716149e-05, + "loss": 1.2082, + "theoretical_loss": 3.358037756437656, + "tokens_seen": 2898722816 + }, + { + "epoch": 0.88, + "learning_rate": 9.847141424272819e-05, + "loss": 1.2727, + "theoretical_loss": 3.3580319282729163, + "tokens_seen": 2898788352 + }, + { + "epoch": 0.88, + "learning_rate": 9.845536609829489e-05, + "loss": 1.2761, + "theoretical_loss": 3.3580261002768315, + "tokens_seen": 2898853888 + }, + { + "epoch": 0.88, + "learning_rate": 9.843931795386159e-05, + "loss": 1.1534, + "theoretical_loss": 3.358020272449393, + "tokens_seen": 2898919424 + }, + { + "epoch": 0.88, + "learning_rate": 9.842326980942829e-05, + "loss": 1.2146, + "theoretical_loss": 3.3580144447905917, + "tokens_seen": 2898984960 + }, + { + "epoch": 0.88, + "learning_rate": 9.840722166499499e-05, + "loss": 1.2477, + "theoretical_loss": 3.35800861730042, + "tokens_seen": 2899050496 + }, + { + "epoch": 0.88, + "learning_rate": 9.839117352056169e-05, + "loss": 1.2585, + "theoretical_loss": 3.358002789978868, + "tokens_seen": 2899116032 + }, + { + "epoch": 0.88, + "learning_rate": 9.837512537612839e-05, + "loss": 1.2421, + "theoretical_loss": 3.357996962825927, + "tokens_seen": 2899181568 + }, + { + "epoch": 0.88, + "learning_rate": 9.835907723169509e-05, + "loss": 1.1841, + "theoretical_loss": 3.3579911358415893, + "tokens_seen": 2899247104 + }, + { + "epoch": 0.88, + "learning_rate": 9.834302908726179e-05, + "loss": 1.2617, + "theoretical_loss": 3.357985309025845, + "tokens_seen": 2899312640 + }, + { + "epoch": 0.88, + "learning_rate": 9.832698094282849e-05, + "loss": 1.2599, + "theoretical_loss": 3.3579794823786866, + "tokens_seen": 2899378176 + }, + { + "epoch": 0.88, + "learning_rate": 9.831093279839519e-05, + "loss": 1.1763, + "theoretical_loss": 3.357973655900105, + "tokens_seen": 2899443712 + }, + { + "epoch": 0.88, + "learning_rate": 9.829488465396189e-05, + "loss": 1.1866, + "theoretical_loss": 3.357967829590091, + "tokens_seen": 2899509248 + }, + { + "epoch": 0.88, + "learning_rate": 9.82788365095286e-05, + "loss": 1.1834, + "theoretical_loss": 3.3579620034486366, + "tokens_seen": 2899574784 + }, + { + "epoch": 0.88, + "learning_rate": 9.82627883650953e-05, + "loss": 1.1682, + "theoretical_loss": 3.3579561774757325, + "tokens_seen": 2899640320 + }, + { + "epoch": 0.88, + "learning_rate": 9.8246740220662e-05, + "loss": 1.3287, + "theoretical_loss": 3.3579503516713705, + "tokens_seen": 2899705856 + }, + { + "epoch": 0.88, + "learning_rate": 9.82306920762287e-05, + "loss": 1.0985, + "theoretical_loss": 3.357944526035542, + "tokens_seen": 2899771392 + }, + { + "epoch": 0.88, + "learning_rate": 9.82146439317954e-05, + "loss": 1.2353, + "theoretical_loss": 3.3579387005682375, + "tokens_seen": 2899836928 + }, + { + "epoch": 0.88, + "learning_rate": 9.81985957873621e-05, + "loss": 1.168, + "theoretical_loss": 3.3579328752694493, + "tokens_seen": 2899902464 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 967401, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2384494543075562, + "objective/train/theoretical_loss": 3.357927050139168, + "objective/train/tokens_used": 2920428000, + "theoretical_loss": 3.357927050139168, + "tokens_seen": 2899968000 + }, + { + "epoch": 0.88, + "learning_rate": 9.818254764292878e-05, + "loss": 1.1846, + "theoretical_loss": 3.357927050139168, + "tokens_seen": 2899968000 + }, + { + "epoch": 0.88, + "learning_rate": 9.816649949849548e-05, + "loss": 1.1505, + "theoretical_loss": 3.3579212251773853, + "tokens_seen": 2900033536 + }, + { + "epoch": 0.88, + "learning_rate": 9.815045135406219e-05, + "loss": 1.2496, + "theoretical_loss": 3.3579154003840923, + "tokens_seen": 2900099072 + }, + { + "epoch": 0.88, + "learning_rate": 9.813440320962889e-05, + "loss": 1.1441, + "theoretical_loss": 3.357909575759281, + "tokens_seen": 2900164608 + }, + { + "epoch": 0.88, + "learning_rate": 9.811835506519559e-05, + "loss": 1.3197, + "theoretical_loss": 3.3579037513029415, + "tokens_seen": 2900230144 + }, + { + "epoch": 0.88, + "learning_rate": 9.810230692076229e-05, + "loss": 1.175, + "theoretical_loss": 3.357897927015066, + "tokens_seen": 2900295680 + }, + { + "epoch": 0.88, + "learning_rate": 9.808625877632899e-05, + "loss": 1.2353, + "theoretical_loss": 3.357892102895646, + "tokens_seen": 2900361216 + }, + { + "epoch": 0.88, + "learning_rate": 9.807021063189569e-05, + "loss": 1.2681, + "theoretical_loss": 3.357886278944672, + "tokens_seen": 2900426752 + }, + { + "epoch": 0.88, + "learning_rate": 9.805416248746239e-05, + "loss": 1.2296, + "theoretical_loss": 3.3578804551621357, + "tokens_seen": 2900492288 + }, + { + "epoch": 0.88, + "learning_rate": 9.80381143430291e-05, + "loss": 1.3441, + "theoretical_loss": 3.357874631548029, + "tokens_seen": 2900557824 + }, + { + "epoch": 0.88, + "learning_rate": 9.802206619859579e-05, + "loss": 1.1672, + "theoretical_loss": 3.357868808102342, + "tokens_seen": 2900623360 + }, + { + "epoch": 0.88, + "learning_rate": 9.800601805416249e-05, + "loss": 1.2232, + "theoretical_loss": 3.3578629848250667, + "tokens_seen": 2900688896 + }, + { + "epoch": 0.88, + "learning_rate": 9.798996990972919e-05, + "loss": 1.2318, + "theoretical_loss": 3.357857161716195, + "tokens_seen": 2900754432 + }, + { + "epoch": 0.88, + "learning_rate": 9.797392176529589e-05, + "loss": 1.1269, + "theoretical_loss": 3.3578513387757174, + "tokens_seen": 2900819968 + }, + { + "epoch": 0.88, + "learning_rate": 9.795787362086259e-05, + "loss": 1.1782, + "theoretical_loss": 3.357845516003626, + "tokens_seen": 2900885504 + }, + { + "epoch": 0.88, + "learning_rate": 9.79418254764293e-05, + "loss": 1.3176, + "theoretical_loss": 3.3578396933999106, + "tokens_seen": 2900951040 + }, + { + "epoch": 0.88, + "learning_rate": 9.7925777331996e-05, + "loss": 1.2302, + "theoretical_loss": 3.3578338709645643, + "tokens_seen": 2901016576 + }, + { + "epoch": 0.88, + "learning_rate": 9.79097291875627e-05, + "loss": 1.1422, + "theoretical_loss": 3.3578280486975776, + "tokens_seen": 2901082112 + }, + { + "epoch": 0.88, + "learning_rate": 9.78936810431294e-05, + "loss": 1.1148, + "theoretical_loss": 3.3578222265989415, + "tokens_seen": 2901147648 + }, + { + "epoch": 0.88, + "learning_rate": 9.78776328986961e-05, + "loss": 1.2533, + "theoretical_loss": 3.357816404668648, + "tokens_seen": 2901213184 + }, + { + "epoch": 0.88, + "learning_rate": 9.78615847542628e-05, + "loss": 1.1812, + "theoretical_loss": 3.3578105829066884, + "tokens_seen": 2901278720 + }, + { + "epoch": 0.88, + "learning_rate": 9.78455366098295e-05, + "loss": 1.1313, + "theoretical_loss": 3.3578047613130533, + "tokens_seen": 2901344256 + }, + { + "epoch": 0.88, + "learning_rate": 9.782948846539618e-05, + "loss": 1.156, + "theoretical_loss": 3.357798939887735, + "tokens_seen": 2901409792 + }, + { + "epoch": 0.88, + "learning_rate": 9.781344032096288e-05, + "loss": 1.274, + "theoretical_loss": 3.357793118630724, + "tokens_seen": 2901475328 + }, + { + "epoch": 0.88, + "learning_rate": 9.779739217652959e-05, + "loss": 1.1413, + "theoretical_loss": 3.3577872975420124, + "tokens_seen": 2901540864 + }, + { + "epoch": 0.88, + "learning_rate": 9.778134403209629e-05, + "loss": 1.1505, + "theoretical_loss": 3.3577814766215908, + "tokens_seen": 2901606400 + }, + { + "epoch": 0.88, + "learning_rate": 9.776529588766299e-05, + "loss": 1.1345, + "theoretical_loss": 3.357775655869451, + "tokens_seen": 2901671936 + }, + { + "epoch": 0.88, + "learning_rate": 9.774924774322969e-05, + "loss": 1.1392, + "theoretical_loss": 3.357769835285584, + "tokens_seen": 2901737472 + }, + { + "epoch": 0.88, + "learning_rate": 9.77331995987964e-05, + "loss": 1.2032, + "theoretical_loss": 3.3577640148699817, + "tokens_seen": 2901803008 + }, + { + "epoch": 0.88, + "learning_rate": 9.77171514543631e-05, + "loss": 1.3242, + "theoretical_loss": 3.357758194622635, + "tokens_seen": 2901868544 + }, + { + "epoch": 0.88, + "learning_rate": 9.77011033099298e-05, + "loss": 1.2269, + "theoretical_loss": 3.357752374543535, + "tokens_seen": 2901934080 + }, + { + "epoch": 0.88, + "learning_rate": 9.76850551654965e-05, + "loss": 1.269, + "theoretical_loss": 3.3577465546326732, + "tokens_seen": 2901999616 + }, + { + "epoch": 0.88, + "learning_rate": 9.766900702106319e-05, + "loss": 1.2711, + "theoretical_loss": 3.3577407348900414, + "tokens_seen": 2902065152 + }, + { + "epoch": 0.88, + "learning_rate": 9.765295887662989e-05, + "loss": 1.2796, + "theoretical_loss": 3.3577349153156306, + "tokens_seen": 2902130688 + }, + { + "epoch": 0.88, + "learning_rate": 9.763691073219659e-05, + "loss": 1.2155, + "theoretical_loss": 3.3577290959094324, + "tokens_seen": 2902196224 + }, + { + "epoch": 0.88, + "learning_rate": 9.762086258776329e-05, + "loss": 1.2133, + "theoretical_loss": 3.3577232766714373, + "tokens_seen": 2902261760 + }, + { + "epoch": 0.88, + "learning_rate": 9.760481444332999e-05, + "loss": 1.3015, + "theoretical_loss": 3.3577174576016375, + "tokens_seen": 2902327296 + }, + { + "epoch": 0.88, + "learning_rate": 9.75887662988967e-05, + "loss": 1.181, + "theoretical_loss": 3.357711638700024, + "tokens_seen": 2902392832 + }, + { + "epoch": 0.88, + "learning_rate": 9.75727181544634e-05, + "loss": 1.1035, + "theoretical_loss": 3.3577058199665886, + "tokens_seen": 2902458368 + }, + { + "epoch": 0.88, + "learning_rate": 9.75566700100301e-05, + "loss": 1.2769, + "theoretical_loss": 3.357700001401322, + "tokens_seen": 2902523904 + }, + { + "epoch": 0.88, + "learning_rate": 9.75406218655968e-05, + "loss": 1.3008, + "theoretical_loss": 3.3576941830042157, + "tokens_seen": 2902589440 + }, + { + "epoch": 0.88, + "learning_rate": 9.75245737211635e-05, + "loss": 1.154, + "theoretical_loss": 3.357688364775261, + "tokens_seen": 2902654976 + }, + { + "epoch": 0.88, + "learning_rate": 9.75085255767302e-05, + "loss": 1.1719, + "theoretical_loss": 3.3576825467144498, + "tokens_seen": 2902720512 + }, + { + "epoch": 0.88, + "learning_rate": 9.74924774322969e-05, + "loss": 1.1179, + "theoretical_loss": 3.357676728821773, + "tokens_seen": 2902786048 + }, + { + "epoch": 0.88, + "learning_rate": 9.747642928786358e-05, + "loss": 1.1862, + "theoretical_loss": 3.3576709110972214, + "tokens_seen": 2902851584 + }, + { + "epoch": 0.88, + "learning_rate": 9.746038114343029e-05, + "loss": 1.1492, + "theoretical_loss": 3.3576650935407875, + "tokens_seen": 2902917120 + }, + { + "epoch": 0.88, + "learning_rate": 9.7444332998997e-05, + "loss": 1.1726, + "theoretical_loss": 3.3576592761524617, + "tokens_seen": 2902982656 + }, + { + "epoch": 0.88, + "learning_rate": 9.74282848545637e-05, + "loss": 1.164, + "theoretical_loss": 3.357653458932236, + "tokens_seen": 2903048192 + }, + { + "epoch": 0.88, + "learning_rate": 9.74122367101304e-05, + "loss": 1.3381, + "theoretical_loss": 3.3576476418801016, + "tokens_seen": 2903113728 + }, + { + "epoch": 0.88, + "learning_rate": 9.73961885656971e-05, + "loss": 1.117, + "theoretical_loss": 3.3576418249960494, + "tokens_seen": 2903179264 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 968601, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.283944010734558, + "objective/train/theoretical_loss": 3.357636008280071, + "objective/train/tokens_used": 2923704800, + "theoretical_loss": 3.357636008280071, + "tokens_seen": 2903244800 + }, + { + "epoch": 0.88, + "learning_rate": 9.73801404212638e-05, + "loss": 1.2509, + "theoretical_loss": 3.357636008280071, + "tokens_seen": 2903244800 + }, + { + "epoch": 0.88, + "learning_rate": 9.73640922768305e-05, + "loss": 1.2814, + "theoretical_loss": 3.357630191732158, + "tokens_seen": 2903310336 + }, + { + "epoch": 0.88, + "learning_rate": 9.73480441323972e-05, + "loss": 1.3274, + "theoretical_loss": 3.3576243753523016, + "tokens_seen": 2903375872 + }, + { + "epoch": 0.88, + "learning_rate": 9.73319959879639e-05, + "loss": 1.2209, + "theoretical_loss": 3.357618559140493, + "tokens_seen": 2903441408 + }, + { + "epoch": 0.88, + "learning_rate": 9.731594784353059e-05, + "loss": 1.2061, + "theoretical_loss": 3.3576127430967238, + "tokens_seen": 2903506944 + }, + { + "epoch": 0.88, + "learning_rate": 9.729989969909729e-05, + "loss": 1.1872, + "theoretical_loss": 3.357606927220985, + "tokens_seen": 2903572480 + }, + { + "epoch": 0.88, + "learning_rate": 9.728385155466399e-05, + "loss": 1.1242, + "theoretical_loss": 3.3576011115132687, + "tokens_seen": 2903638016 + }, + { + "epoch": 0.88, + "learning_rate": 9.726780341023069e-05, + "loss": 1.1698, + "theoretical_loss": 3.357595295973565, + "tokens_seen": 2903703552 + }, + { + "epoch": 0.88, + "learning_rate": 9.72517552657974e-05, + "loss": 1.1621, + "theoretical_loss": 3.3575894806018662, + "tokens_seen": 2903769088 + }, + { + "epoch": 0.88, + "learning_rate": 9.72357071213641e-05, + "loss": 1.2313, + "theoretical_loss": 3.3575836653981637, + "tokens_seen": 2903834624 + }, + { + "epoch": 0.88, + "learning_rate": 9.72196589769308e-05, + "loss": 1.2778, + "theoretical_loss": 3.3575778503624485, + "tokens_seen": 2903900160 + }, + { + "epoch": 0.88, + "learning_rate": 9.72036108324975e-05, + "loss": 1.1687, + "theoretical_loss": 3.357572035494712, + "tokens_seen": 2903965696 + }, + { + "epoch": 0.88, + "learning_rate": 9.71875626880642e-05, + "loss": 1.2359, + "theoretical_loss": 3.3575662207949453, + "tokens_seen": 2904031232 + }, + { + "epoch": 0.88, + "learning_rate": 9.71715145436309e-05, + "loss": 1.1675, + "theoretical_loss": 3.357560406263141, + "tokens_seen": 2904096768 + }, + { + "epoch": 0.88, + "learning_rate": 9.71554663991976e-05, + "loss": 1.1769, + "theoretical_loss": 3.357554591899288, + "tokens_seen": 2904162304 + }, + { + "epoch": 0.88, + "learning_rate": 9.71394182547643e-05, + "loss": 1.1638, + "theoretical_loss": 3.3575487777033803, + "tokens_seen": 2904227840 + }, + { + "epoch": 0.88, + "learning_rate": 9.7123370110331e-05, + "loss": 1.1303, + "theoretical_loss": 3.357542963675408, + "tokens_seen": 2904293376 + }, + { + "epoch": 0.88, + "learning_rate": 9.71073219658977e-05, + "loss": 1.2272, + "theoretical_loss": 3.3575371498153626, + "tokens_seen": 2904358912 + }, + { + "epoch": 0.88, + "learning_rate": 9.70912738214644e-05, + "loss": 1.1473, + "theoretical_loss": 3.357531336123235, + "tokens_seen": 2904424448 + }, + { + "epoch": 0.88, + "learning_rate": 9.70752256770311e-05, + "loss": 1.2511, + "theoretical_loss": 3.3575255225990173, + "tokens_seen": 2904489984 + }, + { + "epoch": 0.88, + "learning_rate": 9.70591775325978e-05, + "loss": 1.2274, + "theoretical_loss": 3.357519709242701, + "tokens_seen": 2904555520 + }, + { + "epoch": 0.88, + "learning_rate": 9.70431293881645e-05, + "loss": 1.1281, + "theoretical_loss": 3.3575138960542765, + "tokens_seen": 2904621056 + }, + { + "epoch": 0.88, + "learning_rate": 9.70270812437312e-05, + "loss": 1.1648, + "theoretical_loss": 3.3575080830337356, + "tokens_seen": 2904686592 + }, + { + "epoch": 0.88, + "learning_rate": 9.70110330992979e-05, + "loss": 1.1334, + "theoretical_loss": 3.35750227018107, + "tokens_seen": 2904752128 + }, + { + "epoch": 0.88, + "learning_rate": 9.69949849548646e-05, + "loss": 1.2971, + "theoretical_loss": 3.3574964574962713, + "tokens_seen": 2904817664 + }, + { + "epoch": 0.88, + "learning_rate": 9.69789368104313e-05, + "loss": 1.1381, + "theoretical_loss": 3.35749064497933, + "tokens_seen": 2904883200 + }, + { + "epoch": 0.88, + "learning_rate": 9.696288866599799e-05, + "loss": 1.2705, + "theoretical_loss": 3.3574848326302376, + "tokens_seen": 2904948736 + }, + { + "epoch": 0.88, + "learning_rate": 9.694684052156469e-05, + "loss": 1.1469, + "theoretical_loss": 3.357479020448986, + "tokens_seen": 2905014272 + }, + { + "epoch": 0.88, + "learning_rate": 9.693079237713139e-05, + "loss": 1.1646, + "theoretical_loss": 3.3574732084355663, + "tokens_seen": 2905079808 + }, + { + "epoch": 0.88, + "learning_rate": 9.69147442326981e-05, + "loss": 1.235, + "theoretical_loss": 3.35746739658997, + "tokens_seen": 2905145344 + }, + { + "epoch": 0.88, + "learning_rate": 9.68986960882648e-05, + "loss": 1.0884, + "theoretical_loss": 3.357461584912188, + "tokens_seen": 2905210880 + }, + { + "epoch": 0.88, + "learning_rate": 9.68826479438315e-05, + "loss": 1.2062, + "theoretical_loss": 3.3574557734022124, + "tokens_seen": 2905276416 + }, + { + "epoch": 0.88, + "learning_rate": 9.68665997993982e-05, + "loss": 1.169, + "theoretical_loss": 3.3574499620600338, + "tokens_seen": 2905341952 + }, + { + "epoch": 0.88, + "learning_rate": 9.68505516549649e-05, + "loss": 1.2616, + "theoretical_loss": 3.357444150885644, + "tokens_seen": 2905407488 + }, + { + "epoch": 0.88, + "learning_rate": 9.683450351053161e-05, + "loss": 1.2104, + "theoretical_loss": 3.3574383398790344, + "tokens_seen": 2905473024 + }, + { + "epoch": 0.88, + "learning_rate": 9.681845536609831e-05, + "loss": 1.2309, + "theoretical_loss": 3.3574325290401963, + "tokens_seen": 2905538560 + }, + { + "epoch": 0.88, + "learning_rate": 9.6802407221665e-05, + "loss": 1.2185, + "theoretical_loss": 3.357426718369121, + "tokens_seen": 2905604096 + }, + { + "epoch": 0.88, + "learning_rate": 9.67863590772317e-05, + "loss": 1.1786, + "theoretical_loss": 3.3574209078658, + "tokens_seen": 2905669632 + }, + { + "epoch": 0.88, + "learning_rate": 9.67703109327984e-05, + "loss": 1.1579, + "theoretical_loss": 3.3574150975302244, + "tokens_seen": 2905735168 + }, + { + "epoch": 0.88, + "learning_rate": 9.67542627883651e-05, + "loss": 1.2117, + "theoretical_loss": 3.357409287362386, + "tokens_seen": 2905800704 + }, + { + "epoch": 0.88, + "learning_rate": 9.67382146439318e-05, + "loss": 1.1775, + "theoretical_loss": 3.357403477362276, + "tokens_seen": 2905866240 + }, + { + "epoch": 0.88, + "learning_rate": 9.67221664994985e-05, + "loss": 1.1108, + "theoretical_loss": 3.3573976675298858, + "tokens_seen": 2905931776 + }, + { + "epoch": 0.88, + "learning_rate": 9.67061183550652e-05, + "loss": 1.1685, + "theoretical_loss": 3.3573918578652067, + "tokens_seen": 2905997312 + }, + { + "epoch": 0.88, + "learning_rate": 9.66900702106319e-05, + "loss": 1.1541, + "theoretical_loss": 3.35738604836823, + "tokens_seen": 2906062848 + }, + { + "epoch": 0.88, + "learning_rate": 9.66740220661986e-05, + "loss": 1.2458, + "theoretical_loss": 3.357380239038947, + "tokens_seen": 2906128384 + }, + { + "epoch": 0.88, + "learning_rate": 9.66579739217653e-05, + "loss": 1.175, + "theoretical_loss": 3.3573744298773494, + "tokens_seen": 2906193920 + }, + { + "epoch": 0.88, + "learning_rate": 9.6641925777332e-05, + "loss": 1.1517, + "theoretical_loss": 3.3573686208834284, + "tokens_seen": 2906259456 + }, + { + "epoch": 0.88, + "learning_rate": 9.66258776328987e-05, + "loss": 1.1813, + "theoretical_loss": 3.357362812057175, + "tokens_seen": 2906324992 + }, + { + "epoch": 0.88, + "learning_rate": 9.660982948846539e-05, + "loss": 1.2654, + "theoretical_loss": 3.3573570033985813, + "tokens_seen": 2906390528 + }, + { + "epoch": 0.88, + "learning_rate": 9.659378134403209e-05, + "loss": 1.2042, + "theoretical_loss": 3.3573511949076384, + "tokens_seen": 2906456064 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 969497, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.176384449005127, + "objective/train/theoretical_loss": 3.3573453865843375, + "objective/train/tokens_used": 2926981600, + "theoretical_loss": 3.3573453865843375, + "tokens_seen": 2906521600 + }, + { + "epoch": 0.88, + "learning_rate": 9.657773319959879e-05, + "loss": 1.1734, + "theoretical_loss": 3.3573453865843375, + "tokens_seen": 2906521600 + }, + { + "epoch": 0.88, + "learning_rate": 9.65616850551655e-05, + "loss": 1.2047, + "theoretical_loss": 3.3573395784286704, + "tokens_seen": 2906587136 + }, + { + "epoch": 0.88, + "learning_rate": 9.654563691073221e-05, + "loss": 1.2187, + "theoretical_loss": 3.3573337704406283, + "tokens_seen": 2906652672 + }, + { + "epoch": 0.88, + "learning_rate": 9.652958876629891e-05, + "loss": 1.1895, + "theoretical_loss": 3.357327962620202, + "tokens_seen": 2906718208 + }, + { + "epoch": 0.88, + "learning_rate": 9.651354062186561e-05, + "loss": 1.1581, + "theoretical_loss": 3.3573221549673837, + "tokens_seen": 2906783744 + }, + { + "epoch": 0.88, + "learning_rate": 9.649749247743231e-05, + "loss": 1.1797, + "theoretical_loss": 3.357316347482164, + "tokens_seen": 2906849280 + }, + { + "epoch": 0.88, + "learning_rate": 9.648144433299901e-05, + "loss": 1.0672, + "theoretical_loss": 3.3573105401645353, + "tokens_seen": 2906914816 + }, + { + "epoch": 0.88, + "learning_rate": 9.64653961885657e-05, + "loss": 1.143, + "theoretical_loss": 3.3573047330144883, + "tokens_seen": 2906980352 + }, + { + "epoch": 0.88, + "learning_rate": 9.64493480441324e-05, + "loss": 1.1743, + "theoretical_loss": 3.3572989260320143, + "tokens_seen": 2907045888 + }, + { + "epoch": 0.88, + "learning_rate": 9.64332998996991e-05, + "loss": 1.1741, + "theoretical_loss": 3.357293119217105, + "tokens_seen": 2907111424 + }, + { + "epoch": 0.88, + "learning_rate": 9.64172517552658e-05, + "loss": 1.1378, + "theoretical_loss": 3.3572873125697518, + "tokens_seen": 2907176960 + }, + { + "epoch": 0.88, + "learning_rate": 9.64012036108325e-05, + "loss": 1.0761, + "theoretical_loss": 3.3572815060899455, + "tokens_seen": 2907242496 + }, + { + "epoch": 0.88, + "learning_rate": 9.63851554663992e-05, + "loss": 1.1761, + "theoretical_loss": 3.3572756997776785, + "tokens_seen": 2907308032 + }, + { + "epoch": 0.88, + "learning_rate": 9.63691073219659e-05, + "loss": 1.223, + "theoretical_loss": 3.3572698936329415, + "tokens_seen": 2907373568 + }, + { + "epoch": 0.88, + "learning_rate": 9.63530591775326e-05, + "loss": 1.1286, + "theoretical_loss": 3.357264087655726, + "tokens_seen": 2907439104 + }, + { + "epoch": 0.88, + "learning_rate": 9.63370110330993e-05, + "loss": 1.147, + "theoretical_loss": 3.3572582818460233, + "tokens_seen": 2907504640 + }, + { + "epoch": 0.88, + "learning_rate": 9.6320962888666e-05, + "loss": 1.298, + "theoretical_loss": 3.357252476203825, + "tokens_seen": 2907570176 + }, + { + "epoch": 0.88, + "learning_rate": 9.63049147442327e-05, + "loss": 1.1615, + "theoretical_loss": 3.357246670729123, + "tokens_seen": 2907635712 + }, + { + "epoch": 0.88, + "learning_rate": 9.62888665997994e-05, + "loss": 1.2383, + "theoretical_loss": 3.357240865421907, + "tokens_seen": 2907701248 + }, + { + "epoch": 0.88, + "learning_rate": 9.627281845536609e-05, + "loss": 1.0316, + "theoretical_loss": 3.35723506028217, + "tokens_seen": 2907766784 + }, + { + "epoch": 0.88, + "learning_rate": 9.625677031093279e-05, + "loss": 1.1922, + "theoretical_loss": 3.3572292553099032, + "tokens_seen": 2907832320 + }, + { + "epoch": 0.88, + "learning_rate": 9.62407221664995e-05, + "loss": 1.1587, + "theoretical_loss": 3.3572234505050975, + "tokens_seen": 2907897856 + }, + { + "epoch": 0.88, + "learning_rate": 9.622467402206621e-05, + "loss": 1.1718, + "theoretical_loss": 3.3572176458677445, + "tokens_seen": 2907963392 + }, + { + "epoch": 0.88, + "learning_rate": 9.620862587763291e-05, + "loss": 1.1571, + "theoretical_loss": 3.3572118413978354, + "tokens_seen": 2908028928 + }, + { + "epoch": 0.88, + "learning_rate": 9.619257773319961e-05, + "loss": 1.1797, + "theoretical_loss": 3.357206037095362, + "tokens_seen": 2908094464 + }, + { + "epoch": 0.88, + "learning_rate": 9.617652958876631e-05, + "loss": 1.2669, + "theoretical_loss": 3.3572002329603152, + "tokens_seen": 2908160000 + }, + { + "epoch": 0.88, + "learning_rate": 9.616048144433301e-05, + "loss": 1.1604, + "theoretical_loss": 3.357194428992687, + "tokens_seen": 2908225536 + }, + { + "epoch": 0.88, + "learning_rate": 9.614443329989971e-05, + "loss": 1.098, + "theoretical_loss": 3.3571886251924683, + "tokens_seen": 2908291072 + }, + { + "epoch": 0.88, + "learning_rate": 9.612838515546641e-05, + "loss": 1.1939, + "theoretical_loss": 3.3571828215596504, + "tokens_seen": 2908356608 + }, + { + "epoch": 0.88, + "learning_rate": 9.61123370110331e-05, + "loss": 1.1411, + "theoretical_loss": 3.3571770180942253, + "tokens_seen": 2908422144 + }, + { + "epoch": 0.88, + "learning_rate": 9.60962888665998e-05, + "loss": 1.3265, + "theoretical_loss": 3.357171214796184, + "tokens_seen": 2908487680 + }, + { + "epoch": 0.88, + "learning_rate": 9.60802407221665e-05, + "loss": 1.2258, + "theoretical_loss": 3.357165411665518, + "tokens_seen": 2908553216 + }, + { + "epoch": 0.88, + "learning_rate": 9.60641925777332e-05, + "loss": 1.1267, + "theoretical_loss": 3.3571596087022186, + "tokens_seen": 2908618752 + }, + { + "epoch": 0.88, + "learning_rate": 9.60481444332999e-05, + "loss": 1.1637, + "theoretical_loss": 3.357153805906277, + "tokens_seen": 2908684288 + }, + { + "epoch": 0.88, + "learning_rate": 9.60320962888666e-05, + "loss": 1.1767, + "theoretical_loss": 3.3571480032776853, + "tokens_seen": 2908749824 + }, + { + "epoch": 0.88, + "learning_rate": 9.60160481444333e-05, + "loss": 1.2409, + "theoretical_loss": 3.3571422008164347, + "tokens_seen": 2908815360 + }, + { + "epoch": 0.88, + "learning_rate": 9.6e-05, + "loss": 1.1865, + "theoretical_loss": 3.357136398522516, + "tokens_seen": 2908880896 + }, + { + "epoch": 0.88, + "learning_rate": 9.59839518555667e-05, + "loss": 1.189, + "theoretical_loss": 3.357130596395921, + "tokens_seen": 2908946432 + }, + { + "epoch": 0.88, + "learning_rate": 9.59679037111334e-05, + "loss": 1.1504, + "theoretical_loss": 3.357124794436641, + "tokens_seen": 2909011968 + }, + { + "epoch": 0.88, + "learning_rate": 9.59518555667001e-05, + "loss": 1.1639, + "theoretical_loss": 3.3571189926446676, + "tokens_seen": 2909077504 + }, + { + "epoch": 0.88, + "learning_rate": 9.59358074222668e-05, + "loss": 1.1843, + "theoretical_loss": 3.3571131910199923, + "tokens_seen": 2909143040 + }, + { + "epoch": 0.88, + "learning_rate": 9.59197592778335e-05, + "loss": 1.2094, + "theoretical_loss": 3.3571073895626062, + "tokens_seen": 2909208576 + }, + { + "epoch": 0.88, + "learning_rate": 9.59037111334002e-05, + "loss": 1.1662, + "theoretical_loss": 3.3571015882725006, + "tokens_seen": 2909274112 + }, + { + "epoch": 0.88, + "learning_rate": 9.588766298896691e-05, + "loss": 1.073, + "theoretical_loss": 3.3570957871496674, + "tokens_seen": 2909339648 + }, + { + "epoch": 0.88, + "learning_rate": 9.587161484453361e-05, + "loss": 1.2033, + "theoretical_loss": 3.3570899861940973, + "tokens_seen": 2909405184 + }, + { + "epoch": 0.88, + "learning_rate": 9.585556670010031e-05, + "loss": 1.2067, + "theoretical_loss": 3.3570841854057827, + "tokens_seen": 2909470720 + }, + { + "epoch": 0.88, + "learning_rate": 9.583951855566701e-05, + "loss": 1.1348, + "theoretical_loss": 3.357078384784714, + "tokens_seen": 2909536256 + }, + { + "epoch": 0.88, + "learning_rate": 9.582347041123371e-05, + "loss": 1.1756, + "theoretical_loss": 3.3570725843308833, + "tokens_seen": 2909601792 + }, + { + "epoch": 0.88, + "learning_rate": 9.580742226680041e-05, + "loss": 1.1332, + "theoretical_loss": 3.3570667840442816, + "tokens_seen": 2909667328 + }, + { + "epoch": 0.88, + "learning_rate": 9.579137412236711e-05, + "loss": 1.2203, + "theoretical_loss": 3.357060983924901, + "tokens_seen": 2909732864 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 970791, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.178485631942749, + "objective/train/theoretical_loss": 3.357055183972732, + "objective/train/tokens_used": 2930258400, + "theoretical_loss": 3.357055183972732, + "tokens_seen": 2909798400 + }, + { + "epoch": 0.88, + "learning_rate": 9.577532597793381e-05, + "loss": 1.1338, + "theoretical_loss": 3.357055183972732, + "tokens_seen": 2909798400 + }, + { + "epoch": 0.88, + "learning_rate": 9.57592778335005e-05, + "loss": 1.271, + "theoretical_loss": 3.3570493841877664, + "tokens_seen": 2909863936 + }, + { + "epoch": 0.88, + "learning_rate": 9.57432296890672e-05, + "loss": 1.0678, + "theoretical_loss": 3.357043584569996, + "tokens_seen": 2909929472 + }, + { + "epoch": 0.88, + "learning_rate": 9.57271815446339e-05, + "loss": 1.1765, + "theoretical_loss": 3.3570377851194113, + "tokens_seen": 2909995008 + }, + { + "epoch": 0.88, + "learning_rate": 9.57111334002006e-05, + "loss": 1.2277, + "theoretical_loss": 3.3570319858360045, + "tokens_seen": 2910060544 + }, + { + "epoch": 0.88, + "learning_rate": 9.56950852557673e-05, + "loss": 1.2249, + "theoretical_loss": 3.357026186719767, + "tokens_seen": 2910126080 + }, + { + "epoch": 0.88, + "learning_rate": 9.5679037111334e-05, + "loss": 1.1621, + "theoretical_loss": 3.3570203877706897, + "tokens_seen": 2910191616 + }, + { + "epoch": 0.88, + "learning_rate": 9.56629889669007e-05, + "loss": 1.2371, + "theoretical_loss": 3.3570145889887644, + "tokens_seen": 2910257152 + }, + { + "epoch": 0.88, + "learning_rate": 9.56469408224674e-05, + "loss": 1.1419, + "theoretical_loss": 3.3570087903739823, + "tokens_seen": 2910322688 + }, + { + "epoch": 0.88, + "learning_rate": 9.563089267803412e-05, + "loss": 1.2188, + "theoretical_loss": 3.3570029919263353, + "tokens_seen": 2910388224 + }, + { + "epoch": 0.88, + "learning_rate": 9.561484453360082e-05, + "loss": 1.1652, + "theoretical_loss": 3.356997193645814, + "tokens_seen": 2910453760 + }, + { + "epoch": 0.88, + "learning_rate": 9.55987963891675e-05, + "loss": 1.2569, + "theoretical_loss": 3.3569913955324107, + "tokens_seen": 2910519296 + }, + { + "epoch": 0.88, + "learning_rate": 9.55827482447342e-05, + "loss": 1.2674, + "theoretical_loss": 3.3569855975861165, + "tokens_seen": 2910584832 + }, + { + "epoch": 0.88, + "learning_rate": 9.55667001003009e-05, + "loss": 1.1345, + "theoretical_loss": 3.356979799806922, + "tokens_seen": 2910650368 + }, + { + "epoch": 0.88, + "learning_rate": 9.55506519558676e-05, + "loss": 1.1071, + "theoretical_loss": 3.35697400219482, + "tokens_seen": 2910715904 + }, + { + "epoch": 0.88, + "learning_rate": 9.553460381143431e-05, + "loss": 1.2853, + "theoretical_loss": 3.3569682047498013, + "tokens_seen": 2910781440 + }, + { + "epoch": 0.88, + "learning_rate": 9.551855566700101e-05, + "loss": 1.1587, + "theoretical_loss": 3.356962407471857, + "tokens_seen": 2910846976 + }, + { + "epoch": 0.88, + "learning_rate": 9.550250752256771e-05, + "loss": 1.1636, + "theoretical_loss": 3.356956610360979, + "tokens_seen": 2910912512 + }, + { + "epoch": 0.88, + "learning_rate": 9.548645937813441e-05, + "loss": 1.128, + "theoretical_loss": 3.3569508134171584, + "tokens_seen": 2910978048 + }, + { + "epoch": 0.88, + "learning_rate": 9.547041123370111e-05, + "loss": 1.1713, + "theoretical_loss": 3.3569450166403865, + "tokens_seen": 2911043584 + }, + { + "epoch": 0.88, + "learning_rate": 9.545436308926781e-05, + "loss": 1.2309, + "theoretical_loss": 3.3569392200306556, + "tokens_seen": 2911109120 + }, + { + "epoch": 0.88, + "learning_rate": 9.543831494483451e-05, + "loss": 1.1546, + "theoretical_loss": 3.3569334235879564, + "tokens_seen": 2911174656 + }, + { + "epoch": 0.88, + "learning_rate": 9.542226680040121e-05, + "loss": 1.2049, + "theoretical_loss": 3.35692762731228, + "tokens_seen": 2911240192 + }, + { + "epoch": 0.88, + "learning_rate": 9.54062186559679e-05, + "loss": 1.1892, + "theoretical_loss": 3.3569218312036186, + "tokens_seen": 2911305728 + }, + { + "epoch": 0.88, + "learning_rate": 9.53901705115346e-05, + "loss": 1.1309, + "theoretical_loss": 3.356916035261963, + "tokens_seen": 2911371264 + }, + { + "epoch": 0.88, + "learning_rate": 9.53741223671013e-05, + "loss": 1.1859, + "theoretical_loss": 3.3569102394873056, + "tokens_seen": 2911436800 + }, + { + "epoch": 0.88, + "learning_rate": 9.5358074222668e-05, + "loss": 1.1811, + "theoretical_loss": 3.3569044438796367, + "tokens_seen": 2911502336 + }, + { + "epoch": 0.88, + "learning_rate": 9.534202607823471e-05, + "loss": 1.2169, + "theoretical_loss": 3.3568986484389485, + "tokens_seen": 2911567872 + }, + { + "epoch": 0.88, + "learning_rate": 9.532597793380142e-05, + "loss": 1.1441, + "theoretical_loss": 3.356892853165232, + "tokens_seen": 2911633408 + }, + { + "epoch": 0.88, + "learning_rate": 9.530992978936812e-05, + "loss": 1.2763, + "theoretical_loss": 3.3568870580584784, + "tokens_seen": 2911698944 + }, + { + "epoch": 0.88, + "learning_rate": 9.529388164493482e-05, + "loss": 1.1657, + "theoretical_loss": 3.3568812631186797, + "tokens_seen": 2911764480 + }, + { + "epoch": 0.88, + "learning_rate": 9.527783350050152e-05, + "loss": 1.187, + "theoretical_loss": 3.3568754683458275, + "tokens_seen": 2911830016 + }, + { + "epoch": 0.88, + "learning_rate": 9.526178535606822e-05, + "loss": 1.1779, + "theoretical_loss": 3.3568696737399124, + "tokens_seen": 2911895552 + }, + { + "epoch": 0.88, + "learning_rate": 9.52457372116349e-05, + "loss": 1.1511, + "theoretical_loss": 3.3568638793009264, + "tokens_seen": 2911961088 + }, + { + "epoch": 0.88, + "learning_rate": 9.52296890672016e-05, + "loss": 1.1868, + "theoretical_loss": 3.3568580850288607, + "tokens_seen": 2912026624 + }, + { + "epoch": 0.88, + "learning_rate": 9.52136409227683e-05, + "loss": 1.1981, + "theoretical_loss": 3.3568522909237073, + "tokens_seen": 2912092160 + }, + { + "epoch": 0.88, + "learning_rate": 9.519759277833501e-05, + "loss": 1.2043, + "theoretical_loss": 3.356846496985457, + "tokens_seen": 2912157696 + }, + { + "epoch": 0.88, + "learning_rate": 9.518154463390171e-05, + "loss": 1.2992, + "theoretical_loss": 3.3568407032141008, + "tokens_seen": 2912223232 + }, + { + "epoch": 0.88, + "learning_rate": 9.516549648946841e-05, + "loss": 1.1748, + "theoretical_loss": 3.3568349096096313, + "tokens_seen": 2912288768 + }, + { + "epoch": 0.88, + "learning_rate": 9.514944834503511e-05, + "loss": 1.1001, + "theoretical_loss": 3.3568291161720394, + "tokens_seen": 2912354304 + }, + { + "epoch": 0.88, + "learning_rate": 9.513340020060181e-05, + "loss": 1.2577, + "theoretical_loss": 3.3568233229013167, + "tokens_seen": 2912419840 + }, + { + "epoch": 0.88, + "learning_rate": 9.511735205616851e-05, + "loss": 1.2246, + "theoretical_loss": 3.3568175297974543, + "tokens_seen": 2912485376 + }, + { + "epoch": 0.88, + "learning_rate": 9.510130391173521e-05, + "loss": 1.0597, + "theoretical_loss": 3.3568117368604438, + "tokens_seen": 2912550912 + }, + { + "epoch": 0.88, + "learning_rate": 9.508525576730191e-05, + "loss": 1.242, + "theoretical_loss": 3.3568059440902767, + "tokens_seen": 2912616448 + }, + { + "epoch": 0.88, + "learning_rate": 9.506920762286861e-05, + "loss": 1.2571, + "theoretical_loss": 3.356800151486944, + "tokens_seen": 2912681984 + }, + { + "epoch": 0.88, + "learning_rate": 9.50531594784353e-05, + "loss": 1.148, + "theoretical_loss": 3.356794359050438, + "tokens_seen": 2912747520 + }, + { + "epoch": 0.88, + "learning_rate": 9.503711133400201e-05, + "loss": 1.2527, + "theoretical_loss": 3.3567885667807498, + "tokens_seen": 2912813056 + }, + { + "epoch": 0.88, + "learning_rate": 9.502106318956871e-05, + "loss": 1.1426, + "theoretical_loss": 3.3567827746778707, + "tokens_seen": 2912878592 + }, + { + "epoch": 0.88, + "learning_rate": 9.500501504513541e-05, + "loss": 1.2853, + "theoretical_loss": 3.356776982741792, + "tokens_seen": 2912944128 + }, + { + "epoch": 0.88, + "learning_rate": 9.498896690070212e-05, + "loss": 1.1919, + "theoretical_loss": 3.356771190972505, + "tokens_seen": 2913009664 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 971732, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1760027408599854, + "objective/train/theoretical_loss": 3.3567653993700017, + "objective/train/tokens_used": 2933535200, + "theoretical_loss": 3.3567653993700017, + "tokens_seen": 2913075200 + }, + { + "epoch": 0.88, + "learning_rate": 9.497291875626882e-05, + "loss": 1.2116, + "theoretical_loss": 3.3567653993700017, + "tokens_seen": 2913075200 + }, + { + "epoch": 0.88, + "learning_rate": 9.495687061183552e-05, + "loss": 1.1771, + "theoretical_loss": 3.3567596079342734, + "tokens_seen": 2913140736 + }, + { + "epoch": 0.88, + "learning_rate": 9.494082246740222e-05, + "loss": 1.1271, + "theoretical_loss": 3.3567538166653117, + "tokens_seen": 2913206272 + }, + { + "epoch": 0.88, + "learning_rate": 9.492477432296892e-05, + "loss": 1.0894, + "theoretical_loss": 3.3567480255631077, + "tokens_seen": 2913271808 + }, + { + "epoch": 0.88, + "learning_rate": 9.490872617853562e-05, + "loss": 1.1868, + "theoretical_loss": 3.3567422346276525, + "tokens_seen": 2913337344 + }, + { + "epoch": 0.88, + "learning_rate": 9.48926780341023e-05, + "loss": 1.2716, + "theoretical_loss": 3.356736443858938, + "tokens_seen": 2913402880 + }, + { + "epoch": 0.88, + "learning_rate": 9.4876629889669e-05, + "loss": 1.1773, + "theoretical_loss": 3.3567306532569563, + "tokens_seen": 2913468416 + }, + { + "epoch": 0.88, + "learning_rate": 9.486058174523571e-05, + "loss": 1.1702, + "theoretical_loss": 3.3567248628216975, + "tokens_seen": 2913533952 + }, + { + "epoch": 0.88, + "learning_rate": 9.484453360080241e-05, + "loss": 1.1667, + "theoretical_loss": 3.356719072553154, + "tokens_seen": 2913599488 + }, + { + "epoch": 0.88, + "learning_rate": 9.482848545636911e-05, + "loss": 1.136, + "theoretical_loss": 3.356713282451317, + "tokens_seen": 2913665024 + }, + { + "epoch": 0.88, + "learning_rate": 9.481243731193581e-05, + "loss": 1.1481, + "theoretical_loss": 3.356707492516178, + "tokens_seen": 2913730560 + }, + { + "epoch": 0.88, + "learning_rate": 9.479638916750251e-05, + "loss": 1.0831, + "theoretical_loss": 3.3567017027477286, + "tokens_seen": 2913796096 + }, + { + "epoch": 0.88, + "learning_rate": 9.478034102306921e-05, + "loss": 1.1325, + "theoretical_loss": 3.3566959131459595, + "tokens_seen": 2913861632 + }, + { + "epoch": 0.88, + "learning_rate": 9.476429287863591e-05, + "loss": 1.1686, + "theoretical_loss": 3.3566901237108633, + "tokens_seen": 2913927168 + }, + { + "epoch": 0.88, + "learning_rate": 9.474824473420261e-05, + "loss": 1.2368, + "theoretical_loss": 3.35668433444243, + "tokens_seen": 2913992704 + }, + { + "epoch": 0.88, + "learning_rate": 9.473219658976931e-05, + "loss": 1.243, + "theoretical_loss": 3.3566785453406527, + "tokens_seen": 2914058240 + }, + { + "epoch": 0.88, + "learning_rate": 9.471614844533601e-05, + "loss": 1.14, + "theoretical_loss": 3.356672756405522, + "tokens_seen": 2914123776 + }, + { + "epoch": 0.88, + "learning_rate": 9.470010030090271e-05, + "loss": 1.2106, + "theoretical_loss": 3.356666967637029, + "tokens_seen": 2914189312 + }, + { + "epoch": 0.88, + "learning_rate": 9.468405215646941e-05, + "loss": 1.2004, + "theoretical_loss": 3.3566611790351657, + "tokens_seen": 2914254848 + }, + { + "epoch": 0.88, + "learning_rate": 9.466800401203611e-05, + "loss": 1.174, + "theoretical_loss": 3.3566553905999235, + "tokens_seen": 2914320384 + }, + { + "epoch": 0.88, + "learning_rate": 9.465195586760282e-05, + "loss": 1.2398, + "theoretical_loss": 3.3566496023312937, + "tokens_seen": 2914385920 + }, + { + "epoch": 0.88, + "learning_rate": 9.463590772316952e-05, + "loss": 1.178, + "theoretical_loss": 3.356643814229268, + "tokens_seen": 2914451456 + }, + { + "epoch": 0.88, + "learning_rate": 9.461985957873622e-05, + "loss": 1.235, + "theoretical_loss": 3.3566380262938376, + "tokens_seen": 2914516992 + }, + { + "epoch": 0.88, + "learning_rate": 9.460381143430292e-05, + "loss": 1.2891, + "theoretical_loss": 3.3566322385249943, + "tokens_seen": 2914582528 + }, + { + "epoch": 0.88, + "learning_rate": 9.458776328986962e-05, + "loss": 1.2529, + "theoretical_loss": 3.356626450922729, + "tokens_seen": 2914648064 + }, + { + "epoch": 0.88, + "learning_rate": 9.457171514543632e-05, + "loss": 1.2535, + "theoretical_loss": 3.3566206634870337, + "tokens_seen": 2914713600 + }, + { + "epoch": 0.88, + "learning_rate": 9.455566700100302e-05, + "loss": 1.2948, + "theoretical_loss": 3.356614876217899, + "tokens_seen": 2914779136 + }, + { + "epoch": 0.88, + "learning_rate": 9.45396188565697e-05, + "loss": 1.2461, + "theoretical_loss": 3.3566090891153175, + "tokens_seen": 2914844672 + }, + { + "epoch": 0.88, + "learning_rate": 9.45235707121364e-05, + "loss": 1.2807, + "theoretical_loss": 3.3566033021792805, + "tokens_seen": 2914910208 + }, + { + "epoch": 0.88, + "learning_rate": 9.450752256770311e-05, + "loss": 1.254, + "theoretical_loss": 3.3565975154097787, + "tokens_seen": 2914975744 + }, + { + "epoch": 0.88, + "learning_rate": 9.449147442326981e-05, + "loss": 1.1824, + "theoretical_loss": 3.356591728806804, + "tokens_seen": 2915041280 + }, + { + "epoch": 0.88, + "learning_rate": 9.447542627883651e-05, + "loss": 1.1443, + "theoretical_loss": 3.356585942370348, + "tokens_seen": 2915106816 + }, + { + "epoch": 0.88, + "learning_rate": 9.445937813440321e-05, + "loss": 1.2506, + "theoretical_loss": 3.356580156100402, + "tokens_seen": 2915172352 + }, + { + "epoch": 0.88, + "learning_rate": 9.444332998996992e-05, + "loss": 1.1554, + "theoretical_loss": 3.3565743699969572, + "tokens_seen": 2915237888 + }, + { + "epoch": 0.88, + "learning_rate": 9.442728184553662e-05, + "loss": 1.16, + "theoretical_loss": 3.356568584060006, + "tokens_seen": 2915303424 + }, + { + "epoch": 0.88, + "learning_rate": 9.441123370110332e-05, + "loss": 1.168, + "theoretical_loss": 3.3565627982895387, + "tokens_seen": 2915368960 + }, + { + "epoch": 0.88, + "learning_rate": 9.439518555667001e-05, + "loss": 1.1825, + "theoretical_loss": 3.3565570126855473, + "tokens_seen": 2915434496 + }, + { + "epoch": 0.88, + "learning_rate": 9.437913741223671e-05, + "loss": 1.1561, + "theoretical_loss": 3.3565512272480236, + "tokens_seen": 2915500032 + }, + { + "epoch": 0.88, + "learning_rate": 9.436308926780341e-05, + "loss": 1.2183, + "theoretical_loss": 3.3565454419769583, + "tokens_seen": 2915565568 + }, + { + "epoch": 0.88, + "learning_rate": 9.434704112337011e-05, + "loss": 1.127, + "theoretical_loss": 3.3565396568723433, + "tokens_seen": 2915631104 + }, + { + "epoch": 0.88, + "learning_rate": 9.433099297893681e-05, + "loss": 1.1872, + "theoretical_loss": 3.35653387193417, + "tokens_seen": 2915696640 + }, + { + "epoch": 0.88, + "learning_rate": 9.431494483450351e-05, + "loss": 1.2506, + "theoretical_loss": 3.35652808716243, + "tokens_seen": 2915762176 + }, + { + "epoch": 0.88, + "learning_rate": 9.429889669007022e-05, + "loss": 1.1255, + "theoretical_loss": 3.3565223025571154, + "tokens_seen": 2915827712 + }, + { + "epoch": 0.88, + "learning_rate": 9.428284854563692e-05, + "loss": 1.1815, + "theoretical_loss": 3.356516518118216, + "tokens_seen": 2915893248 + }, + { + "epoch": 0.88, + "learning_rate": 9.426680040120362e-05, + "loss": 1.1222, + "theoretical_loss": 3.3565107338457247, + "tokens_seen": 2915958784 + }, + { + "epoch": 0.88, + "learning_rate": 9.425075225677032e-05, + "loss": 1.1134, + "theoretical_loss": 3.3565049497396324, + "tokens_seen": 2916024320 + }, + { + "epoch": 0.88, + "learning_rate": 9.423470411233702e-05, + "loss": 1.1258, + "theoretical_loss": 3.356499165799931, + "tokens_seen": 2916089856 + }, + { + "epoch": 0.88, + "learning_rate": 9.421865596790372e-05, + "loss": 1.1567, + "theoretical_loss": 3.356493382026611, + "tokens_seen": 2916155392 + }, + { + "epoch": 0.88, + "learning_rate": 9.42026078234704e-05, + "loss": 1.3214, + "theoretical_loss": 3.3564875984196654, + "tokens_seen": 2916220928 + }, + { + "epoch": 0.88, + "learning_rate": 9.41865596790371e-05, + "loss": 1.2155, + "theoretical_loss": 3.3564818149790843, + "tokens_seen": 2916286464 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 973010, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3331999778747559, + "objective/train/theoretical_loss": 3.35647603170486, + "objective/train/tokens_used": 2936812000, + "theoretical_loss": 3.35647603170486, + "tokens_seen": 2916352000 + }, + { + "epoch": 0.88, + "learning_rate": 9.417051153460381e-05, + "loss": 1.2785, + "theoretical_loss": 3.35647603170486, + "tokens_seen": 2916352000 + }, + { + "epoch": 0.88, + "learning_rate": 9.415446339017051e-05, + "loss": 1.1914, + "theoretical_loss": 3.3564702485969833, + "tokens_seen": 2916417536 + }, + { + "epoch": 0.88, + "learning_rate": 9.413841524573722e-05, + "loss": 1.2151, + "theoretical_loss": 3.356464465655446, + "tokens_seen": 2916483072 + }, + { + "epoch": 0.88, + "learning_rate": 9.412236710130392e-05, + "loss": 1.2058, + "theoretical_loss": 3.35645868288024, + "tokens_seen": 2916548608 + }, + { + "epoch": 0.88, + "learning_rate": 9.410631895687062e-05, + "loss": 1.1751, + "theoretical_loss": 3.356452900271356, + "tokens_seen": 2916614144 + }, + { + "epoch": 0.88, + "learning_rate": 9.409027081243732e-05, + "loss": 1.2127, + "theoretical_loss": 3.356447117828786, + "tokens_seen": 2916679680 + }, + { + "epoch": 0.88, + "learning_rate": 9.407422266800402e-05, + "loss": 1.1711, + "theoretical_loss": 3.3564413355525216, + "tokens_seen": 2916745216 + }, + { + "epoch": 0.88, + "learning_rate": 9.405817452357073e-05, + "loss": 1.2508, + "theoretical_loss": 3.356435553442554, + "tokens_seen": 2916810752 + }, + { + "epoch": 0.88, + "learning_rate": 9.404212637913741e-05, + "loss": 1.1705, + "theoretical_loss": 3.3564297714988744, + "tokens_seen": 2916876288 + }, + { + "epoch": 0.88, + "learning_rate": 9.402607823470411e-05, + "loss": 1.1868, + "theoretical_loss": 3.356423989721475, + "tokens_seen": 2916941824 + }, + { + "epoch": 0.88, + "learning_rate": 9.401003009027081e-05, + "loss": 1.1305, + "theoretical_loss": 3.3564182081103464, + "tokens_seen": 2917007360 + }, + { + "epoch": 0.88, + "learning_rate": 9.399398194583751e-05, + "loss": 1.2645, + "theoretical_loss": 3.356412426665481, + "tokens_seen": 2917072896 + }, + { + "epoch": 0.88, + "learning_rate": 9.397793380140421e-05, + "loss": 1.2301, + "theoretical_loss": 3.3564066453868695, + "tokens_seen": 2917138432 + }, + { + "epoch": 0.88, + "learning_rate": 9.396188565697092e-05, + "loss": 1.1908, + "theoretical_loss": 3.3564008642745042, + "tokens_seen": 2917203968 + }, + { + "epoch": 0.88, + "learning_rate": 9.394583751253762e-05, + "loss": 1.1705, + "theoretical_loss": 3.3563950833283758, + "tokens_seen": 2917269504 + }, + { + "epoch": 0.88, + "learning_rate": 9.392978936810432e-05, + "loss": 1.232, + "theoretical_loss": 3.356389302548476, + "tokens_seen": 2917335040 + }, + { + "epoch": 0.88, + "learning_rate": 9.391374122367102e-05, + "loss": 1.1514, + "theoretical_loss": 3.356383521934797, + "tokens_seen": 2917400576 + }, + { + "epoch": 0.88, + "learning_rate": 9.389769307923772e-05, + "loss": 1.176, + "theoretical_loss": 3.3563777414873286, + "tokens_seen": 2917466112 + }, + { + "epoch": 0.88, + "learning_rate": 9.388164493480442e-05, + "loss": 1.1274, + "theoretical_loss": 3.3563719612060643, + "tokens_seen": 2917531648 + }, + { + "epoch": 0.88, + "learning_rate": 9.386559679037112e-05, + "loss": 1.1862, + "theoretical_loss": 3.356366181090994, + "tokens_seen": 2917597184 + }, + { + "epoch": 0.88, + "learning_rate": 9.38495486459378e-05, + "loss": 1.2521, + "theoretical_loss": 3.3563604011421106, + "tokens_seen": 2917662720 + }, + { + "epoch": 0.88, + "learning_rate": 9.383350050150452e-05, + "loss": 1.1608, + "theoretical_loss": 3.3563546213594044, + "tokens_seen": 2917728256 + }, + { + "epoch": 0.88, + "learning_rate": 9.381745235707122e-05, + "loss": 1.2307, + "theoretical_loss": 3.3563488417428675, + "tokens_seen": 2917793792 + }, + { + "epoch": 0.88, + "learning_rate": 9.380140421263792e-05, + "loss": 1.1588, + "theoretical_loss": 3.356343062292491, + "tokens_seen": 2917859328 + }, + { + "epoch": 0.88, + "learning_rate": 9.378535606820462e-05, + "loss": 1.2581, + "theoretical_loss": 3.3563372830082665, + "tokens_seen": 2917924864 + }, + { + "epoch": 0.88, + "learning_rate": 9.376930792377132e-05, + "loss": 1.1878, + "theoretical_loss": 3.356331503890186, + "tokens_seen": 2917990400 + }, + { + "epoch": 0.88, + "learning_rate": 9.375325977933802e-05, + "loss": 1.2064, + "theoretical_loss": 3.35632572493824, + "tokens_seen": 2918055936 + }, + { + "epoch": 0.88, + "learning_rate": 9.373721163490472e-05, + "loss": 1.2737, + "theoretical_loss": 3.3563199461524214, + "tokens_seen": 2918121472 + }, + { + "epoch": 0.88, + "learning_rate": 9.372116349047142e-05, + "loss": 1.1345, + "theoretical_loss": 3.3563141675327204, + "tokens_seen": 2918187008 + }, + { + "epoch": 0.88, + "learning_rate": 9.370511534603813e-05, + "loss": 1.2823, + "theoretical_loss": 3.3563083890791288, + "tokens_seen": 2918252544 + }, + { + "epoch": 0.88, + "learning_rate": 9.368906720160481e-05, + "loss": 1.226, + "theoretical_loss": 3.3563026107916385, + "tokens_seen": 2918318080 + }, + { + "epoch": 0.88, + "learning_rate": 9.367301905717151e-05, + "loss": 1.2302, + "theoretical_loss": 3.3562968326702407, + "tokens_seen": 2918383616 + }, + { + "epoch": 0.88, + "learning_rate": 9.365697091273821e-05, + "loss": 1.1237, + "theoretical_loss": 3.356291054714927, + "tokens_seen": 2918449152 + }, + { + "epoch": 0.88, + "learning_rate": 9.364092276830491e-05, + "loss": 1.223, + "theoretical_loss": 3.356285276925689, + "tokens_seen": 2918514688 + }, + { + "epoch": 0.88, + "learning_rate": 9.362487462387162e-05, + "loss": 1.2324, + "theoretical_loss": 3.3562794993025173, + "tokens_seen": 2918580224 + }, + { + "epoch": 0.88, + "learning_rate": 9.360882647943832e-05, + "loss": 1.1801, + "theoretical_loss": 3.356273721845405, + "tokens_seen": 2918645760 + }, + { + "epoch": 0.88, + "learning_rate": 9.359277833500502e-05, + "loss": 1.1878, + "theoretical_loss": 3.3562679445543426, + "tokens_seen": 2918711296 + }, + { + "epoch": 0.88, + "learning_rate": 9.357673019057172e-05, + "loss": 1.2139, + "theoretical_loss": 3.3562621674293216, + "tokens_seen": 2918776832 + }, + { + "epoch": 0.88, + "learning_rate": 9.356068204613842e-05, + "loss": 1.2526, + "theoretical_loss": 3.3562563904703335, + "tokens_seen": 2918842368 + }, + { + "epoch": 0.88, + "learning_rate": 9.354463390170512e-05, + "loss": 1.1716, + "theoretical_loss": 3.3562506136773704, + "tokens_seen": 2918907904 + }, + { + "epoch": 0.88, + "learning_rate": 9.352858575727182e-05, + "loss": 1.1206, + "theoretical_loss": 3.356244837050423, + "tokens_seen": 2918973440 + }, + { + "epoch": 0.88, + "learning_rate": 9.351253761283852e-05, + "loss": 1.2294, + "theoretical_loss": 3.356239060589483, + "tokens_seen": 2919038976 + }, + { + "epoch": 0.88, + "learning_rate": 9.349648946840522e-05, + "loss": 1.1318, + "theoretical_loss": 3.356233284294542, + "tokens_seen": 2919104512 + }, + { + "epoch": 0.88, + "learning_rate": 9.348044132397192e-05, + "loss": 1.1159, + "theoretical_loss": 3.356227508165592, + "tokens_seen": 2919170048 + }, + { + "epoch": 0.88, + "learning_rate": 9.346439317953862e-05, + "loss": 1.1134, + "theoretical_loss": 3.356221732202624, + "tokens_seen": 2919235584 + }, + { + "epoch": 0.88, + "learning_rate": 9.344834503510532e-05, + "loss": 1.167, + "theoretical_loss": 3.3562159564056295, + "tokens_seen": 2919301120 + }, + { + "epoch": 0.88, + "learning_rate": 9.343229689067202e-05, + "loss": 1.3176, + "theoretical_loss": 3.3562101807746, + "tokens_seen": 2919366656 + }, + { + "epoch": 0.88, + "learning_rate": 9.341624874623872e-05, + "loss": 1.2261, + "theoretical_loss": 3.3562044053095272, + "tokens_seen": 2919432192 + }, + { + "epoch": 0.88, + "learning_rate": 9.340020060180542e-05, + "loss": 1.3039, + "theoretical_loss": 3.356198630010402, + "tokens_seen": 2919497728 + }, + { + "epoch": 0.88, + "learning_rate": 9.338415245737212e-05, + "loss": 1.0766, + "theoretical_loss": 3.356192854877217, + "tokens_seen": 2919563264 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 973848, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3456711769104004, + "objective/train/theoretical_loss": 3.356187079909963, + "objective/train/tokens_used": 2940088800, + "theoretical_loss": 3.356187079909963, + "tokens_seen": 2919628800 + }, + { + "epoch": 0.88, + "learning_rate": 9.336810431293883e-05, + "loss": 1.2922, + "theoretical_loss": 3.356187079909963, + "tokens_seen": 2919628800 + }, + { + "epoch": 0.88, + "learning_rate": 9.335205616850553e-05, + "loss": 1.2099, + "theoretical_loss": 3.356181305108631, + "tokens_seen": 2919694336 + }, + { + "epoch": 0.88, + "learning_rate": 9.333600802407221e-05, + "loss": 1.1896, + "theoretical_loss": 3.3561755304732137, + "tokens_seen": 2919759872 + }, + { + "epoch": 0.88, + "learning_rate": 9.331995987963891e-05, + "loss": 1.1443, + "theoretical_loss": 3.356169756003702, + "tokens_seen": 2919825408 + }, + { + "epoch": 0.88, + "learning_rate": 9.330391173520561e-05, + "loss": 1.1742, + "theoretical_loss": 3.356163981700087, + "tokens_seen": 2919890944 + }, + { + "epoch": 0.88, + "learning_rate": 9.328786359077231e-05, + "loss": 1.2754, + "theoretical_loss": 3.3561582075623613, + "tokens_seen": 2919956480 + }, + { + "epoch": 0.88, + "learning_rate": 9.327181544633902e-05, + "loss": 1.1519, + "theoretical_loss": 3.3561524335905153, + "tokens_seen": 2920022016 + }, + { + "epoch": 0.88, + "learning_rate": 9.325576730190572e-05, + "loss": 1.129, + "theoretical_loss": 3.356146659784541, + "tokens_seen": 2920087552 + }, + { + "epoch": 0.88, + "learning_rate": 9.323971915747243e-05, + "loss": 1.1061, + "theoretical_loss": 3.3561408861444297, + "tokens_seen": 2920153088 + }, + { + "epoch": 0.88, + "learning_rate": 9.322367101303913e-05, + "loss": 1.1717, + "theoretical_loss": 3.3561351126701737, + "tokens_seen": 2920218624 + }, + { + "epoch": 0.88, + "learning_rate": 9.320762286860583e-05, + "loss": 1.1725, + "theoretical_loss": 3.3561293393617633, + "tokens_seen": 2920284160 + }, + { + "epoch": 0.88, + "learning_rate": 9.319157472417253e-05, + "loss": 1.2415, + "theoretical_loss": 3.356123566219191, + "tokens_seen": 2920349696 + }, + { + "epoch": 0.88, + "learning_rate": 9.317552657973922e-05, + "loss": 1.1389, + "theoretical_loss": 3.3561177932424475, + "tokens_seen": 2920415232 + }, + { + "epoch": 0.88, + "learning_rate": 9.315947843530592e-05, + "loss": 1.1538, + "theoretical_loss": 3.3561120204315253, + "tokens_seen": 2920480768 + }, + { + "epoch": 0.89, + "learning_rate": 9.314343029087262e-05, + "loss": 1.1841, + "theoretical_loss": 3.356106247786415, + "tokens_seen": 2920546304 + }, + { + "epoch": 0.89, + "learning_rate": 9.312738214643932e-05, + "loss": 1.2079, + "theoretical_loss": 3.3561004753071084, + "tokens_seen": 2920611840 + }, + { + "epoch": 0.89, + "learning_rate": 9.311133400200602e-05, + "loss": 1.2335, + "theoretical_loss": 3.3560947029935972, + "tokens_seen": 2920677376 + }, + { + "epoch": 0.89, + "learning_rate": 9.309528585757272e-05, + "loss": 1.1687, + "theoretical_loss": 3.356088930845873, + "tokens_seen": 2920742912 + }, + { + "epoch": 0.89, + "learning_rate": 9.307923771313942e-05, + "loss": 1.1984, + "theoretical_loss": 3.356083158863927, + "tokens_seen": 2920808448 + }, + { + "epoch": 0.89, + "learning_rate": 9.306318956870612e-05, + "loss": 1.2396, + "theoretical_loss": 3.3560773870477507, + "tokens_seen": 2920873984 + }, + { + "epoch": 0.89, + "learning_rate": 9.304714142427282e-05, + "loss": 1.1995, + "theoretical_loss": 3.356071615397336, + "tokens_seen": 2920939520 + }, + { + "epoch": 0.89, + "learning_rate": 9.303109327983952e-05, + "loss": 1.2459, + "theoretical_loss": 3.356065843912674, + "tokens_seen": 2921005056 + }, + { + "epoch": 0.89, + "learning_rate": 9.301504513540623e-05, + "loss": 1.1049, + "theoretical_loss": 3.3560600725937566, + "tokens_seen": 2921070592 + }, + { + "epoch": 0.89, + "learning_rate": 9.299899699097293e-05, + "loss": 1.2165, + "theoretical_loss": 3.356054301440575, + "tokens_seen": 2921136128 + }, + { + "epoch": 0.89, + "learning_rate": 9.298294884653961e-05, + "loss": 1.1165, + "theoretical_loss": 3.356048530453121, + "tokens_seen": 2921201664 + }, + { + "epoch": 0.89, + "learning_rate": 9.296690070210631e-05, + "loss": 1.1143, + "theoretical_loss": 3.3560427596313858, + "tokens_seen": 2921267200 + }, + { + "epoch": 0.89, + "learning_rate": 9.295085255767301e-05, + "loss": 1.0597, + "theoretical_loss": 3.3560369889753616, + "tokens_seen": 2921332736 + }, + { + "epoch": 0.89, + "learning_rate": 9.293480441323973e-05, + "loss": 1.1331, + "theoretical_loss": 3.3560312184850387, + "tokens_seen": 2921398272 + }, + { + "epoch": 0.89, + "learning_rate": 9.291875626880643e-05, + "loss": 1.2192, + "theoretical_loss": 3.35602544816041, + "tokens_seen": 2921463808 + }, + { + "epoch": 0.89, + "learning_rate": 9.290270812437313e-05, + "loss": 1.1906, + "theoretical_loss": 3.356019678001466, + "tokens_seen": 2921529344 + }, + { + "epoch": 0.89, + "learning_rate": 9.288665997993983e-05, + "loss": 1.2269, + "theoretical_loss": 3.3560139080081983, + "tokens_seen": 2921594880 + }, + { + "epoch": 0.89, + "learning_rate": 9.287061183550653e-05, + "loss": 1.2203, + "theoretical_loss": 3.3560081381805995, + "tokens_seen": 2921660416 + }, + { + "epoch": 0.89, + "learning_rate": 9.285456369107323e-05, + "loss": 1.1984, + "theoretical_loss": 3.3560023685186597, + "tokens_seen": 2921725952 + }, + { + "epoch": 0.89, + "learning_rate": 9.283851554663993e-05, + "loss": 1.2025, + "theoretical_loss": 3.3559965990223715, + "tokens_seen": 2921791488 + }, + { + "epoch": 0.89, + "learning_rate": 9.282246740220662e-05, + "loss": 1.1915, + "theoretical_loss": 3.355990829691726, + "tokens_seen": 2921857024 + }, + { + "epoch": 0.89, + "learning_rate": 9.280641925777332e-05, + "loss": 1.2632, + "theoretical_loss": 3.3559850605267147, + "tokens_seen": 2921922560 + }, + { + "epoch": 0.89, + "learning_rate": 9.279037111334002e-05, + "loss": 1.2796, + "theoretical_loss": 3.355979291527329, + "tokens_seen": 2921988096 + }, + { + "epoch": 0.89, + "learning_rate": 9.277432296890672e-05, + "loss": 1.1665, + "theoretical_loss": 3.3559735226935605, + "tokens_seen": 2922053632 + }, + { + "epoch": 0.89, + "learning_rate": 9.275827482447342e-05, + "loss": 1.1897, + "theoretical_loss": 3.355967754025401, + "tokens_seen": 2922119168 + }, + { + "epoch": 0.89, + "learning_rate": 9.274222668004012e-05, + "loss": 1.2191, + "theoretical_loss": 3.3559619855228418, + "tokens_seen": 2922184704 + }, + { + "epoch": 0.89, + "learning_rate": 9.272617853560682e-05, + "loss": 1.2956, + "theoretical_loss": 3.355956217185875, + "tokens_seen": 2922250240 + }, + { + "epoch": 0.89, + "learning_rate": 9.271013039117352e-05, + "loss": 1.0749, + "theoretical_loss": 3.3559504490144914, + "tokens_seen": 2922315776 + }, + { + "epoch": 0.89, + "learning_rate": 9.269408224674022e-05, + "loss": 1.1944, + "theoretical_loss": 3.3559446810086824, + "tokens_seen": 2922381312 + }, + { + "epoch": 0.89, + "learning_rate": 9.267803410230693e-05, + "loss": 1.1785, + "theoretical_loss": 3.35593891316844, + "tokens_seen": 2922446848 + }, + { + "epoch": 0.89, + "learning_rate": 9.266198595787363e-05, + "loss": 1.224, + "theoretical_loss": 3.3559331454937555, + "tokens_seen": 2922512384 + }, + { + "epoch": 0.89, + "learning_rate": 9.264593781344033e-05, + "loss": 1.1769, + "theoretical_loss": 3.3559273779846213, + "tokens_seen": 2922577920 + }, + { + "epoch": 0.89, + "learning_rate": 9.262988966900703e-05, + "loss": 1.284, + "theoretical_loss": 3.3559216106410275, + "tokens_seen": 2922643456 + }, + { + "epoch": 0.89, + "learning_rate": 9.261384152457373e-05, + "loss": 1.2615, + "theoretical_loss": 3.3559158434629666, + "tokens_seen": 2922708992 + }, + { + "epoch": 0.89, + "learning_rate": 9.259779338014043e-05, + "loss": 1.1829, + "theoretical_loss": 3.3559100764504297, + "tokens_seen": 2922774528 + }, + { + "epoch": 0.89, + "learning_rate": 9.258174523570713e-05, + "loss": 1.2646, + "theoretical_loss": 3.355904309603409, + "tokens_seen": 2922840064 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 975070, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0621994733810425, + "objective/train/theoretical_loss": 3.355898542921895, + "objective/train/tokens_used": 2943365600, + "theoretical_loss": 3.355898542921895, + "tokens_seen": 2922905600 + }, + { + "epoch": 0.89, + "learning_rate": 9.256569709127383e-05, + "loss": 1.1328, + "theoretical_loss": 3.355898542921895, + "tokens_seen": 2922905600 + }, + { + "epoch": 0.89, + "learning_rate": 9.254964894684053e-05, + "loss": 1.2399, + "theoretical_loss": 3.35589277640588, + "tokens_seen": 2922971136 + }, + { + "epoch": 0.89, + "learning_rate": 9.253360080240723e-05, + "loss": 1.198, + "theoretical_loss": 3.3558870100553557, + "tokens_seen": 2923036672 + }, + { + "epoch": 0.89, + "learning_rate": 9.251755265797393e-05, + "loss": 1.1924, + "theoretical_loss": 3.3558812438703125, + "tokens_seen": 2923102208 + }, + { + "epoch": 0.89, + "learning_rate": 9.250150451354063e-05, + "loss": 1.1866, + "theoretical_loss": 3.355875477850743, + "tokens_seen": 2923167744 + }, + { + "epoch": 0.89, + "learning_rate": 9.248545636910732e-05, + "loss": 1.1435, + "theoretical_loss": 3.355869711996639, + "tokens_seen": 2923233280 + }, + { + "epoch": 0.89, + "learning_rate": 9.246940822467402e-05, + "loss": 1.1776, + "theoretical_loss": 3.3558639463079913, + "tokens_seen": 2923298816 + }, + { + "epoch": 0.89, + "learning_rate": 9.245336008024072e-05, + "loss": 1.1321, + "theoretical_loss": 3.355858180784791, + "tokens_seen": 2923364352 + }, + { + "epoch": 0.89, + "learning_rate": 9.243731193580742e-05, + "loss": 1.118, + "theoretical_loss": 3.355852415427031, + "tokens_seen": 2923429888 + }, + { + "epoch": 0.89, + "learning_rate": 9.242126379137412e-05, + "loss": 1.116, + "theoretical_loss": 3.355846650234702, + "tokens_seen": 2923495424 + }, + { + "epoch": 0.89, + "learning_rate": 9.240521564694082e-05, + "loss": 1.1931, + "theoretical_loss": 3.3558408852077957, + "tokens_seen": 2923560960 + }, + { + "epoch": 0.89, + "learning_rate": 9.238916750250752e-05, + "loss": 1.2724, + "theoretical_loss": 3.3558351203463035, + "tokens_seen": 2923626496 + }, + { + "epoch": 0.89, + "learning_rate": 9.237311935807422e-05, + "loss": 1.1115, + "theoretical_loss": 3.355829355650217, + "tokens_seen": 2923692032 + }, + { + "epoch": 0.89, + "learning_rate": 9.235707121364092e-05, + "loss": 1.181, + "theoretical_loss": 3.355823591119528, + "tokens_seen": 2923757568 + }, + { + "epoch": 0.89, + "learning_rate": 9.234102306920764e-05, + "loss": 1.2523, + "theoretical_loss": 3.355817826754228, + "tokens_seen": 2923823104 + }, + { + "epoch": 0.89, + "learning_rate": 9.232497492477433e-05, + "loss": 1.2673, + "theoretical_loss": 3.3558120625543078, + "tokens_seen": 2923888640 + }, + { + "epoch": 0.89, + "learning_rate": 9.230892678034103e-05, + "loss": 1.1268, + "theoretical_loss": 3.3558062985197603, + "tokens_seen": 2923954176 + }, + { + "epoch": 0.89, + "learning_rate": 9.229287863590773e-05, + "loss": 1.2524, + "theoretical_loss": 3.3558005346505757, + "tokens_seen": 2924019712 + }, + { + "epoch": 0.89, + "learning_rate": 9.227683049147443e-05, + "loss": 1.1559, + "theoretical_loss": 3.3557947709467464, + "tokens_seen": 2924085248 + }, + { + "epoch": 0.89, + "learning_rate": 9.226078234704113e-05, + "loss": 1.2309, + "theoretical_loss": 3.355789007408264, + "tokens_seen": 2924150784 + }, + { + "epoch": 0.89, + "learning_rate": 9.224473420260783e-05, + "loss": 1.2094, + "theoretical_loss": 3.3557832440351194, + "tokens_seen": 2924216320 + }, + { + "epoch": 0.89, + "learning_rate": 9.222868605817453e-05, + "loss": 1.1532, + "theoretical_loss": 3.3557774808273044, + "tokens_seen": 2924281856 + }, + { + "epoch": 0.89, + "learning_rate": 9.221263791374123e-05, + "loss": 1.1628, + "theoretical_loss": 3.355771717784811, + "tokens_seen": 2924347392 + }, + { + "epoch": 0.89, + "learning_rate": 9.219658976930793e-05, + "loss": 1.2031, + "theoretical_loss": 3.35576595490763, + "tokens_seen": 2924412928 + }, + { + "epoch": 0.89, + "learning_rate": 9.218054162487463e-05, + "loss": 1.1667, + "theoretical_loss": 3.355760192195754, + "tokens_seen": 2924478464 + }, + { + "epoch": 0.89, + "learning_rate": 9.216449348044133e-05, + "loss": 1.1132, + "theoretical_loss": 3.3557544296491733, + "tokens_seen": 2924544000 + }, + { + "epoch": 0.89, + "learning_rate": 9.214844533600803e-05, + "loss": 1.1852, + "theoretical_loss": 3.3557486672678807, + "tokens_seen": 2924609536 + }, + { + "epoch": 0.89, + "learning_rate": 9.213239719157472e-05, + "loss": 1.2572, + "theoretical_loss": 3.3557429050518666, + "tokens_seen": 2924675072 + }, + { + "epoch": 0.89, + "learning_rate": 9.211634904714142e-05, + "loss": 1.1888, + "theoretical_loss": 3.3557371430011234, + "tokens_seen": 2924740608 + }, + { + "epoch": 0.89, + "learning_rate": 9.210030090270812e-05, + "loss": 1.2253, + "theoretical_loss": 3.355731381115642, + "tokens_seen": 2924806144 + }, + { + "epoch": 0.89, + "learning_rate": 9.208425275827482e-05, + "loss": 1.2962, + "theoretical_loss": 3.3557256193954146, + "tokens_seen": 2924871680 + }, + { + "epoch": 0.89, + "learning_rate": 9.206820461384152e-05, + "loss": 1.2145, + "theoretical_loss": 3.3557198578404326, + "tokens_seen": 2924937216 + }, + { + "epoch": 0.89, + "learning_rate": 9.205215646940822e-05, + "loss": 1.1556, + "theoretical_loss": 3.3557140964506873, + "tokens_seen": 2925002752 + }, + { + "epoch": 0.89, + "learning_rate": 9.203610832497494e-05, + "loss": 1.2183, + "theoretical_loss": 3.3557083352261703, + "tokens_seen": 2925068288 + }, + { + "epoch": 0.89, + "learning_rate": 9.202006018054164e-05, + "loss": 1.1758, + "theoretical_loss": 3.355702574166873, + "tokens_seen": 2925133824 + }, + { + "epoch": 0.89, + "learning_rate": 9.200401203610834e-05, + "loss": 1.2296, + "theoretical_loss": 3.355696813272788, + "tokens_seen": 2925199360 + }, + { + "epoch": 0.89, + "learning_rate": 9.198796389167504e-05, + "loss": 1.2646, + "theoretical_loss": 3.3556910525439054, + "tokens_seen": 2925264896 + }, + { + "epoch": 0.89, + "learning_rate": 9.197191574724173e-05, + "loss": 1.1235, + "theoretical_loss": 3.355685291980217, + "tokens_seen": 2925330432 + }, + { + "epoch": 0.89, + "learning_rate": 9.195586760280843e-05, + "loss": 1.1658, + "theoretical_loss": 3.355679531581716, + "tokens_seen": 2925395968 + }, + { + "epoch": 0.89, + "learning_rate": 9.193981945837513e-05, + "loss": 1.2238, + "theoretical_loss": 3.3556737713483917, + "tokens_seen": 2925461504 + }, + { + "epoch": 0.89, + "learning_rate": 9.192377131394183e-05, + "loss": 1.1867, + "theoretical_loss": 3.3556680112802373, + "tokens_seen": 2925527040 + }, + { + "epoch": 0.89, + "learning_rate": 9.190772316950853e-05, + "loss": 1.1137, + "theoretical_loss": 3.3556622513772436, + "tokens_seen": 2925592576 + }, + { + "epoch": 0.89, + "learning_rate": 9.189167502507523e-05, + "loss": 1.1777, + "theoretical_loss": 3.3556564916394023, + "tokens_seen": 2925658112 + }, + { + "epoch": 0.89, + "learning_rate": 9.187562688064193e-05, + "loss": 1.2257, + "theoretical_loss": 3.355650732066705, + "tokens_seen": 2925723648 + }, + { + "epoch": 0.89, + "learning_rate": 9.185957873620863e-05, + "loss": 1.1939, + "theoretical_loss": 3.3556449726591433, + "tokens_seen": 2925789184 + }, + { + "epoch": 0.89, + "learning_rate": 9.184353059177533e-05, + "loss": 1.2344, + "theoretical_loss": 3.355639213416709, + "tokens_seen": 2925854720 + }, + { + "epoch": 0.89, + "learning_rate": 9.182748244734203e-05, + "loss": 1.164, + "theoretical_loss": 3.355633454339393, + "tokens_seen": 2925920256 + }, + { + "epoch": 0.89, + "learning_rate": 9.181143430290873e-05, + "loss": 1.1024, + "theoretical_loss": 3.355627695427187, + "tokens_seen": 2925985792 + }, + { + "epoch": 0.89, + "learning_rate": 9.179538615847543e-05, + "loss": 1.2468, + "theoretical_loss": 3.3556219366800835, + "tokens_seen": 2926051328 + }, + { + "epoch": 0.89, + "learning_rate": 9.177933801404212e-05, + "loss": 1.165, + "theoretical_loss": 3.355616178098073, + "tokens_seen": 2926116864 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 976402, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.200951337814331, + "objective/train/theoretical_loss": 3.355610419681148, + "objective/train/tokens_used": 2946642400, + "theoretical_loss": 3.355610419681148, + "tokens_seen": 2926182400 + }, + { + "epoch": 0.89, + "learning_rate": 9.176328986960882e-05, + "loss": 1.2057, + "theoretical_loss": 3.355610419681148, + "tokens_seen": 2926182400 + }, + { + "epoch": 0.89, + "learning_rate": 9.174724172517552e-05, + "loss": 1.2038, + "theoretical_loss": 3.355604661429299, + "tokens_seen": 2926247936 + }, + { + "epoch": 0.89, + "learning_rate": 9.173119358074224e-05, + "loss": 1.1359, + "theoretical_loss": 3.355598903342518, + "tokens_seen": 2926313472 + }, + { + "epoch": 0.89, + "learning_rate": 9.171514543630894e-05, + "loss": 1.1437, + "theoretical_loss": 3.3555931454207975, + "tokens_seen": 2926379008 + }, + { + "epoch": 0.89, + "learning_rate": 9.169909729187564e-05, + "loss": 1.1608, + "theoretical_loss": 3.355587387664128, + "tokens_seen": 2926444544 + }, + { + "epoch": 0.89, + "learning_rate": 9.168304914744234e-05, + "loss": 1.2084, + "theoretical_loss": 3.3555816300725008, + "tokens_seen": 2926510080 + }, + { + "epoch": 0.89, + "learning_rate": 9.166700100300904e-05, + "loss": 1.1859, + "theoretical_loss": 3.3555758726459084, + "tokens_seen": 2926575616 + }, + { + "epoch": 0.89, + "learning_rate": 9.165095285857574e-05, + "loss": 1.1126, + "theoretical_loss": 3.355570115384342, + "tokens_seen": 2926641152 + }, + { + "epoch": 0.89, + "learning_rate": 9.163490471414244e-05, + "loss": 1.241, + "theoretical_loss": 3.355564358287793, + "tokens_seen": 2926706688 + }, + { + "epoch": 0.89, + "learning_rate": 9.161885656970913e-05, + "loss": 1.1656, + "theoretical_loss": 3.355558601356253, + "tokens_seen": 2926772224 + }, + { + "epoch": 0.89, + "learning_rate": 9.160280842527583e-05, + "loss": 1.1361, + "theoretical_loss": 3.355552844589714, + "tokens_seen": 2926837760 + }, + { + "epoch": 0.89, + "learning_rate": 9.158676028084253e-05, + "loss": 1.2066, + "theoretical_loss": 3.3555470879881675, + "tokens_seen": 2926903296 + }, + { + "epoch": 0.89, + "learning_rate": 9.157071213640923e-05, + "loss": 1.2295, + "theoretical_loss": 3.3555413315516045, + "tokens_seen": 2926968832 + }, + { + "epoch": 0.89, + "learning_rate": 9.155466399197593e-05, + "loss": 1.1365, + "theoretical_loss": 3.355535575280017, + "tokens_seen": 2927034368 + }, + { + "epoch": 0.89, + "learning_rate": 9.153861584754263e-05, + "loss": 1.2263, + "theoretical_loss": 3.3555298191733964, + "tokens_seen": 2927099904 + }, + { + "epoch": 0.89, + "learning_rate": 9.152256770310933e-05, + "loss": 1.1793, + "theoretical_loss": 3.3555240632317345, + "tokens_seen": 2927165440 + }, + { + "epoch": 0.89, + "learning_rate": 9.150651955867603e-05, + "loss": 1.215, + "theoretical_loss": 3.3555183074550228, + "tokens_seen": 2927230976 + }, + { + "epoch": 0.89, + "learning_rate": 9.149047141424273e-05, + "loss": 1.1828, + "theoretical_loss": 3.355512551843253, + "tokens_seen": 2927296512 + }, + { + "epoch": 0.89, + "learning_rate": 9.147442326980943e-05, + "loss": 1.2832, + "theoretical_loss": 3.3555067963964165, + "tokens_seen": 2927362048 + }, + { + "epoch": 0.89, + "learning_rate": 9.145837512537613e-05, + "loss": 1.1982, + "theoretical_loss": 3.355501041114505, + "tokens_seen": 2927427584 + }, + { + "epoch": 0.89, + "learning_rate": 9.144232698094283e-05, + "loss": 1.1328, + "theoretical_loss": 3.3554952859975096, + "tokens_seen": 2927493120 + }, + { + "epoch": 0.89, + "learning_rate": 9.142627883650953e-05, + "loss": 1.1943, + "theoretical_loss": 3.3554895310454227, + "tokens_seen": 2927558656 + }, + { + "epoch": 0.89, + "learning_rate": 9.141023069207623e-05, + "loss": 1.1672, + "theoretical_loss": 3.355483776258235, + "tokens_seen": 2927624192 + }, + { + "epoch": 0.89, + "learning_rate": 9.139418254764294e-05, + "loss": 1.2237, + "theoretical_loss": 3.355478021635939, + "tokens_seen": 2927689728 + }, + { + "epoch": 0.89, + "learning_rate": 9.137813440320964e-05, + "loss": 1.115, + "theoretical_loss": 3.3554722671785258, + "tokens_seen": 2927755264 + }, + { + "epoch": 0.89, + "learning_rate": 9.136208625877634e-05, + "loss": 1.3437, + "theoretical_loss": 3.355466512885987, + "tokens_seen": 2927820800 + }, + { + "epoch": 0.89, + "learning_rate": 9.134603811434304e-05, + "loss": 1.2165, + "theoretical_loss": 3.355460758758314, + "tokens_seen": 2927886336 + }, + { + "epoch": 0.89, + "learning_rate": 9.132998996990974e-05, + "loss": 1.0809, + "theoretical_loss": 3.3554550047954987, + "tokens_seen": 2927951872 + }, + { + "epoch": 0.89, + "learning_rate": 9.131394182547644e-05, + "loss": 1.2583, + "theoretical_loss": 3.3554492509975327, + "tokens_seen": 2928017408 + }, + { + "epoch": 0.89, + "learning_rate": 9.129789368104314e-05, + "loss": 1.244, + "theoretical_loss": 3.355443497364407, + "tokens_seen": 2928082944 + }, + { + "epoch": 0.89, + "learning_rate": 9.128184553660984e-05, + "loss": 1.2405, + "theoretical_loss": 3.3554377438961143, + "tokens_seen": 2928148480 + }, + { + "epoch": 0.89, + "learning_rate": 9.126579739217653e-05, + "loss": 1.1932, + "theoretical_loss": 3.3554319905926455, + "tokens_seen": 2928214016 + }, + { + "epoch": 0.89, + "learning_rate": 9.124974924774323e-05, + "loss": 1.2381, + "theoretical_loss": 3.355426237453992, + "tokens_seen": 2928279552 + }, + { + "epoch": 0.89, + "learning_rate": 9.123370110330993e-05, + "loss": 1.2306, + "theoretical_loss": 3.3554204844801454, + "tokens_seen": 2928345088 + }, + { + "epoch": 0.89, + "learning_rate": 9.121765295887663e-05, + "loss": 1.0897, + "theoretical_loss": 3.3554147316710976, + "tokens_seen": 2928410624 + }, + { + "epoch": 0.89, + "learning_rate": 9.120160481444333e-05, + "loss": 1.2257, + "theoretical_loss": 3.3554089790268407, + "tokens_seen": 2928476160 + }, + { + "epoch": 0.89, + "learning_rate": 9.118555667001003e-05, + "loss": 1.1144, + "theoretical_loss": 3.355403226547365, + "tokens_seen": 2928541696 + }, + { + "epoch": 0.89, + "learning_rate": 9.116950852557673e-05, + "loss": 1.1874, + "theoretical_loss": 3.355397474232663, + "tokens_seen": 2928607232 + }, + { + "epoch": 0.89, + "learning_rate": 9.115346038114343e-05, + "loss": 1.227, + "theoretical_loss": 3.355391722082726, + "tokens_seen": 2928672768 + }, + { + "epoch": 0.89, + "learning_rate": 9.113741223671015e-05, + "loss": 1.1646, + "theoretical_loss": 3.355385970097546, + "tokens_seen": 2928738304 + }, + { + "epoch": 0.89, + "learning_rate": 9.112136409227685e-05, + "loss": 1.185, + "theoretical_loss": 3.355380218277114, + "tokens_seen": 2928803840 + }, + { + "epoch": 0.89, + "learning_rate": 9.110531594784353e-05, + "loss": 1.163, + "theoretical_loss": 3.3553744666214222, + "tokens_seen": 2928869376 + }, + { + "epoch": 0.89, + "learning_rate": 9.108926780341023e-05, + "loss": 1.2073, + "theoretical_loss": 3.3553687151304614, + "tokens_seen": 2928934912 + }, + { + "epoch": 0.89, + "learning_rate": 9.107321965897693e-05, + "loss": 1.1916, + "theoretical_loss": 3.355362963804224, + "tokens_seen": 2929000448 + }, + { + "epoch": 0.89, + "learning_rate": 9.105717151454364e-05, + "loss": 1.1397, + "theoretical_loss": 3.3553572126427014, + "tokens_seen": 2929065984 + }, + { + "epoch": 0.89, + "learning_rate": 9.104112337011034e-05, + "loss": 1.1533, + "theoretical_loss": 3.355351461645885, + "tokens_seen": 2929131520 + }, + { + "epoch": 0.89, + "learning_rate": 9.102507522567704e-05, + "loss": 1.1548, + "theoretical_loss": 3.355345710813766, + "tokens_seen": 2929197056 + }, + { + "epoch": 0.89, + "learning_rate": 9.100902708124374e-05, + "loss": 1.2094, + "theoretical_loss": 3.3553399601463365, + "tokens_seen": 2929262592 + }, + { + "epoch": 0.89, + "learning_rate": 9.099297893681044e-05, + "loss": 1.1916, + "theoretical_loss": 3.3553342096435883, + "tokens_seen": 2929328128 + }, + { + "epoch": 0.89, + "learning_rate": 9.097693079237714e-05, + "loss": 1.1869, + "theoretical_loss": 3.355328459305513, + "tokens_seen": 2929393664 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 977326, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2213685512542725, + "objective/train/theoretical_loss": 3.355322709132101, + "objective/train/tokens_used": 2949919200, + "theoretical_loss": 3.355322709132101, + "tokens_seen": 2929459200 + }, + { + "epoch": 0.89, + "learning_rate": 9.096088264794384e-05, + "loss": 1.2204, + "theoretical_loss": 3.355322709132101, + "tokens_seen": 2929459200 + }, + { + "epoch": 0.89, + "learning_rate": 9.094483450351054e-05, + "loss": 1.1183, + "theoretical_loss": 3.355316959123346, + "tokens_seen": 2929524736 + }, + { + "epoch": 0.89, + "learning_rate": 9.092878635907724e-05, + "loss": 1.1225, + "theoretical_loss": 3.355311209279238, + "tokens_seen": 2929590272 + }, + { + "epoch": 0.89, + "learning_rate": 9.091273821464393e-05, + "loss": 1.1809, + "theoretical_loss": 3.355305459599769, + "tokens_seen": 2929655808 + }, + { + "epoch": 0.89, + "learning_rate": 9.089669007021063e-05, + "loss": 1.2385, + "theoretical_loss": 3.3552997100849304, + "tokens_seen": 2929721344 + }, + { + "epoch": 0.89, + "learning_rate": 9.088064192577733e-05, + "loss": 1.1661, + "theoretical_loss": 3.3552939607347145, + "tokens_seen": 2929786880 + }, + { + "epoch": 0.89, + "learning_rate": 9.086459378134403e-05, + "loss": 1.2276, + "theoretical_loss": 3.3552882115491123, + "tokens_seen": 2929852416 + }, + { + "epoch": 0.89, + "learning_rate": 9.084854563691073e-05, + "loss": 1.1778, + "theoretical_loss": 3.3552824625281152, + "tokens_seen": 2929917952 + }, + { + "epoch": 0.89, + "learning_rate": 9.083249749247744e-05, + "loss": 1.2295, + "theoretical_loss": 3.355276713671716, + "tokens_seen": 2929983488 + }, + { + "epoch": 0.89, + "learning_rate": 9.081644934804414e-05, + "loss": 1.234, + "theoretical_loss": 3.355270964979905, + "tokens_seen": 2930049024 + }, + { + "epoch": 0.89, + "learning_rate": 9.080040120361085e-05, + "loss": 1.082, + "theoretical_loss": 3.355265216452674, + "tokens_seen": 2930114560 + }, + { + "epoch": 0.89, + "learning_rate": 9.078435305917755e-05, + "loss": 1.2195, + "theoretical_loss": 3.3552594680900154, + "tokens_seen": 2930180096 + }, + { + "epoch": 0.89, + "learning_rate": 9.076830491474425e-05, + "loss": 1.2113, + "theoretical_loss": 3.35525371989192, + "tokens_seen": 2930245632 + }, + { + "epoch": 0.89, + "learning_rate": 9.075225677031093e-05, + "loss": 1.2389, + "theoretical_loss": 3.35524797185838, + "tokens_seen": 2930311168 + }, + { + "epoch": 0.89, + "learning_rate": 9.073620862587763e-05, + "loss": 1.277, + "theoretical_loss": 3.3552422239893867, + "tokens_seen": 2930376704 + }, + { + "epoch": 0.89, + "learning_rate": 9.072016048144434e-05, + "loss": 1.2185, + "theoretical_loss": 3.355236476284931, + "tokens_seen": 2930442240 + }, + { + "epoch": 0.89, + "learning_rate": 9.070411233701104e-05, + "loss": 1.1795, + "theoretical_loss": 3.3552307287450063, + "tokens_seen": 2930507776 + }, + { + "epoch": 0.89, + "learning_rate": 9.068806419257774e-05, + "loss": 1.2269, + "theoretical_loss": 3.3552249813696027, + "tokens_seen": 2930573312 + }, + { + "epoch": 0.89, + "learning_rate": 9.067201604814444e-05, + "loss": 1.1688, + "theoretical_loss": 3.3552192341587124, + "tokens_seen": 2930638848 + }, + { + "epoch": 0.89, + "learning_rate": 9.065596790371114e-05, + "loss": 1.2478, + "theoretical_loss": 3.3552134871123265, + "tokens_seen": 2930704384 + }, + { + "epoch": 0.89, + "learning_rate": 9.063991975927784e-05, + "loss": 1.1176, + "theoretical_loss": 3.3552077402304374, + "tokens_seen": 2930769920 + }, + { + "epoch": 0.89, + "learning_rate": 9.062387161484454e-05, + "loss": 1.2498, + "theoretical_loss": 3.355201993513036, + "tokens_seen": 2930835456 + }, + { + "epoch": 0.89, + "learning_rate": 9.060782347041124e-05, + "loss": 1.0976, + "theoretical_loss": 3.3551962469601144, + "tokens_seen": 2930900992 + }, + { + "epoch": 0.89, + "learning_rate": 9.059177532597794e-05, + "loss": 1.2488, + "theoretical_loss": 3.355190500571664, + "tokens_seen": 2930966528 + }, + { + "epoch": 0.89, + "learning_rate": 9.057572718154464e-05, + "loss": 1.1898, + "theoretical_loss": 3.3551847543476763, + "tokens_seen": 2931032064 + }, + { + "epoch": 0.89, + "learning_rate": 9.055967903711133e-05, + "loss": 1.1045, + "theoretical_loss": 3.355179008288143, + "tokens_seen": 2931097600 + }, + { + "epoch": 0.89, + "learning_rate": 9.054363089267803e-05, + "loss": 1.1269, + "theoretical_loss": 3.3551732623930564, + "tokens_seen": 2931163136 + }, + { + "epoch": 0.89, + "learning_rate": 9.052758274824474e-05, + "loss": 1.2469, + "theoretical_loss": 3.355167516662407, + "tokens_seen": 2931228672 + }, + { + "epoch": 0.89, + "learning_rate": 9.051153460381144e-05, + "loss": 1.2616, + "theoretical_loss": 3.3551617710961867, + "tokens_seen": 2931294208 + }, + { + "epoch": 0.89, + "learning_rate": 9.049548645937814e-05, + "loss": 1.1028, + "theoretical_loss": 3.355156025694388, + "tokens_seen": 2931359744 + }, + { + "epoch": 0.89, + "learning_rate": 9.047943831494484e-05, + "loss": 1.2202, + "theoretical_loss": 3.3551502804570013, + "tokens_seen": 2931425280 + }, + { + "epoch": 0.89, + "learning_rate": 9.046339017051155e-05, + "loss": 1.2256, + "theoretical_loss": 3.355144535384019, + "tokens_seen": 2931490816 + }, + { + "epoch": 0.89, + "learning_rate": 9.044734202607825e-05, + "loss": 1.3135, + "theoretical_loss": 3.3551387904754324, + "tokens_seen": 2931556352 + }, + { + "epoch": 0.89, + "learning_rate": 9.043129388164495e-05, + "loss": 1.1661, + "theoretical_loss": 3.3551330457312334, + "tokens_seen": 2931621888 + }, + { + "epoch": 0.89, + "learning_rate": 9.041524573721163e-05, + "loss": 1.2296, + "theoretical_loss": 3.355127301151413, + "tokens_seen": 2931687424 + }, + { + "epoch": 0.89, + "learning_rate": 9.039919759277833e-05, + "loss": 1.1079, + "theoretical_loss": 3.3551215567359636, + "tokens_seen": 2931752960 + }, + { + "epoch": 0.89, + "learning_rate": 9.038314944834503e-05, + "loss": 1.1729, + "theoretical_loss": 3.3551158124848763, + "tokens_seen": 2931818496 + }, + { + "epoch": 0.89, + "learning_rate": 9.036710130391174e-05, + "loss": 1.2376, + "theoretical_loss": 3.3551100683981434, + "tokens_seen": 2931884032 + }, + { + "epoch": 0.89, + "learning_rate": 9.035105315947844e-05, + "loss": 1.1112, + "theoretical_loss": 3.3551043244757555, + "tokens_seen": 2931949568 + }, + { + "epoch": 0.89, + "learning_rate": 9.033500501504514e-05, + "loss": 1.2194, + "theoretical_loss": 3.355098580717705, + "tokens_seen": 2932015104 + }, + { + "epoch": 0.89, + "learning_rate": 9.031895687061184e-05, + "loss": 1.0502, + "theoretical_loss": 3.3550928371239834, + "tokens_seen": 2932080640 + }, + { + "epoch": 0.89, + "learning_rate": 9.030290872617854e-05, + "loss": 1.1853, + "theoretical_loss": 3.355087093694582, + "tokens_seen": 2932146176 + }, + { + "epoch": 0.89, + "learning_rate": 9.028686058174524e-05, + "loss": 1.3084, + "theoretical_loss": 3.3550813504294923, + "tokens_seen": 2932211712 + }, + { + "epoch": 0.89, + "learning_rate": 9.027081243731194e-05, + "loss": 1.1927, + "theoretical_loss": 3.3550756073287067, + "tokens_seen": 2932277248 + }, + { + "epoch": 0.89, + "learning_rate": 9.025476429287864e-05, + "loss": 1.2976, + "theoretical_loss": 3.3550698643922163, + "tokens_seen": 2932342784 + }, + { + "epoch": 0.89, + "learning_rate": 9.023871614844534e-05, + "loss": 1.2145, + "theoretical_loss": 3.3550641216200128, + "tokens_seen": 2932408320 + }, + { + "epoch": 0.89, + "learning_rate": 9.022266800401204e-05, + "loss": 1.2137, + "theoretical_loss": 3.3550583790120876, + "tokens_seen": 2932473856 + }, + { + "epoch": 0.89, + "learning_rate": 9.020661985957874e-05, + "loss": 1.188, + "theoretical_loss": 3.3550526365684328, + "tokens_seen": 2932539392 + }, + { + "epoch": 0.89, + "learning_rate": 9.019057171514544e-05, + "loss": 1.1958, + "theoretical_loss": 3.35504689428904, + "tokens_seen": 2932604928 + }, + { + "epoch": 0.89, + "learning_rate": 9.017452357071214e-05, + "loss": 1.1757, + "theoretical_loss": 3.3550411521739, + "tokens_seen": 2932670464 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 978603, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0632957220077515, + "objective/train/theoretical_loss": 3.3550354102230058, + "objective/train/tokens_used": 2953196000, + "theoretical_loss": 3.3550354102230058, + "tokens_seen": 2932736000 + }, + { + "epoch": 0.89, + "learning_rate": 9.015847542627884e-05, + "loss": 1.1541, + "theoretical_loss": 3.3550354102230058, + "tokens_seen": 2932736000 + }, + { + "epoch": 0.89, + "learning_rate": 9.014242728184554e-05, + "loss": 1.2947, + "theoretical_loss": 3.355029668436348, + "tokens_seen": 2932801536 + }, + { + "epoch": 0.89, + "learning_rate": 9.012637913741224e-05, + "loss": 1.2204, + "theoretical_loss": 3.355023926813918, + "tokens_seen": 2932867072 + }, + { + "epoch": 0.89, + "learning_rate": 9.011033099297895e-05, + "loss": 1.279, + "theoretical_loss": 3.355018185355709, + "tokens_seen": 2932932608 + }, + { + "epoch": 0.89, + "learning_rate": 9.009428284854565e-05, + "loss": 1.2646, + "theoretical_loss": 3.355012444061711, + "tokens_seen": 2932998144 + }, + { + "epoch": 0.89, + "learning_rate": 9.007823470411235e-05, + "loss": 1.1442, + "theoretical_loss": 3.355006702931916, + "tokens_seen": 2933063680 + }, + { + "epoch": 0.89, + "learning_rate": 9.006218655967903e-05, + "loss": 1.0797, + "theoretical_loss": 3.3550009619663164, + "tokens_seen": 2933129216 + }, + { + "epoch": 0.89, + "learning_rate": 9.004613841524573e-05, + "loss": 1.1813, + "theoretical_loss": 3.354995221164903, + "tokens_seen": 2933194752 + }, + { + "epoch": 0.89, + "learning_rate": 9.003009027081244e-05, + "loss": 1.1239, + "theoretical_loss": 3.3549894805276677, + "tokens_seen": 2933260288 + }, + { + "epoch": 0.89, + "learning_rate": 9.001404212637914e-05, + "loss": 1.1563, + "theoretical_loss": 3.3549837400546023, + "tokens_seen": 2933325824 + }, + { + "epoch": 0.89, + "learning_rate": 8.999799398194584e-05, + "loss": 1.2066, + "theoretical_loss": 3.354977999745698, + "tokens_seen": 2933391360 + }, + { + "epoch": 0.89, + "learning_rate": 8.998194583751254e-05, + "loss": 1.0896, + "theoretical_loss": 3.354972259600947, + "tokens_seen": 2933456896 + }, + { + "epoch": 0.89, + "learning_rate": 8.996589769307924e-05, + "loss": 1.2448, + "theoretical_loss": 3.3549665196203406, + "tokens_seen": 2933522432 + }, + { + "epoch": 0.89, + "learning_rate": 8.994984954864594e-05, + "loss": 1.1736, + "theoretical_loss": 3.3549607798038705, + "tokens_seen": 2933587968 + }, + { + "epoch": 0.89, + "learning_rate": 8.993380140421265e-05, + "loss": 1.1855, + "theoretical_loss": 3.3549550401515287, + "tokens_seen": 2933653504 + }, + { + "epoch": 0.89, + "learning_rate": 8.991775325977935e-05, + "loss": 1.2219, + "theoretical_loss": 3.354949300663306, + "tokens_seen": 2933719040 + }, + { + "epoch": 0.89, + "learning_rate": 8.990170511534604e-05, + "loss": 1.2175, + "theoretical_loss": 3.354943561339195, + "tokens_seen": 2933784576 + }, + { + "epoch": 0.89, + "learning_rate": 8.988565697091274e-05, + "loss": 1.0479, + "theoretical_loss": 3.3549378221791866, + "tokens_seen": 2933850112 + }, + { + "epoch": 0.89, + "learning_rate": 8.986960882647944e-05, + "loss": 1.2025, + "theoretical_loss": 3.354932083183273, + "tokens_seen": 2933915648 + }, + { + "epoch": 0.89, + "learning_rate": 8.985356068204614e-05, + "loss": 1.1533, + "theoretical_loss": 3.3549263443514454, + "tokens_seen": 2933981184 + }, + { + "epoch": 0.89, + "learning_rate": 8.983751253761284e-05, + "loss": 1.2665, + "theoretical_loss": 3.3549206056836955, + "tokens_seen": 2934046720 + }, + { + "epoch": 0.89, + "learning_rate": 8.982146439317954e-05, + "loss": 1.1965, + "theoretical_loss": 3.3549148671800153, + "tokens_seen": 2934112256 + }, + { + "epoch": 0.89, + "learning_rate": 8.980541624874624e-05, + "loss": 1.1219, + "theoretical_loss": 3.3549091288403963, + "tokens_seen": 2934177792 + }, + { + "epoch": 0.89, + "learning_rate": 8.978936810431294e-05, + "loss": 1.2128, + "theoretical_loss": 3.3549033906648296, + "tokens_seen": 2934243328 + }, + { + "epoch": 0.89, + "learning_rate": 8.977331995987965e-05, + "loss": 1.1202, + "theoretical_loss": 3.3548976526533076, + "tokens_seen": 2934308864 + }, + { + "epoch": 0.89, + "learning_rate": 8.975727181544635e-05, + "loss": 1.1324, + "theoretical_loss": 3.3548919148058216, + "tokens_seen": 2934374400 + }, + { + "epoch": 0.89, + "learning_rate": 8.974122367101305e-05, + "loss": 1.1683, + "theoretical_loss": 3.3548861771223635, + "tokens_seen": 2934439936 + }, + { + "epoch": 0.89, + "learning_rate": 8.972517552657975e-05, + "loss": 1.2694, + "theoretical_loss": 3.3548804396029244, + "tokens_seen": 2934505472 + }, + { + "epoch": 0.89, + "learning_rate": 8.970912738214643e-05, + "loss": 1.2428, + "theoretical_loss": 3.3548747022474963, + "tokens_seen": 2934571008 + }, + { + "epoch": 0.89, + "learning_rate": 8.969307923771313e-05, + "loss": 1.1398, + "theoretical_loss": 3.354868965056071, + "tokens_seen": 2934636544 + }, + { + "epoch": 0.89, + "learning_rate": 8.967703109327984e-05, + "loss": 1.2281, + "theoretical_loss": 3.3548632280286395, + "tokens_seen": 2934702080 + }, + { + "epoch": 0.89, + "learning_rate": 8.966098294884654e-05, + "loss": 1.1337, + "theoretical_loss": 3.3548574911651947, + "tokens_seen": 2934767616 + }, + { + "epoch": 0.89, + "learning_rate": 8.964493480441324e-05, + "loss": 1.2115, + "theoretical_loss": 3.354851754465727, + "tokens_seen": 2934833152 + }, + { + "epoch": 0.89, + "learning_rate": 8.962888665997995e-05, + "loss": 1.2019, + "theoretical_loss": 3.3548460179302286, + "tokens_seen": 2934898688 + }, + { + "epoch": 0.89, + "learning_rate": 8.961283851554665e-05, + "loss": 1.0729, + "theoretical_loss": 3.3548402815586913, + "tokens_seen": 2934964224 + }, + { + "epoch": 0.89, + "learning_rate": 8.959679037111335e-05, + "loss": 1.2381, + "theoretical_loss": 3.3548345453511064, + "tokens_seen": 2935029760 + }, + { + "epoch": 0.89, + "learning_rate": 8.958074222668005e-05, + "loss": 1.2558, + "theoretical_loss": 3.3548288093074654, + "tokens_seen": 2935095296 + }, + { + "epoch": 0.89, + "learning_rate": 8.956469408224675e-05, + "loss": 1.1776, + "theoretical_loss": 3.3548230734277604, + "tokens_seen": 2935160832 + }, + { + "epoch": 0.89, + "learning_rate": 8.954864593781344e-05, + "loss": 1.2098, + "theoretical_loss": 3.3548173377119834, + "tokens_seen": 2935226368 + }, + { + "epoch": 0.89, + "learning_rate": 8.953259779338014e-05, + "loss": 1.2472, + "theoretical_loss": 3.354811602160125, + "tokens_seen": 2935291904 + }, + { + "epoch": 0.89, + "learning_rate": 8.951654964894684e-05, + "loss": 1.1662, + "theoretical_loss": 3.354805866772178, + "tokens_seen": 2935357440 + }, + { + "epoch": 0.89, + "learning_rate": 8.950050150451354e-05, + "loss": 1.1507, + "theoretical_loss": 3.354800131548133, + "tokens_seen": 2935422976 + }, + { + "epoch": 0.89, + "learning_rate": 8.948445336008024e-05, + "loss": 1.1506, + "theoretical_loss": 3.354794396487982, + "tokens_seen": 2935488512 + }, + { + "epoch": 0.89, + "learning_rate": 8.946840521564694e-05, + "loss": 1.1166, + "theoretical_loss": 3.354788661591717, + "tokens_seen": 2935554048 + }, + { + "epoch": 0.89, + "learning_rate": 8.945235707121364e-05, + "loss": 1.2123, + "theoretical_loss": 3.3547829268593294, + "tokens_seen": 2935619584 + }, + { + "epoch": 0.89, + "learning_rate": 8.943630892678035e-05, + "loss": 1.2114, + "theoretical_loss": 3.354777192290811, + "tokens_seen": 2935685120 + }, + { + "epoch": 0.89, + "learning_rate": 8.942026078234705e-05, + "loss": 1.2395, + "theoretical_loss": 3.3547714578861534, + "tokens_seen": 2935750656 + }, + { + "epoch": 0.89, + "learning_rate": 8.940421263791375e-05, + "loss": 1.2444, + "theoretical_loss": 3.354765723645348, + "tokens_seen": 2935816192 + }, + { + "epoch": 0.89, + "learning_rate": 8.938816449348045e-05, + "loss": 1.1971, + "theoretical_loss": 3.354759989568387, + "tokens_seen": 2935881728 + }, + { + "epoch": 0.89, + "learning_rate": 8.937211634904715e-05, + "loss": 1.1987, + "theoretical_loss": 3.354754255655261, + "tokens_seen": 2935947264 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 979406, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.225745439529419, + "objective/train/theoretical_loss": 3.354748521905963, + "objective/train/tokens_used": 2956472800, + "theoretical_loss": 3.354748521905963, + "tokens_seen": 2936012800 + }, + { + "epoch": 0.89, + "learning_rate": 8.935606820461383e-05, + "loss": 1.2498, + "theoretical_loss": 3.354748521905963, + "tokens_seen": 2936012800 + }, + { + "epoch": 0.89, + "learning_rate": 8.934002006018054e-05, + "loss": 1.1957, + "theoretical_loss": 3.3547427883204843, + "tokens_seen": 2936078336 + }, + { + "epoch": 0.89, + "learning_rate": 8.932397191574725e-05, + "loss": 1.1326, + "theoretical_loss": 3.3547370548988162, + "tokens_seen": 2936143872 + }, + { + "epoch": 0.89, + "learning_rate": 8.930792377131395e-05, + "loss": 1.2742, + "theoretical_loss": 3.35473132164095, + "tokens_seen": 2936209408 + }, + { + "epoch": 0.89, + "learning_rate": 8.929187562688065e-05, + "loss": 1.1716, + "theoretical_loss": 3.3547255885468785, + "tokens_seen": 2936274944 + }, + { + "epoch": 0.89, + "learning_rate": 8.927582748244735e-05, + "loss": 1.1992, + "theoretical_loss": 3.3547198556165925, + "tokens_seen": 2936340480 + }, + { + "epoch": 0.89, + "learning_rate": 8.925977933801405e-05, + "loss": 1.1837, + "theoretical_loss": 3.3547141228500834, + "tokens_seen": 2936406016 + }, + { + "epoch": 0.89, + "learning_rate": 8.924373119358075e-05, + "loss": 1.1257, + "theoretical_loss": 3.3547083902473442, + "tokens_seen": 2936471552 + }, + { + "epoch": 0.89, + "learning_rate": 8.922768304914745e-05, + "loss": 1.2014, + "theoretical_loss": 3.354702657808365, + "tokens_seen": 2936537088 + }, + { + "epoch": 0.89, + "learning_rate": 8.921163490471415e-05, + "loss": 1.1874, + "theoretical_loss": 3.3546969255331387, + "tokens_seen": 2936602624 + }, + { + "epoch": 0.89, + "learning_rate": 8.919558676028084e-05, + "loss": 1.221, + "theoretical_loss": 3.3546911934216563, + "tokens_seen": 2936668160 + }, + { + "epoch": 0.89, + "learning_rate": 8.917953861584754e-05, + "loss": 1.1577, + "theoretical_loss": 3.3546854614739097, + "tokens_seen": 2936733696 + }, + { + "epoch": 0.89, + "learning_rate": 8.916349047141424e-05, + "loss": 1.1524, + "theoretical_loss": 3.35467972968989, + "tokens_seen": 2936799232 + }, + { + "epoch": 0.89, + "learning_rate": 8.914744232698094e-05, + "loss": 1.1739, + "theoretical_loss": 3.3546739980695897, + "tokens_seen": 2936864768 + }, + { + "epoch": 0.89, + "learning_rate": 8.913139418254764e-05, + "loss": 1.1479, + "theoretical_loss": 3.3546682666130003, + "tokens_seen": 2936930304 + }, + { + "epoch": 0.89, + "learning_rate": 8.911534603811434e-05, + "loss": 1.2045, + "theoretical_loss": 3.354662535320113, + "tokens_seen": 2936995840 + }, + { + "epoch": 0.89, + "learning_rate": 8.909929789368104e-05, + "loss": 1.2958, + "theoretical_loss": 3.3546568041909204, + "tokens_seen": 2937061376 + }, + { + "epoch": 0.89, + "learning_rate": 8.908324974924775e-05, + "loss": 1.2045, + "theoretical_loss": 3.354651073225413, + "tokens_seen": 2937126912 + }, + { + "epoch": 0.89, + "learning_rate": 8.906720160481445e-05, + "loss": 1.2039, + "theoretical_loss": 3.354645342423583, + "tokens_seen": 2937192448 + }, + { + "epoch": 0.89, + "learning_rate": 8.905115346038115e-05, + "loss": 1.228, + "theoretical_loss": 3.354639611785422, + "tokens_seen": 2937257984 + }, + { + "epoch": 0.89, + "learning_rate": 8.903510531594785e-05, + "loss": 1.1591, + "theoretical_loss": 3.354633881310922, + "tokens_seen": 2937323520 + }, + { + "epoch": 0.89, + "learning_rate": 8.901905717151455e-05, + "loss": 1.2319, + "theoretical_loss": 3.3546281510000746, + "tokens_seen": 2937389056 + }, + { + "epoch": 0.89, + "learning_rate": 8.900300902708125e-05, + "loss": 1.1658, + "theoretical_loss": 3.354622420852871, + "tokens_seen": 2937454592 + }, + { + "epoch": 0.89, + "learning_rate": 8.898696088264795e-05, + "loss": 1.1512, + "theoretical_loss": 3.3546166908693036, + "tokens_seen": 2937520128 + }, + { + "epoch": 0.89, + "learning_rate": 8.897091273821465e-05, + "loss": 1.3108, + "theoretical_loss": 3.3546109610493637, + "tokens_seen": 2937585664 + }, + { + "epoch": 0.89, + "learning_rate": 8.895486459378135e-05, + "loss": 1.1275, + "theoretical_loss": 3.354605231393043, + "tokens_seen": 2937651200 + }, + { + "epoch": 0.89, + "learning_rate": 8.893881644934805e-05, + "loss": 1.189, + "theoretical_loss": 3.3545995019003323, + "tokens_seen": 2937716736 + }, + { + "epoch": 0.89, + "learning_rate": 8.892276830491475e-05, + "loss": 1.2564, + "theoretical_loss": 3.3545937725712247, + "tokens_seen": 2937782272 + }, + { + "epoch": 0.89, + "learning_rate": 8.890672016048145e-05, + "loss": 1.2163, + "theoretical_loss": 3.354588043405711, + "tokens_seen": 2937847808 + }, + { + "epoch": 0.89, + "learning_rate": 8.889067201604815e-05, + "loss": 1.2177, + "theoretical_loss": 3.3545823144037836, + "tokens_seen": 2937913344 + }, + { + "epoch": 0.89, + "learning_rate": 8.887462387161485e-05, + "loss": 1.1914, + "theoretical_loss": 3.3545765855654333, + "tokens_seen": 2937978880 + }, + { + "epoch": 0.89, + "learning_rate": 8.885857572718155e-05, + "loss": 1.1928, + "theoretical_loss": 3.3545708568906525, + "tokens_seen": 2938044416 + }, + { + "epoch": 0.89, + "learning_rate": 8.884252758274824e-05, + "loss": 1.2318, + "theoretical_loss": 3.3545651283794324, + "tokens_seen": 2938109952 + }, + { + "epoch": 0.89, + "learning_rate": 8.882647943831494e-05, + "loss": 1.2403, + "theoretical_loss": 3.354559400031765, + "tokens_seen": 2938175488 + }, + { + "epoch": 0.89, + "learning_rate": 8.881043129388164e-05, + "loss": 1.2051, + "theoretical_loss": 3.354553671847642, + "tokens_seen": 2938241024 + }, + { + "epoch": 0.89, + "learning_rate": 8.879438314944834e-05, + "loss": 1.2361, + "theoretical_loss": 3.354547943827055, + "tokens_seen": 2938306560 + }, + { + "epoch": 0.89, + "learning_rate": 8.877833500501504e-05, + "loss": 1.1421, + "theoretical_loss": 3.3545422159699956, + "tokens_seen": 2938372096 + }, + { + "epoch": 0.89, + "learning_rate": 8.876228686058174e-05, + "loss": 1.2196, + "theoretical_loss": 3.3545364882764552, + "tokens_seen": 2938437632 + }, + { + "epoch": 0.89, + "learning_rate": 8.874623871614845e-05, + "loss": 1.1785, + "theoretical_loss": 3.3545307607464263, + "tokens_seen": 2938503168 + }, + { + "epoch": 0.89, + "learning_rate": 8.873019057171516e-05, + "loss": 1.2002, + "theoretical_loss": 3.3545250333799, + "tokens_seen": 2938568704 + }, + { + "epoch": 0.89, + "learning_rate": 8.871414242728186e-05, + "loss": 1.2025, + "theoretical_loss": 3.3545193061768677, + "tokens_seen": 2938634240 + }, + { + "epoch": 0.89, + "learning_rate": 8.869809428284855e-05, + "loss": 1.2625, + "theoretical_loss": 3.354513579137322, + "tokens_seen": 2938699776 + }, + { + "epoch": 0.89, + "learning_rate": 8.868204613841525e-05, + "loss": 1.1795, + "theoretical_loss": 3.3545078522612535, + "tokens_seen": 2938765312 + }, + { + "epoch": 0.89, + "learning_rate": 8.866599799398195e-05, + "loss": 1.1677, + "theoretical_loss": 3.3545021255486547, + "tokens_seen": 2938830848 + }, + { + "epoch": 0.89, + "learning_rate": 8.864994984954865e-05, + "loss": 1.2404, + "theoretical_loss": 3.354496398999517, + "tokens_seen": 2938896384 + }, + { + "epoch": 0.89, + "learning_rate": 8.863390170511535e-05, + "loss": 1.2026, + "theoretical_loss": 3.354490672613832, + "tokens_seen": 2938961920 + }, + { + "epoch": 0.89, + "learning_rate": 8.861785356068205e-05, + "loss": 1.2621, + "theoretical_loss": 3.354484946391592, + "tokens_seen": 2939027456 + }, + { + "epoch": 0.89, + "learning_rate": 8.860180541624875e-05, + "loss": 1.094, + "theoretical_loss": 3.3544792203327876, + "tokens_seen": 2939092992 + }, + { + "epoch": 0.89, + "learning_rate": 8.858575727181545e-05, + "loss": 1.2446, + "theoretical_loss": 3.3544734944374115, + "tokens_seen": 2939158528 + }, + { + "epoch": 0.89, + "learning_rate": 8.856970912738215e-05, + "loss": 1.1657, + "theoretical_loss": 3.3544677687054545, + "tokens_seen": 2939224064 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 980796, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1053264141082764, + "objective/train/theoretical_loss": 3.354462043136909, + "objective/train/tokens_used": 2959749600, + "theoretical_loss": 3.354462043136909, + "tokens_seen": 2939289600 + }, + { + "epoch": 0.89, + "learning_rate": 8.855366098294885e-05, + "loss": 1.1924, + "theoretical_loss": 3.354462043136909, + "tokens_seen": 2939289600 + }, + { + "epoch": 0.89, + "learning_rate": 8.853761283851555e-05, + "loss": 1.2513, + "theoretical_loss": 3.354456317731767, + "tokens_seen": 2939355136 + }, + { + "epoch": 0.89, + "learning_rate": 8.852156469408225e-05, + "loss": 1.2294, + "theoretical_loss": 3.3544505924900188, + "tokens_seen": 2939420672 + }, + { + "epoch": 0.89, + "learning_rate": 8.850551654964894e-05, + "loss": 1.2554, + "theoretical_loss": 3.3544448674116576, + "tokens_seen": 2939486208 + }, + { + "epoch": 0.89, + "learning_rate": 8.848946840521564e-05, + "loss": 1.1919, + "theoretical_loss": 3.354439142496674, + "tokens_seen": 2939551744 + }, + { + "epoch": 0.89, + "learning_rate": 8.847342026078234e-05, + "loss": 1.2735, + "theoretical_loss": 3.3544334177450605, + "tokens_seen": 2939617280 + }, + { + "epoch": 0.89, + "learning_rate": 8.845737211634904e-05, + "loss": 1.2316, + "theoretical_loss": 3.354427693156808, + "tokens_seen": 2939682816 + }, + { + "epoch": 0.89, + "learning_rate": 8.844132397191574e-05, + "loss": 1.1876, + "theoretical_loss": 3.3544219687319092, + "tokens_seen": 2939748352 + }, + { + "epoch": 0.89, + "learning_rate": 8.842527582748246e-05, + "loss": 1.2284, + "theoretical_loss": 3.354416244470355, + "tokens_seen": 2939813888 + }, + { + "epoch": 0.89, + "learning_rate": 8.840922768304916e-05, + "loss": 1.2337, + "theoretical_loss": 3.3544105203721375, + "tokens_seen": 2939879424 + }, + { + "epoch": 0.89, + "learning_rate": 8.839317953861586e-05, + "loss": 1.204, + "theoretical_loss": 3.3544047964372483, + "tokens_seen": 2939944960 + }, + { + "epoch": 0.89, + "learning_rate": 8.837713139418256e-05, + "loss": 1.2379, + "theoretical_loss": 3.3543990726656787, + "tokens_seen": 2940010496 + }, + { + "epoch": 0.89, + "learning_rate": 8.836108324974926e-05, + "loss": 1.1896, + "theoretical_loss": 3.3543933490574207, + "tokens_seen": 2940076032 + }, + { + "epoch": 0.89, + "learning_rate": 8.834503510531595e-05, + "loss": 1.1554, + "theoretical_loss": 3.3543876256124663, + "tokens_seen": 2940141568 + }, + { + "epoch": 0.89, + "learning_rate": 8.832898696088265e-05, + "loss": 1.2022, + "theoretical_loss": 3.3543819023308066, + "tokens_seen": 2940207104 + }, + { + "epoch": 0.89, + "learning_rate": 8.831293881644935e-05, + "loss": 1.1373, + "theoretical_loss": 3.354376179212434, + "tokens_seen": 2940272640 + }, + { + "epoch": 0.89, + "learning_rate": 8.829689067201605e-05, + "loss": 1.1963, + "theoretical_loss": 3.3543704562573398, + "tokens_seen": 2940338176 + }, + { + "epoch": 0.89, + "learning_rate": 8.828084252758275e-05, + "loss": 1.2356, + "theoretical_loss": 3.3543647334655153, + "tokens_seen": 2940403712 + }, + { + "epoch": 0.89, + "learning_rate": 8.826479438314945e-05, + "loss": 1.1008, + "theoretical_loss": 3.354359010836953, + "tokens_seen": 2940469248 + }, + { + "epoch": 0.89, + "learning_rate": 8.824874623871615e-05, + "loss": 1.201, + "theoretical_loss": 3.3543532883716445, + "tokens_seen": 2940534784 + }, + { + "epoch": 0.89, + "learning_rate": 8.823269809428285e-05, + "loss": 1.1768, + "theoretical_loss": 3.3543475660695807, + "tokens_seen": 2940600320 + }, + { + "epoch": 0.89, + "learning_rate": 8.821664994984955e-05, + "loss": 1.1473, + "theoretical_loss": 3.354341843930754, + "tokens_seen": 2940665856 + }, + { + "epoch": 0.89, + "learning_rate": 8.820060180541625e-05, + "loss": 1.174, + "theoretical_loss": 3.354336121955156, + "tokens_seen": 2940731392 + }, + { + "epoch": 0.89, + "learning_rate": 8.818455366098295e-05, + "loss": 1.1904, + "theoretical_loss": 3.3543304001427785, + "tokens_seen": 2940796928 + }, + { + "epoch": 0.89, + "learning_rate": 8.816850551654965e-05, + "loss": 1.1844, + "theoretical_loss": 3.354324678493613, + "tokens_seen": 2940862464 + }, + { + "epoch": 0.89, + "learning_rate": 8.815245737211634e-05, + "loss": 1.2085, + "theoretical_loss": 3.354318957007651, + "tokens_seen": 2940928000 + }, + { + "epoch": 0.89, + "learning_rate": 8.813640922768306e-05, + "loss": 1.2156, + "theoretical_loss": 3.3543132356848853, + "tokens_seen": 2940993536 + }, + { + "epoch": 0.89, + "learning_rate": 8.812036108324976e-05, + "loss": 1.1365, + "theoretical_loss": 3.3543075145253063, + "tokens_seen": 2941059072 + }, + { + "epoch": 0.89, + "learning_rate": 8.810431293881646e-05, + "loss": 1.1932, + "theoretical_loss": 3.354301793528906, + "tokens_seen": 2941124608 + }, + { + "epoch": 0.89, + "learning_rate": 8.808826479438316e-05, + "loss": 1.164, + "theoretical_loss": 3.3542960726956763, + "tokens_seen": 2941190144 + }, + { + "epoch": 0.89, + "learning_rate": 8.807221664994986e-05, + "loss": 1.2156, + "theoretical_loss": 3.3542903520256093, + "tokens_seen": 2941255680 + }, + { + "epoch": 0.89, + "learning_rate": 8.805616850551656e-05, + "loss": 1.2374, + "theoretical_loss": 3.354284631518696, + "tokens_seen": 2941321216 + }, + { + "epoch": 0.89, + "learning_rate": 8.804012036108326e-05, + "loss": 1.1795, + "theoretical_loss": 3.3542789111749287, + "tokens_seen": 2941386752 + }, + { + "epoch": 0.89, + "learning_rate": 8.802407221664996e-05, + "loss": 1.2689, + "theoretical_loss": 3.3542731909942987, + "tokens_seen": 2941452288 + }, + { + "epoch": 0.89, + "learning_rate": 8.800802407221666e-05, + "loss": 1.2079, + "theoretical_loss": 3.354267470976798, + "tokens_seen": 2941517824 + }, + { + "epoch": 0.89, + "learning_rate": 8.799197592778335e-05, + "loss": 1.2328, + "theoretical_loss": 3.354261751122418, + "tokens_seen": 2941583360 + }, + { + "epoch": 0.89, + "learning_rate": 8.797592778335005e-05, + "loss": 1.2589, + "theoretical_loss": 3.354256031431151, + "tokens_seen": 2941648896 + }, + { + "epoch": 0.89, + "learning_rate": 8.795987963891675e-05, + "loss": 1.1043, + "theoretical_loss": 3.354250311902988, + "tokens_seen": 2941714432 + }, + { + "epoch": 0.89, + "learning_rate": 8.794383149448345e-05, + "loss": 1.205, + "theoretical_loss": 3.354244592537921, + "tokens_seen": 2941779968 + }, + { + "epoch": 0.89, + "learning_rate": 8.792778335005015e-05, + "loss": 1.2147, + "theoretical_loss": 3.354238873335942, + "tokens_seen": 2941845504 + }, + { + "epoch": 0.89, + "learning_rate": 8.791173520561685e-05, + "loss": 1.2067, + "theoretical_loss": 3.354233154297042, + "tokens_seen": 2941911040 + }, + { + "epoch": 0.89, + "learning_rate": 8.789568706118355e-05, + "loss": 1.2831, + "theoretical_loss": 3.3542274354212136, + "tokens_seen": 2941976576 + }, + { + "epoch": 0.89, + "learning_rate": 8.787963891675025e-05, + "loss": 1.2223, + "theoretical_loss": 3.354221716708448, + "tokens_seen": 2942042112 + }, + { + "epoch": 0.89, + "learning_rate": 8.786359077231695e-05, + "loss": 1.1967, + "theoretical_loss": 3.354215998158737, + "tokens_seen": 2942107648 + }, + { + "epoch": 0.89, + "learning_rate": 8.784754262788365e-05, + "loss": 1.203, + "theoretical_loss": 3.3542102797720723, + "tokens_seen": 2942173184 + }, + { + "epoch": 0.89, + "learning_rate": 8.783149448345035e-05, + "loss": 1.1554, + "theoretical_loss": 3.3542045615484457, + "tokens_seen": 2942238720 + }, + { + "epoch": 0.89, + "learning_rate": 8.781544633901706e-05, + "loss": 1.1388, + "theoretical_loss": 3.354198843487849, + "tokens_seen": 2942304256 + }, + { + "epoch": 0.89, + "learning_rate": 8.779939819458376e-05, + "loss": 1.1863, + "theoretical_loss": 3.3541931255902737, + "tokens_seen": 2942369792 + }, + { + "epoch": 0.89, + "learning_rate": 8.778335005015046e-05, + "loss": 1.2412, + "theoretical_loss": 3.3541874078557115, + "tokens_seen": 2942435328 + }, + { + "epoch": 0.89, + "learning_rate": 8.776730190571716e-05, + "loss": 1.2108, + "theoretical_loss": 3.3541816902841544, + "tokens_seen": 2942500864 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 981758, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0930460691452026, + "objective/train/theoretical_loss": 3.354175972875594, + "objective/train/tokens_used": 2963026400, + "theoretical_loss": 3.354175972875594, + "tokens_seen": 2942566400 + }, + { + "epoch": 0.89, + "learning_rate": 8.775125376128386e-05, + "loss": 1.1157, + "theoretical_loss": 3.354175972875594, + "tokens_seen": 2942566400 + }, + { + "epoch": 0.89, + "learning_rate": 8.773520561685056e-05, + "loss": 1.2022, + "theoretical_loss": 3.354170255630022, + "tokens_seen": 2942631936 + }, + { + "epoch": 0.89, + "learning_rate": 8.771915747241726e-05, + "loss": 1.1633, + "theoretical_loss": 3.3541645385474297, + "tokens_seen": 2942697472 + }, + { + "epoch": 0.89, + "learning_rate": 8.770310932798396e-05, + "loss": 1.1725, + "theoretical_loss": 3.3541588216278098, + "tokens_seen": 2942763008 + }, + { + "epoch": 0.89, + "learning_rate": 8.768706118355066e-05, + "loss": 1.2679, + "theoretical_loss": 3.3541531048711533, + "tokens_seen": 2942828544 + }, + { + "epoch": 0.89, + "learning_rate": 8.767101303911736e-05, + "loss": 1.1775, + "theoretical_loss": 3.3541473882774517, + "tokens_seen": 2942894080 + }, + { + "epoch": 0.89, + "learning_rate": 8.765496489468406e-05, + "loss": 1.1293, + "theoretical_loss": 3.3541416718466976, + "tokens_seen": 2942959616 + }, + { + "epoch": 0.89, + "learning_rate": 8.763891675025075e-05, + "loss": 1.1521, + "theoretical_loss": 3.354135955578882, + "tokens_seen": 2943025152 + }, + { + "epoch": 0.89, + "learning_rate": 8.762286860581745e-05, + "loss": 1.1504, + "theoretical_loss": 3.354130239473997, + "tokens_seen": 2943090688 + }, + { + "epoch": 0.89, + "learning_rate": 8.760682046138415e-05, + "loss": 1.2585, + "theoretical_loss": 3.3541245235320343, + "tokens_seen": 2943156224 + }, + { + "epoch": 0.89, + "learning_rate": 8.759077231695085e-05, + "loss": 1.2806, + "theoretical_loss": 3.354118807752985, + "tokens_seen": 2943221760 + }, + { + "epoch": 0.89, + "learning_rate": 8.757472417251755e-05, + "loss": 1.2505, + "theoretical_loss": 3.354113092136842, + "tokens_seen": 2943287296 + }, + { + "epoch": 0.89, + "learning_rate": 8.755867602808425e-05, + "loss": 1.2446, + "theoretical_loss": 3.354107376683596, + "tokens_seen": 2943352832 + }, + { + "epoch": 0.89, + "learning_rate": 8.754262788365095e-05, + "loss": 1.1917, + "theoretical_loss": 3.3541016613932393, + "tokens_seen": 2943418368 + }, + { + "epoch": 0.89, + "learning_rate": 8.752657973921767e-05, + "loss": 1.1572, + "theoretical_loss": 3.354095946265763, + "tokens_seen": 2943483904 + }, + { + "epoch": 0.89, + "learning_rate": 8.751053159478437e-05, + "loss": 1.2448, + "theoretical_loss": 3.35409023130116, + "tokens_seen": 2943549440 + }, + { + "epoch": 0.89, + "learning_rate": 8.749448345035107e-05, + "loss": 1.181, + "theoretical_loss": 3.3540845164994204, + "tokens_seen": 2943614976 + }, + { + "epoch": 0.89, + "learning_rate": 8.747843530591775e-05, + "loss": 1.2084, + "theoretical_loss": 3.3540788018605374, + "tokens_seen": 2943680512 + }, + { + "epoch": 0.89, + "learning_rate": 8.746238716148446e-05, + "loss": 1.1838, + "theoretical_loss": 3.3540730873845024, + "tokens_seen": 2943746048 + }, + { + "epoch": 0.89, + "learning_rate": 8.744633901705116e-05, + "loss": 1.2221, + "theoretical_loss": 3.354067373071307, + "tokens_seen": 2943811584 + }, + { + "epoch": 0.89, + "learning_rate": 8.743029087261786e-05, + "loss": 1.1845, + "theoretical_loss": 3.3540616589209424, + "tokens_seen": 2943877120 + }, + { + "epoch": 0.89, + "learning_rate": 8.741424272818456e-05, + "loss": 1.2577, + "theoretical_loss": 3.3540559449334006, + "tokens_seen": 2943942656 + }, + { + "epoch": 0.89, + "learning_rate": 8.739819458375126e-05, + "loss": 1.3157, + "theoretical_loss": 3.354050231108674, + "tokens_seen": 2944008192 + }, + { + "epoch": 0.89, + "learning_rate": 8.738214643931796e-05, + "loss": 1.1752, + "theoretical_loss": 3.3540445174467535, + "tokens_seen": 2944073728 + }, + { + "epoch": 0.89, + "learning_rate": 8.736609829488466e-05, + "loss": 1.2324, + "theoretical_loss": 3.354038803947631, + "tokens_seen": 2944139264 + }, + { + "epoch": 0.89, + "learning_rate": 8.735005015045136e-05, + "loss": 1.2386, + "theoretical_loss": 3.354033090611299, + "tokens_seen": 2944204800 + }, + { + "epoch": 0.89, + "learning_rate": 8.733400200601806e-05, + "loss": 1.0924, + "theoretical_loss": 3.3540273774377485, + "tokens_seen": 2944270336 + }, + { + "epoch": 0.89, + "learning_rate": 8.731795386158476e-05, + "loss": 1.2763, + "theoretical_loss": 3.3540216644269716, + "tokens_seen": 2944335872 + }, + { + "epoch": 0.89, + "learning_rate": 8.730190571715146e-05, + "loss": 1.1994, + "theoretical_loss": 3.354015951578959, + "tokens_seen": 2944401408 + }, + { + "epoch": 0.89, + "learning_rate": 8.728585757271815e-05, + "loss": 1.2092, + "theoretical_loss": 3.354010238893704, + "tokens_seen": 2944466944 + }, + { + "epoch": 0.89, + "learning_rate": 8.726980942828485e-05, + "loss": 1.1455, + "theoretical_loss": 3.354004526371198, + "tokens_seen": 2944532480 + }, + { + "epoch": 0.89, + "learning_rate": 8.725376128385155e-05, + "loss": 1.3828, + "theoretical_loss": 3.3539988140114314, + "tokens_seen": 2944598016 + }, + { + "epoch": 0.89, + "learning_rate": 8.723771313941825e-05, + "loss": 1.2557, + "theoretical_loss": 3.3539931018143974, + "tokens_seen": 2944663552 + }, + { + "epoch": 0.89, + "learning_rate": 8.722166499498496e-05, + "loss": 1.2361, + "theoretical_loss": 3.3539873897800874, + "tokens_seen": 2944729088 + }, + { + "epoch": 0.89, + "learning_rate": 8.720561685055167e-05, + "loss": 1.2417, + "theoretical_loss": 3.353981677908493, + "tokens_seen": 2944794624 + }, + { + "epoch": 0.89, + "learning_rate": 8.718956870611837e-05, + "loss": 1.2707, + "theoretical_loss": 3.353975966199606, + "tokens_seen": 2944860160 + }, + { + "epoch": 0.89, + "learning_rate": 8.717352056168507e-05, + "loss": 1.1281, + "theoretical_loss": 3.3539702546534174, + "tokens_seen": 2944925696 + }, + { + "epoch": 0.89, + "learning_rate": 8.715747241725177e-05, + "loss": 1.2662, + "theoretical_loss": 3.35396454326992, + "tokens_seen": 2944991232 + }, + { + "epoch": 0.89, + "learning_rate": 8.714142427281847e-05, + "loss": 1.3029, + "theoretical_loss": 3.353958832049105, + "tokens_seen": 2945056768 + }, + { + "epoch": 0.89, + "learning_rate": 8.712537612838516e-05, + "loss": 1.2829, + "theoretical_loss": 3.3539531209909645, + "tokens_seen": 2945122304 + }, + { + "epoch": 0.89, + "learning_rate": 8.710932798395186e-05, + "loss": 1.1806, + "theoretical_loss": 3.3539474100954907, + "tokens_seen": 2945187840 + }, + { + "epoch": 0.89, + "learning_rate": 8.709327983951856e-05, + "loss": 1.1955, + "theoretical_loss": 3.353941699362674, + "tokens_seen": 2945253376 + }, + { + "epoch": 0.89, + "learning_rate": 8.707723169508526e-05, + "loss": 1.1472, + "theoretical_loss": 3.353935988792507, + "tokens_seen": 2945318912 + }, + { + "epoch": 0.89, + "learning_rate": 8.706118355065196e-05, + "loss": 1.1207, + "theoretical_loss": 3.353930278384981, + "tokens_seen": 2945384448 + }, + { + "epoch": 0.89, + "learning_rate": 8.704513540621866e-05, + "loss": 1.1688, + "theoretical_loss": 3.3539245681400884, + "tokens_seen": 2945449984 + }, + { + "epoch": 0.89, + "learning_rate": 8.702908726178536e-05, + "loss": 1.2739, + "theoretical_loss": 3.3539188580578205, + "tokens_seen": 2945515520 + }, + { + "epoch": 0.89, + "learning_rate": 8.701303911735206e-05, + "loss": 1.1321, + "theoretical_loss": 3.353913148138169, + "tokens_seen": 2945581056 + }, + { + "epoch": 0.89, + "learning_rate": 8.699699097291876e-05, + "loss": 1.1574, + "theoretical_loss": 3.3539074383811265, + "tokens_seen": 2945646592 + }, + { + "epoch": 0.89, + "learning_rate": 8.698094282848546e-05, + "loss": 1.2168, + "theoretical_loss": 3.3539017287866835, + "tokens_seen": 2945712128 + }, + { + "epoch": 0.89, + "learning_rate": 8.696489468405216e-05, + "loss": 1.2209, + "theoretical_loss": 3.353896019354832, + "tokens_seen": 2945777664 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 982993, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4611337184906006, + "objective/train/theoretical_loss": 3.353890310085565, + "objective/train/tokens_used": 2966303200, + "theoretical_loss": 3.353890310085565, + "tokens_seen": 2945843200 + }, + { + "epoch": 0.89, + "learning_rate": 8.694884653961886e-05, + "loss": 1.319, + "theoretical_loss": 3.353890310085565, + "tokens_seen": 2945843200 + }, + { + "epoch": 0.89, + "learning_rate": 8.693279839518556e-05, + "loss": 1.1924, + "theoretical_loss": 3.3538846009788728, + "tokens_seen": 2945908736 + }, + { + "epoch": 0.89, + "learning_rate": 8.691675025075226e-05, + "loss": 1.1329, + "theoretical_loss": 3.3538788920347473, + "tokens_seen": 2945974272 + }, + { + "epoch": 0.89, + "learning_rate": 8.690070210631896e-05, + "loss": 1.2053, + "theoretical_loss": 3.353873183253181, + "tokens_seen": 2946039808 + }, + { + "epoch": 0.89, + "learning_rate": 8.688465396188566e-05, + "loss": 1.1422, + "theoretical_loss": 3.3538674746341655, + "tokens_seen": 2946105344 + }, + { + "epoch": 0.89, + "learning_rate": 8.686860581745237e-05, + "loss": 1.1613, + "theoretical_loss": 3.353861766177692, + "tokens_seen": 2946170880 + }, + { + "epoch": 0.89, + "learning_rate": 8.685255767301907e-05, + "loss": 1.2013, + "theoretical_loss": 3.353856057883753, + "tokens_seen": 2946236416 + }, + { + "epoch": 0.89, + "learning_rate": 8.683650952858577e-05, + "loss": 1.1869, + "theoretical_loss": 3.3538503497523395, + "tokens_seen": 2946301952 + }, + { + "epoch": 0.89, + "learning_rate": 8.682046138415247e-05, + "loss": 1.1782, + "theoretical_loss": 3.3538446417834438, + "tokens_seen": 2946367488 + }, + { + "epoch": 0.89, + "learning_rate": 8.680441323971917e-05, + "loss": 1.2148, + "theoretical_loss": 3.353838933977057, + "tokens_seen": 2946433024 + }, + { + "epoch": 0.89, + "learning_rate": 8.678836509528587e-05, + "loss": 1.1713, + "theoretical_loss": 3.3538332263331716, + "tokens_seen": 2946498560 + }, + { + "epoch": 0.89, + "learning_rate": 8.677231695085256e-05, + "loss": 1.2286, + "theoretical_loss": 3.353827518851779, + "tokens_seen": 2946564096 + }, + { + "epoch": 0.89, + "learning_rate": 8.675626880641926e-05, + "loss": 1.1799, + "theoretical_loss": 3.3538218115328715, + "tokens_seen": 2946629632 + }, + { + "epoch": 0.89, + "learning_rate": 8.674022066198596e-05, + "loss": 1.293, + "theoretical_loss": 3.35381610437644, + "tokens_seen": 2946695168 + }, + { + "epoch": 0.89, + "learning_rate": 8.672417251755266e-05, + "loss": 1.1887, + "theoretical_loss": 3.353810397382477, + "tokens_seen": 2946760704 + }, + { + "epoch": 0.89, + "learning_rate": 8.670812437311936e-05, + "loss": 1.2937, + "theoretical_loss": 3.3538046905509735, + "tokens_seen": 2946826240 + }, + { + "epoch": 0.89, + "learning_rate": 8.669207622868606e-05, + "loss": 1.1639, + "theoretical_loss": 3.353798983881922, + "tokens_seen": 2946891776 + }, + { + "epoch": 0.89, + "learning_rate": 8.667602808425276e-05, + "loss": 1.1883, + "theoretical_loss": 3.353793277375314, + "tokens_seen": 2946957312 + }, + { + "epoch": 0.89, + "learning_rate": 8.665997993981946e-05, + "loss": 1.1981, + "theoretical_loss": 3.353787571031141, + "tokens_seen": 2947022848 + }, + { + "epoch": 0.89, + "learning_rate": 8.664393179538616e-05, + "loss": 1.1662, + "theoretical_loss": 3.353781864849395, + "tokens_seen": 2947088384 + }, + { + "epoch": 0.89, + "learning_rate": 8.662788365095286e-05, + "loss": 1.1321, + "theoretical_loss": 3.353776158830068, + "tokens_seen": 2947153920 + }, + { + "epoch": 0.89, + "learning_rate": 8.661183550651956e-05, + "loss": 1.2042, + "theoretical_loss": 3.3537704529731514, + "tokens_seen": 2947219456 + }, + { + "epoch": 0.89, + "learning_rate": 8.659578736208626e-05, + "loss": 1.1883, + "theoretical_loss": 3.3537647472786367, + "tokens_seen": 2947284992 + }, + { + "epoch": 0.89, + "learning_rate": 8.657973921765296e-05, + "loss": 1.2093, + "theoretical_loss": 3.3537590417465166, + "tokens_seen": 2947350528 + }, + { + "epoch": 0.89, + "learning_rate": 8.656369107321966e-05, + "loss": 1.2674, + "theoretical_loss": 3.3537533363767817, + "tokens_seen": 2947416064 + }, + { + "epoch": 0.89, + "learning_rate": 8.654764292878636e-05, + "loss": 1.142, + "theoretical_loss": 3.353747631169425, + "tokens_seen": 2947481600 + }, + { + "epoch": 0.89, + "learning_rate": 8.653159478435307e-05, + "loss": 1.1835, + "theoretical_loss": 3.353741926124437, + "tokens_seen": 2947547136 + }, + { + "epoch": 0.89, + "learning_rate": 8.651554663991977e-05, + "loss": 1.1331, + "theoretical_loss": 3.3537362212418107, + "tokens_seen": 2947612672 + }, + { + "epoch": 0.89, + "learning_rate": 8.649949849548647e-05, + "loss": 1.1945, + "theoretical_loss": 3.353730516521537, + "tokens_seen": 2947678208 + }, + { + "epoch": 0.89, + "learning_rate": 8.648345035105317e-05, + "loss": 1.2163, + "theoretical_loss": 3.353724811963608, + "tokens_seen": 2947743744 + }, + { + "epoch": 0.89, + "learning_rate": 8.646740220661987e-05, + "loss": 1.5161, + "theoretical_loss": 3.3537191075680157, + "tokens_seen": 2947809280 + }, + { + "epoch": 0.89, + "learning_rate": 8.645135406218657e-05, + "loss": 1.3009, + "theoretical_loss": 3.3537134033347513, + "tokens_seen": 2947874816 + }, + { + "epoch": 0.89, + "learning_rate": 8.643530591775326e-05, + "loss": 1.2726, + "theoretical_loss": 3.353707699263807, + "tokens_seen": 2947940352 + }, + { + "epoch": 0.89, + "learning_rate": 8.641925777331996e-05, + "loss": 1.2366, + "theoretical_loss": 3.3537019953551743, + "tokens_seen": 2948005888 + }, + { + "epoch": 0.89, + "learning_rate": 8.640320962888666e-05, + "loss": 1.1641, + "theoretical_loss": 3.3536962916088453, + "tokens_seen": 2948071424 + }, + { + "epoch": 0.89, + "learning_rate": 8.638716148445336e-05, + "loss": 1.2154, + "theoretical_loss": 3.353690588024812, + "tokens_seen": 2948136960 + }, + { + "epoch": 0.89, + "learning_rate": 8.637111334002006e-05, + "loss": 1.2168, + "theoretical_loss": 3.353684884603065, + "tokens_seen": 2948202496 + }, + { + "epoch": 0.89, + "learning_rate": 8.635506519558676e-05, + "loss": 1.2342, + "theoretical_loss": 3.353679181343597, + "tokens_seen": 2948268032 + }, + { + "epoch": 0.89, + "learning_rate": 8.633901705115346e-05, + "loss": 1.2571, + "theoretical_loss": 3.3536734782463995, + "tokens_seen": 2948333568 + }, + { + "epoch": 0.89, + "learning_rate": 8.632296890672017e-05, + "loss": 1.2251, + "theoretical_loss": 3.353667775311465, + "tokens_seen": 2948399104 + }, + { + "epoch": 0.89, + "learning_rate": 8.630692076228687e-05, + "loss": 1.2873, + "theoretical_loss": 3.3536620725387842, + "tokens_seen": 2948464640 + }, + { + "epoch": 0.89, + "learning_rate": 8.629087261785357e-05, + "loss": 1.2166, + "theoretical_loss": 3.3536563699283493, + "tokens_seen": 2948530176 + }, + { + "epoch": 0.89, + "learning_rate": 8.627482447342026e-05, + "loss": 1.2845, + "theoretical_loss": 3.3536506674801525, + "tokens_seen": 2948595712 + }, + { + "epoch": 0.89, + "learning_rate": 8.625877632898696e-05, + "loss": 1.2186, + "theoretical_loss": 3.353644965194185, + "tokens_seen": 2948661248 + }, + { + "epoch": 0.89, + "learning_rate": 8.624272818455366e-05, + "loss": 1.4395, + "theoretical_loss": 3.3536392630704386, + "tokens_seen": 2948726784 + }, + { + "epoch": 0.89, + "learning_rate": 8.622668004012036e-05, + "loss": 1.2688, + "theoretical_loss": 3.3536335611089054, + "tokens_seen": 2948792320 + }, + { + "epoch": 0.89, + "learning_rate": 8.621063189568706e-05, + "loss": 1.2049, + "theoretical_loss": 3.353627859309577, + "tokens_seen": 2948857856 + }, + { + "epoch": 0.89, + "learning_rate": 8.619458375125376e-05, + "loss": 1.251, + "theoretical_loss": 3.3536221576724454, + "tokens_seen": 2948923392 + }, + { + "epoch": 0.89, + "learning_rate": 8.617853560682047e-05, + "loss": 1.416, + "theoretical_loss": 3.353616456197502, + "tokens_seen": 2948988928 + }, + { + "epoch": 0.89, + "learning_rate": 8.616248746238717e-05, + "loss": 1.2485, + "theoretical_loss": 3.353610754884739, + "tokens_seen": 2949054464 + }, + { + "debugging/Compilability": 0.0, + "debugging/distinct-1-grams": 0.7106720966157244, + "debugging/entropy-1-grams": 5.0846983319489585, + "debugging/length": 550.2727272727273, + "debugging/num_segments": 11, + "debugging/score": 0.07093949686876623, + "debugging/score_std": 0.015367332482443576, + "epoch": 0.89, + "objective/train/docs_used": 983893, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3300191164016724, + "objective/train/theoretical_loss": 3.3536050537341477, + "objective/train/tokens_used": 2969580000, + "theoretical_loss": 3.3536050537341477, + "tokens_seen": 2949120000 + }, + { + "epoch": 0.89, + "learning_rate": 8.614643931795387e-05, + "loss": 1.339, + "theoretical_loss": 3.3536050537341477, + "tokens_seen": 2949120000 + }, + { + "epoch": 0.89, + "learning_rate": 8.613039117352057e-05, + "loss": 1.1255, + "theoretical_loss": 3.3535993527457206, + "tokens_seen": 2949185536 + }, + { + "epoch": 0.89, + "learning_rate": 8.611434302908727e-05, + "loss": 1.1152, + "theoretical_loss": 3.3535936519194487, + "tokens_seen": 2949251072 + }, + { + "epoch": 0.89, + "learning_rate": 8.609829488465397e-05, + "loss": 1.2476, + "theoretical_loss": 3.3535879512553244, + "tokens_seen": 2949316608 + }, + { + "epoch": 0.89, + "learning_rate": 8.608224674022066e-05, + "loss": 1.2447, + "theoretical_loss": 3.353582250753339, + "tokens_seen": 2949382144 + }, + { + "epoch": 0.89, + "learning_rate": 8.606619859578736e-05, + "loss": 1.2461, + "theoretical_loss": 3.353576550413484, + "tokens_seen": 2949447680 + }, + { + "epoch": 0.89, + "learning_rate": 8.605015045135406e-05, + "loss": 1.2541, + "theoretical_loss": 3.3535708502357524, + "tokens_seen": 2949513216 + }, + { + "epoch": 0.89, + "learning_rate": 8.603410230692077e-05, + "loss": 1.2422, + "theoretical_loss": 3.353565150220135, + "tokens_seen": 2949578752 + }, + { + "epoch": 0.89, + "learning_rate": 8.601805416248747e-05, + "loss": 1.1944, + "theoretical_loss": 3.3535594503666237, + "tokens_seen": 2949644288 + }, + { + "epoch": 0.89, + "learning_rate": 8.600200601805417e-05, + "loss": 1.4413, + "theoretical_loss": 3.3535537506752107, + "tokens_seen": 2949709824 + }, + { + "epoch": 0.89, + "learning_rate": 8.598595787362087e-05, + "loss": 1.1464, + "theoretical_loss": 3.3535480511458875, + "tokens_seen": 2949775360 + }, + { + "epoch": 0.89, + "learning_rate": 8.596990972918757e-05, + "loss": 1.2734, + "theoretical_loss": 3.3535423517786462, + "tokens_seen": 2949840896 + }, + { + "epoch": 0.89, + "learning_rate": 8.595386158475427e-05, + "loss": 1.2992, + "theoretical_loss": 3.353536652573478, + "tokens_seen": 2949906432 + }, + { + "epoch": 0.89, + "learning_rate": 8.593781344032098e-05, + "loss": 1.4355, + "theoretical_loss": 3.353530953530375, + "tokens_seen": 2949971968 + }, + { + "epoch": 0.89, + "learning_rate": 8.592176529588766e-05, + "loss": 1.3063, + "theoretical_loss": 3.353525254649329, + "tokens_seen": 2950037504 + }, + { + "epoch": 0.89, + "learning_rate": 8.590571715145436e-05, + "loss": 1.1996, + "theoretical_loss": 3.3535195559303315, + "tokens_seen": 2950103040 + }, + { + "epoch": 0.89, + "learning_rate": 8.588966900702106e-05, + "loss": 1.2553, + "theoretical_loss": 3.353513857373375, + "tokens_seen": 2950168576 + }, + { + "epoch": 0.89, + "learning_rate": 8.587362086258776e-05, + "loss": 1.272, + "theoretical_loss": 3.3535081589784506, + "tokens_seen": 2950234112 + }, + { + "epoch": 0.89, + "learning_rate": 8.585757271815446e-05, + "loss": 1.1328, + "theoretical_loss": 3.353502460745551, + "tokens_seen": 2950299648 + }, + { + "epoch": 0.89, + "learning_rate": 8.584152457372117e-05, + "loss": 1.2265, + "theoretical_loss": 3.3534967626746663, + "tokens_seen": 2950365184 + }, + { + "epoch": 0.89, + "learning_rate": 8.582547642928787e-05, + "loss": 1.2316, + "theoretical_loss": 3.35349106476579, + "tokens_seen": 2950430720 + }, + { + "epoch": 0.89, + "learning_rate": 8.580942828485457e-05, + "loss": 1.1754, + "theoretical_loss": 3.353485367018913, + "tokens_seen": 2950496256 + }, + { + "epoch": 0.89, + "learning_rate": 8.579338014042127e-05, + "loss": 1.214, + "theoretical_loss": 3.353479669434028, + "tokens_seen": 2950561792 + }, + { + "epoch": 0.89, + "learning_rate": 8.577733199598797e-05, + "loss": 1.2524, + "theoretical_loss": 3.3534739720111255, + "tokens_seen": 2950627328 + }, + { + "epoch": 0.89, + "learning_rate": 8.576128385155467e-05, + "loss": 1.1605, + "theoretical_loss": 3.353468274750198, + "tokens_seen": 2950692864 + }, + { + "epoch": 0.89, + "learning_rate": 8.574523570712137e-05, + "loss": 1.2096, + "theoretical_loss": 3.3534625776512375, + "tokens_seen": 2950758400 + }, + { + "epoch": 0.89, + "learning_rate": 8.572918756268807e-05, + "loss": 1.2907, + "theoretical_loss": 3.3534568807142353, + "tokens_seen": 2950823936 + }, + { + "epoch": 0.89, + "learning_rate": 8.571313941825477e-05, + "loss": 1.2578, + "theoretical_loss": 3.3534511839391836, + "tokens_seen": 2950889472 + }, + { + "epoch": 0.89, + "learning_rate": 8.569709127382147e-05, + "loss": 1.292, + "theoretical_loss": 3.3534454873260735, + "tokens_seen": 2950955008 + }, + { + "epoch": 0.89, + "learning_rate": 8.568104312938817e-05, + "loss": 1.3696, + "theoretical_loss": 3.353439790874898, + "tokens_seen": 2951020544 + }, + { + "epoch": 0.89, + "learning_rate": 8.566499498495487e-05, + "loss": 1.3787, + "theoretical_loss": 3.353434094585648, + "tokens_seen": 2951086080 + }, + { + "epoch": 0.89, + "learning_rate": 8.564894684052157e-05, + "loss": 1.31, + "theoretical_loss": 3.3534283984583153, + "tokens_seen": 2951151616 + }, + { + "epoch": 0.89, + "learning_rate": 8.563289869608827e-05, + "loss": 1.1501, + "theoretical_loss": 3.3534227024928924, + "tokens_seen": 2951217152 + }, + { + "epoch": 0.89, + "learning_rate": 8.561685055165497e-05, + "loss": 1.2104, + "theoretical_loss": 3.3534170066893703, + "tokens_seen": 2951282688 + }, + { + "epoch": 0.89, + "learning_rate": 8.560080240722167e-05, + "loss": 1.1614, + "theoretical_loss": 3.3534113110477413, + "tokens_seen": 2951348224 + }, + { + "epoch": 0.89, + "learning_rate": 8.558475426278838e-05, + "loss": 1.1575, + "theoretical_loss": 3.353405615567997, + "tokens_seen": 2951413760 + }, + { + "epoch": 0.89, + "learning_rate": 8.556870611835506e-05, + "loss": 1.1283, + "theoretical_loss": 3.353399920250129, + "tokens_seen": 2951479296 + }, + { + "epoch": 0.89, + "learning_rate": 8.555265797392176e-05, + "loss": 1.2216, + "theoretical_loss": 3.3533942250941298, + "tokens_seen": 2951544832 + }, + { + "epoch": 0.89, + "learning_rate": 8.553660982948846e-05, + "loss": 1.1766, + "theoretical_loss": 3.3533885300999904, + "tokens_seen": 2951610368 + }, + { + "epoch": 0.89, + "learning_rate": 8.552056168505516e-05, + "loss": 1.3054, + "theoretical_loss": 3.3533828352677033, + "tokens_seen": 2951675904 + }, + { + "epoch": 0.89, + "learning_rate": 8.550451354062187e-05, + "loss": 1.3598, + "theoretical_loss": 3.3533771405972597, + "tokens_seen": 2951741440 + }, + { + "epoch": 0.89, + "learning_rate": 8.548846539618857e-05, + "loss": 1.2273, + "theoretical_loss": 3.353371446088652, + "tokens_seen": 2951806976 + }, + { + "epoch": 0.89, + "learning_rate": 8.547241725175527e-05, + "loss": 1.1059, + "theoretical_loss": 3.3533657517418716, + "tokens_seen": 2951872512 + }, + { + "epoch": 0.89, + "learning_rate": 8.545636910732197e-05, + "loss": 1.2425, + "theoretical_loss": 3.35336005755691, + "tokens_seen": 2951938048 + }, + { + "epoch": 0.89, + "learning_rate": 8.544032096288867e-05, + "loss": 1.1939, + "theoretical_loss": 3.3533543635337595, + "tokens_seen": 2952003584 + }, + { + "epoch": 0.89, + "learning_rate": 8.542427281845538e-05, + "loss": 1.32, + "theoretical_loss": 3.353348669672412, + "tokens_seen": 2952069120 + }, + { + "epoch": 0.89, + "learning_rate": 8.540822467402207e-05, + "loss": 1.2158, + "theoretical_loss": 3.3533429759728595, + "tokens_seen": 2952134656 + }, + { + "epoch": 0.89, + "learning_rate": 8.539217652958877e-05, + "loss": 1.2141, + "theoretical_loss": 3.353337282435093, + "tokens_seen": 2952200192 + }, + { + "epoch": 0.89, + "learning_rate": 8.537612838515547e-05, + "loss": 1.2911, + "theoretical_loss": 3.3533315890591044, + "tokens_seen": 2952265728 + }, + { + "epoch": 0.89, + "learning_rate": 8.536008024072217e-05, + "loss": 1.2237, + "theoretical_loss": 3.3533258958448866, + "tokens_seen": 2952331264 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 985064, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3512449264526367, + "objective/train/theoretical_loss": 3.35332020279243, + "objective/train/tokens_used": 2972856800, + "theoretical_loss": 3.35332020279243, + "tokens_seen": 2952396800 + }, + { + "epoch": 0.89, + "learning_rate": 8.534403209628887e-05, + "loss": 1.3152, + "theoretical_loss": 3.35332020279243, + "tokens_seen": 2952396800 + }, + { + "epoch": 0.89, + "learning_rate": 8.532798395185557e-05, + "loss": 1.4255, + "theoretical_loss": 3.3533145099017276, + "tokens_seen": 2952462336 + }, + { + "epoch": 0.89, + "learning_rate": 8.531193580742227e-05, + "loss": 1.1281, + "theoretical_loss": 3.3533088171727705, + "tokens_seen": 2952527872 + }, + { + "epoch": 0.89, + "learning_rate": 8.529588766298897e-05, + "loss": 1.1395, + "theoretical_loss": 3.3533031246055507, + "tokens_seen": 2952593408 + }, + { + "epoch": 0.89, + "learning_rate": 8.527983951855567e-05, + "loss": 1.2041, + "theoretical_loss": 3.35329743220006, + "tokens_seen": 2952658944 + }, + { + "epoch": 0.89, + "learning_rate": 8.526379137412237e-05, + "loss": 1.2156, + "theoretical_loss": 3.3532917399562905, + "tokens_seen": 2952724480 + }, + { + "epoch": 0.89, + "learning_rate": 8.524774322968908e-05, + "loss": 1.1767, + "theoretical_loss": 3.3532860478742337, + "tokens_seen": 2952790016 + }, + { + "epoch": 0.89, + "learning_rate": 8.523169508525578e-05, + "loss": 1.2672, + "theoretical_loss": 3.3532803559538813, + "tokens_seen": 2952855552 + }, + { + "epoch": 0.89, + "learning_rate": 8.521564694082246e-05, + "loss": 1.3094, + "theoretical_loss": 3.3532746641952254, + "tokens_seen": 2952921088 + }, + { + "epoch": 0.89, + "learning_rate": 8.519959879638916e-05, + "loss": 1.25, + "theoretical_loss": 3.353268972598258, + "tokens_seen": 2952986624 + }, + { + "epoch": 0.89, + "learning_rate": 8.518355065195586e-05, + "loss": 1.3744, + "theoretical_loss": 3.35326328116297, + "tokens_seen": 2953052160 + }, + { + "epoch": 0.89, + "learning_rate": 8.516750250752256e-05, + "loss": 1.2548, + "theoretical_loss": 3.353257589889354, + "tokens_seen": 2953117696 + }, + { + "epoch": 0.89, + "learning_rate": 8.515145436308927e-05, + "loss": 1.2448, + "theoretical_loss": 3.353251898777402, + "tokens_seen": 2953183232 + }, + { + "epoch": 0.89, + "learning_rate": 8.513540621865597e-05, + "loss": 1.195, + "theoretical_loss": 3.3532462078271053, + "tokens_seen": 2953248768 + }, + { + "epoch": 0.89, + "learning_rate": 8.511935807422268e-05, + "loss": 1.1531, + "theoretical_loss": 3.353240517038456, + "tokens_seen": 2953314304 + }, + { + "epoch": 0.89, + "learning_rate": 8.510330992978938e-05, + "loss": 1.2502, + "theoretical_loss": 3.3532348264114455, + "tokens_seen": 2953379840 + }, + { + "epoch": 0.89, + "learning_rate": 8.508726178535608e-05, + "loss": 1.2662, + "theoretical_loss": 3.3532291359460666, + "tokens_seen": 2953445376 + }, + { + "epoch": 0.9, + "learning_rate": 8.507121364092278e-05, + "loss": 1.3567, + "theoretical_loss": 3.35322344564231, + "tokens_seen": 2953510912 + }, + { + "epoch": 0.9, + "learning_rate": 8.505516549648947e-05, + "loss": 1.2983, + "theoretical_loss": 3.353217755500168, + "tokens_seen": 2953576448 + }, + { + "epoch": 0.9, + "learning_rate": 8.503911735205617e-05, + "loss": 1.2025, + "theoretical_loss": 3.3532120655196325, + "tokens_seen": 2953641984 + }, + { + "epoch": 0.9, + "learning_rate": 8.502306920762287e-05, + "loss": 1.2277, + "theoretical_loss": 3.3532063757006956, + "tokens_seen": 2953707520 + }, + { + "epoch": 0.9, + "learning_rate": 8.500702106318957e-05, + "loss": 1.1912, + "theoretical_loss": 3.353200686043348, + "tokens_seen": 2953773056 + }, + { + "epoch": 0.9, + "learning_rate": 8.499097291875627e-05, + "loss": 1.1161, + "theoretical_loss": 3.353194996547583, + "tokens_seen": 2953838592 + }, + { + "epoch": 0.9, + "learning_rate": 8.497492477432297e-05, + "loss": 1.1414, + "theoretical_loss": 3.3531893072133916, + "tokens_seen": 2953904128 + }, + { + "epoch": 0.9, + "learning_rate": 8.495887662988967e-05, + "loss": 1.211, + "theoretical_loss": 3.3531836180407657, + "tokens_seen": 2953969664 + }, + { + "epoch": 0.9, + "learning_rate": 8.494282848545637e-05, + "loss": 1.2199, + "theoretical_loss": 3.3531779290296972, + "tokens_seen": 2954035200 + }, + { + "epoch": 0.9, + "learning_rate": 8.492678034102307e-05, + "loss": 1.2721, + "theoretical_loss": 3.353172240180178, + "tokens_seen": 2954100736 + }, + { + "epoch": 0.9, + "learning_rate": 8.491073219658977e-05, + "loss": 1.2604, + "theoretical_loss": 3.3531665514921993, + "tokens_seen": 2954166272 + }, + { + "epoch": 0.9, + "learning_rate": 8.489468405215648e-05, + "loss": 1.2826, + "theoretical_loss": 3.353160862965754, + "tokens_seen": 2954231808 + }, + { + "epoch": 0.9, + "learning_rate": 8.487863590772318e-05, + "loss": 1.2416, + "theoretical_loss": 3.3531551746008335, + "tokens_seen": 2954297344 + }, + { + "epoch": 0.9, + "learning_rate": 8.486258776328986e-05, + "loss": 1.1933, + "theoretical_loss": 3.353149486397429, + "tokens_seen": 2954362880 + }, + { + "epoch": 0.9, + "learning_rate": 8.484653961885656e-05, + "loss": 1.2827, + "theoretical_loss": 3.3531437983555334, + "tokens_seen": 2954428416 + }, + { + "epoch": 0.9, + "learning_rate": 8.483049147442328e-05, + "loss": 1.3201, + "theoretical_loss": 3.353138110475138, + "tokens_seen": 2954493952 + }, + { + "epoch": 0.9, + "learning_rate": 8.481444332998998e-05, + "loss": 1.2545, + "theoretical_loss": 3.353132422756234, + "tokens_seen": 2954559488 + }, + { + "epoch": 0.9, + "learning_rate": 8.479839518555668e-05, + "loss": 1.2795, + "theoretical_loss": 3.3531267351988143, + "tokens_seen": 2954625024 + }, + { + "epoch": 0.9, + "learning_rate": 8.478234704112338e-05, + "loss": 1.2369, + "theoretical_loss": 3.3531210478028703, + "tokens_seen": 2954690560 + }, + { + "epoch": 0.9, + "learning_rate": 8.476629889669008e-05, + "loss": 1.1969, + "theoretical_loss": 3.3531153605683937, + "tokens_seen": 2954756096 + }, + { + "epoch": 0.9, + "learning_rate": 8.475025075225678e-05, + "loss": 1.2077, + "theoretical_loss": 3.3531096734953767, + "tokens_seen": 2954821632 + }, + { + "epoch": 0.9, + "learning_rate": 8.473420260782348e-05, + "loss": 1.1889, + "theoretical_loss": 3.353103986583811, + "tokens_seen": 2954887168 + }, + { + "epoch": 0.9, + "learning_rate": 8.471815446339017e-05, + "loss": 1.2115, + "theoretical_loss": 3.353098299833688, + "tokens_seen": 2954952704 + }, + { + "epoch": 0.9, + "learning_rate": 8.470210631895687e-05, + "loss": 1.2122, + "theoretical_loss": 3.353092613245, + "tokens_seen": 2955018240 + }, + { + "epoch": 0.9, + "learning_rate": 8.468605817452357e-05, + "loss": 1.2232, + "theoretical_loss": 3.3530869268177383, + "tokens_seen": 2955083776 + }, + { + "epoch": 0.9, + "learning_rate": 8.467001003009027e-05, + "loss": 1.2467, + "theoretical_loss": 3.3530812405518957, + "tokens_seen": 2955149312 + }, + { + "epoch": 0.9, + "learning_rate": 8.465396188565697e-05, + "loss": 1.1996, + "theoretical_loss": 3.3530755544474635, + "tokens_seen": 2955214848 + }, + { + "epoch": 0.9, + "learning_rate": 8.463791374122367e-05, + "loss": 1.3338, + "theoretical_loss": 3.353069868504433, + "tokens_seen": 2955280384 + }, + { + "epoch": 0.9, + "learning_rate": 8.462186559679037e-05, + "loss": 1.2376, + "theoretical_loss": 3.3530641827227967, + "tokens_seen": 2955345920 + }, + { + "epoch": 0.9, + "learning_rate": 8.460581745235707e-05, + "loss": 1.1752, + "theoretical_loss": 3.3530584971025466, + "tokens_seen": 2955411456 + }, + { + "epoch": 0.9, + "learning_rate": 8.458976930792377e-05, + "loss": 1.3879, + "theoretical_loss": 3.353052811643674, + "tokens_seen": 2955476992 + }, + { + "epoch": 0.9, + "learning_rate": 8.457372116349047e-05, + "loss": 1.1214, + "theoretical_loss": 3.3530471263461714, + "tokens_seen": 2955542528 + }, + { + "epoch": 0.9, + "learning_rate": 8.455767301905718e-05, + "loss": 1.2637, + "theoretical_loss": 3.3530414412100296, + "tokens_seen": 2955608064 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 985992, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2401387691497803, + "objective/train/theoretical_loss": 3.3530357562352417, + "objective/train/tokens_used": 2976133600, + "theoretical_loss": 3.3530357562352417, + "tokens_seen": 2955673600 + }, + { + "epoch": 0.9, + "learning_rate": 8.454162487462388e-05, + "loss": 1.2375, + "theoretical_loss": 3.3530357562352417, + "tokens_seen": 2955673600 + }, + { + "epoch": 0.9, + "learning_rate": 8.452557673019058e-05, + "loss": 1.2117, + "theoretical_loss": 3.3530300714217987, + "tokens_seen": 2955739136 + }, + { + "epoch": 0.9, + "learning_rate": 8.450952858575728e-05, + "loss": 1.3363, + "theoretical_loss": 3.3530243867696923, + "tokens_seen": 2955804672 + }, + { + "epoch": 0.9, + "learning_rate": 8.449348044132398e-05, + "loss": 1.237, + "theoretical_loss": 3.3530187022789146, + "tokens_seen": 2955870208 + }, + { + "epoch": 0.9, + "learning_rate": 8.447743229689068e-05, + "loss": 1.2105, + "theoretical_loss": 3.353013017949458, + "tokens_seen": 2955935744 + }, + { + "epoch": 0.9, + "learning_rate": 8.446138415245738e-05, + "loss": 1.288, + "theoretical_loss": 3.353007333781314, + "tokens_seen": 2956001280 + }, + { + "epoch": 0.9, + "learning_rate": 8.444533600802408e-05, + "loss": 1.2048, + "theoretical_loss": 3.353001649774474, + "tokens_seen": 2956066816 + }, + { + "epoch": 0.9, + "learning_rate": 8.442928786359078e-05, + "loss": 1.262, + "theoretical_loss": 3.3529959659289306, + "tokens_seen": 2956132352 + }, + { + "epoch": 0.9, + "learning_rate": 8.441323971915748e-05, + "loss": 1.1518, + "theoretical_loss": 3.352990282244675, + "tokens_seen": 2956197888 + }, + { + "epoch": 0.9, + "learning_rate": 8.439719157472418e-05, + "loss": 1.1951, + "theoretical_loss": 3.352984598721699, + "tokens_seen": 2956263424 + }, + { + "epoch": 0.9, + "learning_rate": 8.438114343029088e-05, + "loss": 1.1446, + "theoretical_loss": 3.3529789153599947, + "tokens_seen": 2956328960 + }, + { + "epoch": 0.9, + "learning_rate": 8.436509528585757e-05, + "loss": 1.1414, + "theoretical_loss": 3.3529732321595542, + "tokens_seen": 2956394496 + }, + { + "epoch": 0.9, + "learning_rate": 8.434904714142427e-05, + "loss": 1.2498, + "theoretical_loss": 3.352967549120369, + "tokens_seen": 2956460032 + }, + { + "epoch": 0.9, + "learning_rate": 8.433299899699097e-05, + "loss": 1.2487, + "theoretical_loss": 3.352961866242431, + "tokens_seen": 2956525568 + }, + { + "epoch": 0.9, + "learning_rate": 8.431695085255767e-05, + "loss": 1.3016, + "theoretical_loss": 3.3529561835257327, + "tokens_seen": 2956591104 + }, + { + "epoch": 0.9, + "learning_rate": 8.430090270812437e-05, + "loss": 1.2353, + "theoretical_loss": 3.352950500970265, + "tokens_seen": 2956656640 + }, + { + "epoch": 0.9, + "learning_rate": 8.428485456369107e-05, + "loss": 1.2164, + "theoretical_loss": 3.3529448185760202, + "tokens_seen": 2956722176 + }, + { + "epoch": 0.9, + "learning_rate": 8.426880641925777e-05, + "loss": 1.2208, + "theoretical_loss": 3.35293913634299, + "tokens_seen": 2956787712 + }, + { + "epoch": 0.9, + "learning_rate": 8.425275827482447e-05, + "loss": 1.1709, + "theoretical_loss": 3.352933454271166, + "tokens_seen": 2956853248 + }, + { + "epoch": 0.9, + "learning_rate": 8.423671013039117e-05, + "loss": 1.2023, + "theoretical_loss": 3.3529277723605406, + "tokens_seen": 2956918784 + }, + { + "epoch": 0.9, + "learning_rate": 8.422066198595789e-05, + "loss": 1.2253, + "theoretical_loss": 3.3529220906111057, + "tokens_seen": 2956984320 + }, + { + "epoch": 0.9, + "learning_rate": 8.420461384152458e-05, + "loss": 1.2421, + "theoretical_loss": 3.352916409022853, + "tokens_seen": 2957049856 + }, + { + "epoch": 0.9, + "learning_rate": 8.418856569709128e-05, + "loss": 1.3134, + "theoretical_loss": 3.352910727595774, + "tokens_seen": 2957115392 + }, + { + "epoch": 0.9, + "learning_rate": 8.417251755265798e-05, + "loss": 1.2168, + "theoretical_loss": 3.3529050463298606, + "tokens_seen": 2957180928 + }, + { + "epoch": 0.9, + "learning_rate": 8.415646940822468e-05, + "loss": 1.4207, + "theoretical_loss": 3.352899365225105, + "tokens_seen": 2957246464 + }, + { + "epoch": 0.9, + "learning_rate": 8.414042126379138e-05, + "loss": 1.1684, + "theoretical_loss": 3.352893684281499, + "tokens_seen": 2957312000 + }, + { + "epoch": 0.9, + "learning_rate": 8.412437311935808e-05, + "loss": 1.1895, + "theoretical_loss": 3.3528880034990345, + "tokens_seen": 2957377536 + }, + { + "epoch": 0.9, + "learning_rate": 8.410832497492478e-05, + "loss": 1.1174, + "theoretical_loss": 3.352882322877703, + "tokens_seen": 2957443072 + }, + { + "epoch": 0.9, + "learning_rate": 8.409227683049148e-05, + "loss": 1.2531, + "theoretical_loss": 3.3528766424174967, + "tokens_seen": 2957508608 + }, + { + "epoch": 0.9, + "learning_rate": 8.407622868605818e-05, + "loss": 1.3752, + "theoretical_loss": 3.3528709621184074, + "tokens_seen": 2957574144 + }, + { + "epoch": 0.9, + "learning_rate": 8.406018054162488e-05, + "loss": 1.1847, + "theoretical_loss": 3.352865281980427, + "tokens_seen": 2957639680 + }, + { + "epoch": 0.9, + "learning_rate": 8.404413239719158e-05, + "loss": 1.1624, + "theoretical_loss": 3.352859602003547, + "tokens_seen": 2957705216 + }, + { + "epoch": 0.9, + "learning_rate": 8.402808425275828e-05, + "loss": 1.2626, + "theoretical_loss": 3.3528539221877596, + "tokens_seen": 2957770752 + }, + { + "epoch": 0.9, + "learning_rate": 8.401203610832497e-05, + "loss": 1.2337, + "theoretical_loss": 3.3528482425330566, + "tokens_seen": 2957836288 + }, + { + "epoch": 0.9, + "learning_rate": 8.399598796389167e-05, + "loss": 1.2552, + "theoretical_loss": 3.35284256303943, + "tokens_seen": 2957901824 + }, + { + "epoch": 0.9, + "learning_rate": 8.397993981945837e-05, + "loss": 1.31, + "theoretical_loss": 3.352836883706871, + "tokens_seen": 2957967360 + }, + { + "epoch": 0.9, + "learning_rate": 8.396389167502507e-05, + "loss": 1.2558, + "theoretical_loss": 3.3528312045353728, + "tokens_seen": 2958032896 + }, + { + "epoch": 0.9, + "learning_rate": 8.394784353059177e-05, + "loss": 1.1992, + "theoretical_loss": 3.352825525524926, + "tokens_seen": 2958098432 + }, + { + "epoch": 0.9, + "learning_rate": 8.393179538615849e-05, + "loss": 1.1358, + "theoretical_loss": 3.3528198466755232, + "tokens_seen": 2958163968 + }, + { + "epoch": 0.9, + "learning_rate": 8.391574724172519e-05, + "loss": 1.14, + "theoretical_loss": 3.3528141679871557, + "tokens_seen": 2958229504 + }, + { + "epoch": 0.9, + "learning_rate": 8.389969909729189e-05, + "loss": 1.1915, + "theoretical_loss": 3.352808489459816, + "tokens_seen": 2958295040 + }, + { + "epoch": 0.9, + "learning_rate": 8.388365095285859e-05, + "loss": 1.1422, + "theoretical_loss": 3.352802811093495, + "tokens_seen": 2958360576 + }, + { + "epoch": 0.9, + "learning_rate": 8.386760280842529e-05, + "loss": 1.2244, + "theoretical_loss": 3.3527971328881856, + "tokens_seen": 2958426112 + }, + { + "epoch": 0.9, + "learning_rate": 8.385155466399198e-05, + "loss": 1.1887, + "theoretical_loss": 3.3527914548438793, + "tokens_seen": 2958491648 + }, + { + "epoch": 0.9, + "learning_rate": 8.383550651955868e-05, + "loss": 1.039, + "theoretical_loss": 3.352785776960568, + "tokens_seen": 2958557184 + }, + { + "epoch": 0.9, + "learning_rate": 8.381945837512538e-05, + "loss": 1.2036, + "theoretical_loss": 3.352780099238243, + "tokens_seen": 2958622720 + }, + { + "epoch": 0.9, + "learning_rate": 8.380341023069208e-05, + "loss": 1.218, + "theoretical_loss": 3.352774421676897, + "tokens_seen": 2958688256 + }, + { + "epoch": 0.9, + "learning_rate": 8.378736208625878e-05, + "loss": 1.1938, + "theoretical_loss": 3.3527687442765215, + "tokens_seen": 2958753792 + }, + { + "epoch": 0.9, + "learning_rate": 8.377131394182548e-05, + "loss": 1.2792, + "theoretical_loss": 3.3527630670371087, + "tokens_seen": 2958819328 + }, + { + "epoch": 0.9, + "learning_rate": 8.375526579739218e-05, + "loss": 1.2, + "theoretical_loss": 3.35275738995865, + "tokens_seen": 2958884864 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 987227, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2336535453796387, + "objective/train/theoretical_loss": 3.352751713041137, + "objective/train/tokens_used": 2979410400, + "theoretical_loss": 3.352751713041137, + "tokens_seen": 2958950400 + }, + { + "epoch": 0.9, + "learning_rate": 8.373921765295888e-05, + "loss": 1.208, + "theoretical_loss": 3.352751713041137, + "tokens_seen": 2958950400 + }, + { + "epoch": 0.9, + "learning_rate": 8.372316950852558e-05, + "loss": 1.175, + "theoretical_loss": 3.3527460362845627, + "tokens_seen": 2959015936 + }, + { + "epoch": 0.9, + "learning_rate": 8.370712136409228e-05, + "loss": 1.1981, + "theoretical_loss": 3.3527403596889176, + "tokens_seen": 2959081472 + }, + { + "epoch": 0.9, + "learning_rate": 8.369107321965898e-05, + "loss": 1.2108, + "theoretical_loss": 3.3527346832541944, + "tokens_seen": 2959147008 + }, + { + "epoch": 0.9, + "learning_rate": 8.367502507522568e-05, + "loss": 1.1647, + "theoretical_loss": 3.3527290069803852, + "tokens_seen": 2959212544 + }, + { + "epoch": 0.9, + "learning_rate": 8.365897693079237e-05, + "loss": 1.1439, + "theoretical_loss": 3.3527233308674815, + "tokens_seen": 2959278080 + }, + { + "epoch": 0.9, + "learning_rate": 8.364292878635907e-05, + "loss": 1.3276, + "theoretical_loss": 3.352717654915475, + "tokens_seen": 2959343616 + }, + { + "epoch": 0.9, + "learning_rate": 8.362688064192579e-05, + "loss": 1.2524, + "theoretical_loss": 3.352711979124358, + "tokens_seen": 2959409152 + }, + { + "epoch": 0.9, + "learning_rate": 8.361083249749249e-05, + "loss": 1.2558, + "theoretical_loss": 3.352706303494122, + "tokens_seen": 2959474688 + }, + { + "epoch": 0.9, + "learning_rate": 8.359478435305919e-05, + "loss": 1.1445, + "theoretical_loss": 3.3527006280247584, + "tokens_seen": 2959540224 + }, + { + "epoch": 0.9, + "learning_rate": 8.357873620862589e-05, + "loss": 1.1703, + "theoretical_loss": 3.35269495271626, + "tokens_seen": 2959605760 + }, + { + "epoch": 0.9, + "learning_rate": 8.356268806419259e-05, + "loss": 1.2507, + "theoretical_loss": 3.352689277568619, + "tokens_seen": 2959671296 + }, + { + "epoch": 0.9, + "learning_rate": 8.354663991975929e-05, + "loss": 1.2116, + "theoretical_loss": 3.352683602581826, + "tokens_seen": 2959736832 + }, + { + "epoch": 0.9, + "learning_rate": 8.353059177532599e-05, + "loss": 1.1631, + "theoretical_loss": 3.3526779277558743, + "tokens_seen": 2959802368 + }, + { + "epoch": 0.9, + "learning_rate": 8.351454363089269e-05, + "loss": 1.1053, + "theoretical_loss": 3.3526722530907542, + "tokens_seen": 2959867904 + }, + { + "epoch": 0.9, + "learning_rate": 8.349849548645938e-05, + "loss": 1.1284, + "theoretical_loss": 3.352666578586459, + "tokens_seen": 2959933440 + }, + { + "epoch": 0.9, + "learning_rate": 8.348244734202608e-05, + "loss": 1.1545, + "theoretical_loss": 3.3526609042429794, + "tokens_seen": 2959998976 + }, + { + "epoch": 0.9, + "learning_rate": 8.346639919759278e-05, + "loss": 1.22, + "theoretical_loss": 3.352655230060308, + "tokens_seen": 2960064512 + }, + { + "epoch": 0.9, + "learning_rate": 8.345035105315948e-05, + "loss": 1.1667, + "theoretical_loss": 3.352649556038437, + "tokens_seen": 2960130048 + }, + { + "epoch": 0.9, + "learning_rate": 8.343430290872618e-05, + "loss": 1.2356, + "theoretical_loss": 3.3526438821773574, + "tokens_seen": 2960195584 + }, + { + "epoch": 0.9, + "learning_rate": 8.341825476429288e-05, + "loss": 1.2288, + "theoretical_loss": 3.3526382084770616, + "tokens_seen": 2960261120 + }, + { + "epoch": 0.9, + "learning_rate": 8.340220661985958e-05, + "loss": 1.2646, + "theoretical_loss": 3.352632534937541, + "tokens_seen": 2960326656 + }, + { + "epoch": 0.9, + "learning_rate": 8.338615847542628e-05, + "loss": 1.2119, + "theoretical_loss": 3.3526268615587886, + "tokens_seen": 2960392192 + }, + { + "epoch": 0.9, + "learning_rate": 8.337011033099298e-05, + "loss": 1.2809, + "theoretical_loss": 3.3526211883407955, + "tokens_seen": 2960457728 + }, + { + "epoch": 0.9, + "learning_rate": 8.335406218655968e-05, + "loss": 1.2761, + "theoretical_loss": 3.3526155152835533, + "tokens_seen": 2960523264 + }, + { + "epoch": 0.9, + "learning_rate": 8.333801404212638e-05, + "loss": 1.1208, + "theoretical_loss": 3.3526098423870545, + "tokens_seen": 2960588800 + }, + { + "epoch": 0.9, + "learning_rate": 8.332196589769308e-05, + "loss": 1.1741, + "theoretical_loss": 3.3526041696512903, + "tokens_seen": 2960654336 + }, + { + "epoch": 0.9, + "learning_rate": 8.330591775325978e-05, + "loss": 1.181, + "theoretical_loss": 3.3525984970762535, + "tokens_seen": 2960719872 + }, + { + "epoch": 0.9, + "learning_rate": 8.328986960882648e-05, + "loss": 1.1529, + "theoretical_loss": 3.3525928246619348, + "tokens_seen": 2960785408 + }, + { + "epoch": 0.9, + "learning_rate": 8.327382146439319e-05, + "loss": 1.3091, + "theoretical_loss": 3.3525871524083275, + "tokens_seen": 2960850944 + }, + { + "epoch": 0.9, + "learning_rate": 8.325777331995989e-05, + "loss": 1.1442, + "theoretical_loss": 3.3525814803154224, + "tokens_seen": 2960916480 + }, + { + "epoch": 0.9, + "learning_rate": 8.324172517552659e-05, + "loss": 1.2768, + "theoretical_loss": 3.352575808383212, + "tokens_seen": 2960982016 + }, + { + "epoch": 0.9, + "learning_rate": 8.322567703109329e-05, + "loss": 1.2153, + "theoretical_loss": 3.352570136611688, + "tokens_seen": 2961047552 + }, + { + "epoch": 0.9, + "learning_rate": 8.320962888665999e-05, + "loss": 1.2624, + "theoretical_loss": 3.352564465000842, + "tokens_seen": 2961113088 + }, + { + "epoch": 0.9, + "learning_rate": 8.319358074222669e-05, + "loss": 1.268, + "theoretical_loss": 3.352558793550666, + "tokens_seen": 2961178624 + }, + { + "epoch": 0.9, + "learning_rate": 8.317753259779339e-05, + "loss": 1.168, + "theoretical_loss": 3.3525531222611527, + "tokens_seen": 2961244160 + }, + { + "epoch": 0.9, + "learning_rate": 8.316148445336009e-05, + "loss": 1.1685, + "theoretical_loss": 3.352547451132293, + "tokens_seen": 2961309696 + }, + { + "epoch": 0.9, + "learning_rate": 8.314543630892678e-05, + "loss": 1.2845, + "theoretical_loss": 3.3525417801640787, + "tokens_seen": 2961375232 + }, + { + "epoch": 0.9, + "learning_rate": 8.312938816449348e-05, + "loss": 1.2099, + "theoretical_loss": 3.3525361093565023, + "tokens_seen": 2961440768 + }, + { + "epoch": 0.9, + "learning_rate": 8.311334002006018e-05, + "loss": 1.2149, + "theoretical_loss": 3.352530438709556, + "tokens_seen": 2961506304 + }, + { + "epoch": 0.9, + "learning_rate": 8.309729187562688e-05, + "loss": 1.2282, + "theoretical_loss": 3.3525247682232306, + "tokens_seen": 2961571840 + }, + { + "epoch": 0.9, + "learning_rate": 8.308124373119358e-05, + "loss": 1.1445, + "theoretical_loss": 3.352519097897519, + "tokens_seen": 2961637376 + }, + { + "epoch": 0.9, + "learning_rate": 8.306519558676028e-05, + "loss": 1.2658, + "theoretical_loss": 3.3525134277324127, + "tokens_seen": 2961702912 + }, + { + "epoch": 0.9, + "learning_rate": 8.304914744232698e-05, + "loss": 1.2399, + "theoretical_loss": 3.352507757727903, + "tokens_seen": 2961768448 + }, + { + "epoch": 0.9, + "learning_rate": 8.303309929789368e-05, + "loss": 1.19, + "theoretical_loss": 3.352502087883983, + "tokens_seen": 2961833984 + }, + { + "epoch": 0.9, + "learning_rate": 8.30170511534604e-05, + "loss": 1.1964, + "theoretical_loss": 3.3524964182006443, + "tokens_seen": 2961899520 + }, + { + "epoch": 0.9, + "learning_rate": 8.30010030090271e-05, + "loss": 1.2329, + "theoretical_loss": 3.352490748677878, + "tokens_seen": 2961965056 + }, + { + "epoch": 0.9, + "learning_rate": 8.298495486459378e-05, + "loss": 1.1607, + "theoretical_loss": 3.3524850793156764, + "tokens_seen": 2962030592 + }, + { + "epoch": 0.9, + "learning_rate": 8.296890672016048e-05, + "loss": 1.1686, + "theoretical_loss": 3.3524794101140314, + "tokens_seen": 2962096128 + }, + { + "epoch": 0.9, + "learning_rate": 8.295285857572718e-05, + "loss": 1.1949, + "theoretical_loss": 3.3524737410729353, + "tokens_seen": 2962161664 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 988366, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.269360065460205, + "objective/train/theoretical_loss": 3.3524680721923796, + "objective/train/tokens_used": 2982687200, + "theoretical_loss": 3.3524680721923796, + "tokens_seen": 2962227200 + }, + { + "epoch": 0.9, + "learning_rate": 8.293681043129389e-05, + "loss": 1.2918, + "theoretical_loss": 3.3524680721923796, + "tokens_seen": 2962227200 + }, + { + "epoch": 0.9, + "learning_rate": 8.292076228686059e-05, + "loss": 1.2558, + "theoretical_loss": 3.3524624034723565, + "tokens_seen": 2962292736 + }, + { + "epoch": 0.9, + "learning_rate": 8.290471414242729e-05, + "loss": 1.3503, + "theoretical_loss": 3.3524567349128573, + "tokens_seen": 2962358272 + }, + { + "epoch": 0.9, + "learning_rate": 8.288866599799399e-05, + "loss": 1.1942, + "theoretical_loss": 3.3524510665138747, + "tokens_seen": 2962423808 + }, + { + "epoch": 0.9, + "learning_rate": 8.287261785356069e-05, + "loss": 1.2065, + "theoretical_loss": 3.3524453982754, + "tokens_seen": 2962489344 + }, + { + "epoch": 0.9, + "learning_rate": 8.285656970912739e-05, + "loss": 1.2036, + "theoretical_loss": 3.3524397301974256, + "tokens_seen": 2962554880 + }, + { + "epoch": 0.9, + "learning_rate": 8.284052156469409e-05, + "loss": 1.2753, + "theoretical_loss": 3.3524340622799427, + "tokens_seen": 2962620416 + }, + { + "epoch": 0.9, + "learning_rate": 8.282447342026079e-05, + "loss": 1.207, + "theoretical_loss": 3.352428394522944, + "tokens_seen": 2962685952 + }, + { + "epoch": 0.9, + "learning_rate": 8.280842527582749e-05, + "loss": 1.1992, + "theoretical_loss": 3.3524227269264206, + "tokens_seen": 2962751488 + }, + { + "epoch": 0.9, + "learning_rate": 8.279237713139418e-05, + "loss": 1.2342, + "theoretical_loss": 3.352417059490365, + "tokens_seen": 2962817024 + }, + { + "epoch": 0.9, + "learning_rate": 8.277632898696088e-05, + "loss": 1.2175, + "theoretical_loss": 3.352411392214769, + "tokens_seen": 2962882560 + }, + { + "epoch": 0.9, + "learning_rate": 8.276028084252758e-05, + "loss": 1.235, + "theoretical_loss": 3.3524057250996244, + "tokens_seen": 2962948096 + }, + { + "epoch": 0.9, + "learning_rate": 8.274423269809428e-05, + "loss": 1.2284, + "theoretical_loss": 3.3524000581449234, + "tokens_seen": 2963013632 + }, + { + "epoch": 0.9, + "learning_rate": 8.2728184553661e-05, + "loss": 1.2267, + "theoretical_loss": 3.352394391350657, + "tokens_seen": 2963079168 + }, + { + "epoch": 0.9, + "learning_rate": 8.27121364092277e-05, + "loss": 1.2801, + "theoretical_loss": 3.3523887247168185, + "tokens_seen": 2963144704 + }, + { + "epoch": 0.9, + "learning_rate": 8.26960882647944e-05, + "loss": 1.3047, + "theoretical_loss": 3.352383058243399, + "tokens_seen": 2963210240 + }, + { + "epoch": 0.9, + "learning_rate": 8.26800401203611e-05, + "loss": 1.1336, + "theoretical_loss": 3.35237739193039, + "tokens_seen": 2963275776 + }, + { + "epoch": 0.9, + "learning_rate": 8.26639919759278e-05, + "loss": 1.1817, + "theoretical_loss": 3.3523717257777843, + "tokens_seen": 2963341312 + }, + { + "epoch": 0.9, + "learning_rate": 8.264794383149448e-05, + "loss": 1.1944, + "theoretical_loss": 3.352366059785574, + "tokens_seen": 2963406848 + }, + { + "epoch": 0.9, + "learning_rate": 8.263189568706118e-05, + "loss": 1.193, + "theoretical_loss": 3.3523603939537496, + "tokens_seen": 2963472384 + }, + { + "epoch": 0.9, + "learning_rate": 8.261584754262788e-05, + "loss": 1.1899, + "theoretical_loss": 3.352354728282304, + "tokens_seen": 2963537920 + }, + { + "epoch": 0.9, + "learning_rate": 8.259979939819459e-05, + "loss": 1.2067, + "theoretical_loss": 3.352349062771229, + "tokens_seen": 2963603456 + }, + { + "epoch": 0.9, + "learning_rate": 8.258375125376129e-05, + "loss": 1.1707, + "theoretical_loss": 3.3523433974205163, + "tokens_seen": 2963668992 + }, + { + "epoch": 0.9, + "learning_rate": 8.256770310932799e-05, + "loss": 1.2262, + "theoretical_loss": 3.352337732230158, + "tokens_seen": 2963734528 + }, + { + "epoch": 0.9, + "learning_rate": 8.255165496489469e-05, + "loss": 1.1621, + "theoretical_loss": 3.3523320672001464, + "tokens_seen": 2963800064 + }, + { + "epoch": 0.9, + "learning_rate": 8.253560682046139e-05, + "loss": 1.125, + "theoretical_loss": 3.3523264023304726, + "tokens_seen": 2963865600 + }, + { + "epoch": 0.9, + "learning_rate": 8.251955867602809e-05, + "loss": 1.2217, + "theoretical_loss": 3.3523207376211293, + "tokens_seen": 2963931136 + }, + { + "epoch": 0.9, + "learning_rate": 8.250351053159479e-05, + "loss": 1.2149, + "theoretical_loss": 3.352315073072108, + "tokens_seen": 2963996672 + }, + { + "epoch": 0.9, + "learning_rate": 8.248746238716149e-05, + "loss": 1.25, + "theoretical_loss": 3.3523094086834004, + "tokens_seen": 2964062208 + }, + { + "epoch": 0.9, + "learning_rate": 8.247141424272819e-05, + "loss": 1.2123, + "theoretical_loss": 3.352303744454999, + "tokens_seen": 2964127744 + }, + { + "epoch": 0.9, + "learning_rate": 8.245536609829488e-05, + "loss": 1.2773, + "theoretical_loss": 3.3522980803868956, + "tokens_seen": 2964193280 + }, + { + "epoch": 0.9, + "learning_rate": 8.243931795386158e-05, + "loss": 1.2015, + "theoretical_loss": 3.3522924164790813, + "tokens_seen": 2964258816 + }, + { + "epoch": 0.9, + "learning_rate": 8.242326980942829e-05, + "loss": 1.2084, + "theoretical_loss": 3.352286752731549, + "tokens_seen": 2964324352 + }, + { + "epoch": 0.9, + "learning_rate": 8.240722166499499e-05, + "loss": 1.2715, + "theoretical_loss": 3.3522810891442902, + "tokens_seen": 2964389888 + }, + { + "epoch": 0.9, + "learning_rate": 8.23911735205617e-05, + "loss": 1.2582, + "theoretical_loss": 3.352275425717297, + "tokens_seen": 2964455424 + }, + { + "epoch": 0.9, + "learning_rate": 8.23751253761284e-05, + "loss": 1.2255, + "theoretical_loss": 3.352269762450561, + "tokens_seen": 2964520960 + }, + { + "epoch": 0.9, + "learning_rate": 8.23590772316951e-05, + "loss": 1.0908, + "theoretical_loss": 3.3522640993440747, + "tokens_seen": 2964586496 + }, + { + "epoch": 0.9, + "learning_rate": 8.23430290872618e-05, + "loss": 1.2653, + "theoretical_loss": 3.3522584363978294, + "tokens_seen": 2964652032 + }, + { + "epoch": 0.9, + "learning_rate": 8.23269809428285e-05, + "loss": 1.2053, + "theoretical_loss": 3.3522527736118173, + "tokens_seen": 2964717568 + }, + { + "epoch": 0.9, + "learning_rate": 8.23109327983952e-05, + "loss": 1.1766, + "theoretical_loss": 3.3522471109860303, + "tokens_seen": 2964783104 + }, + { + "epoch": 0.9, + "learning_rate": 8.229488465396188e-05, + "loss": 1.2795, + "theoretical_loss": 3.3522414485204606, + "tokens_seen": 2964848640 + }, + { + "epoch": 0.9, + "learning_rate": 8.227883650952858e-05, + "loss": 1.1392, + "theoretical_loss": 3.3522357862150995, + "tokens_seen": 2964914176 + }, + { + "epoch": 0.9, + "learning_rate": 8.226278836509528e-05, + "loss": 1.3555, + "theoretical_loss": 3.3522301240699393, + "tokens_seen": 2964979712 + }, + { + "epoch": 0.9, + "learning_rate": 8.224674022066199e-05, + "loss": 1.2917, + "theoretical_loss": 3.352224462084972, + "tokens_seen": 2965045248 + }, + { + "epoch": 0.9, + "learning_rate": 8.223069207622869e-05, + "loss": 1.2315, + "theoretical_loss": 3.35221880026019, + "tokens_seen": 2965110784 + }, + { + "epoch": 0.9, + "learning_rate": 8.221464393179539e-05, + "loss": 1.1651, + "theoretical_loss": 3.352213138595584, + "tokens_seen": 2965176320 + }, + { + "epoch": 0.9, + "learning_rate": 8.219859578736209e-05, + "loss": 1.1283, + "theoretical_loss": 3.3522074770911465, + "tokens_seen": 2965241856 + }, + { + "epoch": 0.9, + "learning_rate": 8.218254764292879e-05, + "loss": 1.0844, + "theoretical_loss": 3.35220181574687, + "tokens_seen": 2965307392 + }, + { + "epoch": 0.9, + "learning_rate": 8.216649949849549e-05, + "loss": 1.2447, + "theoretical_loss": 3.352196154562746, + "tokens_seen": 2965372928 + }, + { + "epoch": 0.9, + "learning_rate": 8.215045135406219e-05, + "loss": 1.2008, + "theoretical_loss": 3.3521904935387656, + "tokens_seen": 2965438464 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 989280, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1992205381393433, + "objective/train/theoretical_loss": 3.352184832674922, + "objective/train/tokens_used": 2985964000, + "theoretical_loss": 3.352184832674922, + "tokens_seen": 2965504000 + }, + { + "epoch": 0.9, + "learning_rate": 8.213440320962889e-05, + "loss": 1.2043, + "theoretical_loss": 3.352184832674922, + "tokens_seen": 2965504000 + }, + { + "epoch": 0.9, + "learning_rate": 8.211835506519559e-05, + "loss": 1.1981, + "theoretical_loss": 3.3521791719712066, + "tokens_seen": 2965569536 + }, + { + "epoch": 0.9, + "learning_rate": 8.210230692076229e-05, + "loss": 1.0814, + "theoretical_loss": 3.352173511427612, + "tokens_seen": 2965635072 + }, + { + "epoch": 0.9, + "learning_rate": 8.208625877632899e-05, + "loss": 1.2681, + "theoretical_loss": 3.3521678510441286, + "tokens_seen": 2965700608 + }, + { + "epoch": 0.9, + "learning_rate": 8.207021063189569e-05, + "loss": 1.2386, + "theoretical_loss": 3.3521621908207497, + "tokens_seen": 2965766144 + }, + { + "epoch": 0.9, + "learning_rate": 8.205416248746239e-05, + "loss": 1.2667, + "theoretical_loss": 3.352156530757467, + "tokens_seen": 2965831680 + }, + { + "epoch": 0.9, + "learning_rate": 8.20381143430291e-05, + "loss": 1.286, + "theoretical_loss": 3.3521508708542718, + "tokens_seen": 2965897216 + }, + { + "epoch": 0.9, + "learning_rate": 8.20220661985958e-05, + "loss": 1.3253, + "theoretical_loss": 3.3521452111111567, + "tokens_seen": 2965962752 + }, + { + "epoch": 0.9, + "learning_rate": 8.20060180541625e-05, + "loss": 1.233, + "theoretical_loss": 3.352139551528113, + "tokens_seen": 2966028288 + }, + { + "epoch": 0.9, + "learning_rate": 8.19899699097292e-05, + "loss": 1.0891, + "theoretical_loss": 3.3521338921051336, + "tokens_seen": 2966093824 + }, + { + "epoch": 0.9, + "learning_rate": 8.19739217652959e-05, + "loss": 1.1662, + "theoretical_loss": 3.352128232842209, + "tokens_seen": 2966159360 + }, + { + "epoch": 0.9, + "learning_rate": 8.19578736208626e-05, + "loss": 1.1574, + "theoretical_loss": 3.352122573739333, + "tokens_seen": 2966224896 + }, + { + "epoch": 0.9, + "learning_rate": 8.194182547642928e-05, + "loss": 1.2324, + "theoretical_loss": 3.352116914796496, + "tokens_seen": 2966290432 + }, + { + "epoch": 0.9, + "learning_rate": 8.192577733199598e-05, + "loss": 1.2877, + "theoretical_loss": 3.3521112560136905, + "tokens_seen": 2966355968 + }, + { + "epoch": 0.9, + "learning_rate": 8.190972918756269e-05, + "loss": 1.1679, + "theoretical_loss": 3.3521055973909086, + "tokens_seen": 2966421504 + }, + { + "epoch": 0.9, + "learning_rate": 8.189368104312939e-05, + "loss": 1.1106, + "theoretical_loss": 3.352099938928142, + "tokens_seen": 2966487040 + }, + { + "epoch": 0.9, + "learning_rate": 8.187763289869609e-05, + "loss": 1.1583, + "theoretical_loss": 3.352094280625382, + "tokens_seen": 2966552576 + }, + { + "epoch": 0.9, + "learning_rate": 8.186158475426279e-05, + "loss": 1.2446, + "theoretical_loss": 3.352088622482622, + "tokens_seen": 2966618112 + }, + { + "epoch": 0.9, + "learning_rate": 8.184553660982949e-05, + "loss": 1.2121, + "theoretical_loss": 3.352082964499853, + "tokens_seen": 2966683648 + }, + { + "epoch": 0.9, + "learning_rate": 8.18294884653962e-05, + "loss": 1.2699, + "theoretical_loss": 3.3520773066770673, + "tokens_seen": 2966749184 + }, + { + "epoch": 0.9, + "learning_rate": 8.18134403209629e-05, + "loss": 1.151, + "theoretical_loss": 3.352071649014256, + "tokens_seen": 2966814720 + }, + { + "epoch": 0.9, + "learning_rate": 8.17973921765296e-05, + "loss": 1.2754, + "theoretical_loss": 3.3520659915114126, + "tokens_seen": 2966880256 + }, + { + "epoch": 0.9, + "learning_rate": 8.178134403209629e-05, + "loss": 1.1794, + "theoretical_loss": 3.3520603341685273, + "tokens_seen": 2966945792 + }, + { + "epoch": 0.9, + "learning_rate": 8.176529588766299e-05, + "loss": 1.2582, + "theoretical_loss": 3.3520546769855937, + "tokens_seen": 2967011328 + }, + { + "epoch": 0.9, + "learning_rate": 8.174924774322969e-05, + "loss": 1.1818, + "theoretical_loss": 3.3520490199626023, + "tokens_seen": 2967076864 + }, + { + "epoch": 0.9, + "learning_rate": 8.173319959879639e-05, + "loss": 1.1467, + "theoretical_loss": 3.352043363099546, + "tokens_seen": 2967142400 + }, + { + "epoch": 0.9, + "learning_rate": 8.171715145436309e-05, + "loss": 1.1239, + "theoretical_loss": 3.352037706396416, + "tokens_seen": 2967207936 + }, + { + "epoch": 0.9, + "learning_rate": 8.17011033099298e-05, + "loss": 1.2748, + "theoretical_loss": 3.3520320498532055, + "tokens_seen": 2967273472 + }, + { + "epoch": 0.9, + "learning_rate": 8.16850551654965e-05, + "loss": 1.1852, + "theoretical_loss": 3.352026393469905, + "tokens_seen": 2967339008 + }, + { + "epoch": 0.9, + "learning_rate": 8.16690070210632e-05, + "loss": 1.2183, + "theoretical_loss": 3.352020737246507, + "tokens_seen": 2967404544 + }, + { + "epoch": 0.9, + "learning_rate": 8.16529588766299e-05, + "loss": 1.2541, + "theoretical_loss": 3.3520150811830036, + "tokens_seen": 2967470080 + }, + { + "epoch": 0.9, + "learning_rate": 8.16369107321966e-05, + "loss": 1.1847, + "theoretical_loss": 3.3520094252793866, + "tokens_seen": 2967535616 + }, + { + "epoch": 0.9, + "learning_rate": 8.16208625877633e-05, + "loss": 1.1741, + "theoretical_loss": 3.3520037695356484, + "tokens_seen": 2967601152 + }, + { + "epoch": 0.9, + "learning_rate": 8.160481444333e-05, + "loss": 1.2257, + "theoretical_loss": 3.35199811395178, + "tokens_seen": 2967666688 + }, + { + "epoch": 0.9, + "learning_rate": 8.158876629889668e-05, + "loss": 1.1775, + "theoretical_loss": 3.3519924585277745, + "tokens_seen": 2967732224 + }, + { + "epoch": 0.9, + "learning_rate": 8.157271815446339e-05, + "loss": 1.1866, + "theoretical_loss": 3.351986803263623, + "tokens_seen": 2967797760 + }, + { + "epoch": 0.9, + "learning_rate": 8.155667001003009e-05, + "loss": 1.2281, + "theoretical_loss": 3.3519811481593176, + "tokens_seen": 2967863296 + }, + { + "epoch": 0.9, + "learning_rate": 8.154062186559679e-05, + "loss": 1.2054, + "theoretical_loss": 3.3519754932148507, + "tokens_seen": 2967928832 + }, + { + "epoch": 0.9, + "learning_rate": 8.15245737211635e-05, + "loss": 1.2385, + "theoretical_loss": 3.3519698384302137, + "tokens_seen": 2967994368 + }, + { + "epoch": 0.9, + "learning_rate": 8.15085255767302e-05, + "loss": 1.2596, + "theoretical_loss": 3.351964183805398, + "tokens_seen": 2968059904 + }, + { + "epoch": 0.9, + "learning_rate": 8.14924774322969e-05, + "loss": 1.2476, + "theoretical_loss": 3.3519585293403975, + "tokens_seen": 2968125440 + }, + { + "epoch": 0.9, + "learning_rate": 8.14764292878636e-05, + "loss": 1.2139, + "theoretical_loss": 3.351952875035203, + "tokens_seen": 2968190976 + }, + { + "epoch": 0.9, + "learning_rate": 8.14603811434303e-05, + "loss": 1.1488, + "theoretical_loss": 3.3519472208898056, + "tokens_seen": 2968256512 + }, + { + "epoch": 0.9, + "learning_rate": 8.1444332998997e-05, + "loss": 1.1399, + "theoretical_loss": 3.3519415669041983, + "tokens_seen": 2968322048 + }, + { + "epoch": 0.9, + "learning_rate": 8.142828485456369e-05, + "loss": 1.1924, + "theoretical_loss": 3.351935913078373, + "tokens_seen": 2968387584 + }, + { + "epoch": 0.9, + "learning_rate": 8.141223671013039e-05, + "loss": 1.1588, + "theoretical_loss": 3.3519302594123217, + "tokens_seen": 2968453120 + }, + { + "epoch": 0.9, + "learning_rate": 8.139618856569709e-05, + "loss": 1.1219, + "theoretical_loss": 3.351924605906036, + "tokens_seen": 2968518656 + }, + { + "epoch": 0.9, + "learning_rate": 8.138014042126379e-05, + "loss": 1.3429, + "theoretical_loss": 3.351918952559508, + "tokens_seen": 2968584192 + }, + { + "epoch": 0.9, + "learning_rate": 8.13640922768305e-05, + "loss": 1.1131, + "theoretical_loss": 3.35191329937273, + "tokens_seen": 2968649728 + }, + { + "epoch": 0.9, + "learning_rate": 8.13480441323972e-05, + "loss": 1.32, + "theoretical_loss": 3.351907646345693, + "tokens_seen": 2968715264 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 990556, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.225261926651001, + "objective/train/theoretical_loss": 3.35190199347839, + "objective/train/tokens_used": 2989240800, + "theoretical_loss": 3.35190199347839, + "tokens_seen": 2968780800 + }, + { + "epoch": 0.9, + "learning_rate": 8.13319959879639e-05, + "loss": 1.1264, + "theoretical_loss": 3.35190199347839, + "tokens_seen": 2968780800 + }, + { + "epoch": 0.9, + "learning_rate": 8.13159478435306e-05, + "loss": 1.195, + "theoretical_loss": 3.3518963407708124, + "tokens_seen": 2968846336 + }, + { + "epoch": 0.9, + "learning_rate": 8.12998996990973e-05, + "loss": 1.1651, + "theoretical_loss": 3.351890688222953, + "tokens_seen": 2968911872 + }, + { + "epoch": 0.9, + "learning_rate": 8.1283851554664e-05, + "loss": 1.1608, + "theoretical_loss": 3.351885035834802, + "tokens_seen": 2968977408 + }, + { + "epoch": 0.9, + "learning_rate": 8.12678034102307e-05, + "loss": 1.1954, + "theoretical_loss": 3.351879383606353, + "tokens_seen": 2969042944 + }, + { + "epoch": 0.9, + "learning_rate": 8.12517552657974e-05, + "loss": 1.2436, + "theoretical_loss": 3.3518737315375975, + "tokens_seen": 2969108480 + }, + { + "epoch": 0.9, + "learning_rate": 8.123570712136408e-05, + "loss": 1.2872, + "theoretical_loss": 3.3518680796285274, + "tokens_seen": 2969174016 + }, + { + "epoch": 0.9, + "learning_rate": 8.12196589769308e-05, + "loss": 1.1989, + "theoretical_loss": 3.3518624278791345, + "tokens_seen": 2969239552 + }, + { + "epoch": 0.9, + "learning_rate": 8.12036108324975e-05, + "loss": 1.1779, + "theoretical_loss": 3.3518567762894107, + "tokens_seen": 2969305088 + }, + { + "epoch": 0.9, + "learning_rate": 8.11875626880642e-05, + "loss": 1.127, + "theoretical_loss": 3.3518511248593486, + "tokens_seen": 2969370624 + }, + { + "epoch": 0.9, + "learning_rate": 8.11715145436309e-05, + "loss": 1.1436, + "theoretical_loss": 3.3518454735889396, + "tokens_seen": 2969436160 + }, + { + "epoch": 0.9, + "learning_rate": 8.11554663991976e-05, + "loss": 1.2408, + "theoretical_loss": 3.3518398224781754, + "tokens_seen": 2969501696 + }, + { + "epoch": 0.9, + "learning_rate": 8.11394182547643e-05, + "loss": 1.2455, + "theoretical_loss": 3.3518341715270488, + "tokens_seen": 2969567232 + }, + { + "epoch": 0.9, + "learning_rate": 8.1123370110331e-05, + "loss": 1.1877, + "theoretical_loss": 3.351828520735551, + "tokens_seen": 2969632768 + }, + { + "epoch": 0.9, + "learning_rate": 8.11073219658977e-05, + "loss": 1.1724, + "theoretical_loss": 3.3518228701036747, + "tokens_seen": 2969698304 + }, + { + "epoch": 0.9, + "learning_rate": 8.10912738214644e-05, + "loss": 1.2723, + "theoretical_loss": 3.3518172196314113, + "tokens_seen": 2969763840 + }, + { + "epoch": 0.9, + "learning_rate": 8.107522567703109e-05, + "loss": 1.2191, + "theoretical_loss": 3.3518115693187527, + "tokens_seen": 2969829376 + }, + { + "epoch": 0.9, + "learning_rate": 8.105917753259779e-05, + "loss": 1.2653, + "theoretical_loss": 3.3518059191656913, + "tokens_seen": 2969894912 + }, + { + "epoch": 0.9, + "learning_rate": 8.104312938816449e-05, + "loss": 1.182, + "theoretical_loss": 3.351800269172219, + "tokens_seen": 2969960448 + }, + { + "epoch": 0.9, + "learning_rate": 8.102708124373119e-05, + "loss": 1.1364, + "theoretical_loss": 3.3517946193383272, + "tokens_seen": 2970025984 + }, + { + "epoch": 0.9, + "learning_rate": 8.10110330992979e-05, + "loss": 1.1579, + "theoretical_loss": 3.3517889696640086, + "tokens_seen": 2970091520 + }, + { + "epoch": 0.9, + "learning_rate": 8.09949849548646e-05, + "loss": 1.213, + "theoretical_loss": 3.3517833201492553, + "tokens_seen": 2970157056 + }, + { + "epoch": 0.9, + "learning_rate": 8.09789368104313e-05, + "loss": 1.2621, + "theoretical_loss": 3.3517776707940583, + "tokens_seen": 2970222592 + }, + { + "epoch": 0.9, + "learning_rate": 8.0962888665998e-05, + "loss": 1.2646, + "theoretical_loss": 3.3517720215984106, + "tokens_seen": 2970288128 + }, + { + "epoch": 0.9, + "learning_rate": 8.09468405215647e-05, + "loss": 1.1281, + "theoretical_loss": 3.3517663725623033, + "tokens_seen": 2970353664 + }, + { + "epoch": 0.9, + "learning_rate": 8.09307923771314e-05, + "loss": 1.304, + "theoretical_loss": 3.351760723685729, + "tokens_seen": 2970419200 + }, + { + "epoch": 0.9, + "learning_rate": 8.09147442326981e-05, + "loss": 1.202, + "theoretical_loss": 3.3517550749686795, + "tokens_seen": 2970484736 + }, + { + "epoch": 0.9, + "learning_rate": 8.08986960882648e-05, + "loss": 1.2446, + "theoretical_loss": 3.3517494264111467, + "tokens_seen": 2970550272 + }, + { + "epoch": 0.9, + "learning_rate": 8.08826479438315e-05, + "loss": 1.2854, + "theoretical_loss": 3.3517437780131223, + "tokens_seen": 2970615808 + }, + { + "epoch": 0.9, + "learning_rate": 8.08665997993982e-05, + "loss": 1.2293, + "theoretical_loss": 3.351738129774599, + "tokens_seen": 2970681344 + }, + { + "epoch": 0.9, + "learning_rate": 8.08505516549649e-05, + "loss": 1.2008, + "theoretical_loss": 3.3517324816955685, + "tokens_seen": 2970746880 + }, + { + "epoch": 0.9, + "learning_rate": 8.08345035105316e-05, + "loss": 1.2136, + "theoretical_loss": 3.351726833776022, + "tokens_seen": 2970812416 + }, + { + "epoch": 0.9, + "learning_rate": 8.08184553660983e-05, + "loss": 1.157, + "theoretical_loss": 3.351721186015953, + "tokens_seen": 2970877952 + }, + { + "epoch": 0.9, + "learning_rate": 8.0802407221665e-05, + "loss": 1.2988, + "theoretical_loss": 3.351715538415352, + "tokens_seen": 2970943488 + }, + { + "epoch": 0.9, + "learning_rate": 8.07863590772317e-05, + "loss": 1.2016, + "theoretical_loss": 3.3517098909742113, + "tokens_seen": 2971009024 + }, + { + "epoch": 0.9, + "learning_rate": 8.07703109327984e-05, + "loss": 1.21, + "theoretical_loss": 3.3517042436925237, + "tokens_seen": 2971074560 + }, + { + "epoch": 0.9, + "learning_rate": 8.07542627883651e-05, + "loss": 1.2215, + "theoretical_loss": 3.3516985965702806, + "tokens_seen": 2971140096 + }, + { + "epoch": 0.9, + "learning_rate": 8.073821464393179e-05, + "loss": 1.2199, + "theoretical_loss": 3.351692949607474, + "tokens_seen": 2971205632 + }, + { + "epoch": 0.9, + "learning_rate": 8.072216649949849e-05, + "loss": 1.2046, + "theoretical_loss": 3.3516873028040957, + "tokens_seen": 2971271168 + }, + { + "epoch": 0.9, + "learning_rate": 8.070611835506519e-05, + "loss": 1.2395, + "theoretical_loss": 3.3516816561601384, + "tokens_seen": 2971336704 + }, + { + "epoch": 0.9, + "learning_rate": 8.069007021063189e-05, + "loss": 1.2431, + "theoretical_loss": 3.351676009675593, + "tokens_seen": 2971402240 + }, + { + "epoch": 0.9, + "learning_rate": 8.06740220661986e-05, + "loss": 1.1718, + "theoretical_loss": 3.3516703633504523, + "tokens_seen": 2971467776 + }, + { + "epoch": 0.9, + "learning_rate": 8.06579739217653e-05, + "loss": 1.2629, + "theoretical_loss": 3.351664717184708, + "tokens_seen": 2971533312 + }, + { + "epoch": 0.9, + "learning_rate": 8.0641925777332e-05, + "loss": 1.1929, + "theoretical_loss": 3.3516590711783523, + "tokens_seen": 2971598848 + }, + { + "epoch": 0.9, + "learning_rate": 8.062587763289871e-05, + "loss": 1.2505, + "theoretical_loss": 3.351653425331377, + "tokens_seen": 2971664384 + }, + { + "epoch": 0.9, + "learning_rate": 8.060982948846541e-05, + "loss": 1.3137, + "theoretical_loss": 3.351647779643774, + "tokens_seen": 2971729920 + }, + { + "epoch": 0.9, + "learning_rate": 8.059378134403211e-05, + "loss": 1.1906, + "theoretical_loss": 3.351642134115535, + "tokens_seen": 2971795456 + }, + { + "epoch": 0.9, + "learning_rate": 8.05777331995988e-05, + "loss": 1.1658, + "theoretical_loss": 3.351636488746653, + "tokens_seen": 2971860992 + }, + { + "epoch": 0.9, + "learning_rate": 8.05616850551655e-05, + "loss": 1.1504, + "theoretical_loss": 3.351630843537119, + "tokens_seen": 2971926528 + }, + { + "epoch": 0.9, + "learning_rate": 8.05456369107322e-05, + "loss": 1.2138, + "theoretical_loss": 3.3516251984869254, + "tokens_seen": 2971992064 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 991429, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2303760051727295, + "objective/train/theoretical_loss": 3.3516195535960644, + "objective/train/tokens_used": 2992517600, + "theoretical_loss": 3.3516195535960644, + "tokens_seen": 2972057600 + }, + { + "epoch": 0.9, + "learning_rate": 8.05295887662989e-05, + "loss": 1.1854, + "theoretical_loss": 3.3516195535960644, + "tokens_seen": 2972057600 + }, + { + "epoch": 0.9, + "learning_rate": 8.05135406218656e-05, + "loss": 1.2909, + "theoretical_loss": 3.351613908864527, + "tokens_seen": 2972123136 + }, + { + "epoch": 0.9, + "learning_rate": 8.04974924774323e-05, + "loss": 1.2008, + "theoretical_loss": 3.351608264292307, + "tokens_seen": 2972188672 + }, + { + "epoch": 0.9, + "learning_rate": 8.0481444332999e-05, + "loss": 1.2199, + "theoretical_loss": 3.3516026198793947, + "tokens_seen": 2972254208 + }, + { + "epoch": 0.9, + "learning_rate": 8.04653961885657e-05, + "loss": 1.2092, + "theoretical_loss": 3.3515969756257826, + "tokens_seen": 2972319744 + }, + { + "epoch": 0.9, + "learning_rate": 8.04493480441324e-05, + "loss": 1.1919, + "theoretical_loss": 3.3515913315314627, + "tokens_seen": 2972385280 + }, + { + "epoch": 0.9, + "learning_rate": 8.04332998996991e-05, + "loss": 1.3041, + "theoretical_loss": 3.3515856875964274, + "tokens_seen": 2972450816 + }, + { + "epoch": 0.9, + "learning_rate": 8.04172517552658e-05, + "loss": 1.185, + "theoretical_loss": 3.351580043820668, + "tokens_seen": 2972516352 + }, + { + "epoch": 0.9, + "learning_rate": 8.04012036108325e-05, + "loss": 1.1213, + "theoretical_loss": 3.3515744002041776, + "tokens_seen": 2972581888 + }, + { + "epoch": 0.9, + "learning_rate": 8.038515546639919e-05, + "loss": 1.2906, + "theoretical_loss": 3.351568756746947, + "tokens_seen": 2972647424 + }, + { + "epoch": 0.9, + "learning_rate": 8.036910732196589e-05, + "loss": 1.1677, + "theoretical_loss": 3.351563113448969, + "tokens_seen": 2972712960 + }, + { + "epoch": 0.9, + "learning_rate": 8.035305917753259e-05, + "loss": 1.2341, + "theoretical_loss": 3.351557470310235, + "tokens_seen": 2972778496 + }, + { + "epoch": 0.9, + "learning_rate": 8.033701103309929e-05, + "loss": 1.1468, + "theoretical_loss": 3.351551827330737, + "tokens_seen": 2972844032 + }, + { + "epoch": 0.9, + "learning_rate": 8.032096288866601e-05, + "loss": 1.1996, + "theoretical_loss": 3.3515461845104673, + "tokens_seen": 2972909568 + }, + { + "epoch": 0.9, + "learning_rate": 8.030491474423271e-05, + "loss": 1.2289, + "theoretical_loss": 3.351540541849418, + "tokens_seen": 2972975104 + }, + { + "epoch": 0.9, + "learning_rate": 8.028886659979941e-05, + "loss": 1.1917, + "theoretical_loss": 3.3515348993475813, + "tokens_seen": 2973040640 + }, + { + "epoch": 0.9, + "learning_rate": 8.027281845536611e-05, + "loss": 1.3031, + "theoretical_loss": 3.351529257004948, + "tokens_seen": 2973106176 + }, + { + "epoch": 0.9, + "learning_rate": 8.025677031093281e-05, + "loss": 1.2019, + "theoretical_loss": 3.3515236148215117, + "tokens_seen": 2973171712 + }, + { + "epoch": 0.9, + "learning_rate": 8.024072216649951e-05, + "loss": 1.1626, + "theoretical_loss": 3.3515179727972635, + "tokens_seen": 2973237248 + }, + { + "epoch": 0.9, + "learning_rate": 8.02246740220662e-05, + "loss": 1.1663, + "theoretical_loss": 3.3515123309321955, + "tokens_seen": 2973302784 + }, + { + "epoch": 0.9, + "learning_rate": 8.02086258776329e-05, + "loss": 1.2321, + "theoretical_loss": 3.3515066892262997, + "tokens_seen": 2973368320 + }, + { + "epoch": 0.9, + "learning_rate": 8.01925777331996e-05, + "loss": 1.3185, + "theoretical_loss": 3.3515010476795677, + "tokens_seen": 2973433856 + }, + { + "epoch": 0.9, + "learning_rate": 8.01765295887663e-05, + "loss": 1.2274, + "theoretical_loss": 3.351495406291993, + "tokens_seen": 2973499392 + }, + { + "epoch": 0.9, + "learning_rate": 8.0160481444333e-05, + "loss": 1.1293, + "theoretical_loss": 3.3514897650635653, + "tokens_seen": 2973564928 + }, + { + "epoch": 0.9, + "learning_rate": 8.01444332998997e-05, + "loss": 1.1647, + "theoretical_loss": 3.3514841239942785, + "tokens_seen": 2973630464 + }, + { + "epoch": 0.9, + "learning_rate": 8.01283851554664e-05, + "loss": 1.2208, + "theoretical_loss": 3.351478483084124, + "tokens_seen": 2973696000 + }, + { + "epoch": 0.9, + "learning_rate": 8.01123370110331e-05, + "loss": 1.1673, + "theoretical_loss": 3.3514728423330937, + "tokens_seen": 2973761536 + }, + { + "epoch": 0.9, + "learning_rate": 8.00962888665998e-05, + "loss": 1.2084, + "theoretical_loss": 3.3514672017411797, + "tokens_seen": 2973827072 + }, + { + "epoch": 0.9, + "learning_rate": 8.00802407221665e-05, + "loss": 1.1881, + "theoretical_loss": 3.3514615613083736, + "tokens_seen": 2973892608 + }, + { + "epoch": 0.9, + "learning_rate": 8.00641925777332e-05, + "loss": 1.2887, + "theoretical_loss": 3.351455921034668, + "tokens_seen": 2973958144 + }, + { + "epoch": 0.9, + "learning_rate": 8.00481444332999e-05, + "loss": 1.2094, + "theoretical_loss": 3.351450280920055, + "tokens_seen": 2974023680 + }, + { + "epoch": 0.9, + "learning_rate": 8.003209628886659e-05, + "loss": 1.24, + "theoretical_loss": 3.3514446409645258, + "tokens_seen": 2974089216 + }, + { + "epoch": 0.9, + "learning_rate": 8.00160481444333e-05, + "loss": 1.1581, + "theoretical_loss": 3.3514390011680733, + "tokens_seen": 2974154752 + }, + { + "epoch": 0.9, + "learning_rate": 8e-05, + "loss": 1.1622, + "theoretical_loss": 3.351433361530689, + "tokens_seen": 2974220288 + }, + { + "epoch": 0.9, + "learning_rate": 7.998395185556671e-05, + "loss": 1.2051, + "theoretical_loss": 3.3514277220523647, + "tokens_seen": 2974285824 + }, + { + "epoch": 0.9, + "learning_rate": 7.996790371113341e-05, + "loss": 1.1598, + "theoretical_loss": 3.351422082733093, + "tokens_seen": 2974351360 + }, + { + "epoch": 0.9, + "learning_rate": 7.995185556670011e-05, + "loss": 1.0993, + "theoretical_loss": 3.3514164435728655, + "tokens_seen": 2974416896 + }, + { + "epoch": 0.9, + "learning_rate": 7.993580742226681e-05, + "loss": 1.1727, + "theoretical_loss": 3.3514108045716746, + "tokens_seen": 2974482432 + }, + { + "epoch": 0.9, + "learning_rate": 7.991975927783351e-05, + "loss": 1.1768, + "theoretical_loss": 3.3514051657295116, + "tokens_seen": 2974547968 + }, + { + "epoch": 0.9, + "learning_rate": 7.990371113340021e-05, + "loss": 1.2532, + "theoretical_loss": 3.3513995270463695, + "tokens_seen": 2974613504 + }, + { + "epoch": 0.9, + "learning_rate": 7.988766298896691e-05, + "loss": 1.2492, + "theoretical_loss": 3.3513938885222396, + "tokens_seen": 2974679040 + }, + { + "epoch": 0.9, + "learning_rate": 7.98716148445336e-05, + "loss": 1.2126, + "theoretical_loss": 3.3513882501571137, + "tokens_seen": 2974744576 + }, + { + "epoch": 0.9, + "learning_rate": 7.98555667001003e-05, + "loss": 1.1361, + "theoretical_loss": 3.3513826119509846, + "tokens_seen": 2974810112 + }, + { + "epoch": 0.9, + "learning_rate": 7.9839518555667e-05, + "loss": 1.2802, + "theoretical_loss": 3.351376973903844, + "tokens_seen": 2974875648 + }, + { + "epoch": 0.9, + "learning_rate": 7.98234704112337e-05, + "loss": 1.2572, + "theoretical_loss": 3.3513713360156836, + "tokens_seen": 2974941184 + }, + { + "epoch": 0.9, + "learning_rate": 7.98074222668004e-05, + "loss": 1.1491, + "theoretical_loss": 3.3513656982864957, + "tokens_seen": 2975006720 + }, + { + "epoch": 0.9, + "learning_rate": 7.97913741223671e-05, + "loss": 1.196, + "theoretical_loss": 3.351360060716272, + "tokens_seen": 2975072256 + }, + { + "epoch": 0.9, + "learning_rate": 7.97753259779338e-05, + "loss": 1.1043, + "theoretical_loss": 3.351354423305005, + "tokens_seen": 2975137792 + }, + { + "epoch": 0.9, + "learning_rate": 7.97592778335005e-05, + "loss": 1.2049, + "theoretical_loss": 3.3513487860526867, + "tokens_seen": 2975203328 + }, + { + "epoch": 0.9, + "learning_rate": 7.97432296890672e-05, + "loss": 1.1775, + "theoretical_loss": 3.351343148959309, + "tokens_seen": 2975268864 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 992447, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1126525402069092, + "objective/train/theoretical_loss": 3.3513375120248634, + "objective/train/tokens_used": 2995794400, + "theoretical_loss": 3.3513375120248634, + "tokens_seen": 2975334400 + }, + { + "epoch": 0.9, + "learning_rate": 7.972718154463392e-05, + "loss": 1.1681, + "theoretical_loss": 3.3513375120248634, + "tokens_seen": 2975334400 + }, + { + "epoch": 0.9, + "learning_rate": 7.97111334002006e-05, + "loss": 1.2595, + "theoretical_loss": 3.3513318752493424, + "tokens_seen": 2975399936 + }, + { + "epoch": 0.9, + "learning_rate": 7.96950852557673e-05, + "loss": 1.2041, + "theoretical_loss": 3.351326238632738, + "tokens_seen": 2975465472 + }, + { + "epoch": 0.9, + "learning_rate": 7.9679037111334e-05, + "loss": 1.2021, + "theoretical_loss": 3.351320602175042, + "tokens_seen": 2975531008 + }, + { + "epoch": 0.9, + "learning_rate": 7.96629889669007e-05, + "loss": 1.2154, + "theoretical_loss": 3.351314965876247, + "tokens_seen": 2975596544 + }, + { + "epoch": 0.9, + "learning_rate": 7.964694082246741e-05, + "loss": 1.2801, + "theoretical_loss": 3.3513093297363445, + "tokens_seen": 2975662080 + }, + { + "epoch": 0.9, + "learning_rate": 7.963089267803411e-05, + "loss": 1.1159, + "theoretical_loss": 3.3513036937553267, + "tokens_seen": 2975727616 + }, + { + "epoch": 0.9, + "learning_rate": 7.961484453360081e-05, + "loss": 1.1623, + "theoretical_loss": 3.3512980579331852, + "tokens_seen": 2975793152 + }, + { + "epoch": 0.9, + "learning_rate": 7.959879638916751e-05, + "loss": 1.2897, + "theoretical_loss": 3.3512924222699128, + "tokens_seen": 2975858688 + }, + { + "epoch": 0.9, + "learning_rate": 7.958274824473421e-05, + "loss": 1.2264, + "theoretical_loss": 3.351286786765501, + "tokens_seen": 2975924224 + }, + { + "epoch": 0.9, + "learning_rate": 7.956670010030091e-05, + "loss": 1.1509, + "theoretical_loss": 3.351281151419942, + "tokens_seen": 2975989760 + }, + { + "epoch": 0.9, + "learning_rate": 7.955065195586761e-05, + "loss": 1.2105, + "theoretical_loss": 3.3512755162332275, + "tokens_seen": 2976055296 + }, + { + "epoch": 0.9, + "learning_rate": 7.953460381143431e-05, + "loss": 1.154, + "theoretical_loss": 3.35126988120535, + "tokens_seen": 2976120832 + }, + { + "epoch": 0.9, + "learning_rate": 7.9518555667001e-05, + "loss": 1.2381, + "theoretical_loss": 3.3512642463363016, + "tokens_seen": 2976186368 + }, + { + "epoch": 0.9, + "learning_rate": 7.95025075225677e-05, + "loss": 1.3048, + "theoretical_loss": 3.3512586116260734, + "tokens_seen": 2976251904 + }, + { + "epoch": 0.9, + "learning_rate": 7.94864593781344e-05, + "loss": 1.1065, + "theoretical_loss": 3.3512529770746586, + "tokens_seen": 2976317440 + }, + { + "epoch": 0.9, + "learning_rate": 7.94704112337011e-05, + "loss": 1.1807, + "theoretical_loss": 3.3512473426820484, + "tokens_seen": 2976382976 + }, + { + "epoch": 0.9, + "learning_rate": 7.94543630892678e-05, + "loss": 1.2849, + "theoretical_loss": 3.351241708448235, + "tokens_seen": 2976448512 + }, + { + "epoch": 0.9, + "learning_rate": 7.94383149448345e-05, + "loss": 1.2454, + "theoretical_loss": 3.351236074373211, + "tokens_seen": 2976514048 + }, + { + "epoch": 0.9, + "learning_rate": 7.942226680040122e-05, + "loss": 1.2326, + "theoretical_loss": 3.3512304404569675, + "tokens_seen": 2976579584 + }, + { + "epoch": 0.9, + "learning_rate": 7.940621865596792e-05, + "loss": 1.2492, + "theoretical_loss": 3.3512248066994976, + "tokens_seen": 2976645120 + }, + { + "epoch": 0.9, + "learning_rate": 7.939017051153462e-05, + "loss": 1.2556, + "theoretical_loss": 3.351219173100792, + "tokens_seen": 2976710656 + }, + { + "epoch": 0.9, + "learning_rate": 7.937412236710132e-05, + "loss": 1.0917, + "theoretical_loss": 3.351213539660844, + "tokens_seen": 2976776192 + }, + { + "epoch": 0.9, + "learning_rate": 7.9358074222668e-05, + "loss": 1.2877, + "theoretical_loss": 3.3512079063796447, + "tokens_seen": 2976841728 + }, + { + "epoch": 0.9, + "learning_rate": 7.93420260782347e-05, + "loss": 1.1564, + "theoretical_loss": 3.351202273257187, + "tokens_seen": 2976907264 + }, + { + "epoch": 0.9, + "learning_rate": 7.93259779338014e-05, + "loss": 1.2974, + "theoretical_loss": 3.3511966402934625, + "tokens_seen": 2976972800 + }, + { + "epoch": 0.9, + "learning_rate": 7.93099297893681e-05, + "loss": 1.1538, + "theoretical_loss": 3.3511910074884628, + "tokens_seen": 2977038336 + }, + { + "epoch": 0.9, + "learning_rate": 7.929388164493481e-05, + "loss": 1.1903, + "theoretical_loss": 3.3511853748421805, + "tokens_seen": 2977103872 + }, + { + "epoch": 0.9, + "learning_rate": 7.927783350050151e-05, + "loss": 1.1821, + "theoretical_loss": 3.3511797423546072, + "tokens_seen": 2977169408 + }, + { + "epoch": 0.9, + "learning_rate": 7.926178535606821e-05, + "loss": 1.1377, + "theoretical_loss": 3.3511741100257355, + "tokens_seen": 2977234944 + }, + { + "epoch": 0.9, + "learning_rate": 7.924573721163491e-05, + "loss": 1.2061, + "theoretical_loss": 3.351168477855557, + "tokens_seen": 2977300480 + }, + { + "epoch": 0.9, + "learning_rate": 7.922968906720161e-05, + "loss": 1.2742, + "theoretical_loss": 3.3511628458440637, + "tokens_seen": 2977366016 + }, + { + "epoch": 0.9, + "learning_rate": 7.921364092276831e-05, + "loss": 1.1617, + "theoretical_loss": 3.351157213991248, + "tokens_seen": 2977431552 + }, + { + "epoch": 0.9, + "learning_rate": 7.919759277833501e-05, + "loss": 1.2388, + "theoretical_loss": 3.351151582297102, + "tokens_seen": 2977497088 + }, + { + "epoch": 0.9, + "learning_rate": 7.918154463390171e-05, + "loss": 1.1842, + "theoretical_loss": 3.3511459507616173, + "tokens_seen": 2977562624 + }, + { + "epoch": 0.9, + "learning_rate": 7.91654964894684e-05, + "loss": 1.2124, + "theoretical_loss": 3.3511403193847857, + "tokens_seen": 2977628160 + }, + { + "epoch": 0.9, + "learning_rate": 7.91494483450351e-05, + "loss": 1.1766, + "theoretical_loss": 3.3511346881666, + "tokens_seen": 2977693696 + }, + { + "epoch": 0.9, + "learning_rate": 7.91334002006018e-05, + "loss": 1.2576, + "theoretical_loss": 3.3511290571070522, + "tokens_seen": 2977759232 + }, + { + "epoch": 0.9, + "learning_rate": 7.911735205616851e-05, + "loss": 1.1594, + "theoretical_loss": 3.351123426206134, + "tokens_seen": 2977824768 + }, + { + "epoch": 0.9, + "learning_rate": 7.910130391173521e-05, + "loss": 1.248, + "theoretical_loss": 3.351117795463837, + "tokens_seen": 2977890304 + }, + { + "epoch": 0.9, + "learning_rate": 7.908525576730192e-05, + "loss": 1.1945, + "theoretical_loss": 3.3511121648801536, + "tokens_seen": 2977955840 + }, + { + "epoch": 0.9, + "learning_rate": 7.906920762286862e-05, + "loss": 1.3066, + "theoretical_loss": 3.3511065344550763, + "tokens_seen": 2978021376 + }, + { + "epoch": 0.9, + "learning_rate": 7.905315947843532e-05, + "loss": 1.2546, + "theoretical_loss": 3.351100904188597, + "tokens_seen": 2978086912 + }, + { + "epoch": 0.9, + "learning_rate": 7.903711133400202e-05, + "loss": 1.2022, + "theoretical_loss": 3.3510952740807074, + "tokens_seen": 2978152448 + }, + { + "epoch": 0.9, + "learning_rate": 7.902106318956872e-05, + "loss": 1.172, + "theoretical_loss": 3.3510896441313998, + "tokens_seen": 2978217984 + }, + { + "epoch": 0.9, + "learning_rate": 7.90050150451354e-05, + "loss": 1.2158, + "theoretical_loss": 3.3510840143406657, + "tokens_seen": 2978283520 + }, + { + "epoch": 0.9, + "learning_rate": 7.89889669007021e-05, + "loss": 1.2464, + "theoretical_loss": 3.3510783847084977, + "tokens_seen": 2978349056 + }, + { + "epoch": 0.9, + "learning_rate": 7.89729187562688e-05, + "loss": 1.1393, + "theoretical_loss": 3.351072755234888, + "tokens_seen": 2978414592 + }, + { + "epoch": 0.9, + "learning_rate": 7.895687061183551e-05, + "loss": 1.1164, + "theoretical_loss": 3.351067125919828, + "tokens_seen": 2978480128 + }, + { + "epoch": 0.9, + "learning_rate": 7.894082246740221e-05, + "loss": 1.22, + "theoretical_loss": 3.3510614967633106, + "tokens_seen": 2978545664 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 993339, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2312157154083252, + "objective/train/theoretical_loss": 3.351055867765327, + "objective/train/tokens_used": 2999071200, + "theoretical_loss": 3.351055867765327, + "tokens_seen": 2978611200 + }, + { + "epoch": 0.9, + "learning_rate": 7.892477432296891e-05, + "loss": 1.2696, + "theoretical_loss": 3.351055867765327, + "tokens_seen": 2978611200 + }, + { + "epoch": 0.9, + "learning_rate": 7.890872617853561e-05, + "loss": 1.1732, + "theoretical_loss": 3.3510502389258696, + "tokens_seen": 2978676736 + }, + { + "epoch": 0.9, + "learning_rate": 7.889267803410231e-05, + "loss": 1.1949, + "theoretical_loss": 3.3510446102449305, + "tokens_seen": 2978742272 + }, + { + "epoch": 0.9, + "learning_rate": 7.887662988966901e-05, + "loss": 1.1277, + "theoretical_loss": 3.3510389817225015, + "tokens_seen": 2978807808 + }, + { + "epoch": 0.9, + "learning_rate": 7.886058174523571e-05, + "loss": 1.2028, + "theoretical_loss": 3.3510333533585754, + "tokens_seen": 2978873344 + }, + { + "epoch": 0.9, + "learning_rate": 7.884453360080241e-05, + "loss": 1.152, + "theoretical_loss": 3.351027725153143, + "tokens_seen": 2978938880 + }, + { + "epoch": 0.9, + "learning_rate": 7.882848545636911e-05, + "loss": 1.2543, + "theoretical_loss": 3.3510220971061977, + "tokens_seen": 2979004416 + }, + { + "epoch": 0.9, + "learning_rate": 7.881243731193581e-05, + "loss": 1.1773, + "theoretical_loss": 3.3510164692177304, + "tokens_seen": 2979069952 + }, + { + "epoch": 0.9, + "learning_rate": 7.879638916750251e-05, + "loss": 1.1522, + "theoretical_loss": 3.351010841487734, + "tokens_seen": 2979135488 + }, + { + "epoch": 0.9, + "learning_rate": 7.878034102306921e-05, + "loss": 1.2033, + "theoretical_loss": 3.3510052139162, + "tokens_seen": 2979201024 + }, + { + "epoch": 0.9, + "learning_rate": 7.876429287863591e-05, + "loss": 1.3154, + "theoretical_loss": 3.3509995865031206, + "tokens_seen": 2979266560 + }, + { + "epoch": 0.9, + "learning_rate": 7.874824473420262e-05, + "loss": 1.1451, + "theoretical_loss": 3.350993959248488, + "tokens_seen": 2979332096 + }, + { + "epoch": 0.9, + "learning_rate": 7.873219658976932e-05, + "loss": 1.232, + "theoretical_loss": 3.350988332152294, + "tokens_seen": 2979397632 + }, + { + "epoch": 0.9, + "learning_rate": 7.871614844533602e-05, + "loss": 1.1875, + "theoretical_loss": 3.3509827052145313, + "tokens_seen": 2979463168 + }, + { + "epoch": 0.9, + "learning_rate": 7.870010030090272e-05, + "loss": 1.1847, + "theoretical_loss": 3.350977078435191, + "tokens_seen": 2979528704 + }, + { + "epoch": 0.9, + "learning_rate": 7.868405215646942e-05, + "loss": 1.205, + "theoretical_loss": 3.350971451814266, + "tokens_seen": 2979594240 + }, + { + "epoch": 0.9, + "learning_rate": 7.86680040120361e-05, + "loss": 1.1682, + "theoretical_loss": 3.350965825351748, + "tokens_seen": 2979659776 + }, + { + "epoch": 0.9, + "learning_rate": 7.86519558676028e-05, + "loss": 1.2225, + "theoretical_loss": 3.3509601990476288, + "tokens_seen": 2979725312 + }, + { + "epoch": 0.9, + "learning_rate": 7.86359077231695e-05, + "loss": 1.3226, + "theoretical_loss": 3.350954572901901, + "tokens_seen": 2979790848 + }, + { + "epoch": 0.9, + "learning_rate": 7.861985957873621e-05, + "loss": 1.2519, + "theoretical_loss": 3.3509489469145555, + "tokens_seen": 2979856384 + }, + { + "epoch": 0.9, + "learning_rate": 7.860381143430291e-05, + "loss": 1.2117, + "theoretical_loss": 3.3509433210855857, + "tokens_seen": 2979921920 + }, + { + "epoch": 0.9, + "learning_rate": 7.858776328986961e-05, + "loss": 1.1091, + "theoretical_loss": 3.3509376954149834, + "tokens_seen": 2979987456 + }, + { + "epoch": 0.9, + "learning_rate": 7.857171514543631e-05, + "loss": 1.1574, + "theoretical_loss": 3.3509320699027403, + "tokens_seen": 2980052992 + }, + { + "epoch": 0.9, + "learning_rate": 7.855566700100301e-05, + "loss": 1.2011, + "theoretical_loss": 3.3509264445488487, + "tokens_seen": 2980118528 + }, + { + "epoch": 0.9, + "learning_rate": 7.853961885656971e-05, + "loss": 1.2374, + "theoretical_loss": 3.3509208193533, + "tokens_seen": 2980184064 + }, + { + "epoch": 0.9, + "learning_rate": 7.852357071213642e-05, + "loss": 1.1826, + "theoretical_loss": 3.3509151943160873, + "tokens_seen": 2980249600 + }, + { + "epoch": 0.9, + "learning_rate": 7.850752256770311e-05, + "loss": 1.1669, + "theoretical_loss": 3.3509095694372024, + "tokens_seen": 2980315136 + }, + { + "epoch": 0.9, + "learning_rate": 7.849147442326981e-05, + "loss": 1.1628, + "theoretical_loss": 3.3509039447166367, + "tokens_seen": 2980380672 + }, + { + "epoch": 0.9, + "learning_rate": 7.847542627883651e-05, + "loss": 1.2685, + "theoretical_loss": 3.350898320154383, + "tokens_seen": 2980446208 + }, + { + "epoch": 0.9, + "learning_rate": 7.845937813440321e-05, + "loss": 1.1798, + "theoretical_loss": 3.350892695750433, + "tokens_seen": 2980511744 + }, + { + "epoch": 0.9, + "learning_rate": 7.844332998996991e-05, + "loss": 1.1415, + "theoretical_loss": 3.3508870715047783, + "tokens_seen": 2980577280 + }, + { + "epoch": 0.9, + "learning_rate": 7.842728184553661e-05, + "loss": 1.1664, + "theoretical_loss": 3.3508814474174122, + "tokens_seen": 2980642816 + }, + { + "epoch": 0.9, + "learning_rate": 7.841123370110332e-05, + "loss": 1.156, + "theoretical_loss": 3.350875823488326, + "tokens_seen": 2980708352 + }, + { + "epoch": 0.9, + "learning_rate": 7.839518555667002e-05, + "loss": 1.1459, + "theoretical_loss": 3.3508701997175114, + "tokens_seen": 2980773888 + }, + { + "epoch": 0.9, + "learning_rate": 7.837913741223672e-05, + "loss": 1.2436, + "theoretical_loss": 3.3508645761049607, + "tokens_seen": 2980839424 + }, + { + "epoch": 0.9, + "learning_rate": 7.836308926780342e-05, + "loss": 1.2033, + "theoretical_loss": 3.3508589526506665, + "tokens_seen": 2980904960 + }, + { + "epoch": 0.9, + "learning_rate": 7.834704112337012e-05, + "loss": 1.1164, + "theoretical_loss": 3.3508533293546208, + "tokens_seen": 2980970496 + }, + { + "epoch": 0.9, + "learning_rate": 7.833099297893682e-05, + "loss": 1.2336, + "theoretical_loss": 3.3508477062168147, + "tokens_seen": 2981036032 + }, + { + "epoch": 0.9, + "learning_rate": 7.83149448345035e-05, + "loss": 1.0902, + "theoretical_loss": 3.3508420832372416, + "tokens_seen": 2981101568 + }, + { + "epoch": 0.9, + "learning_rate": 7.82988966900702e-05, + "loss": 1.1847, + "theoretical_loss": 3.3508364604158927, + "tokens_seen": 2981167104 + }, + { + "epoch": 0.9, + "learning_rate": 7.82828485456369e-05, + "loss": 1.2506, + "theoretical_loss": 3.3508308377527602, + "tokens_seen": 2981232640 + }, + { + "epoch": 0.9, + "learning_rate": 7.826680040120361e-05, + "loss": 1.1554, + "theoretical_loss": 3.3508252152478364, + "tokens_seen": 2981298176 + }, + { + "epoch": 0.9, + "learning_rate": 7.825075225677031e-05, + "loss": 1.204, + "theoretical_loss": 3.350819592901113, + "tokens_seen": 2981363712 + }, + { + "epoch": 0.9, + "learning_rate": 7.823470411233701e-05, + "loss": 1.1712, + "theoretical_loss": 3.350813970712583, + "tokens_seen": 2981429248 + }, + { + "epoch": 0.9, + "learning_rate": 7.821865596790372e-05, + "loss": 1.2476, + "theoretical_loss": 3.3508083486822366, + "tokens_seen": 2981494784 + }, + { + "epoch": 0.9, + "learning_rate": 7.820260782347042e-05, + "loss": 1.1399, + "theoretical_loss": 3.350802726810068, + "tokens_seen": 2981560320 + }, + { + "epoch": 0.9, + "learning_rate": 7.818655967903712e-05, + "loss": 1.2407, + "theoretical_loss": 3.350797105096068, + "tokens_seen": 2981625856 + }, + { + "epoch": 0.9, + "learning_rate": 7.817051153460382e-05, + "loss": 1.2366, + "theoretical_loss": 3.3507914835402293, + "tokens_seen": 2981691392 + }, + { + "epoch": 0.9, + "learning_rate": 7.815446339017051e-05, + "loss": 1.1956, + "theoretical_loss": 3.3507858621425433, + "tokens_seen": 2981756928 + }, + { + "epoch": 0.9, + "learning_rate": 7.813841524573721e-05, + "loss": 1.1005, + "theoretical_loss": 3.3507802409030023, + "tokens_seen": 2981822464 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 994656, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.172619104385376, + "objective/train/theoretical_loss": 3.350774619821599, + "objective/train/tokens_used": 3002348000, + "theoretical_loss": 3.350774619821599, + "tokens_seen": 2981888000 + }, + { + "epoch": 0.9, + "learning_rate": 7.812236710130391e-05, + "loss": 1.1267, + "theoretical_loss": 3.350774619821599, + "tokens_seen": 2981888000 + }, + { + "epoch": 0.9, + "learning_rate": 7.810631895687061e-05, + "loss": 1.2277, + "theoretical_loss": 3.3507689988983245, + "tokens_seen": 2981953536 + }, + { + "epoch": 0.9, + "learning_rate": 7.809027081243731e-05, + "loss": 1.1885, + "theoretical_loss": 3.3507633781331716, + "tokens_seen": 2982019072 + }, + { + "epoch": 0.9, + "learning_rate": 7.807422266800401e-05, + "loss": 1.1606, + "theoretical_loss": 3.3507577575261323, + "tokens_seen": 2982084608 + }, + { + "epoch": 0.9, + "learning_rate": 7.805817452357072e-05, + "loss": 1.1788, + "theoretical_loss": 3.3507521370771984, + "tokens_seen": 2982150144 + }, + { + "epoch": 0.9, + "learning_rate": 7.804212637913742e-05, + "loss": 1.0985, + "theoretical_loss": 3.350746516786362, + "tokens_seen": 2982215680 + }, + { + "epoch": 0.9, + "learning_rate": 7.802607823470412e-05, + "loss": 1.1137, + "theoretical_loss": 3.3507408966536154, + "tokens_seen": 2982281216 + }, + { + "epoch": 0.9, + "learning_rate": 7.801003009027082e-05, + "loss": 1.2999, + "theoretical_loss": 3.3507352766789507, + "tokens_seen": 2982346752 + }, + { + "epoch": 0.9, + "learning_rate": 7.799398194583752e-05, + "loss": 1.1111, + "theoretical_loss": 3.3507296568623595, + "tokens_seen": 2982412288 + }, + { + "epoch": 0.9, + "learning_rate": 7.797793380140422e-05, + "loss": 1.1923, + "theoretical_loss": 3.3507240372038347, + "tokens_seen": 2982477824 + }, + { + "epoch": 0.9, + "learning_rate": 7.79618856569709e-05, + "loss": 1.1529, + "theoretical_loss": 3.3507184177033675, + "tokens_seen": 2982543360 + }, + { + "epoch": 0.9, + "learning_rate": 7.79458375125376e-05, + "loss": 1.1057, + "theoretical_loss": 3.3507127983609504, + "tokens_seen": 2982608896 + }, + { + "epoch": 0.9, + "learning_rate": 7.792978936810431e-05, + "loss": 1.1864, + "theoretical_loss": 3.3507071791765757, + "tokens_seen": 2982674432 + }, + { + "epoch": 0.9, + "learning_rate": 7.791374122367102e-05, + "loss": 1.2368, + "theoretical_loss": 3.3507015601502355, + "tokens_seen": 2982739968 + }, + { + "epoch": 0.9, + "learning_rate": 7.789769307923772e-05, + "loss": 1.1275, + "theoretical_loss": 3.350695941281921, + "tokens_seen": 2982805504 + }, + { + "epoch": 0.9, + "learning_rate": 7.788164493480442e-05, + "loss": 1.1505, + "theoretical_loss": 3.350690322571625, + "tokens_seen": 2982871040 + }, + { + "epoch": 0.9, + "learning_rate": 7.786559679037112e-05, + "loss": 1.1624, + "theoretical_loss": 3.35068470401934, + "tokens_seen": 2982936576 + }, + { + "epoch": 0.9, + "learning_rate": 7.784954864593782e-05, + "loss": 1.2615, + "theoretical_loss": 3.3506790856250572, + "tokens_seen": 2983002112 + }, + { + "epoch": 0.9, + "learning_rate": 7.783350050150452e-05, + "loss": 1.2179, + "theoretical_loss": 3.350673467388769, + "tokens_seen": 2983067648 + }, + { + "epoch": 0.9, + "learning_rate": 7.781745235707123e-05, + "loss": 1.1619, + "theoretical_loss": 3.350667849310468, + "tokens_seen": 2983133184 + }, + { + "epoch": 0.9, + "learning_rate": 7.780140421263791e-05, + "loss": 1.1207, + "theoretical_loss": 3.3506622313901455, + "tokens_seen": 2983198720 + }, + { + "epoch": 0.9, + "learning_rate": 7.778535606820461e-05, + "loss": 1.1735, + "theoretical_loss": 3.3506566136277938, + "tokens_seen": 2983264256 + }, + { + "epoch": 0.9, + "learning_rate": 7.776930792377131e-05, + "loss": 1.2631, + "theoretical_loss": 3.3506509960234054, + "tokens_seen": 2983329792 + }, + { + "epoch": 0.9, + "learning_rate": 7.775325977933801e-05, + "loss": 1.2207, + "theoretical_loss": 3.350645378576972, + "tokens_seen": 2983395328 + }, + { + "epoch": 0.9, + "learning_rate": 7.773721163490471e-05, + "loss": 1.1523, + "theoretical_loss": 3.3506397612884857, + "tokens_seen": 2983460864 + }, + { + "epoch": 0.9, + "learning_rate": 7.772116349047142e-05, + "loss": 1.1505, + "theoretical_loss": 3.3506341441579393, + "tokens_seen": 2983526400 + }, + { + "epoch": 0.9, + "learning_rate": 7.770511534603812e-05, + "loss": 1.1086, + "theoretical_loss": 3.3506285271853233, + "tokens_seen": 2983591936 + }, + { + "epoch": 0.9, + "learning_rate": 7.768906720160482e-05, + "loss": 1.2215, + "theoretical_loss": 3.3506229103706318, + "tokens_seen": 2983657472 + }, + { + "epoch": 0.9, + "learning_rate": 7.767301905717152e-05, + "loss": 1.1371, + "theoretical_loss": 3.350617293713855, + "tokens_seen": 2983723008 + }, + { + "epoch": 0.9, + "learning_rate": 7.765697091273822e-05, + "loss": 1.2204, + "theoretical_loss": 3.350611677214986, + "tokens_seen": 2983788544 + }, + { + "epoch": 0.9, + "learning_rate": 7.764092276830492e-05, + "loss": 1.2161, + "theoretical_loss": 3.350606060874017, + "tokens_seen": 2983854080 + }, + { + "epoch": 0.9, + "learning_rate": 7.762487462387162e-05, + "loss": 1.2656, + "theoretical_loss": 3.35060044469094, + "tokens_seen": 2983919616 + }, + { + "epoch": 0.9, + "learning_rate": 7.760882647943832e-05, + "loss": 1.0857, + "theoretical_loss": 3.3505948286657468, + "tokens_seen": 2983985152 + }, + { + "epoch": 0.9, + "learning_rate": 7.759277833500502e-05, + "loss": 1.2697, + "theoretical_loss": 3.3505892127984294, + "tokens_seen": 2984050688 + }, + { + "epoch": 0.9, + "learning_rate": 7.757673019057172e-05, + "loss": 1.1793, + "theoretical_loss": 3.3505835970889803, + "tokens_seen": 2984116224 + }, + { + "epoch": 0.9, + "learning_rate": 7.756068204613842e-05, + "loss": 1.1277, + "theoretical_loss": 3.3505779815373913, + "tokens_seen": 2984181760 + }, + { + "epoch": 0.9, + "learning_rate": 7.754463390170512e-05, + "loss": 1.2926, + "theoretical_loss": 3.3505723661436546, + "tokens_seen": 2984247296 + }, + { + "epoch": 0.9, + "learning_rate": 7.752858575727182e-05, + "loss": 1.2311, + "theoretical_loss": 3.3505667509077623, + "tokens_seen": 2984312832 + }, + { + "epoch": 0.9, + "learning_rate": 7.751253761283852e-05, + "loss": 1.2177, + "theoretical_loss": 3.3505611358297065, + "tokens_seen": 2984378368 + }, + { + "epoch": 0.9, + "learning_rate": 7.749648946840522e-05, + "loss": 1.1613, + "theoretical_loss": 3.3505555209094795, + "tokens_seen": 2984443904 + }, + { + "epoch": 0.9, + "learning_rate": 7.748044132397192e-05, + "loss": 1.2138, + "theoretical_loss": 3.3505499061470734, + "tokens_seen": 2984509440 + }, + { + "epoch": 0.9, + "learning_rate": 7.746439317953863e-05, + "loss": 1.1698, + "theoretical_loss": 3.3505442915424797, + "tokens_seen": 2984574976 + }, + { + "epoch": 0.9, + "learning_rate": 7.744834503510531e-05, + "loss": 1.2166, + "theoretical_loss": 3.350538677095691, + "tokens_seen": 2984640512 + }, + { + "epoch": 0.9, + "learning_rate": 7.743229689067201e-05, + "loss": 1.2102, + "theoretical_loss": 3.350533062806699, + "tokens_seen": 2984706048 + }, + { + "epoch": 0.9, + "learning_rate": 7.741624874623871e-05, + "loss": 1.2566, + "theoretical_loss": 3.3505274486754963, + "tokens_seen": 2984771584 + }, + { + "epoch": 0.9, + "learning_rate": 7.740020060180541e-05, + "loss": 1.0669, + "theoretical_loss": 3.350521834702075, + "tokens_seen": 2984837120 + }, + { + "epoch": 0.9, + "learning_rate": 7.738415245737212e-05, + "loss": 1.2832, + "theoretical_loss": 3.350516220886427, + "tokens_seen": 2984902656 + }, + { + "epoch": 0.9, + "learning_rate": 7.736810431293882e-05, + "loss": 1.2573, + "theoretical_loss": 3.350510607228544, + "tokens_seen": 2984968192 + }, + { + "epoch": 0.9, + "learning_rate": 7.735205616850552e-05, + "loss": 1.1374, + "theoretical_loss": 3.350504993728419, + "tokens_seen": 2985033728 + }, + { + "epoch": 0.9, + "learning_rate": 7.733600802407222e-05, + "loss": 1.3922, + "theoretical_loss": 3.3504993803860437, + "tokens_seen": 2985099264 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 995793, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.215075969696045, + "objective/train/theoretical_loss": 3.3504937672014097, + "objective/train/tokens_used": 3005624800, + "theoretical_loss": 3.3504937672014097, + "tokens_seen": 2985164800 + }, + { + "epoch": 0.9, + "learning_rate": 7.731995987963893e-05, + "loss": 1.2134, + "theoretical_loss": 3.3504937672014097, + "tokens_seen": 2985164800 + }, + { + "epoch": 0.9, + "learning_rate": 7.730391173520563e-05, + "loss": 1.1801, + "theoretical_loss": 3.3504881541745095, + "tokens_seen": 2985230336 + }, + { + "epoch": 0.9, + "learning_rate": 7.728786359077232e-05, + "loss": 1.202, + "theoretical_loss": 3.3504825413053356, + "tokens_seen": 2985295872 + }, + { + "epoch": 0.9, + "learning_rate": 7.727181544633902e-05, + "loss": 1.1708, + "theoretical_loss": 3.350476928593879, + "tokens_seen": 2985361408 + }, + { + "epoch": 0.9, + "learning_rate": 7.725576730190572e-05, + "loss": 1.1772, + "theoretical_loss": 3.3504713160401334, + "tokens_seen": 2985426944 + }, + { + "epoch": 0.9, + "learning_rate": 7.723971915747242e-05, + "loss": 1.2319, + "theoretical_loss": 3.3504657036440895, + "tokens_seen": 2985492480 + }, + { + "epoch": 0.9, + "learning_rate": 7.722367101303912e-05, + "loss": 1.2439, + "theoretical_loss": 3.3504600914057403, + "tokens_seen": 2985558016 + }, + { + "epoch": 0.9, + "learning_rate": 7.720762286860582e-05, + "loss": 1.0977, + "theoretical_loss": 3.3504544793250775, + "tokens_seen": 2985623552 + }, + { + "epoch": 0.9, + "learning_rate": 7.719157472417252e-05, + "loss": 1.204, + "theoretical_loss": 3.350448867402093, + "tokens_seen": 2985689088 + }, + { + "epoch": 0.9, + "learning_rate": 7.717552657973922e-05, + "loss": 1.1214, + "theoretical_loss": 3.350443255636779, + "tokens_seen": 2985754624 + }, + { + "epoch": 0.9, + "learning_rate": 7.715947843530592e-05, + "loss": 1.1484, + "theoretical_loss": 3.350437644029128, + "tokens_seen": 2985820160 + }, + { + "epoch": 0.9, + "learning_rate": 7.714343029087262e-05, + "loss": 1.1373, + "theoretical_loss": 3.3504320325791324, + "tokens_seen": 2985885696 + }, + { + "epoch": 0.9, + "learning_rate": 7.712738214643933e-05, + "loss": 1.2181, + "theoretical_loss": 3.3504264212867834, + "tokens_seen": 2985951232 + }, + { + "epoch": 0.9, + "learning_rate": 7.711133400200603e-05, + "loss": 1.1753, + "theoretical_loss": 3.350420810152073, + "tokens_seen": 2986016768 + }, + { + "epoch": 0.9, + "learning_rate": 7.709528585757271e-05, + "loss": 1.1549, + "theoretical_loss": 3.3504151991749946, + "tokens_seen": 2986082304 + }, + { + "epoch": 0.9, + "learning_rate": 7.707923771313941e-05, + "loss": 1.1104, + "theoretical_loss": 3.3504095883555394, + "tokens_seen": 2986147840 + }, + { + "epoch": 0.9, + "learning_rate": 7.706318956870611e-05, + "loss": 1.1663, + "theoretical_loss": 3.3504039776936994, + "tokens_seen": 2986213376 + }, + { + "epoch": 0.9, + "learning_rate": 7.704714142427281e-05, + "loss": 1.2018, + "theoretical_loss": 3.350398367189467, + "tokens_seen": 2986278912 + }, + { + "epoch": 0.9, + "learning_rate": 7.703109327983952e-05, + "loss": 1.1109, + "theoretical_loss": 3.3503927568428344, + "tokens_seen": 2986344448 + }, + { + "epoch": 0.9, + "learning_rate": 7.701504513540623e-05, + "loss": 1.2374, + "theoretical_loss": 3.3503871466537936, + "tokens_seen": 2986409984 + }, + { + "epoch": 0.9, + "learning_rate": 7.699899699097293e-05, + "loss": 1.137, + "theoretical_loss": 3.3503815366223364, + "tokens_seen": 2986475520 + }, + { + "epoch": 0.91, + "learning_rate": 7.698294884653963e-05, + "loss": 1.1997, + "theoretical_loss": 3.3503759267484554, + "tokens_seen": 2986541056 + }, + { + "epoch": 0.91, + "learning_rate": 7.696690070210633e-05, + "loss": 1.2109, + "theoretical_loss": 3.350370317032143, + "tokens_seen": 2986606592 + }, + { + "epoch": 0.91, + "learning_rate": 7.695085255767303e-05, + "loss": 1.1985, + "theoretical_loss": 3.3503647074733904, + "tokens_seen": 2986672128 + }, + { + "epoch": 0.91, + "learning_rate": 7.693480441323972e-05, + "loss": 1.1988, + "theoretical_loss": 3.3503590980721905, + "tokens_seen": 2986737664 + }, + { + "epoch": 0.91, + "learning_rate": 7.691875626880642e-05, + "loss": 1.177, + "theoretical_loss": 3.3503534888285347, + "tokens_seen": 2986803200 + }, + { + "epoch": 0.91, + "learning_rate": 7.690270812437312e-05, + "loss": 1.1589, + "theoretical_loss": 3.3503478797424155, + "tokens_seen": 2986868736 + }, + { + "epoch": 0.91, + "learning_rate": 7.688665997993982e-05, + "loss": 1.1295, + "theoretical_loss": 3.350342270813825, + "tokens_seen": 2986934272 + }, + { + "epoch": 0.91, + "learning_rate": 7.687061183550652e-05, + "loss": 1.2026, + "theoretical_loss": 3.3503366620427557, + "tokens_seen": 2986999808 + }, + { + "epoch": 0.91, + "learning_rate": 7.685456369107322e-05, + "loss": 1.1294, + "theoretical_loss": 3.3503310534291995, + "tokens_seen": 2987065344 + }, + { + "epoch": 0.91, + "learning_rate": 7.683851554663992e-05, + "loss": 1.0717, + "theoretical_loss": 3.350325444973148, + "tokens_seen": 2987130880 + }, + { + "epoch": 0.91, + "learning_rate": 7.682246740220662e-05, + "loss": 1.193, + "theoretical_loss": 3.3503198366745934, + "tokens_seen": 2987196416 + }, + { + "epoch": 0.91, + "learning_rate": 7.680641925777332e-05, + "loss": 1.1706, + "theoretical_loss": 3.350314228533529, + "tokens_seen": 2987261952 + }, + { + "epoch": 0.91, + "learning_rate": 7.679037111334003e-05, + "loss": 1.2131, + "theoretical_loss": 3.3503086205499453, + "tokens_seen": 2987327488 + }, + { + "epoch": 0.91, + "learning_rate": 7.677432296890673e-05, + "loss": 1.2512, + "theoretical_loss": 3.3503030127238356, + "tokens_seen": 2987393024 + }, + { + "epoch": 0.91, + "learning_rate": 7.675827482447343e-05, + "loss": 1.1852, + "theoretical_loss": 3.3502974050551915, + "tokens_seen": 2987458560 + }, + { + "epoch": 0.91, + "learning_rate": 7.674222668004011e-05, + "loss": 1.1639, + "theoretical_loss": 3.350291797544005, + "tokens_seen": 2987524096 + }, + { + "epoch": 0.91, + "learning_rate": 7.672617853560683e-05, + "loss": 1.17, + "theoretical_loss": 3.3502861901902685, + "tokens_seen": 2987589632 + }, + { + "epoch": 0.91, + "learning_rate": 7.671013039117353e-05, + "loss": 1.1625, + "theoretical_loss": 3.3502805829939746, + "tokens_seen": 2987655168 + }, + { + "epoch": 0.91, + "learning_rate": 7.669408224674023e-05, + "loss": 1.2534, + "theoretical_loss": 3.3502749759551147, + "tokens_seen": 2987720704 + }, + { + "epoch": 0.91, + "learning_rate": 7.667803410230693e-05, + "loss": 1.2165, + "theoretical_loss": 3.350269369073681, + "tokens_seen": 2987786240 + }, + { + "epoch": 0.91, + "learning_rate": 7.666198595787363e-05, + "loss": 1.1933, + "theoretical_loss": 3.3502637623496656, + "tokens_seen": 2987851776 + }, + { + "epoch": 0.91, + "learning_rate": 7.664593781344033e-05, + "loss": 1.2805, + "theoretical_loss": 3.350258155783061, + "tokens_seen": 2987917312 + }, + { + "epoch": 0.91, + "learning_rate": 7.662988966900703e-05, + "loss": 1.2112, + "theoretical_loss": 3.3502525493738586, + "tokens_seen": 2987982848 + }, + { + "epoch": 0.91, + "learning_rate": 7.661384152457373e-05, + "loss": 1.155, + "theoretical_loss": 3.350246943122052, + "tokens_seen": 2988048384 + }, + { + "epoch": 0.91, + "learning_rate": 7.659779338014042e-05, + "loss": 1.2038, + "theoretical_loss": 3.3502413370276316, + "tokens_seen": 2988113920 + }, + { + "epoch": 0.91, + "learning_rate": 7.658174523570712e-05, + "loss": 1.225, + "theoretical_loss": 3.3502357310905904, + "tokens_seen": 2988179456 + }, + { + "epoch": 0.91, + "learning_rate": 7.656569709127382e-05, + "loss": 1.1395, + "theoretical_loss": 3.3502301253109206, + "tokens_seen": 2988244992 + }, + { + "epoch": 0.91, + "learning_rate": 7.654964894684052e-05, + "loss": 1.0809, + "theoretical_loss": 3.3502245196886142, + "tokens_seen": 2988310528 + }, + { + "epoch": 0.91, + "learning_rate": 7.653360080240722e-05, + "loss": 1.1239, + "theoretical_loss": 3.3502189142236634, + "tokens_seen": 2988376064 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 997016, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.265026330947876, + "objective/train/theoretical_loss": 3.35021330891606, + "objective/train/tokens_used": 3008901600, + "theoretical_loss": 3.35021330891606, + "tokens_seen": 2988441600 + }, + { + "epoch": 0.91, + "learning_rate": 7.651755265797392e-05, + "loss": 1.1823, + "theoretical_loss": 3.35021330891606, + "tokens_seen": 2988441600 + }, + { + "epoch": 0.91, + "learning_rate": 7.650150451354062e-05, + "loss": 1.2928, + "theoretical_loss": 3.3502077037657965, + "tokens_seen": 2988507136 + }, + { + "epoch": 0.91, + "learning_rate": 7.648545636910732e-05, + "loss": 1.2553, + "theoretical_loss": 3.350202098772865, + "tokens_seen": 2988572672 + }, + { + "epoch": 0.91, + "learning_rate": 7.646940822467402e-05, + "loss": 1.1775, + "theoretical_loss": 3.350196493937257, + "tokens_seen": 2988638208 + }, + { + "epoch": 0.91, + "learning_rate": 7.645336008024072e-05, + "loss": 1.1097, + "theoretical_loss": 3.3501908892589656, + "tokens_seen": 2988703744 + }, + { + "epoch": 0.91, + "learning_rate": 7.643731193580743e-05, + "loss": 1.2055, + "theoretical_loss": 3.3501852847379823, + "tokens_seen": 2988769280 + }, + { + "epoch": 0.91, + "learning_rate": 7.642126379137413e-05, + "loss": 1.1303, + "theoretical_loss": 3.3501796803742994, + "tokens_seen": 2988834816 + }, + { + "epoch": 0.91, + "learning_rate": 7.640521564694083e-05, + "loss": 1.239, + "theoretical_loss": 3.3501740761679093, + "tokens_seen": 2988900352 + }, + { + "epoch": 0.91, + "learning_rate": 7.638916750250753e-05, + "loss": 1.1747, + "theoretical_loss": 3.3501684721188036, + "tokens_seen": 2988965888 + }, + { + "epoch": 0.91, + "learning_rate": 7.637311935807423e-05, + "loss": 1.1844, + "theoretical_loss": 3.3501628682269753, + "tokens_seen": 2989031424 + }, + { + "epoch": 0.91, + "learning_rate": 7.635707121364093e-05, + "loss": 1.2246, + "theoretical_loss": 3.350157264492416, + "tokens_seen": 2989096960 + }, + { + "epoch": 0.91, + "learning_rate": 7.634102306920763e-05, + "loss": 1.1594, + "theoretical_loss": 3.3501516609151167, + "tokens_seen": 2989162496 + }, + { + "epoch": 0.91, + "learning_rate": 7.632497492477433e-05, + "loss": 1.3244, + "theoretical_loss": 3.3501460574950714, + "tokens_seen": 2989228032 + }, + { + "epoch": 0.91, + "learning_rate": 7.630892678034103e-05, + "loss": 1.2491, + "theoretical_loss": 3.350140454232272, + "tokens_seen": 2989293568 + }, + { + "epoch": 0.91, + "learning_rate": 7.629287863590773e-05, + "loss": 1.1803, + "theoretical_loss": 3.3501348511267093, + "tokens_seen": 2989359104 + }, + { + "epoch": 0.91, + "learning_rate": 7.627683049147443e-05, + "loss": 1.2159, + "theoretical_loss": 3.3501292481783764, + "tokens_seen": 2989424640 + }, + { + "epoch": 0.91, + "learning_rate": 7.626078234704113e-05, + "loss": 1.216, + "theoretical_loss": 3.350123645387266, + "tokens_seen": 2989490176 + }, + { + "epoch": 0.91, + "learning_rate": 7.624473420260782e-05, + "loss": 1.1842, + "theoretical_loss": 3.350118042753369, + "tokens_seen": 2989555712 + }, + { + "epoch": 0.91, + "learning_rate": 7.622868605817452e-05, + "loss": 1.2543, + "theoretical_loss": 3.350112440276678, + "tokens_seen": 2989621248 + }, + { + "epoch": 0.91, + "learning_rate": 7.621263791374122e-05, + "loss": 1.1349, + "theoretical_loss": 3.3501068379571857, + "tokens_seen": 2989686784 + }, + { + "epoch": 0.91, + "learning_rate": 7.619658976930792e-05, + "loss": 1.1684, + "theoretical_loss": 3.350101235794883, + "tokens_seen": 2989752320 + }, + { + "epoch": 0.91, + "learning_rate": 7.618054162487462e-05, + "loss": 1.1965, + "theoretical_loss": 3.350095633789764, + "tokens_seen": 2989817856 + }, + { + "epoch": 0.91, + "learning_rate": 7.616449348044132e-05, + "loss": 1.1487, + "theoretical_loss": 3.3500900319418188, + "tokens_seen": 2989883392 + }, + { + "epoch": 0.91, + "learning_rate": 7.614844533600802e-05, + "loss": 1.176, + "theoretical_loss": 3.350084430251041, + "tokens_seen": 2989948928 + }, + { + "epoch": 0.91, + "learning_rate": 7.613239719157472e-05, + "loss": 1.1862, + "theoretical_loss": 3.3500788287174217, + "tokens_seen": 2990014464 + }, + { + "epoch": 0.91, + "learning_rate": 7.611634904714144e-05, + "loss": 1.0962, + "theoretical_loss": 3.3500732273409533, + "tokens_seen": 2990080000 + }, + { + "epoch": 0.91, + "learning_rate": 7.610030090270814e-05, + "loss": 1.1276, + "theoretical_loss": 3.3500676261216285, + "tokens_seen": 2990145536 + }, + { + "epoch": 0.91, + "learning_rate": 7.608425275827483e-05, + "loss": 1.1824, + "theoretical_loss": 3.3500620250594393, + "tokens_seen": 2990211072 + }, + { + "epoch": 0.91, + "learning_rate": 7.606820461384153e-05, + "loss": 1.1692, + "theoretical_loss": 3.3500564241543773, + "tokens_seen": 2990276608 + }, + { + "epoch": 0.91, + "learning_rate": 7.605215646940823e-05, + "loss": 1.1664, + "theoretical_loss": 3.350050823406435, + "tokens_seen": 2990342144 + }, + { + "epoch": 0.91, + "learning_rate": 7.603610832497493e-05, + "loss": 1.2798, + "theoretical_loss": 3.350045222815605, + "tokens_seen": 2990407680 + }, + { + "epoch": 0.91, + "learning_rate": 7.602006018054163e-05, + "loss": 1.2512, + "theoretical_loss": 3.3500396223818787, + "tokens_seen": 2990473216 + }, + { + "epoch": 0.91, + "learning_rate": 7.600401203610833e-05, + "loss": 1.1393, + "theoretical_loss": 3.3500340221052487, + "tokens_seen": 2990538752 + }, + { + "epoch": 0.91, + "learning_rate": 7.598796389167503e-05, + "loss": 1.2199, + "theoretical_loss": 3.350028421985707, + "tokens_seen": 2990604288 + }, + { + "epoch": 0.91, + "learning_rate": 7.597191574724173e-05, + "loss": 1.1745, + "theoretical_loss": 3.3500228220232455, + "tokens_seen": 2990669824 + }, + { + "epoch": 0.91, + "learning_rate": 7.595586760280843e-05, + "loss": 1.2023, + "theoretical_loss": 3.350017222217857, + "tokens_seen": 2990735360 + }, + { + "epoch": 0.91, + "learning_rate": 7.593981945837513e-05, + "loss": 1.1845, + "theoretical_loss": 3.3500116225695327, + "tokens_seen": 2990800896 + }, + { + "epoch": 0.91, + "learning_rate": 7.592377131394183e-05, + "loss": 1.2031, + "theoretical_loss": 3.3500060230782656, + "tokens_seen": 2990866432 + }, + { + "epoch": 0.91, + "learning_rate": 7.590772316950853e-05, + "loss": 1.2319, + "theoretical_loss": 3.3500004237440475, + "tokens_seen": 2990931968 + }, + { + "epoch": 0.91, + "learning_rate": 7.589167502507522e-05, + "loss": 1.2519, + "theoretical_loss": 3.349994824566871, + "tokens_seen": 2990997504 + }, + { + "epoch": 0.91, + "learning_rate": 7.587562688064192e-05, + "loss": 1.2038, + "theoretical_loss": 3.3499892255467274, + "tokens_seen": 2991063040 + }, + { + "epoch": 0.91, + "learning_rate": 7.585957873620862e-05, + "loss": 1.2138, + "theoretical_loss": 3.34998362668361, + "tokens_seen": 2991128576 + }, + { + "epoch": 0.91, + "learning_rate": 7.584353059177532e-05, + "loss": 1.2701, + "theoretical_loss": 3.3499780279775093, + "tokens_seen": 2991194112 + }, + { + "epoch": 0.91, + "learning_rate": 7.582748244734202e-05, + "loss": 1.2226, + "theoretical_loss": 3.349972429428419, + "tokens_seen": 2991259648 + }, + { + "epoch": 0.91, + "learning_rate": 7.581143430290874e-05, + "loss": 1.2787, + "theoretical_loss": 3.3499668310363306, + "tokens_seen": 2991325184 + }, + { + "epoch": 0.91, + "learning_rate": 7.579538615847544e-05, + "loss": 1.2292, + "theoretical_loss": 3.3499612328012365, + "tokens_seen": 2991390720 + }, + { + "epoch": 0.91, + "learning_rate": 7.577933801404214e-05, + "loss": 1.1704, + "theoretical_loss": 3.3499556347231287, + "tokens_seen": 2991456256 + }, + { + "epoch": 0.91, + "learning_rate": 7.576328986960884e-05, + "loss": 1.1975, + "theoretical_loss": 3.3499500368019994, + "tokens_seen": 2991521792 + }, + { + "epoch": 0.91, + "learning_rate": 7.574724172517554e-05, + "loss": 1.2098, + "theoretical_loss": 3.3499444390378406, + "tokens_seen": 2991587328 + }, + { + "epoch": 0.91, + "learning_rate": 7.573119358074223e-05, + "loss": 1.2337, + "theoretical_loss": 3.3499388414306446, + "tokens_seen": 2991652864 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 997972, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2263262271881104, + "objective/train/theoretical_loss": 3.349933243980404, + "objective/train/tokens_used": 3012178400, + "theoretical_loss": 3.349933243980404, + "tokens_seen": 2991718400 + }, + { + "epoch": 0.91, + "learning_rate": 7.571514543630893e-05, + "loss": 1.2037, + "theoretical_loss": 3.349933243980404, + "tokens_seen": 2991718400 + }, + { + "epoch": 0.91, + "learning_rate": 7.569909729187563e-05, + "loss": 1.1419, + "theoretical_loss": 3.3499276466871097, + "tokens_seen": 2991783936 + }, + { + "epoch": 0.91, + "learning_rate": 7.568304914744233e-05, + "loss": 1.229, + "theoretical_loss": 3.3499220495507553, + "tokens_seen": 2991849472 + }, + { + "epoch": 0.91, + "learning_rate": 7.566700100300903e-05, + "loss": 1.3045, + "theoretical_loss": 3.349916452571332, + "tokens_seen": 2991915008 + }, + { + "epoch": 0.91, + "learning_rate": 7.565095285857573e-05, + "loss": 1.2475, + "theoretical_loss": 3.349910855748832, + "tokens_seen": 2991980544 + }, + { + "epoch": 0.91, + "learning_rate": 7.563490471414243e-05, + "loss": 1.2063, + "theoretical_loss": 3.3499052590832488, + "tokens_seen": 2992046080 + }, + { + "epoch": 0.91, + "learning_rate": 7.561885656970913e-05, + "loss": 1.1653, + "theoretical_loss": 3.3498996625745727, + "tokens_seen": 2992111616 + }, + { + "epoch": 0.91, + "learning_rate": 7.560280842527583e-05, + "loss": 1.1571, + "theoretical_loss": 3.349894066222797, + "tokens_seen": 2992177152 + }, + { + "epoch": 0.91, + "learning_rate": 7.558676028084253e-05, + "loss": 1.0845, + "theoretical_loss": 3.3498884700279135, + "tokens_seen": 2992242688 + }, + { + "epoch": 0.91, + "learning_rate": 7.557071213640923e-05, + "loss": 1.1559, + "theoretical_loss": 3.3498828739899142, + "tokens_seen": 2992308224 + }, + { + "epoch": 0.91, + "learning_rate": 7.555466399197593e-05, + "loss": 1.193, + "theoretical_loss": 3.349877278108792, + "tokens_seen": 2992373760 + }, + { + "epoch": 0.91, + "learning_rate": 7.553861584754262e-05, + "loss": 1.1533, + "theoretical_loss": 3.3498716823845385, + "tokens_seen": 2992439296 + }, + { + "epoch": 0.91, + "learning_rate": 7.552256770310933e-05, + "loss": 1.17, + "theoretical_loss": 3.3498660868171455, + "tokens_seen": 2992504832 + }, + { + "epoch": 0.91, + "learning_rate": 7.550651955867604e-05, + "loss": 1.2835, + "theoretical_loss": 3.3498604914066057, + "tokens_seen": 2992570368 + }, + { + "epoch": 0.91, + "learning_rate": 7.549047141424274e-05, + "loss": 1.1008, + "theoretical_loss": 3.349854896152911, + "tokens_seen": 2992635904 + }, + { + "epoch": 0.91, + "learning_rate": 7.547442326980944e-05, + "loss": 1.1878, + "theoretical_loss": 3.349849301056054, + "tokens_seen": 2992701440 + }, + { + "epoch": 0.91, + "learning_rate": 7.545837512537614e-05, + "loss": 1.1916, + "theoretical_loss": 3.349843706116027, + "tokens_seen": 2992766976 + }, + { + "epoch": 0.91, + "learning_rate": 7.544232698094284e-05, + "loss": 1.227, + "theoretical_loss": 3.3498381113328213, + "tokens_seen": 2992832512 + }, + { + "epoch": 0.91, + "learning_rate": 7.542627883650954e-05, + "loss": 1.1806, + "theoretical_loss": 3.3498325167064293, + "tokens_seen": 2992898048 + }, + { + "epoch": 0.91, + "learning_rate": 7.541023069207624e-05, + "loss": 1.2431, + "theoretical_loss": 3.3498269222368435, + "tokens_seen": 2992963584 + }, + { + "epoch": 0.91, + "learning_rate": 7.539418254764294e-05, + "loss": 1.1317, + "theoretical_loss": 3.3498213279240563, + "tokens_seen": 2993029120 + }, + { + "epoch": 0.91, + "learning_rate": 7.537813440320963e-05, + "loss": 1.1014, + "theoretical_loss": 3.3498157337680596, + "tokens_seen": 2993094656 + }, + { + "epoch": 0.91, + "learning_rate": 7.536208625877633e-05, + "loss": 1.1124, + "theoretical_loss": 3.349810139768845, + "tokens_seen": 2993160192 + }, + { + "epoch": 0.91, + "learning_rate": 7.534603811434303e-05, + "loss": 1.1555, + "theoretical_loss": 3.3498045459264056, + "tokens_seen": 2993225728 + }, + { + "epoch": 0.91, + "learning_rate": 7.532998996990973e-05, + "loss": 1.2585, + "theoretical_loss": 3.349798952240733, + "tokens_seen": 2993291264 + }, + { + "epoch": 0.91, + "learning_rate": 7.531394182547643e-05, + "loss": 1.2752, + "theoretical_loss": 3.3497933587118203, + "tokens_seen": 2993356800 + }, + { + "epoch": 0.91, + "learning_rate": 7.529789368104313e-05, + "loss": 1.2134, + "theoretical_loss": 3.349787765339658, + "tokens_seen": 2993422336 + }, + { + "epoch": 0.91, + "learning_rate": 7.528184553660983e-05, + "loss": 1.1984, + "theoretical_loss": 3.3497821721242396, + "tokens_seen": 2993487872 + }, + { + "epoch": 0.91, + "learning_rate": 7.526579739217653e-05, + "loss": 1.1858, + "theoretical_loss": 3.3497765790655567, + "tokens_seen": 2993553408 + }, + { + "epoch": 0.91, + "learning_rate": 7.524974924774323e-05, + "loss": 1.1506, + "theoretical_loss": 3.349770986163602, + "tokens_seen": 2993618944 + }, + { + "epoch": 0.91, + "learning_rate": 7.523370110330993e-05, + "loss": 1.183, + "theoretical_loss": 3.349765393418367, + "tokens_seen": 2993684480 + }, + { + "epoch": 0.91, + "learning_rate": 7.521765295887663e-05, + "loss": 1.0985, + "theoretical_loss": 3.349759800829844, + "tokens_seen": 2993750016 + }, + { + "epoch": 0.91, + "learning_rate": 7.520160481444333e-05, + "loss": 1.1517, + "theoretical_loss": 3.349754208398026, + "tokens_seen": 2993815552 + }, + { + "epoch": 0.91, + "learning_rate": 7.518555667001003e-05, + "loss": 1.2034, + "theoretical_loss": 3.3497486161229038, + "tokens_seen": 2993881088 + }, + { + "epoch": 0.91, + "learning_rate": 7.516950852557673e-05, + "loss": 1.1803, + "theoretical_loss": 3.3497430240044705, + "tokens_seen": 2993946624 + }, + { + "epoch": 0.91, + "learning_rate": 7.515346038114344e-05, + "loss": 1.2334, + "theoretical_loss": 3.3497374320427187, + "tokens_seen": 2994012160 + }, + { + "epoch": 0.91, + "learning_rate": 7.513741223671014e-05, + "loss": 1.2405, + "theoretical_loss": 3.3497318402376397, + "tokens_seen": 2994077696 + }, + { + "epoch": 0.91, + "learning_rate": 7.512136409227684e-05, + "loss": 1.0911, + "theoretical_loss": 3.3497262485892256, + "tokens_seen": 2994143232 + }, + { + "epoch": 0.91, + "learning_rate": 7.510531594784354e-05, + "loss": 1.1557, + "theoretical_loss": 3.3497206570974694, + "tokens_seen": 2994208768 + }, + { + "epoch": 0.91, + "learning_rate": 7.508926780341024e-05, + "loss": 1.1034, + "theoretical_loss": 3.349715065762363, + "tokens_seen": 2994274304 + }, + { + "epoch": 0.91, + "learning_rate": 7.507321965897694e-05, + "loss": 1.1783, + "theoretical_loss": 3.3497094745838982, + "tokens_seen": 2994339840 + }, + { + "epoch": 0.91, + "learning_rate": 7.505717151454364e-05, + "loss": 1.2875, + "theoretical_loss": 3.3497038835620674, + "tokens_seen": 2994405376 + }, + { + "epoch": 0.91, + "learning_rate": 7.504112337011034e-05, + "loss": 1.2159, + "theoretical_loss": 3.349698292696863, + "tokens_seen": 2994470912 + }, + { + "epoch": 0.91, + "learning_rate": 7.502507522567703e-05, + "loss": 1.1364, + "theoretical_loss": 3.3496927019882765, + "tokens_seen": 2994536448 + }, + { + "epoch": 0.91, + "learning_rate": 7.500902708124373e-05, + "loss": 1.1899, + "theoretical_loss": 3.349687111436301, + "tokens_seen": 2994601984 + }, + { + "epoch": 0.91, + "learning_rate": 7.499297893681043e-05, + "loss": 1.3035, + "theoretical_loss": 3.349681521040928, + "tokens_seen": 2994667520 + }, + { + "epoch": 0.91, + "learning_rate": 7.497693079237713e-05, + "loss": 1.0874, + "theoretical_loss": 3.34967593080215, + "tokens_seen": 2994733056 + }, + { + "epoch": 0.91, + "learning_rate": 7.496088264794383e-05, + "loss": 1.2393, + "theoretical_loss": 3.3496703407199595, + "tokens_seen": 2994798592 + }, + { + "epoch": 0.91, + "learning_rate": 7.494483450351053e-05, + "loss": 1.1656, + "theoretical_loss": 3.349664750794348, + "tokens_seen": 2994864128 + }, + { + "epoch": 0.91, + "learning_rate": 7.492878635907723e-05, + "loss": 1.1188, + "theoretical_loss": 3.3496591610253077, + "tokens_seen": 2994929664 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 999187, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1573805809020996, + "objective/train/theoretical_loss": 3.3496535714128317, + "objective/train/tokens_used": 3015455200, + "theoretical_loss": 3.3496535714128317, + "tokens_seen": 2994995200 + }, + { + "epoch": 0.91, + "learning_rate": 7.491273821464395e-05, + "loss": 1.1356, + "theoretical_loss": 3.3496535714128317, + "tokens_seen": 2994995200 + }, + { + "epoch": 0.91, + "learning_rate": 7.489669007021065e-05, + "loss": 1.2328, + "theoretical_loss": 3.3496479819569114, + "tokens_seen": 2995060736 + }, + { + "epoch": 0.91, + "learning_rate": 7.488064192577733e-05, + "loss": 1.2972, + "theoretical_loss": 3.3496423926575387, + "tokens_seen": 2995126272 + }, + { + "epoch": 0.91, + "learning_rate": 7.486459378134403e-05, + "loss": 1.1391, + "theoretical_loss": 3.3496368035147066, + "tokens_seen": 2995191808 + }, + { + "epoch": 0.91, + "learning_rate": 7.484854563691073e-05, + "loss": 1.093, + "theoretical_loss": 3.349631214528407, + "tokens_seen": 2995257344 + }, + { + "epoch": 0.91, + "learning_rate": 7.483249749247743e-05, + "loss": 1.2562, + "theoretical_loss": 3.349625625698632, + "tokens_seen": 2995322880 + }, + { + "epoch": 0.91, + "learning_rate": 7.481644934804414e-05, + "loss": 1.1909, + "theoretical_loss": 3.349620037025374, + "tokens_seen": 2995388416 + }, + { + "epoch": 0.91, + "learning_rate": 7.480040120361084e-05, + "loss": 1.134, + "theoretical_loss": 3.3496144485086248, + "tokens_seen": 2995453952 + }, + { + "epoch": 0.91, + "learning_rate": 7.478435305917754e-05, + "loss": 1.1465, + "theoretical_loss": 3.349608860148377, + "tokens_seen": 2995519488 + }, + { + "epoch": 0.91, + "learning_rate": 7.476830491474424e-05, + "loss": 1.2299, + "theoretical_loss": 3.3496032719446225, + "tokens_seen": 2995585024 + }, + { + "epoch": 0.91, + "learning_rate": 7.475225677031094e-05, + "loss": 1.1863, + "theoretical_loss": 3.3495976838973536, + "tokens_seen": 2995650560 + }, + { + "epoch": 0.91, + "learning_rate": 7.473620862587764e-05, + "loss": 1.1695, + "theoretical_loss": 3.3495920960065626, + "tokens_seen": 2995716096 + }, + { + "epoch": 0.91, + "learning_rate": 7.472016048144434e-05, + "loss": 1.1778, + "theoretical_loss": 3.3495865082722416, + "tokens_seen": 2995781632 + }, + { + "epoch": 0.91, + "learning_rate": 7.470411233701104e-05, + "loss": 1.2805, + "theoretical_loss": 3.3495809206943825, + "tokens_seen": 2995847168 + }, + { + "epoch": 0.91, + "learning_rate": 7.468806419257773e-05, + "loss": 1.2029, + "theoretical_loss": 3.3495753332729783, + "tokens_seen": 2995912704 + }, + { + "epoch": 0.91, + "learning_rate": 7.467201604814443e-05, + "loss": 1.2264, + "theoretical_loss": 3.3495697460080205, + "tokens_seen": 2995978240 + }, + { + "epoch": 0.91, + "learning_rate": 7.465596790371113e-05, + "loss": 1.2593, + "theoretical_loss": 3.349564158899501, + "tokens_seen": 2996043776 + }, + { + "epoch": 0.91, + "learning_rate": 7.463991975927783e-05, + "loss": 1.2237, + "theoretical_loss": 3.349558571947413, + "tokens_seen": 2996109312 + }, + { + "epoch": 0.91, + "learning_rate": 7.462387161484454e-05, + "loss": 1.0763, + "theoretical_loss": 3.349552985151748, + "tokens_seen": 2996174848 + }, + { + "epoch": 0.91, + "learning_rate": 7.460782347041124e-05, + "loss": 1.1552, + "theoretical_loss": 3.3495473985124984, + "tokens_seen": 2996240384 + }, + { + "epoch": 0.91, + "learning_rate": 7.459177532597794e-05, + "loss": 1.2334, + "theoretical_loss": 3.3495418120296563, + "tokens_seen": 2996305920 + }, + { + "epoch": 0.91, + "learning_rate": 7.457572718154464e-05, + "loss": 1.0867, + "theoretical_loss": 3.3495362257032144, + "tokens_seen": 2996371456 + }, + { + "epoch": 0.91, + "learning_rate": 7.455967903711135e-05, + "loss": 1.174, + "theoretical_loss": 3.349530639533164, + "tokens_seen": 2996436992 + }, + { + "epoch": 0.91, + "learning_rate": 7.454363089267805e-05, + "loss": 1.1844, + "theoretical_loss": 3.349525053519498, + "tokens_seen": 2996502528 + }, + { + "epoch": 0.91, + "learning_rate": 7.452758274824473e-05, + "loss": 1.1311, + "theoretical_loss": 3.3495194676622084, + "tokens_seen": 2996568064 + }, + { + "epoch": 0.91, + "learning_rate": 7.451153460381143e-05, + "loss": 1.2457, + "theoretical_loss": 3.349513881961287, + "tokens_seen": 2996633600 + }, + { + "epoch": 0.91, + "learning_rate": 7.449548645937813e-05, + "loss": 1.1321, + "theoretical_loss": 3.349508296416727, + "tokens_seen": 2996699136 + }, + { + "epoch": 0.91, + "learning_rate": 7.447943831494484e-05, + "loss": 1.2024, + "theoretical_loss": 3.34950271102852, + "tokens_seen": 2996764672 + }, + { + "epoch": 0.91, + "learning_rate": 7.446339017051154e-05, + "loss": 1.1228, + "theoretical_loss": 3.3494971257966575, + "tokens_seen": 2996830208 + }, + { + "epoch": 0.91, + "learning_rate": 7.444734202607824e-05, + "loss": 1.1875, + "theoretical_loss": 3.3494915407211328, + "tokens_seen": 2996895744 + }, + { + "epoch": 0.91, + "learning_rate": 7.443129388164494e-05, + "loss": 1.1383, + "theoretical_loss": 3.3494859558019376, + "tokens_seen": 2996961280 + }, + { + "epoch": 0.91, + "learning_rate": 7.441524573721164e-05, + "loss": 1.1297, + "theoretical_loss": 3.3494803710390646, + "tokens_seen": 2997026816 + }, + { + "epoch": 0.91, + "learning_rate": 7.439919759277834e-05, + "loss": 1.2183, + "theoretical_loss": 3.349474786432505, + "tokens_seen": 2997092352 + }, + { + "epoch": 0.91, + "learning_rate": 7.438314944834504e-05, + "loss": 1.1226, + "theoretical_loss": 3.3494692019822523, + "tokens_seen": 2997157888 + }, + { + "epoch": 0.91, + "learning_rate": 7.436710130391174e-05, + "loss": 1.2098, + "theoretical_loss": 3.3494636176882975, + "tokens_seen": 2997223424 + }, + { + "epoch": 0.91, + "learning_rate": 7.435105315947844e-05, + "loss": 1.112, + "theoretical_loss": 3.3494580335506337, + "tokens_seen": 2997288960 + }, + { + "epoch": 0.91, + "learning_rate": 7.433500501504513e-05, + "loss": 1.2252, + "theoretical_loss": 3.3494524495692524, + "tokens_seen": 2997354496 + }, + { + "epoch": 0.91, + "learning_rate": 7.431895687061184e-05, + "loss": 1.2043, + "theoretical_loss": 3.3494468657441465, + "tokens_seen": 2997420032 + }, + { + "epoch": 0.91, + "learning_rate": 7.430290872617854e-05, + "loss": 1.2691, + "theoretical_loss": 3.3494412820753077, + "tokens_seen": 2997485568 + }, + { + "epoch": 0.91, + "learning_rate": 7.428686058174524e-05, + "loss": 1.1304, + "theoretical_loss": 3.3494356985627283, + "tokens_seen": 2997551104 + }, + { + "epoch": 0.91, + "learning_rate": 7.427081243731194e-05, + "loss": 1.0015, + "theoretical_loss": 3.3494301152064008, + "tokens_seen": 2997616640 + }, + { + "epoch": 0.91, + "learning_rate": 7.425476429287864e-05, + "loss": 1.2299, + "theoretical_loss": 3.349424532006317, + "tokens_seen": 2997682176 + }, + { + "epoch": 0.91, + "learning_rate": 7.423871614844534e-05, + "loss": 1.1543, + "theoretical_loss": 3.3494189489624695, + "tokens_seen": 2997747712 + }, + { + "epoch": 0.91, + "learning_rate": 7.422266800401205e-05, + "loss": 1.2849, + "theoretical_loss": 3.34941336607485, + "tokens_seen": 2997813248 + }, + { + "epoch": 0.91, + "learning_rate": 7.420661985957875e-05, + "loss": 1.1765, + "theoretical_loss": 3.3494077833434517, + "tokens_seen": 2997878784 + }, + { + "epoch": 0.91, + "learning_rate": 7.419057171514545e-05, + "loss": 1.1605, + "theoretical_loss": 3.3494022007682656, + "tokens_seen": 2997944320 + }, + { + "epoch": 0.91, + "learning_rate": 7.417452357071213e-05, + "loss": 1.1664, + "theoretical_loss": 3.3493966183492847, + "tokens_seen": 2998009856 + }, + { + "epoch": 0.91, + "learning_rate": 7.415847542627883e-05, + "loss": 1.2114, + "theoretical_loss": 3.3493910360865007, + "tokens_seen": 2998075392 + }, + { + "epoch": 0.91, + "learning_rate": 7.414242728184553e-05, + "loss": 1.1445, + "theoretical_loss": 3.3493854539799064, + "tokens_seen": 2998140928 + }, + { + "epoch": 0.91, + "learning_rate": 7.412637913741224e-05, + "loss": 1.202, + "theoretical_loss": 3.3493798720294934, + "tokens_seen": 2998206464 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 999981, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1404556035995483, + "objective/train/theoretical_loss": 3.3493742902352546, + "objective/train/tokens_used": 3018732000, + "theoretical_loss": 3.3493742902352546, + "tokens_seen": 2998272000 + }, + { + "epoch": 0.91, + "learning_rate": 7.411033099297894e-05, + "loss": 1.148, + "theoretical_loss": 3.3493742902352546, + "tokens_seen": 2998272000 + }, + { + "epoch": 0.91, + "learning_rate": 7.409428284854564e-05, + "loss": 1.2164, + "theoretical_loss": 3.3493687085971815, + "tokens_seen": 2998337536 + }, + { + "epoch": 0.91, + "learning_rate": 7.407823470411234e-05, + "loss": 1.2314, + "theoretical_loss": 3.349363127115267, + "tokens_seen": 2998403072 + }, + { + "epoch": 0.91, + "learning_rate": 7.406218655967904e-05, + "loss": 1.0927, + "theoretical_loss": 3.349357545789503, + "tokens_seen": 2998468608 + }, + { + "epoch": 0.91, + "learning_rate": 7.404613841524574e-05, + "loss": 1.2179, + "theoretical_loss": 3.3493519646198813, + "tokens_seen": 2998534144 + }, + { + "epoch": 0.91, + "learning_rate": 7.403009027081244e-05, + "loss": 1.1652, + "theoretical_loss": 3.349346383606395, + "tokens_seen": 2998599680 + }, + { + "epoch": 0.91, + "learning_rate": 7.401404212637914e-05, + "loss": 1.1599, + "theoretical_loss": 3.3493408027490355, + "tokens_seen": 2998665216 + }, + { + "epoch": 0.91, + "learning_rate": 7.399799398194584e-05, + "loss": 1.1438, + "theoretical_loss": 3.349335222047795, + "tokens_seen": 2998730752 + }, + { + "epoch": 0.91, + "learning_rate": 7.398194583751254e-05, + "loss": 1.2587, + "theoretical_loss": 3.349329641502667, + "tokens_seen": 2998796288 + }, + { + "epoch": 0.91, + "learning_rate": 7.396589769307924e-05, + "loss": 1.2592, + "theoretical_loss": 3.3493240611136423, + "tokens_seen": 2998861824 + }, + { + "epoch": 0.91, + "learning_rate": 7.394984954864594e-05, + "loss": 1.1392, + "theoretical_loss": 3.3493184808807133, + "tokens_seen": 2998927360 + }, + { + "epoch": 0.91, + "learning_rate": 7.393380140421264e-05, + "loss": 1.1525, + "theoretical_loss": 3.349312900803873, + "tokens_seen": 2998992896 + }, + { + "epoch": 0.91, + "learning_rate": 7.391775325977934e-05, + "loss": 1.3027, + "theoretical_loss": 3.3493073208831134, + "tokens_seen": 2999058432 + }, + { + "epoch": 0.91, + "learning_rate": 7.390170511534604e-05, + "loss": 1.2079, + "theoretical_loss": 3.349301741118426, + "tokens_seen": 2999123968 + }, + { + "epoch": 0.91, + "learning_rate": 7.388565697091275e-05, + "loss": 1.1728, + "theoretical_loss": 3.3492961615098036, + "tokens_seen": 2999189504 + }, + { + "epoch": 0.91, + "learning_rate": 7.386960882647945e-05, + "loss": 1.2159, + "theoretical_loss": 3.3492905820572387, + "tokens_seen": 2999255040 + }, + { + "epoch": 0.91, + "learning_rate": 7.385356068204615e-05, + "loss": 1.2061, + "theoretical_loss": 3.349285002760723, + "tokens_seen": 2999320576 + }, + { + "epoch": 0.91, + "learning_rate": 7.383751253761285e-05, + "loss": 1.161, + "theoretical_loss": 3.3492794236202488, + "tokens_seen": 2999386112 + }, + { + "epoch": 0.91, + "learning_rate": 7.382146439317953e-05, + "loss": 1.0895, + "theoretical_loss": 3.349273844635808, + "tokens_seen": 2999451648 + }, + { + "epoch": 0.91, + "learning_rate": 7.380541624874623e-05, + "loss": 1.0892, + "theoretical_loss": 3.349268265807394, + "tokens_seen": 2999517184 + }, + { + "epoch": 0.91, + "learning_rate": 7.378936810431294e-05, + "loss": 1.2045, + "theoretical_loss": 3.3492626871349978, + "tokens_seen": 2999582720 + }, + { + "epoch": 0.91, + "learning_rate": 7.377331995987964e-05, + "loss": 1.2105, + "theoretical_loss": 3.3492571086186125, + "tokens_seen": 2999648256 + }, + { + "epoch": 0.91, + "learning_rate": 7.375727181544634e-05, + "loss": 1.1862, + "theoretical_loss": 3.3492515302582295, + "tokens_seen": 2999713792 + }, + { + "epoch": 0.91, + "learning_rate": 7.374122367101304e-05, + "loss": 1.2191, + "theoretical_loss": 3.349245952053842, + "tokens_seen": 2999779328 + }, + { + "epoch": 0.91, + "learning_rate": 7.372517552657974e-05, + "loss": 1.1821, + "theoretical_loss": 3.349240374005441, + "tokens_seen": 2999844864 + }, + { + "epoch": 0.91, + "learning_rate": 7.370912738214645e-05, + "loss": 1.296, + "theoretical_loss": 3.34923479611302, + "tokens_seen": 2999910400 + }, + { + "epoch": 0.91, + "learning_rate": 7.369307923771315e-05, + "loss": 1.1626, + "theoretical_loss": 3.3492292183765704, + "tokens_seen": 2999975936 + }, + { + "epoch": 0.91, + "learning_rate": 7.367703109327985e-05, + "loss": 1.0906, + "theoretical_loss": 3.3492236407960845, + "tokens_seen": 3000041472 + }, + { + "epoch": 0.91, + "learning_rate": 7.366098294884654e-05, + "loss": 1.2798, + "theoretical_loss": 3.349218063371555, + "tokens_seen": 3000107008 + }, + { + "epoch": 0.91, + "learning_rate": 7.364493480441324e-05, + "loss": 1.394, + "theoretical_loss": 3.3492124861029735, + "tokens_seen": 3000172544 + }, + { + "epoch": 0.91, + "learning_rate": 7.362888665997994e-05, + "loss": 1.2585, + "theoretical_loss": 3.3492069089903334, + "tokens_seen": 3000238080 + }, + { + "epoch": 0.91, + "learning_rate": 7.361283851554664e-05, + "loss": 1.1899, + "theoretical_loss": 3.3492013320336254, + "tokens_seen": 3000303616 + }, + { + "epoch": 0.91, + "learning_rate": 7.359679037111334e-05, + "loss": 1.1945, + "theoretical_loss": 3.3491957552328424, + "tokens_seen": 3000369152 + }, + { + "epoch": 0.91, + "learning_rate": 7.358074222668004e-05, + "loss": 1.1559, + "theoretical_loss": 3.3491901785879774, + "tokens_seen": 3000434688 + }, + { + "epoch": 0.91, + "learning_rate": 7.356469408224674e-05, + "loss": 1.175, + "theoretical_loss": 3.349184602099021, + "tokens_seen": 3000500224 + }, + { + "epoch": 0.91, + "learning_rate": 7.354864593781344e-05, + "loss": 1.1908, + "theoretical_loss": 3.349179025765967, + "tokens_seen": 3000565760 + }, + { + "epoch": 0.91, + "learning_rate": 7.353259779338015e-05, + "loss": 1.1482, + "theoretical_loss": 3.3491734495888066, + "tokens_seen": 3000631296 + }, + { + "epoch": 0.91, + "learning_rate": 7.351654964894685e-05, + "loss": 1.187, + "theoretical_loss": 3.3491678735675325, + "tokens_seen": 3000696832 + }, + { + "epoch": 0.91, + "learning_rate": 7.350050150451355e-05, + "loss": 1.1263, + "theoretical_loss": 3.3491622977021374, + "tokens_seen": 3000762368 + }, + { + "epoch": 0.91, + "learning_rate": 7.348445336008025e-05, + "loss": 1.112, + "theoretical_loss": 3.3491567219926126, + "tokens_seen": 3000827904 + }, + { + "epoch": 0.91, + "learning_rate": 7.346840521564693e-05, + "loss": 1.2218, + "theoretical_loss": 3.3491511464389507, + "tokens_seen": 3000893440 + }, + { + "epoch": 0.91, + "learning_rate": 7.345235707121364e-05, + "loss": 1.1178, + "theoretical_loss": 3.349145571041144, + "tokens_seen": 3000958976 + }, + { + "epoch": 0.91, + "learning_rate": 7.343630892678034e-05, + "loss": 1.1357, + "theoretical_loss": 3.3491399957991845, + "tokens_seen": 3001024512 + }, + { + "epoch": 0.91, + "learning_rate": 7.342026078234705e-05, + "loss": 1.1836, + "theoretical_loss": 3.349134420713065, + "tokens_seen": 3001090048 + }, + { + "epoch": 0.91, + "learning_rate": 7.340421263791375e-05, + "loss": 1.1793, + "theoretical_loss": 3.3491288457827775, + "tokens_seen": 3001155584 + }, + { + "epoch": 0.91, + "learning_rate": 7.338816449348045e-05, + "loss": 1.1895, + "theoretical_loss": 3.349123271008314, + "tokens_seen": 3001221120 + }, + { + "epoch": 0.91, + "learning_rate": 7.337211634904715e-05, + "loss": 1.2014, + "theoretical_loss": 3.349117696389667, + "tokens_seen": 3001286656 + }, + { + "epoch": 0.91, + "learning_rate": 7.335606820461385e-05, + "loss": 1.149, + "theoretical_loss": 3.3491121219268285, + "tokens_seen": 3001352192 + }, + { + "epoch": 0.91, + "learning_rate": 7.334002006018055e-05, + "loss": 1.1667, + "theoretical_loss": 3.3491065476197908, + "tokens_seen": 3001417728 + }, + { + "epoch": 0.91, + "learning_rate": 7.332397191574725e-05, + "loss": 1.2352, + "theoretical_loss": 3.349100973468546, + "tokens_seen": 3001483264 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1001227, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2069125175476074, + "objective/train/theoretical_loss": 3.349095399473087, + "objective/train/tokens_used": 3022008800, + "theoretical_loss": 3.349095399473087, + "tokens_seen": 3001548800 + }, + { + "epoch": 0.91, + "learning_rate": 7.330792377131394e-05, + "loss": 1.245, + "theoretical_loss": 3.349095399473087, + "tokens_seen": 3001548800 + }, + { + "epoch": 0.91, + "learning_rate": 7.329187562688064e-05, + "loss": 1.1281, + "theoretical_loss": 3.3490898256334054, + "tokens_seen": 3001614336 + }, + { + "epoch": 0.91, + "learning_rate": 7.327582748244734e-05, + "loss": 1.3201, + "theoretical_loss": 3.349084251949494, + "tokens_seen": 3001679872 + }, + { + "epoch": 0.91, + "learning_rate": 7.325977933801404e-05, + "loss": 1.1157, + "theoretical_loss": 3.3490786784213444, + "tokens_seen": 3001745408 + }, + { + "epoch": 0.91, + "learning_rate": 7.324373119358074e-05, + "loss": 1.2514, + "theoretical_loss": 3.3490731050489493, + "tokens_seen": 3001810944 + }, + { + "epoch": 0.91, + "learning_rate": 7.322768304914744e-05, + "loss": 1.1843, + "theoretical_loss": 3.3490675318323007, + "tokens_seen": 3001876480 + }, + { + "epoch": 0.91, + "learning_rate": 7.321163490471414e-05, + "loss": 1.1433, + "theoretical_loss": 3.3490619587713906, + "tokens_seen": 3001942016 + }, + { + "epoch": 0.91, + "learning_rate": 7.319558676028085e-05, + "loss": 1.1422, + "theoretical_loss": 3.349056385866212, + "tokens_seen": 3002007552 + }, + { + "epoch": 0.91, + "learning_rate": 7.317953861584755e-05, + "loss": 1.2219, + "theoretical_loss": 3.3490508131167567, + "tokens_seen": 3002073088 + }, + { + "epoch": 0.91, + "learning_rate": 7.316349047141425e-05, + "loss": 1.1473, + "theoretical_loss": 3.349045240523017, + "tokens_seen": 3002138624 + }, + { + "epoch": 0.91, + "learning_rate": 7.314744232698095e-05, + "loss": 1.2058, + "theoretical_loss": 3.349039668084985, + "tokens_seen": 3002204160 + }, + { + "epoch": 0.91, + "learning_rate": 7.313139418254765e-05, + "loss": 1.2265, + "theoretical_loss": 3.3490340958026534, + "tokens_seen": 3002269696 + }, + { + "epoch": 0.91, + "learning_rate": 7.311534603811435e-05, + "loss": 1.0826, + "theoretical_loss": 3.349028523676014, + "tokens_seen": 3002335232 + }, + { + "epoch": 0.91, + "learning_rate": 7.309929789368105e-05, + "loss": 1.1906, + "theoretical_loss": 3.3490229517050585, + "tokens_seen": 3002400768 + }, + { + "epoch": 0.91, + "learning_rate": 7.308324974924775e-05, + "loss": 1.2045, + "theoretical_loss": 3.349017379889781, + "tokens_seen": 3002466304 + }, + { + "epoch": 0.91, + "learning_rate": 7.306720160481445e-05, + "loss": 1.1638, + "theoretical_loss": 3.3490118082301716, + "tokens_seen": 3002531840 + }, + { + "epoch": 0.91, + "learning_rate": 7.305115346038115e-05, + "loss": 1.226, + "theoretical_loss": 3.349006236726224, + "tokens_seen": 3002597376 + }, + { + "epoch": 0.91, + "learning_rate": 7.303510531594785e-05, + "loss": 1.1049, + "theoretical_loss": 3.34900066537793, + "tokens_seen": 3002662912 + }, + { + "epoch": 0.91, + "learning_rate": 7.301905717151455e-05, + "loss": 1.2829, + "theoretical_loss": 3.3489950941852817, + "tokens_seen": 3002728448 + }, + { + "epoch": 0.91, + "learning_rate": 7.300300902708125e-05, + "loss": 1.2711, + "theoretical_loss": 3.348989523148272, + "tokens_seen": 3002793984 + }, + { + "epoch": 0.91, + "learning_rate": 7.298696088264795e-05, + "loss": 1.1382, + "theoretical_loss": 3.348983952266892, + "tokens_seen": 3002859520 + }, + { + "epoch": 0.91, + "learning_rate": 7.297091273821465e-05, + "loss": 1.3466, + "theoretical_loss": 3.3489783815411354, + "tokens_seen": 3002925056 + }, + { + "epoch": 0.91, + "learning_rate": 7.295486459378134e-05, + "loss": 1.2336, + "theoretical_loss": 3.348972810970993, + "tokens_seen": 3002990592 + }, + { + "epoch": 0.91, + "learning_rate": 7.293881644934804e-05, + "loss": 1.256, + "theoretical_loss": 3.348967240556458, + "tokens_seen": 3003056128 + }, + { + "epoch": 0.91, + "learning_rate": 7.292276830491474e-05, + "loss": 1.2501, + "theoretical_loss": 3.348961670297522, + "tokens_seen": 3003121664 + }, + { + "epoch": 0.91, + "learning_rate": 7.290672016048144e-05, + "loss": 1.1163, + "theoretical_loss": 3.348956100194178, + "tokens_seen": 3003187200 + }, + { + "epoch": 0.91, + "learning_rate": 7.289067201604814e-05, + "loss": 1.176, + "theoretical_loss": 3.348950530246418, + "tokens_seen": 3003252736 + }, + { + "epoch": 0.91, + "learning_rate": 7.287462387161484e-05, + "loss": 1.2402, + "theoretical_loss": 3.348944960454234, + "tokens_seen": 3003318272 + }, + { + "epoch": 0.91, + "learning_rate": 7.285857572718154e-05, + "loss": 1.1849, + "theoretical_loss": 3.348939390817619, + "tokens_seen": 3003383808 + }, + { + "epoch": 0.91, + "learning_rate": 7.284252758274825e-05, + "loss": 1.1911, + "theoretical_loss": 3.348933821336564, + "tokens_seen": 3003449344 + }, + { + "epoch": 0.91, + "learning_rate": 7.282647943831495e-05, + "loss": 1.18, + "theoretical_loss": 3.3489282520110617, + "tokens_seen": 3003514880 + }, + { + "epoch": 0.91, + "learning_rate": 7.281043129388165e-05, + "loss": 1.2938, + "theoretical_loss": 3.3489226828411054, + "tokens_seen": 3003580416 + }, + { + "epoch": 0.91, + "learning_rate": 7.279438314944835e-05, + "loss": 1.1849, + "theoretical_loss": 3.348917113826686, + "tokens_seen": 3003645952 + }, + { + "epoch": 0.91, + "learning_rate": 7.277833500501505e-05, + "loss": 1.2394, + "theoretical_loss": 3.348911544967797, + "tokens_seen": 3003711488 + }, + { + "epoch": 0.91, + "learning_rate": 7.276228686058175e-05, + "loss": 1.188, + "theoretical_loss": 3.348905976264429, + "tokens_seen": 3003777024 + }, + { + "epoch": 0.91, + "learning_rate": 7.274623871614845e-05, + "loss": 1.1786, + "theoretical_loss": 3.348900407716576, + "tokens_seen": 3003842560 + }, + { + "epoch": 0.91, + "learning_rate": 7.273019057171515e-05, + "loss": 1.2117, + "theoretical_loss": 3.3488948393242293, + "tokens_seen": 3003908096 + }, + { + "epoch": 0.91, + "learning_rate": 7.271414242728185e-05, + "loss": 1.1623, + "theoretical_loss": 3.3488892710873817, + "tokens_seen": 3003973632 + }, + { + "epoch": 0.91, + "learning_rate": 7.269809428284855e-05, + "loss": 1.2187, + "theoretical_loss": 3.348883703006025, + "tokens_seen": 3004039168 + }, + { + "epoch": 0.91, + "learning_rate": 7.268204613841525e-05, + "loss": 1.164, + "theoretical_loss": 3.3488781350801515, + "tokens_seen": 3004104704 + }, + { + "epoch": 0.91, + "learning_rate": 7.266599799398195e-05, + "loss": 1.1696, + "theoretical_loss": 3.3488725673097535, + "tokens_seen": 3004170240 + }, + { + "epoch": 0.91, + "learning_rate": 7.264994984954865e-05, + "loss": 1.2318, + "theoretical_loss": 3.3488669996948235, + "tokens_seen": 3004235776 + }, + { + "epoch": 0.91, + "learning_rate": 7.263390170511535e-05, + "loss": 1.2433, + "theoretical_loss": 3.3488614322353536, + "tokens_seen": 3004301312 + }, + { + "epoch": 0.91, + "learning_rate": 7.261785356068204e-05, + "loss": 1.2613, + "theoretical_loss": 3.3488558649313362, + "tokens_seen": 3004366848 + }, + { + "epoch": 0.91, + "learning_rate": 7.260180541624874e-05, + "loss": 1.2162, + "theoretical_loss": 3.3488502977827634, + "tokens_seen": 3004432384 + }, + { + "epoch": 0.91, + "learning_rate": 7.258575727181544e-05, + "loss": 1.1649, + "theoretical_loss": 3.3488447307896276, + "tokens_seen": 3004497920 + }, + { + "epoch": 0.91, + "learning_rate": 7.256970912738214e-05, + "loss": 1.2137, + "theoretical_loss": 3.348839163951921, + "tokens_seen": 3004563456 + }, + { + "epoch": 0.91, + "learning_rate": 7.255366098294884e-05, + "loss": 1.1298, + "theoretical_loss": 3.3488335972696355, + "tokens_seen": 3004628992 + }, + { + "epoch": 0.91, + "learning_rate": 7.253761283851554e-05, + "loss": 1.1934, + "theoretical_loss": 3.348828030742764, + "tokens_seen": 3004694528 + }, + { + "epoch": 0.91, + "learning_rate": 7.252156469408226e-05, + "loss": 1.2981, + "theoretical_loss": 3.3488224643712985, + "tokens_seen": 3004760064 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1002101, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1422464847564697, + "objective/train/theoretical_loss": 3.3488168981552313, + "objective/train/tokens_used": 3025285600, + "theoretical_loss": 3.3488168981552313, + "tokens_seen": 3004825600 + }, + { + "epoch": 0.91, + "learning_rate": 7.250551654964896e-05, + "loss": 1.2861, + "theoretical_loss": 3.3488168981552313, + "tokens_seen": 3004825600 + }, + { + "epoch": 0.91, + "learning_rate": 7.248946840521566e-05, + "loss": 1.2019, + "theoretical_loss": 3.348811332094555, + "tokens_seen": 3004891136 + }, + { + "epoch": 0.91, + "learning_rate": 7.247342026078236e-05, + "loss": 1.277, + "theoretical_loss": 3.348805766189261, + "tokens_seen": 3004956672 + }, + { + "epoch": 0.91, + "learning_rate": 7.245737211634905e-05, + "loss": 1.206, + "theoretical_loss": 3.3488002004393422, + "tokens_seen": 3005022208 + }, + { + "epoch": 0.91, + "learning_rate": 7.244132397191575e-05, + "loss": 1.1935, + "theoretical_loss": 3.348794634844791, + "tokens_seen": 3005087744 + }, + { + "epoch": 0.91, + "learning_rate": 7.242527582748245e-05, + "loss": 1.1938, + "theoretical_loss": 3.3487890694055995, + "tokens_seen": 3005153280 + }, + { + "epoch": 0.91, + "learning_rate": 7.240922768304915e-05, + "loss": 1.2473, + "theoretical_loss": 3.3487835041217595, + "tokens_seen": 3005218816 + }, + { + "epoch": 0.91, + "learning_rate": 7.239317953861585e-05, + "loss": 1.2862, + "theoretical_loss": 3.348777938993264, + "tokens_seen": 3005284352 + }, + { + "epoch": 0.91, + "learning_rate": 7.237713139418255e-05, + "loss": 1.2464, + "theoretical_loss": 3.348772374020105, + "tokens_seen": 3005349888 + }, + { + "epoch": 0.91, + "learning_rate": 7.236108324974925e-05, + "loss": 1.1771, + "theoretical_loss": 3.3487668092022744, + "tokens_seen": 3005415424 + }, + { + "epoch": 0.91, + "learning_rate": 7.234503510531595e-05, + "loss": 1.1373, + "theoretical_loss": 3.3487612445397654, + "tokens_seen": 3005480960 + }, + { + "epoch": 0.91, + "learning_rate": 7.232898696088265e-05, + "loss": 1.1981, + "theoretical_loss": 3.3487556800325695, + "tokens_seen": 3005546496 + }, + { + "epoch": 0.91, + "learning_rate": 7.231293881644935e-05, + "loss": 1.1493, + "theoretical_loss": 3.348750115680679, + "tokens_seen": 3005612032 + }, + { + "epoch": 0.91, + "learning_rate": 7.229689067201605e-05, + "loss": 1.1472, + "theoretical_loss": 3.348744551484087, + "tokens_seen": 3005677568 + }, + { + "epoch": 0.91, + "learning_rate": 7.228084252758275e-05, + "loss": 1.1443, + "theoretical_loss": 3.3487389874427844, + "tokens_seen": 3005743104 + }, + { + "epoch": 0.91, + "learning_rate": 7.226479438314944e-05, + "loss": 1.1608, + "theoretical_loss": 3.3487334235567645, + "tokens_seen": 3005808640 + }, + { + "epoch": 0.91, + "learning_rate": 7.224874623871614e-05, + "loss": 1.1558, + "theoretical_loss": 3.3487278598260195, + "tokens_seen": 3005874176 + }, + { + "epoch": 0.91, + "learning_rate": 7.223269809428284e-05, + "loss": 1.2315, + "theoretical_loss": 3.348722296250541, + "tokens_seen": 3005939712 + }, + { + "epoch": 0.91, + "learning_rate": 7.221664994984956e-05, + "loss": 1.2282, + "theoretical_loss": 3.3487167328303222, + "tokens_seen": 3006005248 + }, + { + "epoch": 0.91, + "learning_rate": 7.220060180541626e-05, + "loss": 1.1771, + "theoretical_loss": 3.348711169565355, + "tokens_seen": 3006070784 + }, + { + "epoch": 0.91, + "learning_rate": 7.218455366098296e-05, + "loss": 1.2082, + "theoretical_loss": 3.3487056064556313, + "tokens_seen": 3006136320 + }, + { + "epoch": 0.91, + "learning_rate": 7.216850551654966e-05, + "loss": 1.3104, + "theoretical_loss": 3.348700043501144, + "tokens_seen": 3006201856 + }, + { + "epoch": 0.91, + "learning_rate": 7.215245737211636e-05, + "loss": 1.237, + "theoretical_loss": 3.348694480701885, + "tokens_seen": 3006267392 + }, + { + "epoch": 0.91, + "learning_rate": 7.213640922768306e-05, + "loss": 1.217, + "theoretical_loss": 3.3486889180578467, + "tokens_seen": 3006332928 + }, + { + "epoch": 0.91, + "learning_rate": 7.212036108324976e-05, + "loss": 1.2933, + "theoretical_loss": 3.3486833555690216, + "tokens_seen": 3006398464 + }, + { + "epoch": 0.91, + "learning_rate": 7.210431293881645e-05, + "loss": 1.191, + "theoretical_loss": 3.3486777932354017, + "tokens_seen": 3006464000 + }, + { + "epoch": 0.91, + "learning_rate": 7.208826479438315e-05, + "loss": 1.1983, + "theoretical_loss": 3.3486722310569794, + "tokens_seen": 3006529536 + }, + { + "epoch": 0.91, + "learning_rate": 7.207221664994985e-05, + "loss": 1.1878, + "theoretical_loss": 3.3486666690337463, + "tokens_seen": 3006595072 + }, + { + "epoch": 0.91, + "learning_rate": 7.205616850551655e-05, + "loss": 1.19, + "theoretical_loss": 3.348661107165696, + "tokens_seen": 3006660608 + }, + { + "epoch": 0.91, + "learning_rate": 7.204012036108325e-05, + "loss": 1.228, + "theoretical_loss": 3.3486555454528197, + "tokens_seen": 3006726144 + }, + { + "epoch": 0.91, + "learning_rate": 7.202407221664995e-05, + "loss": 1.1996, + "theoretical_loss": 3.3486499838951107, + "tokens_seen": 3006791680 + }, + { + "epoch": 0.91, + "learning_rate": 7.200802407221665e-05, + "loss": 1.0929, + "theoretical_loss": 3.3486444224925602, + "tokens_seen": 3006857216 + }, + { + "epoch": 0.91, + "learning_rate": 7.199197592778335e-05, + "loss": 1.189, + "theoretical_loss": 3.348638861245161, + "tokens_seen": 3006922752 + }, + { + "epoch": 0.91, + "learning_rate": 7.197592778335005e-05, + "loss": 1.1074, + "theoretical_loss": 3.3486333001529056, + "tokens_seen": 3006988288 + }, + { + "epoch": 0.91, + "learning_rate": 7.195987963891675e-05, + "loss": 1.152, + "theoretical_loss": 3.348627739215786, + "tokens_seen": 3007053824 + }, + { + "epoch": 0.91, + "learning_rate": 7.194383149448345e-05, + "loss": 1.2543, + "theoretical_loss": 3.3486221784337946, + "tokens_seen": 3007119360 + }, + { + "epoch": 0.91, + "learning_rate": 7.192778335005015e-05, + "loss": 1.192, + "theoretical_loss": 3.348616617806923, + "tokens_seen": 3007184896 + }, + { + "epoch": 0.91, + "learning_rate": 7.191173520561686e-05, + "loss": 1.2914, + "theoretical_loss": 3.348611057335165, + "tokens_seen": 3007250432 + }, + { + "epoch": 0.91, + "learning_rate": 7.189568706118356e-05, + "loss": 1.2327, + "theoretical_loss": 3.3486054970185117, + "tokens_seen": 3007315968 + }, + { + "epoch": 0.91, + "learning_rate": 7.187963891675026e-05, + "loss": 1.2502, + "theoretical_loss": 3.3485999368569557, + "tokens_seen": 3007381504 + }, + { + "epoch": 0.91, + "learning_rate": 7.186359077231696e-05, + "loss": 1.1609, + "theoretical_loss": 3.3485943768504893, + "tokens_seen": 3007447040 + }, + { + "epoch": 0.91, + "learning_rate": 7.184754262788366e-05, + "loss": 1.1641, + "theoretical_loss": 3.348588816999105, + "tokens_seen": 3007512576 + }, + { + "epoch": 0.91, + "learning_rate": 7.183149448345036e-05, + "loss": 1.2452, + "theoretical_loss": 3.3485832573027947, + "tokens_seen": 3007578112 + }, + { + "epoch": 0.91, + "learning_rate": 7.181544633901706e-05, + "loss": 1.262, + "theoretical_loss": 3.348577697761551, + "tokens_seen": 3007643648 + }, + { + "epoch": 0.91, + "learning_rate": 7.179939819458376e-05, + "loss": 1.1225, + "theoretical_loss": 3.348572138375366, + "tokens_seen": 3007709184 + }, + { + "epoch": 0.91, + "learning_rate": 7.178335005015046e-05, + "loss": 1.2562, + "theoretical_loss": 3.348566579144232, + "tokens_seen": 3007774720 + }, + { + "epoch": 0.91, + "learning_rate": 7.176730190571716e-05, + "loss": 1.2504, + "theoretical_loss": 3.348561020068142, + "tokens_seen": 3007840256 + }, + { + "epoch": 0.91, + "learning_rate": 7.175125376128385e-05, + "loss": 1.2014, + "theoretical_loss": 3.348555461147087, + "tokens_seen": 3007905792 + }, + { + "epoch": 0.91, + "learning_rate": 7.173520561685055e-05, + "loss": 1.1807, + "theoretical_loss": 3.3485499023810603, + "tokens_seen": 3007971328 + }, + { + "epoch": 0.91, + "learning_rate": 7.171915747241725e-05, + "loss": 1.189, + "theoretical_loss": 3.3485443437700537, + "tokens_seen": 3008036864 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1003197, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.215165138244629, + "objective/train/theoretical_loss": 3.34853878531406, + "objective/train/tokens_used": 3028562400, + "theoretical_loss": 3.34853878531406, + "tokens_seen": 3008102400 + }, + { + "epoch": 0.91, + "learning_rate": 7.170310932798395e-05, + "loss": 1.2396, + "theoretical_loss": 3.34853878531406, + "tokens_seen": 3008102400 + }, + { + "epoch": 0.91, + "learning_rate": 7.168706118355065e-05, + "loss": 1.3155, + "theoretical_loss": 3.348533227013071, + "tokens_seen": 3008167936 + }, + { + "epoch": 0.91, + "learning_rate": 7.167101303911735e-05, + "loss": 1.1555, + "theoretical_loss": 3.3485276688670793, + "tokens_seen": 3008233472 + }, + { + "epoch": 0.91, + "learning_rate": 7.165496489468405e-05, + "loss": 1.2213, + "theoretical_loss": 3.348522110876077, + "tokens_seen": 3008299008 + }, + { + "epoch": 0.91, + "learning_rate": 7.163891675025075e-05, + "loss": 1.2507, + "theoretical_loss": 3.3485165530400565, + "tokens_seen": 3008364544 + }, + { + "epoch": 0.91, + "learning_rate": 7.162286860581745e-05, + "loss": 1.2393, + "theoretical_loss": 3.34851099535901, + "tokens_seen": 3008430080 + }, + { + "epoch": 0.91, + "learning_rate": 7.160682046138417e-05, + "loss": 1.1835, + "theoretical_loss": 3.34850543783293, + "tokens_seen": 3008495616 + }, + { + "epoch": 0.91, + "learning_rate": 7.159077231695085e-05, + "loss": 1.1424, + "theoretical_loss": 3.3484998804618087, + "tokens_seen": 3008561152 + }, + { + "epoch": 0.91, + "learning_rate": 7.157472417251756e-05, + "loss": 1.1059, + "theoretical_loss": 3.3484943232456383, + "tokens_seen": 3008626688 + }, + { + "epoch": 0.91, + "learning_rate": 7.155867602808426e-05, + "loss": 1.1736, + "theoretical_loss": 3.3484887661844116, + "tokens_seen": 3008692224 + }, + { + "epoch": 0.91, + "learning_rate": 7.154262788365096e-05, + "loss": 1.2272, + "theoretical_loss": 3.34848320927812, + "tokens_seen": 3008757760 + }, + { + "epoch": 0.91, + "learning_rate": 7.152657973921766e-05, + "loss": 1.1807, + "theoretical_loss": 3.3484776525267566, + "tokens_seen": 3008823296 + }, + { + "epoch": 0.91, + "learning_rate": 7.151053159478436e-05, + "loss": 1.2624, + "theoretical_loss": 3.3484720959303136, + "tokens_seen": 3008888832 + }, + { + "epoch": 0.91, + "learning_rate": 7.149448345035106e-05, + "loss": 1.2373, + "theoretical_loss": 3.3484665394887827, + "tokens_seen": 3008954368 + }, + { + "epoch": 0.91, + "learning_rate": 7.147843530591776e-05, + "loss": 1.0959, + "theoretical_loss": 3.3484609832021572, + "tokens_seen": 3009019904 + }, + { + "epoch": 0.91, + "learning_rate": 7.146238716148446e-05, + "loss": 1.1559, + "theoretical_loss": 3.3484554270704283, + "tokens_seen": 3009085440 + }, + { + "epoch": 0.91, + "learning_rate": 7.144633901705116e-05, + "loss": 1.2162, + "theoretical_loss": 3.348449871093589, + "tokens_seen": 3009150976 + }, + { + "epoch": 0.91, + "learning_rate": 7.143029087261786e-05, + "loss": 1.2092, + "theoretical_loss": 3.3484443152716317, + "tokens_seen": 3009216512 + }, + { + "epoch": 0.91, + "learning_rate": 7.141424272818456e-05, + "loss": 1.1757, + "theoretical_loss": 3.348438759604548, + "tokens_seen": 3009282048 + }, + { + "epoch": 0.91, + "learning_rate": 7.139819458375125e-05, + "loss": 1.1227, + "theoretical_loss": 3.3484332040923315, + "tokens_seen": 3009347584 + }, + { + "epoch": 0.91, + "learning_rate": 7.138214643931795e-05, + "loss": 1.1485, + "theoretical_loss": 3.348427648734973, + "tokens_seen": 3009413120 + }, + { + "epoch": 0.91, + "learning_rate": 7.136609829488465e-05, + "loss": 1.2711, + "theoretical_loss": 3.3484220935324656, + "tokens_seen": 3009478656 + }, + { + "epoch": 0.91, + "learning_rate": 7.135005015045135e-05, + "loss": 1.1693, + "theoretical_loss": 3.3484165384848015, + "tokens_seen": 3009544192 + }, + { + "epoch": 0.91, + "learning_rate": 7.133400200601805e-05, + "loss": 1.2371, + "theoretical_loss": 3.3484109835919735, + "tokens_seen": 3009609728 + }, + { + "epoch": 0.91, + "learning_rate": 7.131795386158477e-05, + "loss": 1.2767, + "theoretical_loss": 3.348405428853973, + "tokens_seen": 3009675264 + }, + { + "epoch": 0.91, + "learning_rate": 7.130190571715147e-05, + "loss": 1.1709, + "theoretical_loss": 3.3483998742707928, + "tokens_seen": 3009740800 + }, + { + "epoch": 0.91, + "learning_rate": 7.128585757271817e-05, + "loss": 1.2121, + "theoretical_loss": 3.3483943198424253, + "tokens_seen": 3009806336 + }, + { + "epoch": 0.91, + "learning_rate": 7.126980942828487e-05, + "loss": 1.1806, + "theoretical_loss": 3.3483887655688624, + "tokens_seen": 3009871872 + }, + { + "epoch": 0.91, + "learning_rate": 7.125376128385157e-05, + "loss": 1.2008, + "theoretical_loss": 3.348383211450097, + "tokens_seen": 3009937408 + }, + { + "epoch": 0.91, + "learning_rate": 7.123771313941825e-05, + "loss": 1.219, + "theoretical_loss": 3.348377657486121, + "tokens_seen": 3010002944 + }, + { + "epoch": 0.91, + "learning_rate": 7.122166499498496e-05, + "loss": 1.1826, + "theoretical_loss": 3.348372103676927, + "tokens_seen": 3010068480 + }, + { + "epoch": 0.91, + "learning_rate": 7.120561685055166e-05, + "loss": 1.1482, + "theoretical_loss": 3.348366550022507, + "tokens_seen": 3010134016 + }, + { + "epoch": 0.91, + "learning_rate": 7.118956870611836e-05, + "loss": 1.1836, + "theoretical_loss": 3.3483609965228536, + "tokens_seen": 3010199552 + }, + { + "epoch": 0.91, + "learning_rate": 7.117352056168506e-05, + "loss": 1.1847, + "theoretical_loss": 3.348355443177959, + "tokens_seen": 3010265088 + }, + { + "epoch": 0.91, + "learning_rate": 7.115747241725176e-05, + "loss": 1.222, + "theoretical_loss": 3.3483498899878152, + "tokens_seen": 3010330624 + }, + { + "epoch": 0.91, + "learning_rate": 7.114142427281846e-05, + "loss": 1.1843, + "theoretical_loss": 3.348344336952415, + "tokens_seen": 3010396160 + }, + { + "epoch": 0.91, + "learning_rate": 7.112537612838516e-05, + "loss": 1.1677, + "theoretical_loss": 3.3483387840717507, + "tokens_seen": 3010461696 + }, + { + "epoch": 0.91, + "learning_rate": 7.110932798395186e-05, + "loss": 1.1544, + "theoretical_loss": 3.3483332313458143, + "tokens_seen": 3010527232 + }, + { + "epoch": 0.91, + "learning_rate": 7.109327983951856e-05, + "loss": 1.2975, + "theoretical_loss": 3.348327678774598, + "tokens_seen": 3010592768 + }, + { + "epoch": 0.91, + "learning_rate": 7.107723169508526e-05, + "loss": 1.1992, + "theoretical_loss": 3.3483221263580947, + "tokens_seen": 3010658304 + }, + { + "epoch": 0.91, + "learning_rate": 7.106118355065196e-05, + "loss": 1.2225, + "theoretical_loss": 3.3483165740962963, + "tokens_seen": 3010723840 + }, + { + "epoch": 0.91, + "learning_rate": 7.104513540621865e-05, + "loss": 1.2037, + "theoretical_loss": 3.3483110219891956, + "tokens_seen": 3010789376 + }, + { + "epoch": 0.91, + "learning_rate": 7.102908726178535e-05, + "loss": 1.2034, + "theoretical_loss": 3.3483054700367845, + "tokens_seen": 3010854912 + }, + { + "epoch": 0.91, + "learning_rate": 7.101303911735206e-05, + "loss": 1.1707, + "theoretical_loss": 3.348299918239055, + "tokens_seen": 3010920448 + }, + { + "epoch": 0.91, + "learning_rate": 7.099699097291876e-05, + "loss": 1.1928, + "theoretical_loss": 3.348294366596, + "tokens_seen": 3010985984 + }, + { + "epoch": 0.91, + "learning_rate": 7.098094282848547e-05, + "loss": 1.214, + "theoretical_loss": 3.348288815107612, + "tokens_seen": 3011051520 + }, + { + "epoch": 0.91, + "learning_rate": 7.096489468405217e-05, + "loss": 1.1532, + "theoretical_loss": 3.348283263773882, + "tokens_seen": 3011117056 + }, + { + "epoch": 0.91, + "learning_rate": 7.094884653961887e-05, + "loss": 1.2678, + "theoretical_loss": 3.3482777125948044, + "tokens_seen": 3011182592 + }, + { + "epoch": 0.91, + "learning_rate": 7.093279839518557e-05, + "loss": 1.1814, + "theoretical_loss": 3.34827216157037, + "tokens_seen": 3011248128 + }, + { + "epoch": 0.91, + "learning_rate": 7.091675025075227e-05, + "loss": 1.1454, + "theoretical_loss": 3.3482666107005716, + "tokens_seen": 3011313664 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1004330, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2958860397338867, + "objective/train/theoretical_loss": 3.348261059985401, + "objective/train/tokens_used": 3031839200, + "theoretical_loss": 3.348261059985401, + "tokens_seen": 3011379200 + }, + { + "epoch": 0.91, + "learning_rate": 7.090070210631895e-05, + "loss": 1.2207, + "theoretical_loss": 3.348261059985401, + "tokens_seen": 3011379200 + }, + { + "epoch": 0.91, + "learning_rate": 7.088465396188566e-05, + "loss": 1.228, + "theoretical_loss": 3.3482555094248516, + "tokens_seen": 3011444736 + }, + { + "epoch": 0.91, + "learning_rate": 7.086860581745236e-05, + "loss": 1.185, + "theoretical_loss": 3.348249959018915, + "tokens_seen": 3011510272 + }, + { + "epoch": 0.91, + "learning_rate": 7.085255767301906e-05, + "loss": 1.1234, + "theoretical_loss": 3.3482444087675836, + "tokens_seen": 3011575808 + }, + { + "epoch": 0.91, + "learning_rate": 7.083650952858576e-05, + "loss": 1.1322, + "theoretical_loss": 3.3482388586708494, + "tokens_seen": 3011641344 + }, + { + "epoch": 0.91, + "learning_rate": 7.082046138415246e-05, + "loss": 1.1626, + "theoretical_loss": 3.3482333087287053, + "tokens_seen": 3011706880 + }, + { + "epoch": 0.91, + "learning_rate": 7.080441323971916e-05, + "loss": 1.0629, + "theoretical_loss": 3.3482277589411438, + "tokens_seen": 3011772416 + }, + { + "epoch": 0.91, + "learning_rate": 7.078836509528586e-05, + "loss": 1.1157, + "theoretical_loss": 3.348222209308157, + "tokens_seen": 3011837952 + }, + { + "epoch": 0.91, + "learning_rate": 7.077231695085256e-05, + "loss": 1.1105, + "theoretical_loss": 3.3482166598297365, + "tokens_seen": 3011903488 + }, + { + "epoch": 0.91, + "learning_rate": 7.075626880641926e-05, + "loss": 1.2278, + "theoretical_loss": 3.3482111105058756, + "tokens_seen": 3011969024 + }, + { + "epoch": 0.91, + "learning_rate": 7.074022066198596e-05, + "loss": 1.2553, + "theoretical_loss": 3.3482055613365658, + "tokens_seen": 3012034560 + }, + { + "epoch": 0.91, + "learning_rate": 7.072417251755266e-05, + "loss": 1.1502, + "theoretical_loss": 3.3482000123218003, + "tokens_seen": 3012100096 + }, + { + "epoch": 0.91, + "learning_rate": 7.070812437311936e-05, + "loss": 1.2973, + "theoretical_loss": 3.3481944634615712, + "tokens_seen": 3012165632 + }, + { + "epoch": 0.91, + "learning_rate": 7.069207622868606e-05, + "loss": 1.1967, + "theoretical_loss": 3.3481889147558705, + "tokens_seen": 3012231168 + }, + { + "epoch": 0.91, + "learning_rate": 7.067602808425276e-05, + "loss": 1.2283, + "theoretical_loss": 3.3481833662046907, + "tokens_seen": 3012296704 + }, + { + "epoch": 0.91, + "learning_rate": 7.065997993981946e-05, + "loss": 1.1253, + "theoretical_loss": 3.348177817808024, + "tokens_seen": 3012362240 + }, + { + "epoch": 0.91, + "learning_rate": 7.064393179538616e-05, + "loss": 1.2379, + "theoretical_loss": 3.3481722695658633, + "tokens_seen": 3012427776 + }, + { + "epoch": 0.91, + "learning_rate": 7.062788365095287e-05, + "loss": 1.217, + "theoretical_loss": 3.3481667214782, + "tokens_seen": 3012493312 + }, + { + "epoch": 0.91, + "learning_rate": 7.061183550651957e-05, + "loss": 1.1916, + "theoretical_loss": 3.3481611735450274, + "tokens_seen": 3012558848 + }, + { + "epoch": 0.91, + "learning_rate": 7.059578736208627e-05, + "loss": 1.2126, + "theoretical_loss": 3.3481556257663367, + "tokens_seen": 3012624384 + }, + { + "epoch": 0.91, + "learning_rate": 7.057973921765297e-05, + "loss": 1.1263, + "theoretical_loss": 3.3481500781421216, + "tokens_seen": 3012689920 + }, + { + "epoch": 0.91, + "learning_rate": 7.056369107321967e-05, + "loss": 1.1485, + "theoretical_loss": 3.3481445306723736, + "tokens_seen": 3012755456 + }, + { + "epoch": 0.91, + "learning_rate": 7.054764292878636e-05, + "loss": 1.1263, + "theoretical_loss": 3.348138983357085, + "tokens_seen": 3012820992 + }, + { + "epoch": 0.91, + "learning_rate": 7.053159478435306e-05, + "loss": 1.1791, + "theoretical_loss": 3.3481334361962487, + "tokens_seen": 3012886528 + }, + { + "epoch": 0.91, + "learning_rate": 7.051554663991976e-05, + "loss": 1.2312, + "theoretical_loss": 3.3481278891898567, + "tokens_seen": 3012952064 + }, + { + "epoch": 0.91, + "learning_rate": 7.049949849548646e-05, + "loss": 1.1901, + "theoretical_loss": 3.3481223423379007, + "tokens_seen": 3013017600 + }, + { + "epoch": 0.91, + "learning_rate": 7.048345035105316e-05, + "loss": 1.25, + "theoretical_loss": 3.348116795640374, + "tokens_seen": 3013083136 + }, + { + "epoch": 0.91, + "learning_rate": 7.046740220661986e-05, + "loss": 1.0985, + "theoretical_loss": 3.348111249097269, + "tokens_seen": 3013148672 + }, + { + "epoch": 0.91, + "learning_rate": 7.045135406218656e-05, + "loss": 1.246, + "theoretical_loss": 3.348105702708577, + "tokens_seen": 3013214208 + }, + { + "epoch": 0.91, + "learning_rate": 7.043530591775326e-05, + "loss": 1.2422, + "theoretical_loss": 3.348100156474292, + "tokens_seen": 3013279744 + }, + { + "epoch": 0.91, + "learning_rate": 7.041925777331997e-05, + "loss": 1.0969, + "theoretical_loss": 3.348094610394404, + "tokens_seen": 3013345280 + }, + { + "epoch": 0.91, + "learning_rate": 7.040320962888667e-05, + "loss": 1.1267, + "theoretical_loss": 3.3480890644689074, + "tokens_seen": 3013410816 + }, + { + "epoch": 0.91, + "learning_rate": 7.038716148445336e-05, + "loss": 1.2125, + "theoretical_loss": 3.348083518697794, + "tokens_seen": 3013476352 + }, + { + "epoch": 0.91, + "learning_rate": 7.037111334002006e-05, + "loss": 1.1678, + "theoretical_loss": 3.3480779730810557, + "tokens_seen": 3013541888 + }, + { + "epoch": 0.91, + "learning_rate": 7.035506519558676e-05, + "loss": 1.2074, + "theoretical_loss": 3.348072427618685, + "tokens_seen": 3013607424 + }, + { + "epoch": 0.91, + "learning_rate": 7.033901705115346e-05, + "loss": 1.2299, + "theoretical_loss": 3.3480668823106745, + "tokens_seen": 3013672960 + }, + { + "epoch": 0.91, + "learning_rate": 7.032296890672016e-05, + "loss": 1.1942, + "theoretical_loss": 3.3480613371570165, + "tokens_seen": 3013738496 + }, + { + "epoch": 0.91, + "learning_rate": 7.030692076228686e-05, + "loss": 1.1637, + "theoretical_loss": 3.348055792157703, + "tokens_seen": 3013804032 + }, + { + "epoch": 0.91, + "learning_rate": 7.029087261785357e-05, + "loss": 1.2552, + "theoretical_loss": 3.348050247312727, + "tokens_seen": 3013869568 + }, + { + "epoch": 0.91, + "learning_rate": 7.027482447342027e-05, + "loss": 1.0909, + "theoretical_loss": 3.3480447026220803, + "tokens_seen": 3013935104 + }, + { + "epoch": 0.91, + "learning_rate": 7.025877632898697e-05, + "loss": 1.1964, + "theoretical_loss": 3.348039158085755, + "tokens_seen": 3014000640 + }, + { + "epoch": 0.91, + "learning_rate": 7.024272818455367e-05, + "loss": 1.0273, + "theoretical_loss": 3.3480336137037443, + "tokens_seen": 3014066176 + }, + { + "epoch": 0.91, + "learning_rate": 7.022668004012037e-05, + "loss": 1.1838, + "theoretical_loss": 3.3480280694760403, + "tokens_seen": 3014131712 + }, + { + "epoch": 0.91, + "learning_rate": 7.021063189568707e-05, + "loss": 1.2325, + "theoretical_loss": 3.3480225254026346, + "tokens_seen": 3014197248 + }, + { + "epoch": 0.91, + "learning_rate": 7.019458375125376e-05, + "loss": 1.2786, + "theoretical_loss": 3.34801698148352, + "tokens_seen": 3014262784 + }, + { + "epoch": 0.91, + "learning_rate": 7.017853560682046e-05, + "loss": 1.1701, + "theoretical_loss": 3.3480114377186894, + "tokens_seen": 3014328320 + }, + { + "epoch": 0.91, + "learning_rate": 7.016248746238716e-05, + "loss": 1.2282, + "theoretical_loss": 3.3480058941081348, + "tokens_seen": 3014393856 + }, + { + "epoch": 0.91, + "learning_rate": 7.014643931795386e-05, + "loss": 1.2156, + "theoretical_loss": 3.348000350651848, + "tokens_seen": 3014459392 + }, + { + "epoch": 0.91, + "learning_rate": 7.013039117352056e-05, + "loss": 1.2464, + "theoretical_loss": 3.347994807349822, + "tokens_seen": 3014524928 + }, + { + "epoch": 0.91, + "learning_rate": 7.011434302908727e-05, + "loss": 1.2198, + "theoretical_loss": 3.3479892642020492, + "tokens_seen": 3014590464 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1005262, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2559559345245361, + "objective/train/theoretical_loss": 3.3479837212085215, + "objective/train/tokens_used": 3035116000, + "theoretical_loss": 3.3479837212085215, + "tokens_seen": 3014656000 + }, + { + "epoch": 0.91, + "learning_rate": 7.009829488465397e-05, + "loss": 1.2476, + "theoretical_loss": 3.3479837212085215, + "tokens_seen": 3014656000 + }, + { + "epoch": 0.91, + "learning_rate": 7.008224674022067e-05, + "loss": 1.2104, + "theoretical_loss": 3.3479781783692317, + "tokens_seen": 3014721536 + }, + { + "epoch": 0.91, + "learning_rate": 7.006619859578737e-05, + "loss": 1.2086, + "theoretical_loss": 3.3479726356841715, + "tokens_seen": 3014787072 + }, + { + "epoch": 0.91, + "learning_rate": 7.005015045135407e-05, + "loss": 1.1878, + "theoretical_loss": 3.347967093153334, + "tokens_seen": 3014852608 + }, + { + "epoch": 0.91, + "learning_rate": 7.003410230692076e-05, + "loss": 1.2165, + "theoretical_loss": 3.347961550776711, + "tokens_seen": 3014918144 + }, + { + "epoch": 0.91, + "learning_rate": 7.001805416248746e-05, + "loss": 1.1967, + "theoretical_loss": 3.3479560085542954, + "tokens_seen": 3014983680 + }, + { + "epoch": 0.91, + "learning_rate": 7.000200601805416e-05, + "loss": 1.2398, + "theoretical_loss": 3.347950466486079, + "tokens_seen": 3015049216 + }, + { + "epoch": 0.91, + "learning_rate": 6.998595787362086e-05, + "loss": 1.1349, + "theoretical_loss": 3.3479449245720545, + "tokens_seen": 3015114752 + }, + { + "epoch": 0.91, + "learning_rate": 6.996990972918756e-05, + "loss": 1.1244, + "theoretical_loss": 3.347939382812214, + "tokens_seen": 3015180288 + }, + { + "epoch": 0.91, + "learning_rate": 6.995386158475426e-05, + "loss": 1.2009, + "theoretical_loss": 3.3479338412065505, + "tokens_seen": 3015245824 + }, + { + "epoch": 0.91, + "learning_rate": 6.993781344032097e-05, + "loss": 1.0983, + "theoretical_loss": 3.347928299755056, + "tokens_seen": 3015311360 + }, + { + "epoch": 0.91, + "learning_rate": 6.992176529588767e-05, + "loss": 1.0775, + "theoretical_loss": 3.347922758457722, + "tokens_seen": 3015376896 + }, + { + "epoch": 0.91, + "learning_rate": 6.990571715145437e-05, + "loss": 1.1677, + "theoretical_loss": 3.347917217314542, + "tokens_seen": 3015442432 + }, + { + "epoch": 0.91, + "learning_rate": 6.988966900702107e-05, + "loss": 1.3574, + "theoretical_loss": 3.347911676325508, + "tokens_seen": 3015507968 + }, + { + "epoch": 0.91, + "learning_rate": 6.987362086258777e-05, + "loss": 1.2204, + "theoretical_loss": 3.3479061354906126, + "tokens_seen": 3015573504 + }, + { + "epoch": 0.91, + "learning_rate": 6.985757271815447e-05, + "loss": 1.1823, + "theoretical_loss": 3.3479005948098473, + "tokens_seen": 3015639040 + }, + { + "epoch": 0.91, + "learning_rate": 6.984152457372116e-05, + "loss": 1.1865, + "theoretical_loss": 3.3478950542832053, + "tokens_seen": 3015704576 + }, + { + "epoch": 0.91, + "learning_rate": 6.982547642928786e-05, + "loss": 1.2557, + "theoretical_loss": 3.347889513910679, + "tokens_seen": 3015770112 + }, + { + "epoch": 0.91, + "learning_rate": 6.980942828485457e-05, + "loss": 1.18, + "theoretical_loss": 3.34788397369226, + "tokens_seen": 3015835648 + }, + { + "epoch": 0.91, + "learning_rate": 6.979338014042127e-05, + "loss": 1.0566, + "theoretical_loss": 3.3478784336279417, + "tokens_seen": 3015901184 + }, + { + "epoch": 0.91, + "learning_rate": 6.977733199598797e-05, + "loss": 1.2101, + "theoretical_loss": 3.3478728937177156, + "tokens_seen": 3015966720 + }, + { + "epoch": 0.91, + "learning_rate": 6.976128385155467e-05, + "loss": 1.2061, + "theoretical_loss": 3.3478673539615746, + "tokens_seen": 3016032256 + }, + { + "epoch": 0.91, + "learning_rate": 6.974523570712137e-05, + "loss": 1.1255, + "theoretical_loss": 3.3478618143595105, + "tokens_seen": 3016097792 + }, + { + "epoch": 0.91, + "learning_rate": 6.972918756268807e-05, + "loss": 1.2611, + "theoretical_loss": 3.3478562749115164, + "tokens_seen": 3016163328 + }, + { + "epoch": 0.91, + "learning_rate": 6.971313941825477e-05, + "loss": 1.2248, + "theoretical_loss": 3.347850735617584, + "tokens_seen": 3016228864 + }, + { + "epoch": 0.91, + "learning_rate": 6.969709127382148e-05, + "loss": 1.1932, + "theoretical_loss": 3.347845196477706, + "tokens_seen": 3016294400 + }, + { + "epoch": 0.91, + "learning_rate": 6.968104312938816e-05, + "loss": 1.2629, + "theoretical_loss": 3.3478396574918747, + "tokens_seen": 3016359936 + }, + { + "epoch": 0.91, + "learning_rate": 6.966499498495486e-05, + "loss": 1.1529, + "theoretical_loss": 3.347834118660083, + "tokens_seen": 3016425472 + }, + { + "epoch": 0.91, + "learning_rate": 6.964894684052156e-05, + "loss": 1.1914, + "theoretical_loss": 3.347828579982322, + "tokens_seen": 3016491008 + }, + { + "epoch": 0.91, + "learning_rate": 6.963289869608826e-05, + "loss": 1.2033, + "theoretical_loss": 3.3478230414585854, + "tokens_seen": 3016556544 + }, + { + "epoch": 0.91, + "learning_rate": 6.961685055165496e-05, + "loss": 1.25, + "theoretical_loss": 3.347817503088865, + "tokens_seen": 3016622080 + }, + { + "epoch": 0.91, + "learning_rate": 6.960080240722167e-05, + "loss": 1.2275, + "theoretical_loss": 3.3478119648731526, + "tokens_seen": 3016687616 + }, + { + "epoch": 0.91, + "learning_rate": 6.958475426278837e-05, + "loss": 1.2726, + "theoretical_loss": 3.3478064268114416, + "tokens_seen": 3016753152 + }, + { + "epoch": 0.91, + "learning_rate": 6.956870611835507e-05, + "loss": 1.2561, + "theoretical_loss": 3.347800888903724, + "tokens_seen": 3016818688 + }, + { + "epoch": 0.91, + "learning_rate": 6.955265797392177e-05, + "loss": 1.1804, + "theoretical_loss": 3.347795351149992, + "tokens_seen": 3016884224 + }, + { + "epoch": 0.91, + "learning_rate": 6.953660982948847e-05, + "loss": 1.1533, + "theoretical_loss": 3.3477898135502375, + "tokens_seen": 3016949760 + }, + { + "epoch": 0.91, + "learning_rate": 6.952056168505517e-05, + "loss": 1.1912, + "theoretical_loss": 3.347784276104454, + "tokens_seen": 3017015296 + }, + { + "epoch": 0.91, + "learning_rate": 6.950451354062187e-05, + "loss": 1.2314, + "theoretical_loss": 3.347778738812633, + "tokens_seen": 3017080832 + }, + { + "epoch": 0.91, + "learning_rate": 6.948846539618857e-05, + "loss": 1.1869, + "theoretical_loss": 3.3477732016747677, + "tokens_seen": 3017146368 + }, + { + "epoch": 0.91, + "learning_rate": 6.947241725175527e-05, + "loss": 1.182, + "theoretical_loss": 3.3477676646908496, + "tokens_seen": 3017211904 + }, + { + "epoch": 0.91, + "learning_rate": 6.945636910732197e-05, + "loss": 1.1249, + "theoretical_loss": 3.3477621278608716, + "tokens_seen": 3017277440 + }, + { + "epoch": 0.91, + "learning_rate": 6.944032096288867e-05, + "loss": 1.2046, + "theoretical_loss": 3.347756591184826, + "tokens_seen": 3017342976 + }, + { + "epoch": 0.91, + "learning_rate": 6.942427281845537e-05, + "loss": 1.2131, + "theoretical_loss": 3.3477510546627047, + "tokens_seen": 3017408512 + }, + { + "epoch": 0.91, + "learning_rate": 6.940822467402207e-05, + "loss": 1.1845, + "theoretical_loss": 3.347745518294501, + "tokens_seen": 3017474048 + }, + { + "epoch": 0.91, + "learning_rate": 6.939217652958877e-05, + "loss": 1.1501, + "theoretical_loss": 3.3477399820802063, + "tokens_seen": 3017539584 + }, + { + "epoch": 0.91, + "learning_rate": 6.937612838515547e-05, + "loss": 1.0732, + "theoretical_loss": 3.3477344460198135, + "tokens_seen": 3017605120 + }, + { + "epoch": 0.91, + "learning_rate": 6.936008024072217e-05, + "loss": 1.2365, + "theoretical_loss": 3.347728910113315, + "tokens_seen": 3017670656 + }, + { + "epoch": 0.91, + "learning_rate": 6.934403209628888e-05, + "loss": 1.1618, + "theoretical_loss": 3.347723374360703, + "tokens_seen": 3017736192 + }, + { + "epoch": 0.91, + "learning_rate": 6.932798395185556e-05, + "loss": 1.1741, + "theoretical_loss": 3.3477178387619704, + "tokens_seen": 3017801728 + }, + { + "epoch": 0.91, + "learning_rate": 6.931193580742226e-05, + "loss": 1.299, + "theoretical_loss": 3.347712303317109, + "tokens_seen": 3017867264 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1006524, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.271710753440857, + "objective/train/theoretical_loss": 3.347706768026111, + "objective/train/tokens_used": 3038392800, + "theoretical_loss": 3.347706768026111, + "tokens_seen": 3017932800 + }, + { + "epoch": 0.91, + "learning_rate": 6.929588766298896e-05, + "loss": 1.2658, + "theoretical_loss": 3.347706768026111, + "tokens_seen": 3017932800 + }, + { + "epoch": 0.91, + "learning_rate": 6.927983951855566e-05, + "loss": 1.3269, + "theoretical_loss": 3.347701232888969, + "tokens_seen": 3017998336 + }, + { + "epoch": 0.91, + "learning_rate": 6.926379137412237e-05, + "loss": 1.1517, + "theoretical_loss": 3.347695697905676, + "tokens_seen": 3018063872 + }, + { + "epoch": 0.91, + "learning_rate": 6.924774322968907e-05, + "loss": 1.1925, + "theoretical_loss": 3.3476901630762232, + "tokens_seen": 3018129408 + }, + { + "epoch": 0.91, + "learning_rate": 6.923169508525577e-05, + "loss": 1.1824, + "theoretical_loss": 3.347684628400604, + "tokens_seen": 3018194944 + }, + { + "epoch": 0.91, + "learning_rate": 6.921564694082248e-05, + "loss": 1.1837, + "theoretical_loss": 3.347679093878811, + "tokens_seen": 3018260480 + }, + { + "epoch": 0.91, + "learning_rate": 6.919959879638918e-05, + "loss": 1.207, + "theoretical_loss": 3.3476735595108353, + "tokens_seen": 3018326016 + }, + { + "epoch": 0.91, + "learning_rate": 6.918355065195588e-05, + "loss": 1.1051, + "theoretical_loss": 3.34766802529667, + "tokens_seen": 3018391552 + }, + { + "epoch": 0.91, + "learning_rate": 6.916750250752257e-05, + "loss": 1.2093, + "theoretical_loss": 3.3476624912363078, + "tokens_seen": 3018457088 + }, + { + "epoch": 0.91, + "learning_rate": 6.915145436308927e-05, + "loss": 1.079, + "theoretical_loss": 3.347656957329741, + "tokens_seen": 3018522624 + }, + { + "epoch": 0.91, + "learning_rate": 6.913540621865597e-05, + "loss": 1.1853, + "theoretical_loss": 3.3476514235769614, + "tokens_seen": 3018588160 + }, + { + "epoch": 0.91, + "learning_rate": 6.911935807422267e-05, + "loss": 1.1297, + "theoretical_loss": 3.347645889977962, + "tokens_seen": 3018653696 + }, + { + "epoch": 0.91, + "learning_rate": 6.910330992978937e-05, + "loss": 1.1163, + "theoretical_loss": 3.347640356532735, + "tokens_seen": 3018719232 + }, + { + "epoch": 0.91, + "learning_rate": 6.908726178535607e-05, + "loss": 1.1882, + "theoretical_loss": 3.3476348232412723, + "tokens_seen": 3018784768 + }, + { + "epoch": 0.91, + "learning_rate": 6.907121364092277e-05, + "loss": 1.2052, + "theoretical_loss": 3.3476292901035674, + "tokens_seen": 3018850304 + }, + { + "epoch": 0.91, + "learning_rate": 6.905516549648947e-05, + "loss": 1.2335, + "theoretical_loss": 3.3476237571196115, + "tokens_seen": 3018915840 + }, + { + "epoch": 0.91, + "learning_rate": 6.903911735205617e-05, + "loss": 1.212, + "theoretical_loss": 3.3476182242893975, + "tokens_seen": 3018981376 + }, + { + "epoch": 0.91, + "learning_rate": 6.902306920762287e-05, + "loss": 1.2896, + "theoretical_loss": 3.347612691612918, + "tokens_seen": 3019046912 + }, + { + "epoch": 0.91, + "learning_rate": 6.900702106318958e-05, + "loss": 1.2209, + "theoretical_loss": 3.347607159090165, + "tokens_seen": 3019112448 + }, + { + "epoch": 0.91, + "learning_rate": 6.899097291875628e-05, + "loss": 1.1924, + "theoretical_loss": 3.3476016267211315, + "tokens_seen": 3019177984 + }, + { + "epoch": 0.91, + "learning_rate": 6.897492477432296e-05, + "loss": 1.1896, + "theoretical_loss": 3.347596094505809, + "tokens_seen": 3019243520 + }, + { + "epoch": 0.91, + "learning_rate": 6.895887662988966e-05, + "loss": 1.1897, + "theoretical_loss": 3.3475905624441906, + "tokens_seen": 3019309056 + }, + { + "epoch": 0.91, + "learning_rate": 6.894282848545636e-05, + "loss": 1.2482, + "theoretical_loss": 3.347585030536268, + "tokens_seen": 3019374592 + }, + { + "epoch": 0.91, + "learning_rate": 6.892678034102306e-05, + "loss": 1.2147, + "theoretical_loss": 3.347579498782035, + "tokens_seen": 3019440128 + }, + { + "epoch": 0.92, + "learning_rate": 6.891073219658978e-05, + "loss": 1.2183, + "theoretical_loss": 3.3475739671814826, + "tokens_seen": 3019505664 + }, + { + "epoch": 0.92, + "learning_rate": 6.889468405215648e-05, + "loss": 1.3038, + "theoretical_loss": 3.347568435734604, + "tokens_seen": 3019571200 + }, + { + "epoch": 0.92, + "learning_rate": 6.887863590772318e-05, + "loss": 1.1745, + "theoretical_loss": 3.34756290444139, + "tokens_seen": 3019636736 + }, + { + "epoch": 0.92, + "learning_rate": 6.886258776328988e-05, + "loss": 1.2467, + "theoretical_loss": 3.3475573733018353, + "tokens_seen": 3019702272 + }, + { + "epoch": 0.92, + "learning_rate": 6.884653961885658e-05, + "loss": 1.3152, + "theoretical_loss": 3.347551842315931, + "tokens_seen": 3019767808 + }, + { + "epoch": 0.92, + "learning_rate": 6.883049147442327e-05, + "loss": 1.2948, + "theoretical_loss": 3.34754631148367, + "tokens_seen": 3019833344 + }, + { + "epoch": 0.92, + "learning_rate": 6.881444332998997e-05, + "loss": 1.1106, + "theoretical_loss": 3.347540780805044, + "tokens_seen": 3019898880 + }, + { + "epoch": 0.92, + "learning_rate": 6.879839518555667e-05, + "loss": 1.2195, + "theoretical_loss": 3.347535250280046, + "tokens_seen": 3019964416 + }, + { + "epoch": 0.92, + "learning_rate": 6.878234704112337e-05, + "loss": 1.2819, + "theoretical_loss": 3.3475297199086684, + "tokens_seen": 3020029952 + }, + { + "epoch": 0.92, + "learning_rate": 6.876629889669007e-05, + "loss": 1.133, + "theoretical_loss": 3.347524189690903, + "tokens_seen": 3020095488 + }, + { + "epoch": 0.92, + "learning_rate": 6.875025075225677e-05, + "loss": 1.1815, + "theoretical_loss": 3.347518659626743, + "tokens_seen": 3020161024 + }, + { + "epoch": 0.92, + "learning_rate": 6.873420260782347e-05, + "loss": 1.1755, + "theoretical_loss": 3.34751312971618, + "tokens_seen": 3020226560 + }, + { + "epoch": 0.92, + "learning_rate": 6.871815446339017e-05, + "loss": 1.1683, + "theoretical_loss": 3.3475075999592074, + "tokens_seen": 3020292096 + }, + { + "epoch": 0.92, + "learning_rate": 6.870210631895687e-05, + "loss": 1.2666, + "theoretical_loss": 3.3475020703558167, + "tokens_seen": 3020357632 + }, + { + "epoch": 0.92, + "learning_rate": 6.868605817452357e-05, + "loss": 1.1731, + "theoretical_loss": 3.347496540906001, + "tokens_seen": 3020423168 + }, + { + "epoch": 0.92, + "learning_rate": 6.867001003009028e-05, + "loss": 1.2172, + "theoretical_loss": 3.3474910116097516, + "tokens_seen": 3020488704 + }, + { + "epoch": 0.92, + "learning_rate": 6.865396188565698e-05, + "loss": 1.1433, + "theoretical_loss": 3.347485482467062, + "tokens_seen": 3020554240 + }, + { + "epoch": 0.92, + "learning_rate": 6.863791374122366e-05, + "loss": 1.1755, + "theoretical_loss": 3.3474799534779245, + "tokens_seen": 3020619776 + }, + { + "epoch": 0.92, + "learning_rate": 6.862186559679036e-05, + "loss": 1.1592, + "theoretical_loss": 3.347474424642331, + "tokens_seen": 3020685312 + }, + { + "epoch": 0.92, + "learning_rate": 6.860581745235708e-05, + "loss": 1.1794, + "theoretical_loss": 3.3474688959602745, + "tokens_seen": 3020750848 + }, + { + "epoch": 0.92, + "learning_rate": 6.858976930792378e-05, + "loss": 1.1285, + "theoretical_loss": 3.3474633674317467, + "tokens_seen": 3020816384 + }, + { + "epoch": 0.92, + "learning_rate": 6.857372116349048e-05, + "loss": 1.1872, + "theoretical_loss": 3.3474578390567404, + "tokens_seen": 3020881920 + }, + { + "epoch": 0.92, + "learning_rate": 6.855767301905718e-05, + "loss": 1.2461, + "theoretical_loss": 3.347452310835248, + "tokens_seen": 3020947456 + }, + { + "epoch": 0.92, + "learning_rate": 6.854162487462388e-05, + "loss": 1.2819, + "theoretical_loss": 3.347446782767262, + "tokens_seen": 3021012992 + }, + { + "epoch": 0.92, + "learning_rate": 6.852557673019058e-05, + "loss": 1.2149, + "theoretical_loss": 3.3474412548527743, + "tokens_seen": 3021078528 + }, + { + "epoch": 0.92, + "learning_rate": 6.850952858575728e-05, + "loss": 1.2144, + "theoretical_loss": 3.347435727091778, + "tokens_seen": 3021144064 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1007419, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1903594732284546, + "objective/train/theoretical_loss": 3.3474301994842652, + "objective/train/tokens_used": 3041669600, + "theoretical_loss": 3.3474301994842652, + "tokens_seen": 3021209600 + }, + { + "epoch": 0.92, + "learning_rate": 6.849348044132398e-05, + "loss": 1.1682, + "theoretical_loss": 3.3474301994842652, + "tokens_seen": 3021209600 + }, + { + "epoch": 0.92, + "learning_rate": 6.847743229689067e-05, + "loss": 1.1776, + "theoretical_loss": 3.347424672030228, + "tokens_seen": 3021275136 + }, + { + "epoch": 0.92, + "learning_rate": 6.846138415245737e-05, + "loss": 1.2047, + "theoretical_loss": 3.3474191447296597, + "tokens_seen": 3021340672 + }, + { + "epoch": 0.92, + "learning_rate": 6.844533600802407e-05, + "loss": 1.1166, + "theoretical_loss": 3.3474136175825517, + "tokens_seen": 3021406208 + }, + { + "epoch": 0.92, + "learning_rate": 6.842928786359077e-05, + "loss": 1.2151, + "theoretical_loss": 3.347408090588897, + "tokens_seen": 3021471744 + }, + { + "epoch": 0.92, + "learning_rate": 6.841323971915747e-05, + "loss": 1.1734, + "theoretical_loss": 3.3474025637486875, + "tokens_seen": 3021537280 + }, + { + "epoch": 0.92, + "learning_rate": 6.839719157472417e-05, + "loss": 1.2389, + "theoretical_loss": 3.347397037061916, + "tokens_seen": 3021602816 + }, + { + "epoch": 0.92, + "learning_rate": 6.838114343029087e-05, + "loss": 1.2037, + "theoretical_loss": 3.3473915105285754, + "tokens_seen": 3021668352 + }, + { + "epoch": 0.92, + "learning_rate": 6.836509528585757e-05, + "loss": 1.2164, + "theoretical_loss": 3.3473859841486573, + "tokens_seen": 3021733888 + }, + { + "epoch": 0.92, + "learning_rate": 6.834904714142427e-05, + "loss": 1.2048, + "theoretical_loss": 3.3473804579221547, + "tokens_seen": 3021799424 + }, + { + "epoch": 0.92, + "learning_rate": 6.833299899699097e-05, + "loss": 1.3537, + "theoretical_loss": 3.3473749318490595, + "tokens_seen": 3021864960 + }, + { + "epoch": 0.92, + "learning_rate": 6.831695085255768e-05, + "loss": 1.1961, + "theoretical_loss": 3.347369405929364, + "tokens_seen": 3021930496 + }, + { + "epoch": 0.92, + "learning_rate": 6.830090270812438e-05, + "loss": 1.1668, + "theoretical_loss": 3.347363880163061, + "tokens_seen": 3021996032 + }, + { + "epoch": 0.92, + "learning_rate": 6.828485456369108e-05, + "loss": 1.1873, + "theoretical_loss": 3.3473583545501433, + "tokens_seen": 3022061568 + }, + { + "epoch": 0.92, + "learning_rate": 6.826880641925778e-05, + "loss": 1.2328, + "theoretical_loss": 3.3473528290906027, + "tokens_seen": 3022127104 + }, + { + "epoch": 0.92, + "learning_rate": 6.825275827482448e-05, + "loss": 1.1655, + "theoretical_loss": 3.3473473037844315, + "tokens_seen": 3022192640 + }, + { + "epoch": 0.92, + "learning_rate": 6.823671013039118e-05, + "loss": 1.1737, + "theoretical_loss": 3.347341778631623, + "tokens_seen": 3022258176 + }, + { + "epoch": 0.92, + "learning_rate": 6.822066198595788e-05, + "loss": 1.2481, + "theoretical_loss": 3.3473362536321685, + "tokens_seen": 3022323712 + }, + { + "epoch": 0.92, + "learning_rate": 6.820461384152458e-05, + "loss": 1.1445, + "theoretical_loss": 3.347330728786061, + "tokens_seen": 3022389248 + }, + { + "epoch": 0.92, + "learning_rate": 6.818856569709128e-05, + "loss": 1.1043, + "theoretical_loss": 3.347325204093293, + "tokens_seen": 3022454784 + }, + { + "epoch": 0.92, + "learning_rate": 6.817251755265798e-05, + "loss": 1.1707, + "theoretical_loss": 3.347319679553857, + "tokens_seen": 3022520320 + }, + { + "epoch": 0.92, + "learning_rate": 6.815646940822468e-05, + "loss": 1.2444, + "theoretical_loss": 3.3473141551677448, + "tokens_seen": 3022585856 + }, + { + "epoch": 0.92, + "learning_rate": 6.814042126379138e-05, + "loss": 1.2493, + "theoretical_loss": 3.3473086309349496, + "tokens_seen": 3022651392 + }, + { + "epoch": 0.92, + "learning_rate": 6.812437311935807e-05, + "loss": 1.2806, + "theoretical_loss": 3.3473031068554633, + "tokens_seen": 3022716928 + }, + { + "epoch": 0.92, + "learning_rate": 6.810832497492477e-05, + "loss": 1.1616, + "theoretical_loss": 3.3472975829292784, + "tokens_seen": 3022782464 + }, + { + "epoch": 0.92, + "learning_rate": 6.809227683049147e-05, + "loss": 1.1506, + "theoretical_loss": 3.3472920591563873, + "tokens_seen": 3022848000 + }, + { + "epoch": 0.92, + "learning_rate": 6.807622868605817e-05, + "loss": 1.1807, + "theoretical_loss": 3.3472865355367825, + "tokens_seen": 3022913536 + }, + { + "epoch": 0.92, + "learning_rate": 6.806018054162487e-05, + "loss": 1.25, + "theoretical_loss": 3.3472810120704564, + "tokens_seen": 3022979072 + }, + { + "epoch": 0.92, + "learning_rate": 6.804413239719157e-05, + "loss": 1.2257, + "theoretical_loss": 3.347275488757402, + "tokens_seen": 3023044608 + }, + { + "epoch": 0.92, + "learning_rate": 6.802808425275827e-05, + "loss": 1.1512, + "theoretical_loss": 3.3472699655976106, + "tokens_seen": 3023110144 + }, + { + "epoch": 0.92, + "learning_rate": 6.801203610832499e-05, + "loss": 1.0966, + "theoretical_loss": 3.3472644425910754, + "tokens_seen": 3023175680 + }, + { + "epoch": 0.92, + "learning_rate": 6.799598796389169e-05, + "loss": 1.1967, + "theoretical_loss": 3.3472589197377887, + "tokens_seen": 3023241216 + }, + { + "epoch": 0.92, + "learning_rate": 6.797993981945839e-05, + "loss": 1.1962, + "theoretical_loss": 3.3472533970377425, + "tokens_seen": 3023306752 + }, + { + "epoch": 0.92, + "learning_rate": 6.796389167502508e-05, + "loss": 1.1499, + "theoretical_loss": 3.34724787449093, + "tokens_seen": 3023372288 + }, + { + "epoch": 0.92, + "learning_rate": 6.794784353059178e-05, + "loss": 1.2169, + "theoretical_loss": 3.3472423520973433, + "tokens_seen": 3023437824 + }, + { + "epoch": 0.92, + "learning_rate": 6.793179538615848e-05, + "loss": 1.2323, + "theoretical_loss": 3.3472368298569743, + "tokens_seen": 3023503360 + }, + { + "epoch": 0.92, + "learning_rate": 6.791574724172518e-05, + "loss": 1.2527, + "theoretical_loss": 3.347231307769816, + "tokens_seen": 3023568896 + }, + { + "epoch": 0.92, + "learning_rate": 6.789969909729188e-05, + "loss": 1.1997, + "theoretical_loss": 3.347225785835861, + "tokens_seen": 3023634432 + }, + { + "epoch": 0.92, + "learning_rate": 6.788365095285858e-05, + "loss": 1.1742, + "theoretical_loss": 3.3472202640551014, + "tokens_seen": 3023699968 + }, + { + "epoch": 0.92, + "learning_rate": 6.786760280842528e-05, + "loss": 1.1676, + "theoretical_loss": 3.3472147424275294, + "tokens_seen": 3023765504 + }, + { + "epoch": 0.92, + "learning_rate": 6.785155466399198e-05, + "loss": 1.1583, + "theoretical_loss": 3.347209220953138, + "tokens_seen": 3023831040 + }, + { + "epoch": 0.92, + "learning_rate": 6.783550651955868e-05, + "loss": 1.2167, + "theoretical_loss": 3.347203699631919, + "tokens_seen": 3023896576 + }, + { + "epoch": 0.92, + "learning_rate": 6.781945837512538e-05, + "loss": 1.2168, + "theoretical_loss": 3.347198178463865, + "tokens_seen": 3023962112 + }, + { + "epoch": 0.92, + "learning_rate": 6.780341023069208e-05, + "loss": 1.1894, + "theoretical_loss": 3.347192657448969, + "tokens_seen": 3024027648 + }, + { + "epoch": 0.92, + "learning_rate": 6.778736208625878e-05, + "loss": 1.1349, + "theoretical_loss": 3.347187136587223, + "tokens_seen": 3024093184 + }, + { + "epoch": 0.92, + "learning_rate": 6.777131394182547e-05, + "loss": 1.0712, + "theoretical_loss": 3.347181615878619, + "tokens_seen": 3024158720 + }, + { + "epoch": 0.92, + "learning_rate": 6.775526579739217e-05, + "loss": 1.1394, + "theoretical_loss": 3.3471760953231504, + "tokens_seen": 3024224256 + }, + { + "epoch": 0.92, + "learning_rate": 6.773921765295887e-05, + "loss": 1.1481, + "theoretical_loss": 3.347170574920809, + "tokens_seen": 3024289792 + }, + { + "epoch": 0.92, + "learning_rate": 6.772316950852557e-05, + "loss": 1.1362, + "theoretical_loss": 3.347165054671587, + "tokens_seen": 3024355328 + }, + { + "epoch": 0.92, + "learning_rate": 6.770712136409229e-05, + "loss": 1.2712, + "theoretical_loss": 3.3471595345754777, + "tokens_seen": 3024420864 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1008771, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1029722690582275, + "objective/train/theoretical_loss": 3.3471540146324728, + "objective/train/tokens_used": 3044946400, + "theoretical_loss": 3.3471540146324728, + "tokens_seen": 3024486400 + }, + { + "epoch": 0.92, + "learning_rate": 6.769107321965899e-05, + "loss": 1.1519, + "theoretical_loss": 3.3471540146324728, + "tokens_seen": 3024486400 + }, + { + "epoch": 0.92, + "learning_rate": 6.767502507522569e-05, + "loss": 1.2141, + "theoretical_loss": 3.3471484948425645, + "tokens_seen": 3024551936 + }, + { + "epoch": 0.92, + "learning_rate": 6.765897693079239e-05, + "loss": 1.1473, + "theoretical_loss": 3.347142975205746, + "tokens_seen": 3024617472 + }, + { + "epoch": 0.92, + "learning_rate": 6.764292878635909e-05, + "loss": 1.1704, + "theoretical_loss": 3.3471374557220095, + "tokens_seen": 3024683008 + }, + { + "epoch": 0.92, + "learning_rate": 6.762688064192579e-05, + "loss": 1.09, + "theoretical_loss": 3.3471319363913477, + "tokens_seen": 3024748544 + }, + { + "epoch": 0.92, + "learning_rate": 6.761083249749248e-05, + "loss": 1.2446, + "theoretical_loss": 3.3471264172137523, + "tokens_seen": 3024814080 + }, + { + "epoch": 0.92, + "learning_rate": 6.759478435305918e-05, + "loss": 1.1658, + "theoretical_loss": 3.347120898189216, + "tokens_seen": 3024879616 + }, + { + "epoch": 0.92, + "learning_rate": 6.757873620862588e-05, + "loss": 1.1768, + "theoretical_loss": 3.347115379317732, + "tokens_seen": 3024945152 + }, + { + "epoch": 0.92, + "learning_rate": 6.756268806419258e-05, + "loss": 1.2429, + "theoretical_loss": 3.3471098605992915, + "tokens_seen": 3025010688 + }, + { + "epoch": 0.92, + "learning_rate": 6.754663991975928e-05, + "loss": 1.3234, + "theoretical_loss": 3.3471043420338877, + "tokens_seen": 3025076224 + }, + { + "epoch": 0.92, + "learning_rate": 6.753059177532598e-05, + "loss": 1.1758, + "theoretical_loss": 3.347098823621513, + "tokens_seen": 3025141760 + }, + { + "epoch": 0.92, + "learning_rate": 6.751454363089268e-05, + "loss": 1.1762, + "theoretical_loss": 3.3470933053621597, + "tokens_seen": 3025207296 + }, + { + "epoch": 0.92, + "learning_rate": 6.749849548645938e-05, + "loss": 1.2139, + "theoretical_loss": 3.3470877872558207, + "tokens_seen": 3025272832 + }, + { + "epoch": 0.92, + "learning_rate": 6.748244734202608e-05, + "loss": 1.1523, + "theoretical_loss": 3.3470822693024873, + "tokens_seen": 3025338368 + }, + { + "epoch": 0.92, + "learning_rate": 6.746639919759278e-05, + "loss": 1.2048, + "theoretical_loss": 3.3470767515021533, + "tokens_seen": 3025403904 + }, + { + "epoch": 0.92, + "learning_rate": 6.745035105315948e-05, + "loss": 1.1763, + "theoretical_loss": 3.3470712338548103, + "tokens_seen": 3025469440 + }, + { + "epoch": 0.92, + "learning_rate": 6.743430290872618e-05, + "loss": 1.2678, + "theoretical_loss": 3.3470657163604507, + "tokens_seen": 3025534976 + }, + { + "epoch": 0.92, + "learning_rate": 6.741825476429287e-05, + "loss": 1.1271, + "theoretical_loss": 3.3470601990190674, + "tokens_seen": 3025600512 + }, + { + "epoch": 0.92, + "learning_rate": 6.740220661985958e-05, + "loss": 1.2848, + "theoretical_loss": 3.347054681830653, + "tokens_seen": 3025666048 + }, + { + "epoch": 0.92, + "learning_rate": 6.738615847542629e-05, + "loss": 1.1128, + "theoretical_loss": 3.347049164795199, + "tokens_seen": 3025731584 + }, + { + "epoch": 0.92, + "learning_rate": 6.737011033099299e-05, + "loss": 1.1374, + "theoretical_loss": 3.347043647912699, + "tokens_seen": 3025797120 + }, + { + "epoch": 0.92, + "learning_rate": 6.735406218655969e-05, + "loss": 1.0167, + "theoretical_loss": 3.3470381311831443, + "tokens_seen": 3025862656 + }, + { + "epoch": 0.92, + "learning_rate": 6.733801404212639e-05, + "loss": 1.1568, + "theoretical_loss": 3.347032614606528, + "tokens_seen": 3025928192 + }, + { + "epoch": 0.92, + "learning_rate": 6.732196589769309e-05, + "loss": 1.1587, + "theoretical_loss": 3.347027098182843, + "tokens_seen": 3025993728 + }, + { + "epoch": 0.92, + "learning_rate": 6.730591775325979e-05, + "loss": 1.2384, + "theoretical_loss": 3.347021581912081, + "tokens_seen": 3026059264 + }, + { + "epoch": 0.92, + "learning_rate": 6.728986960882649e-05, + "loss": 1.1198, + "theoretical_loss": 3.3470160657942345, + "tokens_seen": 3026124800 + }, + { + "epoch": 0.92, + "learning_rate": 6.727382146439319e-05, + "loss": 1.145, + "theoretical_loss": 3.3470105498292964, + "tokens_seen": 3026190336 + }, + { + "epoch": 0.92, + "learning_rate": 6.725777331995988e-05, + "loss": 1.1222, + "theoretical_loss": 3.3470050340172586, + "tokens_seen": 3026255872 + }, + { + "epoch": 0.92, + "learning_rate": 6.724172517552658e-05, + "loss": 1.1526, + "theoretical_loss": 3.346999518358114, + "tokens_seen": 3026321408 + }, + { + "epoch": 0.92, + "learning_rate": 6.722567703109328e-05, + "loss": 1.2102, + "theoretical_loss": 3.3469940028518548, + "tokens_seen": 3026386944 + }, + { + "epoch": 0.92, + "learning_rate": 6.720962888665998e-05, + "loss": 1.2379, + "theoretical_loss": 3.3469884874984737, + "tokens_seen": 3026452480 + }, + { + "epoch": 0.92, + "learning_rate": 6.719358074222668e-05, + "loss": 1.2397, + "theoretical_loss": 3.3469829722979627, + "tokens_seen": 3026518016 + }, + { + "epoch": 0.92, + "learning_rate": 6.717753259779338e-05, + "loss": 1.1398, + "theoretical_loss": 3.346977457250315, + "tokens_seen": 3026583552 + }, + { + "epoch": 0.92, + "learning_rate": 6.716148445336008e-05, + "loss": 1.1412, + "theoretical_loss": 3.346971942355522, + "tokens_seen": 3026649088 + }, + { + "epoch": 0.92, + "learning_rate": 6.714543630892678e-05, + "loss": 1.0969, + "theoretical_loss": 3.346966427613577, + "tokens_seen": 3026714624 + }, + { + "epoch": 0.92, + "learning_rate": 6.712938816449348e-05, + "loss": 1.2049, + "theoretical_loss": 3.346960913024472, + "tokens_seen": 3026780160 + }, + { + "epoch": 0.92, + "learning_rate": 6.711334002006018e-05, + "loss": 1.2101, + "theoretical_loss": 3.3469553985882, + "tokens_seen": 3026845696 + }, + { + "epoch": 0.92, + "learning_rate": 6.709729187562688e-05, + "loss": 1.1717, + "theoretical_loss": 3.3469498843047534, + "tokens_seen": 3026911232 + }, + { + "epoch": 0.92, + "learning_rate": 6.708124373119358e-05, + "loss": 1.1981, + "theoretical_loss": 3.346944370174124, + "tokens_seen": 3026976768 + }, + { + "epoch": 0.92, + "learning_rate": 6.706519558676028e-05, + "loss": 1.1934, + "theoretical_loss": 3.3469388561963047, + "tokens_seen": 3027042304 + }, + { + "epoch": 0.92, + "learning_rate": 6.704914744232698e-05, + "loss": 1.1492, + "theoretical_loss": 3.3469333423712877, + "tokens_seen": 3027107840 + }, + { + "epoch": 0.92, + "learning_rate": 6.703309929789369e-05, + "loss": 1.1628, + "theoretical_loss": 3.346927828699066, + "tokens_seen": 3027173376 + }, + { + "epoch": 0.92, + "learning_rate": 6.701705115346039e-05, + "loss": 1.2212, + "theoretical_loss": 3.346922315179631, + "tokens_seen": 3027238912 + }, + { + "epoch": 0.92, + "learning_rate": 6.700100300902709e-05, + "loss": 1.1987, + "theoretical_loss": 3.3469168018129762, + "tokens_seen": 3027304448 + }, + { + "epoch": 0.92, + "learning_rate": 6.698495486459379e-05, + "loss": 1.272, + "theoretical_loss": 3.346911288599094, + "tokens_seen": 3027369984 + }, + { + "epoch": 0.92, + "learning_rate": 6.696890672016049e-05, + "loss": 1.1634, + "theoretical_loss": 3.3469057755379765, + "tokens_seen": 3027435520 + }, + { + "epoch": 0.92, + "learning_rate": 6.695285857572719e-05, + "loss": 1.188, + "theoretical_loss": 3.346900262629616, + "tokens_seen": 3027501056 + }, + { + "epoch": 0.92, + "learning_rate": 6.693681043129389e-05, + "loss": 1.2648, + "theoretical_loss": 3.3468947498740054, + "tokens_seen": 3027566592 + }, + { + "epoch": 0.92, + "learning_rate": 6.692076228686058e-05, + "loss": 1.2925, + "theoretical_loss": 3.346889237271137, + "tokens_seen": 3027632128 + }, + { + "epoch": 0.92, + "learning_rate": 6.690471414242728e-05, + "loss": 1.1598, + "theoretical_loss": 3.346883724821003, + "tokens_seen": 3027697664 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1009725, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2452223300933838, + "objective/train/theoretical_loss": 3.3468782125235963, + "objective/train/tokens_used": 3048223200, + "theoretical_loss": 3.3468782125235963, + "tokens_seen": 3027763200 + }, + { + "epoch": 0.92, + "learning_rate": 6.688866599799398e-05, + "loss": 1.1616, + "theoretical_loss": 3.3468782125235963, + "tokens_seen": 3027763200 + }, + { + "epoch": 0.92, + "learning_rate": 6.687261785356068e-05, + "loss": 1.1997, + "theoretical_loss": 3.346872700378909, + "tokens_seen": 3027828736 + }, + { + "epoch": 0.92, + "learning_rate": 6.685656970912738e-05, + "loss": 1.1751, + "theoretical_loss": 3.346867188386933, + "tokens_seen": 3027894272 + }, + { + "epoch": 0.92, + "learning_rate": 6.684052156469408e-05, + "loss": 1.248, + "theoretical_loss": 3.346861676547663, + "tokens_seen": 3027959808 + }, + { + "epoch": 0.92, + "learning_rate": 6.682447342026078e-05, + "loss": 1.1689, + "theoretical_loss": 3.346856164861089, + "tokens_seen": 3028025344 + }, + { + "epoch": 0.92, + "learning_rate": 6.68084252758275e-05, + "loss": 1.1681, + "theoretical_loss": 3.3468506533272047, + "tokens_seen": 3028090880 + }, + { + "epoch": 0.92, + "learning_rate": 6.67923771313942e-05, + "loss": 1.2825, + "theoretical_loss": 3.3468451419460017, + "tokens_seen": 3028156416 + }, + { + "epoch": 0.92, + "learning_rate": 6.67763289869609e-05, + "loss": 1.1354, + "theoretical_loss": 3.3468396307174735, + "tokens_seen": 3028221952 + }, + { + "epoch": 0.92, + "learning_rate": 6.676028084252758e-05, + "loss": 1.1538, + "theoretical_loss": 3.346834119641612, + "tokens_seen": 3028287488 + }, + { + "epoch": 0.92, + "learning_rate": 6.674423269809428e-05, + "loss": 1.2185, + "theoretical_loss": 3.34682860871841, + "tokens_seen": 3028353024 + }, + { + "epoch": 0.92, + "learning_rate": 6.672818455366098e-05, + "loss": 1.199, + "theoretical_loss": 3.3468230979478593, + "tokens_seen": 3028418560 + }, + { + "epoch": 0.92, + "learning_rate": 6.671213640922768e-05, + "loss": 1.141, + "theoretical_loss": 3.3468175873299533, + "tokens_seen": 3028484096 + }, + { + "epoch": 0.92, + "learning_rate": 6.669608826479439e-05, + "loss": 1.1364, + "theoretical_loss": 3.3468120768646834, + "tokens_seen": 3028549632 + }, + { + "epoch": 0.92, + "learning_rate": 6.668004012036109e-05, + "loss": 1.276, + "theoretical_loss": 3.346806566552043, + "tokens_seen": 3028615168 + }, + { + "epoch": 0.92, + "learning_rate": 6.666399197592779e-05, + "loss": 1.2341, + "theoretical_loss": 3.346801056392024, + "tokens_seen": 3028680704 + }, + { + "epoch": 0.92, + "learning_rate": 6.664794383149449e-05, + "loss": 1.1909, + "theoretical_loss": 3.3467955463846195, + "tokens_seen": 3028746240 + }, + { + "epoch": 0.92, + "learning_rate": 6.663189568706119e-05, + "loss": 1.201, + "theoretical_loss": 3.3467900365298213, + "tokens_seen": 3028811776 + }, + { + "epoch": 0.92, + "learning_rate": 6.661584754262789e-05, + "loss": 1.2226, + "theoretical_loss": 3.346784526827622, + "tokens_seen": 3028877312 + }, + { + "epoch": 0.92, + "learning_rate": 6.659979939819459e-05, + "loss": 1.1539, + "theoretical_loss": 3.3467790172780143, + "tokens_seen": 3028942848 + }, + { + "epoch": 0.92, + "learning_rate": 6.658375125376129e-05, + "loss": 1.255, + "theoretical_loss": 3.346773507880991, + "tokens_seen": 3029008384 + }, + { + "epoch": 0.92, + "learning_rate": 6.656770310932798e-05, + "loss": 1.1655, + "theoretical_loss": 3.3467679986365435, + "tokens_seen": 3029073920 + }, + { + "epoch": 0.92, + "learning_rate": 6.655165496489468e-05, + "loss": 1.2346, + "theoretical_loss": 3.3467624895446653, + "tokens_seen": 3029139456 + }, + { + "epoch": 0.92, + "learning_rate": 6.653560682046138e-05, + "loss": 1.2057, + "theoretical_loss": 3.346756980605348, + "tokens_seen": 3029204992 + }, + { + "epoch": 0.92, + "learning_rate": 6.651955867602808e-05, + "loss": 1.2735, + "theoretical_loss": 3.346751471818585, + "tokens_seen": 3029270528 + }, + { + "epoch": 0.92, + "learning_rate": 6.650351053159479e-05, + "loss": 1.2017, + "theoretical_loss": 3.346745963184368, + "tokens_seen": 3029336064 + }, + { + "epoch": 0.92, + "learning_rate": 6.64874623871615e-05, + "loss": 1.182, + "theoretical_loss": 3.34674045470269, + "tokens_seen": 3029401600 + }, + { + "epoch": 0.92, + "learning_rate": 6.64714142427282e-05, + "loss": 1.1601, + "theoretical_loss": 3.3467349463735436, + "tokens_seen": 3029467136 + }, + { + "epoch": 0.92, + "learning_rate": 6.64553660982949e-05, + "loss": 1.1723, + "theoretical_loss": 3.346729438196921, + "tokens_seen": 3029532672 + }, + { + "epoch": 0.92, + "learning_rate": 6.64393179538616e-05, + "loss": 1.2208, + "theoretical_loss": 3.346723930172814, + "tokens_seen": 3029598208 + }, + { + "epoch": 0.92, + "learning_rate": 6.64232698094283e-05, + "loss": 1.179, + "theoretical_loss": 3.3467184223012163, + "tokens_seen": 3029663744 + }, + { + "epoch": 0.92, + "learning_rate": 6.640722166499498e-05, + "loss": 1.1619, + "theoretical_loss": 3.3467129145821195, + "tokens_seen": 3029729280 + }, + { + "epoch": 0.92, + "learning_rate": 6.639117352056168e-05, + "loss": 1.3075, + "theoretical_loss": 3.3467074070155163, + "tokens_seen": 3029794816 + }, + { + "epoch": 0.92, + "learning_rate": 6.637512537612838e-05, + "loss": 1.1905, + "theoretical_loss": 3.3467018996013995, + "tokens_seen": 3029860352 + }, + { + "epoch": 0.92, + "learning_rate": 6.635907723169509e-05, + "loss": 1.2081, + "theoretical_loss": 3.3466963923397612, + "tokens_seen": 3029925888 + }, + { + "epoch": 0.92, + "learning_rate": 6.634302908726179e-05, + "loss": 1.2948, + "theoretical_loss": 3.3466908852305943, + "tokens_seen": 3029991424 + }, + { + "epoch": 0.92, + "learning_rate": 6.632698094282849e-05, + "loss": 1.2008, + "theoretical_loss": 3.3466853782738903, + "tokens_seen": 3030056960 + }, + { + "epoch": 0.92, + "learning_rate": 6.631093279839519e-05, + "loss": 1.2614, + "theoretical_loss": 3.346679871469643, + "tokens_seen": 3030122496 + }, + { + "epoch": 0.92, + "learning_rate": 6.629488465396189e-05, + "loss": 1.1529, + "theoretical_loss": 3.3466743648178445, + "tokens_seen": 3030188032 + }, + { + "epoch": 0.92, + "learning_rate": 6.627883650952859e-05, + "loss": 1.1963, + "theoretical_loss": 3.3466688583184863, + "tokens_seen": 3030253568 + }, + { + "epoch": 0.92, + "learning_rate": 6.626278836509529e-05, + "loss": 1.2199, + "theoretical_loss": 3.346663351971562, + "tokens_seen": 3030319104 + }, + { + "epoch": 0.92, + "learning_rate": 6.624674022066199e-05, + "loss": 1.1787, + "theoretical_loss": 3.346657845777064, + "tokens_seen": 3030384640 + }, + { + "epoch": 0.92, + "learning_rate": 6.623069207622869e-05, + "loss": 1.1808, + "theoretical_loss": 3.346652339734984, + "tokens_seen": 3030450176 + }, + { + "epoch": 0.92, + "learning_rate": 6.621464393179539e-05, + "loss": 1.223, + "theoretical_loss": 3.3466468338453152, + "tokens_seen": 3030515712 + }, + { + "epoch": 0.92, + "learning_rate": 6.619859578736209e-05, + "loss": 1.1786, + "theoretical_loss": 3.34664132810805, + "tokens_seen": 3030581248 + }, + { + "epoch": 0.92, + "learning_rate": 6.618254764292879e-05, + "loss": 1.2556, + "theoretical_loss": 3.3466358225231803, + "tokens_seen": 3030646784 + }, + { + "epoch": 0.92, + "learning_rate": 6.616649949849549e-05, + "loss": 1.2024, + "theoretical_loss": 3.3466303170906997, + "tokens_seen": 3030712320 + }, + { + "epoch": 0.92, + "learning_rate": 6.61504513540622e-05, + "loss": 1.2161, + "theoretical_loss": 3.3466248118105995, + "tokens_seen": 3030777856 + }, + { + "epoch": 0.92, + "learning_rate": 6.61344032096289e-05, + "loss": 1.2659, + "theoretical_loss": 3.346619306682873, + "tokens_seen": 3030843392 + }, + { + "epoch": 0.92, + "learning_rate": 6.61183550651956e-05, + "loss": 1.2477, + "theoretical_loss": 3.3466138017075124, + "tokens_seen": 3030908928 + }, + { + "epoch": 0.92, + "learning_rate": 6.61023069207623e-05, + "loss": 1.2351, + "theoretical_loss": 3.34660829688451, + "tokens_seen": 3030974464 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1010980, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.139613151550293, + "objective/train/theoretical_loss": 3.346602792213859, + "objective/train/tokens_used": 3051500000, + "theoretical_loss": 3.346602792213859, + "tokens_seen": 3031040000 + }, + { + "epoch": 0.92, + "learning_rate": 6.6086258776329e-05, + "loss": 1.1129, + "theoretical_loss": 3.346602792213859, + "tokens_seen": 3031040000 + }, + { + "epoch": 0.92, + "learning_rate": 6.60702106318957e-05, + "loss": 1.2239, + "theoretical_loss": 3.3465972876955505, + "tokens_seen": 3031105536 + }, + { + "epoch": 0.92, + "learning_rate": 6.605416248746238e-05, + "loss": 1.1639, + "theoretical_loss": 3.3465917833295786, + "tokens_seen": 3031171072 + }, + { + "epoch": 0.92, + "learning_rate": 6.603811434302908e-05, + "loss": 1.162, + "theoretical_loss": 3.3465862791159346, + "tokens_seen": 3031236608 + }, + { + "epoch": 0.92, + "learning_rate": 6.602206619859578e-05, + "loss": 1.2629, + "theoretical_loss": 3.3465807750546115, + "tokens_seen": 3031302144 + }, + { + "epoch": 0.92, + "learning_rate": 6.600601805416249e-05, + "loss": 1.2131, + "theoretical_loss": 3.346575271145602, + "tokens_seen": 3031367680 + }, + { + "epoch": 0.92, + "learning_rate": 6.598996990972919e-05, + "loss": 1.2959, + "theoretical_loss": 3.346569767388898, + "tokens_seen": 3031433216 + }, + { + "epoch": 0.92, + "learning_rate": 6.597392176529589e-05, + "loss": 1.2398, + "theoretical_loss": 3.346564263784493, + "tokens_seen": 3031498752 + }, + { + "epoch": 0.92, + "learning_rate": 6.595787362086259e-05, + "loss": 1.1398, + "theoretical_loss": 3.3465587603323783, + "tokens_seen": 3031564288 + }, + { + "epoch": 0.92, + "learning_rate": 6.594182547642929e-05, + "loss": 1.1589, + "theoretical_loss": 3.3465532570325465, + "tokens_seen": 3031629824 + }, + { + "epoch": 0.92, + "learning_rate": 6.592577733199599e-05, + "loss": 1.2782, + "theoretical_loss": 3.346547753884991, + "tokens_seen": 3031695360 + }, + { + "epoch": 0.92, + "learning_rate": 6.59097291875627e-05, + "loss": 1.1409, + "theoretical_loss": 3.346542250889704, + "tokens_seen": 3031760896 + }, + { + "epoch": 0.92, + "learning_rate": 6.589368104312939e-05, + "loss": 1.1595, + "theoretical_loss": 3.3465367480466774, + "tokens_seen": 3031826432 + }, + { + "epoch": 0.92, + "learning_rate": 6.587763289869609e-05, + "loss": 1.2396, + "theoretical_loss": 3.346531245355904, + "tokens_seen": 3031891968 + }, + { + "epoch": 0.92, + "learning_rate": 6.586158475426279e-05, + "loss": 1.225, + "theoretical_loss": 3.3465257428173767, + "tokens_seen": 3031957504 + }, + { + "epoch": 0.92, + "learning_rate": 6.584553660982949e-05, + "loss": 1.2039, + "theoretical_loss": 3.3465202404310874, + "tokens_seen": 3032023040 + }, + { + "epoch": 0.92, + "learning_rate": 6.582948846539619e-05, + "loss": 1.1354, + "theoretical_loss": 3.3465147381970293, + "tokens_seen": 3032088576 + }, + { + "epoch": 0.92, + "learning_rate": 6.581344032096289e-05, + "loss": 1.1794, + "theoretical_loss": 3.3465092361151942, + "tokens_seen": 3032154112 + }, + { + "epoch": 0.92, + "learning_rate": 6.57973921765296e-05, + "loss": 1.1872, + "theoretical_loss": 3.346503734185575, + "tokens_seen": 3032219648 + }, + { + "epoch": 0.92, + "learning_rate": 6.57813440320963e-05, + "loss": 1.2143, + "theoretical_loss": 3.346498232408164, + "tokens_seen": 3032285184 + }, + { + "epoch": 0.92, + "learning_rate": 6.5765295887663e-05, + "loss": 1.2637, + "theoretical_loss": 3.3464927307829537, + "tokens_seen": 3032350720 + }, + { + "epoch": 0.92, + "learning_rate": 6.57492477432297e-05, + "loss": 1.2289, + "theoretical_loss": 3.3464872293099366, + "tokens_seen": 3032416256 + }, + { + "epoch": 0.92, + "learning_rate": 6.57331995987964e-05, + "loss": 1.1636, + "theoretical_loss": 3.3464817279891053, + "tokens_seen": 3032481792 + }, + { + "epoch": 0.92, + "learning_rate": 6.57171514543631e-05, + "loss": 1.2414, + "theoretical_loss": 3.3464762268204526, + "tokens_seen": 3032547328 + }, + { + "epoch": 0.92, + "learning_rate": 6.570110330992978e-05, + "loss": 1.1089, + "theoretical_loss": 3.3464707258039703, + "tokens_seen": 3032612864 + }, + { + "epoch": 0.92, + "learning_rate": 6.568505516549648e-05, + "loss": 1.1908, + "theoretical_loss": 3.3464652249396516, + "tokens_seen": 3032678400 + }, + { + "epoch": 0.92, + "learning_rate": 6.566900702106319e-05, + "loss": 1.1343, + "theoretical_loss": 3.3464597242274885, + "tokens_seen": 3032743936 + }, + { + "epoch": 0.92, + "learning_rate": 6.565295887662989e-05, + "loss": 1.2311, + "theoretical_loss": 3.3464542236674735, + "tokens_seen": 3032809472 + }, + { + "epoch": 0.92, + "learning_rate": 6.563691073219659e-05, + "loss": 1.2298, + "theoretical_loss": 3.3464487232596, + "tokens_seen": 3032875008 + }, + { + "epoch": 0.92, + "learning_rate": 6.562086258776329e-05, + "loss": 1.2512, + "theoretical_loss": 3.346443223003859, + "tokens_seen": 3032940544 + }, + { + "epoch": 0.92, + "learning_rate": 6.560481444333e-05, + "loss": 1.0272, + "theoretical_loss": 3.3464377229002444, + "tokens_seen": 3033006080 + }, + { + "epoch": 0.92, + "learning_rate": 6.55887662988967e-05, + "loss": 1.2083, + "theoretical_loss": 3.3464322229487475, + "tokens_seen": 3033071616 + }, + { + "epoch": 0.92, + "learning_rate": 6.55727181544634e-05, + "loss": 1.2449, + "theoretical_loss": 3.346426723149362, + "tokens_seen": 3033137152 + }, + { + "epoch": 0.92, + "learning_rate": 6.55566700100301e-05, + "loss": 1.2991, + "theoretical_loss": 3.3464212235020794, + "tokens_seen": 3033202688 + }, + { + "epoch": 0.92, + "learning_rate": 6.554062186559679e-05, + "loss": 1.2688, + "theoretical_loss": 3.3464157240068926, + "tokens_seen": 3033268224 + }, + { + "epoch": 0.92, + "learning_rate": 6.552457372116349e-05, + "loss": 1.1704, + "theoretical_loss": 3.3464102246637943, + "tokens_seen": 3033333760 + }, + { + "epoch": 0.92, + "learning_rate": 6.550852557673019e-05, + "loss": 1.2971, + "theoretical_loss": 3.346404725472777, + "tokens_seen": 3033399296 + }, + { + "epoch": 0.92, + "learning_rate": 6.549247743229689e-05, + "loss": 1.2146, + "theoretical_loss": 3.346399226433833, + "tokens_seen": 3033464832 + }, + { + "epoch": 0.92, + "learning_rate": 6.547642928786359e-05, + "loss": 1.2468, + "theoretical_loss": 3.3463937275469546, + "tokens_seen": 3033530368 + }, + { + "epoch": 0.92, + "learning_rate": 6.54603811434303e-05, + "loss": 1.2688, + "theoretical_loss": 3.3463882288121347, + "tokens_seen": 3033595904 + }, + { + "epoch": 0.92, + "learning_rate": 6.5444332998997e-05, + "loss": 1.2033, + "theoretical_loss": 3.346382730229366, + "tokens_seen": 3033661440 + }, + { + "epoch": 0.92, + "learning_rate": 6.54282848545637e-05, + "loss": 1.2832, + "theoretical_loss": 3.34637723179864, + "tokens_seen": 3033726976 + }, + { + "epoch": 0.92, + "learning_rate": 6.54122367101304e-05, + "loss": 1.2914, + "theoretical_loss": 3.346371733519951, + "tokens_seen": 3033792512 + }, + { + "epoch": 0.92, + "learning_rate": 6.53961885656971e-05, + "loss": 1.2227, + "theoretical_loss": 3.3463662353932895, + "tokens_seen": 3033858048 + }, + { + "epoch": 0.92, + "learning_rate": 6.53801404212638e-05, + "loss": 1.185, + "theoretical_loss": 3.346360737418649, + "tokens_seen": 3033923584 + }, + { + "epoch": 0.92, + "learning_rate": 6.53640922768305e-05, + "loss": 1.2049, + "theoretical_loss": 3.346355239596022, + "tokens_seen": 3033989120 + }, + { + "epoch": 0.92, + "learning_rate": 6.534804413239718e-05, + "loss": 1.2282, + "theoretical_loss": 3.346349741925401, + "tokens_seen": 3034054656 + }, + { + "epoch": 0.92, + "learning_rate": 6.533199598796389e-05, + "loss": 1.1753, + "theoretical_loss": 3.3463442444067786, + "tokens_seen": 3034120192 + }, + { + "epoch": 0.92, + "learning_rate": 6.531594784353059e-05, + "loss": 1.1575, + "theoretical_loss": 3.3463387470401473, + "tokens_seen": 3034185728 + }, + { + "epoch": 0.92, + "learning_rate": 6.52998996990973e-05, + "loss": 1.1609, + "theoretical_loss": 3.3463332498254994, + "tokens_seen": 3034251264 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1011897, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.284067153930664, + "objective/train/theoretical_loss": 3.3463277527628277, + "objective/train/tokens_used": 3054776800, + "theoretical_loss": 3.3463277527628277, + "tokens_seen": 3034316800 + }, + { + "epoch": 0.92, + "learning_rate": 6.5283851554664e-05, + "loss": 1.2571, + "theoretical_loss": 3.3463277527628277, + "tokens_seen": 3034316800 + }, + { + "epoch": 0.92, + "learning_rate": 6.52678034102307e-05, + "loss": 1.2697, + "theoretical_loss": 3.3463222558521237, + "tokens_seen": 3034382336 + }, + { + "epoch": 0.92, + "learning_rate": 6.52517552657974e-05, + "loss": 1.1438, + "theoretical_loss": 3.3463167590933818, + "tokens_seen": 3034447872 + }, + { + "epoch": 0.92, + "learning_rate": 6.52357071213641e-05, + "loss": 1.1791, + "theoretical_loss": 3.3463112624865925, + "tokens_seen": 3034513408 + }, + { + "epoch": 0.92, + "learning_rate": 6.52196589769308e-05, + "loss": 1.2153, + "theoretical_loss": 3.34630576603175, + "tokens_seen": 3034578944 + }, + { + "epoch": 0.92, + "learning_rate": 6.52036108324975e-05, + "loss": 1.1644, + "theoretical_loss": 3.3463002697288458, + "tokens_seen": 3034644480 + }, + { + "epoch": 0.92, + "learning_rate": 6.518756268806419e-05, + "loss": 1.2965, + "theoretical_loss": 3.3462947735778727, + "tokens_seen": 3034710016 + }, + { + "epoch": 0.92, + "learning_rate": 6.517151454363089e-05, + "loss": 1.0761, + "theoretical_loss": 3.3462892775788236, + "tokens_seen": 3034775552 + }, + { + "epoch": 0.92, + "learning_rate": 6.515546639919759e-05, + "loss": 1.1346, + "theoretical_loss": 3.34628378173169, + "tokens_seen": 3034841088 + }, + { + "epoch": 0.92, + "learning_rate": 6.513941825476429e-05, + "loss": 1.2676, + "theoretical_loss": 3.3462782860364655, + "tokens_seen": 3034906624 + }, + { + "epoch": 0.92, + "learning_rate": 6.5123370110331e-05, + "loss": 1.1751, + "theoretical_loss": 3.3462727904931424, + "tokens_seen": 3034972160 + }, + { + "epoch": 0.92, + "learning_rate": 6.51073219658977e-05, + "loss": 1.238, + "theoretical_loss": 3.3462672951017125, + "tokens_seen": 3035037696 + }, + { + "epoch": 0.92, + "learning_rate": 6.50912738214644e-05, + "loss": 1.3183, + "theoretical_loss": 3.3462617998621695, + "tokens_seen": 3035103232 + }, + { + "epoch": 0.92, + "learning_rate": 6.50752256770311e-05, + "loss": 1.2484, + "theoretical_loss": 3.3462563047745046, + "tokens_seen": 3035168768 + }, + { + "epoch": 0.92, + "learning_rate": 6.50591775325978e-05, + "loss": 1.126, + "theoretical_loss": 3.346250809838711, + "tokens_seen": 3035234304 + }, + { + "epoch": 0.92, + "learning_rate": 6.50431293881645e-05, + "loss": 1.1176, + "theoretical_loss": 3.346245315054782, + "tokens_seen": 3035299840 + }, + { + "epoch": 0.92, + "learning_rate": 6.50270812437312e-05, + "loss": 1.2134, + "theoretical_loss": 3.3462398204227086, + "tokens_seen": 3035365376 + }, + { + "epoch": 0.92, + "learning_rate": 6.50110330992979e-05, + "loss": 1.2577, + "theoretical_loss": 3.3462343259424845, + "tokens_seen": 3035430912 + }, + { + "epoch": 0.92, + "learning_rate": 6.49949849548646e-05, + "loss": 1.1095, + "theoretical_loss": 3.346228831614101, + "tokens_seen": 3035496448 + }, + { + "epoch": 0.92, + "learning_rate": 6.49789368104313e-05, + "loss": 1.2029, + "theoretical_loss": 3.3462233374375523, + "tokens_seen": 3035561984 + }, + { + "epoch": 0.92, + "learning_rate": 6.4962888665998e-05, + "loss": 1.0561, + "theoretical_loss": 3.3462178434128296, + "tokens_seen": 3035627520 + }, + { + "epoch": 0.92, + "learning_rate": 6.49468405215647e-05, + "loss": 1.1743, + "theoretical_loss": 3.3462123495399263, + "tokens_seen": 3035693056 + }, + { + "epoch": 0.92, + "learning_rate": 6.49307923771314e-05, + "loss": 1.1888, + "theoretical_loss": 3.346206855818834, + "tokens_seen": 3035758592 + }, + { + "epoch": 0.92, + "learning_rate": 6.49147442326981e-05, + "loss": 1.2128, + "theoretical_loss": 3.346201362249546, + "tokens_seen": 3035824128 + }, + { + "epoch": 0.92, + "learning_rate": 6.48986960882648e-05, + "loss": 1.2538, + "theoretical_loss": 3.3461958688320546, + "tokens_seen": 3035889664 + }, + { + "epoch": 0.92, + "learning_rate": 6.48826479438315e-05, + "loss": 1.2574, + "theoretical_loss": 3.3461903755663522, + "tokens_seen": 3035955200 + }, + { + "epoch": 0.92, + "learning_rate": 6.48665997993982e-05, + "loss": 1.2547, + "theoretical_loss": 3.346184882452431, + "tokens_seen": 3036020736 + }, + { + "epoch": 0.92, + "learning_rate": 6.485055165496489e-05, + "loss": 1.2361, + "theoretical_loss": 3.3461793894902843, + "tokens_seen": 3036086272 + }, + { + "epoch": 0.92, + "learning_rate": 6.483450351053159e-05, + "loss": 1.2465, + "theoretical_loss": 3.3461738966799044, + "tokens_seen": 3036151808 + }, + { + "epoch": 0.92, + "learning_rate": 6.481845536609829e-05, + "loss": 1.3295, + "theoretical_loss": 3.3461684040212836, + "tokens_seen": 3036217344 + }, + { + "epoch": 0.92, + "learning_rate": 6.480240722166499e-05, + "loss": 1.184, + "theoretical_loss": 3.3461629115144143, + "tokens_seen": 3036282880 + }, + { + "epoch": 0.92, + "learning_rate": 6.478635907723169e-05, + "loss": 1.2097, + "theoretical_loss": 3.3461574191592893, + "tokens_seen": 3036348416 + }, + { + "epoch": 0.92, + "learning_rate": 6.47703109327984e-05, + "loss": 1.187, + "theoretical_loss": 3.3461519269559012, + "tokens_seen": 3036413952 + }, + { + "epoch": 0.92, + "learning_rate": 6.47542627883651e-05, + "loss": 1.1497, + "theoretical_loss": 3.3461464349042425, + "tokens_seen": 3036479488 + }, + { + "epoch": 0.92, + "learning_rate": 6.47382146439318e-05, + "loss": 1.1932, + "theoretical_loss": 3.346140943004306, + "tokens_seen": 3036545024 + }, + { + "epoch": 0.92, + "learning_rate": 6.47221664994985e-05, + "loss": 1.2667, + "theoretical_loss": 3.3461354512560835, + "tokens_seen": 3036610560 + }, + { + "epoch": 0.92, + "learning_rate": 6.470611835506521e-05, + "loss": 1.1954, + "theoretical_loss": 3.3461299596595677, + "tokens_seen": 3036676096 + }, + { + "epoch": 0.92, + "learning_rate": 6.46900702106319e-05, + "loss": 1.2029, + "theoretical_loss": 3.3461244682147515, + "tokens_seen": 3036741632 + }, + { + "epoch": 0.92, + "learning_rate": 6.46740220661986e-05, + "loss": 1.2664, + "theoretical_loss": 3.3461189769216273, + "tokens_seen": 3036807168 + }, + { + "epoch": 0.92, + "learning_rate": 6.46579739217653e-05, + "loss": 1.1744, + "theoretical_loss": 3.346113485780188, + "tokens_seen": 3036872704 + }, + { + "epoch": 0.92, + "learning_rate": 6.4641925777332e-05, + "loss": 1.1398, + "theoretical_loss": 3.3461079947904255, + "tokens_seen": 3036938240 + }, + { + "epoch": 0.92, + "learning_rate": 6.46258776328987e-05, + "loss": 1.1616, + "theoretical_loss": 3.3461025039523324, + "tokens_seen": 3037003776 + }, + { + "epoch": 0.92, + "learning_rate": 6.46098294884654e-05, + "loss": 1.2259, + "theoretical_loss": 3.3460970132659016, + "tokens_seen": 3037069312 + }, + { + "epoch": 0.92, + "learning_rate": 6.45937813440321e-05, + "loss": 1.0848, + "theoretical_loss": 3.346091522731126, + "tokens_seen": 3037134848 + }, + { + "epoch": 0.92, + "learning_rate": 6.45777331995988e-05, + "loss": 1.2057, + "theoretical_loss": 3.346086032347997, + "tokens_seen": 3037200384 + }, + { + "epoch": 0.92, + "learning_rate": 6.45616850551655e-05, + "loss": 1.1573, + "theoretical_loss": 3.346080542116508, + "tokens_seen": 3037265920 + }, + { + "epoch": 0.92, + "learning_rate": 6.45456369107322e-05, + "loss": 1.1916, + "theoretical_loss": 3.3460750520366513, + "tokens_seen": 3037331456 + }, + { + "epoch": 0.92, + "learning_rate": 6.45295887662989e-05, + "loss": 1.1542, + "theoretical_loss": 3.3460695621084193, + "tokens_seen": 3037396992 + }, + { + "epoch": 0.92, + "learning_rate": 6.45135406218656e-05, + "loss": 1.2265, + "theoretical_loss": 3.3460640723318047, + "tokens_seen": 3037462528 + }, + { + "epoch": 0.92, + "learning_rate": 6.449749247743229e-05, + "loss": 1.2682, + "theoretical_loss": 3.3460585827068003, + "tokens_seen": 3037528064 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1013094, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1800098419189453, + "objective/train/theoretical_loss": 3.3460530932333983, + "objective/train/tokens_used": 3058053600, + "theoretical_loss": 3.3460530932333983, + "tokens_seen": 3037593600 + }, + { + "epoch": 0.92, + "learning_rate": 6.448144433299899e-05, + "loss": 1.2058, + "theoretical_loss": 3.3460530932333983, + "tokens_seen": 3037593600 + }, + { + "epoch": 0.92, + "learning_rate": 6.446539618856569e-05, + "loss": 1.1195, + "theoretical_loss": 3.3460476039115914, + "tokens_seen": 3037659136 + }, + { + "epoch": 0.92, + "learning_rate": 6.444934804413239e-05, + "loss": 1.2069, + "theoretical_loss": 3.3460421147413717, + "tokens_seen": 3037724672 + }, + { + "epoch": 0.92, + "learning_rate": 6.44332998996991e-05, + "loss": 1.1623, + "theoretical_loss": 3.346036625722732, + "tokens_seen": 3037790208 + }, + { + "epoch": 0.92, + "learning_rate": 6.44172517552658e-05, + "loss": 1.1749, + "theoretical_loss": 3.3460311368556654, + "tokens_seen": 3037855744 + }, + { + "epoch": 0.92, + "learning_rate": 6.440120361083251e-05, + "loss": 1.1812, + "theoretical_loss": 3.346025648140164, + "tokens_seen": 3037921280 + }, + { + "epoch": 0.92, + "learning_rate": 6.438515546639921e-05, + "loss": 1.1956, + "theoretical_loss": 3.34602015957622, + "tokens_seen": 3037986816 + }, + { + "epoch": 0.92, + "learning_rate": 6.436910732196591e-05, + "loss": 1.3084, + "theoretical_loss": 3.3460146711638266, + "tokens_seen": 3038052352 + }, + { + "epoch": 0.92, + "learning_rate": 6.435305917753261e-05, + "loss": 1.1777, + "theoretical_loss": 3.346009182902976, + "tokens_seen": 3038117888 + }, + { + "epoch": 0.92, + "learning_rate": 6.43370110330993e-05, + "loss": 1.1764, + "theoretical_loss": 3.3460036947936604, + "tokens_seen": 3038183424 + }, + { + "epoch": 0.92, + "learning_rate": 6.4320962888666e-05, + "loss": 1.2219, + "theoretical_loss": 3.345998206835873, + "tokens_seen": 3038248960 + }, + { + "epoch": 0.92, + "learning_rate": 6.43049147442327e-05, + "loss": 1.2493, + "theoretical_loss": 3.345992719029606, + "tokens_seen": 3038314496 + }, + { + "epoch": 0.92, + "learning_rate": 6.42888665997994e-05, + "loss": 1.1186, + "theoretical_loss": 3.345987231374852, + "tokens_seen": 3038380032 + }, + { + "epoch": 0.92, + "learning_rate": 6.42728184553661e-05, + "loss": 1.2282, + "theoretical_loss": 3.3459817438716035, + "tokens_seen": 3038445568 + }, + { + "epoch": 0.92, + "learning_rate": 6.42567703109328e-05, + "loss": 1.1171, + "theoretical_loss": 3.3459762565198536, + "tokens_seen": 3038511104 + }, + { + "epoch": 0.92, + "learning_rate": 6.42407221664995e-05, + "loss": 1.1275, + "theoretical_loss": 3.3459707693195937, + "tokens_seen": 3038576640 + }, + { + "epoch": 0.92, + "learning_rate": 6.42246740220662e-05, + "loss": 1.1482, + "theoretical_loss": 3.3459652822708175, + "tokens_seen": 3038642176 + }, + { + "epoch": 0.92, + "learning_rate": 6.42086258776329e-05, + "loss": 1.1758, + "theoretical_loss": 3.345959795373517, + "tokens_seen": 3038707712 + }, + { + "epoch": 0.92, + "learning_rate": 6.41925777331996e-05, + "loss": 1.235, + "theoretical_loss": 3.3459543086276846, + "tokens_seen": 3038773248 + }, + { + "epoch": 0.92, + "learning_rate": 6.41765295887663e-05, + "loss": 1.0784, + "theoretical_loss": 3.3459488220333133, + "tokens_seen": 3038838784 + }, + { + "epoch": 0.92, + "learning_rate": 6.4160481444333e-05, + "loss": 1.2003, + "theoretical_loss": 3.345943335590395, + "tokens_seen": 3038904320 + }, + { + "epoch": 0.92, + "learning_rate": 6.414443329989969e-05, + "loss": 1.1733, + "theoretical_loss": 3.3459378492989234, + "tokens_seen": 3038969856 + }, + { + "epoch": 0.92, + "learning_rate": 6.412838515546639e-05, + "loss": 1.1449, + "theoretical_loss": 3.3459323631588895, + "tokens_seen": 3039035392 + }, + { + "epoch": 0.92, + "learning_rate": 6.41123370110331e-05, + "loss": 1.1545, + "theoretical_loss": 3.345926877170287, + "tokens_seen": 3039100928 + }, + { + "epoch": 0.92, + "learning_rate": 6.409628886659981e-05, + "loss": 1.1146, + "theoretical_loss": 3.3459213913331087, + "tokens_seen": 3039166464 + }, + { + "epoch": 0.92, + "learning_rate": 6.408024072216651e-05, + "loss": 1.2161, + "theoretical_loss": 3.345915905647346, + "tokens_seen": 3039232000 + }, + { + "epoch": 0.92, + "learning_rate": 6.406419257773321e-05, + "loss": 1.1449, + "theoretical_loss": 3.3459104201129923, + "tokens_seen": 3039297536 + }, + { + "epoch": 0.92, + "learning_rate": 6.404814443329991e-05, + "loss": 1.2035, + "theoretical_loss": 3.3459049347300396, + "tokens_seen": 3039363072 + }, + { + "epoch": 0.92, + "learning_rate": 6.403209628886661e-05, + "loss": 1.2603, + "theoretical_loss": 3.3458994494984813, + "tokens_seen": 3039428608 + }, + { + "epoch": 0.92, + "learning_rate": 6.401604814443331e-05, + "loss": 1.2526, + "theoretical_loss": 3.345893964418309, + "tokens_seen": 3039494144 + }, + { + "epoch": 0.92, + "learning_rate": 6.400000000000001e-05, + "loss": 1.1528, + "theoretical_loss": 3.3458884794895156, + "tokens_seen": 3039559680 + }, + { + "epoch": 0.92, + "learning_rate": 6.39839518555667e-05, + "loss": 1.2224, + "theoretical_loss": 3.345882994712094, + "tokens_seen": 3039625216 + }, + { + "epoch": 0.92, + "learning_rate": 6.39679037111334e-05, + "loss": 1.172, + "theoretical_loss": 3.3458775100860363, + "tokens_seen": 3039690752 + }, + { + "epoch": 0.92, + "learning_rate": 6.39518555667001e-05, + "loss": 1.1692, + "theoretical_loss": 3.3458720256113352, + "tokens_seen": 3039756288 + }, + { + "epoch": 0.92, + "learning_rate": 6.39358074222668e-05, + "loss": 1.2189, + "theoretical_loss": 3.3458665412879838, + "tokens_seen": 3039821824 + }, + { + "epoch": 0.92, + "learning_rate": 6.39197592778335e-05, + "loss": 1.2744, + "theoretical_loss": 3.345861057115974, + "tokens_seen": 3039887360 + }, + { + "epoch": 0.92, + "learning_rate": 6.39037111334002e-05, + "loss": 1.2561, + "theoretical_loss": 3.345855573095298, + "tokens_seen": 3039952896 + }, + { + "epoch": 0.92, + "learning_rate": 6.38876629889669e-05, + "loss": 1.1751, + "theoretical_loss": 3.3458500892259493, + "tokens_seen": 3040018432 + }, + { + "epoch": 0.92, + "learning_rate": 6.38716148445336e-05, + "loss": 1.1066, + "theoretical_loss": 3.3458446055079203, + "tokens_seen": 3040083968 + }, + { + "epoch": 0.92, + "learning_rate": 6.38555667001003e-05, + "loss": 1.1837, + "theoretical_loss": 3.345839121941203, + "tokens_seen": 3040149504 + }, + { + "epoch": 0.92, + "learning_rate": 6.3839518555667e-05, + "loss": 1.2036, + "theoretical_loss": 3.3458336385257903, + "tokens_seen": 3040215040 + }, + { + "epoch": 0.92, + "learning_rate": 6.38234704112337e-05, + "loss": 1.1099, + "theoretical_loss": 3.345828155261675, + "tokens_seen": 3040280576 + }, + { + "epoch": 0.92, + "learning_rate": 6.38074222668004e-05, + "loss": 1.2115, + "theoretical_loss": 3.345822672148849, + "tokens_seen": 3040346112 + }, + { + "epoch": 0.92, + "learning_rate": 6.37913741223671e-05, + "loss": 1.2207, + "theoretical_loss": 3.3458171891873056, + "tokens_seen": 3040411648 + }, + { + "epoch": 0.92, + "learning_rate": 6.37753259779338e-05, + "loss": 1.2117, + "theoretical_loss": 3.345811706377037, + "tokens_seen": 3040477184 + }, + { + "epoch": 0.92, + "learning_rate": 6.37592778335005e-05, + "loss": 1.0743, + "theoretical_loss": 3.3458062237180357, + "tokens_seen": 3040542720 + }, + { + "epoch": 0.92, + "learning_rate": 6.374322968906721e-05, + "loss": 1.2634, + "theoretical_loss": 3.3458007412102946, + "tokens_seen": 3040608256 + }, + { + "epoch": 0.92, + "learning_rate": 6.372718154463391e-05, + "loss": 1.2889, + "theoretical_loss": 3.345795258853806, + "tokens_seen": 3040673792 + }, + { + "epoch": 0.92, + "learning_rate": 6.371113340020061e-05, + "loss": 1.1076, + "theoretical_loss": 3.3457897766485623, + "tokens_seen": 3040739328 + }, + { + "epoch": 0.92, + "learning_rate": 6.369508525576731e-05, + "loss": 1.18, + "theoretical_loss": 3.3457842945945564, + "tokens_seen": 3040804864 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1014325, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1042473316192627, + "objective/train/theoretical_loss": 3.3457788126917807, + "objective/train/tokens_used": 3061330400, + "theoretical_loss": 3.3457788126917807, + "tokens_seen": 3040870400 + }, + { + "epoch": 0.92, + "learning_rate": 6.367903711133401e-05, + "loss": 1.1574, + "theoretical_loss": 3.3457788126917807, + "tokens_seen": 3040870400 + }, + { + "epoch": 0.92, + "learning_rate": 6.366298896690071e-05, + "loss": 1.1931, + "theoretical_loss": 3.345773330940228, + "tokens_seen": 3040935936 + }, + { + "epoch": 0.92, + "learning_rate": 6.364694082246741e-05, + "loss": 1.1696, + "theoretical_loss": 3.3457678493398904, + "tokens_seen": 3041001472 + }, + { + "epoch": 0.92, + "learning_rate": 6.36308926780341e-05, + "loss": 1.2016, + "theoretical_loss": 3.345762367890761, + "tokens_seen": 3041067008 + }, + { + "epoch": 0.92, + "learning_rate": 6.36148445336008e-05, + "loss": 1.2199, + "theoretical_loss": 3.345756886592832, + "tokens_seen": 3041132544 + }, + { + "epoch": 0.92, + "learning_rate": 6.35987963891675e-05, + "loss": 1.2149, + "theoretical_loss": 3.345751405446096, + "tokens_seen": 3041198080 + }, + { + "epoch": 0.92, + "learning_rate": 6.35827482447342e-05, + "loss": 1.1906, + "theoretical_loss": 3.3457459244505463, + "tokens_seen": 3041263616 + }, + { + "epoch": 0.92, + "learning_rate": 6.35667001003009e-05, + "loss": 1.2204, + "theoretical_loss": 3.345740443606174, + "tokens_seen": 3041329152 + }, + { + "epoch": 0.92, + "learning_rate": 6.35506519558676e-05, + "loss": 1.1857, + "theoretical_loss": 3.345734962912973, + "tokens_seen": 3041394688 + }, + { + "epoch": 0.92, + "learning_rate": 6.35346038114343e-05, + "loss": 1.1988, + "theoretical_loss": 3.345729482370935, + "tokens_seen": 3041460224 + }, + { + "epoch": 0.92, + "learning_rate": 6.3518555667001e-05, + "loss": 1.2388, + "theoretical_loss": 3.3457240019800536, + "tokens_seen": 3041525760 + }, + { + "epoch": 0.92, + "learning_rate": 6.350250752256772e-05, + "loss": 1.3116, + "theoretical_loss": 3.34571852174032, + "tokens_seen": 3041591296 + }, + { + "epoch": 0.92, + "learning_rate": 6.348645937813442e-05, + "loss": 1.2756, + "theoretical_loss": 3.3457130416517282, + "tokens_seen": 3041656832 + }, + { + "epoch": 0.92, + "learning_rate": 6.34704112337011e-05, + "loss": 1.2117, + "theoretical_loss": 3.3457075617142698, + "tokens_seen": 3041722368 + }, + { + "epoch": 0.92, + "learning_rate": 6.34543630892678e-05, + "loss": 1.1022, + "theoretical_loss": 3.3457020819279375, + "tokens_seen": 3041787904 + }, + { + "epoch": 0.92, + "learning_rate": 6.34383149448345e-05, + "loss": 1.1004, + "theoretical_loss": 3.3456966022927244, + "tokens_seen": 3041853440 + }, + { + "epoch": 0.92, + "learning_rate": 6.34222668004012e-05, + "loss": 1.2024, + "theoretical_loss": 3.3456911228086224, + "tokens_seen": 3041918976 + }, + { + "epoch": 0.92, + "learning_rate": 6.340621865596791e-05, + "loss": 1.2961, + "theoretical_loss": 3.3456856434756244, + "tokens_seen": 3041984512 + }, + { + "epoch": 0.92, + "learning_rate": 6.339017051153461e-05, + "loss": 1.2387, + "theoretical_loss": 3.3456801642937233, + "tokens_seen": 3042050048 + }, + { + "epoch": 0.92, + "learning_rate": 6.337412236710131e-05, + "loss": 1.1748, + "theoretical_loss": 3.345674685262911, + "tokens_seen": 3042115584 + }, + { + "epoch": 0.92, + "learning_rate": 6.335807422266801e-05, + "loss": 1.2055, + "theoretical_loss": 3.3456692063831803, + "tokens_seen": 3042181120 + }, + { + "epoch": 0.92, + "learning_rate": 6.334202607823471e-05, + "loss": 1.178, + "theoretical_loss": 3.3456637276545242, + "tokens_seen": 3042246656 + }, + { + "epoch": 0.92, + "learning_rate": 6.332597793380141e-05, + "loss": 1.1792, + "theoretical_loss": 3.345658249076935, + "tokens_seen": 3042312192 + }, + { + "epoch": 0.92, + "learning_rate": 6.330992978936811e-05, + "loss": 1.207, + "theoretical_loss": 3.345652770650405, + "tokens_seen": 3042377728 + }, + { + "epoch": 0.92, + "learning_rate": 6.329388164493481e-05, + "loss": 1.2007, + "theoretical_loss": 3.3456472923749274, + "tokens_seen": 3042443264 + }, + { + "epoch": 0.92, + "learning_rate": 6.32778335005015e-05, + "loss": 1.2583, + "theoretical_loss": 3.3456418142504942, + "tokens_seen": 3042508800 + }, + { + "epoch": 0.92, + "learning_rate": 6.32617853560682e-05, + "loss": 1.1835, + "theoretical_loss": 3.3456363362770984, + "tokens_seen": 3042574336 + }, + { + "epoch": 0.92, + "learning_rate": 6.32457372116349e-05, + "loss": 1.1987, + "theoretical_loss": 3.345630858454732, + "tokens_seen": 3042639872 + }, + { + "epoch": 0.92, + "learning_rate": 6.32296890672016e-05, + "loss": 1.1558, + "theoretical_loss": 3.3456253807833884, + "tokens_seen": 3042705408 + }, + { + "epoch": 0.92, + "learning_rate": 6.32136409227683e-05, + "loss": 1.1476, + "theoretical_loss": 3.3456199032630596, + "tokens_seen": 3042770944 + }, + { + "epoch": 0.92, + "learning_rate": 6.319759277833502e-05, + "loss": 1.239, + "theoretical_loss": 3.3456144258937384, + "tokens_seen": 3042836480 + }, + { + "epoch": 0.92, + "learning_rate": 6.318154463390172e-05, + "loss": 1.2201, + "theoretical_loss": 3.3456089486754172, + "tokens_seen": 3042902016 + }, + { + "epoch": 0.92, + "learning_rate": 6.316549648946842e-05, + "loss": 1.2181, + "theoretical_loss": 3.345603471608089, + "tokens_seen": 3042967552 + }, + { + "epoch": 0.92, + "learning_rate": 6.314944834503512e-05, + "loss": 1.3039, + "theoretical_loss": 3.345597994691746, + "tokens_seen": 3043033088 + }, + { + "epoch": 0.92, + "learning_rate": 6.313340020060182e-05, + "loss": 1.2382, + "theoretical_loss": 3.345592517926381, + "tokens_seen": 3043098624 + }, + { + "epoch": 0.92, + "learning_rate": 6.31173520561685e-05, + "loss": 1.1667, + "theoretical_loss": 3.345587041311986, + "tokens_seen": 3043164160 + }, + { + "epoch": 0.92, + "learning_rate": 6.31013039117352e-05, + "loss": 1.1679, + "theoretical_loss": 3.3455815648485543, + "tokens_seen": 3043229696 + }, + { + "epoch": 0.92, + "learning_rate": 6.30852557673019e-05, + "loss": 1.2203, + "theoretical_loss": 3.3455760885360784, + "tokens_seen": 3043295232 + }, + { + "epoch": 0.92, + "learning_rate": 6.306920762286861e-05, + "loss": 1.1536, + "theoretical_loss": 3.3455706123745506, + "tokens_seen": 3043360768 + }, + { + "epoch": 0.92, + "learning_rate": 6.305315947843531e-05, + "loss": 1.2211, + "theoretical_loss": 3.3455651363639634, + "tokens_seen": 3043426304 + }, + { + "epoch": 0.92, + "learning_rate": 6.303711133400201e-05, + "loss": 1.1633, + "theoretical_loss": 3.34555966050431, + "tokens_seen": 3043491840 + }, + { + "epoch": 0.92, + "learning_rate": 6.302106318956871e-05, + "loss": 1.1202, + "theoretical_loss": 3.3455541847955823, + "tokens_seen": 3043557376 + }, + { + "epoch": 0.92, + "learning_rate": 6.300501504513541e-05, + "loss": 1.1977, + "theoretical_loss": 3.3455487092377734, + "tokens_seen": 3043622912 + }, + { + "epoch": 0.92, + "learning_rate": 6.298896690070211e-05, + "loss": 1.2109, + "theoretical_loss": 3.3455432338308757, + "tokens_seen": 3043688448 + }, + { + "epoch": 0.92, + "learning_rate": 6.297291875626881e-05, + "loss": 1.2995, + "theoretical_loss": 3.345537758574882, + "tokens_seen": 3043753984 + }, + { + "epoch": 0.92, + "learning_rate": 6.295687061183551e-05, + "loss": 1.1729, + "theoretical_loss": 3.345532283469784, + "tokens_seen": 3043819520 + }, + { + "epoch": 0.92, + "learning_rate": 6.294082246740221e-05, + "loss": 1.1699, + "theoretical_loss": 3.345526808515576, + "tokens_seen": 3043885056 + }, + { + "epoch": 0.92, + "learning_rate": 6.29247743229689e-05, + "loss": 1.1183, + "theoretical_loss": 3.3455213337122487, + "tokens_seen": 3043950592 + }, + { + "epoch": 0.92, + "learning_rate": 6.290872617853561e-05, + "loss": 1.231, + "theoretical_loss": 3.3455158590597955, + "tokens_seen": 3044016128 + }, + { + "epoch": 0.92, + "learning_rate": 6.289267803410231e-05, + "loss": 1.2251, + "theoretical_loss": 3.3455103845582093, + "tokens_seen": 3044081664 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1015283, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1950149536132812, + "objective/train/theoretical_loss": 3.3455049102074828, + "objective/train/tokens_used": 3064607200, + "theoretical_loss": 3.3455049102074828, + "tokens_seen": 3044147200 + }, + { + "epoch": 0.92, + "learning_rate": 6.287662988966901e-05, + "loss": 1.1922, + "theoretical_loss": 3.3455049102074828, + "tokens_seen": 3044147200 + }, + { + "epoch": 0.92, + "learning_rate": 6.286058174523572e-05, + "loss": 1.1376, + "theoretical_loss": 3.345499436007608, + "tokens_seen": 3044212736 + }, + { + "epoch": 0.92, + "learning_rate": 6.284453360080242e-05, + "loss": 1.2258, + "theoretical_loss": 3.3454939619585775, + "tokens_seen": 3044278272 + }, + { + "epoch": 0.92, + "learning_rate": 6.282848545636912e-05, + "loss": 1.1749, + "theoretical_loss": 3.345488488060384, + "tokens_seen": 3044343808 + }, + { + "epoch": 0.92, + "learning_rate": 6.281243731193582e-05, + "loss": 1.1295, + "theoretical_loss": 3.3454830143130208, + "tokens_seen": 3044409344 + }, + { + "epoch": 0.92, + "learning_rate": 6.279638916750252e-05, + "loss": 1.2259, + "theoretical_loss": 3.3454775407164794, + "tokens_seen": 3044474880 + }, + { + "epoch": 0.92, + "learning_rate": 6.27803410230692e-05, + "loss": 1.1812, + "theoretical_loss": 3.3454720672707534, + "tokens_seen": 3044540416 + }, + { + "epoch": 0.92, + "learning_rate": 6.27642928786359e-05, + "loss": 1.2186, + "theoretical_loss": 3.345466593975835, + "tokens_seen": 3044605952 + }, + { + "epoch": 0.92, + "learning_rate": 6.27482447342026e-05, + "loss": 1.2193, + "theoretical_loss": 3.345461120831716, + "tokens_seen": 3044671488 + }, + { + "epoch": 0.92, + "learning_rate": 6.27321965897693e-05, + "loss": 1.164, + "theoretical_loss": 3.34545564783839, + "tokens_seen": 3044737024 + }, + { + "epoch": 0.92, + "learning_rate": 6.271614844533601e-05, + "loss": 1.2217, + "theoretical_loss": 3.3454501749958494, + "tokens_seen": 3044802560 + }, + { + "epoch": 0.92, + "learning_rate": 6.270010030090271e-05, + "loss": 1.2226, + "theoretical_loss": 3.3454447023040865, + "tokens_seen": 3044868096 + }, + { + "epoch": 0.92, + "learning_rate": 6.268405215646941e-05, + "loss": 1.0948, + "theoretical_loss": 3.3454392297630946, + "tokens_seen": 3044933632 + }, + { + "epoch": 0.92, + "learning_rate": 6.266800401203611e-05, + "loss": 1.1563, + "theoretical_loss": 3.3454337573728656, + "tokens_seen": 3044999168 + }, + { + "epoch": 0.92, + "learning_rate": 6.265195586760281e-05, + "loss": 1.2741, + "theoretical_loss": 3.345428285133392, + "tokens_seen": 3045064704 + }, + { + "epoch": 0.92, + "learning_rate": 6.263590772316951e-05, + "loss": 1.1993, + "theoretical_loss": 3.3454228130446673, + "tokens_seen": 3045130240 + }, + { + "epoch": 0.92, + "learning_rate": 6.261985957873621e-05, + "loss": 1.1696, + "theoretical_loss": 3.3454173411066828, + "tokens_seen": 3045195776 + }, + { + "epoch": 0.92, + "learning_rate": 6.260381143430291e-05, + "loss": 1.2469, + "theoretical_loss": 3.3454118693194324, + "tokens_seen": 3045261312 + }, + { + "epoch": 0.92, + "learning_rate": 6.258776328986961e-05, + "loss": 1.1539, + "theoretical_loss": 3.345406397682908, + "tokens_seen": 3045326848 + }, + { + "epoch": 0.92, + "learning_rate": 6.257171514543631e-05, + "loss": 1.1495, + "theoretical_loss": 3.3454009261971023, + "tokens_seen": 3045392384 + }, + { + "epoch": 0.92, + "learning_rate": 6.255566700100301e-05, + "loss": 1.208, + "theoretical_loss": 3.345395454862008, + "tokens_seen": 3045457920 + }, + { + "epoch": 0.92, + "learning_rate": 6.253961885656971e-05, + "loss": 1.0951, + "theoretical_loss": 3.3453899836776175, + "tokens_seen": 3045523456 + }, + { + "epoch": 0.92, + "learning_rate": 6.252357071213641e-05, + "loss": 1.256, + "theoretical_loss": 3.345384512643924, + "tokens_seen": 3045588992 + }, + { + "epoch": 0.92, + "learning_rate": 6.250752256770312e-05, + "loss": 1.2182, + "theoretical_loss": 3.345379041760919, + "tokens_seen": 3045654528 + }, + { + "epoch": 0.92, + "learning_rate": 6.249147442326982e-05, + "loss": 1.1649, + "theoretical_loss": 3.3453735710285963, + "tokens_seen": 3045720064 + }, + { + "epoch": 0.92, + "learning_rate": 6.247542627883652e-05, + "loss": 1.2585, + "theoretical_loss": 3.3453681004469473, + "tokens_seen": 3045785600 + }, + { + "epoch": 0.92, + "learning_rate": 6.245937813440322e-05, + "loss": 1.2137, + "theoretical_loss": 3.3453626300159662, + "tokens_seen": 3045851136 + }, + { + "epoch": 0.92, + "learning_rate": 6.244332998996992e-05, + "loss": 1.2628, + "theoretical_loss": 3.345357159735644, + "tokens_seen": 3045916672 + }, + { + "epoch": 0.92, + "learning_rate": 6.24272818455366e-05, + "loss": 1.2376, + "theoretical_loss": 3.345351689605974, + "tokens_seen": 3045982208 + }, + { + "epoch": 0.92, + "learning_rate": 6.24112337011033e-05, + "loss": 1.2016, + "theoretical_loss": 3.3453462196269497, + "tokens_seen": 3046047744 + }, + { + "epoch": 0.92, + "learning_rate": 6.239518555667e-05, + "loss": 1.2134, + "theoretical_loss": 3.345340749798562, + "tokens_seen": 3046113280 + }, + { + "epoch": 0.92, + "learning_rate": 6.237913741223671e-05, + "loss": 1.1674, + "theoretical_loss": 3.3453352801208043, + "tokens_seen": 3046178816 + }, + { + "epoch": 0.92, + "learning_rate": 6.236308926780341e-05, + "loss": 1.1767, + "theoretical_loss": 3.3453298105936695, + "tokens_seen": 3046244352 + }, + { + "epoch": 0.92, + "learning_rate": 6.234704112337011e-05, + "loss": 1.2543, + "theoretical_loss": 3.3453243412171503, + "tokens_seen": 3046309888 + }, + { + "epoch": 0.92, + "learning_rate": 6.233099297893681e-05, + "loss": 1.2097, + "theoretical_loss": 3.345318871991238, + "tokens_seen": 3046375424 + }, + { + "epoch": 0.92, + "learning_rate": 6.231494483450351e-05, + "loss": 1.2191, + "theoretical_loss": 3.3453134029159273, + "tokens_seen": 3046440960 + }, + { + "epoch": 0.92, + "learning_rate": 6.229889669007022e-05, + "loss": 1.1504, + "theoretical_loss": 3.345307933991209, + "tokens_seen": 3046506496 + }, + { + "epoch": 0.92, + "learning_rate": 6.228284854563692e-05, + "loss": 1.1774, + "theoretical_loss": 3.3453024652170766, + "tokens_seen": 3046572032 + }, + { + "epoch": 0.92, + "learning_rate": 6.226680040120361e-05, + "loss": 1.2593, + "theoretical_loss": 3.345296996593522, + "tokens_seen": 3046637568 + }, + { + "epoch": 0.92, + "learning_rate": 6.225075225677031e-05, + "loss": 1.1692, + "theoretical_loss": 3.345291528120539, + "tokens_seen": 3046703104 + }, + { + "epoch": 0.92, + "learning_rate": 6.223470411233701e-05, + "loss": 1.1714, + "theoretical_loss": 3.3452860597981195, + "tokens_seen": 3046768640 + }, + { + "epoch": 0.92, + "learning_rate": 6.221865596790371e-05, + "loss": 1.1551, + "theoretical_loss": 3.3452805916262562, + "tokens_seen": 3046834176 + }, + { + "epoch": 0.92, + "learning_rate": 6.220260782347041e-05, + "loss": 1.12, + "theoretical_loss": 3.3452751236049414, + "tokens_seen": 3046899712 + }, + { + "epoch": 0.92, + "learning_rate": 6.218655967903711e-05, + "loss": 1.21, + "theoretical_loss": 3.345269655734168, + "tokens_seen": 3046965248 + }, + { + "epoch": 0.92, + "learning_rate": 6.217051153460382e-05, + "loss": 1.258, + "theoretical_loss": 3.345264188013929, + "tokens_seen": 3047030784 + }, + { + "epoch": 0.92, + "learning_rate": 6.215446339017052e-05, + "loss": 1.2546, + "theoretical_loss": 3.345258720444216, + "tokens_seen": 3047096320 + }, + { + "epoch": 0.92, + "learning_rate": 6.213841524573722e-05, + "loss": 1.2244, + "theoretical_loss": 3.3452532530250227, + "tokens_seen": 3047161856 + }, + { + "epoch": 0.92, + "learning_rate": 6.212236710130392e-05, + "loss": 1.137, + "theoretical_loss": 3.3452477857563414, + "tokens_seen": 3047227392 + }, + { + "epoch": 0.92, + "learning_rate": 6.210631895687062e-05, + "loss": 1.1942, + "theoretical_loss": 3.3452423186381646, + "tokens_seen": 3047292928 + }, + { + "epoch": 0.92, + "learning_rate": 6.209027081243732e-05, + "loss": 1.2366, + "theoretical_loss": 3.3452368516704847, + "tokens_seen": 3047358464 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1016512, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1774486303329468, + "objective/train/theoretical_loss": 3.3452313848532946, + "objective/train/tokens_used": 3067884000, + "theoretical_loss": 3.3452313848532946, + "tokens_seen": 3047424000 + }, + { + "epoch": 0.92, + "learning_rate": 6.2074222668004e-05, + "loss": 1.1966, + "theoretical_loss": 3.3452313848532946, + "tokens_seen": 3047424000 + }, + { + "epoch": 0.92, + "learning_rate": 6.20581745235707e-05, + "loss": 1.4011, + "theoretical_loss": 3.345225918186587, + "tokens_seen": 3047489536 + }, + { + "epoch": 0.92, + "learning_rate": 6.20421263791374e-05, + "loss": 1.159, + "theoretical_loss": 3.345220451670354, + "tokens_seen": 3047555072 + }, + { + "epoch": 0.92, + "learning_rate": 6.202607823470411e-05, + "loss": 1.2058, + "theoretical_loss": 3.345214985304589, + "tokens_seen": 3047620608 + }, + { + "epoch": 0.92, + "learning_rate": 6.201003009027082e-05, + "loss": 1.2161, + "theoretical_loss": 3.345209519089284, + "tokens_seen": 3047686144 + }, + { + "epoch": 0.92, + "learning_rate": 6.199398194583752e-05, + "loss": 1.2625, + "theoretical_loss": 3.345204053024432, + "tokens_seen": 3047751680 + }, + { + "epoch": 0.92, + "learning_rate": 6.197793380140422e-05, + "loss": 1.2634, + "theoretical_loss": 3.3451985871100254, + "tokens_seen": 3047817216 + }, + { + "epoch": 0.92, + "learning_rate": 6.196188565697092e-05, + "loss": 1.2435, + "theoretical_loss": 3.345193121346057, + "tokens_seen": 3047882752 + }, + { + "epoch": 0.92, + "learning_rate": 6.194583751253762e-05, + "loss": 1.2529, + "theoretical_loss": 3.345187655732519, + "tokens_seen": 3047948288 + }, + { + "epoch": 0.92, + "learning_rate": 6.192978936810432e-05, + "loss": 1.2249, + "theoretical_loss": 3.345182190269404, + "tokens_seen": 3048013824 + }, + { + "epoch": 0.92, + "learning_rate": 6.191374122367101e-05, + "loss": 1.2273, + "theoretical_loss": 3.3451767249567057, + "tokens_seen": 3048079360 + }, + { + "epoch": 0.92, + "learning_rate": 6.189769307923771e-05, + "loss": 1.1791, + "theoretical_loss": 3.345171259794416, + "tokens_seen": 3048144896 + }, + { + "epoch": 0.92, + "learning_rate": 6.188164493480441e-05, + "loss": 1.1824, + "theoretical_loss": 3.3451657947825275, + "tokens_seen": 3048210432 + }, + { + "epoch": 0.92, + "learning_rate": 6.186559679037111e-05, + "loss": 1.1922, + "theoretical_loss": 3.3451603299210326, + "tokens_seen": 3048275968 + }, + { + "epoch": 0.92, + "learning_rate": 6.184954864593781e-05, + "loss": 1.2355, + "theoretical_loss": 3.345154865209924, + "tokens_seen": 3048341504 + }, + { + "epoch": 0.92, + "learning_rate": 6.183350050150452e-05, + "loss": 1.1904, + "theoretical_loss": 3.345149400649195, + "tokens_seen": 3048407040 + }, + { + "epoch": 0.92, + "learning_rate": 6.181745235707122e-05, + "loss": 1.1474, + "theoretical_loss": 3.3451439362388378, + "tokens_seen": 3048472576 + }, + { + "epoch": 0.92, + "learning_rate": 6.180140421263792e-05, + "loss": 1.3197, + "theoretical_loss": 3.3451384719788444, + "tokens_seen": 3048538112 + }, + { + "epoch": 0.92, + "learning_rate": 6.178535606820462e-05, + "loss": 1.1461, + "theoretical_loss": 3.3451330078692085, + "tokens_seen": 3048603648 + }, + { + "epoch": 0.92, + "learning_rate": 6.176930792377132e-05, + "loss": 1.155, + "theoretical_loss": 3.345127543909922, + "tokens_seen": 3048669184 + }, + { + "epoch": 0.92, + "learning_rate": 6.175325977933802e-05, + "loss": 1.0713, + "theoretical_loss": 3.345122080100978, + "tokens_seen": 3048734720 + }, + { + "epoch": 0.92, + "learning_rate": 6.173721163490472e-05, + "loss": 1.1403, + "theoretical_loss": 3.3451166164423682, + "tokens_seen": 3048800256 + }, + { + "epoch": 0.92, + "learning_rate": 6.17211634904714e-05, + "loss": 1.1952, + "theoretical_loss": 3.345111152934087, + "tokens_seen": 3048865792 + }, + { + "epoch": 0.92, + "learning_rate": 6.170511534603812e-05, + "loss": 1.1105, + "theoretical_loss": 3.345105689576125, + "tokens_seen": 3048931328 + }, + { + "epoch": 0.92, + "learning_rate": 6.168906720160482e-05, + "loss": 1.2419, + "theoretical_loss": 3.3451002263684764, + "tokens_seen": 3048996864 + }, + { + "epoch": 0.92, + "learning_rate": 6.167301905717152e-05, + "loss": 1.2133, + "theoretical_loss": 3.345094763311133, + "tokens_seen": 3049062400 + }, + { + "epoch": 0.92, + "learning_rate": 6.165697091273822e-05, + "loss": 1.1963, + "theoretical_loss": 3.345089300404087, + "tokens_seen": 3049127936 + }, + { + "epoch": 0.92, + "learning_rate": 6.164092276830492e-05, + "loss": 1.1756, + "theoretical_loss": 3.3450838376473326, + "tokens_seen": 3049193472 + }, + { + "epoch": 0.92, + "learning_rate": 6.162487462387162e-05, + "loss": 1.1514, + "theoretical_loss": 3.3450783750408615, + "tokens_seen": 3049259008 + }, + { + "epoch": 0.92, + "learning_rate": 6.160882647943832e-05, + "loss": 1.1536, + "theoretical_loss": 3.345072912584666, + "tokens_seen": 3049324544 + }, + { + "epoch": 0.92, + "learning_rate": 6.159277833500502e-05, + "loss": 1.2268, + "theoretical_loss": 3.3450674502787394, + "tokens_seen": 3049390080 + }, + { + "epoch": 0.92, + "learning_rate": 6.157673019057173e-05, + "loss": 1.3068, + "theoretical_loss": 3.345061988123074, + "tokens_seen": 3049455616 + }, + { + "epoch": 0.92, + "learning_rate": 6.156068204613841e-05, + "loss": 1.2425, + "theoretical_loss": 3.345056526117662, + "tokens_seen": 3049521152 + }, + { + "epoch": 0.92, + "learning_rate": 6.154463390170511e-05, + "loss": 1.2064, + "theoretical_loss": 3.345051064262497, + "tokens_seen": 3049586688 + }, + { + "epoch": 0.92, + "learning_rate": 6.152858575727181e-05, + "loss": 1.1717, + "theoretical_loss": 3.3450456025575708, + "tokens_seen": 3049652224 + }, + { + "epoch": 0.92, + "learning_rate": 6.151253761283851e-05, + "loss": 1.2557, + "theoretical_loss": 3.345040141002877, + "tokens_seen": 3049717760 + }, + { + "epoch": 0.92, + "learning_rate": 6.149648946840521e-05, + "loss": 1.1861, + "theoretical_loss": 3.345034679598407, + "tokens_seen": 3049783296 + }, + { + "epoch": 0.92, + "learning_rate": 6.148044132397192e-05, + "loss": 1.1428, + "theoretical_loss": 3.3450292183441546, + "tokens_seen": 3049848832 + }, + { + "epoch": 0.92, + "learning_rate": 6.146439317953862e-05, + "loss": 1.1555, + "theoretical_loss": 3.3450237572401114, + "tokens_seen": 3049914368 + }, + { + "epoch": 0.92, + "learning_rate": 6.144834503510532e-05, + "loss": 1.1922, + "theoretical_loss": 3.3450182962862707, + "tokens_seen": 3049979904 + }, + { + "epoch": 0.92, + "learning_rate": 6.143229689067202e-05, + "loss": 1.1876, + "theoretical_loss": 3.345012835482625, + "tokens_seen": 3050045440 + }, + { + "epoch": 0.92, + "learning_rate": 6.141624874623872e-05, + "loss": 1.138, + "theoretical_loss": 3.345007374829167, + "tokens_seen": 3050110976 + }, + { + "epoch": 0.92, + "learning_rate": 6.140020060180542e-05, + "loss": 1.2149, + "theoretical_loss": 3.3450019143258896, + "tokens_seen": 3050176512 + }, + { + "epoch": 0.92, + "learning_rate": 6.138415245737212e-05, + "loss": 1.3028, + "theoretical_loss": 3.3449964539727848, + "tokens_seen": 3050242048 + }, + { + "epoch": 0.92, + "learning_rate": 6.136810431293882e-05, + "loss": 1.1837, + "theoretical_loss": 3.344990993769845, + "tokens_seen": 3050307584 + }, + { + "epoch": 0.92, + "learning_rate": 6.135205616850552e-05, + "loss": 1.2192, + "theoretical_loss": 3.3449855337170646, + "tokens_seen": 3050373120 + }, + { + "epoch": 0.92, + "learning_rate": 6.133600802407222e-05, + "loss": 1.2412, + "theoretical_loss": 3.344980073814434, + "tokens_seen": 3050438656 + }, + { + "epoch": 0.92, + "learning_rate": 6.131995987963892e-05, + "loss": 1.178, + "theoretical_loss": 3.3449746140619476, + "tokens_seen": 3050504192 + }, + { + "epoch": 0.92, + "learning_rate": 6.130391173520562e-05, + "loss": 1.2074, + "theoretical_loss": 3.344969154459597, + "tokens_seen": 3050569728 + }, + { + "epoch": 0.92, + "learning_rate": 6.128786359077232e-05, + "loss": 1.2082, + "theoretical_loss": 3.344963695007375, + "tokens_seen": 3050635264 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1017419, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.188966989517212, + "objective/train/theoretical_loss": 3.344958235705275, + "objective/train/tokens_used": 3071160800, + "theoretical_loss": 3.344958235705275, + "tokens_seen": 3050700800 + }, + { + "epoch": 0.92, + "learning_rate": 6.127181544633902e-05, + "loss": 1.2469, + "theoretical_loss": 3.344958235705275, + "tokens_seen": 3050700800 + }, + { + "epoch": 0.92, + "learning_rate": 6.125576730190572e-05, + "loss": 1.2738, + "theoretical_loss": 3.3449527765532885, + "tokens_seen": 3050766336 + }, + { + "epoch": 0.92, + "learning_rate": 6.123971915747242e-05, + "loss": 1.1784, + "theoretical_loss": 3.344947317551409, + "tokens_seen": 3050831872 + }, + { + "epoch": 0.92, + "learning_rate": 6.122367101303913e-05, + "loss": 1.1549, + "theoretical_loss": 3.344941858699629, + "tokens_seen": 3050897408 + }, + { + "epoch": 0.92, + "learning_rate": 6.120762286860581e-05, + "loss": 1.1863, + "theoretical_loss": 3.344936399997941, + "tokens_seen": 3050962944 + }, + { + "epoch": 0.92, + "learning_rate": 6.119157472417251e-05, + "loss": 1.1816, + "theoretical_loss": 3.3449309414463375, + "tokens_seen": 3051028480 + }, + { + "epoch": 0.92, + "learning_rate": 6.117552657973921e-05, + "loss": 1.1012, + "theoretical_loss": 3.3449254830448116, + "tokens_seen": 3051094016 + }, + { + "epoch": 0.92, + "learning_rate": 6.115947843530591e-05, + "loss": 1.1155, + "theoretical_loss": 3.3449200247933555, + "tokens_seen": 3051159552 + }, + { + "epoch": 0.92, + "learning_rate": 6.114343029087262e-05, + "loss": 1.2022, + "theoretical_loss": 3.344914566691962, + "tokens_seen": 3051225088 + }, + { + "epoch": 0.92, + "learning_rate": 6.112738214643932e-05, + "loss": 1.1163, + "theoretical_loss": 3.344909108740624, + "tokens_seen": 3051290624 + }, + { + "epoch": 0.92, + "learning_rate": 6.111133400200602e-05, + "loss": 1.186, + "theoretical_loss": 3.3449036509393335, + "tokens_seen": 3051356160 + }, + { + "epoch": 0.92, + "learning_rate": 6.109528585757273e-05, + "loss": 1.1045, + "theoretical_loss": 3.344898193288084, + "tokens_seen": 3051421696 + }, + { + "epoch": 0.92, + "learning_rate": 6.107923771313943e-05, + "loss": 1.2195, + "theoretical_loss": 3.3448927357868676, + "tokens_seen": 3051487232 + }, + { + "epoch": 0.92, + "learning_rate": 6.106318956870612e-05, + "loss": 1.2294, + "theoretical_loss": 3.344887278435677, + "tokens_seen": 3051552768 + }, + { + "epoch": 0.92, + "learning_rate": 6.104714142427282e-05, + "loss": 1.1257, + "theoretical_loss": 3.344881821234505, + "tokens_seen": 3051618304 + }, + { + "epoch": 0.92, + "learning_rate": 6.1031093279839526e-05, + "loss": 1.3356, + "theoretical_loss": 3.344876364183344, + "tokens_seen": 3051683840 + }, + { + "epoch": 0.92, + "learning_rate": 6.101504513540622e-05, + "loss": 1.152, + "theoretical_loss": 3.3448709072821874, + "tokens_seen": 3051749376 + }, + { + "epoch": 0.92, + "learning_rate": 6.099899699097292e-05, + "loss": 1.1374, + "theoretical_loss": 3.3448654505310267, + "tokens_seen": 3051814912 + }, + { + "epoch": 0.92, + "learning_rate": 6.098294884653962e-05, + "loss": 1.2235, + "theoretical_loss": 3.3448599939298553, + "tokens_seen": 3051880448 + }, + { + "epoch": 0.92, + "learning_rate": 6.096690070210632e-05, + "loss": 1.2719, + "theoretical_loss": 3.344854537478666, + "tokens_seen": 3051945984 + }, + { + "epoch": 0.92, + "learning_rate": 6.095085255767302e-05, + "loss": 1.1902, + "theoretical_loss": 3.344849081177451, + "tokens_seen": 3052011520 + }, + { + "epoch": 0.92, + "learning_rate": 6.0934804413239723e-05, + "loss": 1.2246, + "theoretical_loss": 3.3448436250262032, + "tokens_seen": 3052077056 + }, + { + "epoch": 0.92, + "learning_rate": 6.091875626880642e-05, + "loss": 1.1825, + "theoretical_loss": 3.344838169024915, + "tokens_seen": 3052142592 + }, + { + "epoch": 0.92, + "learning_rate": 6.090270812437312e-05, + "loss": 1.0475, + "theoretical_loss": 3.3448327131735796, + "tokens_seen": 3052208128 + }, + { + "epoch": 0.92, + "learning_rate": 6.088665997993982e-05, + "loss": 1.2712, + "theoretical_loss": 3.3448272574721893, + "tokens_seen": 3052273664 + }, + { + "epoch": 0.92, + "learning_rate": 6.087061183550652e-05, + "loss": 1.2298, + "theoretical_loss": 3.3448218019207365, + "tokens_seen": 3052339200 + }, + { + "epoch": 0.92, + "learning_rate": 6.085456369107322e-05, + "loss": 1.2494, + "theoretical_loss": 3.3448163465192144, + "tokens_seen": 3052404736 + }, + { + "epoch": 0.92, + "learning_rate": 6.083851554663992e-05, + "loss": 1.2532, + "theoretical_loss": 3.344810891267615, + "tokens_seen": 3052470272 + }, + { + "epoch": 0.93, + "learning_rate": 6.0822467402206614e-05, + "loss": 1.1821, + "theoretical_loss": 3.344805436165932, + "tokens_seen": 3052535808 + }, + { + "epoch": 0.93, + "learning_rate": 6.080641925777333e-05, + "loss": 1.2423, + "theoretical_loss": 3.3447999812141567, + "tokens_seen": 3052601344 + }, + { + "epoch": 0.93, + "learning_rate": 6.079037111334003e-05, + "loss": 1.1476, + "theoretical_loss": 3.344794526412283, + "tokens_seen": 3052666880 + }, + { + "epoch": 0.93, + "learning_rate": 6.077432296890673e-05, + "loss": 1.2396, + "theoretical_loss": 3.3447890717603026, + "tokens_seen": 3052732416 + }, + { + "epoch": 0.93, + "learning_rate": 6.0758274824473424e-05, + "loss": 1.1819, + "theoretical_loss": 3.3447836172582086, + "tokens_seen": 3052797952 + }, + { + "epoch": 0.93, + "learning_rate": 6.0742226680040124e-05, + "loss": 1.2123, + "theoretical_loss": 3.3447781629059943, + "tokens_seen": 3052863488 + }, + { + "epoch": 0.93, + "learning_rate": 6.0726178535606825e-05, + "loss": 1.1408, + "theoretical_loss": 3.344772708703651, + "tokens_seen": 3052929024 + }, + { + "epoch": 0.93, + "learning_rate": 6.0710130391173526e-05, + "loss": 1.2438, + "theoretical_loss": 3.3447672546511726, + "tokens_seen": 3052994560 + }, + { + "epoch": 0.93, + "learning_rate": 6.0694082246740226e-05, + "loss": 1.1801, + "theoretical_loss": 3.344761800748551, + "tokens_seen": 3053060096 + }, + { + "epoch": 0.93, + "learning_rate": 6.067803410230693e-05, + "loss": 1.2517, + "theoretical_loss": 3.344756346995779, + "tokens_seen": 3053125632 + }, + { + "epoch": 0.93, + "learning_rate": 6.066198595787362e-05, + "loss": 1.176, + "theoretical_loss": 3.34475089339285, + "tokens_seen": 3053191168 + }, + { + "epoch": 0.93, + "learning_rate": 6.064593781344032e-05, + "loss": 1.2213, + "theoretical_loss": 3.3447454399397554, + "tokens_seen": 3053256704 + }, + { + "epoch": 0.93, + "learning_rate": 6.062988966900702e-05, + "loss": 1.183, + "theoretical_loss": 3.344739986636489, + "tokens_seen": 3053322240 + }, + { + "epoch": 0.93, + "learning_rate": 6.061384152457372e-05, + "loss": 1.1874, + "theoretical_loss": 3.3447345334830425, + "tokens_seen": 3053387776 + }, + { + "epoch": 0.93, + "learning_rate": 6.059779338014042e-05, + "loss": 1.1887, + "theoretical_loss": 3.34472908047941, + "tokens_seen": 3053453312 + }, + { + "epoch": 0.93, + "learning_rate": 6.0581745235707124e-05, + "loss": 1.1315, + "theoretical_loss": 3.3447236276255823, + "tokens_seen": 3053518848 + }, + { + "epoch": 0.93, + "learning_rate": 6.056569709127382e-05, + "loss": 1.1378, + "theoretical_loss": 3.344718174921553, + "tokens_seen": 3053584384 + }, + { + "epoch": 0.93, + "learning_rate": 6.054964894684052e-05, + "loss": 1.258, + "theoretical_loss": 3.3447127223673156, + "tokens_seen": 3053649920 + }, + { + "epoch": 0.93, + "learning_rate": 6.053360080240722e-05, + "loss": 1.1877, + "theoretical_loss": 3.3447072699628615, + "tokens_seen": 3053715456 + }, + { + "epoch": 0.93, + "learning_rate": 6.051755265797392e-05, + "loss": 1.2115, + "theoretical_loss": 3.3447018177081835, + "tokens_seen": 3053780992 + }, + { + "epoch": 0.93, + "learning_rate": 6.050150451354063e-05, + "loss": 1.1843, + "theoretical_loss": 3.3446963656032747, + "tokens_seen": 3053846528 + }, + { + "epoch": 0.93, + "learning_rate": 6.048545636910733e-05, + "loss": 1.2211, + "theoretical_loss": 3.3446909136481278, + "tokens_seen": 3053912064 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1018719, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3165093660354614, + "objective/train/theoretical_loss": 3.3446854618427353, + "objective/train/tokens_used": 3074437600, + "theoretical_loss": 3.3446854618427353, + "tokens_seen": 3053977600 + }, + { + "epoch": 0.93, + "learning_rate": 6.046940822467403e-05, + "loss": 1.2439, + "theoretical_loss": 3.3446854618427353, + "tokens_seen": 3053977600 + }, + { + "epoch": 0.93, + "learning_rate": 6.045336008024073e-05, + "loss": 1.0692, + "theoretical_loss": 3.34468001018709, + "tokens_seen": 3054043136 + }, + { + "epoch": 0.93, + "learning_rate": 6.043731193580743e-05, + "loss": 1.2788, + "theoretical_loss": 3.3446745586811844, + "tokens_seen": 3054108672 + }, + { + "epoch": 0.93, + "learning_rate": 6.042126379137413e-05, + "loss": 1.1966, + "theoretical_loss": 3.344669107325011, + "tokens_seen": 3054174208 + }, + { + "epoch": 0.93, + "learning_rate": 6.0405215646940824e-05, + "loss": 1.1498, + "theoretical_loss": 3.344663656118563, + "tokens_seen": 3054239744 + }, + { + "epoch": 0.93, + "learning_rate": 6.0389167502507525e-05, + "loss": 1.2082, + "theoretical_loss": 3.344658205061833, + "tokens_seen": 3054305280 + }, + { + "epoch": 0.93, + "learning_rate": 6.0373119358074225e-05, + "loss": 1.1813, + "theoretical_loss": 3.3446527541548132, + "tokens_seen": 3054370816 + }, + { + "epoch": 0.93, + "learning_rate": 6.0357071213640926e-05, + "loss": 1.1292, + "theoretical_loss": 3.3446473033974966, + "tokens_seen": 3054436352 + }, + { + "epoch": 0.93, + "learning_rate": 6.0341023069207627e-05, + "loss": 1.213, + "theoretical_loss": 3.344641852789876, + "tokens_seen": 3054501888 + }, + { + "epoch": 0.93, + "learning_rate": 6.032497492477433e-05, + "loss": 1.1687, + "theoretical_loss": 3.3446364023319437, + "tokens_seen": 3054567424 + }, + { + "epoch": 0.93, + "learning_rate": 6.030892678034102e-05, + "loss": 1.1719, + "theoretical_loss": 3.3446309520236923, + "tokens_seen": 3054632960 + }, + { + "epoch": 0.93, + "learning_rate": 6.029287863590772e-05, + "loss": 1.2355, + "theoretical_loss": 3.3446255018651154, + "tokens_seen": 3054698496 + }, + { + "epoch": 0.93, + "learning_rate": 6.027683049147442e-05, + "loss": 1.1774, + "theoretical_loss": 3.3446200518562046, + "tokens_seen": 3054764032 + }, + { + "epoch": 0.93, + "learning_rate": 6.026078234704112e-05, + "loss": 1.2142, + "theoretical_loss": 3.344614601996953, + "tokens_seen": 3054829568 + }, + { + "epoch": 0.93, + "learning_rate": 6.0244734202607824e-05, + "loss": 1.1601, + "theoretical_loss": 3.3446091522873536, + "tokens_seen": 3054895104 + }, + { + "epoch": 0.93, + "learning_rate": 6.0228686058174524e-05, + "loss": 1.134, + "theoretical_loss": 3.3446037027273987, + "tokens_seen": 3054960640 + }, + { + "epoch": 0.93, + "learning_rate": 6.021263791374122e-05, + "loss": 1.1363, + "theoretical_loss": 3.344598253317081, + "tokens_seen": 3055026176 + }, + { + "epoch": 0.93, + "learning_rate": 6.019658976930793e-05, + "loss": 1.2222, + "theoretical_loss": 3.3445928040563935, + "tokens_seen": 3055091712 + }, + { + "epoch": 0.93, + "learning_rate": 6.018054162487463e-05, + "loss": 1.2161, + "theoretical_loss": 3.344587354945328, + "tokens_seen": 3055157248 + }, + { + "epoch": 0.93, + "learning_rate": 6.0164493480441334e-05, + "loss": 1.3042, + "theoretical_loss": 3.3445819059838784, + "tokens_seen": 3055222784 + }, + { + "epoch": 0.93, + "learning_rate": 6.014844533600803e-05, + "loss": 1.169, + "theoretical_loss": 3.3445764571720367, + "tokens_seen": 3055288320 + }, + { + "epoch": 0.93, + "learning_rate": 6.013239719157473e-05, + "loss": 1.2401, + "theoretical_loss": 3.3445710085097953, + "tokens_seen": 3055353856 + }, + { + "epoch": 0.93, + "learning_rate": 6.011634904714143e-05, + "loss": 1.1802, + "theoretical_loss": 3.3445655599971476, + "tokens_seen": 3055419392 + }, + { + "epoch": 0.93, + "learning_rate": 6.010030090270813e-05, + "loss": 1.2511, + "theoretical_loss": 3.344560111634086, + "tokens_seen": 3055484928 + }, + { + "epoch": 0.93, + "learning_rate": 6.008425275827483e-05, + "loss": 1.2066, + "theoretical_loss": 3.344554663420603, + "tokens_seen": 3055550464 + }, + { + "epoch": 0.93, + "learning_rate": 6.006820461384153e-05, + "loss": 1.1922, + "theoretical_loss": 3.3445492153566914, + "tokens_seen": 3055616000 + }, + { + "epoch": 0.93, + "learning_rate": 6.0052156469408224e-05, + "loss": 1.1216, + "theoretical_loss": 3.3445437674423437, + "tokens_seen": 3055681536 + }, + { + "epoch": 0.93, + "learning_rate": 6.0036108324974925e-05, + "loss": 1.2359, + "theoretical_loss": 3.344538319677553, + "tokens_seen": 3055747072 + }, + { + "epoch": 0.93, + "learning_rate": 6.0020060180541626e-05, + "loss": 1.34, + "theoretical_loss": 3.3445328720623118, + "tokens_seen": 3055812608 + }, + { + "epoch": 0.93, + "learning_rate": 6.0004012036108326e-05, + "loss": 1.2053, + "theoretical_loss": 3.3445274245966123, + "tokens_seen": 3055878144 + }, + { + "epoch": 0.93, + "learning_rate": 5.998796389167503e-05, + "loss": 1.149, + "theoretical_loss": 3.3445219772804484, + "tokens_seen": 3055943680 + }, + { + "epoch": 0.93, + "learning_rate": 5.997191574724173e-05, + "loss": 1.0948, + "theoretical_loss": 3.3445165301138116, + "tokens_seen": 3056009216 + }, + { + "epoch": 0.93, + "learning_rate": 5.995586760280842e-05, + "loss": 1.1294, + "theoretical_loss": 3.344511083096695, + "tokens_seen": 3056074752 + }, + { + "epoch": 0.93, + "learning_rate": 5.993981945837512e-05, + "loss": 1.2577, + "theoretical_loss": 3.3445056362290915, + "tokens_seen": 3056140288 + }, + { + "epoch": 0.93, + "learning_rate": 5.992377131394182e-05, + "loss": 1.2259, + "theoretical_loss": 3.3445001895109936, + "tokens_seen": 3056205824 + }, + { + "epoch": 0.93, + "learning_rate": 5.990772316950854e-05, + "loss": 1.1612, + "theoretical_loss": 3.344494742942394, + "tokens_seen": 3056271360 + }, + { + "epoch": 0.93, + "learning_rate": 5.989167502507523e-05, + "loss": 1.2187, + "theoretical_loss": 3.3444892965232853, + "tokens_seen": 3056336896 + }, + { + "epoch": 0.93, + "learning_rate": 5.987562688064193e-05, + "loss": 1.2369, + "theoretical_loss": 3.3444838502536602, + "tokens_seen": 3056402432 + }, + { + "epoch": 0.93, + "learning_rate": 5.985957873620863e-05, + "loss": 1.2324, + "theoretical_loss": 3.3444784041335116, + "tokens_seen": 3056467968 + }, + { + "epoch": 0.93, + "learning_rate": 5.984353059177533e-05, + "loss": 1.1988, + "theoretical_loss": 3.344472958162832, + "tokens_seen": 3056533504 + }, + { + "epoch": 0.93, + "learning_rate": 5.982748244734203e-05, + "loss": 1.2375, + "theoretical_loss": 3.344467512341614, + "tokens_seen": 3056599040 + }, + { + "epoch": 0.93, + "learning_rate": 5.9811434302908734e-05, + "loss": 1.1623, + "theoretical_loss": 3.3444620666698506, + "tokens_seen": 3056664576 + }, + { + "epoch": 0.93, + "learning_rate": 5.979538615847543e-05, + "loss": 1.2251, + "theoretical_loss": 3.3444566211475344, + "tokens_seen": 3056730112 + }, + { + "epoch": 0.93, + "learning_rate": 5.977933801404213e-05, + "loss": 1.233, + "theoretical_loss": 3.3444511757746582, + "tokens_seen": 3056795648 + }, + { + "epoch": 0.93, + "learning_rate": 5.976328986960883e-05, + "loss": 1.2095, + "theoretical_loss": 3.3444457305512145, + "tokens_seen": 3056861184 + }, + { + "epoch": 0.93, + "learning_rate": 5.974724172517553e-05, + "loss": 1.1644, + "theoretical_loss": 3.3444402854771957, + "tokens_seen": 3056926720 + }, + { + "epoch": 0.93, + "learning_rate": 5.973119358074223e-05, + "loss": 1.1798, + "theoretical_loss": 3.3444348405525948, + "tokens_seen": 3056992256 + }, + { + "epoch": 0.93, + "learning_rate": 5.971514543630893e-05, + "loss": 1.2311, + "theoretical_loss": 3.3444293957774045, + "tokens_seen": 3057057792 + }, + { + "epoch": 0.93, + "learning_rate": 5.9699097291875625e-05, + "loss": 1.211, + "theoretical_loss": 3.344423951151618, + "tokens_seen": 3057123328 + }, + { + "epoch": 0.93, + "learning_rate": 5.9683049147442326e-05, + "loss": 1.1097, + "theoretical_loss": 3.344418506675227, + "tokens_seen": 3057188864 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1019718, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0997182130813599, + "objective/train/theoretical_loss": 3.344413062348225, + "objective/train/tokens_used": 3077714400, + "theoretical_loss": 3.344413062348225, + "tokens_seen": 3057254400 + }, + { + "epoch": 0.93, + "learning_rate": 5.9667001003009026e-05, + "loss": 1.2005, + "theoretical_loss": 3.344413062348225, + "tokens_seen": 3057254400 + }, + { + "epoch": 0.93, + "learning_rate": 5.965095285857573e-05, + "loss": 1.1644, + "theoretical_loss": 3.344407618170605, + "tokens_seen": 3057319936 + }, + { + "epoch": 0.93, + "learning_rate": 5.963490471414243e-05, + "loss": 1.2715, + "theoretical_loss": 3.344402174142358, + "tokens_seen": 3057385472 + }, + { + "epoch": 0.93, + "learning_rate": 5.961885656970913e-05, + "loss": 1.1617, + "theoretical_loss": 3.3443967302634783, + "tokens_seen": 3057451008 + }, + { + "epoch": 0.93, + "learning_rate": 5.9602808425275835e-05, + "loss": 1.3596, + "theoretical_loss": 3.3443912865339582, + "tokens_seen": 3057516544 + }, + { + "epoch": 0.93, + "learning_rate": 5.9586760280842536e-05, + "loss": 1.1863, + "theoretical_loss": 3.3443858429537903, + "tokens_seen": 3057582080 + }, + { + "epoch": 0.93, + "learning_rate": 5.957071213640924e-05, + "loss": 1.1587, + "theoretical_loss": 3.3443803995229673, + "tokens_seen": 3057647616 + }, + { + "epoch": 0.93, + "learning_rate": 5.955466399197594e-05, + "loss": 1.1839, + "theoretical_loss": 3.3443749562414817, + "tokens_seen": 3057713152 + }, + { + "epoch": 0.93, + "learning_rate": 5.953861584754263e-05, + "loss": 1.3163, + "theoretical_loss": 3.3443695131093265, + "tokens_seen": 3057778688 + }, + { + "epoch": 0.93, + "learning_rate": 5.952256770310933e-05, + "loss": 1.2154, + "theoretical_loss": 3.3443640701264945, + "tokens_seen": 3057844224 + }, + { + "epoch": 0.93, + "learning_rate": 5.950651955867603e-05, + "loss": 1.1786, + "theoretical_loss": 3.344358627292978, + "tokens_seen": 3057909760 + }, + { + "epoch": 0.93, + "learning_rate": 5.949047141424273e-05, + "loss": 1.1591, + "theoretical_loss": 3.34435318460877, + "tokens_seen": 3057975296 + }, + { + "epoch": 0.93, + "learning_rate": 5.9474423269809434e-05, + "loss": 1.1052, + "theoretical_loss": 3.3443477420738636, + "tokens_seen": 3058040832 + }, + { + "epoch": 0.93, + "learning_rate": 5.9458375125376134e-05, + "loss": 1.1857, + "theoretical_loss": 3.3443422996882504, + "tokens_seen": 3058106368 + }, + { + "epoch": 0.93, + "learning_rate": 5.944232698094283e-05, + "loss": 1.167, + "theoretical_loss": 3.344336857451924, + "tokens_seen": 3058171904 + }, + { + "epoch": 0.93, + "learning_rate": 5.942627883650953e-05, + "loss": 1.3157, + "theoretical_loss": 3.344331415364877, + "tokens_seen": 3058237440 + }, + { + "epoch": 0.93, + "learning_rate": 5.941023069207623e-05, + "loss": 1.0867, + "theoretical_loss": 3.3443259734271016, + "tokens_seen": 3058302976 + }, + { + "epoch": 0.93, + "learning_rate": 5.939418254764293e-05, + "loss": 1.2896, + "theoretical_loss": 3.344320531638591, + "tokens_seen": 3058368512 + }, + { + "epoch": 0.93, + "learning_rate": 5.937813440320963e-05, + "loss": 1.1946, + "theoretical_loss": 3.344315089999338, + "tokens_seen": 3058434048 + }, + { + "epoch": 0.93, + "learning_rate": 5.936208625877633e-05, + "loss": 1.1049, + "theoretical_loss": 3.344309648509335, + "tokens_seen": 3058499584 + }, + { + "epoch": 0.93, + "learning_rate": 5.9346038114343025e-05, + "loss": 1.1827, + "theoretical_loss": 3.3443042071685745, + "tokens_seen": 3058565120 + }, + { + "epoch": 0.93, + "learning_rate": 5.9329989969909726e-05, + "loss": 1.3104, + "theoretical_loss": 3.3442987659770496, + "tokens_seen": 3058630656 + }, + { + "epoch": 0.93, + "learning_rate": 5.9313941825476427e-05, + "loss": 1.1619, + "theoretical_loss": 3.3442933249347533, + "tokens_seen": 3058696192 + }, + { + "epoch": 0.93, + "learning_rate": 5.929789368104314e-05, + "loss": 1.187, + "theoretical_loss": 3.3442878840416776, + "tokens_seen": 3058761728 + }, + { + "epoch": 0.93, + "learning_rate": 5.9281845536609835e-05, + "loss": 1.2273, + "theoretical_loss": 3.344282443297815, + "tokens_seen": 3058827264 + }, + { + "epoch": 0.93, + "learning_rate": 5.9265797392176535e-05, + "loss": 1.1464, + "theoretical_loss": 3.3442770027031594, + "tokens_seen": 3058892800 + }, + { + "epoch": 0.93, + "learning_rate": 5.9249749247743236e-05, + "loss": 1.107, + "theoretical_loss": 3.344271562257703, + "tokens_seen": 3058958336 + }, + { + "epoch": 0.93, + "learning_rate": 5.9233701103309936e-05, + "loss": 1.2013, + "theoretical_loss": 3.344266121961438, + "tokens_seen": 3059023872 + }, + { + "epoch": 0.93, + "learning_rate": 5.921765295887664e-05, + "loss": 1.2067, + "theoretical_loss": 3.3442606818143576, + "tokens_seen": 3059089408 + }, + { + "epoch": 0.93, + "learning_rate": 5.920160481444334e-05, + "loss": 1.2124, + "theoretical_loss": 3.3442552418164544, + "tokens_seen": 3059154944 + }, + { + "epoch": 0.93, + "learning_rate": 5.918555667001003e-05, + "loss": 1.2352, + "theoretical_loss": 3.344249801967721, + "tokens_seen": 3059220480 + }, + { + "epoch": 0.93, + "learning_rate": 5.916950852557673e-05, + "loss": 1.2089, + "theoretical_loss": 3.3442443622681504, + "tokens_seen": 3059286016 + }, + { + "epoch": 0.93, + "learning_rate": 5.915346038114343e-05, + "loss": 1.1534, + "theoretical_loss": 3.344238922717735, + "tokens_seen": 3059351552 + }, + { + "epoch": 0.93, + "learning_rate": 5.9137412236710134e-05, + "loss": 1.2577, + "theoretical_loss": 3.3442334833164677, + "tokens_seen": 3059417088 + }, + { + "epoch": 0.93, + "learning_rate": 5.9121364092276834e-05, + "loss": 1.2225, + "theoretical_loss": 3.344228044064341, + "tokens_seen": 3059482624 + }, + { + "epoch": 0.93, + "learning_rate": 5.9105315947843535e-05, + "loss": 1.2026, + "theoretical_loss": 3.344222604961348, + "tokens_seen": 3059548160 + }, + { + "epoch": 0.93, + "learning_rate": 5.908926780341023e-05, + "loss": 1.1826, + "theoretical_loss": 3.344217166007481, + "tokens_seen": 3059613696 + }, + { + "epoch": 0.93, + "learning_rate": 5.907321965897693e-05, + "loss": 1.1948, + "theoretical_loss": 3.344211727202733, + "tokens_seen": 3059679232 + }, + { + "epoch": 0.93, + "learning_rate": 5.905717151454363e-05, + "loss": 1.1727, + "theoretical_loss": 3.3442062885470967, + "tokens_seen": 3059744768 + }, + { + "epoch": 0.93, + "learning_rate": 5.904112337011033e-05, + "loss": 1.1777, + "theoretical_loss": 3.3442008500405644, + "tokens_seen": 3059810304 + }, + { + "epoch": 0.93, + "learning_rate": 5.902507522567703e-05, + "loss": 1.1036, + "theoretical_loss": 3.3441954116831294, + "tokens_seen": 3059875840 + }, + { + "epoch": 0.93, + "learning_rate": 5.900902708124373e-05, + "loss": 1.3401, + "theoretical_loss": 3.3441899734747844, + "tokens_seen": 3059941376 + }, + { + "epoch": 0.93, + "learning_rate": 5.899297893681044e-05, + "loss": 1.2117, + "theoretical_loss": 3.3441845354155215, + "tokens_seen": 3060006912 + }, + { + "epoch": 0.93, + "learning_rate": 5.897693079237714e-05, + "loss": 1.2838, + "theoretical_loss": 3.344179097505334, + "tokens_seen": 3060072448 + }, + { + "epoch": 0.93, + "learning_rate": 5.896088264794384e-05, + "loss": 1.302, + "theoretical_loss": 3.344173659744215, + "tokens_seen": 3060137984 + }, + { + "epoch": 0.93, + "learning_rate": 5.8944834503510534e-05, + "loss": 1.2556, + "theoretical_loss": 3.344168222132156, + "tokens_seen": 3060203520 + }, + { + "epoch": 0.93, + "learning_rate": 5.8928786359077235e-05, + "loss": 1.1499, + "theoretical_loss": 3.3441627846691504, + "tokens_seen": 3060269056 + }, + { + "epoch": 0.93, + "learning_rate": 5.8912738214643936e-05, + "loss": 1.2775, + "theoretical_loss": 3.3441573473551913, + "tokens_seen": 3060334592 + }, + { + "epoch": 0.93, + "learning_rate": 5.8896690070210636e-05, + "loss": 1.1034, + "theoretical_loss": 3.344151910190271, + "tokens_seen": 3060400128 + }, + { + "epoch": 0.93, + "learning_rate": 5.888064192577734e-05, + "loss": 1.3319, + "theoretical_loss": 3.3441464731743817, + "tokens_seen": 3060465664 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1020954, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2900712490081787, + "objective/train/theoretical_loss": 3.344141036307517, + "objective/train/tokens_used": 3080991200, + "theoretical_loss": 3.344141036307517, + "tokens_seen": 3060531200 + }, + { + "epoch": 0.93, + "learning_rate": 5.886459378134404e-05, + "loss": 1.3049, + "theoretical_loss": 3.344141036307517, + "tokens_seen": 3060531200 + }, + { + "epoch": 0.93, + "learning_rate": 5.884854563691073e-05, + "loss": 1.2438, + "theoretical_loss": 3.3441355995896695, + "tokens_seen": 3060596736 + }, + { + "epoch": 0.93, + "learning_rate": 5.883249749247743e-05, + "loss": 1.0585, + "theoretical_loss": 3.3441301630208313, + "tokens_seen": 3060662272 + }, + { + "epoch": 0.93, + "learning_rate": 5.881644934804413e-05, + "loss": 1.2233, + "theoretical_loss": 3.344124726600996, + "tokens_seen": 3060727808 + }, + { + "epoch": 0.93, + "learning_rate": 5.880040120361083e-05, + "loss": 1.2376, + "theoretical_loss": 3.344119290330156, + "tokens_seen": 3060793344 + }, + { + "epoch": 0.93, + "learning_rate": 5.8784353059177534e-05, + "loss": 1.1817, + "theoretical_loss": 3.3441138542083033, + "tokens_seen": 3060858880 + }, + { + "epoch": 0.93, + "learning_rate": 5.8768304914744235e-05, + "loss": 1.1863, + "theoretical_loss": 3.344108418235432, + "tokens_seen": 3060924416 + }, + { + "epoch": 0.93, + "learning_rate": 5.875225677031093e-05, + "loss": 1.2606, + "theoretical_loss": 3.3441029824115334, + "tokens_seen": 3060989952 + }, + { + "epoch": 0.93, + "learning_rate": 5.873620862587763e-05, + "loss": 1.1963, + "theoretical_loss": 3.344097546736601, + "tokens_seen": 3061055488 + }, + { + "epoch": 0.93, + "learning_rate": 5.872016048144433e-05, + "loss": 1.3474, + "theoretical_loss": 3.3440921112106277, + "tokens_seen": 3061121024 + }, + { + "epoch": 0.93, + "learning_rate": 5.8704112337011044e-05, + "loss": 1.1867, + "theoretical_loss": 3.344086675833606, + "tokens_seen": 3061186560 + }, + { + "epoch": 0.93, + "learning_rate": 5.868806419257774e-05, + "loss": 1.1629, + "theoretical_loss": 3.344081240605528, + "tokens_seen": 3061252096 + }, + { + "epoch": 0.93, + "learning_rate": 5.867201604814444e-05, + "loss": 1.0855, + "theoretical_loss": 3.3440758055263875, + "tokens_seen": 3061317632 + }, + { + "epoch": 0.93, + "learning_rate": 5.865596790371114e-05, + "loss": 1.2581, + "theoretical_loss": 3.3440703705961767, + "tokens_seen": 3061383168 + }, + { + "epoch": 0.93, + "learning_rate": 5.863991975927784e-05, + "loss": 1.1682, + "theoretical_loss": 3.344064935814888, + "tokens_seen": 3061448704 + }, + { + "epoch": 0.93, + "learning_rate": 5.862387161484454e-05, + "loss": 1.2159, + "theoretical_loss": 3.344059501182515, + "tokens_seen": 3061514240 + }, + { + "epoch": 0.93, + "learning_rate": 5.860782347041124e-05, + "loss": 1.1801, + "theoretical_loss": 3.3440540666990497, + "tokens_seen": 3061579776 + }, + { + "epoch": 0.93, + "learning_rate": 5.8591775325977935e-05, + "loss": 1.2056, + "theoretical_loss": 3.344048632364485, + "tokens_seen": 3061645312 + }, + { + "epoch": 0.93, + "learning_rate": 5.8575727181544635e-05, + "loss": 1.1541, + "theoretical_loss": 3.344043198178814, + "tokens_seen": 3061710848 + }, + { + "epoch": 0.93, + "learning_rate": 5.8559679037111336e-05, + "loss": 1.1866, + "theoretical_loss": 3.344037764142029, + "tokens_seen": 3061776384 + }, + { + "epoch": 0.93, + "learning_rate": 5.854363089267804e-05, + "loss": 1.2229, + "theoretical_loss": 3.3440323302541226, + "tokens_seen": 3061841920 + }, + { + "epoch": 0.93, + "learning_rate": 5.852758274824474e-05, + "loss": 1.2407, + "theoretical_loss": 3.3440268965150883, + "tokens_seen": 3061907456 + }, + { + "epoch": 0.93, + "learning_rate": 5.851153460381144e-05, + "loss": 1.1711, + "theoretical_loss": 3.3440214629249176, + "tokens_seen": 3061972992 + }, + { + "epoch": 0.93, + "learning_rate": 5.849548645937813e-05, + "loss": 1.1703, + "theoretical_loss": 3.3440160294836048, + "tokens_seen": 3062038528 + }, + { + "epoch": 0.93, + "learning_rate": 5.847943831494483e-05, + "loss": 1.1812, + "theoretical_loss": 3.3440105961911413, + "tokens_seen": 3062104064 + }, + { + "epoch": 0.93, + "learning_rate": 5.846339017051153e-05, + "loss": 1.2575, + "theoretical_loss": 3.3440051630475205, + "tokens_seen": 3062169600 + }, + { + "epoch": 0.93, + "learning_rate": 5.8447342026078234e-05, + "loss": 1.2475, + "theoretical_loss": 3.343999730052735, + "tokens_seen": 3062235136 + }, + { + "epoch": 0.93, + "learning_rate": 5.8431293881644934e-05, + "loss": 1.1089, + "theoretical_loss": 3.3439942972067773, + "tokens_seen": 3062300672 + }, + { + "epoch": 0.93, + "learning_rate": 5.8415245737211635e-05, + "loss": 1.1434, + "theoretical_loss": 3.3439888645096407, + "tokens_seen": 3062366208 + }, + { + "epoch": 0.93, + "learning_rate": 5.839919759277834e-05, + "loss": 1.1149, + "theoretical_loss": 3.343983431961317, + "tokens_seen": 3062431744 + }, + { + "epoch": 0.93, + "learning_rate": 5.838314944834504e-05, + "loss": 1.2458, + "theoretical_loss": 3.3439779995618, + "tokens_seen": 3062497280 + }, + { + "epoch": 0.93, + "learning_rate": 5.8367101303911744e-05, + "loss": 1.2256, + "theoretical_loss": 3.343972567311082, + "tokens_seen": 3062562816 + }, + { + "epoch": 0.93, + "learning_rate": 5.8351053159478444e-05, + "loss": 1.2808, + "theoretical_loss": 3.343967135209156, + "tokens_seen": 3062628352 + }, + { + "epoch": 0.93, + "learning_rate": 5.833500501504514e-05, + "loss": 1.2854, + "theoretical_loss": 3.343961703256014, + "tokens_seen": 3062693888 + }, + { + "epoch": 0.93, + "learning_rate": 5.831895687061184e-05, + "loss": 1.1219, + "theoretical_loss": 3.343956271451649, + "tokens_seen": 3062759424 + }, + { + "epoch": 0.93, + "learning_rate": 5.830290872617854e-05, + "loss": 1.2164, + "theoretical_loss": 3.3439508397960545, + "tokens_seen": 3062824960 + }, + { + "epoch": 0.93, + "learning_rate": 5.828686058174524e-05, + "loss": 1.2404, + "theoretical_loss": 3.3439454082892226, + "tokens_seen": 3062890496 + }, + { + "epoch": 0.93, + "learning_rate": 5.827081243731194e-05, + "loss": 1.1879, + "theoretical_loss": 3.343939976931146, + "tokens_seen": 3062956032 + }, + { + "epoch": 0.93, + "learning_rate": 5.825476429287864e-05, + "loss": 1.1453, + "theoretical_loss": 3.3439345457218175, + "tokens_seen": 3063021568 + }, + { + "epoch": 0.93, + "learning_rate": 5.8238716148445335e-05, + "loss": 1.2253, + "theoretical_loss": 3.34392911466123, + "tokens_seen": 3063087104 + }, + { + "epoch": 0.93, + "learning_rate": 5.8222668004012036e-05, + "loss": 1.0852, + "theoretical_loss": 3.343923683749376, + "tokens_seen": 3063152640 + }, + { + "epoch": 0.93, + "learning_rate": 5.8206619859578736e-05, + "loss": 1.2345, + "theoretical_loss": 3.3439182529862483, + "tokens_seen": 3063218176 + }, + { + "epoch": 0.93, + "learning_rate": 5.819057171514544e-05, + "loss": 1.1918, + "theoretical_loss": 3.34391282237184, + "tokens_seen": 3063283712 + }, + { + "epoch": 0.93, + "learning_rate": 5.817452357071214e-05, + "loss": 1.2852, + "theoretical_loss": 3.3439073919061437, + "tokens_seen": 3063349248 + }, + { + "epoch": 0.93, + "learning_rate": 5.815847542627884e-05, + "loss": 1.129, + "theoretical_loss": 3.3439019615891517, + "tokens_seen": 3063414784 + }, + { + "epoch": 0.93, + "learning_rate": 5.814242728184553e-05, + "loss": 1.2079, + "theoretical_loss": 3.3438965314208575, + "tokens_seen": 3063480320 + }, + { + "epoch": 0.93, + "learning_rate": 5.812637913741223e-05, + "loss": 1.2285, + "theoretical_loss": 3.343891101401253, + "tokens_seen": 3063545856 + }, + { + "epoch": 0.93, + "learning_rate": 5.8110330992978933e-05, + "loss": 1.1741, + "theoretical_loss": 3.3438856715303316, + "tokens_seen": 3063611392 + }, + { + "epoch": 0.93, + "learning_rate": 5.809428284854565e-05, + "loss": 1.1955, + "theoretical_loss": 3.3438802418080855, + "tokens_seen": 3063676928 + }, + { + "epoch": 0.93, + "learning_rate": 5.807823470411234e-05, + "loss": 1.0644, + "theoretical_loss": 3.3438748122345086, + "tokens_seen": 3063742464 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1022167, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.264589786529541, + "objective/train/theoretical_loss": 3.343869382809592, + "objective/train/tokens_used": 3084268000, + "theoretical_loss": 3.343869382809592, + "tokens_seen": 3063808000 + }, + { + "epoch": 0.93, + "learning_rate": 5.806218655967904e-05, + "loss": 1.2341, + "theoretical_loss": 3.343869382809592, + "tokens_seen": 3063808000 + }, + { + "epoch": 0.93, + "learning_rate": 5.804613841524574e-05, + "loss": 1.0991, + "theoretical_loss": 3.3438639535333294, + "tokens_seen": 3063873536 + }, + { + "epoch": 0.93, + "learning_rate": 5.8030090270812443e-05, + "loss": 1.2473, + "theoretical_loss": 3.343858524405714, + "tokens_seen": 3063939072 + }, + { + "epoch": 0.93, + "learning_rate": 5.8014042126379144e-05, + "loss": 1.2059, + "theoretical_loss": 3.3438530954267374, + "tokens_seen": 3064004608 + }, + { + "epoch": 0.93, + "learning_rate": 5.7997993981945845e-05, + "loss": 1.142, + "theoretical_loss": 3.343847666596393, + "tokens_seen": 3064070144 + }, + { + "epoch": 0.93, + "learning_rate": 5.798194583751254e-05, + "loss": 1.2013, + "theoretical_loss": 3.3438422379146737, + "tokens_seen": 3064135680 + }, + { + "epoch": 0.93, + "learning_rate": 5.796589769307924e-05, + "loss": 1.2004, + "theoretical_loss": 3.343836809381572, + "tokens_seen": 3064201216 + }, + { + "epoch": 0.93, + "learning_rate": 5.794984954864594e-05, + "loss": 1.1651, + "theoretical_loss": 3.3438313809970808, + "tokens_seen": 3064266752 + }, + { + "epoch": 0.93, + "learning_rate": 5.793380140421264e-05, + "loss": 1.1793, + "theoretical_loss": 3.3438259527611924, + "tokens_seen": 3064332288 + }, + { + "epoch": 0.93, + "learning_rate": 5.791775325977934e-05, + "loss": 1.2353, + "theoretical_loss": 3.3438205246739, + "tokens_seen": 3064397824 + }, + { + "epoch": 0.93, + "learning_rate": 5.790170511534604e-05, + "loss": 1.1363, + "theoretical_loss": 3.343815096735196, + "tokens_seen": 3064463360 + }, + { + "epoch": 0.93, + "learning_rate": 5.7885656970912736e-05, + "loss": 1.1915, + "theoretical_loss": 3.343809668945074, + "tokens_seen": 3064528896 + }, + { + "epoch": 0.93, + "learning_rate": 5.7869608826479436e-05, + "loss": 1.2626, + "theoretical_loss": 3.343804241303526, + "tokens_seen": 3064594432 + }, + { + "epoch": 0.93, + "learning_rate": 5.785356068204614e-05, + "loss": 1.2451, + "theoretical_loss": 3.343798813810545, + "tokens_seen": 3064659968 + }, + { + "epoch": 0.93, + "learning_rate": 5.783751253761284e-05, + "loss": 1.2229, + "theoretical_loss": 3.343793386466123, + "tokens_seen": 3064725504 + }, + { + "epoch": 0.93, + "learning_rate": 5.782146439317954e-05, + "loss": 1.2043, + "theoretical_loss": 3.343787959270254, + "tokens_seen": 3064791040 + }, + { + "epoch": 0.93, + "learning_rate": 5.7805416248746246e-05, + "loss": 1.3114, + "theoretical_loss": 3.3437825322229298, + "tokens_seen": 3064856576 + }, + { + "epoch": 0.93, + "learning_rate": 5.7789368104312946e-05, + "loss": 1.1629, + "theoretical_loss": 3.343777105324144, + "tokens_seen": 3064922112 + }, + { + "epoch": 0.93, + "learning_rate": 5.777331995987965e-05, + "loss": 1.1338, + "theoretical_loss": 3.343771678573889, + "tokens_seen": 3064987648 + }, + { + "epoch": 0.93, + "learning_rate": 5.775727181544635e-05, + "loss": 1.2569, + "theoretical_loss": 3.343766251972157, + "tokens_seen": 3065053184 + }, + { + "epoch": 0.93, + "learning_rate": 5.774122367101305e-05, + "loss": 1.1529, + "theoretical_loss": 3.3437608255189413, + "tokens_seen": 3065118720 + }, + { + "epoch": 0.93, + "learning_rate": 5.772517552657974e-05, + "loss": 1.1804, + "theoretical_loss": 3.343755399214235, + "tokens_seen": 3065184256 + }, + { + "epoch": 0.93, + "learning_rate": 5.770912738214644e-05, + "loss": 1.2662, + "theoretical_loss": 3.3437499730580296, + "tokens_seen": 3065249792 + }, + { + "epoch": 0.93, + "learning_rate": 5.769307923771314e-05, + "loss": 1.1925, + "theoretical_loss": 3.3437445470503193, + "tokens_seen": 3065315328 + }, + { + "epoch": 0.93, + "learning_rate": 5.7677031093279844e-05, + "loss": 1.1349, + "theoretical_loss": 3.3437391211910965, + "tokens_seen": 3065380864 + }, + { + "epoch": 0.93, + "learning_rate": 5.7660982948846544e-05, + "loss": 1.2917, + "theoretical_loss": 3.343733695480353, + "tokens_seen": 3065446400 + }, + { + "epoch": 0.93, + "learning_rate": 5.7644934804413245e-05, + "loss": 1.1717, + "theoretical_loss": 3.343728269918083, + "tokens_seen": 3065511936 + }, + { + "epoch": 0.93, + "learning_rate": 5.762888665997994e-05, + "loss": 1.0905, + "theoretical_loss": 3.343722844504278, + "tokens_seen": 3065577472 + }, + { + "epoch": 0.93, + "learning_rate": 5.761283851554664e-05, + "loss": 1.1469, + "theoretical_loss": 3.343717419238932, + "tokens_seen": 3065643008 + }, + { + "epoch": 0.93, + "learning_rate": 5.759679037111334e-05, + "loss": 1.1586, + "theoretical_loss": 3.3437119941220366, + "tokens_seen": 3065708544 + }, + { + "epoch": 0.93, + "learning_rate": 5.758074222668004e-05, + "loss": 1.207, + "theoretical_loss": 3.3437065691535848, + "tokens_seen": 3065774080 + }, + { + "epoch": 0.93, + "learning_rate": 5.756469408224674e-05, + "loss": 1.1481, + "theoretical_loss": 3.3437011443335702, + "tokens_seen": 3065839616 + }, + { + "epoch": 0.93, + "learning_rate": 5.754864593781344e-05, + "loss": 1.2225, + "theoretical_loss": 3.3436957196619845, + "tokens_seen": 3065905152 + }, + { + "epoch": 0.93, + "learning_rate": 5.7532597793380136e-05, + "loss": 1.0913, + "theoretical_loss": 3.3436902951388214, + "tokens_seen": 3065970688 + }, + { + "epoch": 0.93, + "learning_rate": 5.751654964894684e-05, + "loss": 1.2569, + "theoretical_loss": 3.343684870764073, + "tokens_seen": 3066036224 + }, + { + "epoch": 0.93, + "learning_rate": 5.750050150451355e-05, + "loss": 1.1265, + "theoretical_loss": 3.343679446537732, + "tokens_seen": 3066101760 + }, + { + "epoch": 0.93, + "learning_rate": 5.748445336008025e-05, + "loss": 1.2057, + "theoretical_loss": 3.3436740224597914, + "tokens_seen": 3066167296 + }, + { + "epoch": 0.93, + "learning_rate": 5.7468405215646945e-05, + "loss": 1.1663, + "theoretical_loss": 3.3436685985302446, + "tokens_seen": 3066232832 + }, + { + "epoch": 0.93, + "learning_rate": 5.7452357071213646e-05, + "loss": 1.2857, + "theoretical_loss": 3.3436631747490835, + "tokens_seen": 3066298368 + }, + { + "epoch": 0.93, + "learning_rate": 5.7436308926780347e-05, + "loss": 1.1587, + "theoretical_loss": 3.343657751116301, + "tokens_seen": 3066363904 + }, + { + "epoch": 0.93, + "learning_rate": 5.742026078234705e-05, + "loss": 1.233, + "theoretical_loss": 3.34365232763189, + "tokens_seen": 3066429440 + }, + { + "epoch": 0.93, + "learning_rate": 5.740421263791375e-05, + "loss": 1.2756, + "theoretical_loss": 3.3436469042958437, + "tokens_seen": 3066494976 + }, + { + "epoch": 0.93, + "learning_rate": 5.738816449348045e-05, + "loss": 1.2324, + "theoretical_loss": 3.343641481108154, + "tokens_seen": 3066560512 + }, + { + "epoch": 0.93, + "learning_rate": 5.737211634904714e-05, + "loss": 1.1093, + "theoretical_loss": 3.3436360580688143, + "tokens_seen": 3066626048 + }, + { + "epoch": 0.93, + "learning_rate": 5.735606820461384e-05, + "loss": 1.2018, + "theoretical_loss": 3.343630635177817, + "tokens_seen": 3066691584 + }, + { + "epoch": 0.93, + "learning_rate": 5.7340020060180544e-05, + "loss": 1.2157, + "theoretical_loss": 3.3436252124351555, + "tokens_seen": 3066757120 + }, + { + "epoch": 0.93, + "learning_rate": 5.7323971915747244e-05, + "loss": 1.2309, + "theoretical_loss": 3.343619789840822, + "tokens_seen": 3066822656 + }, + { + "epoch": 0.93, + "learning_rate": 5.7307923771313945e-05, + "loss": 1.2239, + "theoretical_loss": 3.3436143673948093, + "tokens_seen": 3066888192 + }, + { + "epoch": 0.93, + "learning_rate": 5.7291875626880646e-05, + "loss": 1.1462, + "theoretical_loss": 3.3436089450971105, + "tokens_seen": 3066953728 + }, + { + "epoch": 0.93, + "learning_rate": 5.727582748244734e-05, + "loss": 1.3169, + "theoretical_loss": 3.343603522947718, + "tokens_seen": 3067019264 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1023102, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2192015647888184, + "objective/train/theoretical_loss": 3.3435981009466245, + "objective/train/tokens_used": 3087544800, + "theoretical_loss": 3.3435981009466245, + "tokens_seen": 3067084800 + }, + { + "epoch": 0.93, + "learning_rate": 5.725977933801404e-05, + "loss": 1.2409, + "theoretical_loss": 3.3435981009466245, + "tokens_seen": 3067084800 + }, + { + "epoch": 0.93, + "learning_rate": 5.724373119358074e-05, + "loss": 1.1554, + "theoretical_loss": 3.343592679093823, + "tokens_seen": 3067150336 + }, + { + "epoch": 0.93, + "learning_rate": 5.722768304914744e-05, + "loss": 1.1733, + "theoretical_loss": 3.3435872573893066, + "tokens_seen": 3067215872 + }, + { + "epoch": 0.93, + "learning_rate": 5.721163490471414e-05, + "loss": 1.2143, + "theoretical_loss": 3.343581835833068, + "tokens_seen": 3067281408 + }, + { + "epoch": 0.93, + "learning_rate": 5.719558676028085e-05, + "loss": 1.2196, + "theoretical_loss": 3.3435764144250992, + "tokens_seen": 3067346944 + }, + { + "epoch": 0.93, + "learning_rate": 5.717953861584755e-05, + "loss": 1.1656, + "theoretical_loss": 3.343570993165394, + "tokens_seen": 3067412480 + }, + { + "epoch": 0.93, + "learning_rate": 5.716349047141425e-05, + "loss": 1.1667, + "theoretical_loss": 3.3435655720539446, + "tokens_seen": 3067478016 + }, + { + "epoch": 0.93, + "learning_rate": 5.714744232698095e-05, + "loss": 1.2366, + "theoretical_loss": 3.3435601510907436, + "tokens_seen": 3067543552 + }, + { + "epoch": 0.93, + "learning_rate": 5.7131394182547645e-05, + "loss": 1.1987, + "theoretical_loss": 3.3435547302757844, + "tokens_seen": 3067609088 + }, + { + "epoch": 0.93, + "learning_rate": 5.7115346038114346e-05, + "loss": 1.1818, + "theoretical_loss": 3.3435493096090596, + "tokens_seen": 3067674624 + }, + { + "epoch": 0.93, + "learning_rate": 5.7099297893681046e-05, + "loss": 1.1485, + "theoretical_loss": 3.343543889090561, + "tokens_seen": 3067740160 + }, + { + "epoch": 0.93, + "learning_rate": 5.708324974924775e-05, + "loss": 1.2309, + "theoretical_loss": 3.3435384687202827, + "tokens_seen": 3067805696 + }, + { + "epoch": 0.93, + "learning_rate": 5.706720160481445e-05, + "loss": 1.1552, + "theoretical_loss": 3.3435330484982173, + "tokens_seen": 3067871232 + }, + { + "epoch": 0.93, + "learning_rate": 5.705115346038115e-05, + "loss": 1.2071, + "theoretical_loss": 3.343527628424357, + "tokens_seen": 3067936768 + }, + { + "epoch": 0.93, + "learning_rate": 5.703510531594784e-05, + "loss": 1.2034, + "theoretical_loss": 3.343522208498695, + "tokens_seen": 3068002304 + }, + { + "epoch": 0.93, + "learning_rate": 5.701905717151454e-05, + "loss": 1.2296, + "theoretical_loss": 3.3435167887212236, + "tokens_seen": 3068067840 + }, + { + "epoch": 0.93, + "learning_rate": 5.700300902708124e-05, + "loss": 1.1693, + "theoretical_loss": 3.343511369091936, + "tokens_seen": 3068133376 + }, + { + "epoch": 0.93, + "learning_rate": 5.6986960882647944e-05, + "loss": 1.2553, + "theoretical_loss": 3.343505949610825, + "tokens_seen": 3068198912 + }, + { + "epoch": 0.93, + "learning_rate": 5.6970912738214645e-05, + "loss": 1.1955, + "theoretical_loss": 3.343500530277883, + "tokens_seen": 3068264448 + }, + { + "epoch": 0.93, + "learning_rate": 5.6954864593781345e-05, + "loss": 1.1917, + "theoretical_loss": 3.343495111093103, + "tokens_seen": 3068329984 + }, + { + "epoch": 0.93, + "learning_rate": 5.693881644934804e-05, + "loss": 1.1857, + "theoretical_loss": 3.3434896920564783, + "tokens_seen": 3068395520 + }, + { + "epoch": 0.93, + "learning_rate": 5.692276830491474e-05, + "loss": 1.1799, + "theoretical_loss": 3.3434842731680012, + "tokens_seen": 3068461056 + }, + { + "epoch": 0.93, + "learning_rate": 5.6906720160481454e-05, + "loss": 1.1834, + "theoretical_loss": 3.343478854427664, + "tokens_seen": 3068526592 + }, + { + "epoch": 0.93, + "learning_rate": 5.6890672016048155e-05, + "loss": 1.1637, + "theoretical_loss": 3.3434734358354605, + "tokens_seen": 3068592128 + }, + { + "epoch": 0.93, + "learning_rate": 5.687462387161485e-05, + "loss": 1.2693, + "theoretical_loss": 3.343468017391383, + "tokens_seen": 3068657664 + }, + { + "epoch": 0.93, + "learning_rate": 5.685857572718155e-05, + "loss": 1.1893, + "theoretical_loss": 3.343462599095424, + "tokens_seen": 3068723200 + }, + { + "epoch": 0.93, + "learning_rate": 5.684252758274825e-05, + "loss": 1.233, + "theoretical_loss": 3.343457180947577, + "tokens_seen": 3068788736 + }, + { + "epoch": 0.93, + "learning_rate": 5.682647943831495e-05, + "loss": 1.1808, + "theoretical_loss": 3.3434517629478337, + "tokens_seen": 3068854272 + }, + { + "epoch": 0.93, + "learning_rate": 5.681043129388165e-05, + "loss": 1.2754, + "theoretical_loss": 3.3434463450961878, + "tokens_seen": 3068919808 + }, + { + "epoch": 0.93, + "learning_rate": 5.679438314944835e-05, + "loss": 1.1915, + "theoretical_loss": 3.343440927392632, + "tokens_seen": 3068985344 + }, + { + "epoch": 0.93, + "learning_rate": 5.6778335005015045e-05, + "loss": 1.231, + "theoretical_loss": 3.343435509837159, + "tokens_seen": 3069050880 + }, + { + "epoch": 0.93, + "learning_rate": 5.6762286860581746e-05, + "loss": 1.13, + "theoretical_loss": 3.3434300924297613, + "tokens_seen": 3069116416 + }, + { + "epoch": 0.93, + "learning_rate": 5.674623871614845e-05, + "loss": 1.3233, + "theoretical_loss": 3.343424675170432, + "tokens_seen": 3069181952 + }, + { + "epoch": 0.93, + "learning_rate": 5.673019057171515e-05, + "loss": 1.136, + "theoretical_loss": 3.3434192580591637, + "tokens_seen": 3069247488 + }, + { + "epoch": 0.93, + "learning_rate": 5.671414242728185e-05, + "loss": 1.3231, + "theoretical_loss": 3.3434138410959493, + "tokens_seen": 3069313024 + }, + { + "epoch": 0.93, + "learning_rate": 5.669809428284855e-05, + "loss": 1.2076, + "theoretical_loss": 3.3434084242807818, + "tokens_seen": 3069378560 + }, + { + "epoch": 0.93, + "learning_rate": 5.668204613841524e-05, + "loss": 1.1995, + "theoretical_loss": 3.343403007613653, + "tokens_seen": 3069444096 + }, + { + "epoch": 0.93, + "learning_rate": 5.666599799398194e-05, + "loss": 1.2317, + "theoretical_loss": 3.3433975910945577, + "tokens_seen": 3069509632 + }, + { + "epoch": 0.93, + "learning_rate": 5.6649949849548644e-05, + "loss": 1.1513, + "theoretical_loss": 3.3433921747234865, + "tokens_seen": 3069575168 + }, + { + "epoch": 0.93, + "learning_rate": 5.6633901705115344e-05, + "loss": 1.1653, + "theoretical_loss": 3.3433867585004338, + "tokens_seen": 3069640704 + }, + { + "epoch": 0.93, + "learning_rate": 5.6617853560682045e-05, + "loss": 1.2137, + "theoretical_loss": 3.343381342425391, + "tokens_seen": 3069706240 + }, + { + "epoch": 0.93, + "learning_rate": 5.660180541624875e-05, + "loss": 1.2078, + "theoretical_loss": 3.3433759264983522, + "tokens_seen": 3069771776 + }, + { + "epoch": 0.93, + "learning_rate": 5.658575727181545e-05, + "loss": 1.2585, + "theoretical_loss": 3.3433705107193097, + "tokens_seen": 3069837312 + }, + { + "epoch": 0.93, + "learning_rate": 5.6569709127382154e-05, + "loss": 1.1659, + "theoretical_loss": 3.343365095088256, + "tokens_seen": 3069902848 + }, + { + "epoch": 0.93, + "learning_rate": 5.6553660982948854e-05, + "loss": 1.2817, + "theoretical_loss": 3.343359679605184, + "tokens_seen": 3069968384 + }, + { + "epoch": 0.93, + "learning_rate": 5.6537612838515555e-05, + "loss": 1.2233, + "theoretical_loss": 3.343354264270087, + "tokens_seen": 3070033920 + }, + { + "epoch": 0.93, + "learning_rate": 5.652156469408225e-05, + "loss": 1.1801, + "theoretical_loss": 3.343348849082957, + "tokens_seen": 3070099456 + }, + { + "epoch": 0.93, + "learning_rate": 5.650551654964895e-05, + "loss": 1.2751, + "theoretical_loss": 3.3433434340437875, + "tokens_seen": 3070164992 + }, + { + "epoch": 0.93, + "learning_rate": 5.648946840521565e-05, + "loss": 1.2284, + "theoretical_loss": 3.3433380191525712, + "tokens_seen": 3070230528 + }, + { + "epoch": 0.93, + "learning_rate": 5.647342026078235e-05, + "loss": 1.2147, + "theoretical_loss": 3.3433326044093, + "tokens_seen": 3070296064 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1024459, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2639639377593994, + "objective/train/theoretical_loss": 3.3433271898139685, + "objective/train/tokens_used": 3090821600, + "theoretical_loss": 3.3433271898139685, + "tokens_seen": 3070361600 + }, + { + "epoch": 0.93, + "learning_rate": 5.645737211634905e-05, + "loss": 1.2506, + "theoretical_loss": 3.3433271898139685, + "tokens_seen": 3070361600 + }, + { + "epoch": 0.93, + "learning_rate": 5.644132397191575e-05, + "loss": 1.313, + "theoretical_loss": 3.343321775366568, + "tokens_seen": 3070427136 + }, + { + "epoch": 0.93, + "learning_rate": 5.6425275827482446e-05, + "loss": 1.2314, + "theoretical_loss": 3.343316361067091, + "tokens_seen": 3070492672 + }, + { + "epoch": 0.93, + "learning_rate": 5.6409227683049147e-05, + "loss": 1.1714, + "theoretical_loss": 3.3433109469155315, + "tokens_seen": 3070558208 + }, + { + "epoch": 0.93, + "learning_rate": 5.639317953861585e-05, + "loss": 1.2319, + "theoretical_loss": 3.343305532911882, + "tokens_seen": 3070623744 + }, + { + "epoch": 0.93, + "learning_rate": 5.637713139418255e-05, + "loss": 1.2415, + "theoretical_loss": 3.3433001190561353, + "tokens_seen": 3070689280 + }, + { + "epoch": 0.93, + "learning_rate": 5.636108324974925e-05, + "loss": 1.2124, + "theoretical_loss": 3.343294705348284, + "tokens_seen": 3070754816 + }, + { + "epoch": 0.93, + "learning_rate": 5.634503510531595e-05, + "loss": 1.2458, + "theoretical_loss": 3.3432892917883206, + "tokens_seen": 3070820352 + }, + { + "epoch": 0.93, + "learning_rate": 5.632898696088264e-05, + "loss": 1.1181, + "theoretical_loss": 3.3432838783762384, + "tokens_seen": 3070885888 + }, + { + "epoch": 0.93, + "learning_rate": 5.6312938816449344e-05, + "loss": 1.2494, + "theoretical_loss": 3.34327846511203, + "tokens_seen": 3070951424 + }, + { + "epoch": 0.93, + "learning_rate": 5.629689067201606e-05, + "loss": 1.1627, + "theoretical_loss": 3.3432730519956886, + "tokens_seen": 3071016960 + }, + { + "epoch": 0.93, + "learning_rate": 5.628084252758276e-05, + "loss": 1.3425, + "theoretical_loss": 3.3432676390272063, + "tokens_seen": 3071082496 + }, + { + "epoch": 0.93, + "learning_rate": 5.626479438314945e-05, + "loss": 1.3473, + "theoretical_loss": 3.3432622262065763, + "tokens_seen": 3071148032 + }, + { + "epoch": 0.93, + "learning_rate": 5.624874623871615e-05, + "loss": 1.2162, + "theoretical_loss": 3.3432568135337912, + "tokens_seen": 3071213568 + }, + { + "epoch": 0.93, + "learning_rate": 5.6232698094282854e-05, + "loss": 1.1819, + "theoretical_loss": 3.3432514010088443, + "tokens_seen": 3071279104 + }, + { + "epoch": 0.93, + "learning_rate": 5.6216649949849554e-05, + "loss": 1.2358, + "theoretical_loss": 3.343245988631728, + "tokens_seen": 3071344640 + }, + { + "epoch": 0.93, + "learning_rate": 5.6200601805416255e-05, + "loss": 1.1714, + "theoretical_loss": 3.3432405764024353, + "tokens_seen": 3071410176 + }, + { + "epoch": 0.93, + "learning_rate": 5.6184553660982955e-05, + "loss": 1.1566, + "theoretical_loss": 3.3432351643209586, + "tokens_seen": 3071475712 + }, + { + "epoch": 0.93, + "learning_rate": 5.616850551654965e-05, + "loss": 1.2491, + "theoretical_loss": 3.343229752387291, + "tokens_seen": 3071541248 + }, + { + "epoch": 0.93, + "learning_rate": 5.615245737211635e-05, + "loss": 1.2235, + "theoretical_loss": 3.343224340601426, + "tokens_seen": 3071606784 + }, + { + "epoch": 0.93, + "learning_rate": 5.613640922768305e-05, + "loss": 1.1902, + "theoretical_loss": 3.343218928963355, + "tokens_seen": 3071672320 + }, + { + "epoch": 0.93, + "learning_rate": 5.612036108324975e-05, + "loss": 1.2595, + "theoretical_loss": 3.343213517473072, + "tokens_seen": 3071737856 + }, + { + "epoch": 0.93, + "learning_rate": 5.610431293881645e-05, + "loss": 1.2228, + "theoretical_loss": 3.343208106130569, + "tokens_seen": 3071803392 + }, + { + "epoch": 0.93, + "learning_rate": 5.608826479438315e-05, + "loss": 1.2009, + "theoretical_loss": 3.3432026949358393, + "tokens_seen": 3071868928 + }, + { + "epoch": 0.93, + "learning_rate": 5.6072216649949846e-05, + "loss": 1.1578, + "theoretical_loss": 3.3431972838888755, + "tokens_seen": 3071934464 + }, + { + "epoch": 0.93, + "learning_rate": 5.605616850551655e-05, + "loss": 1.2194, + "theoretical_loss": 3.343191872989671, + "tokens_seen": 3072000000 + }, + { + "epoch": 0.93, + "learning_rate": 5.604012036108325e-05, + "loss": 1.1973, + "theoretical_loss": 3.3431864622382172, + "tokens_seen": 3072065536 + }, + { + "epoch": 0.93, + "learning_rate": 5.602407221664995e-05, + "loss": 1.2689, + "theoretical_loss": 3.3431810516345086, + "tokens_seen": 3072131072 + }, + { + "epoch": 0.93, + "learning_rate": 5.600802407221665e-05, + "loss": 1.1317, + "theoretical_loss": 3.3431756411785365, + "tokens_seen": 3072196608 + }, + { + "epoch": 0.93, + "learning_rate": 5.5991975927783356e-05, + "loss": 1.2263, + "theoretical_loss": 3.343170230870295, + "tokens_seen": 3072262144 + }, + { + "epoch": 0.93, + "learning_rate": 5.597592778335006e-05, + "loss": 1.164, + "theoretical_loss": 3.3431648207097764, + "tokens_seen": 3072327680 + }, + { + "epoch": 0.93, + "learning_rate": 5.595987963891676e-05, + "loss": 1.1978, + "theoretical_loss": 3.3431594106969733, + "tokens_seen": 3072393216 + }, + { + "epoch": 0.93, + "learning_rate": 5.594383149448346e-05, + "loss": 1.272, + "theoretical_loss": 3.3431540008318787, + "tokens_seen": 3072458752 + }, + { + "epoch": 0.93, + "learning_rate": 5.592778335005016e-05, + "loss": 1.2449, + "theoretical_loss": 3.3431485911144856, + "tokens_seen": 3072524288 + }, + { + "epoch": 0.93, + "learning_rate": 5.591173520561685e-05, + "loss": 1.2611, + "theoretical_loss": 3.3431431815447863, + "tokens_seen": 3072589824 + }, + { + "epoch": 0.93, + "learning_rate": 5.589568706118355e-05, + "loss": 1.3412, + "theoretical_loss": 3.343137772122774, + "tokens_seen": 3072655360 + }, + { + "epoch": 0.93, + "learning_rate": 5.5879638916750254e-05, + "loss": 1.1368, + "theoretical_loss": 3.3431323628484417, + "tokens_seen": 3072720896 + }, + { + "epoch": 0.93, + "learning_rate": 5.5863590772316955e-05, + "loss": 1.1329, + "theoretical_loss": 3.343126953721782, + "tokens_seen": 3072786432 + }, + { + "epoch": 0.93, + "learning_rate": 5.5847542627883655e-05, + "loss": 1.1911, + "theoretical_loss": 3.3431215447427873, + "tokens_seen": 3072851968 + }, + { + "epoch": 0.93, + "learning_rate": 5.5831494483450356e-05, + "loss": 1.1471, + "theoretical_loss": 3.3431161359114507, + "tokens_seen": 3072917504 + }, + { + "epoch": 0.93, + "learning_rate": 5.581544633901705e-05, + "loss": 1.1933, + "theoretical_loss": 3.343110727227766, + "tokens_seen": 3072983040 + }, + { + "epoch": 0.93, + "learning_rate": 5.579939819458375e-05, + "loss": 1.124, + "theoretical_loss": 3.3431053186917246, + "tokens_seen": 3073048576 + }, + { + "epoch": 0.93, + "learning_rate": 5.578335005015045e-05, + "loss": 1.2025, + "theoretical_loss": 3.3430999103033194, + "tokens_seen": 3073114112 + }, + { + "epoch": 0.93, + "learning_rate": 5.576730190571715e-05, + "loss": 1.1995, + "theoretical_loss": 3.3430945020625447, + "tokens_seen": 3073179648 + }, + { + "epoch": 0.93, + "learning_rate": 5.575125376128385e-05, + "loss": 1.1714, + "theoretical_loss": 3.343089093969392, + "tokens_seen": 3073245184 + }, + { + "epoch": 0.93, + "learning_rate": 5.573520561685055e-05, + "loss": 1.1908, + "theoretical_loss": 3.343083686023854, + "tokens_seen": 3073310720 + }, + { + "epoch": 0.93, + "learning_rate": 5.571915747241725e-05, + "loss": 1.1741, + "theoretical_loss": 3.3430782782259243, + "tokens_seen": 3073376256 + }, + { + "epoch": 0.93, + "learning_rate": 5.570310932798396e-05, + "loss": 1.1855, + "theoretical_loss": 3.3430728705755954, + "tokens_seen": 3073441792 + }, + { + "epoch": 0.93, + "learning_rate": 5.568706118355066e-05, + "loss": 1.2295, + "theoretical_loss": 3.34306746307286, + "tokens_seen": 3073507328 + }, + { + "epoch": 0.93, + "learning_rate": 5.567101303911736e-05, + "loss": 1.2256, + "theoretical_loss": 3.343062055717711, + "tokens_seen": 3073572864 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1025368, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1682112216949463, + "objective/train/theoretical_loss": 3.3430566485101414, + "objective/train/tokens_used": 3094098400, + "theoretical_loss": 3.3430566485101414, + "tokens_seen": 3073638400 + }, + { + "epoch": 0.93, + "learning_rate": 5.5654964894684056e-05, + "loss": 1.2084, + "theoretical_loss": 3.3430566485101414, + "tokens_seen": 3073638400 + }, + { + "epoch": 0.93, + "learning_rate": 5.563891675025076e-05, + "loss": 1.2342, + "theoretical_loss": 3.3430512414501443, + "tokens_seen": 3073703936 + }, + { + "epoch": 0.93, + "learning_rate": 5.562286860581746e-05, + "loss": 1.1633, + "theoretical_loss": 3.3430458345377114, + "tokens_seen": 3073769472 + }, + { + "epoch": 0.93, + "learning_rate": 5.560682046138416e-05, + "loss": 1.2324, + "theoretical_loss": 3.343040427772837, + "tokens_seen": 3073835008 + }, + { + "epoch": 0.93, + "learning_rate": 5.559077231695086e-05, + "loss": 1.2389, + "theoretical_loss": 3.3430350211555124, + "tokens_seen": 3073900544 + }, + { + "epoch": 0.93, + "learning_rate": 5.557472417251756e-05, + "loss": 1.1563, + "theoretical_loss": 3.3430296146857317, + "tokens_seen": 3073966080 + }, + { + "epoch": 0.93, + "learning_rate": 5.555867602808425e-05, + "loss": 1.2562, + "theoretical_loss": 3.343024208363487, + "tokens_seen": 3074031616 + }, + { + "epoch": 0.93, + "learning_rate": 5.5542627883650954e-05, + "loss": 1.2523, + "theoretical_loss": 3.3430188021887712, + "tokens_seen": 3074097152 + }, + { + "epoch": 0.93, + "learning_rate": 5.5526579739217654e-05, + "loss": 1.2626, + "theoretical_loss": 3.3430133961615778, + "tokens_seen": 3074162688 + }, + { + "epoch": 0.93, + "learning_rate": 5.5510531594784355e-05, + "loss": 1.0933, + "theoretical_loss": 3.3430079902818988, + "tokens_seen": 3074228224 + }, + { + "epoch": 0.93, + "learning_rate": 5.5494483450351056e-05, + "loss": 1.1274, + "theoretical_loss": 3.3430025845497275, + "tokens_seen": 3074293760 + }, + { + "epoch": 0.93, + "learning_rate": 5.5478435305917756e-05, + "loss": 1.1742, + "theoretical_loss": 3.3429971789650565, + "tokens_seen": 3074359296 + }, + { + "epoch": 0.93, + "learning_rate": 5.546238716148445e-05, + "loss": 1.115, + "theoretical_loss": 3.3429917735278787, + "tokens_seen": 3074424832 + }, + { + "epoch": 0.93, + "learning_rate": 5.544633901705115e-05, + "loss": 1.1736, + "theoretical_loss": 3.342986368238187, + "tokens_seen": 3074490368 + }, + { + "epoch": 0.93, + "learning_rate": 5.543029087261785e-05, + "loss": 1.2993, + "theoretical_loss": 3.342980963095974, + "tokens_seen": 3074555904 + }, + { + "epoch": 0.93, + "learning_rate": 5.541424272818455e-05, + "loss": 1.3163, + "theoretical_loss": 3.342975558101233, + "tokens_seen": 3074621440 + }, + { + "epoch": 0.93, + "learning_rate": 5.539819458375126e-05, + "loss": 1.1727, + "theoretical_loss": 3.342970153253956, + "tokens_seen": 3074686976 + }, + { + "epoch": 0.93, + "learning_rate": 5.538214643931796e-05, + "loss": 1.1435, + "theoretical_loss": 3.3429647485541367, + "tokens_seen": 3074752512 + }, + { + "epoch": 0.93, + "learning_rate": 5.536609829488466e-05, + "loss": 1.1977, + "theoretical_loss": 3.3429593440017675, + "tokens_seen": 3074818048 + }, + { + "epoch": 0.93, + "learning_rate": 5.535005015045136e-05, + "loss": 1.2456, + "theoretical_loss": 3.3429539395968417, + "tokens_seen": 3074883584 + }, + { + "epoch": 0.93, + "learning_rate": 5.533400200601806e-05, + "loss": 1.3058, + "theoretical_loss": 3.3429485353393513, + "tokens_seen": 3074949120 + }, + { + "epoch": 0.93, + "learning_rate": 5.531795386158476e-05, + "loss": 1.0952, + "theoretical_loss": 3.34294313122929, + "tokens_seen": 3075014656 + }, + { + "epoch": 0.93, + "learning_rate": 5.5301905717151456e-05, + "loss": 1.1714, + "theoretical_loss": 3.34293772726665, + "tokens_seen": 3075080192 + }, + { + "epoch": 0.93, + "learning_rate": 5.528585757271816e-05, + "loss": 1.1821, + "theoretical_loss": 3.3429323234514245, + "tokens_seen": 3075145728 + }, + { + "epoch": 0.93, + "learning_rate": 5.526980942828486e-05, + "loss": 1.3374, + "theoretical_loss": 3.3429269197836065, + "tokens_seen": 3075211264 + }, + { + "epoch": 0.93, + "learning_rate": 5.525376128385156e-05, + "loss": 1.2761, + "theoretical_loss": 3.3429215162631882, + "tokens_seen": 3075276800 + }, + { + "epoch": 0.93, + "learning_rate": 5.523771313941826e-05, + "loss": 1.2472, + "theoretical_loss": 3.3429161128901628, + "tokens_seen": 3075342336 + }, + { + "epoch": 0.93, + "learning_rate": 5.522166499498496e-05, + "loss": 1.1995, + "theoretical_loss": 3.3429107096645234, + "tokens_seen": 3075407872 + }, + { + "epoch": 0.93, + "learning_rate": 5.5205616850551653e-05, + "loss": 1.203, + "theoretical_loss": 3.342905306586262, + "tokens_seen": 3075473408 + }, + { + "epoch": 0.93, + "learning_rate": 5.5189568706118354e-05, + "loss": 1.2417, + "theoretical_loss": 3.342899903655373, + "tokens_seen": 3075538944 + }, + { + "epoch": 0.93, + "learning_rate": 5.5173520561685055e-05, + "loss": 1.2121, + "theoretical_loss": 3.3428945008718474, + "tokens_seen": 3075604480 + }, + { + "epoch": 0.93, + "learning_rate": 5.5157472417251755e-05, + "loss": 1.2009, + "theoretical_loss": 3.342889098235679, + "tokens_seen": 3075670016 + }, + { + "epoch": 0.93, + "learning_rate": 5.5141424272818456e-05, + "loss": 1.1285, + "theoretical_loss": 3.3428836957468606, + "tokens_seen": 3075735552 + }, + { + "epoch": 0.93, + "learning_rate": 5.512537612838516e-05, + "loss": 1.1972, + "theoretical_loss": 3.342878293405385, + "tokens_seen": 3075801088 + }, + { + "epoch": 0.93, + "learning_rate": 5.510932798395185e-05, + "loss": 1.1872, + "theoretical_loss": 3.3428728912112455, + "tokens_seen": 3075866624 + }, + { + "epoch": 0.93, + "learning_rate": 5.5093279839518565e-05, + "loss": 1.1179, + "theoretical_loss": 3.342867489164434, + "tokens_seen": 3075932160 + }, + { + "epoch": 0.93, + "learning_rate": 5.5077231695085265e-05, + "loss": 1.1217, + "theoretical_loss": 3.342862087264944, + "tokens_seen": 3075997696 + }, + { + "epoch": 0.93, + "learning_rate": 5.506118355065196e-05, + "loss": 1.1278, + "theoretical_loss": 3.3428566855127677, + "tokens_seen": 3076063232 + }, + { + "epoch": 0.93, + "learning_rate": 5.504513540621866e-05, + "loss": 1.1622, + "theoretical_loss": 3.342851283907899, + "tokens_seen": 3076128768 + }, + { + "epoch": 0.93, + "learning_rate": 5.502908726178536e-05, + "loss": 1.2613, + "theoretical_loss": 3.3428458824503298, + "tokens_seen": 3076194304 + }, + { + "epoch": 0.93, + "learning_rate": 5.501303911735206e-05, + "loss": 1.1274, + "theoretical_loss": 3.3428404811400534, + "tokens_seen": 3076259840 + }, + { + "epoch": 0.93, + "learning_rate": 5.499699097291876e-05, + "loss": 1.1356, + "theoretical_loss": 3.3428350799770623, + "tokens_seen": 3076325376 + }, + { + "epoch": 0.93, + "learning_rate": 5.498094282848546e-05, + "loss": 1.2277, + "theoretical_loss": 3.34282967896135, + "tokens_seen": 3076390912 + }, + { + "epoch": 0.93, + "learning_rate": 5.4964894684052156e-05, + "loss": 1.255, + "theoretical_loss": 3.3428242780929085, + "tokens_seen": 3076456448 + }, + { + "epoch": 0.93, + "learning_rate": 5.494884653961886e-05, + "loss": 1.1895, + "theoretical_loss": 3.342818877371731, + "tokens_seen": 3076521984 + }, + { + "epoch": 0.93, + "learning_rate": 5.493279839518556e-05, + "loss": 1.1531, + "theoretical_loss": 3.342813476797811, + "tokens_seen": 3076587520 + }, + { + "epoch": 0.93, + "learning_rate": 5.491675025075226e-05, + "loss": 1.2644, + "theoretical_loss": 3.3428080763711403, + "tokens_seen": 3076653056 + }, + { + "epoch": 0.93, + "learning_rate": 5.490070210631896e-05, + "loss": 1.0767, + "theoretical_loss": 3.3428026760917127, + "tokens_seen": 3076718592 + }, + { + "epoch": 0.93, + "learning_rate": 5.488465396188566e-05, + "loss": 1.1485, + "theoretical_loss": 3.34279727595952, + "tokens_seen": 3076784128 + }, + { + "epoch": 0.93, + "learning_rate": 5.486860581745235e-05, + "loss": 1.0807, + "theoretical_loss": 3.342791875974556, + "tokens_seen": 3076849664 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1026647, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3857640027999878, + "objective/train/theoretical_loss": 3.342786476136813, + "objective/train/tokens_used": 3097375200, + "theoretical_loss": 3.342786476136813, + "tokens_seen": 3076915200 + }, + { + "epoch": 0.93, + "learning_rate": 5.4852557673019054e-05, + "loss": 1.3027, + "theoretical_loss": 3.342786476136813, + "tokens_seen": 3076915200 + }, + { + "epoch": 0.93, + "learning_rate": 5.4836509528585755e-05, + "loss": 1.2497, + "theoretical_loss": 3.342781076446284, + "tokens_seen": 3076980736 + }, + { + "epoch": 0.93, + "learning_rate": 5.4820461384152455e-05, + "loss": 1.204, + "theoretical_loss": 3.3427756769029617, + "tokens_seen": 3077046272 + }, + { + "epoch": 0.93, + "learning_rate": 5.480441323971916e-05, + "loss": 1.2166, + "theoretical_loss": 3.3427702775068395, + "tokens_seen": 3077111808 + }, + { + "epoch": 0.93, + "learning_rate": 5.478836509528586e-05, + "loss": 1.2526, + "theoretical_loss": 3.34276487825791, + "tokens_seen": 3077177344 + }, + { + "epoch": 0.93, + "learning_rate": 5.4772316950852564e-05, + "loss": 1.239, + "theoretical_loss": 3.3427594791561654, + "tokens_seen": 3077242880 + }, + { + "epoch": 0.93, + "learning_rate": 5.4756268806419264e-05, + "loss": 1.1385, + "theoretical_loss": 3.3427540802015994, + "tokens_seen": 3077308416 + }, + { + "epoch": 0.93, + "learning_rate": 5.4740220661985965e-05, + "loss": 1.1047, + "theoretical_loss": 3.3427486813942044, + "tokens_seen": 3077373952 + }, + { + "epoch": 0.93, + "learning_rate": 5.4724172517552666e-05, + "loss": 1.1716, + "theoretical_loss": 3.3427432827339736, + "tokens_seen": 3077439488 + }, + { + "epoch": 0.93, + "learning_rate": 5.470812437311936e-05, + "loss": 1.1895, + "theoretical_loss": 3.3427378842208992, + "tokens_seen": 3077505024 + }, + { + "epoch": 0.93, + "learning_rate": 5.469207622868606e-05, + "loss": 1.1948, + "theoretical_loss": 3.3427324858549747, + "tokens_seen": 3077570560 + }, + { + "epoch": 0.93, + "learning_rate": 5.467602808425276e-05, + "loss": 1.2696, + "theoretical_loss": 3.342727087636193, + "tokens_seen": 3077636096 + }, + { + "epoch": 0.93, + "learning_rate": 5.465997993981946e-05, + "loss": 1.2767, + "theoretical_loss": 3.3427216895645464, + "tokens_seen": 3077701632 + }, + { + "epoch": 0.93, + "learning_rate": 5.464393179538616e-05, + "loss": 1.2411, + "theoretical_loss": 3.3427162916400284, + "tokens_seen": 3077767168 + }, + { + "epoch": 0.93, + "learning_rate": 5.462788365095286e-05, + "loss": 1.1349, + "theoretical_loss": 3.3427108938626313, + "tokens_seen": 3077832704 + }, + { + "epoch": 0.93, + "learning_rate": 5.461183550651956e-05, + "loss": 1.1395, + "theoretical_loss": 3.342705496232348, + "tokens_seen": 3077898240 + }, + { + "epoch": 0.93, + "learning_rate": 5.459578736208626e-05, + "loss": 1.2229, + "theoretical_loss": 3.3427000987491717, + "tokens_seen": 3077963776 + }, + { + "epoch": 0.93, + "learning_rate": 5.457973921765296e-05, + "loss": 1.1007, + "theoretical_loss": 3.3426947014130954, + "tokens_seen": 3078029312 + }, + { + "epoch": 0.93, + "learning_rate": 5.456369107321966e-05, + "loss": 1.2588, + "theoretical_loss": 3.3426893042241113, + "tokens_seen": 3078094848 + }, + { + "epoch": 0.93, + "learning_rate": 5.454764292878636e-05, + "loss": 1.1367, + "theoretical_loss": 3.3426839071822125, + "tokens_seen": 3078160384 + }, + { + "epoch": 0.93, + "learning_rate": 5.453159478435306e-05, + "loss": 1.2238, + "theoretical_loss": 3.342678510287392, + "tokens_seen": 3078225920 + }, + { + "epoch": 0.93, + "learning_rate": 5.4515546639919754e-05, + "loss": 1.2252, + "theoretical_loss": 3.342673113539643, + "tokens_seen": 3078291456 + }, + { + "epoch": 0.93, + "learning_rate": 5.449949849548647e-05, + "loss": 1.065, + "theoretical_loss": 3.3426677169389576, + "tokens_seen": 3078356992 + }, + { + "epoch": 0.93, + "learning_rate": 5.448345035105317e-05, + "loss": 1.258, + "theoretical_loss": 3.3426623204853296, + "tokens_seen": 3078422528 + }, + { + "epoch": 0.93, + "learning_rate": 5.446740220661987e-05, + "loss": 1.2409, + "theoretical_loss": 3.3426569241787507, + "tokens_seen": 3078488064 + }, + { + "epoch": 0.93, + "learning_rate": 5.445135406218656e-05, + "loss": 1.2213, + "theoretical_loss": 3.342651528019215, + "tokens_seen": 3078553600 + }, + { + "epoch": 0.93, + "learning_rate": 5.4435305917753264e-05, + "loss": 1.1334, + "theoretical_loss": 3.3426461320067142, + "tokens_seen": 3078619136 + }, + { + "epoch": 0.93, + "learning_rate": 5.4419257773319964e-05, + "loss": 1.1593, + "theoretical_loss": 3.342640736141242, + "tokens_seen": 3078684672 + }, + { + "epoch": 0.93, + "learning_rate": 5.4403209628886665e-05, + "loss": 1.1197, + "theoretical_loss": 3.342635340422791, + "tokens_seen": 3078750208 + }, + { + "epoch": 0.93, + "learning_rate": 5.4387161484453365e-05, + "loss": 1.1903, + "theoretical_loss": 3.3426299448513537, + "tokens_seen": 3078815744 + }, + { + "epoch": 0.93, + "learning_rate": 5.4371113340020066e-05, + "loss": 1.2514, + "theoretical_loss": 3.3426245494269233, + "tokens_seen": 3078881280 + }, + { + "epoch": 0.93, + "learning_rate": 5.435506519558676e-05, + "loss": 1.2215, + "theoretical_loss": 3.342619154149493, + "tokens_seen": 3078946816 + }, + { + "epoch": 0.93, + "learning_rate": 5.433901705115346e-05, + "loss": 1.1081, + "theoretical_loss": 3.342613759019055, + "tokens_seen": 3079012352 + }, + { + "epoch": 0.93, + "learning_rate": 5.432296890672016e-05, + "loss": 1.2735, + "theoretical_loss": 3.3426083640356032, + "tokens_seen": 3079077888 + }, + { + "epoch": 0.93, + "learning_rate": 5.430692076228686e-05, + "loss": 1.2538, + "theoretical_loss": 3.3426029691991292, + "tokens_seen": 3079143424 + }, + { + "epoch": 0.93, + "learning_rate": 5.429087261785356e-05, + "loss": 1.2115, + "theoretical_loss": 3.3425975745096266, + "tokens_seen": 3079208960 + }, + { + "epoch": 0.93, + "learning_rate": 5.427482447342026e-05, + "loss": 1.2527, + "theoretical_loss": 3.342592179967088, + "tokens_seen": 3079274496 + }, + { + "epoch": 0.93, + "learning_rate": 5.425877632898696e-05, + "loss": 1.1806, + "theoretical_loss": 3.3425867855715063, + "tokens_seen": 3079340032 + }, + { + "epoch": 0.93, + "learning_rate": 5.424272818455366e-05, + "loss": 1.1011, + "theoretical_loss": 3.3425813913228746, + "tokens_seen": 3079405568 + }, + { + "epoch": 0.93, + "learning_rate": 5.422668004012036e-05, + "loss": 1.1966, + "theoretical_loss": 3.3425759972211857, + "tokens_seen": 3079471104 + }, + { + "epoch": 0.93, + "learning_rate": 5.421063189568706e-05, + "loss": 1.2306, + "theoretical_loss": 3.342570603266432, + "tokens_seen": 3079536640 + }, + { + "epoch": 0.93, + "learning_rate": 5.4194583751253766e-05, + "loss": 1.2721, + "theoretical_loss": 3.3425652094586074, + "tokens_seen": 3079602176 + }, + { + "epoch": 0.93, + "learning_rate": 5.417853560682047e-05, + "loss": 1.2102, + "theoretical_loss": 3.3425598157977037, + "tokens_seen": 3079667712 + }, + { + "epoch": 0.93, + "learning_rate": 5.416248746238717e-05, + "loss": 1.2707, + "theoretical_loss": 3.3425544222837145, + "tokens_seen": 3079733248 + }, + { + "epoch": 0.93, + "learning_rate": 5.414643931795387e-05, + "loss": 1.2141, + "theoretical_loss": 3.3425490289166317, + "tokens_seen": 3079798784 + }, + { + "epoch": 0.93, + "learning_rate": 5.413039117352057e-05, + "loss": 1.2584, + "theoretical_loss": 3.3425436356964493, + "tokens_seen": 3079864320 + }, + { + "epoch": 0.93, + "learning_rate": 5.411434302908727e-05, + "loss": 1.1272, + "theoretical_loss": 3.34253824262316, + "tokens_seen": 3079929856 + }, + { + "epoch": 0.93, + "learning_rate": 5.409829488465396e-05, + "loss": 1.1762, + "theoretical_loss": 3.342532849696756, + "tokens_seen": 3079995392 + }, + { + "epoch": 0.93, + "learning_rate": 5.4082246740220664e-05, + "loss": 1.1415, + "theoretical_loss": 3.3425274569172303, + "tokens_seen": 3080060928 + }, + { + "epoch": 0.93, + "learning_rate": 5.4066198595787365e-05, + "loss": 1.2478, + "theoretical_loss": 3.3425220642845765, + "tokens_seen": 3080126464 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1027922, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1871562004089355, + "objective/train/theoretical_loss": 3.342516671798787, + "objective/train/tokens_used": 3100652000, + "theoretical_loss": 3.342516671798787, + "tokens_seen": 3080192000 + }, + { + "epoch": 0.93, + "learning_rate": 5.4050150451354065e-05, + "loss": 1.2054, + "theoretical_loss": 3.342516671798787, + "tokens_seen": 3080192000 + }, + { + "epoch": 0.93, + "learning_rate": 5.4034102306920766e-05, + "loss": 1.1552, + "theoretical_loss": 3.3425112794598544, + "tokens_seen": 3080257536 + }, + { + "epoch": 0.93, + "learning_rate": 5.4018054162487467e-05, + "loss": 1.2133, + "theoretical_loss": 3.342505887267772, + "tokens_seen": 3080323072 + }, + { + "epoch": 0.93, + "learning_rate": 5.400200601805416e-05, + "loss": 1.1561, + "theoretical_loss": 3.3425004952225326, + "tokens_seen": 3080388608 + }, + { + "epoch": 0.93, + "learning_rate": 5.398595787362086e-05, + "loss": 1.2817, + "theoretical_loss": 3.3424951033241292, + "tokens_seen": 3080454144 + }, + { + "epoch": 0.93, + "learning_rate": 5.396990972918756e-05, + "loss": 1.2329, + "theoretical_loss": 3.3424897115725543, + "tokens_seen": 3080519680 + }, + { + "epoch": 0.93, + "learning_rate": 5.395386158475426e-05, + "loss": 1.3129, + "theoretical_loss": 3.3424843199678005, + "tokens_seen": 3080585216 + }, + { + "epoch": 0.93, + "learning_rate": 5.393781344032096e-05, + "loss": 1.2063, + "theoretical_loss": 3.3424789285098617, + "tokens_seen": 3080650752 + }, + { + "epoch": 0.93, + "learning_rate": 5.3921765295887664e-05, + "loss": 1.2756, + "theoretical_loss": 3.34247353719873, + "tokens_seen": 3080716288 + }, + { + "epoch": 0.93, + "learning_rate": 5.390571715145436e-05, + "loss": 1.1541, + "theoretical_loss": 3.342468146034399, + "tokens_seen": 3080781824 + }, + { + "epoch": 0.93, + "learning_rate": 5.388966900702107e-05, + "loss": 1.2698, + "theoretical_loss": 3.342462755016861, + "tokens_seen": 3080847360 + }, + { + "epoch": 0.93, + "learning_rate": 5.387362086258777e-05, + "loss": 1.2003, + "theoretical_loss": 3.3424573641461084, + "tokens_seen": 3080912896 + }, + { + "epoch": 0.93, + "learning_rate": 5.385757271815447e-05, + "loss": 1.1029, + "theoretical_loss": 3.342451973422135, + "tokens_seen": 3080978432 + }, + { + "epoch": 0.93, + "learning_rate": 5.384152457372117e-05, + "loss": 1.1326, + "theoretical_loss": 3.342446582844933, + "tokens_seen": 3081043968 + }, + { + "epoch": 0.93, + "learning_rate": 5.382547642928787e-05, + "loss": 1.2176, + "theoretical_loss": 3.342441192414496, + "tokens_seen": 3081109504 + }, + { + "epoch": 0.93, + "learning_rate": 5.380942828485457e-05, + "loss": 1.1528, + "theoretical_loss": 3.3424358021308165, + "tokens_seen": 3081175040 + }, + { + "epoch": 0.93, + "learning_rate": 5.379338014042127e-05, + "loss": 1.3489, + "theoretical_loss": 3.3424304119938872, + "tokens_seen": 3081240576 + }, + { + "epoch": 0.93, + "learning_rate": 5.377733199598797e-05, + "loss": 1.1538, + "theoretical_loss": 3.342425022003701, + "tokens_seen": 3081306112 + }, + { + "epoch": 0.93, + "learning_rate": 5.376128385155467e-05, + "loss": 1.1874, + "theoretical_loss": 3.3424196321602517, + "tokens_seen": 3081371648 + }, + { + "epoch": 0.93, + "learning_rate": 5.3745235707121364e-05, + "loss": 1.2336, + "theoretical_loss": 3.342414242463531, + "tokens_seen": 3081437184 + }, + { + "epoch": 0.93, + "learning_rate": 5.3729187562688064e-05, + "loss": 1.2003, + "theoretical_loss": 3.3424088529135316, + "tokens_seen": 3081502720 + }, + { + "epoch": 0.93, + "learning_rate": 5.3713139418254765e-05, + "loss": 1.1789, + "theoretical_loss": 3.342403463510248, + "tokens_seen": 3081568256 + }, + { + "epoch": 0.93, + "learning_rate": 5.3697091273821466e-05, + "loss": 1.2385, + "theoretical_loss": 3.3423980742536714, + "tokens_seen": 3081633792 + }, + { + "epoch": 0.93, + "learning_rate": 5.3681043129388166e-05, + "loss": 1.1055, + "theoretical_loss": 3.3423926851437957, + "tokens_seen": 3081699328 + }, + { + "epoch": 0.93, + "learning_rate": 5.366499498495487e-05, + "loss": 1.3292, + "theoretical_loss": 3.3423872961806134, + "tokens_seen": 3081764864 + }, + { + "epoch": 0.93, + "learning_rate": 5.364894684052156e-05, + "loss": 1.1474, + "theoretical_loss": 3.342381907364117, + "tokens_seen": 3081830400 + }, + { + "epoch": 0.93, + "learning_rate": 5.363289869608826e-05, + "loss": 1.1748, + "theoretical_loss": 3.3423765186943006, + "tokens_seen": 3081895936 + }, + { + "epoch": 0.93, + "learning_rate": 5.361685055165496e-05, + "loss": 1.2334, + "theoretical_loss": 3.342371130171156, + "tokens_seen": 3081961472 + }, + { + "epoch": 0.93, + "learning_rate": 5.3600802407221676e-05, + "loss": 1.2262, + "theoretical_loss": 3.3423657417946764, + "tokens_seen": 3082027008 + }, + { + "epoch": 0.93, + "learning_rate": 5.358475426278837e-05, + "loss": 1.3116, + "theoretical_loss": 3.342360353564855, + "tokens_seen": 3082092544 + }, + { + "epoch": 0.93, + "learning_rate": 5.356870611835507e-05, + "loss": 1.2033, + "theoretical_loss": 3.3423549654816838, + "tokens_seen": 3082158080 + }, + { + "epoch": 0.93, + "learning_rate": 5.355265797392177e-05, + "loss": 1.2668, + "theoretical_loss": 3.3423495775451566, + "tokens_seen": 3082223616 + }, + { + "epoch": 0.93, + "learning_rate": 5.353660982948847e-05, + "loss": 1.1365, + "theoretical_loss": 3.3423441897552664, + "tokens_seen": 3082289152 + }, + { + "epoch": 0.93, + "learning_rate": 5.352056168505517e-05, + "loss": 1.2003, + "theoretical_loss": 3.342338802112005, + "tokens_seen": 3082354688 + }, + { + "epoch": 0.93, + "learning_rate": 5.350451354062187e-05, + "loss": 1.2076, + "theoretical_loss": 3.342333414615366, + "tokens_seen": 3082420224 + }, + { + "epoch": 0.93, + "learning_rate": 5.348846539618857e-05, + "loss": 1.2446, + "theoretical_loss": 3.3423280272653426, + "tokens_seen": 3082485760 + }, + { + "epoch": 0.93, + "learning_rate": 5.347241725175527e-05, + "loss": 1.1873, + "theoretical_loss": 3.3423226400619273, + "tokens_seen": 3082551296 + }, + { + "epoch": 0.93, + "learning_rate": 5.345636910732197e-05, + "loss": 1.2185, + "theoretical_loss": 3.3423172530051133, + "tokens_seen": 3082616832 + }, + { + "epoch": 0.93, + "learning_rate": 5.344032096288867e-05, + "loss": 1.3384, + "theoretical_loss": 3.3423118660948923, + "tokens_seen": 3082682368 + }, + { + "epoch": 0.93, + "learning_rate": 5.342427281845537e-05, + "loss": 1.2132, + "theoretical_loss": 3.342306479331259, + "tokens_seen": 3082747904 + }, + { + "epoch": 0.93, + "learning_rate": 5.340822467402207e-05, + "loss": 1.1647, + "theoretical_loss": 3.342301092714205, + "tokens_seen": 3082813440 + }, + { + "epoch": 0.93, + "learning_rate": 5.3392176529588764e-05, + "loss": 1.1732, + "theoretical_loss": 3.342295706243724, + "tokens_seen": 3082878976 + }, + { + "epoch": 0.93, + "learning_rate": 5.3376128385155465e-05, + "loss": 1.2411, + "theoretical_loss": 3.342290319919808, + "tokens_seen": 3082944512 + }, + { + "epoch": 0.93, + "learning_rate": 5.3360080240722165e-05, + "loss": 1.304, + "theoretical_loss": 3.3422849337424507, + "tokens_seen": 3083010048 + }, + { + "epoch": 0.93, + "learning_rate": 5.3344032096288866e-05, + "loss": 1.195, + "theoretical_loss": 3.342279547711645, + "tokens_seen": 3083075584 + }, + { + "epoch": 0.93, + "learning_rate": 5.332798395185557e-05, + "loss": 1.2053, + "theoretical_loss": 3.3422741618273832, + "tokens_seen": 3083141120 + }, + { + "epoch": 0.93, + "learning_rate": 5.331193580742227e-05, + "loss": 1.2628, + "theoretical_loss": 3.3422687760896586, + "tokens_seen": 3083206656 + }, + { + "epoch": 0.93, + "learning_rate": 5.3295887662988975e-05, + "loss": 1.1645, + "theoretical_loss": 3.3422633904984638, + "tokens_seen": 3083272192 + }, + { + "epoch": 0.93, + "learning_rate": 5.3279839518555675e-05, + "loss": 1.1919, + "theoretical_loss": 3.3422580050537922, + "tokens_seen": 3083337728 + }, + { + "epoch": 0.93, + "learning_rate": 5.3263791374122376e-05, + "loss": 1.2392, + "theoretical_loss": 3.342252619755636, + "tokens_seen": 3083403264 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1028967, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1248587369918823, + "objective/train/theoretical_loss": 3.3422472346039886, + "objective/train/tokens_used": 3103928800, + "theoretical_loss": 3.3422472346039886, + "tokens_seen": 3083468800 + }, + { + "epoch": 0.93, + "learning_rate": 5.324774322968907e-05, + "loss": 1.1851, + "theoretical_loss": 3.3422472346039886, + "tokens_seen": 3083468800 + }, + { + "epoch": 0.93, + "learning_rate": 5.323169508525577e-05, + "loss": 1.2705, + "theoretical_loss": 3.3422418495988433, + "tokens_seen": 3083534336 + }, + { + "epoch": 0.93, + "learning_rate": 5.321564694082247e-05, + "loss": 1.1465, + "theoretical_loss": 3.342236464740192, + "tokens_seen": 3083599872 + }, + { + "epoch": 0.93, + "learning_rate": 5.319959879638917e-05, + "loss": 1.2172, + "theoretical_loss": 3.342231080028028, + "tokens_seen": 3083665408 + }, + { + "epoch": 0.93, + "learning_rate": 5.318355065195587e-05, + "loss": 1.2116, + "theoretical_loss": 3.3422256954623446, + "tokens_seen": 3083730944 + }, + { + "epoch": 0.93, + "learning_rate": 5.316750250752257e-05, + "loss": 1.1414, + "theoretical_loss": 3.3422203110431346, + "tokens_seen": 3083796480 + }, + { + "epoch": 0.93, + "learning_rate": 5.315145436308927e-05, + "loss": 1.209, + "theoretical_loss": 3.3422149267703904, + "tokens_seen": 3083862016 + }, + { + "epoch": 0.93, + "learning_rate": 5.313540621865597e-05, + "loss": 1.2272, + "theoretical_loss": 3.342209542644105, + "tokens_seen": 3083927552 + }, + { + "epoch": 0.93, + "learning_rate": 5.311935807422267e-05, + "loss": 1.201, + "theoretical_loss": 3.3422041586642717, + "tokens_seen": 3083993088 + }, + { + "epoch": 0.93, + "learning_rate": 5.310330992978937e-05, + "loss": 1.2096, + "theoretical_loss": 3.3421987748308837, + "tokens_seen": 3084058624 + }, + { + "epoch": 0.93, + "learning_rate": 5.308726178535607e-05, + "loss": 1.1986, + "theoretical_loss": 3.3421933911439328, + "tokens_seen": 3084124160 + }, + { + "epoch": 0.93, + "learning_rate": 5.307121364092277e-05, + "loss": 1.1669, + "theoretical_loss": 3.3421880076034127, + "tokens_seen": 3084189696 + }, + { + "epoch": 0.93, + "learning_rate": 5.3055165496489464e-05, + "loss": 1.2249, + "theoretical_loss": 3.3421826242093164, + "tokens_seen": 3084255232 + }, + { + "epoch": 0.93, + "learning_rate": 5.3039117352056165e-05, + "loss": 1.0907, + "theoretical_loss": 3.3421772409616364, + "tokens_seen": 3084320768 + }, + { + "epoch": 0.93, + "learning_rate": 5.3023069207622865e-05, + "loss": 1.2123, + "theoretical_loss": 3.3421718578603654, + "tokens_seen": 3084386304 + }, + { + "epoch": 0.93, + "learning_rate": 5.3007021063189566e-05, + "loss": 1.2303, + "theoretical_loss": 3.3421664749054973, + "tokens_seen": 3084451840 + }, + { + "epoch": 0.93, + "learning_rate": 5.299097291875627e-05, + "loss": 1.1514, + "theoretical_loss": 3.342161092097024, + "tokens_seen": 3084517376 + }, + { + "epoch": 0.93, + "learning_rate": 5.2974924774322974e-05, + "loss": 1.0966, + "theoretical_loss": 3.342155709434939, + "tokens_seen": 3084582912 + }, + { + "epoch": 0.93, + "learning_rate": 5.2958876629889675e-05, + "loss": 1.2607, + "theoretical_loss": 3.3421503269192345, + "tokens_seen": 3084648448 + }, + { + "epoch": 0.93, + "learning_rate": 5.2942828485456375e-05, + "loss": 1.1725, + "theoretical_loss": 3.3421449445499043, + "tokens_seen": 3084713984 + }, + { + "epoch": 0.93, + "learning_rate": 5.2926780341023076e-05, + "loss": 1.2627, + "theoretical_loss": 3.342139562326941, + "tokens_seen": 3084779520 + }, + { + "epoch": 0.93, + "learning_rate": 5.2910732196589776e-05, + "loss": 1.2423, + "theoretical_loss": 3.342134180250337, + "tokens_seen": 3084845056 + }, + { + "epoch": 0.93, + "learning_rate": 5.289468405215647e-05, + "loss": 1.193, + "theoretical_loss": 3.342128798320086, + "tokens_seen": 3084910592 + }, + { + "epoch": 0.93, + "learning_rate": 5.287863590772317e-05, + "loss": 1.2295, + "theoretical_loss": 3.34212341653618, + "tokens_seen": 3084976128 + }, + { + "epoch": 0.93, + "learning_rate": 5.286258776328987e-05, + "loss": 1.2413, + "theoretical_loss": 3.342118034898613, + "tokens_seen": 3085041664 + }, + { + "epoch": 0.93, + "learning_rate": 5.284653961885657e-05, + "loss": 1.1963, + "theoretical_loss": 3.3421126534073773, + "tokens_seen": 3085107200 + }, + { + "epoch": 0.93, + "learning_rate": 5.283049147442327e-05, + "loss": 1.1982, + "theoretical_loss": 3.3421072720624654, + "tokens_seen": 3085172736 + }, + { + "epoch": 0.93, + "learning_rate": 5.2814443329989973e-05, + "loss": 1.2219, + "theoretical_loss": 3.342101890863871, + "tokens_seen": 3085238272 + }, + { + "epoch": 0.93, + "learning_rate": 5.279839518555667e-05, + "loss": 1.2004, + "theoretical_loss": 3.342096509811587, + "tokens_seen": 3085303808 + }, + { + "epoch": 0.93, + "learning_rate": 5.278234704112337e-05, + "loss": 1.2291, + "theoretical_loss": 3.3420911289056057, + "tokens_seen": 3085369344 + }, + { + "epoch": 0.93, + "learning_rate": 5.276629889669007e-05, + "loss": 1.2227, + "theoretical_loss": 3.3420857481459203, + "tokens_seen": 3085434880 + }, + { + "epoch": 0.94, + "learning_rate": 5.275025075225677e-05, + "loss": 1.1793, + "theoretical_loss": 3.3420803675325237, + "tokens_seen": 3085500416 + }, + { + "epoch": 0.94, + "learning_rate": 5.273420260782347e-05, + "loss": 1.2418, + "theoretical_loss": 3.342074987065409, + "tokens_seen": 3085565952 + }, + { + "epoch": 0.94, + "learning_rate": 5.271815446339017e-05, + "loss": 1.3196, + "theoretical_loss": 3.342069606744569, + "tokens_seen": 3085631488 + }, + { + "epoch": 0.94, + "learning_rate": 5.270210631895688e-05, + "loss": 1.2397, + "theoretical_loss": 3.3420642265699967, + "tokens_seen": 3085697024 + }, + { + "epoch": 0.94, + "learning_rate": 5.268605817452358e-05, + "loss": 1.1822, + "theoretical_loss": 3.3420588465416845, + "tokens_seen": 3085762560 + }, + { + "epoch": 0.94, + "learning_rate": 5.267001003009028e-05, + "loss": 1.1843, + "theoretical_loss": 3.342053466659626, + "tokens_seen": 3085828096 + }, + { + "epoch": 0.94, + "learning_rate": 5.265396188565698e-05, + "loss": 1.1791, + "theoretical_loss": 3.342048086923814, + "tokens_seen": 3085893632 + }, + { + "epoch": 0.94, + "learning_rate": 5.2637913741223674e-05, + "loss": 1.2775, + "theoretical_loss": 3.342042707334241, + "tokens_seen": 3085959168 + }, + { + "epoch": 0.94, + "learning_rate": 5.2621865596790374e-05, + "loss": 1.1637, + "theoretical_loss": 3.3420373278909006, + "tokens_seen": 3086024704 + }, + { + "epoch": 0.94, + "learning_rate": 5.2605817452357075e-05, + "loss": 1.3028, + "theoretical_loss": 3.342031948593785, + "tokens_seen": 3086090240 + }, + { + "epoch": 0.94, + "learning_rate": 5.2589769307923776e-05, + "loss": 1.1638, + "theoretical_loss": 3.3420265694428872, + "tokens_seen": 3086155776 + }, + { + "epoch": 0.94, + "learning_rate": 5.2573721163490476e-05, + "loss": 1.3147, + "theoretical_loss": 3.3420211904382007, + "tokens_seen": 3086221312 + }, + { + "epoch": 0.94, + "learning_rate": 5.255767301905718e-05, + "loss": 1.181, + "theoretical_loss": 3.3420158115797176, + "tokens_seen": 3086286848 + }, + { + "epoch": 0.94, + "learning_rate": 5.254162487462387e-05, + "loss": 1.2154, + "theoretical_loss": 3.342010432867432, + "tokens_seen": 3086352384 + }, + { + "epoch": 0.94, + "learning_rate": 5.252557673019057e-05, + "loss": 1.2515, + "theoretical_loss": 3.3420050543013353, + "tokens_seen": 3086417920 + }, + { + "epoch": 0.94, + "learning_rate": 5.250952858575727e-05, + "loss": 1.2095, + "theoretical_loss": 3.341999675881422, + "tokens_seen": 3086483456 + }, + { + "epoch": 0.94, + "learning_rate": 5.249348044132397e-05, + "loss": 1.1826, + "theoretical_loss": 3.3419942976076835, + "tokens_seen": 3086548992 + }, + { + "epoch": 0.94, + "learning_rate": 5.247743229689067e-05, + "loss": 1.2855, + "theoretical_loss": 3.341988919480114, + "tokens_seen": 3086614528 + }, + { + "epoch": 0.94, + "learning_rate": 5.2461384152457374e-05, + "loss": 1.1427, + "theoretical_loss": 3.341983541498706, + "tokens_seen": 3086680064 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1030255, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.322845220565796, + "objective/train/theoretical_loss": 3.341978163663452, + "objective/train/tokens_used": 3107205600, + "theoretical_loss": 3.341978163663452, + "tokens_seen": 3086745600 + }, + { + "epoch": 0.94, + "learning_rate": 5.244533600802407e-05, + "loss": 1.2905, + "theoretical_loss": 3.341978163663452, + "tokens_seen": 3086745600 + }, + { + "epoch": 0.94, + "learning_rate": 5.242928786359077e-05, + "loss": 1.0852, + "theoretical_loss": 3.3419727859743453, + "tokens_seen": 3086811136 + }, + { + "epoch": 0.94, + "learning_rate": 5.241323971915747e-05, + "loss": 1.1107, + "theoretical_loss": 3.341967408431379, + "tokens_seen": 3086876672 + }, + { + "epoch": 0.94, + "learning_rate": 5.239719157472418e-05, + "loss": 1.1625, + "theoretical_loss": 3.3419620310345453, + "tokens_seen": 3086942208 + }, + { + "epoch": 0.94, + "learning_rate": 5.238114343029088e-05, + "loss": 1.2345, + "theoretical_loss": 3.3419566537838383, + "tokens_seen": 3087007744 + }, + { + "epoch": 0.94, + "learning_rate": 5.236509528585758e-05, + "loss": 1.2005, + "theoretical_loss": 3.34195127667925, + "tokens_seen": 3087073280 + }, + { + "epoch": 0.94, + "learning_rate": 5.234904714142428e-05, + "loss": 1.1841, + "theoretical_loss": 3.3419458997207734, + "tokens_seen": 3087138816 + }, + { + "epoch": 0.94, + "learning_rate": 5.233299899699098e-05, + "loss": 1.2401, + "theoretical_loss": 3.3419405229084016, + "tokens_seen": 3087204352 + }, + { + "epoch": 0.94, + "learning_rate": 5.231695085255768e-05, + "loss": 1.1384, + "theoretical_loss": 3.3419351462421276, + "tokens_seen": 3087269888 + }, + { + "epoch": 0.94, + "learning_rate": 5.230090270812438e-05, + "loss": 1.177, + "theoretical_loss": 3.3419297697219443, + "tokens_seen": 3087335424 + }, + { + "epoch": 0.94, + "learning_rate": 5.2284854563691074e-05, + "loss": 1.2187, + "theoretical_loss": 3.3419243933478446, + "tokens_seen": 3087400960 + }, + { + "epoch": 0.94, + "learning_rate": 5.2268806419257775e-05, + "loss": 1.2091, + "theoretical_loss": 3.341919017119822, + "tokens_seen": 3087466496 + }, + { + "epoch": 0.94, + "learning_rate": 5.2252758274824475e-05, + "loss": 1.1184, + "theoretical_loss": 3.341913641037868, + "tokens_seen": 3087532032 + }, + { + "epoch": 0.94, + "learning_rate": 5.2236710130391176e-05, + "loss": 1.1957, + "theoretical_loss": 3.341908265101977, + "tokens_seen": 3087597568 + }, + { + "epoch": 0.94, + "learning_rate": 5.222066198595788e-05, + "loss": 1.2451, + "theoretical_loss": 3.3419028893121405, + "tokens_seen": 3087663104 + }, + { + "epoch": 0.94, + "learning_rate": 5.220461384152458e-05, + "loss": 1.1667, + "theoretical_loss": 3.341897513668353, + "tokens_seen": 3087728640 + }, + { + "epoch": 0.94, + "learning_rate": 5.218856569709127e-05, + "loss": 1.1822, + "theoretical_loss": 3.3418921381706066, + "tokens_seen": 3087794176 + }, + { + "epoch": 0.94, + "learning_rate": 5.217251755265797e-05, + "loss": 1.2042, + "theoretical_loss": 3.341886762818894, + "tokens_seen": 3087859712 + }, + { + "epoch": 0.94, + "learning_rate": 5.215646940822467e-05, + "loss": 1.2004, + "theoretical_loss": 3.341881387613209, + "tokens_seen": 3087925248 + }, + { + "epoch": 0.94, + "learning_rate": 5.214042126379137e-05, + "loss": 1.1589, + "theoretical_loss": 3.3418760125535436, + "tokens_seen": 3087990784 + }, + { + "epoch": 0.94, + "learning_rate": 5.2124373119358074e-05, + "loss": 1.1647, + "theoretical_loss": 3.3418706376398912, + "tokens_seen": 3088056320 + }, + { + "epoch": 0.94, + "learning_rate": 5.2108324974924774e-05, + "loss": 1.2096, + "theoretical_loss": 3.3418652628722443, + "tokens_seen": 3088121856 + }, + { + "epoch": 0.94, + "learning_rate": 5.209227683049148e-05, + "loss": 1.1804, + "theoretical_loss": 3.3418598882505965, + "tokens_seen": 3088187392 + }, + { + "epoch": 0.94, + "learning_rate": 5.207622868605818e-05, + "loss": 1.264, + "theoretical_loss": 3.3418545137749405, + "tokens_seen": 3088252928 + }, + { + "epoch": 0.94, + "learning_rate": 5.206018054162488e-05, + "loss": 1.2517, + "theoretical_loss": 3.3418491394452694, + "tokens_seen": 3088318464 + }, + { + "epoch": 0.94, + "learning_rate": 5.2044132397191584e-05, + "loss": 1.2268, + "theoretical_loss": 3.3418437652615753, + "tokens_seen": 3088384000 + }, + { + "epoch": 0.94, + "learning_rate": 5.202808425275828e-05, + "loss": 1.2573, + "theoretical_loss": 3.3418383912238525, + "tokens_seen": 3088449536 + }, + { + "epoch": 0.94, + "learning_rate": 5.201203610832498e-05, + "loss": 1.3183, + "theoretical_loss": 3.3418330173320925, + "tokens_seen": 3088515072 + }, + { + "epoch": 0.94, + "learning_rate": 5.199598796389168e-05, + "loss": 1.1608, + "theoretical_loss": 3.3418276435862895, + "tokens_seen": 3088580608 + }, + { + "epoch": 0.94, + "learning_rate": 5.197993981945838e-05, + "loss": 1.2395, + "theoretical_loss": 3.341822269986435, + "tokens_seen": 3088646144 + }, + { + "epoch": 0.94, + "learning_rate": 5.196389167502508e-05, + "loss": 1.1488, + "theoretical_loss": 3.3418168965325235, + "tokens_seen": 3088711680 + }, + { + "epoch": 0.94, + "learning_rate": 5.194784353059178e-05, + "loss": 1.1846, + "theoretical_loss": 3.341811523224547, + "tokens_seen": 3088777216 + }, + { + "epoch": 0.94, + "learning_rate": 5.1931795386158474e-05, + "loss": 1.2982, + "theoretical_loss": 3.341806150062499, + "tokens_seen": 3088842752 + }, + { + "epoch": 0.94, + "learning_rate": 5.1915747241725175e-05, + "loss": 1.2439, + "theoretical_loss": 3.341800777046372, + "tokens_seen": 3088908288 + }, + { + "epoch": 0.94, + "learning_rate": 5.1899699097291876e-05, + "loss": 1.1394, + "theoretical_loss": 3.3417954041761586, + "tokens_seen": 3088973824 + }, + { + "epoch": 0.94, + "learning_rate": 5.1883650952858576e-05, + "loss": 1.2585, + "theoretical_loss": 3.3417900314518523, + "tokens_seen": 3089039360 + }, + { + "epoch": 0.94, + "learning_rate": 5.186760280842528e-05, + "loss": 1.2125, + "theoretical_loss": 3.341784658873446, + "tokens_seen": 3089104896 + }, + { + "epoch": 0.94, + "learning_rate": 5.185155466399198e-05, + "loss": 1.3279, + "theoretical_loss": 3.341779286440933, + "tokens_seen": 3089170432 + }, + { + "epoch": 0.94, + "learning_rate": 5.183550651955867e-05, + "loss": 1.2561, + "theoretical_loss": 3.3417739141543055, + "tokens_seen": 3089235968 + }, + { + "epoch": 0.94, + "learning_rate": 5.181945837512537e-05, + "loss": 1.1851, + "theoretical_loss": 3.3417685420135568, + "tokens_seen": 3089301504 + }, + { + "epoch": 0.94, + "learning_rate": 5.180341023069207e-05, + "loss": 1.1901, + "theoretical_loss": 3.34176317001868, + "tokens_seen": 3089367040 + }, + { + "epoch": 0.94, + "learning_rate": 5.178736208625879e-05, + "loss": 1.2223, + "theoretical_loss": 3.3417577981696676, + "tokens_seen": 3089432576 + }, + { + "epoch": 0.94, + "learning_rate": 5.177131394182548e-05, + "loss": 1.1698, + "theoretical_loss": 3.341752426466513, + "tokens_seen": 3089498112 + }, + { + "epoch": 0.94, + "learning_rate": 5.175526579739218e-05, + "loss": 1.2196, + "theoretical_loss": 3.3417470549092085, + "tokens_seen": 3089563648 + }, + { + "epoch": 0.94, + "learning_rate": 5.173921765295888e-05, + "loss": 1.1173, + "theoretical_loss": 3.341741683497748, + "tokens_seen": 3089629184 + }, + { + "epoch": 0.94, + "learning_rate": 5.172316950852558e-05, + "loss": 1.1245, + "theoretical_loss": 3.3417363122321238, + "tokens_seen": 3089694720 + }, + { + "epoch": 0.94, + "learning_rate": 5.170712136409228e-05, + "loss": 1.2118, + "theoretical_loss": 3.341730941112329, + "tokens_seen": 3089760256 + }, + { + "epoch": 0.94, + "learning_rate": 5.1691073219658984e-05, + "loss": 1.1949, + "theoretical_loss": 3.3417255701383564, + "tokens_seen": 3089825792 + }, + { + "epoch": 0.94, + "learning_rate": 5.167502507522568e-05, + "loss": 1.2172, + "theoretical_loss": 3.3417201993101995, + "tokens_seen": 3089891328 + }, + { + "epoch": 0.94, + "learning_rate": 5.165897693079238e-05, + "loss": 1.1794, + "theoretical_loss": 3.3417148286278504, + "tokens_seen": 3089956864 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1031188, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1749727725982666, + "objective/train/theoretical_loss": 3.3417094580913025, + "objective/train/tokens_used": 3110482400, + "theoretical_loss": 3.3417094580913025, + "tokens_seen": 3090022400 + }, + { + "epoch": 0.94, + "learning_rate": 5.164292878635908e-05, + "loss": 1.1502, + "theoretical_loss": 3.3417094580913025, + "tokens_seen": 3090022400 + }, + { + "epoch": 0.94, + "learning_rate": 5.162688064192578e-05, + "loss": 1.1925, + "theoretical_loss": 3.341704087700549, + "tokens_seen": 3090087936 + }, + { + "epoch": 0.94, + "learning_rate": 5.161083249749248e-05, + "loss": 1.1943, + "theoretical_loss": 3.3416987174555826, + "tokens_seen": 3090153472 + }, + { + "epoch": 0.94, + "learning_rate": 5.159478435305918e-05, + "loss": 1.1758, + "theoretical_loss": 3.3416933473563963, + "tokens_seen": 3090219008 + }, + { + "epoch": 0.94, + "learning_rate": 5.1578736208625875e-05, + "loss": 1.2413, + "theoretical_loss": 3.3416879774029824, + "tokens_seen": 3090284544 + }, + { + "epoch": 0.94, + "learning_rate": 5.1562688064192576e-05, + "loss": 1.1974, + "theoretical_loss": 3.341682607595335, + "tokens_seen": 3090350080 + }, + { + "epoch": 0.94, + "learning_rate": 5.1546639919759276e-05, + "loss": 1.2193, + "theoretical_loss": 3.341677237933446, + "tokens_seen": 3090415616 + }, + { + "epoch": 0.94, + "learning_rate": 5.153059177532598e-05, + "loss": 1.165, + "theoretical_loss": 3.3416718684173095, + "tokens_seen": 3090481152 + }, + { + "epoch": 0.94, + "learning_rate": 5.151454363089268e-05, + "loss": 1.209, + "theoretical_loss": 3.3416664990469176, + "tokens_seen": 3090546688 + }, + { + "epoch": 0.94, + "learning_rate": 5.1498495486459385e-05, + "loss": 1.2425, + "theoretical_loss": 3.341661129822263, + "tokens_seen": 3090612224 + }, + { + "epoch": 0.94, + "learning_rate": 5.1482447342026085e-05, + "loss": 1.2871, + "theoretical_loss": 3.3416557607433397, + "tokens_seen": 3090677760 + }, + { + "epoch": 0.94, + "learning_rate": 5.1466399197592786e-05, + "loss": 1.1852, + "theoretical_loss": 3.3416503918101395, + "tokens_seen": 3090743296 + }, + { + "epoch": 0.94, + "learning_rate": 5.145035105315949e-05, + "loss": 1.1611, + "theoretical_loss": 3.3416450230226564, + "tokens_seen": 3090808832 + }, + { + "epoch": 0.94, + "learning_rate": 5.143430290872619e-05, + "loss": 1.1658, + "theoretical_loss": 3.3416396543808826, + "tokens_seen": 3090874368 + }, + { + "epoch": 0.94, + "learning_rate": 5.141825476429288e-05, + "loss": 1.2014, + "theoretical_loss": 3.3416342858848114, + "tokens_seen": 3090939904 + }, + { + "epoch": 0.94, + "learning_rate": 5.140220661985958e-05, + "loss": 1.2905, + "theoretical_loss": 3.3416289175344356, + "tokens_seen": 3091005440 + }, + { + "epoch": 0.94, + "learning_rate": 5.138615847542628e-05, + "loss": 1.2467, + "theoretical_loss": 3.341623549329748, + "tokens_seen": 3091070976 + }, + { + "epoch": 0.94, + "learning_rate": 5.137011033099298e-05, + "loss": 1.1588, + "theoretical_loss": 3.3416181812707424, + "tokens_seen": 3091136512 + }, + { + "epoch": 0.94, + "learning_rate": 5.1354062186559684e-05, + "loss": 1.238, + "theoretical_loss": 3.341612813357411, + "tokens_seen": 3091202048 + }, + { + "epoch": 0.94, + "learning_rate": 5.1338014042126384e-05, + "loss": 1.2964, + "theoretical_loss": 3.3416074455897466, + "tokens_seen": 3091267584 + }, + { + "epoch": 0.94, + "learning_rate": 5.132196589769308e-05, + "loss": 1.2009, + "theoretical_loss": 3.3416020779677424, + "tokens_seen": 3091333120 + }, + { + "epoch": 0.94, + "learning_rate": 5.130591775325978e-05, + "loss": 1.1605, + "theoretical_loss": 3.341596710491392, + "tokens_seen": 3091398656 + }, + { + "epoch": 0.94, + "learning_rate": 5.128986960882648e-05, + "loss": 1.2091, + "theoretical_loss": 3.341591343160687, + "tokens_seen": 3091464192 + }, + { + "epoch": 0.94, + "learning_rate": 5.127382146439318e-05, + "loss": 1.3094, + "theoretical_loss": 3.341585975975622, + "tokens_seen": 3091529728 + }, + { + "epoch": 0.94, + "learning_rate": 5.125777331995988e-05, + "loss": 1.1988, + "theoretical_loss": 3.3415806089361886, + "tokens_seen": 3091595264 + }, + { + "epoch": 0.94, + "learning_rate": 5.124172517552658e-05, + "loss": 1.2177, + "theoretical_loss": 3.3415752420423805, + "tokens_seen": 3091660800 + }, + { + "epoch": 0.94, + "learning_rate": 5.1225677031093275e-05, + "loss": 1.1132, + "theoretical_loss": 3.3415698752941903, + "tokens_seen": 3091726336 + }, + { + "epoch": 0.94, + "learning_rate": 5.1209628886659976e-05, + "loss": 1.2207, + "theoretical_loss": 3.341564508691611, + "tokens_seen": 3091791872 + }, + { + "epoch": 0.94, + "learning_rate": 5.119358074222669e-05, + "loss": 1.2406, + "theoretical_loss": 3.341559142234636, + "tokens_seen": 3091857408 + }, + { + "epoch": 0.94, + "learning_rate": 5.1177532597793384e-05, + "loss": 1.1285, + "theoretical_loss": 3.3415537759232574, + "tokens_seen": 3091922944 + }, + { + "epoch": 0.94, + "learning_rate": 5.1161484453360085e-05, + "loss": 1.1549, + "theoretical_loss": 3.3415484097574693, + "tokens_seen": 3091988480 + }, + { + "epoch": 0.94, + "learning_rate": 5.1145436308926785e-05, + "loss": 1.1081, + "theoretical_loss": 3.3415430437372637, + "tokens_seen": 3092054016 + }, + { + "epoch": 0.94, + "learning_rate": 5.1129388164493486e-05, + "loss": 1.2339, + "theoretical_loss": 3.341537677862634, + "tokens_seen": 3092119552 + }, + { + "epoch": 0.94, + "learning_rate": 5.1113340020060187e-05, + "loss": 1.2039, + "theoretical_loss": 3.341532312133573, + "tokens_seen": 3092185088 + }, + { + "epoch": 0.94, + "learning_rate": 5.109729187562689e-05, + "loss": 1.2147, + "theoretical_loss": 3.341526946550074, + "tokens_seen": 3092250624 + }, + { + "epoch": 0.94, + "learning_rate": 5.108124373119358e-05, + "loss": 1.2493, + "theoretical_loss": 3.3415215811121293, + "tokens_seen": 3092316160 + }, + { + "epoch": 0.94, + "learning_rate": 5.106519558676028e-05, + "loss": 1.2174, + "theoretical_loss": 3.3415162158197327, + "tokens_seen": 3092381696 + }, + { + "epoch": 0.94, + "learning_rate": 5.104914744232698e-05, + "loss": 1.1754, + "theoretical_loss": 3.3415108506728766, + "tokens_seen": 3092447232 + }, + { + "epoch": 0.94, + "learning_rate": 5.103309929789368e-05, + "loss": 1.1657, + "theoretical_loss": 3.341505485671554, + "tokens_seen": 3092512768 + }, + { + "epoch": 0.94, + "learning_rate": 5.1017051153460384e-05, + "loss": 1.283, + "theoretical_loss": 3.341500120815758, + "tokens_seen": 3092578304 + }, + { + "epoch": 0.94, + "learning_rate": 5.1001003009027084e-05, + "loss": 1.1465, + "theoretical_loss": 3.3414947561054817, + "tokens_seen": 3092643840 + }, + { + "epoch": 0.94, + "learning_rate": 5.098495486459378e-05, + "loss": 1.2965, + "theoretical_loss": 3.341489391540718, + "tokens_seen": 3092709376 + }, + { + "epoch": 0.94, + "learning_rate": 5.096890672016048e-05, + "loss": 1.1959, + "theoretical_loss": 3.341484027121459, + "tokens_seen": 3092774912 + }, + { + "epoch": 0.94, + "learning_rate": 5.095285857572718e-05, + "loss": 1.2496, + "theoretical_loss": 3.3414786628476993, + "tokens_seen": 3092840448 + }, + { + "epoch": 0.94, + "learning_rate": 5.093681043129388e-05, + "loss": 1.1675, + "theoretical_loss": 3.341473298719431, + "tokens_seen": 3092905984 + }, + { + "epoch": 0.94, + "learning_rate": 5.092076228686058e-05, + "loss": 1.1839, + "theoretical_loss": 3.341467934736647, + "tokens_seen": 3092971520 + }, + { + "epoch": 0.94, + "learning_rate": 5.090471414242728e-05, + "loss": 1.1491, + "theoretical_loss": 3.34146257089934, + "tokens_seen": 3093037056 + }, + { + "epoch": 0.94, + "learning_rate": 5.088866599799399e-05, + "loss": 1.1373, + "theoretical_loss": 3.3414572072075037, + "tokens_seen": 3093102592 + }, + { + "epoch": 0.94, + "learning_rate": 5.087261785356069e-05, + "loss": 1.1073, + "theoretical_loss": 3.341451843661131, + "tokens_seen": 3093168128 + }, + { + "epoch": 0.94, + "learning_rate": 5.085656970912739e-05, + "loss": 1.1786, + "theoretical_loss": 3.341446480260214, + "tokens_seen": 3093233664 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1032333, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.269102931022644, + "objective/train/theoretical_loss": 3.341441117004747, + "objective/train/tokens_used": 3113759200, + "theoretical_loss": 3.341441117004747, + "tokens_seen": 3093299200 + }, + { + "epoch": 0.94, + "learning_rate": 5.084052156469409e-05, + "loss": 1.1697, + "theoretical_loss": 3.341441117004747, + "tokens_seen": 3093299200 + }, + { + "epoch": 0.94, + "learning_rate": 5.0824473420260784e-05, + "loss": 1.1744, + "theoretical_loss": 3.3414357538947215, + "tokens_seen": 3093364736 + }, + { + "epoch": 0.94, + "learning_rate": 5.0808425275827485e-05, + "loss": 1.2415, + "theoretical_loss": 3.3414303909301317, + "tokens_seen": 3093430272 + }, + { + "epoch": 0.94, + "learning_rate": 5.0792377131394186e-05, + "loss": 1.111, + "theoretical_loss": 3.3414250281109696, + "tokens_seen": 3093495808 + }, + { + "epoch": 0.94, + "learning_rate": 5.0776328986960886e-05, + "loss": 1.1526, + "theoretical_loss": 3.341419665437229, + "tokens_seen": 3093561344 + }, + { + "epoch": 0.94, + "learning_rate": 5.076028084252759e-05, + "loss": 1.2208, + "theoretical_loss": 3.3414143029089027, + "tokens_seen": 3093626880 + }, + { + "epoch": 0.94, + "learning_rate": 5.074423269809429e-05, + "loss": 1.2632, + "theoretical_loss": 3.3414089405259837, + "tokens_seen": 3093692416 + }, + { + "epoch": 0.94, + "learning_rate": 5.072818455366098e-05, + "loss": 1.2508, + "theoretical_loss": 3.3414035782884643, + "tokens_seen": 3093757952 + }, + { + "epoch": 0.94, + "learning_rate": 5.071213640922768e-05, + "loss": 1.1212, + "theoretical_loss": 3.341398216196338, + "tokens_seen": 3093823488 + }, + { + "epoch": 0.94, + "learning_rate": 5.069608826479438e-05, + "loss": 1.1932, + "theoretical_loss": 3.341392854249598, + "tokens_seen": 3093889024 + }, + { + "epoch": 0.94, + "learning_rate": 5.068004012036108e-05, + "loss": 1.2117, + "theoretical_loss": 3.341387492448237, + "tokens_seen": 3093954560 + }, + { + "epoch": 0.94, + "learning_rate": 5.0663991975927784e-05, + "loss": 1.109, + "theoretical_loss": 3.3413821307922476, + "tokens_seen": 3094020096 + }, + { + "epoch": 0.94, + "learning_rate": 5.0647943831494485e-05, + "loss": 1.2818, + "theoretical_loss": 3.3413767692816236, + "tokens_seen": 3094085632 + }, + { + "epoch": 0.94, + "learning_rate": 5.063189568706118e-05, + "loss": 1.0341, + "theoretical_loss": 3.3413714079163577, + "tokens_seen": 3094151168 + }, + { + "epoch": 0.94, + "learning_rate": 5.061584754262788e-05, + "loss": 1.1634, + "theoretical_loss": 3.3413660466964425, + "tokens_seen": 3094216704 + }, + { + "epoch": 0.94, + "learning_rate": 5.059979939819459e-05, + "loss": 1.2591, + "theoretical_loss": 3.341360685621871, + "tokens_seen": 3094282240 + }, + { + "epoch": 0.94, + "learning_rate": 5.0583751253761294e-05, + "loss": 1.2131, + "theoretical_loss": 3.3413553246926373, + "tokens_seen": 3094347776 + }, + { + "epoch": 0.94, + "learning_rate": 5.056770310932799e-05, + "loss": 1.2713, + "theoretical_loss": 3.3413499639087325, + "tokens_seen": 3094413312 + }, + { + "epoch": 0.94, + "learning_rate": 5.055165496489469e-05, + "loss": 1.1802, + "theoretical_loss": 3.341344603270151, + "tokens_seen": 3094478848 + }, + { + "epoch": 0.94, + "learning_rate": 5.053560682046139e-05, + "loss": 1.2609, + "theoretical_loss": 3.3413392427768853, + "tokens_seen": 3094544384 + }, + { + "epoch": 0.94, + "learning_rate": 5.051955867602809e-05, + "loss": 1.2216, + "theoretical_loss": 3.3413338824289287, + "tokens_seen": 3094609920 + }, + { + "epoch": 0.94, + "learning_rate": 5.050351053159479e-05, + "loss": 1.1393, + "theoretical_loss": 3.3413285222262736, + "tokens_seen": 3094675456 + }, + { + "epoch": 0.94, + "learning_rate": 5.048746238716149e-05, + "loss": 1.2133, + "theoretical_loss": 3.3413231621689135, + "tokens_seen": 3094740992 + }, + { + "epoch": 0.94, + "learning_rate": 5.0471414242728185e-05, + "loss": 1.2302, + "theoretical_loss": 3.341317802256841, + "tokens_seen": 3094806528 + }, + { + "epoch": 0.94, + "learning_rate": 5.0455366098294885e-05, + "loss": 1.2412, + "theoretical_loss": 3.3413124424900493, + "tokens_seen": 3094872064 + }, + { + "epoch": 0.94, + "learning_rate": 5.0439317953861586e-05, + "loss": 1.1796, + "theoretical_loss": 3.3413070828685316, + "tokens_seen": 3094937600 + }, + { + "epoch": 0.94, + "learning_rate": 5.042326980942829e-05, + "loss": 1.131, + "theoretical_loss": 3.341301723392281, + "tokens_seen": 3095003136 + }, + { + "epoch": 0.94, + "learning_rate": 5.040722166499499e-05, + "loss": 1.1638, + "theoretical_loss": 3.3412963640612894, + "tokens_seen": 3095068672 + }, + { + "epoch": 0.94, + "learning_rate": 5.039117352056169e-05, + "loss": 1.3051, + "theoretical_loss": 3.3412910048755506, + "tokens_seen": 3095134208 + }, + { + "epoch": 0.94, + "learning_rate": 5.037512537612838e-05, + "loss": 1.2207, + "theoretical_loss": 3.341285645835058, + "tokens_seen": 3095199744 + }, + { + "epoch": 0.94, + "learning_rate": 5.035907723169508e-05, + "loss": 1.2778, + "theoretical_loss": 3.3412802869398037, + "tokens_seen": 3095265280 + }, + { + "epoch": 0.94, + "learning_rate": 5.034302908726178e-05, + "loss": 1.2227, + "theoretical_loss": 3.3412749281897813, + "tokens_seen": 3095330816 + }, + { + "epoch": 0.94, + "learning_rate": 5.0326980942828484e-05, + "loss": 1.1403, + "theoretical_loss": 3.3412695695849832, + "tokens_seen": 3095396352 + }, + { + "epoch": 0.94, + "learning_rate": 5.0310932798395184e-05, + "loss": 1.2036, + "theoretical_loss": 3.341264211125403, + "tokens_seen": 3095461888 + }, + { + "epoch": 0.94, + "learning_rate": 5.029488465396189e-05, + "loss": 1.2116, + "theoretical_loss": 3.341258852811034, + "tokens_seen": 3095527424 + }, + { + "epoch": 0.94, + "learning_rate": 5.027883650952859e-05, + "loss": 1.2184, + "theoretical_loss": 3.341253494641868, + "tokens_seen": 3095592960 + }, + { + "epoch": 0.94, + "learning_rate": 5.026278836509529e-05, + "loss": 1.2033, + "theoretical_loss": 3.341248136617899, + "tokens_seen": 3095658496 + }, + { + "epoch": 0.94, + "learning_rate": 5.0246740220661994e-05, + "loss": 1.2062, + "theoretical_loss": 3.3412427787391197, + "tokens_seen": 3095724032 + }, + { + "epoch": 0.94, + "learning_rate": 5.0230692076228694e-05, + "loss": 1.227, + "theoretical_loss": 3.3412374210055225, + "tokens_seen": 3095789568 + }, + { + "epoch": 0.94, + "learning_rate": 5.021464393179539e-05, + "loss": 1.211, + "theoretical_loss": 3.341232063417101, + "tokens_seen": 3095855104 + }, + { + "epoch": 0.94, + "learning_rate": 5.019859578736209e-05, + "loss": 1.1458, + "theoretical_loss": 3.3412267059738485, + "tokens_seen": 3095920640 + }, + { + "epoch": 0.94, + "learning_rate": 5.018254764292879e-05, + "loss": 1.1728, + "theoretical_loss": 3.3412213486757576, + "tokens_seen": 3095986176 + }, + { + "epoch": 0.94, + "learning_rate": 5.016649949849549e-05, + "loss": 1.1729, + "theoretical_loss": 3.3412159915228212, + "tokens_seen": 3096051712 + }, + { + "epoch": 0.94, + "learning_rate": 5.015045135406219e-05, + "loss": 1.1988, + "theoretical_loss": 3.3412106345150323, + "tokens_seen": 3096117248 + }, + { + "epoch": 0.94, + "learning_rate": 5.013440320962889e-05, + "loss": 1.2191, + "theoretical_loss": 3.341205277652384, + "tokens_seen": 3096182784 + }, + { + "epoch": 0.94, + "learning_rate": 5.0118355065195585e-05, + "loss": 1.2665, + "theoretical_loss": 3.3411999209348693, + "tokens_seen": 3096248320 + }, + { + "epoch": 0.94, + "learning_rate": 5.0102306920762286e-05, + "loss": 1.1781, + "theoretical_loss": 3.3411945643624814, + "tokens_seen": 3096313856 + }, + { + "epoch": 0.94, + "learning_rate": 5.0086258776328986e-05, + "loss": 1.1391, + "theoretical_loss": 3.341189207935213, + "tokens_seen": 3096379392 + }, + { + "epoch": 0.94, + "learning_rate": 5.007021063189569e-05, + "loss": 1.1102, + "theoretical_loss": 3.341183851653057, + "tokens_seen": 3096444928 + }, + { + "epoch": 0.94, + "learning_rate": 5.005416248746239e-05, + "loss": 1.2644, + "theoretical_loss": 3.3411784955160067, + "tokens_seen": 3096510464 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1033188, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2854031324386597, + "objective/train/theoretical_loss": 3.341173139524055, + "objective/train/tokens_used": 3117036000, + "theoretical_loss": 3.341173139524055, + "tokens_seen": 3096576000 + }, + { + "epoch": 0.94, + "learning_rate": 5.003811434302909e-05, + "loss": 1.2613, + "theoretical_loss": 3.341173139524055, + "tokens_seen": 3096576000 + }, + { + "epoch": 0.94, + "learning_rate": 5.002206619859578e-05, + "loss": 1.1744, + "theoretical_loss": 3.3411677836771947, + "tokens_seen": 3096641536 + }, + { + "epoch": 0.94, + "learning_rate": 5.000601805416248e-05, + "loss": 1.2434, + "theoretical_loss": 3.3411624279754193, + "tokens_seen": 3096707072 + }, + { + "epoch": 0.94, + "learning_rate": 4.998996990972919e-05, + "loss": 1.2094, + "theoretical_loss": 3.341157072418721, + "tokens_seen": 3096772608 + }, + { + "epoch": 0.94, + "learning_rate": 4.997392176529589e-05, + "loss": 1.2582, + "theoretical_loss": 3.3411517170070937, + "tokens_seen": 3096838144 + }, + { + "epoch": 0.94, + "learning_rate": 4.995787362086259e-05, + "loss": 1.2063, + "theoretical_loss": 3.34114636174053, + "tokens_seen": 3096903680 + }, + { + "epoch": 0.94, + "learning_rate": 4.9941825476429285e-05, + "loss": 1.1899, + "theoretical_loss": 3.3411410066190226, + "tokens_seen": 3096969216 + }, + { + "epoch": 0.94, + "learning_rate": 4.9925777331995986e-05, + "loss": 1.1795, + "theoretical_loss": 3.341135651642565, + "tokens_seen": 3097034752 + }, + { + "epoch": 0.94, + "learning_rate": 4.9909729187562693e-05, + "loss": 1.1471, + "theoretical_loss": 3.3411302968111496, + "tokens_seen": 3097100288 + }, + { + "epoch": 0.94, + "learning_rate": 4.9893681043129394e-05, + "loss": 1.2222, + "theoretical_loss": 3.34112494212477, + "tokens_seen": 3097165824 + }, + { + "epoch": 0.94, + "learning_rate": 4.9877632898696095e-05, + "loss": 1.2349, + "theoretical_loss": 3.341119587583419, + "tokens_seen": 3097231360 + }, + { + "epoch": 0.94, + "learning_rate": 4.986158475426279e-05, + "loss": 1.1349, + "theoretical_loss": 3.3411142331870898, + "tokens_seen": 3097296896 + }, + { + "epoch": 0.94, + "learning_rate": 4.984553660982949e-05, + "loss": 1.1969, + "theoretical_loss": 3.341108878935775, + "tokens_seen": 3097362432 + }, + { + "epoch": 0.94, + "learning_rate": 4.982948846539619e-05, + "loss": 1.1576, + "theoretical_loss": 3.341103524829468, + "tokens_seen": 3097427968 + }, + { + "epoch": 0.94, + "learning_rate": 4.981344032096289e-05, + "loss": 1.2001, + "theoretical_loss": 3.341098170868161, + "tokens_seen": 3097493504 + }, + { + "epoch": 0.94, + "learning_rate": 4.979739217652959e-05, + "loss": 1.2581, + "theoretical_loss": 3.341092817051848, + "tokens_seen": 3097559040 + }, + { + "epoch": 0.94, + "learning_rate": 4.978134403209629e-05, + "loss": 1.2607, + "theoretical_loss": 3.3410874633805214, + "tokens_seen": 3097624576 + }, + { + "epoch": 0.94, + "learning_rate": 4.976529588766299e-05, + "loss": 1.2834, + "theoretical_loss": 3.3410821098541748, + "tokens_seen": 3097690112 + }, + { + "epoch": 0.94, + "learning_rate": 4.974924774322969e-05, + "loss": 1.2363, + "theoretical_loss": 3.3410767564728006, + "tokens_seen": 3097755648 + }, + { + "epoch": 0.94, + "learning_rate": 4.9733199598796394e-05, + "loss": 1.1825, + "theoretical_loss": 3.341071403236392, + "tokens_seen": 3097821184 + }, + { + "epoch": 0.94, + "learning_rate": 4.9717151454363094e-05, + "loss": 1.1702, + "theoretical_loss": 3.341066050144942, + "tokens_seen": 3097886720 + }, + { + "epoch": 0.94, + "learning_rate": 4.9701103309929795e-05, + "loss": 1.2406, + "theoretical_loss": 3.341060697198444, + "tokens_seen": 3097952256 + }, + { + "epoch": 0.94, + "learning_rate": 4.968505516549649e-05, + "loss": 1.094, + "theoretical_loss": 3.3410553443968904, + "tokens_seen": 3098017792 + }, + { + "epoch": 0.94, + "learning_rate": 4.966900702106319e-05, + "loss": 1.1924, + "theoretical_loss": 3.3410499917402747, + "tokens_seen": 3098083328 + }, + { + "epoch": 0.94, + "learning_rate": 4.965295887662989e-05, + "loss": 1.3344, + "theoretical_loss": 3.3410446392285893, + "tokens_seen": 3098148864 + }, + { + "epoch": 0.94, + "learning_rate": 4.963691073219659e-05, + "loss": 1.204, + "theoretical_loss": 3.3410392868618275, + "tokens_seen": 3098214400 + }, + { + "epoch": 0.94, + "learning_rate": 4.96208625877633e-05, + "loss": 1.2746, + "theoretical_loss": 3.3410339346399827, + "tokens_seen": 3098279936 + }, + { + "epoch": 0.94, + "learning_rate": 4.960481444332999e-05, + "loss": 1.172, + "theoretical_loss": 3.3410285825630477, + "tokens_seen": 3098345472 + }, + { + "epoch": 0.94, + "learning_rate": 4.958876629889669e-05, + "loss": 1.1681, + "theoretical_loss": 3.341023230631015, + "tokens_seen": 3098411008 + }, + { + "epoch": 0.94, + "learning_rate": 4.957271815446339e-05, + "loss": 1.1714, + "theoretical_loss": 3.3410178788438785, + "tokens_seen": 3098476544 + }, + { + "epoch": 0.94, + "learning_rate": 4.9556670010030094e-05, + "loss": 1.2558, + "theoretical_loss": 3.3410125272016304, + "tokens_seen": 3098542080 + }, + { + "epoch": 0.94, + "learning_rate": 4.9540621865596795e-05, + "loss": 1.2606, + "theoretical_loss": 3.341007175704264, + "tokens_seen": 3098607616 + }, + { + "epoch": 0.94, + "learning_rate": 4.9524573721163495e-05, + "loss": 1.2037, + "theoretical_loss": 3.3410018243517725, + "tokens_seen": 3098673152 + }, + { + "epoch": 0.94, + "learning_rate": 4.950852557673019e-05, + "loss": 1.2746, + "theoretical_loss": 3.340996473144149, + "tokens_seen": 3098738688 + }, + { + "epoch": 0.94, + "learning_rate": 4.949247743229689e-05, + "loss": 1.2202, + "theoretical_loss": 3.3409911220813857, + "tokens_seen": 3098804224 + }, + { + "epoch": 0.94, + "learning_rate": 4.947642928786359e-05, + "loss": 1.1797, + "theoretical_loss": 3.340985771163477, + "tokens_seen": 3098869760 + }, + { + "epoch": 0.94, + "learning_rate": 4.94603811434303e-05, + "loss": 1.2024, + "theoretical_loss": 3.3409804203904145, + "tokens_seen": 3098935296 + }, + { + "epoch": 0.94, + "learning_rate": 4.9444332998997e-05, + "loss": 1.2664, + "theoretical_loss": 3.3409750697621923, + "tokens_seen": 3099000832 + }, + { + "epoch": 0.94, + "learning_rate": 4.942828485456369e-05, + "loss": 1.155, + "theoretical_loss": 3.3409697192788026, + "tokens_seen": 3099066368 + }, + { + "epoch": 0.94, + "learning_rate": 4.941223671013039e-05, + "loss": 1.2733, + "theoretical_loss": 3.340964368940239, + "tokens_seen": 3099131904 + }, + { + "epoch": 0.94, + "learning_rate": 4.9396188565697093e-05, + "loss": 1.1873, + "theoretical_loss": 3.3409590187464944, + "tokens_seen": 3099197440 + }, + { + "epoch": 0.94, + "learning_rate": 4.9380140421263794e-05, + "loss": 1.2282, + "theoretical_loss": 3.340953668697561, + "tokens_seen": 3099262976 + }, + { + "epoch": 0.94, + "learning_rate": 4.9364092276830495e-05, + "loss": 1.1828, + "theoretical_loss": 3.3409483187934335, + "tokens_seen": 3099328512 + }, + { + "epoch": 0.94, + "learning_rate": 4.9348044132397195e-05, + "loss": 1.2357, + "theoretical_loss": 3.3409429690341033, + "tokens_seen": 3099394048 + }, + { + "epoch": 0.94, + "learning_rate": 4.933199598796389e-05, + "loss": 1.1702, + "theoretical_loss": 3.3409376194195644, + "tokens_seen": 3099459584 + }, + { + "epoch": 0.94, + "learning_rate": 4.93159478435306e-05, + "loss": 1.2404, + "theoretical_loss": 3.340932269949809, + "tokens_seen": 3099525120 + }, + { + "epoch": 0.94, + "learning_rate": 4.92998996990973e-05, + "loss": 1.2574, + "theoretical_loss": 3.340926920624831, + "tokens_seen": 3099590656 + }, + { + "epoch": 0.94, + "learning_rate": 4.9283851554664e-05, + "loss": 1.2097, + "theoretical_loss": 3.340921571444623, + "tokens_seen": 3099656192 + }, + { + "epoch": 0.94, + "learning_rate": 4.926780341023069e-05, + "loss": 1.1576, + "theoretical_loss": 3.340916222409178, + "tokens_seen": 3099721728 + }, + { + "epoch": 0.94, + "learning_rate": 4.925175526579739e-05, + "loss": 1.2419, + "theoretical_loss": 3.3409108735184896, + "tokens_seen": 3099787264 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1034453, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2311272621154785, + "objective/train/theoretical_loss": 3.3409055247725497, + "objective/train/tokens_used": 3120312800, + "theoretical_loss": 3.3409055247725497, + "tokens_seen": 3099852800 + }, + { + "epoch": 0.94, + "learning_rate": 4.923570712136409e-05, + "loss": 1.1915, + "theoretical_loss": 3.3409055247725497, + "tokens_seen": 3099852800 + }, + { + "epoch": 0.94, + "learning_rate": 4.9219658976930794e-05, + "loss": 1.2389, + "theoretical_loss": 3.3409001761713517, + "tokens_seen": 3099918336 + }, + { + "epoch": 0.94, + "learning_rate": 4.9203610832497494e-05, + "loss": 1.2652, + "theoretical_loss": 3.3408948277148895, + "tokens_seen": 3099983872 + }, + { + "epoch": 0.94, + "learning_rate": 4.9187562688064195e-05, + "loss": 1.2598, + "theoretical_loss": 3.340889479403155, + "tokens_seen": 3100049408 + }, + { + "epoch": 0.94, + "learning_rate": 4.9171514543630896e-05, + "loss": 1.1036, + "theoretical_loss": 3.340884131236142, + "tokens_seen": 3100114944 + }, + { + "epoch": 0.94, + "learning_rate": 4.9155466399197596e-05, + "loss": 1.1471, + "theoretical_loss": 3.340878783213843, + "tokens_seen": 3100180480 + }, + { + "epoch": 0.94, + "learning_rate": 4.91394182547643e-05, + "loss": 1.2101, + "theoretical_loss": 3.3408734353362513, + "tokens_seen": 3100246016 + }, + { + "epoch": 0.94, + "learning_rate": 4.9123370110331e-05, + "loss": 1.223, + "theoretical_loss": 3.3408680876033596, + "tokens_seen": 3100311552 + }, + { + "epoch": 0.94, + "learning_rate": 4.91073219658977e-05, + "loss": 1.156, + "theoretical_loss": 3.3408627400151616, + "tokens_seen": 3100377088 + }, + { + "epoch": 0.94, + "learning_rate": 4.909127382146439e-05, + "loss": 1.24, + "theoretical_loss": 3.3408573925716496, + "tokens_seen": 3100442624 + }, + { + "epoch": 0.94, + "learning_rate": 4.907522567703109e-05, + "loss": 1.1329, + "theoretical_loss": 3.3408520452728174, + "tokens_seen": 3100508160 + }, + { + "epoch": 0.94, + "learning_rate": 4.905917753259779e-05, + "loss": 1.1988, + "theoretical_loss": 3.340846698118657, + "tokens_seen": 3100573696 + }, + { + "epoch": 0.94, + "learning_rate": 4.9043129388164494e-05, + "loss": 1.2258, + "theoretical_loss": 3.3408413511091624, + "tokens_seen": 3100639232 + }, + { + "epoch": 0.94, + "learning_rate": 4.9027081243731194e-05, + "loss": 1.1229, + "theoretical_loss": 3.340836004244326, + "tokens_seen": 3100704768 + }, + { + "epoch": 0.94, + "learning_rate": 4.9011033099297895e-05, + "loss": 1.2106, + "theoretical_loss": 3.3408306575241413, + "tokens_seen": 3100770304 + }, + { + "epoch": 0.94, + "learning_rate": 4.8994984954864596e-05, + "loss": 1.2528, + "theoretical_loss": 3.340825310948601, + "tokens_seen": 3100835840 + }, + { + "epoch": 0.94, + "learning_rate": 4.8978936810431296e-05, + "loss": 1.3026, + "theoretical_loss": 3.3408199645176984, + "tokens_seen": 3100901376 + }, + { + "epoch": 0.94, + "learning_rate": 4.8962888665998e-05, + "loss": 1.126, + "theoretical_loss": 3.340814618231426, + "tokens_seen": 3100966912 + }, + { + "epoch": 0.94, + "learning_rate": 4.89468405215647e-05, + "loss": 1.1285, + "theoretical_loss": 3.340809272089777, + "tokens_seen": 3101032448 + }, + { + "epoch": 0.94, + "learning_rate": 4.89307923771314e-05, + "loss": 1.1693, + "theoretical_loss": 3.340803926092745, + "tokens_seen": 3101097984 + }, + { + "epoch": 0.94, + "learning_rate": 4.891474423269809e-05, + "loss": 1.174, + "theoretical_loss": 3.3407985802403224, + "tokens_seen": 3101163520 + }, + { + "epoch": 0.94, + "learning_rate": 4.889869608826479e-05, + "loss": 1.1659, + "theoretical_loss": 3.340793234532503, + "tokens_seen": 3101229056 + }, + { + "epoch": 0.94, + "learning_rate": 4.8882647943831493e-05, + "loss": 1.2108, + "theoretical_loss": 3.340787888969279, + "tokens_seen": 3101294592 + }, + { + "epoch": 0.94, + "learning_rate": 4.88665997993982e-05, + "loss": 1.1742, + "theoretical_loss": 3.3407825435506435, + "tokens_seen": 3101360128 + }, + { + "epoch": 0.94, + "learning_rate": 4.88505516549649e-05, + "loss": 1.1052, + "theoretical_loss": 3.34077719827659, + "tokens_seen": 3101425664 + }, + { + "epoch": 0.94, + "learning_rate": 4.8834503510531595e-05, + "loss": 1.2941, + "theoretical_loss": 3.340771853147111, + "tokens_seen": 3101491200 + }, + { + "epoch": 0.94, + "learning_rate": 4.8818455366098296e-05, + "loss": 1.196, + "theoretical_loss": 3.3407665081622, + "tokens_seen": 3101556736 + }, + { + "epoch": 0.94, + "learning_rate": 4.8802407221664997e-05, + "loss": 1.161, + "theoretical_loss": 3.34076116332185, + "tokens_seen": 3101622272 + }, + { + "epoch": 0.94, + "learning_rate": 4.87863590772317e-05, + "loss": 1.1233, + "theoretical_loss": 3.3407558186260538, + "tokens_seen": 3101687808 + }, + { + "epoch": 0.94, + "learning_rate": 4.87703109327984e-05, + "loss": 1.2373, + "theoretical_loss": 3.3407504740748046, + "tokens_seen": 3101753344 + }, + { + "epoch": 0.94, + "learning_rate": 4.87542627883651e-05, + "loss": 1.0787, + "theoretical_loss": 3.3407451296680954, + "tokens_seen": 3101818880 + }, + { + "epoch": 0.94, + "learning_rate": 4.873821464393179e-05, + "loss": 1.1647, + "theoretical_loss": 3.3407397854059195, + "tokens_seen": 3101884416 + }, + { + "epoch": 0.94, + "learning_rate": 4.87221664994985e-05, + "loss": 1.2259, + "theoretical_loss": 3.3407344412882694, + "tokens_seen": 3101949952 + }, + { + "epoch": 0.94, + "learning_rate": 4.87061183550652e-05, + "loss": 1.162, + "theoretical_loss": 3.3407290973151387, + "tokens_seen": 3102015488 + }, + { + "epoch": 0.94, + "learning_rate": 4.86900702106319e-05, + "loss": 1.0934, + "theoretical_loss": 3.3407237534865195, + "tokens_seen": 3102081024 + }, + { + "epoch": 0.94, + "learning_rate": 4.86740220661986e-05, + "loss": 1.1737, + "theoretical_loss": 3.340718409802406, + "tokens_seen": 3102146560 + }, + { + "epoch": 0.94, + "learning_rate": 4.8657973921765296e-05, + "loss": 1.2174, + "theoretical_loss": 3.3407130662627904, + "tokens_seen": 3102212096 + }, + { + "epoch": 0.94, + "learning_rate": 4.8641925777331996e-05, + "loss": 1.2311, + "theoretical_loss": 3.3407077228676663, + "tokens_seen": 3102277632 + }, + { + "epoch": 0.94, + "learning_rate": 4.86258776328987e-05, + "loss": 1.1821, + "theoretical_loss": 3.340702379617026, + "tokens_seen": 3102343168 + }, + { + "epoch": 0.94, + "learning_rate": 4.86098294884654e-05, + "loss": 1.1945, + "theoretical_loss": 3.3406970365108637, + "tokens_seen": 3102408704 + }, + { + "epoch": 0.94, + "learning_rate": 4.85937813440321e-05, + "loss": 1.1288, + "theoretical_loss": 3.3406916935491715, + "tokens_seen": 3102474240 + }, + { + "epoch": 0.94, + "learning_rate": 4.85777331995988e-05, + "loss": 1.2342, + "theoretical_loss": 3.340686350731943, + "tokens_seen": 3102539776 + }, + { + "epoch": 0.94, + "learning_rate": 4.85616850551655e-05, + "loss": 1.1298, + "theoretical_loss": 3.3406810080591707, + "tokens_seen": 3102605312 + }, + { + "epoch": 0.94, + "learning_rate": 4.85456369107322e-05, + "loss": 1.3062, + "theoretical_loss": 3.340675665530848, + "tokens_seen": 3102670848 + }, + { + "epoch": 0.94, + "learning_rate": 4.85295887662989e-05, + "loss": 1.3909, + "theoretical_loss": 3.3406703231469677, + "tokens_seen": 3102736384 + }, + { + "epoch": 0.94, + "learning_rate": 4.85135406218656e-05, + "loss": 1.2238, + "theoretical_loss": 3.340664980907523, + "tokens_seen": 3102801920 + }, + { + "epoch": 0.94, + "learning_rate": 4.84974924774323e-05, + "loss": 1.2092, + "theoretical_loss": 3.340659638812507, + "tokens_seen": 3102867456 + }, + { + "epoch": 0.94, + "learning_rate": 4.8481444332998996e-05, + "loss": 1.1842, + "theoretical_loss": 3.3406542968619126, + "tokens_seen": 3102932992 + }, + { + "epoch": 0.94, + "learning_rate": 4.8465396188565696e-05, + "loss": 1.1758, + "theoretical_loss": 3.340648955055733, + "tokens_seen": 3102998528 + }, + { + "epoch": 0.94, + "learning_rate": 4.84493480441324e-05, + "loss": 1.1921, + "theoretical_loss": 3.3406436133939614, + "tokens_seen": 3103064064 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1035795, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3543717861175537, + "objective/train/theoretical_loss": 3.3406382718765903, + "objective/train/tokens_used": 3123589600, + "theoretical_loss": 3.3406382718765903, + "tokens_seen": 3103129600 + }, + { + "epoch": 0.94, + "learning_rate": 4.84332998996991e-05, + "loss": 1.1947, + "theoretical_loss": 3.3406382718765903, + "tokens_seen": 3103129600 + }, + { + "epoch": 0.94, + "learning_rate": 4.8417251755265805e-05, + "loss": 1.1948, + "theoretical_loss": 3.340632930503613, + "tokens_seen": 3103195136 + }, + { + "epoch": 0.94, + "learning_rate": 4.84012036108325e-05, + "loss": 1.1446, + "theoretical_loss": 3.340627589275023, + "tokens_seen": 3103260672 + }, + { + "epoch": 0.94, + "learning_rate": 4.83851554663992e-05, + "loss": 1.1939, + "theoretical_loss": 3.340622248190813, + "tokens_seen": 3103326208 + }, + { + "epoch": 0.94, + "learning_rate": 4.83691073219659e-05, + "loss": 1.2015, + "theoretical_loss": 3.340616907250976, + "tokens_seen": 3103391744 + }, + { + "epoch": 0.94, + "learning_rate": 4.83530591775326e-05, + "loss": 1.1882, + "theoretical_loss": 3.3406115664555047, + "tokens_seen": 3103457280 + }, + { + "epoch": 0.94, + "learning_rate": 4.83370110330993e-05, + "loss": 1.21, + "theoretical_loss": 3.3406062258043927, + "tokens_seen": 3103522816 + }, + { + "epoch": 0.94, + "learning_rate": 4.8320962888666e-05, + "loss": 1.194, + "theoretical_loss": 3.3406008852976328, + "tokens_seen": 3103588352 + }, + { + "epoch": 0.94, + "learning_rate": 4.8304914744232696e-05, + "loss": 1.2025, + "theoretical_loss": 3.3405955449352183, + "tokens_seen": 3103653888 + }, + { + "epoch": 0.94, + "learning_rate": 4.8288866599799397e-05, + "loss": 1.1629, + "theoretical_loss": 3.3405902047171416, + "tokens_seen": 3103719424 + }, + { + "epoch": 0.94, + "learning_rate": 4.8272818455366104e-05, + "loss": 1.2267, + "theoretical_loss": 3.3405848646433967, + "tokens_seen": 3103784960 + }, + { + "epoch": 0.94, + "learning_rate": 4.8256770310932805e-05, + "loss": 1.188, + "theoretical_loss": 3.3405795247139762, + "tokens_seen": 3103850496 + }, + { + "epoch": 0.94, + "learning_rate": 4.8240722166499505e-05, + "loss": 1.1398, + "theoretical_loss": 3.340574184928873, + "tokens_seen": 3103916032 + }, + { + "epoch": 0.94, + "learning_rate": 4.82246740220662e-05, + "loss": 1.2808, + "theoretical_loss": 3.34056884528808, + "tokens_seen": 3103981568 + }, + { + "epoch": 0.94, + "learning_rate": 4.82086258776329e-05, + "loss": 1.2157, + "theoretical_loss": 3.340563505791591, + "tokens_seen": 3104047104 + }, + { + "epoch": 0.94, + "learning_rate": 4.81925777331996e-05, + "loss": 1.2, + "theoretical_loss": 3.340558166439398, + "tokens_seen": 3104112640 + }, + { + "epoch": 0.94, + "learning_rate": 4.81765295887663e-05, + "loss": 1.1969, + "theoretical_loss": 3.340552827231495, + "tokens_seen": 3104178176 + }, + { + "epoch": 0.94, + "learning_rate": 4.8160481444333e-05, + "loss": 1.2035, + "theoretical_loss": 3.340547488167875, + "tokens_seen": 3104243712 + }, + { + "epoch": 0.94, + "learning_rate": 4.81444332998997e-05, + "loss": 1.1747, + "theoretical_loss": 3.34054214924853, + "tokens_seen": 3104309248 + }, + { + "epoch": 0.94, + "learning_rate": 4.8128385155466396e-05, + "loss": 1.1748, + "theoretical_loss": 3.3405368104734543, + "tokens_seen": 3104374784 + }, + { + "epoch": 0.94, + "learning_rate": 4.8112337011033104e-05, + "loss": 1.1727, + "theoretical_loss": 3.3405314718426404, + "tokens_seen": 3104440320 + }, + { + "epoch": 0.94, + "learning_rate": 4.8096288866599804e-05, + "loss": 1.2187, + "theoretical_loss": 3.3405261333560814, + "tokens_seen": 3104505856 + }, + { + "epoch": 0.94, + "learning_rate": 4.8080240722166505e-05, + "loss": 1.1856, + "theoretical_loss": 3.34052079501377, + "tokens_seen": 3104571392 + }, + { + "epoch": 0.94, + "learning_rate": 4.8064192577733205e-05, + "loss": 1.1647, + "theoretical_loss": 3.3405154568157003, + "tokens_seen": 3104636928 + }, + { + "epoch": 0.94, + "learning_rate": 4.80481444332999e-05, + "loss": 1.1884, + "theoretical_loss": 3.340510118761864, + "tokens_seen": 3104702464 + }, + { + "epoch": 0.94, + "learning_rate": 4.80320962888666e-05, + "loss": 1.236, + "theoretical_loss": 3.3405047808522554, + "tokens_seen": 3104768000 + }, + { + "epoch": 0.94, + "learning_rate": 4.80160481444333e-05, + "loss": 1.2375, + "theoretical_loss": 3.3404994430868666, + "tokens_seen": 3104833536 + }, + { + "epoch": 0.94, + "learning_rate": 4.8e-05, + "loss": 1.1194, + "theoretical_loss": 3.3404941054656914, + "tokens_seen": 3104899072 + }, + { + "epoch": 0.94, + "learning_rate": 4.79839518555667e-05, + "loss": 1.2356, + "theoretical_loss": 3.340488767988722, + "tokens_seen": 3104964608 + }, + { + "epoch": 0.94, + "learning_rate": 4.79679037111334e-05, + "loss": 1.198, + "theoretical_loss": 3.3404834306559525, + "tokens_seen": 3105030144 + }, + { + "epoch": 0.94, + "learning_rate": 4.79518555667001e-05, + "loss": 1.209, + "theoretical_loss": 3.3404780934673752, + "tokens_seen": 3105095680 + }, + { + "epoch": 0.94, + "learning_rate": 4.7935807422266804e-05, + "loss": 1.2894, + "theoretical_loss": 3.3404727564229835, + "tokens_seen": 3105161216 + }, + { + "epoch": 0.94, + "learning_rate": 4.7919759277833504e-05, + "loss": 1.1935, + "theoretical_loss": 3.3404674195227706, + "tokens_seen": 3105226752 + }, + { + "epoch": 0.94, + "learning_rate": 4.7903711133400205e-05, + "loss": 1.2186, + "theoretical_loss": 3.340462082766729, + "tokens_seen": 3105292288 + }, + { + "epoch": 0.94, + "learning_rate": 4.7887662988966906e-05, + "loss": 1.1553, + "theoretical_loss": 3.340456746154852, + "tokens_seen": 3105357824 + }, + { + "epoch": 0.94, + "learning_rate": 4.78716148445336e-05, + "loss": 1.1875, + "theoretical_loss": 3.340451409687133, + "tokens_seen": 3105423360 + }, + { + "epoch": 0.94, + "learning_rate": 4.78555667001003e-05, + "loss": 1.2175, + "theoretical_loss": 3.3404460733635646, + "tokens_seen": 3105488896 + }, + { + "epoch": 0.94, + "learning_rate": 4.7839518555667e-05, + "loss": 1.2299, + "theoretical_loss": 3.3404407371841405, + "tokens_seen": 3105554432 + }, + { + "epoch": 0.94, + "learning_rate": 4.78234704112337e-05, + "loss": 1.1829, + "theoretical_loss": 3.3404354011488526, + "tokens_seen": 3105619968 + }, + { + "epoch": 0.94, + "learning_rate": 4.780742226680041e-05, + "loss": 1.2465, + "theoretical_loss": 3.3404300652576953, + "tokens_seen": 3105685504 + }, + { + "epoch": 0.94, + "learning_rate": 4.77913741223671e-05, + "loss": 1.1677, + "theoretical_loss": 3.340424729510661, + "tokens_seen": 3105751040 + }, + { + "epoch": 0.94, + "learning_rate": 4.77753259779338e-05, + "loss": 1.1622, + "theoretical_loss": 3.3404193939077427, + "tokens_seen": 3105816576 + }, + { + "epoch": 0.94, + "learning_rate": 4.7759277833500504e-05, + "loss": 1.1855, + "theoretical_loss": 3.3404140584489337, + "tokens_seen": 3105882112 + }, + { + "epoch": 0.94, + "learning_rate": 4.7743229689067205e-05, + "loss": 1.1102, + "theoretical_loss": 3.3404087231342268, + "tokens_seen": 3105947648 + }, + { + "epoch": 0.94, + "learning_rate": 4.7727181544633905e-05, + "loss": 1.2258, + "theoretical_loss": 3.3404033879636157, + "tokens_seen": 3106013184 + }, + { + "epoch": 0.94, + "learning_rate": 4.7711133400200606e-05, + "loss": 1.1863, + "theoretical_loss": 3.3403980529370925, + "tokens_seen": 3106078720 + }, + { + "epoch": 0.94, + "learning_rate": 4.76950852557673e-05, + "loss": 1.3038, + "theoretical_loss": 3.340392718054651, + "tokens_seen": 3106144256 + }, + { + "epoch": 0.94, + "learning_rate": 4.7679037111334e-05, + "loss": 1.1821, + "theoretical_loss": 3.3403873833162843, + "tokens_seen": 3106209792 + }, + { + "epoch": 0.94, + "learning_rate": 4.766298896690071e-05, + "loss": 1.2209, + "theoretical_loss": 3.3403820487219846, + "tokens_seen": 3106275328 + }, + { + "epoch": 0.94, + "learning_rate": 4.764694082246741e-05, + "loss": 1.222, + "theoretical_loss": 3.340376714271746, + "tokens_seen": 3106340864 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1036701, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2195053100585938, + "objective/train/theoretical_loss": 3.340371379965561, + "objective/train/tokens_used": 3126866400, + "theoretical_loss": 3.340371379965561, + "tokens_seen": 3106406400 + }, + { + "epoch": 0.94, + "learning_rate": 4.763089267803411e-05, + "loss": 1.19, + "theoretical_loss": 3.340371379965561, + "tokens_seen": 3106406400 + }, + { + "epoch": 0.94, + "learning_rate": 4.76148445336008e-05, + "loss": 1.1471, + "theoretical_loss": 3.3403660458034232, + "tokens_seen": 3106471936 + }, + { + "epoch": 0.94, + "learning_rate": 4.7598796389167504e-05, + "loss": 1.3348, + "theoretical_loss": 3.340360711785325, + "tokens_seen": 3106537472 + }, + { + "epoch": 0.94, + "learning_rate": 4.7582748244734204e-05, + "loss": 1.202, + "theoretical_loss": 3.34035537791126, + "tokens_seen": 3106603008 + }, + { + "epoch": 0.94, + "learning_rate": 4.7566700100300905e-05, + "loss": 1.222, + "theoretical_loss": 3.3403500441812204, + "tokens_seen": 3106668544 + }, + { + "epoch": 0.94, + "learning_rate": 4.7550651955867605e-05, + "loss": 1.2736, + "theoretical_loss": 3.3403447105952004, + "tokens_seen": 3106734080 + }, + { + "epoch": 0.94, + "learning_rate": 4.7534603811434306e-05, + "loss": 1.2707, + "theoretical_loss": 3.340339377153193, + "tokens_seen": 3106799616 + }, + { + "epoch": 0.94, + "learning_rate": 4.751855566700101e-05, + "loss": 1.2112, + "theoretical_loss": 3.34033404385519, + "tokens_seen": 3106865152 + }, + { + "epoch": 0.94, + "learning_rate": 4.750250752256771e-05, + "loss": 1.1769, + "theoretical_loss": 3.340328710701186, + "tokens_seen": 3106930688 + }, + { + "epoch": 0.94, + "learning_rate": 4.748645937813441e-05, + "loss": 1.1154, + "theoretical_loss": 3.340323377691173, + "tokens_seen": 3106996224 + }, + { + "epoch": 0.94, + "learning_rate": 4.747041123370111e-05, + "loss": 1.222, + "theoretical_loss": 3.340318044825145, + "tokens_seen": 3107061760 + }, + { + "epoch": 0.94, + "learning_rate": 4.745436308926781e-05, + "loss": 1.1392, + "theoretical_loss": 3.3403127121030938, + "tokens_seen": 3107127296 + }, + { + "epoch": 0.94, + "learning_rate": 4.74383149448345e-05, + "loss": 1.2578, + "theoretical_loss": 3.3403073795250138, + "tokens_seen": 3107192832 + }, + { + "epoch": 0.94, + "learning_rate": 4.7422266800401204e-05, + "loss": 1.175, + "theoretical_loss": 3.3403020470908973, + "tokens_seen": 3107258368 + }, + { + "epoch": 0.94, + "learning_rate": 4.7406218655967904e-05, + "loss": 1.2346, + "theoretical_loss": 3.3402967148007376, + "tokens_seen": 3107323904 + }, + { + "epoch": 0.94, + "learning_rate": 4.7390170511534605e-05, + "loss": 1.2807, + "theoretical_loss": 3.340291382654528, + "tokens_seen": 3107389440 + }, + { + "epoch": 0.94, + "learning_rate": 4.7374122367101306e-05, + "loss": 1.166, + "theoretical_loss": 3.3402860506522614, + "tokens_seen": 3107454976 + }, + { + "epoch": 0.94, + "learning_rate": 4.7358074222668006e-05, + "loss": 1.1488, + "theoretical_loss": 3.3402807187939305, + "tokens_seen": 3107520512 + }, + { + "epoch": 0.94, + "learning_rate": 4.734202607823471e-05, + "loss": 1.196, + "theoretical_loss": 3.3402753870795285, + "tokens_seen": 3107586048 + }, + { + "epoch": 0.94, + "learning_rate": 4.732597793380141e-05, + "loss": 1.1622, + "theoretical_loss": 3.340270055509049, + "tokens_seen": 3107651584 + }, + { + "epoch": 0.94, + "learning_rate": 4.730992978936811e-05, + "loss": 1.3064, + "theoretical_loss": 3.340264724082485, + "tokens_seen": 3107717120 + }, + { + "epoch": 0.94, + "learning_rate": 4.729388164493481e-05, + "loss": 1.2134, + "theoretical_loss": 3.3402593927998296, + "tokens_seen": 3107782656 + }, + { + "epoch": 0.94, + "learning_rate": 4.727783350050151e-05, + "loss": 1.1965, + "theoretical_loss": 3.340254061661075, + "tokens_seen": 3107848192 + }, + { + "epoch": 0.94, + "learning_rate": 4.72617853560682e-05, + "loss": 1.2565, + "theoretical_loss": 3.340248730666215, + "tokens_seen": 3107913728 + }, + { + "epoch": 0.94, + "learning_rate": 4.7245737211634904e-05, + "loss": 1.2458, + "theoretical_loss": 3.3402433998152428, + "tokens_seen": 3107979264 + }, + { + "epoch": 0.94, + "learning_rate": 4.7229689067201605e-05, + "loss": 1.157, + "theoretical_loss": 3.340238069108151, + "tokens_seen": 3108044800 + }, + { + "epoch": 0.94, + "learning_rate": 4.721364092276831e-05, + "loss": 1.2238, + "theoretical_loss": 3.340232738544933, + "tokens_seen": 3108110336 + }, + { + "epoch": 0.94, + "learning_rate": 4.7197592778335006e-05, + "loss": 1.1294, + "theoretical_loss": 3.3402274081255823, + "tokens_seen": 3108175872 + }, + { + "epoch": 0.94, + "learning_rate": 4.7181544633901706e-05, + "loss": 1.2358, + "theoretical_loss": 3.340222077850091, + "tokens_seen": 3108241408 + }, + { + "epoch": 0.94, + "learning_rate": 4.716549648946841e-05, + "loss": 1.1735, + "theoretical_loss": 3.340216747718453, + "tokens_seen": 3108306944 + }, + { + "epoch": 0.94, + "learning_rate": 4.714944834503511e-05, + "loss": 1.274, + "theoretical_loss": 3.3402114177306608, + "tokens_seen": 3108372480 + }, + { + "epoch": 0.94, + "learning_rate": 4.713340020060181e-05, + "loss": 1.1472, + "theoretical_loss": 3.340206087886708, + "tokens_seen": 3108438016 + }, + { + "epoch": 0.94, + "learning_rate": 4.711735205616851e-05, + "loss": 1.2254, + "theoretical_loss": 3.3402007581865876, + "tokens_seen": 3108503552 + }, + { + "epoch": 0.94, + "learning_rate": 4.71013039117352e-05, + "loss": 1.2298, + "theoretical_loss": 3.3401954286302926, + "tokens_seen": 3108569088 + }, + { + "epoch": 0.94, + "learning_rate": 4.7085255767301904e-05, + "loss": 1.2443, + "theoretical_loss": 3.3401900992178155, + "tokens_seen": 3108634624 + }, + { + "epoch": 0.94, + "learning_rate": 4.706920762286861e-05, + "loss": 1.2122, + "theoretical_loss": 3.3401847699491505, + "tokens_seen": 3108700160 + }, + { + "epoch": 0.94, + "learning_rate": 4.705315947843531e-05, + "loss": 1.2214, + "theoretical_loss": 3.34017944082429, + "tokens_seen": 3108765696 + }, + { + "epoch": 0.94, + "learning_rate": 4.703711133400201e-05, + "loss": 1.2041, + "theoretical_loss": 3.340174111843227, + "tokens_seen": 3108831232 + }, + { + "epoch": 0.94, + "learning_rate": 4.7021063189568706e-05, + "loss": 1.1791, + "theoretical_loss": 3.3401687830059545, + "tokens_seen": 3108896768 + }, + { + "epoch": 0.94, + "learning_rate": 4.700501504513541e-05, + "loss": 1.2198, + "theoretical_loss": 3.3401634543124663, + "tokens_seen": 3108962304 + }, + { + "epoch": 0.94, + "learning_rate": 4.698896690070211e-05, + "loss": 1.1681, + "theoretical_loss": 3.340158125762755, + "tokens_seen": 3109027840 + }, + { + "epoch": 0.94, + "learning_rate": 4.697291875626881e-05, + "loss": 1.2183, + "theoretical_loss": 3.340152797356814, + "tokens_seen": 3109093376 + }, + { + "epoch": 0.94, + "learning_rate": 4.695687061183551e-05, + "loss": 1.0511, + "theoretical_loss": 3.340147469094636, + "tokens_seen": 3109158912 + }, + { + "epoch": 0.94, + "learning_rate": 4.694082246740221e-05, + "loss": 1.1698, + "theoretical_loss": 3.340142140976214, + "tokens_seen": 3109224448 + }, + { + "epoch": 0.94, + "learning_rate": 4.69247743229689e-05, + "loss": 1.2798, + "theoretical_loss": 3.3401368130015414, + "tokens_seen": 3109289984 + }, + { + "epoch": 0.94, + "learning_rate": 4.690872617853561e-05, + "loss": 1.2133, + "theoretical_loss": 3.3401314851706116, + "tokens_seen": 3109355520 + }, + { + "epoch": 0.94, + "learning_rate": 4.689267803410231e-05, + "loss": 1.1979, + "theoretical_loss": 3.340126157483417, + "tokens_seen": 3109421056 + }, + { + "epoch": 0.94, + "learning_rate": 4.687662988966901e-05, + "loss": 1.1873, + "theoretical_loss": 3.340120829939951, + "tokens_seen": 3109486592 + }, + { + "epoch": 0.94, + "learning_rate": 4.686058174523571e-05, + "loss": 1.1869, + "theoretical_loss": 3.3401155025402067, + "tokens_seen": 3109552128 + }, + { + "epoch": 0.94, + "learning_rate": 4.6844533600802406e-05, + "loss": 1.1647, + "theoretical_loss": 3.340110175284177, + "tokens_seen": 3109617664 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1037523, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2590134143829346, + "objective/train/theoretical_loss": 3.3401048481718556, + "objective/train/tokens_used": 3130143200, + "theoretical_loss": 3.3401048481718556, + "tokens_seen": 3109683200 + }, + { + "epoch": 0.94, + "learning_rate": 4.682848545636911e-05, + "loss": 1.2087, + "theoretical_loss": 3.3401048481718556, + "tokens_seen": 3109683200 + }, + { + "epoch": 0.94, + "learning_rate": 4.681243731193581e-05, + "loss": 1.1546, + "theoretical_loss": 3.340099521203235, + "tokens_seen": 3109748736 + }, + { + "epoch": 0.94, + "learning_rate": 4.679638916750251e-05, + "loss": 1.1755, + "theoretical_loss": 3.3400941943783087, + "tokens_seen": 3109814272 + }, + { + "epoch": 0.94, + "learning_rate": 4.678034102306921e-05, + "loss": 1.1968, + "theoretical_loss": 3.3400888676970695, + "tokens_seen": 3109879808 + }, + { + "epoch": 0.94, + "learning_rate": 4.676429287863591e-05, + "loss": 1.2205, + "theoretical_loss": 3.34008354115951, + "tokens_seen": 3109945344 + }, + { + "epoch": 0.94, + "learning_rate": 4.674824473420261e-05, + "loss": 1.1736, + "theoretical_loss": 3.3400782147656245, + "tokens_seen": 3110010880 + }, + { + "epoch": 0.94, + "learning_rate": 4.673219658976931e-05, + "loss": 1.2514, + "theoretical_loss": 3.3400728885154054, + "tokens_seen": 3110076416 + }, + { + "epoch": 0.94, + "learning_rate": 4.671614844533601e-05, + "loss": 1.1779, + "theoretical_loss": 3.3400675624088456, + "tokens_seen": 3110141952 + }, + { + "epoch": 0.94, + "learning_rate": 4.670010030090271e-05, + "loss": 1.1396, + "theoretical_loss": 3.340062236445939, + "tokens_seen": 3110207488 + }, + { + "epoch": 0.94, + "learning_rate": 4.668405215646941e-05, + "loss": 1.2581, + "theoretical_loss": 3.3400569106266778, + "tokens_seen": 3110273024 + }, + { + "epoch": 0.94, + "learning_rate": 4.6668004012036106e-05, + "loss": 1.2474, + "theoretical_loss": 3.340051584951055, + "tokens_seen": 3110338560 + }, + { + "epoch": 0.94, + "learning_rate": 4.665195586760281e-05, + "loss": 1.1466, + "theoretical_loss": 3.340046259419065, + "tokens_seen": 3110404096 + }, + { + "epoch": 0.94, + "learning_rate": 4.663590772316951e-05, + "loss": 1.2337, + "theoretical_loss": 3.3400409340306996, + "tokens_seen": 3110469632 + }, + { + "epoch": 0.94, + "learning_rate": 4.6619859578736215e-05, + "loss": 1.1046, + "theoretical_loss": 3.3400356087859526, + "tokens_seen": 3110535168 + }, + { + "epoch": 0.94, + "learning_rate": 4.6603811434302916e-05, + "loss": 1.1966, + "theoretical_loss": 3.3400302836848166, + "tokens_seen": 3110600704 + }, + { + "epoch": 0.94, + "learning_rate": 4.658776328986961e-05, + "loss": 1.2152, + "theoretical_loss": 3.3400249587272848, + "tokens_seen": 3110666240 + }, + { + "epoch": 0.94, + "learning_rate": 4.657171514543631e-05, + "loss": 1.215, + "theoretical_loss": 3.340019633913351, + "tokens_seen": 3110731776 + }, + { + "epoch": 0.94, + "learning_rate": 4.655566700100301e-05, + "loss": 1.2112, + "theoretical_loss": 3.3400143092430072, + "tokens_seen": 3110797312 + }, + { + "epoch": 0.94, + "learning_rate": 4.653961885656971e-05, + "loss": 1.17, + "theoretical_loss": 3.3400089847162473, + "tokens_seen": 3110862848 + }, + { + "epoch": 0.94, + "learning_rate": 4.652357071213641e-05, + "loss": 1.2368, + "theoretical_loss": 3.3400036603330645, + "tokens_seen": 3110928384 + }, + { + "epoch": 0.94, + "learning_rate": 4.650752256770311e-05, + "loss": 1.2303, + "theoretical_loss": 3.3399983360934513, + "tokens_seen": 3110993920 + }, + { + "epoch": 0.94, + "learning_rate": 4.649147442326981e-05, + "loss": 1.1885, + "theoretical_loss": 3.339993011997401, + "tokens_seen": 3111059456 + }, + { + "epoch": 0.94, + "learning_rate": 4.647542627883651e-05, + "loss": 1.306, + "theoretical_loss": 3.339987688044907, + "tokens_seen": 3111124992 + }, + { + "epoch": 0.94, + "learning_rate": 4.6459378134403215e-05, + "loss": 1.259, + "theoretical_loss": 3.3399823642359623, + "tokens_seen": 3111190528 + }, + { + "epoch": 0.94, + "learning_rate": 4.6443329989969915e-05, + "loss": 1.187, + "theoretical_loss": 3.3399770405705596, + "tokens_seen": 3111256064 + }, + { + "epoch": 0.94, + "learning_rate": 4.6427281845536616e-05, + "loss": 1.2701, + "theoretical_loss": 3.3399717170486922, + "tokens_seen": 3111321600 + }, + { + "epoch": 0.94, + "learning_rate": 4.641123370110331e-05, + "loss": 1.2126, + "theoretical_loss": 3.3399663936703536, + "tokens_seen": 3111387136 + }, + { + "epoch": 0.94, + "learning_rate": 4.639518555667001e-05, + "loss": 1.1122, + "theoretical_loss": 3.3399610704355363, + "tokens_seen": 3111452672 + }, + { + "epoch": 0.94, + "learning_rate": 4.637913741223671e-05, + "loss": 1.1931, + "theoretical_loss": 3.3399557473442343, + "tokens_seen": 3111518208 + }, + { + "epoch": 0.94, + "learning_rate": 4.636308926780341e-05, + "loss": 1.1981, + "theoretical_loss": 3.3399504243964397, + "tokens_seen": 3111583744 + }, + { + "epoch": 0.94, + "learning_rate": 4.634704112337011e-05, + "loss": 1.2255, + "theoretical_loss": 3.339945101592146, + "tokens_seen": 3111649280 + }, + { + "epoch": 0.94, + "learning_rate": 4.633099297893681e-05, + "loss": 1.1868, + "theoretical_loss": 3.3399397789313467, + "tokens_seen": 3111714816 + }, + { + "epoch": 0.94, + "learning_rate": 4.6314944834503514e-05, + "loss": 1.187, + "theoretical_loss": 3.339934456414034, + "tokens_seen": 3111780352 + }, + { + "epoch": 0.94, + "learning_rate": 4.6298896690070214e-05, + "loss": 1.1404, + "theoretical_loss": 3.339929134040202, + "tokens_seen": 3111845888 + }, + { + "epoch": 0.94, + "learning_rate": 4.6282848545636915e-05, + "loss": 1.2141, + "theoretical_loss": 3.3399238118098435, + "tokens_seen": 3111911424 + }, + { + "epoch": 0.94, + "learning_rate": 4.6266800401203616e-05, + "loss": 1.0655, + "theoretical_loss": 3.339918489722951, + "tokens_seen": 3111976960 + }, + { + "epoch": 0.94, + "learning_rate": 4.6250752256770316e-05, + "loss": 1.2136, + "theoretical_loss": 3.3399131677795184, + "tokens_seen": 3112042496 + }, + { + "epoch": 0.94, + "learning_rate": 4.623470411233701e-05, + "loss": 1.1698, + "theoretical_loss": 3.339907845979539, + "tokens_seen": 3112108032 + }, + { + "epoch": 0.94, + "learning_rate": 4.621865596790371e-05, + "loss": 1.0989, + "theoretical_loss": 3.339902524323005, + "tokens_seen": 3112173568 + }, + { + "epoch": 0.94, + "learning_rate": 4.620260782347041e-05, + "loss": 1.0884, + "theoretical_loss": 3.33989720280991, + "tokens_seen": 3112239104 + }, + { + "epoch": 0.94, + "learning_rate": 4.618655967903711e-05, + "loss": 1.2321, + "theoretical_loss": 3.3398918814402467, + "tokens_seen": 3112304640 + }, + { + "epoch": 0.94, + "learning_rate": 4.617051153460382e-05, + "loss": 1.3067, + "theoretical_loss": 3.3398865602140084, + "tokens_seen": 3112370176 + }, + { + "epoch": 0.94, + "learning_rate": 4.615446339017051e-05, + "loss": 1.2123, + "theoretical_loss": 3.339881239131189, + "tokens_seen": 3112435712 + }, + { + "epoch": 0.94, + "learning_rate": 4.6138415245737214e-05, + "loss": 1.1789, + "theoretical_loss": 3.339875918191781, + "tokens_seen": 3112501248 + }, + { + "epoch": 0.94, + "learning_rate": 4.6122367101303914e-05, + "loss": 1.1345, + "theoretical_loss": 3.339870597395777, + "tokens_seen": 3112566784 + }, + { + "epoch": 0.94, + "learning_rate": 4.6106318956870615e-05, + "loss": 1.2252, + "theoretical_loss": 3.339865276743171, + "tokens_seen": 3112632320 + }, + { + "epoch": 0.94, + "learning_rate": 4.6090270812437316e-05, + "loss": 1.0877, + "theoretical_loss": 3.3398599562339557, + "tokens_seen": 3112697856 + }, + { + "epoch": 0.94, + "learning_rate": 4.6074222668004016e-05, + "loss": 1.2382, + "theoretical_loss": 3.3398546358681243, + "tokens_seen": 3112763392 + }, + { + "epoch": 0.94, + "learning_rate": 4.605817452357071e-05, + "loss": 1.1848, + "theoretical_loss": 3.33984931564567, + "tokens_seen": 3112828928 + }, + { + "epoch": 0.94, + "learning_rate": 4.604212637913741e-05, + "loss": 1.1632, + "theoretical_loss": 3.3398439955665857, + "tokens_seen": 3112894464 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1038754, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2280139923095703, + "objective/train/theoretical_loss": 3.3398386756308645, + "objective/train/tokens_used": 3133420000, + "theoretical_loss": 3.3398386756308645, + "tokens_seen": 3112960000 + }, + { + "epoch": 0.94, + "learning_rate": 4.602607823470411e-05, + "loss": 1.2611, + "theoretical_loss": 3.3398386756308645, + "tokens_seen": 3112960000 + }, + { + "epoch": 0.94, + "learning_rate": 4.601003009027082e-05, + "loss": 1.2037, + "theoretical_loss": 3.3398333558384996, + "tokens_seen": 3113025536 + }, + { + "epoch": 0.94, + "learning_rate": 4.599398194583752e-05, + "loss": 1.1951, + "theoretical_loss": 3.3398280361894845, + "tokens_seen": 3113091072 + }, + { + "epoch": 0.94, + "learning_rate": 4.5977933801404213e-05, + "loss": 1.2489, + "theoretical_loss": 3.3398227166838117, + "tokens_seen": 3113156608 + }, + { + "epoch": 0.94, + "learning_rate": 4.5961885656970914e-05, + "loss": 1.1258, + "theoretical_loss": 3.3398173973214744, + "tokens_seen": 3113222144 + }, + { + "epoch": 0.94, + "learning_rate": 4.5945837512537615e-05, + "loss": 1.1124, + "theoretical_loss": 3.3398120781024665, + "tokens_seen": 3113287680 + }, + { + "epoch": 0.94, + "learning_rate": 4.5929789368104315e-05, + "loss": 1.1328, + "theoretical_loss": 3.3398067590267804, + "tokens_seen": 3113353216 + }, + { + "epoch": 0.94, + "learning_rate": 4.5913741223671016e-05, + "loss": 1.2825, + "theoretical_loss": 3.339801440094409, + "tokens_seen": 3113418752 + }, + { + "epoch": 0.94, + "learning_rate": 4.5897693079237717e-05, + "loss": 1.1825, + "theoretical_loss": 3.339796121305346, + "tokens_seen": 3113484288 + }, + { + "epoch": 0.94, + "learning_rate": 4.588164493480441e-05, + "loss": 1.1075, + "theoretical_loss": 3.339790802659584, + "tokens_seen": 3113549824 + }, + { + "epoch": 0.94, + "learning_rate": 4.586559679037112e-05, + "loss": 1.1425, + "theoretical_loss": 3.3397854841571166, + "tokens_seen": 3113615360 + }, + { + "epoch": 0.94, + "learning_rate": 4.584954864593782e-05, + "loss": 1.2432, + "theoretical_loss": 3.339780165797937, + "tokens_seen": 3113680896 + }, + { + "epoch": 0.94, + "learning_rate": 4.583350050150452e-05, + "loss": 1.261, + "theoretical_loss": 3.3397748475820377, + "tokens_seen": 3113746432 + }, + { + "epoch": 0.94, + "learning_rate": 4.581745235707122e-05, + "loss": 1.1437, + "theoretical_loss": 3.3397695295094123, + "tokens_seen": 3113811968 + }, + { + "epoch": 0.94, + "learning_rate": 4.5801404212637914e-05, + "loss": 1.2998, + "theoretical_loss": 3.339764211580054, + "tokens_seen": 3113877504 + }, + { + "epoch": 0.94, + "learning_rate": 4.5785356068204614e-05, + "loss": 1.1479, + "theoretical_loss": 3.339758893793956, + "tokens_seen": 3113943040 + }, + { + "epoch": 0.94, + "learning_rate": 4.5769307923771315e-05, + "loss": 1.1712, + "theoretical_loss": 3.339753576151111, + "tokens_seen": 3114008576 + }, + { + "epoch": 0.94, + "learning_rate": 4.5753259779338016e-05, + "loss": 1.2484, + "theoretical_loss": 3.3397482586515115, + "tokens_seen": 3114074112 + }, + { + "epoch": 0.94, + "learning_rate": 4.5737211634904716e-05, + "loss": 1.3082, + "theoretical_loss": 3.3397429412951523, + "tokens_seen": 3114139648 + }, + { + "epoch": 0.94, + "learning_rate": 4.572116349047142e-05, + "loss": 1.2566, + "theoretical_loss": 3.339737624082025, + "tokens_seen": 3114205184 + }, + { + "epoch": 0.94, + "learning_rate": 4.570511534603812e-05, + "loss": 1.2014, + "theoretical_loss": 3.339732307012124, + "tokens_seen": 3114270720 + }, + { + "epoch": 0.94, + "learning_rate": 4.568906720160482e-05, + "loss": 1.2029, + "theoretical_loss": 3.339726990085442, + "tokens_seen": 3114336256 + }, + { + "epoch": 0.94, + "learning_rate": 4.567301905717152e-05, + "loss": 1.2693, + "theoretical_loss": 3.339721673301971, + "tokens_seen": 3114401792 + }, + { + "epoch": 0.94, + "learning_rate": 4.565697091273822e-05, + "loss": 1.0998, + "theoretical_loss": 3.3397163566617056, + "tokens_seen": 3114467328 + }, + { + "epoch": 0.94, + "learning_rate": 4.564092276830492e-05, + "loss": 1.2038, + "theoretical_loss": 3.3397110401646386, + "tokens_seen": 3114532864 + }, + { + "epoch": 0.94, + "learning_rate": 4.5624874623871614e-05, + "loss": 1.2264, + "theoretical_loss": 3.339705723810763, + "tokens_seen": 3114598400 + }, + { + "epoch": 0.94, + "learning_rate": 4.5608826479438314e-05, + "loss": 1.2772, + "theoretical_loss": 3.339700407600071, + "tokens_seen": 3114663936 + }, + { + "epoch": 0.94, + "learning_rate": 4.5592778335005015e-05, + "loss": 1.2591, + "theoretical_loss": 3.3396950915325574, + "tokens_seen": 3114729472 + }, + { + "epoch": 0.94, + "learning_rate": 4.5576730190571716e-05, + "loss": 1.2201, + "theoretical_loss": 3.339689775608214, + "tokens_seen": 3114795008 + }, + { + "epoch": 0.94, + "learning_rate": 4.556068204613842e-05, + "loss": 1.1792, + "theoretical_loss": 3.339684459827035, + "tokens_seen": 3114860544 + }, + { + "epoch": 0.94, + "learning_rate": 4.554463390170512e-05, + "loss": 1.1554, + "theoretical_loss": 3.3396791441890126, + "tokens_seen": 3114926080 + }, + { + "epoch": 0.94, + "learning_rate": 4.552858575727182e-05, + "loss": 1.1868, + "theoretical_loss": 3.3396738286941403, + "tokens_seen": 3114991616 + }, + { + "epoch": 0.94, + "learning_rate": 4.551253761283852e-05, + "loss": 1.1972, + "theoretical_loss": 3.3396685133424118, + "tokens_seen": 3115057152 + }, + { + "epoch": 0.94, + "learning_rate": 4.549648946840522e-05, + "loss": 1.0994, + "theoretical_loss": 3.339663198133819, + "tokens_seen": 3115122688 + }, + { + "epoch": 0.94, + "learning_rate": 4.548044132397192e-05, + "loss": 1.1664, + "theoretical_loss": 3.339657883068356, + "tokens_seen": 3115188224 + }, + { + "epoch": 0.94, + "learning_rate": 4.546439317953862e-05, + "loss": 1.2134, + "theoretical_loss": 3.3396525681460156, + "tokens_seen": 3115253760 + }, + { + "epoch": 0.94, + "learning_rate": 4.5448345035105314e-05, + "loss": 1.2008, + "theoretical_loss": 3.339647253366791, + "tokens_seen": 3115319296 + }, + { + "epoch": 0.94, + "learning_rate": 4.5432296890672015e-05, + "loss": 1.1572, + "theoretical_loss": 3.3396419387306757, + "tokens_seen": 3115384832 + }, + { + "epoch": 0.94, + "learning_rate": 4.541624874623872e-05, + "loss": 1.1802, + "theoretical_loss": 3.339636624237662, + "tokens_seen": 3115450368 + }, + { + "epoch": 0.94, + "learning_rate": 4.540020060180542e-05, + "loss": 1.1447, + "theoretical_loss": 3.3396313098877437, + "tokens_seen": 3115515904 + }, + { + "epoch": 0.94, + "learning_rate": 4.538415245737212e-05, + "loss": 1.3335, + "theoretical_loss": 3.3396259956809136, + "tokens_seen": 3115581440 + }, + { + "epoch": 0.94, + "learning_rate": 4.536810431293882e-05, + "loss": 1.1531, + "theoretical_loss": 3.339620681617165, + "tokens_seen": 3115646976 + }, + { + "epoch": 0.94, + "learning_rate": 4.535205616850552e-05, + "loss": 1.1736, + "theoretical_loss": 3.339615367696491, + "tokens_seen": 3115712512 + }, + { + "epoch": 0.94, + "learning_rate": 4.533600802407222e-05, + "loss": 1.063, + "theoretical_loss": 3.339610053918885, + "tokens_seen": 3115778048 + }, + { + "epoch": 0.94, + "learning_rate": 4.531995987963892e-05, + "loss": 1.2019, + "theoretical_loss": 3.339604740284339, + "tokens_seen": 3115843584 + }, + { + "epoch": 0.94, + "learning_rate": 4.530391173520562e-05, + "loss": 1.1176, + "theoretical_loss": 3.339599426792848, + "tokens_seen": 3115909120 + }, + { + "epoch": 0.94, + "learning_rate": 4.528786359077232e-05, + "loss": 1.2327, + "theoretical_loss": 3.339594113444404, + "tokens_seen": 3115974656 + }, + { + "epoch": 0.94, + "learning_rate": 4.5271815446339014e-05, + "loss": 1.1642, + "theoretical_loss": 3.339588800239, + "tokens_seen": 3116040192 + }, + { + "epoch": 0.94, + "learning_rate": 4.525576730190572e-05, + "loss": 1.1412, + "theoretical_loss": 3.33958348717663, + "tokens_seen": 3116105728 + }, + { + "epoch": 0.94, + "learning_rate": 4.523971915747242e-05, + "loss": 1.1451, + "theoretical_loss": 3.339578174257286, + "tokens_seen": 3116171264 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1040123, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2071770429611206, + "objective/train/theoretical_loss": 3.3395728614809617, + "objective/train/tokens_used": 3136696800, + "theoretical_loss": 3.3395728614809617, + "tokens_seen": 3116236800 + }, + { + "epoch": 0.94, + "learning_rate": 4.522367101303912e-05, + "loss": 1.2078, + "theoretical_loss": 3.3395728614809617, + "tokens_seen": 3116236800 + }, + { + "epoch": 0.94, + "learning_rate": 4.520762286860582e-05, + "loss": 1.3028, + "theoretical_loss": 3.3395675488476506, + "tokens_seen": 3116302336 + }, + { + "epoch": 0.94, + "learning_rate": 4.519157472417252e-05, + "loss": 1.0762, + "theoretical_loss": 3.339562236357345, + "tokens_seen": 3116367872 + }, + { + "epoch": 0.94, + "learning_rate": 4.517552657973922e-05, + "loss": 1.1706, + "theoretical_loss": 3.339556924010039, + "tokens_seen": 3116433408 + }, + { + "epoch": 0.94, + "learning_rate": 4.515947843530592e-05, + "loss": 1.1224, + "theoretical_loss": 3.339551611805726, + "tokens_seen": 3116498944 + }, + { + "epoch": 0.94, + "learning_rate": 4.514343029087262e-05, + "loss": 1.1419, + "theoretical_loss": 3.3395462997443977, + "tokens_seen": 3116564480 + }, + { + "epoch": 0.94, + "learning_rate": 4.512738214643932e-05, + "loss": 1.2143, + "theoretical_loss": 3.339540987826048, + "tokens_seen": 3116630016 + }, + { + "epoch": 0.94, + "learning_rate": 4.511133400200602e-05, + "loss": 1.2657, + "theoretical_loss": 3.33953567605067, + "tokens_seen": 3116695552 + }, + { + "epoch": 0.94, + "learning_rate": 4.509528585757272e-05, + "loss": 1.1883, + "theoretical_loss": 3.339530364418257, + "tokens_seen": 3116761088 + }, + { + "epoch": 0.94, + "learning_rate": 4.507923771313942e-05, + "loss": 1.284, + "theoretical_loss": 3.339525052928802, + "tokens_seen": 3116826624 + }, + { + "epoch": 0.94, + "learning_rate": 4.506318956870612e-05, + "loss": 1.2445, + "theoretical_loss": 3.339519741582298, + "tokens_seen": 3116892160 + }, + { + "epoch": 0.94, + "learning_rate": 4.504714142427282e-05, + "loss": 1.2016, + "theoretical_loss": 3.3395144303787383, + "tokens_seen": 3116957696 + }, + { + "epoch": 0.94, + "learning_rate": 4.503109327983952e-05, + "loss": 1.1929, + "theoretical_loss": 3.3395091193181163, + "tokens_seen": 3117023232 + }, + { + "epoch": 0.94, + "learning_rate": 4.501504513540622e-05, + "loss": 1.225, + "theoretical_loss": 3.339503808400425, + "tokens_seen": 3117088768 + }, + { + "epoch": 0.94, + "learning_rate": 4.499899699097292e-05, + "loss": 1.148, + "theoretical_loss": 3.339498497625657, + "tokens_seen": 3117154304 + }, + { + "epoch": 0.94, + "learning_rate": 4.498294884653962e-05, + "loss": 1.2356, + "theoretical_loss": 3.3394931869938063, + "tokens_seen": 3117219840 + }, + { + "epoch": 0.94, + "learning_rate": 4.4966900702106326e-05, + "loss": 1.3051, + "theoretical_loss": 3.3394878765048657, + "tokens_seen": 3117285376 + }, + { + "epoch": 0.94, + "learning_rate": 4.495085255767302e-05, + "loss": 1.2657, + "theoretical_loss": 3.3394825661588285, + "tokens_seen": 3117350912 + }, + { + "epoch": 0.94, + "learning_rate": 4.493480441323972e-05, + "loss": 1.1911, + "theoretical_loss": 3.3394772559556873, + "tokens_seen": 3117416448 + }, + { + "epoch": 0.94, + "learning_rate": 4.491875626880642e-05, + "loss": 1.1796, + "theoretical_loss": 3.339471945895436, + "tokens_seen": 3117481984 + }, + { + "epoch": 0.94, + "learning_rate": 4.490270812437312e-05, + "loss": 1.2583, + "theoretical_loss": 3.339466635978067, + "tokens_seen": 3117547520 + }, + { + "epoch": 0.94, + "learning_rate": 4.488665997993982e-05, + "loss": 1.2034, + "theoretical_loss": 3.3394613262035735, + "tokens_seen": 3117613056 + }, + { + "epoch": 0.94, + "learning_rate": 4.487061183550652e-05, + "loss": 1.1787, + "theoretical_loss": 3.33945601657195, + "tokens_seen": 3117678592 + }, + { + "epoch": 0.94, + "learning_rate": 4.485456369107322e-05, + "loss": 1.1876, + "theoretical_loss": 3.339450707083188, + "tokens_seen": 3117744128 + }, + { + "epoch": 0.94, + "learning_rate": 4.483851554663992e-05, + "loss": 1.1876, + "theoretical_loss": 3.339445397737281, + "tokens_seen": 3117809664 + }, + { + "epoch": 0.94, + "learning_rate": 4.482246740220662e-05, + "loss": 1.2156, + "theoretical_loss": 3.339440088534223, + "tokens_seen": 3117875200 + }, + { + "epoch": 0.94, + "learning_rate": 4.4806419257773326e-05, + "loss": 1.1317, + "theoretical_loss": 3.339434779474006, + "tokens_seen": 3117940736 + }, + { + "epoch": 0.94, + "learning_rate": 4.4790371113340026e-05, + "loss": 1.2891, + "theoretical_loss": 3.339429470556624, + "tokens_seen": 3118006272 + }, + { + "epoch": 0.94, + "learning_rate": 4.477432296890672e-05, + "loss": 1.1536, + "theoretical_loss": 3.3394241617820697, + "tokens_seen": 3118071808 + }, + { + "epoch": 0.94, + "learning_rate": 4.475827482447342e-05, + "loss": 1.1863, + "theoretical_loss": 3.339418853150337, + "tokens_seen": 3118137344 + }, + { + "epoch": 0.94, + "learning_rate": 4.474222668004012e-05, + "loss": 1.2917, + "theoretical_loss": 3.339413544661418, + "tokens_seen": 3118202880 + }, + { + "epoch": 0.94, + "learning_rate": 4.472617853560682e-05, + "loss": 1.1582, + "theoretical_loss": 3.3394082363153066, + "tokens_seen": 3118268416 + }, + { + "epoch": 0.94, + "learning_rate": 4.471013039117352e-05, + "loss": 1.14, + "theoretical_loss": 3.339402928111996, + "tokens_seen": 3118333952 + }, + { + "epoch": 0.94, + "learning_rate": 4.4694082246740224e-05, + "loss": 1.0947, + "theoretical_loss": 3.3393976200514786, + "tokens_seen": 3118399488 + }, + { + "epoch": 0.94, + "learning_rate": 4.467803410230692e-05, + "loss": 1.2215, + "theoretical_loss": 3.3393923121337483, + "tokens_seen": 3118465024 + }, + { + "epoch": 0.95, + "learning_rate": 4.4661985957873625e-05, + "loss": 1.2735, + "theoretical_loss": 3.339387004358798, + "tokens_seen": 3118530560 + }, + { + "epoch": 0.95, + "learning_rate": 4.4645937813440325e-05, + "loss": 1.2187, + "theoretical_loss": 3.3393816967266208, + "tokens_seen": 3118596096 + }, + { + "epoch": 0.95, + "learning_rate": 4.4629889669007026e-05, + "loss": 1.2171, + "theoretical_loss": 3.3393763892372097, + "tokens_seen": 3118661632 + }, + { + "epoch": 0.95, + "learning_rate": 4.461384152457373e-05, + "loss": 1.1387, + "theoretical_loss": 3.339371081890558, + "tokens_seen": 3118727168 + }, + { + "epoch": 0.95, + "learning_rate": 4.459779338014042e-05, + "loss": 1.1109, + "theoretical_loss": 3.3393657746866596, + "tokens_seen": 3118792704 + }, + { + "epoch": 0.95, + "learning_rate": 4.458174523570712e-05, + "loss": 1.1687, + "theoretical_loss": 3.3393604676255064, + "tokens_seen": 3118858240 + }, + { + "epoch": 0.95, + "learning_rate": 4.456569709127382e-05, + "loss": 1.1358, + "theoretical_loss": 3.3393551607070924, + "tokens_seen": 3118923776 + }, + { + "epoch": 0.95, + "learning_rate": 4.454964894684052e-05, + "loss": 1.2258, + "theoretical_loss": 3.3393498539314104, + "tokens_seen": 3118989312 + }, + { + "epoch": 0.95, + "learning_rate": 4.453360080240722e-05, + "loss": 1.213, + "theoretical_loss": 3.3393445472984538, + "tokens_seen": 3119054848 + }, + { + "epoch": 0.95, + "learning_rate": 4.4517552657973924e-05, + "loss": 1.2591, + "theoretical_loss": 3.3393392408082154, + "tokens_seen": 3119120384 + }, + { + "epoch": 0.95, + "learning_rate": 4.4501504513540624e-05, + "loss": 1.2082, + "theoretical_loss": 3.339333934460689, + "tokens_seen": 3119185920 + }, + { + "epoch": 0.95, + "learning_rate": 4.4485456369107325e-05, + "loss": 1.2429, + "theoretical_loss": 3.339328628255867, + "tokens_seen": 3119251456 + }, + { + "epoch": 0.95, + "learning_rate": 4.4469408224674026e-05, + "loss": 1.2759, + "theoretical_loss": 3.339323322193743, + "tokens_seen": 3119316992 + }, + { + "epoch": 0.95, + "learning_rate": 4.4453360080240726e-05, + "loss": 1.157, + "theoretical_loss": 3.3393180162743104, + "tokens_seen": 3119382528 + }, + { + "epoch": 0.95, + "learning_rate": 4.443731193580743e-05, + "loss": 1.1898, + "theoretical_loss": 3.3393127104975617, + "tokens_seen": 3119448064 + }, + { + "epoch": 0.95, + "objective/train/docs_used": 1041049, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3701893091201782, + "objective/train/theoretical_loss": 3.3393074048634905, + "objective/train/tokens_used": 3139973600, + "theoretical_loss": 3.3393074048634905, + "tokens_seen": 3119513600 + }, + { + "epoch": 0.95, + "learning_rate": 4.442126379137412e-05, + "loss": 1.2862, + "theoretical_loss": 3.3393074048634905, + "tokens_seen": 3119513600 + }, + { + "epoch": 0.95, + "learning_rate": 4.440521564694082e-05, + "loss": 1.1673, + "theoretical_loss": 3.33930209937209, + "tokens_seen": 3119579136 + }, + { + "epoch": 0.95, + "learning_rate": 4.438916750250752e-05, + "loss": 1.2166, + "theoretical_loss": 3.3392967940233533, + "tokens_seen": 3119644672 + }, + { + "epoch": 0.95, + "learning_rate": 4.437311935807422e-05, + "loss": 1.2548, + "theoretical_loss": 3.3392914888172736, + "tokens_seen": 3119710208 + }, + { + "epoch": 0.95, + "learning_rate": 4.435707121364093e-05, + "loss": 1.2278, + "theoretical_loss": 3.3392861837538437, + "tokens_seen": 3119775744 + }, + { + "epoch": 0.95, + "learning_rate": 4.4341023069207624e-05, + "loss": 1.2132, + "theoretical_loss": 3.3392808788330575, + "tokens_seen": 3119841280 + }, + { + "epoch": 0.95, + "learning_rate": 4.4324974924774325e-05, + "loss": 1.1273, + "theoretical_loss": 3.3392755740549074, + "tokens_seen": 3119906816 + }, + { + "epoch": 0.95, + "learning_rate": 4.4308926780341025e-05, + "loss": 1.2199, + "theoretical_loss": 3.3392702694193868, + "tokens_seen": 3119972352 + }, + { + "epoch": 0.95, + "learning_rate": 4.4292878635907726e-05, + "loss": 1.2356, + "theoretical_loss": 3.339264964926489, + "tokens_seen": 3120037888 + }, + { + "epoch": 0.95, + "learning_rate": 4.4276830491474426e-05, + "loss": 1.1124, + "theoretical_loss": 3.3392596605762073, + "tokens_seen": 3120103424 + }, + { + "epoch": 0.95, + "learning_rate": 4.426078234704113e-05, + "loss": 1.1723, + "theoretical_loss": 3.3392543563685346, + "tokens_seen": 3120168960 + }, + { + "epoch": 0.95, + "learning_rate": 4.424473420260782e-05, + "loss": 1.1717, + "theoretical_loss": 3.3392490523034644, + "tokens_seen": 3120234496 + }, + { + "epoch": 0.95, + "learning_rate": 4.422868605817452e-05, + "loss": 1.2247, + "theoretical_loss": 3.3392437483809894, + "tokens_seen": 3120300032 + }, + { + "epoch": 0.95, + "learning_rate": 4.421263791374123e-05, + "loss": 1.1748, + "theoretical_loss": 3.339238444601103, + "tokens_seen": 3120365568 + }, + { + "epoch": 0.95, + "learning_rate": 4.419658976930793e-05, + "loss": 1.2454, + "theoretical_loss": 3.339233140963799, + "tokens_seen": 3120431104 + }, + { + "epoch": 0.95, + "learning_rate": 4.418054162487463e-05, + "loss": 1.2514, + "theoretical_loss": 3.3392278374690694, + "tokens_seen": 3120496640 + }, + { + "epoch": 0.95, + "learning_rate": 4.4164493480441324e-05, + "loss": 1.1436, + "theoretical_loss": 3.339222534116908, + "tokens_seen": 3120562176 + }, + { + "epoch": 0.95, + "learning_rate": 4.4148445336008025e-05, + "loss": 1.1943, + "theoretical_loss": 3.3392172309073076, + "tokens_seen": 3120627712 + }, + { + "epoch": 0.95, + "learning_rate": 4.4132397191574725e-05, + "loss": 1.2419, + "theoretical_loss": 3.339211927840262, + "tokens_seen": 3120693248 + }, + { + "epoch": 0.95, + "learning_rate": 4.4116349047141426e-05, + "loss": 1.2824, + "theoretical_loss": 3.3392066249157644, + "tokens_seen": 3120758784 + }, + { + "epoch": 0.95, + "learning_rate": 4.410030090270813e-05, + "loss": 1.2008, + "theoretical_loss": 3.3392013221338073, + "tokens_seen": 3120824320 + }, + { + "epoch": 0.95, + "learning_rate": 4.408425275827483e-05, + "loss": 1.1442, + "theoretical_loss": 3.339196019494384, + "tokens_seen": 3120889856 + }, + { + "epoch": 0.95, + "learning_rate": 4.406820461384153e-05, + "loss": 1.3265, + "theoretical_loss": 3.3391907169974884, + "tokens_seen": 3120955392 + }, + { + "epoch": 0.95, + "learning_rate": 4.405215646940823e-05, + "loss": 1.2094, + "theoretical_loss": 3.339185414643113, + "tokens_seen": 3121020928 + }, + { + "epoch": 0.95, + "learning_rate": 4.403610832497493e-05, + "loss": 1.2546, + "theoretical_loss": 3.339180112431251, + "tokens_seen": 3121086464 + }, + { + "epoch": 0.95, + "learning_rate": 4.402006018054163e-05, + "loss": 1.1817, + "theoretical_loss": 3.3391748103618957, + "tokens_seen": 3121152000 + }, + { + "epoch": 0.95, + "learning_rate": 4.400401203610833e-05, + "loss": 1.1966, + "theoretical_loss": 3.33916950843504, + "tokens_seen": 3121217536 + }, + { + "epoch": 0.95, + "learning_rate": 4.3987963891675024e-05, + "loss": 1.1747, + "theoretical_loss": 3.339164206650678, + "tokens_seen": 3121283072 + }, + { + "epoch": 0.95, + "learning_rate": 4.3971915747241725e-05, + "loss": 1.134, + "theoretical_loss": 3.339158905008802, + "tokens_seen": 3121348608 + }, + { + "epoch": 0.95, + "learning_rate": 4.3955867602808426e-05, + "loss": 1.2897, + "theoretical_loss": 3.339153603509405, + "tokens_seen": 3121414144 + }, + { + "epoch": 0.95, + "learning_rate": 4.3939819458375126e-05, + "loss": 1.1578, + "theoretical_loss": 3.3391483021524815, + "tokens_seen": 3121479680 + }, + { + "epoch": 0.95, + "learning_rate": 4.392377131394183e-05, + "loss": 1.2198, + "theoretical_loss": 3.339143000938023, + "tokens_seen": 3121545216 + }, + { + "epoch": 0.95, + "learning_rate": 4.390772316950853e-05, + "loss": 1.244, + "theoretical_loss": 3.339137699866024, + "tokens_seen": 3121610752 + }, + { + "epoch": 0.95, + "learning_rate": 4.389167502507523e-05, + "loss": 1.2008, + "theoretical_loss": 3.339132398936477, + "tokens_seen": 3121676288 + }, + { + "epoch": 0.95, + "learning_rate": 4.387562688064193e-05, + "loss": 1.1534, + "theoretical_loss": 3.3391270981493753, + "tokens_seen": 3121741824 + }, + { + "epoch": 0.95, + "learning_rate": 4.385957873620863e-05, + "loss": 1.2087, + "theoretical_loss": 3.339121797504712, + "tokens_seen": 3121807360 + }, + { + "epoch": 0.95, + "learning_rate": 4.384353059177533e-05, + "loss": 1.2965, + "theoretical_loss": 3.3391164970024807, + "tokens_seen": 3121872896 + }, + { + "epoch": 0.95, + "learning_rate": 4.382748244734203e-05, + "loss": 1.2742, + "theoretical_loss": 3.3391111966426736, + "tokens_seen": 3121938432 + }, + { + "epoch": 0.95, + "learning_rate": 4.3811434302908725e-05, + "loss": 1.2231, + "theoretical_loss": 3.339105896425285, + "tokens_seen": 3122003968 + }, + { + "epoch": 0.95, + "learning_rate": 4.3795386158475425e-05, + "loss": 1.2004, + "theoretical_loss": 3.339100596350308, + "tokens_seen": 3122069504 + }, + { + "epoch": 0.95, + "learning_rate": 4.3779338014042126e-05, + "loss": 1.0852, + "theoretical_loss": 3.339095296417735, + "tokens_seen": 3122135040 + }, + { + "epoch": 0.95, + "learning_rate": 4.376328986960883e-05, + "loss": 1.1772, + "theoretical_loss": 3.3390899966275596, + "tokens_seen": 3122200576 + }, + { + "epoch": 0.95, + "learning_rate": 4.3747241725175534e-05, + "loss": 1.204, + "theoretical_loss": 3.3390846969797754, + "tokens_seen": 3122266112 + }, + { + "epoch": 0.95, + "learning_rate": 4.373119358074223e-05, + "loss": 1.1523, + "theoretical_loss": 3.339079397474375, + "tokens_seen": 3122331648 + }, + { + "epoch": 0.95, + "learning_rate": 4.371514543630893e-05, + "loss": 1.2695, + "theoretical_loss": 3.3390740981113516, + "tokens_seen": 3122397184 + }, + { + "epoch": 0.95, + "learning_rate": 4.369909729187563e-05, + "loss": 1.2324, + "theoretical_loss": 3.3390687988906986, + "tokens_seen": 3122462720 + }, + { + "epoch": 0.95, + "learning_rate": 4.368304914744233e-05, + "loss": 1.2307, + "theoretical_loss": 3.3390634998124096, + "tokens_seen": 3122528256 + }, + { + "epoch": 0.95, + "learning_rate": 4.366700100300903e-05, + "loss": 1.2158, + "theoretical_loss": 3.339058200876477, + "tokens_seen": 3122593792 + }, + { + "epoch": 0.95, + "learning_rate": 4.365095285857573e-05, + "loss": 1.1776, + "theoretical_loss": 3.339052902082894, + "tokens_seen": 3122659328 + }, + { + "epoch": 0.95, + "learning_rate": 4.3634904714142425e-05, + "loss": 1.2496, + "theoretical_loss": 3.3390476034316543, + "tokens_seen": 3122724864 + }, + { + "epoch": 0.95, + "objective/train/docs_used": 1042309, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.127990484237671, + "objective/train/theoretical_loss": 3.339042304922751, + "objective/train/tokens_used": 3143250400, + "theoretical_loss": 3.339042304922751, + "tokens_seen": 3122790400 + }, + { + "epoch": 0.95, + "learning_rate": 4.3618856569709125e-05, + "loss": 1.1658, + "theoretical_loss": 3.339042304922751, + "tokens_seen": 3122790400 + }, + { + "epoch": 0.95, + "learning_rate": 4.360280842527583e-05, + "loss": 1.3042, + "theoretical_loss": 3.339037006556177, + "tokens_seen": 3122855936 + }, + { + "epoch": 0.95, + "learning_rate": 4.3586760280842533e-05, + "loss": 1.2631, + "theoretical_loss": 3.3390317083319263, + "tokens_seen": 3122921472 + }, + { + "epoch": 0.95, + "learning_rate": 4.3570712136409234e-05, + "loss": 1.0979, + "theoretical_loss": 3.339026410249991, + "tokens_seen": 3122987008 + }, + { + "epoch": 0.95, + "learning_rate": 4.355466399197593e-05, + "loss": 1.2667, + "theoretical_loss": 3.339021112310365, + "tokens_seen": 3123052544 + }, + { + "epoch": 0.95, + "learning_rate": 4.353861584754263e-05, + "loss": 1.1758, + "theoretical_loss": 3.339015814513041, + "tokens_seen": 3123118080 + }, + { + "epoch": 0.95, + "learning_rate": 4.352256770310933e-05, + "loss": 1.2682, + "theoretical_loss": 3.3390105168580124, + "tokens_seen": 3123183616 + }, + { + "epoch": 0.95, + "learning_rate": 4.350651955867603e-05, + "loss": 1.2301, + "theoretical_loss": 3.3390052193452724, + "tokens_seen": 3123249152 + }, + { + "epoch": 0.95, + "learning_rate": 4.349047141424273e-05, + "loss": 1.1843, + "theoretical_loss": 3.3389999219748145, + "tokens_seen": 3123314688 + }, + { + "epoch": 0.95, + "learning_rate": 4.347442326980943e-05, + "loss": 1.2265, + "theoretical_loss": 3.3389946247466313, + "tokens_seen": 3123380224 + }, + { + "epoch": 0.95, + "learning_rate": 4.345837512537613e-05, + "loss": 1.1276, + "theoretical_loss": 3.338989327660717, + "tokens_seen": 3123445760 + }, + { + "epoch": 0.95, + "learning_rate": 4.344232698094283e-05, + "loss": 1.1698, + "theoretical_loss": 3.3389840307170635, + "tokens_seen": 3123511296 + }, + { + "epoch": 0.95, + "learning_rate": 4.342627883650953e-05, + "loss": 1.139, + "theoretical_loss": 3.3389787339156647, + "tokens_seen": 3123576832 + }, + { + "epoch": 0.95, + "learning_rate": 4.3410230692076234e-05, + "loss": 1.1222, + "theoretical_loss": 3.338973437256514, + "tokens_seen": 3123642368 + }, + { + "epoch": 0.95, + "learning_rate": 4.3394182547642934e-05, + "loss": 1.2245, + "theoretical_loss": 3.3389681407396035, + "tokens_seen": 3123707904 + }, + { + "epoch": 0.95, + "learning_rate": 4.337813440320963e-05, + "loss": 1.2563, + "theoretical_loss": 3.3389628443649277, + "tokens_seen": 3123773440 + }, + { + "epoch": 0.95, + "learning_rate": 4.336208625877633e-05, + "loss": 1.139, + "theoretical_loss": 3.3389575481324796, + "tokens_seen": 3123838976 + }, + { + "epoch": 0.95, + "learning_rate": 4.334603811434303e-05, + "loss": 1.2263, + "theoretical_loss": 3.338952252042252, + "tokens_seen": 3123904512 + }, + { + "epoch": 0.95, + "learning_rate": 4.332998996990973e-05, + "loss": 1.1071, + "theoretical_loss": 3.338946956094238, + "tokens_seen": 3123970048 + }, + { + "epoch": 0.95, + "learning_rate": 4.331394182547643e-05, + "loss": 1.1596, + "theoretical_loss": 3.3389416602884308, + "tokens_seen": 3124035584 + }, + { + "epoch": 0.95, + "learning_rate": 4.329789368104313e-05, + "loss": 1.1579, + "theoretical_loss": 3.338936364624824, + "tokens_seen": 3124101120 + }, + { + "epoch": 0.95, + "learning_rate": 4.328184553660983e-05, + "loss": 1.2487, + "theoretical_loss": 3.3389310691034106, + "tokens_seen": 3124166656 + }, + { + "epoch": 0.95, + "learning_rate": 4.326579739217653e-05, + "loss": 1.255, + "theoretical_loss": 3.3389257737241835, + "tokens_seen": 3124232192 + }, + { + "epoch": 0.95, + "learning_rate": 4.324974924774323e-05, + "loss": 1.2265, + "theoretical_loss": 3.3389204784871365, + "tokens_seen": 3124297728 + }, + { + "epoch": 0.95, + "learning_rate": 4.3233701103309934e-05, + "loss": 1.2145, + "theoretical_loss": 3.3389151833922623, + "tokens_seen": 3124363264 + }, + { + "epoch": 0.95, + "learning_rate": 4.321765295887663e-05, + "loss": 1.1116, + "theoretical_loss": 3.3389098884395545, + "tokens_seen": 3124428800 + }, + { + "epoch": 0.95, + "learning_rate": 4.320160481444333e-05, + "loss": 1.1692, + "theoretical_loss": 3.338904593629006, + "tokens_seen": 3124494336 + }, + { + "epoch": 0.95, + "learning_rate": 4.318555667001003e-05, + "loss": 1.1713, + "theoretical_loss": 3.33889929896061, + "tokens_seen": 3124559872 + }, + { + "epoch": 0.95, + "learning_rate": 4.316950852557673e-05, + "loss": 1.2689, + "theoretical_loss": 3.33889400443436, + "tokens_seen": 3124625408 + }, + { + "epoch": 0.95, + "learning_rate": 4.315346038114344e-05, + "loss": 1.1877, + "theoretical_loss": 3.3388887100502487, + "tokens_seen": 3124690944 + }, + { + "epoch": 0.95, + "learning_rate": 4.313741223671013e-05, + "loss": 1.1841, + "theoretical_loss": 3.3388834158082696, + "tokens_seen": 3124756480 + }, + { + "epoch": 0.95, + "learning_rate": 4.312136409227683e-05, + "loss": 1.2464, + "theoretical_loss": 3.338878121708416, + "tokens_seen": 3124822016 + }, + { + "epoch": 0.95, + "learning_rate": 4.310531594784353e-05, + "loss": 1.262, + "theoretical_loss": 3.338872827750681, + "tokens_seen": 3124887552 + }, + { + "epoch": 0.95, + "learning_rate": 4.308926780341023e-05, + "loss": 1.2397, + "theoretical_loss": 3.3388675339350575, + "tokens_seen": 3124953088 + }, + { + "epoch": 0.95, + "learning_rate": 4.3073219658976933e-05, + "loss": 1.1913, + "theoretical_loss": 3.3388622402615393, + "tokens_seen": 3125018624 + }, + { + "epoch": 0.95, + "learning_rate": 4.3057171514543634e-05, + "loss": 1.2553, + "theoretical_loss": 3.33885694673012, + "tokens_seen": 3125084160 + }, + { + "epoch": 0.95, + "learning_rate": 4.304112337011033e-05, + "loss": 1.1393, + "theoretical_loss": 3.338851653340791, + "tokens_seen": 3125149696 + }, + { + "epoch": 0.95, + "learning_rate": 4.302507522567703e-05, + "loss": 1.1955, + "theoretical_loss": 3.338846360093547, + "tokens_seen": 3125215232 + }, + { + "epoch": 0.95, + "learning_rate": 4.3009027081243736e-05, + "loss": 1.2574, + "theoretical_loss": 3.338841066988381, + "tokens_seen": 3125280768 + }, + { + "epoch": 0.95, + "learning_rate": 4.2992978936810437e-05, + "loss": 1.2227, + "theoretical_loss": 3.338835774025286, + "tokens_seen": 3125346304 + }, + { + "epoch": 0.95, + "learning_rate": 4.297693079237714e-05, + "loss": 1.1089, + "theoretical_loss": 3.338830481204255, + "tokens_seen": 3125411840 + }, + { + "epoch": 0.95, + "learning_rate": 4.296088264794383e-05, + "loss": 1.2316, + "theoretical_loss": 3.3388251885252815, + "tokens_seen": 3125477376 + }, + { + "epoch": 0.95, + "learning_rate": 4.294483450351053e-05, + "loss": 1.3019, + "theoretical_loss": 3.338819895988359, + "tokens_seen": 3125542912 + }, + { + "epoch": 0.95, + "learning_rate": 4.292878635907723e-05, + "loss": 1.1527, + "theoretical_loss": 3.3388146035934803, + "tokens_seen": 3125608448 + }, + { + "epoch": 0.95, + "learning_rate": 4.291273821464393e-05, + "loss": 1.1079, + "theoretical_loss": 3.338809311340638, + "tokens_seen": 3125673984 + }, + { + "epoch": 0.95, + "learning_rate": 4.2896690070210634e-05, + "loss": 1.1975, + "theoretical_loss": 3.338804019229827, + "tokens_seen": 3125739520 + }, + { + "epoch": 0.95, + "learning_rate": 4.2880641925777334e-05, + "loss": 1.2818, + "theoretical_loss": 3.3387987272610387, + "tokens_seen": 3125805056 + }, + { + "epoch": 0.95, + "learning_rate": 4.2864593781344035e-05, + "loss": 1.1615, + "theoretical_loss": 3.3387934354342677, + "tokens_seen": 3125870592 + }, + { + "epoch": 0.95, + "learning_rate": 4.2848545636910736e-05, + "loss": 1.2403, + "theoretical_loss": 3.338788143749506, + "tokens_seen": 3125936128 + }, + { + "epoch": 0.95, + "learning_rate": 4.2832497492477436e-05, + "loss": 1.1911, + "theoretical_loss": 3.338782852206748, + "tokens_seen": 3126001664 + }, + { + "epoch": 0.95, + "objective/train/docs_used": 1043199, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2547998428344727, + "objective/train/theoretical_loss": 3.338777560805986, + "objective/train/tokens_used": 3146527200, + "theoretical_loss": 3.338777560805986, + "tokens_seen": 3126067200 + }, + { + "epoch": 0.95, + "learning_rate": 4.281644934804414e-05, + "loss": 1.2063, + "theoretical_loss": 3.338777560805986, + "tokens_seen": 3126067200 + }, + { + "epoch": 0.95, + "learning_rate": 4.280040120361084e-05, + "loss": 1.2768, + "theoretical_loss": 3.338772269547214, + "tokens_seen": 3126132736 + }, + { + "epoch": 0.95, + "learning_rate": 4.278435305917753e-05, + "loss": 1.1482, + "theoretical_loss": 3.3387669784304244, + "tokens_seen": 3126198272 + }, + { + "epoch": 0.95, + "learning_rate": 4.276830491474423e-05, + "loss": 1.2318, + "theoretical_loss": 3.338761687455611, + "tokens_seen": 3126263808 + }, + { + "epoch": 0.95, + "learning_rate": 4.275225677031093e-05, + "loss": 1.1564, + "theoretical_loss": 3.3387563966227667, + "tokens_seen": 3126329344 + }, + { + "epoch": 0.95, + "learning_rate": 4.273620862587763e-05, + "loss": 1.1544, + "theoretical_loss": 3.338751105931885, + "tokens_seen": 3126394880 + }, + { + "epoch": 0.95, + "learning_rate": 4.2720160481444334e-05, + "loss": 1.2529, + "theoretical_loss": 3.3387458153829583, + "tokens_seen": 3126460416 + }, + { + "epoch": 0.95, + "learning_rate": 4.2704112337011034e-05, + "loss": 1.1437, + "theoretical_loss": 3.338740524975981, + "tokens_seen": 3126525952 + }, + { + "epoch": 0.95, + "learning_rate": 4.2688064192577735e-05, + "loss": 1.3013, + "theoretical_loss": 3.3387352347109456, + "tokens_seen": 3126591488 + }, + { + "epoch": 0.95, + "learning_rate": 4.2672016048144436e-05, + "loss": 1.1923, + "theoretical_loss": 3.3387299445878456, + "tokens_seen": 3126657024 + }, + { + "epoch": 0.95, + "learning_rate": 4.2655967903711136e-05, + "loss": 1.2061, + "theoretical_loss": 3.338724654606674, + "tokens_seen": 3126722560 + }, + { + "epoch": 0.95, + "learning_rate": 4.263991975927784e-05, + "loss": 1.1826, + "theoretical_loss": 3.3387193647674245, + "tokens_seen": 3126788096 + }, + { + "epoch": 0.95, + "learning_rate": 4.262387161484454e-05, + "loss": 1.3055, + "theoretical_loss": 3.338714075070089, + "tokens_seen": 3126853632 + }, + { + "epoch": 0.95, + "learning_rate": 4.260782347041123e-05, + "loss": 1.2663, + "theoretical_loss": 3.3387087855146627, + "tokens_seen": 3126919168 + }, + { + "epoch": 0.95, + "learning_rate": 4.259177532597793e-05, + "loss": 1.1584, + "theoretical_loss": 3.338703496101137, + "tokens_seen": 3126984704 + }, + { + "epoch": 0.95, + "learning_rate": 4.257572718154463e-05, + "loss": 1.2074, + "theoretical_loss": 3.3386982068295064, + "tokens_seen": 3127050240 + }, + { + "epoch": 0.95, + "learning_rate": 4.255967903711134e-05, + "loss": 1.2157, + "theoretical_loss": 3.3386929176997633, + "tokens_seen": 3127115776 + }, + { + "epoch": 0.95, + "learning_rate": 4.254363089267804e-05, + "loss": 1.2145, + "theoretical_loss": 3.3386876287119014, + "tokens_seen": 3127181312 + }, + { + "epoch": 0.95, + "learning_rate": 4.2527582748244735e-05, + "loss": 1.248, + "theoretical_loss": 3.3386823398659136, + "tokens_seen": 3127246848 + }, + { + "epoch": 0.95, + "learning_rate": 4.2511534603811435e-05, + "loss": 1.1919, + "theoretical_loss": 3.3386770511617936, + "tokens_seen": 3127312384 + }, + { + "epoch": 0.95, + "learning_rate": 4.2495486459378136e-05, + "loss": 1.2764, + "theoretical_loss": 3.338671762599534, + "tokens_seen": 3127377920 + }, + { + "epoch": 0.95, + "learning_rate": 4.2479438314944837e-05, + "loss": 1.1323, + "theoretical_loss": 3.3386664741791283, + "tokens_seen": 3127443456 + }, + { + "epoch": 0.95, + "learning_rate": 4.246339017051154e-05, + "loss": 1.2583, + "theoretical_loss": 3.3386611859005697, + "tokens_seen": 3127508992 + }, + { + "epoch": 0.95, + "learning_rate": 4.244734202607824e-05, + "loss": 1.2446, + "theoretical_loss": 3.338655897763852, + "tokens_seen": 3127574528 + }, + { + "epoch": 0.95, + "learning_rate": 4.243129388164493e-05, + "loss": 1.2001, + "theoretical_loss": 3.338650609768967, + "tokens_seen": 3127640064 + }, + { + "epoch": 0.95, + "learning_rate": 4.241524573721164e-05, + "loss": 1.2617, + "theoretical_loss": 3.338645321915909, + "tokens_seen": 3127705600 + }, + { + "epoch": 0.95, + "learning_rate": 4.239919759277834e-05, + "loss": 1.1826, + "theoretical_loss": 3.338640034204672, + "tokens_seen": 3127771136 + }, + { + "epoch": 0.95, + "learning_rate": 4.238314944834504e-05, + "loss": 1.191, + "theoretical_loss": 3.3386347466352473, + "tokens_seen": 3127836672 + }, + { + "epoch": 0.95, + "learning_rate": 4.236710130391174e-05, + "loss": 1.2587, + "theoretical_loss": 3.3386294592076293, + "tokens_seen": 3127902208 + }, + { + "epoch": 0.95, + "learning_rate": 4.2351053159478435e-05, + "loss": 1.1974, + "theoretical_loss": 3.3386241719218113, + "tokens_seen": 3127967744 + }, + { + "epoch": 0.95, + "learning_rate": 4.2335005015045135e-05, + "loss": 1.2301, + "theoretical_loss": 3.3386188847777856, + "tokens_seen": 3128033280 + }, + { + "epoch": 0.95, + "learning_rate": 4.2318956870611836e-05, + "loss": 1.1611, + "theoretical_loss": 3.3386135977755464, + "tokens_seen": 3128098816 + }, + { + "epoch": 0.95, + "learning_rate": 4.230290872617854e-05, + "loss": 1.2414, + "theoretical_loss": 3.338608310915087, + "tokens_seen": 3128164352 + }, + { + "epoch": 0.95, + "learning_rate": 4.228686058174524e-05, + "loss": 1.2681, + "theoretical_loss": 3.3386030241963995, + "tokens_seen": 3128229888 + }, + { + "epoch": 0.95, + "learning_rate": 4.227081243731194e-05, + "loss": 1.2025, + "theoretical_loss": 3.3385977376194784, + "tokens_seen": 3128295424 + }, + { + "epoch": 0.95, + "learning_rate": 4.225476429287864e-05, + "loss": 1.2441, + "theoretical_loss": 3.3385924511843164, + "tokens_seen": 3128360960 + }, + { + "epoch": 0.95, + "learning_rate": 4.223871614844534e-05, + "loss": 1.2438, + "theoretical_loss": 3.3385871648909062, + "tokens_seen": 3128426496 + }, + { + "epoch": 0.95, + "learning_rate": 4.222266800401204e-05, + "loss": 1.1908, + "theoretical_loss": 3.338581878739242, + "tokens_seen": 3128492032 + }, + { + "epoch": 0.95, + "learning_rate": 4.220661985957874e-05, + "loss": 1.2345, + "theoretical_loss": 3.3385765927293165, + "tokens_seen": 3128557568 + }, + { + "epoch": 0.95, + "learning_rate": 4.219057171514544e-05, + "loss": 1.2383, + "theoretical_loss": 3.3385713068611227, + "tokens_seen": 3128623104 + }, + { + "epoch": 0.95, + "learning_rate": 4.2174523570712135e-05, + "loss": 1.1875, + "theoretical_loss": 3.338566021134654, + "tokens_seen": 3128688640 + }, + { + "epoch": 0.95, + "learning_rate": 4.2158475426278836e-05, + "loss": 1.1357, + "theoretical_loss": 3.3385607355499043, + "tokens_seen": 3128754176 + }, + { + "epoch": 0.95, + "learning_rate": 4.2142427281845536e-05, + "loss": 1.2058, + "theoretical_loss": 3.338555450106866, + "tokens_seen": 3128819712 + }, + { + "epoch": 0.95, + "learning_rate": 4.212637913741224e-05, + "loss": 1.2071, + "theoretical_loss": 3.338550164805533, + "tokens_seen": 3128885248 + }, + { + "epoch": 0.95, + "learning_rate": 4.2110330992978944e-05, + "loss": 1.2949, + "theoretical_loss": 3.3385448796458976, + "tokens_seen": 3128950784 + }, + { + "epoch": 0.95, + "learning_rate": 4.209428284854564e-05, + "loss": 1.2396, + "theoretical_loss": 3.338539594627954, + "tokens_seen": 3129016320 + }, + { + "epoch": 0.95, + "learning_rate": 4.207823470411234e-05, + "loss": 1.2169, + "theoretical_loss": 3.3385343097516946, + "tokens_seen": 3129081856 + }, + { + "epoch": 0.95, + "learning_rate": 4.206218655967904e-05, + "loss": 1.1957, + "theoretical_loss": 3.3385290250171136, + "tokens_seen": 3129147392 + }, + { + "epoch": 0.95, + "learning_rate": 4.204613841524574e-05, + "loss": 1.2772, + "theoretical_loss": 3.338523740424203, + "tokens_seen": 3129212928 + }, + { + "epoch": 0.95, + "learning_rate": 4.203009027081244e-05, + "loss": 1.1826, + "theoretical_loss": 3.3385184559729573, + "tokens_seen": 3129278464 + }, + { + "epoch": 0.95, + "objective/train/docs_used": 1044466, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2057443857192993, + "objective/train/theoretical_loss": 3.338513171663369, + "objective/train/tokens_used": 3149804000, + "theoretical_loss": 3.338513171663369, + "tokens_seen": 3129344000 + }, + { + "epoch": 0.95, + "learning_rate": 4.201404212637914e-05, + "loss": 1.2528, + "theoretical_loss": 3.338513171663369, + "tokens_seen": 3129344000 + }, + { + "epoch": 0.95, + "learning_rate": 4.1997993981945835e-05, + "loss": 1.1122, + "theoretical_loss": 3.3385078874954317, + "tokens_seen": 3129409536 + }, + { + "epoch": 0.95, + "learning_rate": 4.1981945837512536e-05, + "loss": 1.1964, + "theoretical_loss": 3.3385026034691383, + "tokens_seen": 3129475072 + }, + { + "epoch": 0.95, + "learning_rate": 4.196589769307924e-05, + "loss": 1.1227, + "theoretical_loss": 3.338497319584482, + "tokens_seen": 3129540608 + }, + { + "epoch": 0.95, + "learning_rate": 4.1949849548645944e-05, + "loss": 1.2253, + "theoretical_loss": 3.3384920358414565, + "tokens_seen": 3129606144 + }, + { + "epoch": 0.95, + "learning_rate": 4.1933801404212645e-05, + "loss": 1.3222, + "theoretical_loss": 3.3384867522400543, + "tokens_seen": 3129671680 + }, + { + "epoch": 0.95, + "learning_rate": 4.191775325977934e-05, + "loss": 1.2185, + "theoretical_loss": 3.3384814687802695, + "tokens_seen": 3129737216 + }, + { + "epoch": 0.95, + "learning_rate": 4.190170511534604e-05, + "loss": 1.1985, + "theoretical_loss": 3.3384761854620946, + "tokens_seen": 3129802752 + }, + { + "epoch": 0.95, + "learning_rate": 4.188565697091274e-05, + "loss": 1.2677, + "theoretical_loss": 3.3384709022855232, + "tokens_seen": 3129868288 + }, + { + "epoch": 0.95, + "learning_rate": 4.186960882647944e-05, + "loss": 1.1601, + "theoretical_loss": 3.338465619250549, + "tokens_seen": 3129933824 + }, + { + "epoch": 0.95, + "learning_rate": 4.185356068204614e-05, + "loss": 1.1725, + "theoretical_loss": 3.3384603363571643, + "tokens_seen": 3129999360 + }, + { + "epoch": 0.95, + "learning_rate": 4.183751253761284e-05, + "loss": 1.1762, + "theoretical_loss": 3.3384550536053625, + "tokens_seen": 3130064896 + }, + { + "epoch": 0.95, + "learning_rate": 4.1821464393179535e-05, + "loss": 1.2203, + "theoretical_loss": 3.3384497709951377, + "tokens_seen": 3130130432 + }, + { + "epoch": 0.95, + "learning_rate": 4.180541624874624e-05, + "loss": 1.22, + "theoretical_loss": 3.3384444885264823, + "tokens_seen": 3130195968 + }, + { + "epoch": 0.95, + "learning_rate": 4.1789368104312944e-05, + "loss": 1.1645, + "theoretical_loss": 3.33843920619939, + "tokens_seen": 3130261504 + }, + { + "epoch": 0.95, + "learning_rate": 4.1773319959879644e-05, + "loss": 1.1177, + "theoretical_loss": 3.3384339240138536, + "tokens_seen": 3130327040 + }, + { + "epoch": 0.95, + "learning_rate": 4.1757271815446345e-05, + "loss": 1.1496, + "theoretical_loss": 3.338428641969867, + "tokens_seen": 3130392576 + }, + { + "epoch": 0.95, + "learning_rate": 4.174122367101304e-05, + "loss": 1.2225, + "theoretical_loss": 3.3384233600674227, + "tokens_seen": 3130458112 + }, + { + "epoch": 0.95, + "learning_rate": 4.172517552657974e-05, + "loss": 1.2399, + "theoretical_loss": 3.338418078306514, + "tokens_seen": 3130523648 + }, + { + "epoch": 0.95, + "learning_rate": 4.170912738214644e-05, + "loss": 1.1758, + "theoretical_loss": 3.338412796687135, + "tokens_seen": 3130589184 + }, + { + "epoch": 0.95, + "learning_rate": 4.169307923771314e-05, + "loss": 1.2015, + "theoretical_loss": 3.338407515209278, + "tokens_seen": 3130654720 + }, + { + "epoch": 0.95, + "learning_rate": 4.167703109327984e-05, + "loss": 1.1376, + "theoretical_loss": 3.338402233872937, + "tokens_seen": 3130720256 + }, + { + "epoch": 0.95, + "learning_rate": 4.166098294884654e-05, + "loss": 1.1885, + "theoretical_loss": 3.3383969526781043, + "tokens_seen": 3130785792 + }, + { + "epoch": 0.95, + "learning_rate": 4.164493480441324e-05, + "loss": 1.1653, + "theoretical_loss": 3.338391671624774, + "tokens_seen": 3130851328 + }, + { + "epoch": 0.95, + "learning_rate": 4.162888665997994e-05, + "loss": 1.2304, + "theoretical_loss": 3.338386390712939, + "tokens_seen": 3130916864 + }, + { + "epoch": 0.95, + "learning_rate": 4.1612838515546644e-05, + "loss": 1.1909, + "theoretical_loss": 3.338381109942593, + "tokens_seen": 3130982400 + }, + { + "epoch": 0.95, + "learning_rate": 4.1596790371113344e-05, + "loss": 1.2035, + "theoretical_loss": 3.3383758293137284, + "tokens_seen": 3131047936 + }, + { + "epoch": 0.95, + "learning_rate": 4.1580742226680045e-05, + "loss": 1.2657, + "theoretical_loss": 3.3383705488263393, + "tokens_seen": 3131113472 + }, + { + "epoch": 0.95, + "learning_rate": 4.156469408224674e-05, + "loss": 1.2132, + "theoretical_loss": 3.338365268480418, + "tokens_seen": 3131179008 + }, + { + "epoch": 0.95, + "learning_rate": 4.154864593781344e-05, + "loss": 1.2189, + "theoretical_loss": 3.3383599882759585, + "tokens_seen": 3131244544 + }, + { + "epoch": 0.95, + "learning_rate": 4.153259779338014e-05, + "loss": 1.268, + "theoretical_loss": 3.338354708212954, + "tokens_seen": 3131310080 + }, + { + "epoch": 0.95, + "learning_rate": 4.151654964894684e-05, + "loss": 1.2204, + "theoretical_loss": 3.338349428291398, + "tokens_seen": 3131375616 + }, + { + "epoch": 0.95, + "learning_rate": 4.150050150451355e-05, + "loss": 1.2143, + "theoretical_loss": 3.3383441485112826, + "tokens_seen": 3131441152 + }, + { + "epoch": 0.95, + "learning_rate": 4.148445336008024e-05, + "loss": 1.2, + "theoretical_loss": 3.338338868872602, + "tokens_seen": 3131506688 + }, + { + "epoch": 0.95, + "learning_rate": 4.146840521564694e-05, + "loss": 1.3146, + "theoretical_loss": 3.3383335893753494, + "tokens_seen": 3131572224 + }, + { + "epoch": 0.95, + "learning_rate": 4.145235707121364e-05, + "loss": 1.2332, + "theoretical_loss": 3.338328310019518, + "tokens_seen": 3131637760 + }, + { + "epoch": 0.95, + "learning_rate": 4.1436308926780344e-05, + "loss": 1.2621, + "theoretical_loss": 3.3383230308051006, + "tokens_seen": 3131703296 + }, + { + "epoch": 0.95, + "learning_rate": 4.1420260782347045e-05, + "loss": 1.2263, + "theoretical_loss": 3.3383177517320908, + "tokens_seen": 3131768832 + }, + { + "epoch": 0.95, + "learning_rate": 4.1404212637913745e-05, + "loss": 1.2354, + "theoretical_loss": 3.3383124728004825, + "tokens_seen": 3131834368 + }, + { + "epoch": 0.95, + "learning_rate": 4.138816449348044e-05, + "loss": 1.1891, + "theoretical_loss": 3.3383071940102678, + "tokens_seen": 3131899904 + }, + { + "epoch": 0.95, + "learning_rate": 4.137211634904714e-05, + "loss": 1.1383, + "theoretical_loss": 3.3383019153614404, + "tokens_seen": 3131965440 + }, + { + "epoch": 0.95, + "learning_rate": 4.135606820461385e-05, + "loss": 1.242, + "theoretical_loss": 3.3382966368539937, + "tokens_seen": 3132030976 + }, + { + "epoch": 0.95, + "learning_rate": 4.134002006018055e-05, + "loss": 1.1368, + "theoretical_loss": 3.338291358487921, + "tokens_seen": 3132096512 + }, + { + "epoch": 0.95, + "learning_rate": 4.132397191574724e-05, + "loss": 1.1764, + "theoretical_loss": 3.3382860802632157, + "tokens_seen": 3132162048 + }, + { + "epoch": 0.95, + "learning_rate": 4.130792377131394e-05, + "loss": 1.2022, + "theoretical_loss": 3.33828080217987, + "tokens_seen": 3132227584 + }, + { + "epoch": 0.95, + "learning_rate": 4.129187562688064e-05, + "loss": 1.1987, + "theoretical_loss": 3.3382755242378788, + "tokens_seen": 3132293120 + }, + { + "epoch": 0.95, + "learning_rate": 4.1275827482447343e-05, + "loss": 1.232, + "theoretical_loss": 3.338270246437234, + "tokens_seen": 3132358656 + }, + { + "epoch": 0.95, + "learning_rate": 4.1259779338014044e-05, + "loss": 1.2709, + "theoretical_loss": 3.3382649687779296, + "tokens_seen": 3132424192 + }, + { + "epoch": 0.95, + "learning_rate": 4.1243731193580745e-05, + "loss": 1.1558, + "theoretical_loss": 3.3382596912599585, + "tokens_seen": 3132489728 + }, + { + "epoch": 0.95, + "learning_rate": 4.122768304914744e-05, + "loss": 1.2144, + "theoretical_loss": 3.338254413883314, + "tokens_seen": 3132555264 + }, + { + "epoch": 0.95, + "objective/train/docs_used": 1045385, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.27974534034729, + "objective/train/theoretical_loss": 3.3382491366479896, + "objective/train/tokens_used": 3153080800, + "theoretical_loss": 3.3382491366479896, + "tokens_seen": 3132620800 + }, + { + "epoch": 0.95, + "learning_rate": 4.1211634904714146e-05, + "loss": 1.2223, + "theoretical_loss": 3.3382491366479896, + "tokens_seen": 3132620800 + }, + { + "epoch": 0.95, + "learning_rate": 4.119558676028085e-05, + "loss": 1.152, + "theoretical_loss": 3.338243859553978, + "tokens_seen": 3132686336 + }, + { + "epoch": 0.95, + "learning_rate": 4.117953861584755e-05, + "loss": 1.2409, + "theoretical_loss": 3.3382385826012735, + "tokens_seen": 3132751872 + }, + { + "epoch": 0.95, + "learning_rate": 4.116349047141425e-05, + "loss": 1.1428, + "theoretical_loss": 3.3382333057898683, + "tokens_seen": 3132817408 + }, + { + "epoch": 0.95, + "learning_rate": 4.114744232698094e-05, + "loss": 1.2162, + "theoretical_loss": 3.3382280291197564, + "tokens_seen": 3132882944 + }, + { + "epoch": 0.95, + "learning_rate": 4.113139418254764e-05, + "loss": 1.2263, + "theoretical_loss": 3.33822275259093, + "tokens_seen": 3132948480 + }, + { + "epoch": 0.95, + "learning_rate": 4.111534603811434e-05, + "loss": 1.1405, + "theoretical_loss": 3.338217476203384, + "tokens_seen": 3133014016 + }, + { + "epoch": 0.95, + "learning_rate": 4.1099297893681044e-05, + "loss": 1.2916, + "theoretical_loss": 3.3382121999571104, + "tokens_seen": 3133079552 + }, + { + "epoch": 0.95, + "learning_rate": 4.1083249749247744e-05, + "loss": 1.2443, + "theoretical_loss": 3.338206923852103, + "tokens_seen": 3133145088 + }, + { + "epoch": 0.95, + "learning_rate": 4.1067201604814445e-05, + "loss": 1.177, + "theoretical_loss": 3.3382016478883543, + "tokens_seen": 3133210624 + }, + { + "epoch": 0.95, + "learning_rate": 4.1051153460381146e-05, + "loss": 1.1683, + "theoretical_loss": 3.3381963720658585, + "tokens_seen": 3133276160 + }, + { + "epoch": 0.95, + "learning_rate": 4.1035105315947846e-05, + "loss": 1.1964, + "theoretical_loss": 3.338191096384609, + "tokens_seen": 3133341696 + }, + { + "epoch": 0.95, + "learning_rate": 4.101905717151455e-05, + "loss": 1.2253, + "theoretical_loss": 3.338185820844598, + "tokens_seen": 3133407232 + }, + { + "epoch": 0.95, + "learning_rate": 4.100300902708125e-05, + "loss": 1.2347, + "theoretical_loss": 3.338180545445819, + "tokens_seen": 3133472768 + }, + { + "epoch": 0.95, + "learning_rate": 4.098696088264795e-05, + "loss": 1.2379, + "theoretical_loss": 3.338175270188266, + "tokens_seen": 3133538304 + }, + { + "epoch": 0.95, + "learning_rate": 4.097091273821464e-05, + "loss": 1.2401, + "theoretical_loss": 3.338169995071932, + "tokens_seen": 3133603840 + }, + { + "epoch": 0.95, + "learning_rate": 4.095486459378134e-05, + "loss": 1.2482, + "theoretical_loss": 3.3381647200968105, + "tokens_seen": 3133669376 + }, + { + "epoch": 0.95, + "learning_rate": 4.093881644934804e-05, + "loss": 1.2668, + "theoretical_loss": 3.3381594452628938, + "tokens_seen": 3133734912 + }, + { + "epoch": 0.95, + "learning_rate": 4.0922768304914744e-05, + "loss": 1.1326, + "theoretical_loss": 3.338154170570176, + "tokens_seen": 3133800448 + }, + { + "epoch": 0.95, + "learning_rate": 4.090672016048145e-05, + "loss": 1.1288, + "theoretical_loss": 3.3381488960186503, + "tokens_seen": 3133865984 + }, + { + "epoch": 0.95, + "learning_rate": 4.0890672016048145e-05, + "loss": 1.2375, + "theoretical_loss": 3.3381436216083094, + "tokens_seen": 3133931520 + }, + { + "epoch": 0.95, + "learning_rate": 4.0874623871614846e-05, + "loss": 1.2198, + "theoretical_loss": 3.3381383473391475, + "tokens_seen": 3133997056 + }, + { + "epoch": 0.95, + "learning_rate": 4.0858575727181546e-05, + "loss": 1.2165, + "theoretical_loss": 3.3381330732111567, + "tokens_seen": 3134062592 + }, + { + "epoch": 0.95, + "learning_rate": 4.084252758274825e-05, + "loss": 1.2584, + "theoretical_loss": 3.3381277992243312, + "tokens_seen": 3134128128 + }, + { + "epoch": 0.95, + "learning_rate": 4.082647943831495e-05, + "loss": 1.2384, + "theoretical_loss": 3.3381225253786644, + "tokens_seen": 3134193664 + }, + { + "epoch": 0.95, + "learning_rate": 4.081043129388165e-05, + "loss": 1.1226, + "theoretical_loss": 3.3381172516741486, + "tokens_seen": 3134259200 + }, + { + "epoch": 0.95, + "learning_rate": 4.079438314944834e-05, + "loss": 1.2036, + "theoretical_loss": 3.3381119781107778, + "tokens_seen": 3134324736 + }, + { + "epoch": 0.95, + "learning_rate": 4.077833500501504e-05, + "loss": 1.1533, + "theoretical_loss": 3.3381067046885455, + "tokens_seen": 3134390272 + }, + { + "epoch": 0.95, + "learning_rate": 4.076228686058175e-05, + "loss": 1.1959, + "theoretical_loss": 3.338101431407444, + "tokens_seen": 3134455808 + }, + { + "epoch": 0.95, + "learning_rate": 4.074623871614845e-05, + "loss": 1.1056, + "theoretical_loss": 3.3380961582674673, + "tokens_seen": 3134521344 + }, + { + "epoch": 0.95, + "learning_rate": 4.073019057171515e-05, + "loss": 1.2475, + "theoretical_loss": 3.338090885268609, + "tokens_seen": 3134586880 + }, + { + "epoch": 0.95, + "learning_rate": 4.0714142427281845e-05, + "loss": 1.2497, + "theoretical_loss": 3.338085612410861, + "tokens_seen": 3134652416 + }, + { + "epoch": 0.95, + "learning_rate": 4.0698094282848546e-05, + "loss": 1.1998, + "theoretical_loss": 3.3380803396942182, + "tokens_seen": 3134717952 + }, + { + "epoch": 0.95, + "learning_rate": 4.068204613841525e-05, + "loss": 1.2649, + "theoretical_loss": 3.3380750671186727, + "tokens_seen": 3134783488 + }, + { + "epoch": 0.95, + "learning_rate": 4.066599799398195e-05, + "loss": 1.1796, + "theoretical_loss": 3.3380697946842184, + "tokens_seen": 3134849024 + }, + { + "epoch": 0.95, + "learning_rate": 4.064994984954865e-05, + "loss": 1.1989, + "theoretical_loss": 3.3380645223908485, + "tokens_seen": 3134914560 + }, + { + "epoch": 0.95, + "learning_rate": 4.063390170511535e-05, + "loss": 1.2645, + "theoretical_loss": 3.338059250238556, + "tokens_seen": 3134980096 + }, + { + "epoch": 0.95, + "learning_rate": 4.061785356068204e-05, + "loss": 1.2233, + "theoretical_loss": 3.338053978227334, + "tokens_seen": 3135045632 + }, + { + "epoch": 0.95, + "learning_rate": 4.060180541624875e-05, + "loss": 1.1905, + "theoretical_loss": 3.338048706357177, + "tokens_seen": 3135111168 + }, + { + "epoch": 0.95, + "learning_rate": 4.058575727181545e-05, + "loss": 1.2135, + "theoretical_loss": 3.338043434628077, + "tokens_seen": 3135176704 + }, + { + "epoch": 0.95, + "learning_rate": 4.056970912738215e-05, + "loss": 1.1411, + "theoretical_loss": 3.3380381630400273, + "tokens_seen": 3135242240 + }, + { + "epoch": 0.95, + "learning_rate": 4.055366098294885e-05, + "loss": 1.0669, + "theoretical_loss": 3.3380328915930217, + "tokens_seen": 3135307776 + }, + { + "epoch": 0.95, + "learning_rate": 4.0537612838515546e-05, + "loss": 1.2185, + "theoretical_loss": 3.3380276202870536, + "tokens_seen": 3135373312 + }, + { + "epoch": 0.95, + "learning_rate": 4.0521564694082246e-05, + "loss": 1.249, + "theoretical_loss": 3.3380223491221157, + "tokens_seen": 3135438848 + }, + { + "epoch": 0.95, + "learning_rate": 4.050551654964895e-05, + "loss": 1.1656, + "theoretical_loss": 3.338017078098202, + "tokens_seen": 3135504384 + }, + { + "epoch": 0.95, + "learning_rate": 4.048946840521565e-05, + "loss": 1.22, + "theoretical_loss": 3.338011807215305, + "tokens_seen": 3135569920 + }, + { + "epoch": 0.95, + "learning_rate": 4.047342026078235e-05, + "loss": 1.2706, + "theoretical_loss": 3.3380065364734186, + "tokens_seen": 3135635456 + }, + { + "epoch": 0.95, + "learning_rate": 4.045737211634905e-05, + "loss": 1.206, + "theoretical_loss": 3.3380012658725353, + "tokens_seen": 3135700992 + }, + { + "epoch": 0.95, + "learning_rate": 4.044132397191575e-05, + "loss": 1.236, + "theoretical_loss": 3.3379959954126495, + "tokens_seen": 3135766528 + }, + { + "epoch": 0.95, + "learning_rate": 4.042527582748245e-05, + "loss": 1.1295, + "theoretical_loss": 3.3379907250937535, + "tokens_seen": 3135832064 + }, + { + "epoch": 0.95, + "objective/train/docs_used": 1046487, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1893852949142456, + "objective/train/theoretical_loss": 3.337985454915841, + "objective/train/tokens_used": 3156357600, + "theoretical_loss": 3.337985454915841, + "tokens_seen": 3135897600 + }, + { + "epoch": 0.95, + "learning_rate": 4.040922768304915e-05, + "loss": 1.2323, + "theoretical_loss": 3.337985454915841, + "tokens_seen": 3135897600 + }, + { + "epoch": 0.95, + "learning_rate": 4.039317953861585e-05, + "loss": 1.2677, + "theoretical_loss": 3.3379801848789055, + "tokens_seen": 3135963136 + }, + { + "epoch": 0.95, + "learning_rate": 4.037713139418255e-05, + "loss": 1.1125, + "theoretical_loss": 3.3379749149829396, + "tokens_seen": 3136028672 + }, + { + "epoch": 0.95, + "learning_rate": 4.0361083249749246e-05, + "loss": 1.2819, + "theoretical_loss": 3.3379696452279375, + "tokens_seen": 3136094208 + }, + { + "epoch": 0.95, + "learning_rate": 4.0345035105315946e-05, + "loss": 1.2091, + "theoretical_loss": 3.337964375613892, + "tokens_seen": 3136159744 + }, + { + "epoch": 0.95, + "learning_rate": 4.032898696088265e-05, + "loss": 1.1352, + "theoretical_loss": 3.3379591061407963, + "tokens_seen": 3136225280 + }, + { + "epoch": 0.95, + "learning_rate": 4.0312938816449354e-05, + "loss": 1.172, + "theoretical_loss": 3.3379538368086434, + "tokens_seen": 3136290816 + }, + { + "epoch": 0.95, + "learning_rate": 4.0296890672016055e-05, + "loss": 1.1163, + "theoretical_loss": 3.337948567617427, + "tokens_seen": 3136356352 + }, + { + "epoch": 0.95, + "learning_rate": 4.028084252758275e-05, + "loss": 1.2031, + "theoretical_loss": 3.337943298567141, + "tokens_seen": 3136421888 + }, + { + "epoch": 0.95, + "learning_rate": 4.026479438314945e-05, + "loss": 1.2146, + "theoretical_loss": 3.3379380296577774, + "tokens_seen": 3136487424 + }, + { + "epoch": 0.95, + "learning_rate": 4.024874623871615e-05, + "loss": 1.1245, + "theoretical_loss": 3.33793276088933, + "tokens_seen": 3136552960 + }, + { + "epoch": 0.95, + "learning_rate": 4.023269809428285e-05, + "loss": 1.1014, + "theoretical_loss": 3.3379274922617928, + "tokens_seen": 3136618496 + }, + { + "epoch": 0.95, + "learning_rate": 4.021664994984955e-05, + "loss": 1.2728, + "theoretical_loss": 3.3379222237751582, + "tokens_seen": 3136684032 + }, + { + "epoch": 0.95, + "learning_rate": 4.020060180541625e-05, + "loss": 1.1356, + "theoretical_loss": 3.33791695542942, + "tokens_seen": 3136749568 + }, + { + "epoch": 0.95, + "learning_rate": 4.0184553660982946e-05, + "loss": 1.1772, + "theoretical_loss": 3.3379116872245707, + "tokens_seen": 3136815104 + }, + { + "epoch": 0.95, + "learning_rate": 4.0168505516549647e-05, + "loss": 1.1827, + "theoretical_loss": 3.3379064191606043, + "tokens_seen": 3136880640 + }, + { + "epoch": 0.95, + "learning_rate": 4.0152457372116354e-05, + "loss": 1.1647, + "theoretical_loss": 3.3379011512375145, + "tokens_seen": 3136946176 + }, + { + "epoch": 0.95, + "learning_rate": 4.0136409227683055e-05, + "loss": 1.2788, + "theoretical_loss": 3.3378958834552934, + "tokens_seen": 3137011712 + }, + { + "epoch": 0.95, + "learning_rate": 4.0120361083249755e-05, + "loss": 1.2385, + "theoretical_loss": 3.337890615813935, + "tokens_seen": 3137077248 + }, + { + "epoch": 0.95, + "learning_rate": 4.010431293881645e-05, + "loss": 1.1802, + "theoretical_loss": 3.3378853483134328, + "tokens_seen": 3137142784 + }, + { + "epoch": 0.95, + "learning_rate": 4.008826479438315e-05, + "loss": 1.1914, + "theoretical_loss": 3.33788008095378, + "tokens_seen": 3137208320 + }, + { + "epoch": 0.95, + "learning_rate": 4.007221664994985e-05, + "loss": 1.194, + "theoretical_loss": 3.337874813734969, + "tokens_seen": 3137273856 + }, + { + "epoch": 0.95, + "learning_rate": 4.005616850551655e-05, + "loss": 1.1905, + "theoretical_loss": 3.3378695466569943, + "tokens_seen": 3137339392 + }, + { + "epoch": 0.95, + "learning_rate": 4.004012036108325e-05, + "loss": 1.2149, + "theoretical_loss": 3.3378642797198483, + "tokens_seen": 3137404928 + }, + { + "epoch": 0.95, + "learning_rate": 4.002407221664995e-05, + "loss": 1.1734, + "theoretical_loss": 3.3378590129235253, + "tokens_seen": 3137470464 + }, + { + "epoch": 0.95, + "learning_rate": 4.000802407221665e-05, + "loss": 1.2129, + "theoretical_loss": 3.337853746268017, + "tokens_seen": 3137536000 + }, + { + "epoch": 0.95, + "learning_rate": 3.9991975927783354e-05, + "loss": 1.1742, + "theoretical_loss": 3.3378484797533186, + "tokens_seen": 3137601536 + }, + { + "epoch": 0.95, + "learning_rate": 3.9975927783350054e-05, + "loss": 1.1924, + "theoretical_loss": 3.337843213379422, + "tokens_seen": 3137667072 + }, + { + "epoch": 0.95, + "learning_rate": 3.9959879638916755e-05, + "loss": 1.2261, + "theoretical_loss": 3.337837947146321, + "tokens_seen": 3137732608 + }, + { + "epoch": 0.95, + "learning_rate": 3.9943831494483455e-05, + "loss": 1.1413, + "theoretical_loss": 3.337832681054009, + "tokens_seen": 3137798144 + }, + { + "epoch": 0.95, + "learning_rate": 3.992778335005015e-05, + "loss": 1.181, + "theoretical_loss": 3.337827415102479, + "tokens_seen": 3137863680 + }, + { + "epoch": 0.95, + "learning_rate": 3.991173520561685e-05, + "loss": 1.1959, + "theoretical_loss": 3.3378221492917244, + "tokens_seen": 3137929216 + }, + { + "epoch": 0.95, + "learning_rate": 3.989568706118355e-05, + "loss": 1.1369, + "theoretical_loss": 3.3378168836217386, + "tokens_seen": 3137994752 + }, + { + "epoch": 0.95, + "learning_rate": 3.987963891675025e-05, + "loss": 1.117, + "theoretical_loss": 3.337811618092515, + "tokens_seen": 3138060288 + }, + { + "epoch": 0.95, + "learning_rate": 3.986359077231696e-05, + "loss": 1.2543, + "theoretical_loss": 3.3378063527040465, + "tokens_seen": 3138125824 + }, + { + "epoch": 0.95, + "learning_rate": 3.984754262788365e-05, + "loss": 1.2791, + "theoretical_loss": 3.3378010874563264, + "tokens_seen": 3138191360 + }, + { + "epoch": 0.95, + "learning_rate": 3.983149448345035e-05, + "loss": 1.1606, + "theoretical_loss": 3.3377958223493485, + "tokens_seen": 3138256896 + }, + { + "epoch": 0.95, + "learning_rate": 3.9815446339017054e-05, + "loss": 1.1805, + "theoretical_loss": 3.337790557383106, + "tokens_seen": 3138322432 + }, + { + "epoch": 0.95, + "learning_rate": 3.9799398194583754e-05, + "loss": 1.1758, + "theoretical_loss": 3.3377852925575917, + "tokens_seen": 3138387968 + }, + { + "epoch": 0.95, + "learning_rate": 3.9783350050150455e-05, + "loss": 1.2207, + "theoretical_loss": 3.3377800278727996, + "tokens_seen": 3138453504 + }, + { + "epoch": 0.95, + "learning_rate": 3.9767301905717156e-05, + "loss": 1.0986, + "theoretical_loss": 3.3377747633287225, + "tokens_seen": 3138519040 + }, + { + "epoch": 0.95, + "learning_rate": 3.975125376128385e-05, + "loss": 1.2719, + "theoretical_loss": 3.3377694989253532, + "tokens_seen": 3138584576 + }, + { + "epoch": 0.95, + "learning_rate": 3.973520561685055e-05, + "loss": 1.2372, + "theoretical_loss": 3.3377642346626866, + "tokens_seen": 3138650112 + }, + { + "epoch": 0.95, + "learning_rate": 3.971915747241725e-05, + "loss": 1.2465, + "theoretical_loss": 3.3377589705407145, + "tokens_seen": 3138715648 + }, + { + "epoch": 0.95, + "learning_rate": 3.970310932798396e-05, + "loss": 1.207, + "theoretical_loss": 3.3377537065594307, + "tokens_seen": 3138781184 + }, + { + "epoch": 0.95, + "learning_rate": 3.968706118355066e-05, + "loss": 1.0983, + "theoretical_loss": 3.3377484427188286, + "tokens_seen": 3138846720 + }, + { + "epoch": 0.95, + "learning_rate": 3.967101303911735e-05, + "loss": 1.2596, + "theoretical_loss": 3.3377431790189016, + "tokens_seen": 3138912256 + }, + { + "epoch": 0.95, + "learning_rate": 3.965496489468405e-05, + "loss": 1.2252, + "theoretical_loss": 3.3377379154596425, + "tokens_seen": 3138977792 + }, + { + "epoch": 0.95, + "learning_rate": 3.9638916750250754e-05, + "loss": 1.2602, + "theoretical_loss": 3.337732652041045, + "tokens_seen": 3139043328 + }, + { + "epoch": 0.95, + "learning_rate": 3.9622868605817455e-05, + "loss": 1.2101, + "theoretical_loss": 3.3377273887631027, + "tokens_seen": 3139108864 + }, + { + "epoch": 0.95, + "objective/train/docs_used": 1047554, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2115662097930908, + "objective/train/theoretical_loss": 3.3377221256258083, + "objective/train/tokens_used": 3159634400, + "theoretical_loss": 3.3377221256258083, + "tokens_seen": 3139174400 + }, + { + "epoch": 0.95, + "learning_rate": 3.9606820461384155e-05, + "loss": 1.2328, + "theoretical_loss": 3.3377221256258083, + "tokens_seen": 3139174400 + }, + { + "epoch": 0.95, + "learning_rate": 3.9590772316950856e-05, + "loss": 1.2251, + "theoretical_loss": 3.337716862629155, + "tokens_seen": 3139239936 + }, + { + "epoch": 0.95, + "learning_rate": 3.957472417251755e-05, + "loss": 1.1348, + "theoretical_loss": 3.337711599773137, + "tokens_seen": 3139305472 + }, + { + "epoch": 0.95, + "learning_rate": 3.955867602808426e-05, + "loss": 1.2298, + "theoretical_loss": 3.337706337057747, + "tokens_seen": 3139371008 + }, + { + "epoch": 0.95, + "learning_rate": 3.954262788365096e-05, + "loss": 1.1652, + "theoretical_loss": 3.337701074482978, + "tokens_seen": 3139436544 + }, + { + "epoch": 0.95, + "learning_rate": 3.952657973921766e-05, + "loss": 1.1813, + "theoretical_loss": 3.3376958120488243, + "tokens_seen": 3139502080 + }, + { + "epoch": 0.95, + "learning_rate": 3.951053159478436e-05, + "loss": 1.1731, + "theoretical_loss": 3.337690549755278, + "tokens_seen": 3139567616 + }, + { + "epoch": 0.95, + "learning_rate": 3.949448345035105e-05, + "loss": 1.2044, + "theoretical_loss": 3.3376852876023335, + "tokens_seen": 3139633152 + }, + { + "epoch": 0.95, + "learning_rate": 3.9478435305917754e-05, + "loss": 1.1622, + "theoretical_loss": 3.3376800255899832, + "tokens_seen": 3139698688 + }, + { + "epoch": 0.95, + "learning_rate": 3.9462387161484454e-05, + "loss": 1.2866, + "theoretical_loss": 3.337674763718221, + "tokens_seen": 3139764224 + }, + { + "epoch": 0.95, + "learning_rate": 3.9446339017051155e-05, + "loss": 1.236, + "theoretical_loss": 3.33766950198704, + "tokens_seen": 3139829760 + }, + { + "epoch": 0.95, + "learning_rate": 3.9430290872617855e-05, + "loss": 1.2615, + "theoretical_loss": 3.3376642403964336, + "tokens_seen": 3139895296 + }, + { + "epoch": 0.95, + "learning_rate": 3.9414242728184556e-05, + "loss": 1.2156, + "theoretical_loss": 3.337658978946395, + "tokens_seen": 3139960832 + }, + { + "epoch": 0.95, + "learning_rate": 3.939819458375126e-05, + "loss": 1.1664, + "theoretical_loss": 3.3376537176369174, + "tokens_seen": 3140026368 + }, + { + "epoch": 0.95, + "learning_rate": 3.938214643931796e-05, + "loss": 1.2042, + "theoretical_loss": 3.3376484564679942, + "tokens_seen": 3140091904 + }, + { + "epoch": 0.95, + "learning_rate": 3.936609829488466e-05, + "loss": 1.1134, + "theoretical_loss": 3.337643195439619, + "tokens_seen": 3140157440 + }, + { + "epoch": 0.95, + "learning_rate": 3.935005015045136e-05, + "loss": 1.2183, + "theoretical_loss": 3.3376379345517853, + "tokens_seen": 3140222976 + }, + { + "epoch": 0.95, + "learning_rate": 3.933400200601805e-05, + "loss": 1.258, + "theoretical_loss": 3.3376326738044853, + "tokens_seen": 3140288512 + }, + { + "epoch": 0.95, + "learning_rate": 3.931795386158475e-05, + "loss": 1.2075, + "theoretical_loss": 3.3376274131977133, + "tokens_seen": 3140354048 + }, + { + "epoch": 0.95, + "learning_rate": 3.9301905717151454e-05, + "loss": 1.1421, + "theoretical_loss": 3.3376221527314622, + "tokens_seen": 3140419584 + }, + { + "epoch": 0.95, + "learning_rate": 3.9285857572718154e-05, + "loss": 1.224, + "theoretical_loss": 3.337616892405726, + "tokens_seen": 3140485120 + }, + { + "epoch": 0.95, + "learning_rate": 3.9269809428284855e-05, + "loss": 1.3017, + "theoretical_loss": 3.3376116322204967, + "tokens_seen": 3140550656 + }, + { + "epoch": 0.95, + "learning_rate": 3.9253761283851556e-05, + "loss": 1.1808, + "theoretical_loss": 3.3376063721757685, + "tokens_seen": 3140616192 + }, + { + "epoch": 0.95, + "learning_rate": 3.9237713139418256e-05, + "loss": 1.1922, + "theoretical_loss": 3.337601112271535, + "tokens_seen": 3140681728 + }, + { + "epoch": 0.95, + "learning_rate": 3.922166499498496e-05, + "loss": 1.1794, + "theoretical_loss": 3.3375958525077887, + "tokens_seen": 3140747264 + }, + { + "epoch": 0.95, + "learning_rate": 3.920561685055166e-05, + "loss": 1.1443, + "theoretical_loss": 3.3375905928845238, + "tokens_seen": 3140812800 + }, + { + "epoch": 0.95, + "learning_rate": 3.918956870611836e-05, + "loss": 1.1508, + "theoretical_loss": 3.3375853334017327, + "tokens_seen": 3140878336 + }, + { + "epoch": 0.95, + "learning_rate": 3.917352056168506e-05, + "loss": 1.1655, + "theoretical_loss": 3.3375800740594093, + "tokens_seen": 3140943872 + }, + { + "epoch": 0.95, + "learning_rate": 3.915747241725175e-05, + "loss": 1.2136, + "theoretical_loss": 3.337574814857547, + "tokens_seen": 3141009408 + }, + { + "epoch": 0.95, + "learning_rate": 3.914142427281845e-05, + "loss": 1.1894, + "theoretical_loss": 3.3375695557961382, + "tokens_seen": 3141074944 + }, + { + "epoch": 0.95, + "learning_rate": 3.9125376128385154e-05, + "loss": 1.2556, + "theoretical_loss": 3.3375642968751777, + "tokens_seen": 3141140480 + }, + { + "epoch": 0.95, + "learning_rate": 3.910932798395186e-05, + "loss": 1.1725, + "theoretical_loss": 3.337559038094658, + "tokens_seen": 3141206016 + }, + { + "epoch": 0.95, + "learning_rate": 3.909327983951856e-05, + "loss": 1.161, + "theoretical_loss": 3.3375537794545718, + "tokens_seen": 3141271552 + }, + { + "epoch": 0.95, + "learning_rate": 3.9077231695085256e-05, + "loss": 1.2016, + "theoretical_loss": 3.3375485209549134, + "tokens_seen": 3141337088 + }, + { + "epoch": 0.95, + "learning_rate": 3.9061183550651957e-05, + "loss": 1.1808, + "theoretical_loss": 3.337543262595676, + "tokens_seen": 3141402624 + }, + { + "epoch": 0.95, + "learning_rate": 3.904513540621866e-05, + "loss": 1.2534, + "theoretical_loss": 3.337538004376852, + "tokens_seen": 3141468160 + }, + { + "epoch": 0.95, + "learning_rate": 3.902908726178536e-05, + "loss": 1.2203, + "theoretical_loss": 3.3375327462984363, + "tokens_seen": 3141533696 + }, + { + "epoch": 0.95, + "learning_rate": 3.901303911735206e-05, + "loss": 1.2098, + "theoretical_loss": 3.337527488360421, + "tokens_seen": 3141599232 + }, + { + "epoch": 0.95, + "learning_rate": 3.899699097291876e-05, + "loss": 1.2567, + "theoretical_loss": 3.3375222305628, + "tokens_seen": 3141664768 + }, + { + "epoch": 0.95, + "learning_rate": 3.898094282848545e-05, + "loss": 1.1686, + "theoretical_loss": 3.337516972905566, + "tokens_seen": 3141730304 + }, + { + "epoch": 0.95, + "learning_rate": 3.8964894684052154e-05, + "loss": 1.106, + "theoretical_loss": 3.337511715388713, + "tokens_seen": 3141795840 + }, + { + "epoch": 0.95, + "learning_rate": 3.894884653961886e-05, + "loss": 1.1089, + "theoretical_loss": 3.337506458012234, + "tokens_seen": 3141861376 + }, + { + "epoch": 0.95, + "learning_rate": 3.893279839518556e-05, + "loss": 1.2244, + "theoretical_loss": 3.337501200776122, + "tokens_seen": 3141926912 + }, + { + "epoch": 0.95, + "learning_rate": 3.891675025075226e-05, + "loss": 1.1893, + "theoretical_loss": 3.3374959436803713, + "tokens_seen": 3141992448 + }, + { + "epoch": 0.95, + "learning_rate": 3.8900702106318956e-05, + "loss": 1.2588, + "theoretical_loss": 3.337490686724974, + "tokens_seen": 3142057984 + }, + { + "epoch": 0.95, + "learning_rate": 3.888465396188566e-05, + "loss": 1.2237, + "theoretical_loss": 3.3374854299099246, + "tokens_seen": 3142123520 + }, + { + "epoch": 0.95, + "learning_rate": 3.886860581745236e-05, + "loss": 1.226, + "theoretical_loss": 3.3374801732352157, + "tokens_seen": 3142189056 + }, + { + "epoch": 0.95, + "learning_rate": 3.885255767301906e-05, + "loss": 1.1598, + "theoretical_loss": 3.337474916700841, + "tokens_seen": 3142254592 + }, + { + "epoch": 0.95, + "learning_rate": 3.883650952858576e-05, + "loss": 1.2335, + "theoretical_loss": 3.337469660306793, + "tokens_seen": 3142320128 + }, + { + "epoch": 0.95, + "learning_rate": 3.882046138415246e-05, + "loss": 1.134, + "theoretical_loss": 3.3374644040530663, + "tokens_seen": 3142385664 + }, + { + "epoch": 0.95, + "objective/train/docs_used": 1048829, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2685470581054688, + "objective/train/theoretical_loss": 3.3374591479396534, + "objective/train/tokens_used": 3162911200, + "theoretical_loss": 3.3374591479396534, + "tokens_seen": 3142451200 + }, + { + "epoch": 0.95, + "learning_rate": 3.880441323971916e-05, + "loss": 1.18, + "theoretical_loss": 3.3374591479396534, + "tokens_seen": 3142451200 + }, + { + "epoch": 0.95, + "learning_rate": 3.878836509528586e-05, + "loss": 1.1487, + "theoretical_loss": 3.3374538919665473, + "tokens_seen": 3142516736 + }, + { + "epoch": 0.95, + "learning_rate": 3.877231695085256e-05, + "loss": 1.1488, + "theoretical_loss": 3.3374486361337423, + "tokens_seen": 3142582272 + }, + { + "epoch": 0.95, + "learning_rate": 3.875626880641926e-05, + "loss": 1.2317, + "theoretical_loss": 3.3374433804412313, + "tokens_seen": 3142647808 + }, + { + "epoch": 0.95, + "learning_rate": 3.874022066198596e-05, + "loss": 1.1637, + "theoretical_loss": 3.3374381248890073, + "tokens_seen": 3142713344 + }, + { + "epoch": 0.95, + "learning_rate": 3.8724172517552656e-05, + "loss": 1.2996, + "theoretical_loss": 3.3374328694770643, + "tokens_seen": 3142778880 + }, + { + "epoch": 0.95, + "learning_rate": 3.870812437311936e-05, + "loss": 1.222, + "theoretical_loss": 3.337427614205395, + "tokens_seen": 3142844416 + }, + { + "epoch": 0.95, + "learning_rate": 3.869207622868606e-05, + "loss": 1.262, + "theoretical_loss": 3.3374223590739933, + "tokens_seen": 3142909952 + }, + { + "epoch": 0.95, + "learning_rate": 3.867602808425276e-05, + "loss": 1.1453, + "theoretical_loss": 3.337417104082852, + "tokens_seen": 3142975488 + }, + { + "epoch": 0.95, + "learning_rate": 3.8659979939819466e-05, + "loss": 1.1783, + "theoretical_loss": 3.337411849231964, + "tokens_seen": 3143041024 + }, + { + "epoch": 0.95, + "learning_rate": 3.864393179538616e-05, + "loss": 1.2513, + "theoretical_loss": 3.3374065945213243, + "tokens_seen": 3143106560 + }, + { + "epoch": 0.95, + "learning_rate": 3.862788365095286e-05, + "loss": 1.1607, + "theoretical_loss": 3.337401339950925, + "tokens_seen": 3143172096 + }, + { + "epoch": 0.95, + "learning_rate": 3.861183550651956e-05, + "loss": 1.1948, + "theoretical_loss": 3.3373960855207594, + "tokens_seen": 3143237632 + }, + { + "epoch": 0.95, + "learning_rate": 3.859578736208626e-05, + "loss": 1.192, + "theoretical_loss": 3.337390831230821, + "tokens_seen": 3143303168 + }, + { + "epoch": 0.95, + "learning_rate": 3.857973921765296e-05, + "loss": 1.1298, + "theoretical_loss": 3.3373855770811036, + "tokens_seen": 3143368704 + }, + { + "epoch": 0.95, + "learning_rate": 3.856369107321966e-05, + "loss": 1.2695, + "theoretical_loss": 3.3373803230715997, + "tokens_seen": 3143434240 + }, + { + "epoch": 0.95, + "learning_rate": 3.8547642928786356e-05, + "loss": 1.133, + "theoretical_loss": 3.337375069202303, + "tokens_seen": 3143499776 + }, + { + "epoch": 0.95, + "learning_rate": 3.853159478435306e-05, + "loss": 1.2318, + "theoretical_loss": 3.3373698154732074, + "tokens_seen": 3143565312 + }, + { + "epoch": 0.95, + "learning_rate": 3.851554663991976e-05, + "loss": 1.2032, + "theoretical_loss": 3.3373645618843057, + "tokens_seen": 3143630848 + }, + { + "epoch": 0.95, + "learning_rate": 3.8499498495486465e-05, + "loss": 1.2051, + "theoretical_loss": 3.3373593084355915, + "tokens_seen": 3143696384 + }, + { + "epoch": 0.95, + "learning_rate": 3.8483450351053166e-05, + "loss": 1.1936, + "theoretical_loss": 3.337354055127057, + "tokens_seen": 3143761920 + }, + { + "epoch": 0.95, + "learning_rate": 3.846740220661986e-05, + "loss": 1.1888, + "theoretical_loss": 3.3373488019586977, + "tokens_seen": 3143827456 + }, + { + "epoch": 0.95, + "learning_rate": 3.845135406218656e-05, + "loss": 1.1454, + "theoretical_loss": 3.3373435489305048, + "tokens_seen": 3143892992 + }, + { + "epoch": 0.95, + "learning_rate": 3.843530591775326e-05, + "loss": 1.1885, + "theoretical_loss": 3.3373382960424727, + "tokens_seen": 3143958528 + }, + { + "epoch": 0.95, + "learning_rate": 3.841925777331996e-05, + "loss": 1.1978, + "theoretical_loss": 3.3373330432945947, + "tokens_seen": 3144024064 + }, + { + "epoch": 0.95, + "learning_rate": 3.840320962888666e-05, + "loss": 1.2109, + "theoretical_loss": 3.337327790686864, + "tokens_seen": 3144089600 + }, + { + "epoch": 0.95, + "learning_rate": 3.838716148445336e-05, + "loss": 1.1802, + "theoretical_loss": 3.3373225382192744, + "tokens_seen": 3144155136 + }, + { + "epoch": 0.95, + "learning_rate": 3.837111334002006e-05, + "loss": 1.1403, + "theoretical_loss": 3.3373172858918183, + "tokens_seen": 3144220672 + }, + { + "epoch": 0.95, + "learning_rate": 3.8355065195586764e-05, + "loss": 1.2135, + "theoretical_loss": 3.3373120337044897, + "tokens_seen": 3144286208 + }, + { + "epoch": 0.95, + "learning_rate": 3.8339017051153465e-05, + "loss": 1.145, + "theoretical_loss": 3.3373067816572815, + "tokens_seen": 3144351744 + }, + { + "epoch": 0.95, + "learning_rate": 3.8322968906720165e-05, + "loss": 1.1435, + "theoretical_loss": 3.337301529750188, + "tokens_seen": 3144417280 + }, + { + "epoch": 0.95, + "learning_rate": 3.8306920762286866e-05, + "loss": 1.1695, + "theoretical_loss": 3.337296277983201, + "tokens_seen": 3144482816 + }, + { + "epoch": 0.95, + "learning_rate": 3.829087261785356e-05, + "loss": 1.1492, + "theoretical_loss": 3.337291026356315, + "tokens_seen": 3144548352 + }, + { + "epoch": 0.95, + "learning_rate": 3.827482447342026e-05, + "loss": 1.1654, + "theoretical_loss": 3.3372857748695233, + "tokens_seen": 3144613888 + }, + { + "epoch": 0.95, + "learning_rate": 3.825877632898696e-05, + "loss": 1.2189, + "theoretical_loss": 3.3372805235228187, + "tokens_seen": 3144679424 + }, + { + "epoch": 0.95, + "learning_rate": 3.824272818455366e-05, + "loss": 1.1804, + "theoretical_loss": 3.3372752723161954, + "tokens_seen": 3144744960 + }, + { + "epoch": 0.95, + "learning_rate": 3.822668004012036e-05, + "loss": 1.1869, + "theoretical_loss": 3.337270021249646, + "tokens_seen": 3144810496 + }, + { + "epoch": 0.95, + "learning_rate": 3.821063189568706e-05, + "loss": 1.1622, + "theoretical_loss": 3.3372647703231633, + "tokens_seen": 3144876032 + }, + { + "epoch": 0.95, + "learning_rate": 3.8194583751253764e-05, + "loss": 1.2567, + "theoretical_loss": 3.337259519536742, + "tokens_seen": 3144941568 + }, + { + "epoch": 0.95, + "learning_rate": 3.8178535606820464e-05, + "loss": 1.1538, + "theoretical_loss": 3.3372542688903746, + "tokens_seen": 3145007104 + }, + { + "epoch": 0.95, + "learning_rate": 3.8162487462387165e-05, + "loss": 1.2791, + "theoretical_loss": 3.3372490183840546, + "tokens_seen": 3145072640 + }, + { + "epoch": 0.95, + "learning_rate": 3.8146439317953866e-05, + "loss": 1.1927, + "theoretical_loss": 3.3372437680177756, + "tokens_seen": 3145138176 + }, + { + "epoch": 0.95, + "learning_rate": 3.8130391173520566e-05, + "loss": 1.2341, + "theoretical_loss": 3.3372385177915307, + "tokens_seen": 3145203712 + }, + { + "epoch": 0.95, + "learning_rate": 3.811434302908726e-05, + "loss": 1.1764, + "theoretical_loss": 3.3372332677053134, + "tokens_seen": 3145269248 + }, + { + "epoch": 0.95, + "learning_rate": 3.809829488465396e-05, + "loss": 1.2337, + "theoretical_loss": 3.337228017759117, + "tokens_seen": 3145334784 + }, + { + "epoch": 0.95, + "learning_rate": 3.808224674022066e-05, + "loss": 1.1654, + "theoretical_loss": 3.3372227679529347, + "tokens_seen": 3145400320 + }, + { + "epoch": 0.95, + "learning_rate": 3.806619859578736e-05, + "loss": 1.2544, + "theoretical_loss": 3.33721751828676, + "tokens_seen": 3145465856 + }, + { + "epoch": 0.95, + "learning_rate": 3.805015045135407e-05, + "loss": 1.3321, + "theoretical_loss": 3.337212268760586, + "tokens_seen": 3145531392 + }, + { + "epoch": 0.95, + "learning_rate": 3.803410230692076e-05, + "loss": 1.2653, + "theoretical_loss": 3.3372070193744063, + "tokens_seen": 3145596928 + }, + { + "epoch": 0.95, + "learning_rate": 3.8018054162487464e-05, + "loss": 1.2022, + "theoretical_loss": 3.3372017701282144, + "tokens_seen": 3145662464 + }, + { + "epoch": 0.95, + "objective/train/docs_used": 1050110, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.0779855251312256, + "objective/train/theoretical_loss": 3.337196521022003, + "objective/train/tokens_used": 3166188000, + "theoretical_loss": 3.337196521022003, + "tokens_seen": 3145728000 + }, + { + "epoch": 0.95, + "learning_rate": 3.8002006018054165e-05, + "loss": 1.1847, + "theoretical_loss": 3.337196521022003, + "tokens_seen": 3145728000 + }, + { + "epoch": 0.95, + "learning_rate": 3.7985957873620865e-05, + "loss": 1.1561, + "theoretical_loss": 3.337191272055766, + "tokens_seen": 3145793536 + }, + { + "epoch": 0.95, + "learning_rate": 3.7969909729187566e-05, + "loss": 1.2599, + "theoretical_loss": 3.3371860232294974, + "tokens_seen": 3145859072 + }, + { + "epoch": 0.95, + "learning_rate": 3.7953861584754266e-05, + "loss": 1.184, + "theoretical_loss": 3.337180774543189, + "tokens_seen": 3145924608 + }, + { + "epoch": 0.95, + "learning_rate": 3.793781344032096e-05, + "loss": 1.1974, + "theoretical_loss": 3.3371755259968356, + "tokens_seen": 3145990144 + }, + { + "epoch": 0.95, + "learning_rate": 3.792176529588766e-05, + "loss": 1.169, + "theoretical_loss": 3.3371702775904293, + "tokens_seen": 3146055680 + }, + { + "epoch": 0.95, + "learning_rate": 3.790571715145437e-05, + "loss": 1.2521, + "theoretical_loss": 3.3371650293239643, + "tokens_seen": 3146121216 + }, + { + "epoch": 0.95, + "learning_rate": 3.788966900702107e-05, + "loss": 1.1398, + "theoretical_loss": 3.3371597811974336, + "tokens_seen": 3146186752 + }, + { + "epoch": 0.95, + "learning_rate": 3.787362086258777e-05, + "loss": 1.0855, + "theoretical_loss": 3.337154533210831, + "tokens_seen": 3146252288 + }, + { + "epoch": 0.95, + "learning_rate": 3.7857572718154463e-05, + "loss": 1.2746, + "theoretical_loss": 3.3371492853641493, + "tokens_seen": 3146317824 + }, + { + "epoch": 0.95, + "learning_rate": 3.7841524573721164e-05, + "loss": 1.2017, + "theoretical_loss": 3.337144037657382, + "tokens_seen": 3146383360 + }, + { + "epoch": 0.95, + "learning_rate": 3.7825476429287865e-05, + "loss": 1.1168, + "theoretical_loss": 3.3371387900905227, + "tokens_seen": 3146448896 + }, + { + "epoch": 0.95, + "learning_rate": 3.7809428284854565e-05, + "loss": 1.1562, + "theoretical_loss": 3.3371335426635644, + "tokens_seen": 3146514432 + }, + { + "epoch": 0.95, + "learning_rate": 3.7793380140421266e-05, + "loss": 1.1966, + "theoretical_loss": 3.337128295376501, + "tokens_seen": 3146579968 + }, + { + "epoch": 0.95, + "learning_rate": 3.777733199598797e-05, + "loss": 1.2024, + "theoretical_loss": 3.337123048229325, + "tokens_seen": 3146645504 + }, + { + "epoch": 0.95, + "learning_rate": 3.776128385155467e-05, + "loss": 1.249, + "theoretical_loss": 3.3371178012220306, + "tokens_seen": 3146711040 + }, + { + "epoch": 0.95, + "learning_rate": 3.774523570712137e-05, + "loss": 1.2347, + "theoretical_loss": 3.3371125543546105, + "tokens_seen": 3146776576 + }, + { + "epoch": 0.95, + "learning_rate": 3.772918756268807e-05, + "loss": 1.2483, + "theoretical_loss": 3.3371073076270585, + "tokens_seen": 3146842112 + }, + { + "epoch": 0.95, + "learning_rate": 3.771313941825477e-05, + "loss": 1.2426, + "theoretical_loss": 3.337102061039368, + "tokens_seen": 3146907648 + }, + { + "epoch": 0.95, + "learning_rate": 3.769709127382147e-05, + "loss": 1.2066, + "theoretical_loss": 3.3370968145915323, + "tokens_seen": 3146973184 + }, + { + "epoch": 0.95, + "learning_rate": 3.7681043129388164e-05, + "loss": 1.1233, + "theoretical_loss": 3.3370915682835447, + "tokens_seen": 3147038720 + }, + { + "epoch": 0.95, + "learning_rate": 3.7664994984954864e-05, + "loss": 1.1297, + "theoretical_loss": 3.337086322115398, + "tokens_seen": 3147104256 + }, + { + "epoch": 0.95, + "learning_rate": 3.7648946840521565e-05, + "loss": 1.208, + "theoretical_loss": 3.3370810760870864, + "tokens_seen": 3147169792 + }, + { + "epoch": 0.95, + "learning_rate": 3.7632898696088266e-05, + "loss": 1.175, + "theoretical_loss": 3.337075830198603, + "tokens_seen": 3147235328 + }, + { + "epoch": 0.95, + "learning_rate": 3.7616850551654966e-05, + "loss": 1.1343, + "theoretical_loss": 3.337070584449941, + "tokens_seen": 3147300864 + }, + { + "epoch": 0.95, + "learning_rate": 3.760080240722167e-05, + "loss": 1.2335, + "theoretical_loss": 3.337065338841094, + "tokens_seen": 3147366400 + }, + { + "epoch": 0.95, + "learning_rate": 3.758475426278837e-05, + "loss": 1.1154, + "theoretical_loss": 3.337060093372055, + "tokens_seen": 3147431936 + }, + { + "epoch": 0.95, + "learning_rate": 3.756870611835507e-05, + "loss": 1.1378, + "theoretical_loss": 3.337054848042818, + "tokens_seen": 3147497472 + }, + { + "epoch": 0.95, + "learning_rate": 3.755265797392177e-05, + "loss": 1.1166, + "theoretical_loss": 3.3370496028533756, + "tokens_seen": 3147563008 + }, + { + "epoch": 0.95, + "learning_rate": 3.753660982948847e-05, + "loss": 1.1777, + "theoretical_loss": 3.3370443578037214, + "tokens_seen": 3147628544 + }, + { + "epoch": 0.95, + "learning_rate": 3.752056168505517e-05, + "loss": 1.1624, + "theoretical_loss": 3.337039112893849, + "tokens_seen": 3147694080 + }, + { + "epoch": 0.95, + "learning_rate": 3.7504513540621864e-05, + "loss": 1.1785, + "theoretical_loss": 3.3370338681237515, + "tokens_seen": 3147759616 + }, + { + "epoch": 0.95, + "learning_rate": 3.7488465396188564e-05, + "loss": 1.1393, + "theoretical_loss": 3.3370286234934228, + "tokens_seen": 3147825152 + }, + { + "epoch": 0.95, + "learning_rate": 3.7472417251755265e-05, + "loss": 1.1923, + "theoretical_loss": 3.3370233790028556, + "tokens_seen": 3147890688 + }, + { + "epoch": 0.95, + "learning_rate": 3.745636910732197e-05, + "loss": 1.2983, + "theoretical_loss": 3.3370181346520438, + "tokens_seen": 3147956224 + }, + { + "epoch": 0.95, + "learning_rate": 3.7440320962888666e-05, + "loss": 1.1267, + "theoretical_loss": 3.33701289044098, + "tokens_seen": 3148021760 + }, + { + "epoch": 0.95, + "learning_rate": 3.742427281845537e-05, + "loss": 1.2608, + "theoretical_loss": 3.337007646369658, + "tokens_seen": 3148087296 + }, + { + "epoch": 0.95, + "learning_rate": 3.740822467402207e-05, + "loss": 1.2682, + "theoretical_loss": 3.3370024024380722, + "tokens_seen": 3148152832 + }, + { + "epoch": 0.95, + "learning_rate": 3.739217652958877e-05, + "loss": 1.2241, + "theoretical_loss": 3.336997158646214, + "tokens_seen": 3148218368 + }, + { + "epoch": 0.95, + "learning_rate": 3.737612838515547e-05, + "loss": 1.0725, + "theoretical_loss": 3.336991914994078, + "tokens_seen": 3148283904 + }, + { + "epoch": 0.95, + "learning_rate": 3.736008024072217e-05, + "loss": 1.2063, + "theoretical_loss": 3.336986671481658, + "tokens_seen": 3148349440 + }, + { + "epoch": 0.95, + "learning_rate": 3.7344032096288863e-05, + "loss": 1.098, + "theoretical_loss": 3.3369814281089463, + "tokens_seen": 3148414976 + }, + { + "epoch": 0.95, + "learning_rate": 3.7327983951855564e-05, + "loss": 1.1897, + "theoretical_loss": 3.3369761848759363, + "tokens_seen": 3148480512 + }, + { + "epoch": 0.95, + "learning_rate": 3.731193580742227e-05, + "loss": 1.116, + "theoretical_loss": 3.336970941782622, + "tokens_seen": 3148546048 + }, + { + "epoch": 0.95, + "learning_rate": 3.729588766298897e-05, + "loss": 1.2157, + "theoretical_loss": 3.3369656988289966, + "tokens_seen": 3148611584 + }, + { + "epoch": 0.95, + "learning_rate": 3.727983951855567e-05, + "loss": 1.2261, + "theoretical_loss": 3.336960456015053, + "tokens_seen": 3148677120 + }, + { + "epoch": 0.95, + "learning_rate": 3.7263791374122367e-05, + "loss": 1.188, + "theoretical_loss": 3.3369552133407856, + "tokens_seen": 3148742656 + }, + { + "epoch": 0.95, + "learning_rate": 3.724774322968907e-05, + "loss": 1.188, + "theoretical_loss": 3.336949970806187, + "tokens_seen": 3148808192 + }, + { + "epoch": 0.95, + "learning_rate": 3.723169508525577e-05, + "loss": 1.1841, + "theoretical_loss": 3.3369447284112503, + "tokens_seen": 3148873728 + }, + { + "epoch": 0.95, + "learning_rate": 3.721564694082247e-05, + "loss": 1.2168, + "theoretical_loss": 3.33693948615597, + "tokens_seen": 3148939264 + }, + { + "epoch": 0.95, + "objective/train/docs_used": 1051136, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1774084568023682, + "objective/train/theoretical_loss": 3.336934244040338, + "objective/train/tokens_used": 3169464800, + "theoretical_loss": 3.336934244040338, + "tokens_seen": 3149004800 + }, + { + "epoch": 0.95, + "learning_rate": 3.719959879638917e-05, + "loss": 1.13, + "theoretical_loss": 3.336934244040338, + "tokens_seen": 3149004800 + }, + { + "epoch": 0.95, + "learning_rate": 3.718355065195587e-05, + "loss": 1.2367, + "theoretical_loss": 3.3369290020643487, + "tokens_seen": 3149070336 + }, + { + "epoch": 0.95, + "learning_rate": 3.7167502507522564e-05, + "loss": 1.2362, + "theoretical_loss": 3.3369237602279953, + "tokens_seen": 3149135872 + }, + { + "epoch": 0.95, + "learning_rate": 3.715145436308927e-05, + "loss": 1.1926, + "theoretical_loss": 3.336918518531271, + "tokens_seen": 3149201408 + }, + { + "epoch": 0.95, + "learning_rate": 3.713540621865597e-05, + "loss": 1.2027, + "theoretical_loss": 3.3369132769741694, + "tokens_seen": 3149266944 + }, + { + "epoch": 0.95, + "learning_rate": 3.711935807422267e-05, + "loss": 1.1844, + "theoretical_loss": 3.3369080355566836, + "tokens_seen": 3149332480 + }, + { + "epoch": 0.95, + "learning_rate": 3.710330992978937e-05, + "loss": 1.2062, + "theoretical_loss": 3.336902794278807, + "tokens_seen": 3149398016 + }, + { + "epoch": 0.95, + "learning_rate": 3.708726178535607e-05, + "loss": 1.3871, + "theoretical_loss": 3.3368975531405334, + "tokens_seen": 3149463552 + }, + { + "epoch": 0.95, + "learning_rate": 3.707121364092277e-05, + "loss": 1.2441, + "theoretical_loss": 3.336892312141855, + "tokens_seen": 3149529088 + }, + { + "epoch": 0.95, + "learning_rate": 3.705516549648947e-05, + "loss": 1.2331, + "theoretical_loss": 3.336887071282767, + "tokens_seen": 3149594624 + }, + { + "epoch": 0.95, + "learning_rate": 3.703911735205617e-05, + "loss": 1.2282, + "theoretical_loss": 3.3368818305632617, + "tokens_seen": 3149660160 + }, + { + "epoch": 0.95, + "learning_rate": 3.702306920762287e-05, + "loss": 1.0863, + "theoretical_loss": 3.336876589983332, + "tokens_seen": 3149725696 + }, + { + "epoch": 0.95, + "learning_rate": 3.700702106318957e-05, + "loss": 1.1898, + "theoretical_loss": 3.3368713495429727, + "tokens_seen": 3149791232 + }, + { + "epoch": 0.95, + "learning_rate": 3.699097291875627e-05, + "loss": 1.2086, + "theoretical_loss": 3.3368661092421754, + "tokens_seen": 3149856768 + }, + { + "epoch": 0.95, + "learning_rate": 3.697492477432297e-05, + "loss": 1.2563, + "theoretical_loss": 3.336860869080935, + "tokens_seen": 3149922304 + }, + { + "epoch": 0.95, + "learning_rate": 3.695887662988967e-05, + "loss": 1.1869, + "theoretical_loss": 3.336855629059244, + "tokens_seen": 3149987840 + }, + { + "epoch": 0.95, + "learning_rate": 3.694282848545637e-05, + "loss": 1.2172, + "theoretical_loss": 3.3368503891770964, + "tokens_seen": 3150053376 + }, + { + "epoch": 0.95, + "learning_rate": 3.692678034102307e-05, + "loss": 1.2261, + "theoretical_loss": 3.3368451494344855, + "tokens_seen": 3150118912 + }, + { + "epoch": 0.95, + "learning_rate": 3.691073219658977e-05, + "loss": 1.2614, + "theoretical_loss": 3.336839909831404, + "tokens_seen": 3150184448 + }, + { + "epoch": 0.95, + "learning_rate": 3.689468405215647e-05, + "loss": 1.1559, + "theoretical_loss": 3.3368346703678458, + "tokens_seen": 3150249984 + }, + { + "epoch": 0.95, + "learning_rate": 3.687863590772317e-05, + "loss": 1.2533, + "theoretical_loss": 3.336829431043804, + "tokens_seen": 3150315520 + }, + { + "epoch": 0.95, + "learning_rate": 3.686258776328987e-05, + "loss": 1.1463, + "theoretical_loss": 3.3368241918592725, + "tokens_seen": 3150381056 + }, + { + "epoch": 0.95, + "learning_rate": 3.6846539618856576e-05, + "loss": 1.1193, + "theoretical_loss": 3.3368189528142445, + "tokens_seen": 3150446592 + }, + { + "epoch": 0.95, + "learning_rate": 3.683049147442327e-05, + "loss": 1.1767, + "theoretical_loss": 3.3368137139087133, + "tokens_seen": 3150512128 + }, + { + "epoch": 0.95, + "learning_rate": 3.681444332998997e-05, + "loss": 1.161, + "theoretical_loss": 3.336808475142672, + "tokens_seen": 3150577664 + }, + { + "epoch": 0.95, + "learning_rate": 3.679839518555667e-05, + "loss": 1.1352, + "theoretical_loss": 3.336803236516114, + "tokens_seen": 3150643200 + }, + { + "epoch": 0.95, + "learning_rate": 3.678234704112337e-05, + "loss": 1.3094, + "theoretical_loss": 3.3367979980290334, + "tokens_seen": 3150708736 + }, + { + "epoch": 0.95, + "learning_rate": 3.676629889669007e-05, + "loss": 1.3894, + "theoretical_loss": 3.3367927596814226, + "tokens_seen": 3150774272 + }, + { + "epoch": 0.95, + "learning_rate": 3.675025075225677e-05, + "loss": 1.1795, + "theoretical_loss": 3.336787521473276, + "tokens_seen": 3150839808 + }, + { + "epoch": 0.95, + "learning_rate": 3.673420260782347e-05, + "loss": 1.3053, + "theoretical_loss": 3.3367822834045864, + "tokens_seen": 3150905344 + }, + { + "epoch": 0.95, + "learning_rate": 3.671815446339017e-05, + "loss": 1.1601, + "theoretical_loss": 3.3367770454753467, + "tokens_seen": 3150970880 + }, + { + "epoch": 0.95, + "learning_rate": 3.6702106318956875e-05, + "loss": 1.2307, + "theoretical_loss": 3.3367718076855515, + "tokens_seen": 3151036416 + }, + { + "epoch": 0.95, + "learning_rate": 3.6686058174523576e-05, + "loss": 1.2416, + "theoretical_loss": 3.336766570035193, + "tokens_seen": 3151101952 + }, + { + "epoch": 0.95, + "learning_rate": 3.6670010030090277e-05, + "loss": 1.2291, + "theoretical_loss": 3.3367613325242655, + "tokens_seen": 3151167488 + }, + { + "epoch": 0.95, + "learning_rate": 3.665396188565697e-05, + "loss": 1.1555, + "theoretical_loss": 3.336756095152762, + "tokens_seen": 3151233024 + }, + { + "epoch": 0.95, + "learning_rate": 3.663791374122367e-05, + "loss": 1.2763, + "theoretical_loss": 3.3367508579206757, + "tokens_seen": 3151298560 + }, + { + "epoch": 0.95, + "learning_rate": 3.662186559679037e-05, + "loss": 1.2146, + "theoretical_loss": 3.336745620828, + "tokens_seen": 3151364096 + }, + { + "epoch": 0.95, + "learning_rate": 3.660581745235707e-05, + "loss": 1.1634, + "theoretical_loss": 3.336740383874729, + "tokens_seen": 3151429632 + }, + { + "epoch": 0.95, + "learning_rate": 3.658976930792377e-05, + "loss": 1.0855, + "theoretical_loss": 3.3367351470608555, + "tokens_seen": 3151495168 + }, + { + "epoch": 0.96, + "learning_rate": 3.6573721163490474e-05, + "loss": 1.1528, + "theoretical_loss": 3.336729910386373, + "tokens_seen": 3151560704 + }, + { + "epoch": 0.96, + "learning_rate": 3.6557673019057174e-05, + "loss": 1.2326, + "theoretical_loss": 3.3367246738512746, + "tokens_seen": 3151626240 + }, + { + "epoch": 0.96, + "learning_rate": 3.6541624874623875e-05, + "loss": 1.1455, + "theoretical_loss": 3.336719437455554, + "tokens_seen": 3151691776 + }, + { + "epoch": 0.96, + "learning_rate": 3.6525576730190575e-05, + "loss": 1.2241, + "theoretical_loss": 3.3367142011992046, + "tokens_seen": 3151757312 + }, + { + "epoch": 0.96, + "learning_rate": 3.6509528585757276e-05, + "loss": 1.1795, + "theoretical_loss": 3.3367089650822197, + "tokens_seen": 3151822848 + }, + { + "epoch": 0.96, + "learning_rate": 3.649348044132398e-05, + "loss": 1.1628, + "theoretical_loss": 3.3367037291045927, + "tokens_seen": 3151888384 + }, + { + "epoch": 0.96, + "learning_rate": 3.647743229689067e-05, + "loss": 1.2664, + "theoretical_loss": 3.336698493266317, + "tokens_seen": 3151953920 + }, + { + "epoch": 0.96, + "learning_rate": 3.646138415245737e-05, + "loss": 1.2238, + "theoretical_loss": 3.336693257567386, + "tokens_seen": 3152019456 + }, + { + "epoch": 0.96, + "learning_rate": 3.644533600802407e-05, + "loss": 1.3047, + "theoretical_loss": 3.336688022007793, + "tokens_seen": 3152084992 + }, + { + "epoch": 0.96, + "learning_rate": 3.642928786359077e-05, + "loss": 1.0544, + "theoretical_loss": 3.3366827865875317, + "tokens_seen": 3152150528 + }, + { + "epoch": 0.96, + "learning_rate": 3.641323971915747e-05, + "loss": 1.2014, + "theoretical_loss": 3.3366775513065954, + "tokens_seen": 3152216064 + }, + { + "epoch": 0.96, + "objective/train/docs_used": 1052315, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2111128568649292, + "objective/train/theoretical_loss": 3.336672316164977, + "objective/train/tokens_used": 3172741600, + "theoretical_loss": 3.336672316164977, + "tokens_seen": 3152281600 + }, + { + "epoch": 0.96, + "learning_rate": 3.6397191574724174e-05, + "loss": 1.1182, + "theoretical_loss": 3.336672316164977, + "tokens_seen": 3152281600 + }, + { + "epoch": 0.96, + "learning_rate": 3.6381143430290874e-05, + "loss": 1.1244, + "theoretical_loss": 3.3366670811626706, + "tokens_seen": 3152347136 + }, + { + "epoch": 0.96, + "learning_rate": 3.6365095285857575e-05, + "loss": 1.1948, + "theoretical_loss": 3.336661846299669, + "tokens_seen": 3152412672 + }, + { + "epoch": 0.96, + "learning_rate": 3.6349047141424276e-05, + "loss": 1.1526, + "theoretical_loss": 3.3366566115759664, + "tokens_seen": 3152478208 + }, + { + "epoch": 0.96, + "learning_rate": 3.6332998996990976e-05, + "loss": 1.2482, + "theoretical_loss": 3.336651376991555, + "tokens_seen": 3152543744 + }, + { + "epoch": 0.96, + "learning_rate": 3.631695085255768e-05, + "loss": 1.2167, + "theoretical_loss": 3.3366461425464293, + "tokens_seen": 3152609280 + }, + { + "epoch": 0.96, + "learning_rate": 3.630090270812437e-05, + "loss": 1.1288, + "theoretical_loss": 3.3366409082405823, + "tokens_seen": 3152674816 + }, + { + "epoch": 0.96, + "learning_rate": 3.628485456369107e-05, + "loss": 1.106, + "theoretical_loss": 3.3366356740740075, + "tokens_seen": 3152740352 + }, + { + "epoch": 0.96, + "learning_rate": 3.626880641925777e-05, + "loss": 1.2779, + "theoretical_loss": 3.3366304400466977, + "tokens_seen": 3152805888 + }, + { + "epoch": 0.96, + "learning_rate": 3.625275827482448e-05, + "loss": 1.1425, + "theoretical_loss": 3.336625206158647, + "tokens_seen": 3152871424 + }, + { + "epoch": 0.96, + "learning_rate": 3.623671013039118e-05, + "loss": 1.1879, + "theoretical_loss": 3.3366199724098484, + "tokens_seen": 3152936960 + }, + { + "epoch": 0.96, + "learning_rate": 3.6220661985957874e-05, + "loss": 1.15, + "theoretical_loss": 3.336614738800296, + "tokens_seen": 3153002496 + }, + { + "epoch": 0.96, + "learning_rate": 3.6204613841524575e-05, + "loss": 1.2066, + "theoretical_loss": 3.3366095053299825, + "tokens_seen": 3153068032 + }, + { + "epoch": 0.96, + "learning_rate": 3.6188565697091275e-05, + "loss": 1.1411, + "theoretical_loss": 3.336604271998901, + "tokens_seen": 3153133568 + }, + { + "epoch": 0.96, + "learning_rate": 3.6172517552657976e-05, + "loss": 1.142, + "theoretical_loss": 3.3365990388070457, + "tokens_seen": 3153199104 + }, + { + "epoch": 0.96, + "learning_rate": 3.6156469408224677e-05, + "loss": 1.2968, + "theoretical_loss": 3.3365938057544096, + "tokens_seen": 3153264640 + }, + { + "epoch": 0.96, + "learning_rate": 3.614042126379138e-05, + "loss": 1.1586, + "theoretical_loss": 3.3365885728409865, + "tokens_seen": 3153330176 + }, + { + "epoch": 0.96, + "learning_rate": 3.612437311935807e-05, + "loss": 1.2916, + "theoretical_loss": 3.336583340066769, + "tokens_seen": 3153395712 + }, + { + "epoch": 0.96, + "learning_rate": 3.610832497492478e-05, + "loss": 1.196, + "theoretical_loss": 3.3365781074317513, + "tokens_seen": 3153461248 + }, + { + "epoch": 0.96, + "learning_rate": 3.609227683049148e-05, + "loss": 1.1894, + "theoretical_loss": 3.3365728749359267, + "tokens_seen": 3153526784 + }, + { + "epoch": 0.96, + "learning_rate": 3.607622868605818e-05, + "loss": 1.1576, + "theoretical_loss": 3.336567642579288, + "tokens_seen": 3153592320 + }, + { + "epoch": 0.96, + "learning_rate": 3.606018054162488e-05, + "loss": 1.0799, + "theoretical_loss": 3.3365624103618288, + "tokens_seen": 3153657856 + }, + { + "epoch": 0.96, + "learning_rate": 3.6044132397191574e-05, + "loss": 1.132, + "theoretical_loss": 3.3365571782835435, + "tokens_seen": 3153723392 + }, + { + "epoch": 0.96, + "learning_rate": 3.6028084252758275e-05, + "loss": 1.1687, + "theoretical_loss": 3.336551946344424, + "tokens_seen": 3153788928 + }, + { + "epoch": 0.96, + "learning_rate": 3.6012036108324975e-05, + "loss": 1.2013, + "theoretical_loss": 3.3365467145444647, + "tokens_seen": 3153854464 + }, + { + "epoch": 0.96, + "learning_rate": 3.5995987963891676e-05, + "loss": 1.2372, + "theoretical_loss": 3.336541482883659, + "tokens_seen": 3153920000 + }, + { + "epoch": 0.96, + "learning_rate": 3.597993981945838e-05, + "loss": 1.2097, + "theoretical_loss": 3.3365362513619994, + "tokens_seen": 3153985536 + }, + { + "epoch": 0.96, + "learning_rate": 3.596389167502508e-05, + "loss": 1.2675, + "theoretical_loss": 3.33653101997948, + "tokens_seen": 3154051072 + }, + { + "epoch": 0.96, + "learning_rate": 3.594784353059178e-05, + "loss": 1.1912, + "theoretical_loss": 3.3365257887360946, + "tokens_seen": 3154116608 + }, + { + "epoch": 0.96, + "learning_rate": 3.593179538615848e-05, + "loss": 1.1714, + "theoretical_loss": 3.3365205576318364, + "tokens_seen": 3154182144 + }, + { + "epoch": 0.96, + "learning_rate": 3.591574724172518e-05, + "loss": 1.17, + "theoretical_loss": 3.336515326666698, + "tokens_seen": 3154247680 + }, + { + "epoch": 0.96, + "learning_rate": 3.589969909729188e-05, + "loss": 1.118, + "theoretical_loss": 3.3365100958406737, + "tokens_seen": 3154313216 + }, + { + "epoch": 0.96, + "learning_rate": 3.588365095285858e-05, + "loss": 1.1556, + "theoretical_loss": 3.3365048651537563, + "tokens_seen": 3154378752 + }, + { + "epoch": 0.96, + "learning_rate": 3.5867602808425274e-05, + "loss": 1.1364, + "theoretical_loss": 3.3364996346059397, + "tokens_seen": 3154444288 + }, + { + "epoch": 0.96, + "learning_rate": 3.5851554663991975e-05, + "loss": 1.1202, + "theoretical_loss": 3.336494404197217, + "tokens_seen": 3154509824 + }, + { + "epoch": 0.96, + "learning_rate": 3.5835506519558676e-05, + "loss": 1.2323, + "theoretical_loss": 3.336489173927582, + "tokens_seen": 3154575360 + }, + { + "epoch": 0.96, + "learning_rate": 3.5819458375125376e-05, + "loss": 1.1847, + "theoretical_loss": 3.3364839437970275, + "tokens_seen": 3154640896 + }, + { + "epoch": 0.96, + "learning_rate": 3.5803410230692084e-05, + "loss": 1.2008, + "theoretical_loss": 3.3364787138055476, + "tokens_seen": 3154706432 + }, + { + "epoch": 0.96, + "learning_rate": 3.578736208625878e-05, + "loss": 1.1616, + "theoretical_loss": 3.336473483953135, + "tokens_seen": 3154771968 + }, + { + "epoch": 0.96, + "learning_rate": 3.577131394182548e-05, + "loss": 1.1555, + "theoretical_loss": 3.3364682542397834, + "tokens_seen": 3154837504 + }, + { + "epoch": 0.96, + "learning_rate": 3.575526579739218e-05, + "loss": 1.2054, + "theoretical_loss": 3.3364630246654867, + "tokens_seen": 3154903040 + }, + { + "epoch": 0.96, + "learning_rate": 3.573921765295888e-05, + "loss": 1.28, + "theoretical_loss": 3.336457795230238, + "tokens_seen": 3154968576 + }, + { + "epoch": 0.96, + "learning_rate": 3.572316950852558e-05, + "loss": 1.2347, + "theoretical_loss": 3.33645256593403, + "tokens_seen": 3155034112 + }, + { + "epoch": 0.96, + "learning_rate": 3.570712136409228e-05, + "loss": 1.1587, + "theoretical_loss": 3.3364473367768572, + "tokens_seen": 3155099648 + }, + { + "epoch": 0.96, + "learning_rate": 3.5691073219658975e-05, + "loss": 1.0827, + "theoretical_loss": 3.3364421077587125, + "tokens_seen": 3155165184 + }, + { + "epoch": 0.96, + "learning_rate": 3.5675025075225675e-05, + "loss": 1.1325, + "theoretical_loss": 3.3364368788795895, + "tokens_seen": 3155230720 + }, + { + "epoch": 0.96, + "learning_rate": 3.565897693079238e-05, + "loss": 1.1751, + "theoretical_loss": 3.336431650139481, + "tokens_seen": 3155296256 + }, + { + "epoch": 0.96, + "learning_rate": 3.564292878635908e-05, + "loss": 1.2126, + "theoretical_loss": 3.3364264215383814, + "tokens_seen": 3155361792 + }, + { + "epoch": 0.96, + "learning_rate": 3.5626880641925784e-05, + "loss": 1.2726, + "theoretical_loss": 3.3364211930762835, + "tokens_seen": 3155427328 + }, + { + "epoch": 0.96, + "learning_rate": 3.561083249749248e-05, + "loss": 1.2548, + "theoretical_loss": 3.33641596475318, + "tokens_seen": 3155492864 + }, + { + "epoch": 0.96, + "objective/train/docs_used": 1053331, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1865750551223755, + "objective/train/theoretical_loss": 3.336410736569066, + "objective/train/tokens_used": 3176018400, + "theoretical_loss": 3.336410736569066, + "tokens_seen": 3155558400 + }, + { + "epoch": 0.96, + "learning_rate": 3.559478435305918e-05, + "loss": 1.1412, + "theoretical_loss": 3.336410736569066, + "tokens_seen": 3155558400 + }, + { + "epoch": 0.96, + "learning_rate": 3.557873620862588e-05, + "loss": 1.1804, + "theoretical_loss": 3.336405508523934, + "tokens_seen": 3155623936 + }, + { + "epoch": 0.96, + "learning_rate": 3.556268806419258e-05, + "loss": 1.1661, + "theoretical_loss": 3.3364002806177777, + "tokens_seen": 3155689472 + }, + { + "epoch": 0.96, + "learning_rate": 3.554663991975928e-05, + "loss": 1.2739, + "theoretical_loss": 3.33639505285059, + "tokens_seen": 3155755008 + }, + { + "epoch": 0.96, + "learning_rate": 3.553059177532598e-05, + "loss": 1.1947, + "theoretical_loss": 3.3363898252223647, + "tokens_seen": 3155820544 + }, + { + "epoch": 0.96, + "learning_rate": 3.5514543630892675e-05, + "loss": 1.201, + "theoretical_loss": 3.3363845977330953, + "tokens_seen": 3155886080 + }, + { + "epoch": 0.96, + "learning_rate": 3.549849548645938e-05, + "loss": 1.2167, + "theoretical_loss": 3.3363793703827747, + "tokens_seen": 3155951616 + }, + { + "epoch": 0.96, + "learning_rate": 3.548244734202608e-05, + "loss": 1.1719, + "theoretical_loss": 3.336374143171397, + "tokens_seen": 3156017152 + }, + { + "epoch": 0.96, + "learning_rate": 3.5466399197592783e-05, + "loss": 1.2667, + "theoretical_loss": 3.336368916098955, + "tokens_seen": 3156082688 + }, + { + "epoch": 0.96, + "learning_rate": 3.545035105315948e-05, + "loss": 1.2553, + "theoretical_loss": 3.336363689165443, + "tokens_seen": 3156148224 + }, + { + "epoch": 0.96, + "learning_rate": 3.543430290872618e-05, + "loss": 1.2687, + "theoretical_loss": 3.3363584623708533, + "tokens_seen": 3156213760 + }, + { + "epoch": 0.96, + "learning_rate": 3.541825476429288e-05, + "loss": 1.2157, + "theoretical_loss": 3.33635323571518, + "tokens_seen": 3156279296 + }, + { + "epoch": 0.96, + "learning_rate": 3.540220661985958e-05, + "loss": 1.195, + "theoretical_loss": 3.336348009198417, + "tokens_seen": 3156344832 + }, + { + "epoch": 0.96, + "learning_rate": 3.538615847542628e-05, + "loss": 1.1703, + "theoretical_loss": 3.336342782820556, + "tokens_seen": 3156410368 + }, + { + "epoch": 0.96, + "learning_rate": 3.537011033099298e-05, + "loss": 1.1869, + "theoretical_loss": 3.3363375565815927, + "tokens_seen": 3156475904 + }, + { + "epoch": 0.96, + "learning_rate": 3.535406218655968e-05, + "loss": 1.2334, + "theoretical_loss": 3.3363323304815187, + "tokens_seen": 3156541440 + }, + { + "epoch": 0.96, + "learning_rate": 3.533801404212638e-05, + "loss": 1.2597, + "theoretical_loss": 3.336327104520328, + "tokens_seen": 3156606976 + }, + { + "epoch": 0.96, + "learning_rate": 3.532196589769308e-05, + "loss": 1.1964, + "theoretical_loss": 3.3363218786980147, + "tokens_seen": 3156672512 + }, + { + "epoch": 0.96, + "learning_rate": 3.530591775325978e-05, + "loss": 1.1694, + "theoretical_loss": 3.3363166530145714, + "tokens_seen": 3156738048 + }, + { + "epoch": 0.96, + "learning_rate": 3.5289869608826484e-05, + "loss": 1.1994, + "theoretical_loss": 3.3363114274699917, + "tokens_seen": 3156803584 + }, + { + "epoch": 0.96, + "learning_rate": 3.527382146439318e-05, + "loss": 1.1402, + "theoretical_loss": 3.336306202064269, + "tokens_seen": 3156869120 + }, + { + "epoch": 0.96, + "learning_rate": 3.525777331995988e-05, + "loss": 1.1964, + "theoretical_loss": 3.3363009767973972, + "tokens_seen": 3156934656 + }, + { + "epoch": 0.96, + "learning_rate": 3.524172517552658e-05, + "loss": 1.1654, + "theoretical_loss": 3.336295751669369, + "tokens_seen": 3157000192 + }, + { + "epoch": 0.96, + "learning_rate": 3.522567703109328e-05, + "loss": 1.2185, + "theoretical_loss": 3.3362905266801786, + "tokens_seen": 3157065728 + }, + { + "epoch": 0.96, + "learning_rate": 3.520962888665999e-05, + "loss": 1.1677, + "theoretical_loss": 3.336285301829818, + "tokens_seen": 3157131264 + }, + { + "epoch": 0.96, + "learning_rate": 3.519358074222668e-05, + "loss": 1.1605, + "theoretical_loss": 3.336280077118283, + "tokens_seen": 3157196800 + }, + { + "epoch": 0.96, + "learning_rate": 3.517753259779338e-05, + "loss": 1.1985, + "theoretical_loss": 3.336274852545565, + "tokens_seen": 3157262336 + }, + { + "epoch": 0.96, + "learning_rate": 3.516148445336008e-05, + "loss": 1.1736, + "theoretical_loss": 3.3362696281116584, + "tokens_seen": 3157327872 + }, + { + "epoch": 0.96, + "learning_rate": 3.514543630892678e-05, + "loss": 1.2465, + "theoretical_loss": 3.336264403816556, + "tokens_seen": 3157393408 + }, + { + "epoch": 0.96, + "learning_rate": 3.512938816449348e-05, + "loss": 1.2087, + "theoretical_loss": 3.3362591796602517, + "tokens_seen": 3157458944 + }, + { + "epoch": 0.96, + "learning_rate": 3.5113340020060184e-05, + "loss": 1.1313, + "theoretical_loss": 3.336253955642739, + "tokens_seen": 3157524480 + }, + { + "epoch": 0.96, + "learning_rate": 3.509729187562688e-05, + "loss": 1.1759, + "theoretical_loss": 3.336248731764011, + "tokens_seen": 3157590016 + }, + { + "epoch": 0.96, + "learning_rate": 3.508124373119358e-05, + "loss": 1.1137, + "theoretical_loss": 3.336243508024061, + "tokens_seen": 3157655552 + }, + { + "epoch": 0.96, + "learning_rate": 3.506519558676028e-05, + "loss": 1.2173, + "theoretical_loss": 3.3362382844228833, + "tokens_seen": 3157721088 + }, + { + "epoch": 0.96, + "learning_rate": 3.5049147442326986e-05, + "loss": 1.2049, + "theoretical_loss": 3.3362330609604705, + "tokens_seen": 3157786624 + }, + { + "epoch": 0.96, + "learning_rate": 3.503309929789369e-05, + "loss": 1.1807, + "theoretical_loss": 3.336227837636816, + "tokens_seen": 3157852160 + }, + { + "epoch": 0.96, + "learning_rate": 3.501705115346038e-05, + "loss": 1.2766, + "theoretical_loss": 3.3362226144519136, + "tokens_seen": 3157917696 + }, + { + "epoch": 0.96, + "learning_rate": 3.500100300902708e-05, + "loss": 1.1707, + "theoretical_loss": 3.3362173914057567, + "tokens_seen": 3157983232 + }, + { + "epoch": 0.96, + "learning_rate": 3.498495486459378e-05, + "loss": 1.2596, + "theoretical_loss": 3.3362121684983386, + "tokens_seen": 3158048768 + }, + { + "epoch": 0.96, + "learning_rate": 3.496890672016048e-05, + "loss": 1.1674, + "theoretical_loss": 3.336206945729653, + "tokens_seen": 3158114304 + }, + { + "epoch": 0.96, + "learning_rate": 3.4952858575727183e-05, + "loss": 1.1512, + "theoretical_loss": 3.336201723099693, + "tokens_seen": 3158179840 + }, + { + "epoch": 0.96, + "learning_rate": 3.4936810431293884e-05, + "loss": 1.1643, + "theoretical_loss": 3.3361965006084526, + "tokens_seen": 3158245376 + }, + { + "epoch": 0.96, + "learning_rate": 3.492076228686058e-05, + "loss": 1.1198, + "theoretical_loss": 3.3361912782559244, + "tokens_seen": 3158310912 + }, + { + "epoch": 0.96, + "learning_rate": 3.4904714142427285e-05, + "loss": 1.1647, + "theoretical_loss": 3.336186056042102, + "tokens_seen": 3158376448 + }, + { + "epoch": 0.96, + "learning_rate": 3.4888665997993986e-05, + "loss": 1.2201, + "theoretical_loss": 3.3361808339669796, + "tokens_seen": 3158441984 + }, + { + "epoch": 0.96, + "learning_rate": 3.4872617853560687e-05, + "loss": 1.224, + "theoretical_loss": 3.33617561203055, + "tokens_seen": 3158507520 + }, + { + "epoch": 0.96, + "learning_rate": 3.485656970912739e-05, + "loss": 1.1632, + "theoretical_loss": 3.336170390232807, + "tokens_seen": 3158573056 + }, + { + "epoch": 0.96, + "learning_rate": 3.484052156469408e-05, + "loss": 1.168, + "theoretical_loss": 3.3361651685737437, + "tokens_seen": 3158638592 + }, + { + "epoch": 0.96, + "learning_rate": 3.482447342026078e-05, + "loss": 1.1198, + "theoretical_loss": 3.3361599470533534, + "tokens_seen": 3158704128 + }, + { + "epoch": 0.96, + "learning_rate": 3.480842527582748e-05, + "loss": 1.1792, + "theoretical_loss": 3.33615472567163, + "tokens_seen": 3158769664 + }, + { + "epoch": 0.96, + "objective/train/docs_used": 1054650, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.138359546661377, + "objective/train/theoretical_loss": 3.3361495044285663, + "objective/train/tokens_used": 3179295200, + "theoretical_loss": 3.3361495044285663, + "tokens_seen": 3158835200 + }, + { + "epoch": 0.96, + "learning_rate": 3.479237713139418e-05, + "loss": 1.2238, + "theoretical_loss": 3.3361495044285663, + "tokens_seen": 3158835200 + }, + { + "epoch": 0.96, + "learning_rate": 3.4776328986960884e-05, + "loss": 1.195, + "theoretical_loss": 3.3361442833241566, + "tokens_seen": 3158900736 + }, + { + "epoch": 0.96, + "learning_rate": 3.4760280842527584e-05, + "loss": 1.1914, + "theoretical_loss": 3.3361390623583937, + "tokens_seen": 3158966272 + }, + { + "epoch": 0.96, + "learning_rate": 3.4744232698094285e-05, + "loss": 1.1895, + "theoretical_loss": 3.3361338415312716, + "tokens_seen": 3159031808 + }, + { + "epoch": 0.96, + "learning_rate": 3.4728184553660986e-05, + "loss": 1.2942, + "theoretical_loss": 3.3361286208427834, + "tokens_seen": 3159097344 + }, + { + "epoch": 0.96, + "learning_rate": 3.4712136409227686e-05, + "loss": 1.2508, + "theoretical_loss": 3.336123400292922, + "tokens_seen": 3159162880 + }, + { + "epoch": 0.96, + "learning_rate": 3.469608826479439e-05, + "loss": 1.193, + "theoretical_loss": 3.336118179881682, + "tokens_seen": 3159228416 + }, + { + "epoch": 0.96, + "learning_rate": 3.468004012036109e-05, + "loss": 1.151, + "theoretical_loss": 3.336112959609056, + "tokens_seen": 3159293952 + }, + { + "epoch": 0.96, + "learning_rate": 3.466399197592778e-05, + "loss": 1.1737, + "theoretical_loss": 3.336107739475038, + "tokens_seen": 3159359488 + }, + { + "epoch": 0.96, + "learning_rate": 3.464794383149448e-05, + "loss": 1.2264, + "theoretical_loss": 3.3361025194796206, + "tokens_seen": 3159425024 + }, + { + "epoch": 0.96, + "learning_rate": 3.463189568706118e-05, + "loss": 1.1285, + "theoretical_loss": 3.3360972996227978, + "tokens_seen": 3159490560 + }, + { + "epoch": 0.96, + "learning_rate": 3.461584754262788e-05, + "loss": 1.2106, + "theoretical_loss": 3.336092079904563, + "tokens_seen": 3159556096 + }, + { + "epoch": 0.96, + "learning_rate": 3.459979939819459e-05, + "loss": 1.2183, + "theoretical_loss": 3.33608686032491, + "tokens_seen": 3159621632 + }, + { + "epoch": 0.96, + "learning_rate": 3.4583751253761284e-05, + "loss": 1.1798, + "theoretical_loss": 3.3360816408838314, + "tokens_seen": 3159687168 + }, + { + "epoch": 0.96, + "learning_rate": 3.4567703109327985e-05, + "loss": 1.1802, + "theoretical_loss": 3.3360764215813217, + "tokens_seen": 3159752704 + }, + { + "epoch": 0.96, + "learning_rate": 3.4551654964894686e-05, + "loss": 1.2209, + "theoretical_loss": 3.3360712024173735, + "tokens_seen": 3159818240 + }, + { + "epoch": 0.96, + "learning_rate": 3.4535606820461386e-05, + "loss": 1.1201, + "theoretical_loss": 3.336065983391981, + "tokens_seen": 3159883776 + }, + { + "epoch": 0.96, + "learning_rate": 3.451955867602809e-05, + "loss": 1.2133, + "theoretical_loss": 3.3360607645051363, + "tokens_seen": 3159949312 + }, + { + "epoch": 0.96, + "learning_rate": 3.450351053159479e-05, + "loss": 1.1084, + "theoretical_loss": 3.3360555457568344, + "tokens_seen": 3160014848 + }, + { + "epoch": 0.96, + "learning_rate": 3.448746238716148e-05, + "loss": 1.168, + "theoretical_loss": 3.336050327147068, + "tokens_seen": 3160080384 + }, + { + "epoch": 0.96, + "learning_rate": 3.447141424272818e-05, + "loss": 1.2174, + "theoretical_loss": 3.3360451086758305, + "tokens_seen": 3160145920 + }, + { + "epoch": 0.96, + "learning_rate": 3.445536609829489e-05, + "loss": 1.1762, + "theoretical_loss": 3.3360398903431157, + "tokens_seen": 3160211456 + }, + { + "epoch": 0.96, + "learning_rate": 3.443931795386159e-05, + "loss": 1.2207, + "theoretical_loss": 3.3360346721489167, + "tokens_seen": 3160276992 + }, + { + "epoch": 0.96, + "learning_rate": 3.442326980942829e-05, + "loss": 1.2062, + "theoretical_loss": 3.336029454093227, + "tokens_seen": 3160342528 + }, + { + "epoch": 0.96, + "learning_rate": 3.4407221664994985e-05, + "loss": 1.1073, + "theoretical_loss": 3.33602423617604, + "tokens_seen": 3160408064 + }, + { + "epoch": 0.96, + "learning_rate": 3.4391173520561685e-05, + "loss": 1.2477, + "theoretical_loss": 3.3360190183973497, + "tokens_seen": 3160473600 + }, + { + "epoch": 0.96, + "learning_rate": 3.4375125376128386e-05, + "loss": 1.2317, + "theoretical_loss": 3.336013800757149, + "tokens_seen": 3160539136 + }, + { + "epoch": 0.96, + "learning_rate": 3.4359077231695087e-05, + "loss": 1.1827, + "theoretical_loss": 3.3360085832554316, + "tokens_seen": 3160604672 + }, + { + "epoch": 0.96, + "learning_rate": 3.434302908726179e-05, + "loss": 1.2048, + "theoretical_loss": 3.3360033658921906, + "tokens_seen": 3160670208 + }, + { + "epoch": 0.96, + "learning_rate": 3.432698094282849e-05, + "loss": 1.2816, + "theoretical_loss": 3.3359981486674197, + "tokens_seen": 3160735744 + }, + { + "epoch": 0.96, + "learning_rate": 3.431093279839518e-05, + "loss": 1.0959, + "theoretical_loss": 3.3359929315811128, + "tokens_seen": 3160801280 + }, + { + "epoch": 0.96, + "learning_rate": 3.429488465396189e-05, + "loss": 1.2017, + "theoretical_loss": 3.3359877146332626, + "tokens_seen": 3160866816 + }, + { + "epoch": 0.96, + "learning_rate": 3.427883650952859e-05, + "loss": 1.1978, + "theoretical_loss": 3.3359824978238626, + "tokens_seen": 3160932352 + }, + { + "epoch": 0.96, + "learning_rate": 3.426278836509529e-05, + "loss": 1.1811, + "theoretical_loss": 3.335977281152907, + "tokens_seen": 3160997888 + }, + { + "epoch": 0.96, + "learning_rate": 3.424674022066199e-05, + "loss": 1.2452, + "theoretical_loss": 3.335972064620389, + "tokens_seen": 3161063424 + }, + { + "epoch": 0.96, + "learning_rate": 3.4230692076228685e-05, + "loss": 1.2393, + "theoretical_loss": 3.335966848226301, + "tokens_seen": 3161128960 + }, + { + "epoch": 0.96, + "learning_rate": 3.4214643931795386e-05, + "loss": 1.1628, + "theoretical_loss": 3.3359616319706378, + "tokens_seen": 3161194496 + }, + { + "epoch": 0.96, + "learning_rate": 3.4198595787362086e-05, + "loss": 1.3137, + "theoretical_loss": 3.3359564158533925, + "tokens_seen": 3161260032 + }, + { + "epoch": 0.96, + "learning_rate": 3.418254764292879e-05, + "loss": 1.1098, + "theoretical_loss": 3.335951199874558, + "tokens_seen": 3161325568 + }, + { + "epoch": 0.96, + "learning_rate": 3.416649949849549e-05, + "loss": 1.1895, + "theoretical_loss": 3.3359459840341286, + "tokens_seen": 3161391104 + }, + { + "epoch": 0.96, + "learning_rate": 3.415045135406219e-05, + "loss": 1.2265, + "theoretical_loss": 3.335940768332097, + "tokens_seen": 3161456640 + }, + { + "epoch": 0.96, + "learning_rate": 3.413440320962889e-05, + "loss": 1.2756, + "theoretical_loss": 3.335935552768457, + "tokens_seen": 3161522176 + }, + { + "epoch": 0.96, + "learning_rate": 3.411835506519559e-05, + "loss": 1.1593, + "theoretical_loss": 3.335930337343202, + "tokens_seen": 3161587712 + }, + { + "epoch": 0.96, + "learning_rate": 3.410230692076229e-05, + "loss": 1.19, + "theoretical_loss": 3.3359251220563255, + "tokens_seen": 3161653248 + }, + { + "epoch": 0.96, + "learning_rate": 3.408625877632899e-05, + "loss": 1.1123, + "theoretical_loss": 3.3359199069078214, + "tokens_seen": 3161718784 + }, + { + "epoch": 0.96, + "learning_rate": 3.407021063189569e-05, + "loss": 1.2242, + "theoretical_loss": 3.335914691897682, + "tokens_seen": 3161784320 + }, + { + "epoch": 0.96, + "learning_rate": 3.4054162487462385e-05, + "loss": 1.2319, + "theoretical_loss": 3.3359094770259023, + "tokens_seen": 3161849856 + }, + { + "epoch": 0.96, + "learning_rate": 3.4038114343029086e-05, + "loss": 1.2305, + "theoretical_loss": 3.3359042622924746, + "tokens_seen": 3161915392 + }, + { + "epoch": 0.96, + "learning_rate": 3.4022066198595786e-05, + "loss": 1.0451, + "theoretical_loss": 3.3358990476973926, + "tokens_seen": 3161980928 + }, + { + "epoch": 0.96, + "learning_rate": 3.4006018054162494e-05, + "loss": 1.1509, + "theoretical_loss": 3.33589383324065, + "tokens_seen": 3162046464 + }, + { + "epoch": 0.96, + "objective/train/docs_used": 1055661, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.131192922592163, + "objective/train/theoretical_loss": 3.33588861892224, + "objective/train/tokens_used": 3182572000, + "theoretical_loss": 3.33588861892224, + "tokens_seen": 3162112000 + }, + { + "epoch": 0.96, + "learning_rate": 3.3989969909729194e-05, + "loss": 1.102, + "theoretical_loss": 3.33588861892224, + "tokens_seen": 3162112000 + }, + { + "epoch": 0.96, + "learning_rate": 3.397392176529589e-05, + "loss": 1.1783, + "theoretical_loss": 3.335883404742156, + "tokens_seen": 3162177536 + }, + { + "epoch": 0.96, + "learning_rate": 3.395787362086259e-05, + "loss": 1.1952, + "theoretical_loss": 3.335878190700392, + "tokens_seen": 3162243072 + }, + { + "epoch": 0.96, + "learning_rate": 3.394182547642929e-05, + "loss": 1.2828, + "theoretical_loss": 3.3358729767969413, + "tokens_seen": 3162308608 + }, + { + "epoch": 0.96, + "learning_rate": 3.392577733199599e-05, + "loss": 1.2374, + "theoretical_loss": 3.3358677630317968, + "tokens_seen": 3162374144 + }, + { + "epoch": 0.96, + "learning_rate": 3.390972918756269e-05, + "loss": 1.1033, + "theoretical_loss": 3.3358625494049523, + "tokens_seen": 3162439680 + }, + { + "epoch": 0.96, + "learning_rate": 3.389368104312939e-05, + "loss": 1.168, + "theoretical_loss": 3.3358573359164017, + "tokens_seen": 3162505216 + }, + { + "epoch": 0.96, + "learning_rate": 3.3877632898696085e-05, + "loss": 1.2163, + "theoretical_loss": 3.3358521225661377, + "tokens_seen": 3162570752 + }, + { + "epoch": 0.96, + "learning_rate": 3.3861584754262786e-05, + "loss": 1.2052, + "theoretical_loss": 3.3358469093541547, + "tokens_seen": 3162636288 + }, + { + "epoch": 0.96, + "learning_rate": 3.384553660982949e-05, + "loss": 1.2448, + "theoretical_loss": 3.335841696280445, + "tokens_seen": 3162701824 + }, + { + "epoch": 0.96, + "learning_rate": 3.3829488465396194e-05, + "loss": 1.1866, + "theoretical_loss": 3.335836483345003, + "tokens_seen": 3162767360 + }, + { + "epoch": 0.96, + "learning_rate": 3.3813440320962895e-05, + "loss": 1.2274, + "theoretical_loss": 3.335831270547822, + "tokens_seen": 3162832896 + }, + { + "epoch": 0.96, + "learning_rate": 3.379739217652959e-05, + "loss": 1.0887, + "theoretical_loss": 3.335826057888895, + "tokens_seen": 3162898432 + }, + { + "epoch": 0.96, + "learning_rate": 3.378134403209629e-05, + "loss": 1.1592, + "theoretical_loss": 3.335820845368216, + "tokens_seen": 3162963968 + }, + { + "epoch": 0.96, + "learning_rate": 3.376529588766299e-05, + "loss": 1.1982, + "theoretical_loss": 3.335815632985778, + "tokens_seen": 3163029504 + }, + { + "epoch": 0.96, + "learning_rate": 3.374924774322969e-05, + "loss": 1.1763, + "theoretical_loss": 3.335810420741575, + "tokens_seen": 3163095040 + }, + { + "epoch": 0.96, + "learning_rate": 3.373319959879639e-05, + "loss": 1.1519, + "theoretical_loss": 3.3358052086356, + "tokens_seen": 3163160576 + }, + { + "epoch": 0.96, + "learning_rate": 3.371715145436309e-05, + "loss": 1.1365, + "theoretical_loss": 3.3357999966678467, + "tokens_seen": 3163226112 + }, + { + "epoch": 0.96, + "learning_rate": 3.370110330992979e-05, + "loss": 1.223, + "theoretical_loss": 3.335794784838309, + "tokens_seen": 3163291648 + }, + { + "epoch": 0.96, + "learning_rate": 3.368505516549649e-05, + "loss": 1.2516, + "theoretical_loss": 3.3357895731469793, + "tokens_seen": 3163357184 + }, + { + "epoch": 0.96, + "learning_rate": 3.3669007021063194e-05, + "loss": 1.261, + "theoretical_loss": 3.335784361593852, + "tokens_seen": 3163422720 + }, + { + "epoch": 0.96, + "learning_rate": 3.3652958876629894e-05, + "loss": 1.2575, + "theoretical_loss": 3.3357791501789205, + "tokens_seen": 3163488256 + }, + { + "epoch": 0.96, + "learning_rate": 3.3636910732196595e-05, + "loss": 1.128, + "theoretical_loss": 3.3357739389021774, + "tokens_seen": 3163553792 + }, + { + "epoch": 0.96, + "learning_rate": 3.362086258776329e-05, + "loss": 1.0829, + "theoretical_loss": 3.335768727763617, + "tokens_seen": 3163619328 + }, + { + "epoch": 0.96, + "learning_rate": 3.360481444332999e-05, + "loss": 1.1825, + "theoretical_loss": 3.335763516763233, + "tokens_seen": 3163684864 + }, + { + "epoch": 0.96, + "learning_rate": 3.358876629889669e-05, + "loss": 1.1453, + "theoretical_loss": 3.335758305901018, + "tokens_seen": 3163750400 + }, + { + "epoch": 0.96, + "learning_rate": 3.357271815446339e-05, + "loss": 1.2461, + "theoretical_loss": 3.3357530951769663, + "tokens_seen": 3163815936 + }, + { + "epoch": 0.96, + "learning_rate": 3.355667001003009e-05, + "loss": 1.2324, + "theoretical_loss": 3.3357478845910706, + "tokens_seen": 3163881472 + }, + { + "epoch": 0.96, + "learning_rate": 3.354062186559679e-05, + "loss": 1.2358, + "theoretical_loss": 3.335742674143325, + "tokens_seen": 3163947008 + }, + { + "epoch": 0.96, + "learning_rate": 3.352457372116349e-05, + "loss": 1.1265, + "theoretical_loss": 3.3357374638337225, + "tokens_seen": 3164012544 + }, + { + "epoch": 0.96, + "learning_rate": 3.350852557673019e-05, + "loss": 1.1677, + "theoretical_loss": 3.335732253662257, + "tokens_seen": 3164078080 + }, + { + "epoch": 0.96, + "learning_rate": 3.3492477432296894e-05, + "loss": 1.1833, + "theoretical_loss": 3.335727043628922, + "tokens_seen": 3164143616 + }, + { + "epoch": 0.96, + "learning_rate": 3.3476429287863594e-05, + "loss": 1.2009, + "theoretical_loss": 3.335721833733711, + "tokens_seen": 3164209152 + }, + { + "epoch": 0.96, + "learning_rate": 3.346038114343029e-05, + "loss": 1.1148, + "theoretical_loss": 3.3357166239766167, + "tokens_seen": 3164274688 + }, + { + "epoch": 0.96, + "learning_rate": 3.344433299899699e-05, + "loss": 1.1355, + "theoretical_loss": 3.3357114143576334, + "tokens_seen": 3164340224 + }, + { + "epoch": 0.96, + "learning_rate": 3.342828485456369e-05, + "loss": 1.2013, + "theoretical_loss": 3.335706204876754, + "tokens_seen": 3164405760 + }, + { + "epoch": 0.96, + "learning_rate": 3.341223671013039e-05, + "loss": 1.3122, + "theoretical_loss": 3.3357009955339727, + "tokens_seen": 3164471296 + }, + { + "epoch": 0.96, + "learning_rate": 3.33961885656971e-05, + "loss": 1.2249, + "theoretical_loss": 3.335695786329282, + "tokens_seen": 3164536832 + }, + { + "epoch": 0.96, + "learning_rate": 3.338014042126379e-05, + "loss": 1.1542, + "theoretical_loss": 3.3356905772626764, + "tokens_seen": 3164602368 + }, + { + "epoch": 0.96, + "learning_rate": 3.336409227683049e-05, + "loss": 1.2343, + "theoretical_loss": 3.335685368334149, + "tokens_seen": 3164667904 + }, + { + "epoch": 0.96, + "learning_rate": 3.334804413239719e-05, + "loss": 1.2061, + "theoretical_loss": 3.335680159543693, + "tokens_seen": 3164733440 + }, + { + "epoch": 0.96, + "learning_rate": 3.333199598796389e-05, + "loss": 1.1448, + "theoretical_loss": 3.335674950891302, + "tokens_seen": 3164798976 + }, + { + "epoch": 0.96, + "learning_rate": 3.3315947843530594e-05, + "loss": 1.2371, + "theoretical_loss": 3.33566974237697, + "tokens_seen": 3164864512 + }, + { + "epoch": 0.96, + "learning_rate": 3.3299899699097295e-05, + "loss": 1.1881, + "theoretical_loss": 3.33566453400069, + "tokens_seen": 3164930048 + }, + { + "epoch": 0.96, + "learning_rate": 3.328385155466399e-05, + "loss": 1.2929, + "theoretical_loss": 3.335659325762455, + "tokens_seen": 3164995584 + }, + { + "epoch": 0.96, + "learning_rate": 3.326780341023069e-05, + "loss": 1.2706, + "theoretical_loss": 3.3356541176622594, + "tokens_seen": 3165061120 + }, + { + "epoch": 0.96, + "learning_rate": 3.3251755265797396e-05, + "loss": 1.2174, + "theoretical_loss": 3.3356489097000965, + "tokens_seen": 3165126656 + }, + { + "epoch": 0.96, + "learning_rate": 3.32357071213641e-05, + "loss": 1.2409, + "theoretical_loss": 3.3356437018759593, + "tokens_seen": 3165192192 + }, + { + "epoch": 0.96, + "learning_rate": 3.32196589769308e-05, + "loss": 1.2794, + "theoretical_loss": 3.335638494189842, + "tokens_seen": 3165257728 + }, + { + "epoch": 0.96, + "learning_rate": 3.320361083249749e-05, + "loss": 1.2367, + "theoretical_loss": 3.335633286641737, + "tokens_seen": 3165323264 + }, + { + "epoch": 0.96, + "objective/train/docs_used": 1056749, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2029361724853516, + "objective/train/theoretical_loss": 3.335628079231639, + "objective/train/tokens_used": 3185848800, + "theoretical_loss": 3.335628079231639, + "tokens_seen": 3165388800 + }, + { + "epoch": 0.96, + "learning_rate": 3.318756268806419e-05, + "loss": 1.2539, + "theoretical_loss": 3.335628079231639, + "tokens_seen": 3165388800 + }, + { + "epoch": 0.96, + "learning_rate": 3.317151454363089e-05, + "loss": 1.2402, + "theoretical_loss": 3.3356228719595404, + "tokens_seen": 3165454336 + }, + { + "epoch": 0.96, + "learning_rate": 3.3155466399197594e-05, + "loss": 1.1965, + "theoretical_loss": 3.3356176648254356, + "tokens_seen": 3165519872 + }, + { + "epoch": 0.96, + "learning_rate": 3.3139418254764294e-05, + "loss": 1.1928, + "theoretical_loss": 3.335612457829318, + "tokens_seen": 3165585408 + }, + { + "epoch": 0.96, + "learning_rate": 3.3123370110330995e-05, + "loss": 1.1708, + "theoretical_loss": 3.3356072509711803, + "tokens_seen": 3165650944 + }, + { + "epoch": 0.96, + "learning_rate": 3.3107321965897695e-05, + "loss": 1.1927, + "theoretical_loss": 3.3356020442510164, + "tokens_seen": 3165716480 + }, + { + "epoch": 0.96, + "learning_rate": 3.3091273821464396e-05, + "loss": 1.2013, + "theoretical_loss": 3.33559683766882, + "tokens_seen": 3165782016 + }, + { + "epoch": 0.96, + "learning_rate": 3.30752256770311e-05, + "loss": 1.2353, + "theoretical_loss": 3.3355916312245846, + "tokens_seen": 3165847552 + }, + { + "epoch": 0.96, + "learning_rate": 3.30591775325978e-05, + "loss": 1.2833, + "theoretical_loss": 3.3355864249183034, + "tokens_seen": 3165913088 + }, + { + "epoch": 0.96, + "learning_rate": 3.30431293881645e-05, + "loss": 1.1798, + "theoretical_loss": 3.33558121874997, + "tokens_seen": 3165978624 + }, + { + "epoch": 0.96, + "learning_rate": 3.302708124373119e-05, + "loss": 1.2503, + "theoretical_loss": 3.335576012719578, + "tokens_seen": 3166044160 + }, + { + "epoch": 0.96, + "learning_rate": 3.301103309929789e-05, + "loss": 1.2233, + "theoretical_loss": 3.335570806827121, + "tokens_seen": 3166109696 + }, + { + "epoch": 0.96, + "learning_rate": 3.299498495486459e-05, + "loss": 1.1721, + "theoretical_loss": 3.3355656010725916, + "tokens_seen": 3166175232 + }, + { + "epoch": 0.96, + "learning_rate": 3.2978936810431294e-05, + "loss": 1.1503, + "theoretical_loss": 3.3355603954559845, + "tokens_seen": 3166240768 + }, + { + "epoch": 0.96, + "learning_rate": 3.2962888665997994e-05, + "loss": 1.298, + "theoretical_loss": 3.335555189977293, + "tokens_seen": 3166306304 + }, + { + "epoch": 0.96, + "learning_rate": 3.2946840521564695e-05, + "loss": 1.1486, + "theoretical_loss": 3.3355499846365095, + "tokens_seen": 3166371840 + }, + { + "epoch": 0.96, + "learning_rate": 3.2930792377131396e-05, + "loss": 1.172, + "theoretical_loss": 3.335544779433629, + "tokens_seen": 3166437376 + }, + { + "epoch": 0.96, + "learning_rate": 3.2914744232698096e-05, + "loss": 1.202, + "theoretical_loss": 3.335539574368644, + "tokens_seen": 3166502912 + }, + { + "epoch": 0.96, + "learning_rate": 3.28986960882648e-05, + "loss": 1.2503, + "theoretical_loss": 3.335534369441548, + "tokens_seen": 3166568448 + }, + { + "epoch": 0.96, + "learning_rate": 3.28826479438315e-05, + "loss": 1.2947, + "theoretical_loss": 3.335529164652335, + "tokens_seen": 3166633984 + }, + { + "epoch": 0.96, + "learning_rate": 3.28665997993982e-05, + "loss": 1.1531, + "theoretical_loss": 3.335523960000998, + "tokens_seen": 3166699520 + }, + { + "epoch": 0.96, + "learning_rate": 3.285055165496489e-05, + "loss": 1.285, + "theoretical_loss": 3.335518755487531, + "tokens_seen": 3166765056 + }, + { + "epoch": 0.96, + "learning_rate": 3.283450351053159e-05, + "loss": 1.1918, + "theoretical_loss": 3.3355135511119274, + "tokens_seen": 3166830592 + }, + { + "epoch": 0.96, + "learning_rate": 3.281845536609829e-05, + "loss": 1.1678, + "theoretical_loss": 3.33550834687418, + "tokens_seen": 3166896128 + }, + { + "epoch": 0.96, + "learning_rate": 3.2802407221665e-05, + "loss": 1.2028, + "theoretical_loss": 3.335503142774283, + "tokens_seen": 3166961664 + }, + { + "epoch": 0.96, + "learning_rate": 3.27863590772317e-05, + "loss": 1.0758, + "theoretical_loss": 3.33549793881223, + "tokens_seen": 3167027200 + }, + { + "epoch": 0.96, + "learning_rate": 3.2770310932798395e-05, + "loss": 1.1597, + "theoretical_loss": 3.335492734988014, + "tokens_seen": 3167092736 + }, + { + "epoch": 0.96, + "learning_rate": 3.2754262788365096e-05, + "loss": 1.2375, + "theoretical_loss": 3.335487531301629, + "tokens_seen": 3167158272 + }, + { + "epoch": 0.96, + "learning_rate": 3.2738214643931796e-05, + "loss": 1.2511, + "theoretical_loss": 3.335482327753068, + "tokens_seen": 3167223808 + }, + { + "epoch": 0.96, + "learning_rate": 3.27221664994985e-05, + "loss": 1.2076, + "theoretical_loss": 3.3354771243423245, + "tokens_seen": 3167289344 + }, + { + "epoch": 0.96, + "learning_rate": 3.27061183550652e-05, + "loss": 1.1999, + "theoretical_loss": 3.3354719210693924, + "tokens_seen": 3167354880 + }, + { + "epoch": 0.96, + "learning_rate": 3.26900702106319e-05, + "loss": 1.2105, + "theoretical_loss": 3.3354667179342647, + "tokens_seen": 3167420416 + }, + { + "epoch": 0.96, + "learning_rate": 3.267402206619859e-05, + "loss": 1.2254, + "theoretical_loss": 3.3354615149369353, + "tokens_seen": 3167485952 + }, + { + "epoch": 0.96, + "learning_rate": 3.265797392176529e-05, + "loss": 1.1573, + "theoretical_loss": 3.335456312077398, + "tokens_seen": 3167551488 + }, + { + "epoch": 0.96, + "learning_rate": 3.2641925777332e-05, + "loss": 1.2428, + "theoretical_loss": 3.3354511093556454, + "tokens_seen": 3167617024 + }, + { + "epoch": 0.96, + "learning_rate": 3.26258776328987e-05, + "loss": 1.2381, + "theoretical_loss": 3.335445906771672, + "tokens_seen": 3167682560 + }, + { + "epoch": 0.96, + "learning_rate": 3.26098294884654e-05, + "loss": 1.2732, + "theoretical_loss": 3.33544070432547, + "tokens_seen": 3167748096 + }, + { + "epoch": 0.96, + "learning_rate": 3.2593781344032095e-05, + "loss": 1.1929, + "theoretical_loss": 3.3354355020170345, + "tokens_seen": 3167813632 + }, + { + "epoch": 0.96, + "learning_rate": 3.2577733199598796e-05, + "loss": 1.2257, + "theoretical_loss": 3.335430299846358, + "tokens_seen": 3167879168 + }, + { + "epoch": 0.96, + "learning_rate": 3.25616850551655e-05, + "loss": 1.1687, + "theoretical_loss": 3.335425097813434, + "tokens_seen": 3167944704 + }, + { + "epoch": 0.96, + "learning_rate": 3.25456369107322e-05, + "loss": 1.2876, + "theoretical_loss": 3.335419895918257, + "tokens_seen": 3168010240 + }, + { + "epoch": 0.96, + "learning_rate": 3.25295887662989e-05, + "loss": 1.1797, + "theoretical_loss": 3.3354146941608187, + "tokens_seen": 3168075776 + }, + { + "epoch": 0.96, + "learning_rate": 3.25135406218656e-05, + "loss": 1.1892, + "theoretical_loss": 3.335409492541114, + "tokens_seen": 3168141312 + }, + { + "epoch": 0.96, + "learning_rate": 3.24974924774323e-05, + "loss": 1.2204, + "theoretical_loss": 3.335404291059136, + "tokens_seen": 3168206848 + }, + { + "epoch": 0.96, + "learning_rate": 3.2481444332999e-05, + "loss": 1.1843, + "theoretical_loss": 3.335399089714878, + "tokens_seen": 3168272384 + }, + { + "epoch": 0.96, + "learning_rate": 3.24653961885657e-05, + "loss": 1.2279, + "theoretical_loss": 3.3353938885083343, + "tokens_seen": 3168337920 + }, + { + "epoch": 0.96, + "learning_rate": 3.24493480441324e-05, + "loss": 1.2846, + "theoretical_loss": 3.3353886874394973, + "tokens_seen": 3168403456 + }, + { + "epoch": 0.96, + "learning_rate": 3.24332998996991e-05, + "loss": 1.141, + "theoretical_loss": 3.335383486508361, + "tokens_seen": 3168468992 + }, + { + "epoch": 0.96, + "learning_rate": 3.2417251755265796e-05, + "loss": 1.1195, + "theoretical_loss": 3.3353782857149192, + "tokens_seen": 3168534528 + }, + { + "epoch": 0.96, + "learning_rate": 3.2401203610832496e-05, + "loss": 1.2669, + "theoretical_loss": 3.335373085059165, + "tokens_seen": 3168600064 + }, + { + "epoch": 0.96, + "objective/train/docs_used": 1057640, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2869290113449097, + "objective/train/theoretical_loss": 3.3353678845410926, + "objective/train/tokens_used": 3189125600, + "theoretical_loss": 3.3353678845410926, + "tokens_seen": 3168665600 + }, + { + "epoch": 0.96, + "learning_rate": 3.23851554663992e-05, + "loss": 1.2245, + "theoretical_loss": 3.3353678845410926, + "tokens_seen": 3168665600 + }, + { + "epoch": 0.96, + "learning_rate": 3.23691073219659e-05, + "loss": 1.241, + "theoretical_loss": 3.3353626841606943, + "tokens_seen": 3168731136 + }, + { + "epoch": 0.96, + "learning_rate": 3.2353059177532605e-05, + "loss": 1.1651, + "theoretical_loss": 3.3353574839179645, + "tokens_seen": 3168796672 + }, + { + "epoch": 0.96, + "learning_rate": 3.23370110330993e-05, + "loss": 1.2533, + "theoretical_loss": 3.3353522838128966, + "tokens_seen": 3168862208 + }, + { + "epoch": 0.96, + "learning_rate": 3.2320962888666e-05, + "loss": 1.2293, + "theoretical_loss": 3.335347083845484, + "tokens_seen": 3168927744 + }, + { + "epoch": 0.96, + "learning_rate": 3.23049147442327e-05, + "loss": 1.25, + "theoretical_loss": 3.33534188401572, + "tokens_seen": 3168993280 + }, + { + "epoch": 0.96, + "learning_rate": 3.22888665997994e-05, + "loss": 1.1914, + "theoretical_loss": 3.3353366843235985, + "tokens_seen": 3169058816 + }, + { + "epoch": 0.96, + "learning_rate": 3.22728184553661e-05, + "loss": 1.1716, + "theoretical_loss": 3.335331484769113, + "tokens_seen": 3169124352 + }, + { + "epoch": 0.96, + "learning_rate": 3.22567703109328e-05, + "loss": 1.2682, + "theoretical_loss": 3.3353262853522563, + "tokens_seen": 3169189888 + }, + { + "epoch": 0.96, + "learning_rate": 3.2240722166499496e-05, + "loss": 1.1894, + "theoretical_loss": 3.335321086073023, + "tokens_seen": 3169255424 + }, + { + "epoch": 0.96, + "learning_rate": 3.2224674022066196e-05, + "loss": 1.1343, + "theoretical_loss": 3.3353158869314052, + "tokens_seen": 3169320960 + }, + { + "epoch": 0.96, + "learning_rate": 3.22086258776329e-05, + "loss": 1.0836, + "theoretical_loss": 3.3353106879273984, + "tokens_seen": 3169386496 + }, + { + "epoch": 0.96, + "learning_rate": 3.2192577733199604e-05, + "loss": 1.17, + "theoretical_loss": 3.3353054890609943, + "tokens_seen": 3169452032 + }, + { + "epoch": 0.96, + "learning_rate": 3.2176529588766305e-05, + "loss": 1.2466, + "theoretical_loss": 3.335300290332187, + "tokens_seen": 3169517568 + }, + { + "epoch": 0.96, + "learning_rate": 3.2160481444333e-05, + "loss": 1.2105, + "theoretical_loss": 3.3352950917409703, + "tokens_seen": 3169583104 + }, + { + "epoch": 0.96, + "learning_rate": 3.21444332998997e-05, + "loss": 1.1754, + "theoretical_loss": 3.335289893287338, + "tokens_seen": 3169648640 + }, + { + "epoch": 0.96, + "learning_rate": 3.21283851554664e-05, + "loss": 1.1245, + "theoretical_loss": 3.3352846949712824, + "tokens_seen": 3169714176 + }, + { + "epoch": 0.96, + "learning_rate": 3.21123370110331e-05, + "loss": 1.1741, + "theoretical_loss": 3.335279496792798, + "tokens_seen": 3169779712 + }, + { + "epoch": 0.96, + "learning_rate": 3.20962888665998e-05, + "loss": 1.2956, + "theoretical_loss": 3.335274298751878, + "tokens_seen": 3169845248 + }, + { + "epoch": 0.96, + "learning_rate": 3.20802407221665e-05, + "loss": 1.2307, + "theoretical_loss": 3.335269100848516, + "tokens_seen": 3169910784 + }, + { + "epoch": 0.96, + "learning_rate": 3.2064192577733196e-05, + "loss": 1.127, + "theoretical_loss": 3.335263903082706, + "tokens_seen": 3169976320 + }, + { + "epoch": 0.96, + "learning_rate": 3.2048144433299903e-05, + "loss": 1.2857, + "theoretical_loss": 3.3352587054544403, + "tokens_seen": 3170041856 + }, + { + "epoch": 0.96, + "learning_rate": 3.2032096288866604e-05, + "loss": 1.1865, + "theoretical_loss": 3.3352535079637136, + "tokens_seen": 3170107392 + }, + { + "epoch": 0.96, + "learning_rate": 3.2016048144433305e-05, + "loss": 1.1768, + "theoretical_loss": 3.3352483106105186, + "tokens_seen": 3170172928 + }, + { + "epoch": 0.96, + "learning_rate": 3.2000000000000005e-05, + "loss": 1.2914, + "theoretical_loss": 3.3352431133948492, + "tokens_seen": 3170238464 + }, + { + "epoch": 0.96, + "learning_rate": 3.19839518555667e-05, + "loss": 1.1811, + "theoretical_loss": 3.335237916316699, + "tokens_seen": 3170304000 + }, + { + "epoch": 0.96, + "learning_rate": 3.19679037111334e-05, + "loss": 1.1649, + "theoretical_loss": 3.3352327193760614, + "tokens_seen": 3170369536 + }, + { + "epoch": 0.96, + "learning_rate": 3.19518555667001e-05, + "loss": 1.2667, + "theoretical_loss": 3.33522752257293, + "tokens_seen": 3170435072 + }, + { + "epoch": 0.96, + "learning_rate": 3.19358074222668e-05, + "loss": 1.1936, + "theoretical_loss": 3.335222325907298, + "tokens_seen": 3170500608 + }, + { + "epoch": 0.96, + "learning_rate": 3.19197592778335e-05, + "loss": 1.1206, + "theoretical_loss": 3.335217129379159, + "tokens_seen": 3170566144 + }, + { + "epoch": 0.96, + "learning_rate": 3.19037111334002e-05, + "loss": 1.2808, + "theoretical_loss": 3.3352119329885075, + "tokens_seen": 3170631680 + }, + { + "epoch": 0.96, + "learning_rate": 3.18876629889669e-05, + "loss": 1.3079, + "theoretical_loss": 3.3352067367353353, + "tokens_seen": 3170697216 + }, + { + "epoch": 0.96, + "learning_rate": 3.1871614844533604e-05, + "loss": 1.205, + "theoretical_loss": 3.3352015406196376, + "tokens_seen": 3170762752 + }, + { + "epoch": 0.96, + "learning_rate": 3.1855566700100304e-05, + "loss": 1.0987, + "theoretical_loss": 3.335196344641407, + "tokens_seen": 3170828288 + }, + { + "epoch": 0.96, + "learning_rate": 3.1839518555667005e-05, + "loss": 1.2354, + "theoretical_loss": 3.3351911488006367, + "tokens_seen": 3170893824 + }, + { + "epoch": 0.96, + "learning_rate": 3.1823470411233706e-05, + "loss": 1.2383, + "theoretical_loss": 3.3351859530973207, + "tokens_seen": 3170959360 + }, + { + "epoch": 0.96, + "learning_rate": 3.18074222668004e-05, + "loss": 1.2483, + "theoretical_loss": 3.3351807575314525, + "tokens_seen": 3171024896 + }, + { + "epoch": 0.96, + "learning_rate": 3.17913741223671e-05, + "loss": 1.1949, + "theoretical_loss": 3.335175562103026, + "tokens_seen": 3171090432 + }, + { + "epoch": 0.96, + "learning_rate": 3.17753259779338e-05, + "loss": 1.1481, + "theoretical_loss": 3.3351703668120343, + "tokens_seen": 3171155968 + }, + { + "epoch": 0.96, + "learning_rate": 3.17592778335005e-05, + "loss": 1.2337, + "theoretical_loss": 3.335165171658471, + "tokens_seen": 3171221504 + }, + { + "epoch": 0.96, + "learning_rate": 3.174322968906721e-05, + "loss": 1.2584, + "theoretical_loss": 3.335159976642329, + "tokens_seen": 3171287040 + }, + { + "epoch": 0.96, + "learning_rate": 3.17271815446339e-05, + "loss": 1.2559, + "theoretical_loss": 3.3351547817636034, + "tokens_seen": 3171352576 + }, + { + "epoch": 0.96, + "learning_rate": 3.17111334002006e-05, + "loss": 1.3172, + "theoretical_loss": 3.3351495870222863, + "tokens_seen": 3171418112 + }, + { + "epoch": 0.96, + "learning_rate": 3.1695085255767304e-05, + "loss": 1.1596, + "theoretical_loss": 3.3351443924183712, + "tokens_seen": 3171483648 + }, + { + "epoch": 0.96, + "learning_rate": 3.1679037111334004e-05, + "loss": 1.1666, + "theoretical_loss": 3.335139197951853, + "tokens_seen": 3171549184 + }, + { + "epoch": 0.96, + "learning_rate": 3.1662988966900705e-05, + "loss": 1.1866, + "theoretical_loss": 3.3351340036227235, + "tokens_seen": 3171614720 + }, + { + "epoch": 0.96, + "learning_rate": 3.1646940822467406e-05, + "loss": 1.1159, + "theoretical_loss": 3.335128809430978, + "tokens_seen": 3171680256 + }, + { + "epoch": 0.96, + "learning_rate": 3.16308926780341e-05, + "loss": 1.16, + "theoretical_loss": 3.335123615376608, + "tokens_seen": 3171745792 + }, + { + "epoch": 0.96, + "learning_rate": 3.16148445336008e-05, + "loss": 1.2688, + "theoretical_loss": 3.335118421459609, + "tokens_seen": 3171811328 + }, + { + "epoch": 0.96, + "learning_rate": 3.159879638916751e-05, + "loss": 1.161, + "theoretical_loss": 3.3351132276799733, + "tokens_seen": 3171876864 + }, + { + "epoch": 0.96, + "objective/train/docs_used": 1058840, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2894023656845093, + "objective/train/theoretical_loss": 3.335108034037695, + "objective/train/tokens_used": 3192402400, + "theoretical_loss": 3.335108034037695, + "tokens_seen": 3171942400 + }, + { + "epoch": 0.96, + "learning_rate": 3.158274824473421e-05, + "loss": 1.2353, + "theoretical_loss": 3.335108034037695, + "tokens_seen": 3171942400 + }, + { + "epoch": 0.96, + "learning_rate": 3.156670010030091e-05, + "loss": 1.0872, + "theoretical_loss": 3.335102840532767, + "tokens_seen": 3172007936 + }, + { + "epoch": 0.96, + "learning_rate": 3.15506519558676e-05, + "loss": 1.2286, + "theoretical_loss": 3.3350976471651834, + "tokens_seen": 3172073472 + }, + { + "epoch": 0.96, + "learning_rate": 3.1534603811434303e-05, + "loss": 1.0999, + "theoretical_loss": 3.3350924539349376, + "tokens_seen": 3172139008 + }, + { + "epoch": 0.96, + "learning_rate": 3.1518555667001004e-05, + "loss": 1.1825, + "theoretical_loss": 3.3350872608420232, + "tokens_seen": 3172204544 + }, + { + "epoch": 0.96, + "learning_rate": 3.1502507522567705e-05, + "loss": 1.1969, + "theoretical_loss": 3.3350820678864337, + "tokens_seen": 3172270080 + }, + { + "epoch": 0.96, + "learning_rate": 3.1486459378134405e-05, + "loss": 1.2095, + "theoretical_loss": 3.3350768750681623, + "tokens_seen": 3172335616 + }, + { + "epoch": 0.96, + "learning_rate": 3.1470411233701106e-05, + "loss": 1.1722, + "theoretical_loss": 3.335071682387203, + "tokens_seen": 3172401152 + }, + { + "epoch": 0.96, + "learning_rate": 3.1454363089267807e-05, + "loss": 1.113, + "theoretical_loss": 3.3350664898435487, + "tokens_seen": 3172466688 + }, + { + "epoch": 0.96, + "learning_rate": 3.143831494483451e-05, + "loss": 1.2512, + "theoretical_loss": 3.3350612974371936, + "tokens_seen": 3172532224 + }, + { + "epoch": 0.96, + "learning_rate": 3.142226680040121e-05, + "loss": 1.2179, + "theoretical_loss": 3.3350561051681313, + "tokens_seen": 3172597760 + }, + { + "epoch": 0.96, + "learning_rate": 3.140621865596791e-05, + "loss": 1.1775, + "theoretical_loss": 3.335050913036355, + "tokens_seen": 3172663296 + }, + { + "epoch": 0.96, + "learning_rate": 3.13901705115346e-05, + "loss": 1.2089, + "theoretical_loss": 3.335045721041858, + "tokens_seen": 3172728832 + }, + { + "epoch": 0.96, + "learning_rate": 3.13741223671013e-05, + "loss": 1.1516, + "theoretical_loss": 3.335040529184634, + "tokens_seen": 3172794368 + }, + { + "epoch": 0.96, + "learning_rate": 3.1358074222668004e-05, + "loss": 1.1623, + "theoretical_loss": 3.3350353374646766, + "tokens_seen": 3172859904 + }, + { + "epoch": 0.96, + "learning_rate": 3.1342026078234704e-05, + "loss": 1.2109, + "theoretical_loss": 3.33503014588198, + "tokens_seen": 3172925440 + }, + { + "epoch": 0.96, + "learning_rate": 3.1325977933801405e-05, + "loss": 1.1702, + "theoretical_loss": 3.3350249544365362, + "tokens_seen": 3172990976 + }, + { + "epoch": 0.96, + "learning_rate": 3.1309929789368106e-05, + "loss": 1.1828, + "theoretical_loss": 3.3350197631283405, + "tokens_seen": 3173056512 + }, + { + "epoch": 0.96, + "learning_rate": 3.1293881644934806e-05, + "loss": 1.235, + "theoretical_loss": 3.3350145719573847, + "tokens_seen": 3173122048 + }, + { + "epoch": 0.96, + "learning_rate": 3.127783350050151e-05, + "loss": 1.1494, + "theoretical_loss": 3.335009380923664, + "tokens_seen": 3173187584 + }, + { + "epoch": 0.96, + "learning_rate": 3.126178535606821e-05, + "loss": 1.1743, + "theoretical_loss": 3.335004190027171, + "tokens_seen": 3173253120 + }, + { + "epoch": 0.96, + "learning_rate": 3.124573721163491e-05, + "loss": 1.1678, + "theoretical_loss": 3.3349989992678992, + "tokens_seen": 3173318656 + }, + { + "epoch": 0.96, + "learning_rate": 3.122968906720161e-05, + "loss": 1.2594, + "theoretical_loss": 3.334993808645842, + "tokens_seen": 3173384192 + }, + { + "epoch": 0.96, + "learning_rate": 3.12136409227683e-05, + "loss": 1.0813, + "theoretical_loss": 3.3349886181609936, + "tokens_seen": 3173449728 + }, + { + "epoch": 0.96, + "learning_rate": 3.1197592778335e-05, + "loss": 1.2229, + "theoretical_loss": 3.3349834278133477, + "tokens_seen": 3173515264 + }, + { + "epoch": 0.96, + "learning_rate": 3.1181544633901704e-05, + "loss": 1.2112, + "theoretical_loss": 3.334978237602897, + "tokens_seen": 3173580800 + }, + { + "epoch": 0.96, + "learning_rate": 3.1165496489468404e-05, + "loss": 1.1946, + "theoretical_loss": 3.334973047529635, + "tokens_seen": 3173646336 + }, + { + "epoch": 0.96, + "learning_rate": 3.114944834503511e-05, + "loss": 1.1834, + "theoretical_loss": 3.334967857593556, + "tokens_seen": 3173711872 + }, + { + "epoch": 0.96, + "learning_rate": 3.1133400200601806e-05, + "loss": 1.1543, + "theoretical_loss": 3.334962667794653, + "tokens_seen": 3173777408 + }, + { + "epoch": 0.96, + "learning_rate": 3.1117352056168506e-05, + "loss": 1.2508, + "theoretical_loss": 3.3349574781329197, + "tokens_seen": 3173842944 + }, + { + "epoch": 0.96, + "learning_rate": 3.110130391173521e-05, + "loss": 1.1909, + "theoretical_loss": 3.3349522886083496, + "tokens_seen": 3173908480 + }, + { + "epoch": 0.96, + "learning_rate": 3.108525576730191e-05, + "loss": 1.1429, + "theoretical_loss": 3.3349470992209365, + "tokens_seen": 3173974016 + }, + { + "epoch": 0.96, + "learning_rate": 3.106920762286861e-05, + "loss": 1.2132, + "theoretical_loss": 3.3349419099706736, + "tokens_seen": 3174039552 + }, + { + "epoch": 0.96, + "learning_rate": 3.105315947843531e-05, + "loss": 1.2136, + "theoretical_loss": 3.334936720857555, + "tokens_seen": 3174105088 + }, + { + "epoch": 0.96, + "learning_rate": 3.1037111334002e-05, + "loss": 1.2289, + "theoretical_loss": 3.334931531881573, + "tokens_seen": 3174170624 + }, + { + "epoch": 0.96, + "learning_rate": 3.10210631895687e-05, + "loss": 1.1957, + "theoretical_loss": 3.3349263430427225, + "tokens_seen": 3174236160 + }, + { + "epoch": 0.96, + "learning_rate": 3.100501504513541e-05, + "loss": 1.1465, + "theoretical_loss": 3.3349211543409965, + "tokens_seen": 3174301696 + }, + { + "epoch": 0.96, + "learning_rate": 3.098896690070211e-05, + "loss": 1.1519, + "theoretical_loss": 3.3349159657763887, + "tokens_seen": 3174367232 + }, + { + "epoch": 0.96, + "learning_rate": 3.097291875626881e-05, + "loss": 1.1418, + "theoretical_loss": 3.334910777348892, + "tokens_seen": 3174432768 + }, + { + "epoch": 0.96, + "learning_rate": 3.0956870611835506e-05, + "loss": 1.2248, + "theoretical_loss": 3.334905589058501, + "tokens_seen": 3174498304 + }, + { + "epoch": 0.96, + "learning_rate": 3.0940822467402207e-05, + "loss": 1.1714, + "theoretical_loss": 3.3349004009052083, + "tokens_seen": 3174563840 + }, + { + "epoch": 0.96, + "learning_rate": 3.092477432296891e-05, + "loss": 1.1232, + "theoretical_loss": 3.334895212889008, + "tokens_seen": 3174629376 + }, + { + "epoch": 0.96, + "learning_rate": 3.090872617853561e-05, + "loss": 1.1651, + "theoretical_loss": 3.334890025009894, + "tokens_seen": 3174694912 + }, + { + "epoch": 0.96, + "learning_rate": 3.089267803410231e-05, + "loss": 1.1227, + "theoretical_loss": 3.334884837267859, + "tokens_seen": 3174760448 + }, + { + "epoch": 0.96, + "learning_rate": 3.087662988966901e-05, + "loss": 1.233, + "theoretical_loss": 3.3348796496628963, + "tokens_seen": 3174825984 + }, + { + "epoch": 0.96, + "learning_rate": 3.08605817452357e-05, + "loss": 1.1772, + "theoretical_loss": 3.3348744621950006, + "tokens_seen": 3174891520 + }, + { + "epoch": 0.96, + "learning_rate": 3.084453360080241e-05, + "loss": 1.219, + "theoretical_loss": 3.3348692748641646, + "tokens_seen": 3174957056 + }, + { + "epoch": 0.96, + "learning_rate": 3.082848545636911e-05, + "loss": 1.0847, + "theoretical_loss": 3.3348640876703826, + "tokens_seen": 3175022592 + }, + { + "epoch": 0.96, + "learning_rate": 3.081243731193581e-05, + "loss": 1.2564, + "theoretical_loss": 3.3348589006136473, + "tokens_seen": 3175088128 + }, + { + "epoch": 0.96, + "learning_rate": 3.079638916750251e-05, + "loss": 1.2234, + "theoretical_loss": 3.3348537136939527, + "tokens_seen": 3175153664 + }, + { + "epoch": 0.96, + "objective/train/docs_used": 1060049, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.101107120513916, + "objective/train/theoretical_loss": 3.3348485269112924, + "objective/train/tokens_used": 3195679200, + "theoretical_loss": 3.3348485269112924, + "tokens_seen": 3175219200 + }, + { + "epoch": 0.96, + "learning_rate": 3.0780341023069206e-05, + "loss": 1.1817, + "theoretical_loss": 3.3348485269112924, + "tokens_seen": 3175219200 + }, + { + "epoch": 0.96, + "learning_rate": 3.076429287863591e-05, + "loss": 1.2783, + "theoretical_loss": 3.33484334026566, + "tokens_seen": 3175284736 + }, + { + "epoch": 0.96, + "learning_rate": 3.074824473420261e-05, + "loss": 1.1974, + "theoretical_loss": 3.3348381537570484, + "tokens_seen": 3175350272 + }, + { + "epoch": 0.96, + "learning_rate": 3.073219658976931e-05, + "loss": 1.1282, + "theoretical_loss": 3.334832967385452, + "tokens_seen": 3175415808 + }, + { + "epoch": 0.96, + "learning_rate": 3.071614844533601e-05, + "loss": 1.2035, + "theoretical_loss": 3.334827781150864, + "tokens_seen": 3175481344 + }, + { + "epoch": 0.96, + "learning_rate": 3.070010030090271e-05, + "loss": 1.1305, + "theoretical_loss": 3.3348225950532777, + "tokens_seen": 3175546880 + }, + { + "epoch": 0.96, + "learning_rate": 3.068405215646941e-05, + "loss": 1.138, + "theoretical_loss": 3.3348174090926874, + "tokens_seen": 3175612416 + }, + { + "epoch": 0.96, + "learning_rate": 3.066800401203611e-05, + "loss": 1.1963, + "theoretical_loss": 3.3348122232690858, + "tokens_seen": 3175677952 + }, + { + "epoch": 0.96, + "learning_rate": 3.065195586760281e-05, + "loss": 1.242, + "theoretical_loss": 3.334807037582467, + "tokens_seen": 3175743488 + }, + { + "epoch": 0.96, + "learning_rate": 3.063590772316951e-05, + "loss": 1.1912, + "theoretical_loss": 3.3348018520328244, + "tokens_seen": 3175809024 + }, + { + "epoch": 0.96, + "learning_rate": 3.061985957873621e-05, + "loss": 1.1863, + "theoretical_loss": 3.334796666620151, + "tokens_seen": 3175874560 + }, + { + "epoch": 0.96, + "learning_rate": 3.0603811434302906e-05, + "loss": 1.2861, + "theoretical_loss": 3.3347914813444413, + "tokens_seen": 3175940096 + }, + { + "epoch": 0.96, + "learning_rate": 3.058776328986961e-05, + "loss": 1.206, + "theoretical_loss": 3.3347862962056887, + "tokens_seen": 3176005632 + }, + { + "epoch": 0.96, + "learning_rate": 3.057171514543631e-05, + "loss": 1.2117, + "theoretical_loss": 3.334781111203886, + "tokens_seen": 3176071168 + }, + { + "epoch": 0.96, + "learning_rate": 3.055566700100301e-05, + "loss": 1.2322, + "theoretical_loss": 3.3347759263390273, + "tokens_seen": 3176136704 + }, + { + "epoch": 0.96, + "learning_rate": 3.0539618856569716e-05, + "loss": 1.247, + "theoretical_loss": 3.3347707416111065, + "tokens_seen": 3176202240 + }, + { + "epoch": 0.96, + "learning_rate": 3.052357071213641e-05, + "loss": 1.1911, + "theoretical_loss": 3.3347655570201162, + "tokens_seen": 3176267776 + }, + { + "epoch": 0.96, + "learning_rate": 3.050752256770311e-05, + "loss": 1.2038, + "theoretical_loss": 3.334760372566051, + "tokens_seen": 3176333312 + }, + { + "epoch": 0.96, + "learning_rate": 3.049147442326981e-05, + "loss": 1.1548, + "theoretical_loss": 3.334755188248904, + "tokens_seen": 3176398848 + }, + { + "epoch": 0.96, + "learning_rate": 3.047542627883651e-05, + "loss": 1.2088, + "theoretical_loss": 3.334750004068668, + "tokens_seen": 3176464384 + }, + { + "epoch": 0.96, + "learning_rate": 3.045937813440321e-05, + "loss": 1.2016, + "theoretical_loss": 3.334744820025338, + "tokens_seen": 3176529920 + }, + { + "epoch": 0.96, + "learning_rate": 3.044332998996991e-05, + "loss": 1.1443, + "theoretical_loss": 3.334739636118907, + "tokens_seen": 3176595456 + }, + { + "epoch": 0.96, + "learning_rate": 3.042728184553661e-05, + "loss": 1.1214, + "theoretical_loss": 3.334734452349368, + "tokens_seen": 3176660992 + }, + { + "epoch": 0.96, + "learning_rate": 3.0411233701103307e-05, + "loss": 1.1809, + "theoretical_loss": 3.334729268716715, + "tokens_seen": 3176726528 + }, + { + "epoch": 0.96, + "learning_rate": 3.0395185556670015e-05, + "loss": 1.224, + "theoretical_loss": 3.334724085220942, + "tokens_seen": 3176792064 + }, + { + "epoch": 0.96, + "learning_rate": 3.0379137412236712e-05, + "loss": 1.1671, + "theoretical_loss": 3.3347189018620416, + "tokens_seen": 3176857600 + }, + { + "epoch": 0.96, + "learning_rate": 3.0363089267803412e-05, + "loss": 1.2043, + "theoretical_loss": 3.3347137186400078, + "tokens_seen": 3176923136 + }, + { + "epoch": 0.96, + "learning_rate": 3.0347041123370113e-05, + "loss": 1.1311, + "theoretical_loss": 3.3347085355548347, + "tokens_seen": 3176988672 + }, + { + "epoch": 0.96, + "learning_rate": 3.033099297893681e-05, + "loss": 1.2603, + "theoretical_loss": 3.334703352606515, + "tokens_seen": 3177054208 + }, + { + "epoch": 0.96, + "learning_rate": 3.031494483450351e-05, + "loss": 1.154, + "theoretical_loss": 3.334698169795043, + "tokens_seen": 3177119744 + }, + { + "epoch": 0.96, + "learning_rate": 3.029889669007021e-05, + "loss": 1.1854, + "theoretical_loss": 3.3346929871204116, + "tokens_seen": 3177185280 + }, + { + "epoch": 0.96, + "learning_rate": 3.028284854563691e-05, + "loss": 1.0796, + "theoretical_loss": 3.334687804582615, + "tokens_seen": 3177250816 + }, + { + "epoch": 0.96, + "learning_rate": 3.026680040120361e-05, + "loss": 1.1572, + "theoretical_loss": 3.334682622181646, + "tokens_seen": 3177316352 + }, + { + "epoch": 0.96, + "learning_rate": 3.0250752256770314e-05, + "loss": 1.2959, + "theoretical_loss": 3.334677439917499, + "tokens_seen": 3177381888 + }, + { + "epoch": 0.96, + "learning_rate": 3.0234704112337014e-05, + "loss": 1.123, + "theoretical_loss": 3.3346722577901673, + "tokens_seen": 3177447424 + }, + { + "epoch": 0.96, + "learning_rate": 3.0218655967903715e-05, + "loss": 1.2435, + "theoretical_loss": 3.334667075799644, + "tokens_seen": 3177512960 + }, + { + "epoch": 0.96, + "learning_rate": 3.0202607823470412e-05, + "loss": 1.1483, + "theoretical_loss": 3.334661893945923, + "tokens_seen": 3177578496 + }, + { + "epoch": 0.96, + "learning_rate": 3.0186559679037113e-05, + "loss": 1.1976, + "theoretical_loss": 3.3346567122289983, + "tokens_seen": 3177644032 + }, + { + "epoch": 0.96, + "learning_rate": 3.0170511534603813e-05, + "loss": 1.1474, + "theoretical_loss": 3.3346515306488627, + "tokens_seen": 3177709568 + }, + { + "epoch": 0.96, + "learning_rate": 3.015446339017051e-05, + "loss": 1.2014, + "theoretical_loss": 3.33464634920551, + "tokens_seen": 3177775104 + }, + { + "epoch": 0.96, + "learning_rate": 3.015446339017051e-05, + "loss": 1.1563, + "theoretical_loss": 3.3346411678989343, + "tokens_seen": 3177840640 + }, + { + "epoch": 0.96, + "learning_rate": 3.013841524573721e-05, + "loss": 1.1881, + "theoretical_loss": 3.3346359867291286, + "tokens_seen": 3177906176 + }, + { + "epoch": 0.96, + "learning_rate": 3.0122367101303912e-05, + "loss": 1.1634, + "theoretical_loss": 3.3346308056960865, + "tokens_seen": 3177971712 + }, + { + "epoch": 0.96, + "learning_rate": 3.010631895687061e-05, + "loss": 1.158, + "theoretical_loss": 3.334625624799802, + "tokens_seen": 3178037248 + }, + { + "epoch": 0.96, + "learning_rate": 3.0090270812437316e-05, + "loss": 1.122, + "theoretical_loss": 3.334620444040268, + "tokens_seen": 3178102784 + }, + { + "epoch": 0.96, + "learning_rate": 3.0074222668004014e-05, + "loss": 1.152, + "theoretical_loss": 3.3346152634174784, + "tokens_seen": 3178168320 + }, + { + "epoch": 0.96, + "learning_rate": 3.0058174523570714e-05, + "loss": 1.2041, + "theoretical_loss": 3.334610082931427, + "tokens_seen": 3178233856 + }, + { + "epoch": 0.96, + "learning_rate": 3.0042126379137415e-05, + "loss": 1.1823, + "theoretical_loss": 3.3346049025821074, + "tokens_seen": 3178299392 + }, + { + "epoch": 0.96, + "learning_rate": 3.0026078234704112e-05, + "loss": 1.2544, + "theoretical_loss": 3.3345997223695125, + "tokens_seen": 3178364928 + }, + { + "epoch": 0.96, + "learning_rate": 3.0010030090270813e-05, + "loss": 1.1097, + "theoretical_loss": 3.3345945422936367, + "tokens_seen": 3178430464 + }, + { + "epoch": 0.96, + "objective/train/docs_used": 1060911, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1653929948806763, + "objective/train/theoretical_loss": 3.334589362354473, + "objective/train/tokens_used": 3198956000, + "theoretical_loss": 3.334589362354473, + "tokens_seen": 3178496000 + }, + { + "epoch": 0.96, + "learning_rate": 2.9993981945837513e-05, + "loss": 1.1967, + "theoretical_loss": 3.334589362354473, + "tokens_seen": 3178496000 + }, + { + "epoch": 0.96, + "learning_rate": 2.997793380140421e-05, + "loss": 1.2292, + "theoretical_loss": 3.3345841825520153, + "tokens_seen": 3178561536 + }, + { + "epoch": 0.96, + "learning_rate": 2.996188565697091e-05, + "loss": 1.2186, + "theoretical_loss": 3.3345790028862567, + "tokens_seen": 3178627072 + }, + { + "epoch": 0.96, + "learning_rate": 2.9945837512537615e-05, + "loss": 1.2112, + "theoretical_loss": 3.3345738233571915, + "tokens_seen": 3178692608 + }, + { + "epoch": 0.96, + "learning_rate": 2.9929789368104316e-05, + "loss": 1.189, + "theoretical_loss": 3.334568643964813, + "tokens_seen": 3178758144 + }, + { + "epoch": 0.96, + "learning_rate": 2.9913741223671017e-05, + "loss": 1.1702, + "theoretical_loss": 3.334563464709114, + "tokens_seen": 3178823680 + }, + { + "epoch": 0.96, + "learning_rate": 2.9897693079237714e-05, + "loss": 1.1545, + "theoretical_loss": 3.3345582855900893, + "tokens_seen": 3178889216 + }, + { + "epoch": 0.96, + "learning_rate": 2.9881644934804415e-05, + "loss": 1.2597, + "theoretical_loss": 3.334553106607732, + "tokens_seen": 3178954752 + }, + { + "epoch": 0.96, + "learning_rate": 2.9865596790371115e-05, + "loss": 1.1176, + "theoretical_loss": 3.3345479277620353, + "tokens_seen": 3179020288 + }, + { + "epoch": 0.96, + "learning_rate": 2.9849548645937812e-05, + "loss": 1.1997, + "theoretical_loss": 3.334542749052993, + "tokens_seen": 3179085824 + }, + { + "epoch": 0.96, + "learning_rate": 2.9833500501504513e-05, + "loss": 1.1565, + "theoretical_loss": 3.334537570480599, + "tokens_seen": 3179151360 + }, + { + "epoch": 0.96, + "learning_rate": 2.9817452357071214e-05, + "loss": 1.2529, + "theoretical_loss": 3.3345323920448466, + "tokens_seen": 3179216896 + }, + { + "epoch": 0.96, + "learning_rate": 2.9801404212637918e-05, + "loss": 1.2452, + "theoretical_loss": 3.334527213745729, + "tokens_seen": 3179282432 + }, + { + "epoch": 0.96, + "learning_rate": 2.978535606820462e-05, + "loss": 1.1664, + "theoretical_loss": 3.3345220355832406, + "tokens_seen": 3179347968 + }, + { + "epoch": 0.96, + "learning_rate": 2.9769307923771316e-05, + "loss": 1.1888, + "theoretical_loss": 3.3345168575573743, + "tokens_seen": 3179413504 + }, + { + "epoch": 0.96, + "learning_rate": 2.9753259779338016e-05, + "loss": 1.1634, + "theoretical_loss": 3.3345116796681245, + "tokens_seen": 3179479040 + }, + { + "epoch": 0.96, + "learning_rate": 2.9737211634904717e-05, + "loss": 1.1975, + "theoretical_loss": 3.3345065019154836, + "tokens_seen": 3179544576 + }, + { + "epoch": 0.96, + "learning_rate": 2.9721163490471414e-05, + "loss": 1.2035, + "theoretical_loss": 3.334501324299446, + "tokens_seen": 3179610112 + }, + { + "epoch": 0.96, + "learning_rate": 2.9705115346038115e-05, + "loss": 1.1905, + "theoretical_loss": 3.334496146820005, + "tokens_seen": 3179675648 + }, + { + "epoch": 0.96, + "learning_rate": 2.9689067201604815e-05, + "loss": 1.1254, + "theoretical_loss": 3.3344909694771543, + "tokens_seen": 3179741184 + }, + { + "epoch": 0.96, + "learning_rate": 2.9673019057171513e-05, + "loss": 1.2151, + "theoretical_loss": 3.3344857922708875, + "tokens_seen": 3179806720 + }, + { + "epoch": 0.96, + "learning_rate": 2.9656970912738213e-05, + "loss": 1.2505, + "theoretical_loss": 3.334480615201198, + "tokens_seen": 3179872256 + }, + { + "epoch": 0.96, + "learning_rate": 2.9640922768304917e-05, + "loss": 1.3134, + "theoretical_loss": 3.33447543826808, + "tokens_seen": 3179937792 + }, + { + "epoch": 0.96, + "learning_rate": 2.9624874623871618e-05, + "loss": 1.156, + "theoretical_loss": 3.3344702614715263, + "tokens_seen": 3180003328 + }, + { + "epoch": 0.96, + "learning_rate": 2.960882647943832e-05, + "loss": 1.2467, + "theoretical_loss": 3.3344650848115305, + "tokens_seen": 3180068864 + }, + { + "epoch": 0.96, + "learning_rate": 2.9592778335005016e-05, + "loss": 1.258, + "theoretical_loss": 3.334459908288087, + "tokens_seen": 3180134400 + }, + { + "epoch": 0.96, + "learning_rate": 2.9576730190571716e-05, + "loss": 1.1351, + "theoretical_loss": 3.334454731901188, + "tokens_seen": 3180199936 + }, + { + "epoch": 0.96, + "learning_rate": 2.9560682046138417e-05, + "loss": 1.2528, + "theoretical_loss": 3.3344495556508287, + "tokens_seen": 3180265472 + }, + { + "epoch": 0.96, + "learning_rate": 2.9544633901705114e-05, + "loss": 1.1078, + "theoretical_loss": 3.3344443795370013, + "tokens_seen": 3180331008 + }, + { + "epoch": 0.96, + "learning_rate": 2.9528585757271815e-05, + "loss": 1.2, + "theoretical_loss": 3.3344392035597004, + "tokens_seen": 3180396544 + }, + { + "epoch": 0.96, + "learning_rate": 2.9512537612838516e-05, + "loss": 1.1371, + "theoretical_loss": 3.334434027718919, + "tokens_seen": 3180462080 + }, + { + "epoch": 0.96, + "learning_rate": 2.949648946840522e-05, + "loss": 1.2243, + "theoretical_loss": 3.3344288520146512, + "tokens_seen": 3180527616 + }, + { + "epoch": 0.96, + "learning_rate": 2.948044132397192e-05, + "loss": 1.1596, + "theoretical_loss": 3.3344236764468898, + "tokens_seen": 3180593152 + }, + { + "epoch": 0.96, + "learning_rate": 2.9464393179538617e-05, + "loss": 1.1453, + "theoretical_loss": 3.3344185010156293, + "tokens_seen": 3180658688 + }, + { + "epoch": 0.96, + "learning_rate": 2.9448345035105318e-05, + "loss": 1.1649, + "theoretical_loss": 3.3344133257208624, + "tokens_seen": 3180724224 + }, + { + "epoch": 0.96, + "learning_rate": 2.943229689067202e-05, + "loss": 1.1879, + "theoretical_loss": 3.3344081505625836, + "tokens_seen": 3180789760 + }, + { + "epoch": 0.96, + "learning_rate": 2.9416248746238716e-05, + "loss": 1.2212, + "theoretical_loss": 3.3344029755407854, + "tokens_seen": 3180855296 + }, + { + "epoch": 0.96, + "learning_rate": 2.9400200601805417e-05, + "loss": 1.169, + "theoretical_loss": 3.3343978006554624, + "tokens_seen": 3180920832 + }, + { + "epoch": 0.96, + "learning_rate": 2.9384152457372117e-05, + "loss": 1.2075, + "theoretical_loss": 3.3343926259066077, + "tokens_seen": 3180986368 + }, + { + "epoch": 0.96, + "learning_rate": 2.9368104312938815e-05, + "loss": 1.2612, + "theoretical_loss": 3.334387451294215, + "tokens_seen": 3181051904 + }, + { + "epoch": 0.96, + "learning_rate": 2.9352056168505522e-05, + "loss": 1.1981, + "theoretical_loss": 3.3343822768182774, + "tokens_seen": 3181117440 + }, + { + "epoch": 0.96, + "learning_rate": 2.933600802407222e-05, + "loss": 1.2252, + "theoretical_loss": 3.33437710247879, + "tokens_seen": 3181182976 + }, + { + "epoch": 0.96, + "learning_rate": 2.931995987963892e-05, + "loss": 1.1467, + "theoretical_loss": 3.3343719282757442, + "tokens_seen": 3181248512 + }, + { + "epoch": 0.96, + "learning_rate": 2.930391173520562e-05, + "loss": 1.1422, + "theoretical_loss": 3.3343667542091353, + "tokens_seen": 3181314048 + }, + { + "epoch": 0.96, + "learning_rate": 2.9287863590772318e-05, + "loss": 1.181, + "theoretical_loss": 3.3343615802789563, + "tokens_seen": 3181379584 + }, + { + "epoch": 0.96, + "learning_rate": 2.927181544633902e-05, + "loss": 1.2313, + "theoretical_loss": 3.3343564064852007, + "tokens_seen": 3181445120 + }, + { + "epoch": 0.96, + "learning_rate": 2.925576730190572e-05, + "loss": 1.2646, + "theoretical_loss": 3.3343512328278626, + "tokens_seen": 3181510656 + }, + { + "epoch": 0.96, + "learning_rate": 2.9239719157472416e-05, + "loss": 1.2001, + "theoretical_loss": 3.3343460593069345, + "tokens_seen": 3181576192 + }, + { + "epoch": 0.96, + "learning_rate": 2.9223671013039117e-05, + "loss": 1.1991, + "theoretical_loss": 3.334340885922411, + "tokens_seen": 3181641728 + }, + { + "epoch": 0.96, + "learning_rate": 2.9207622868605817e-05, + "loss": 1.1496, + "theoretical_loss": 3.334335712674286, + "tokens_seen": 3181707264 + }, + { + "epoch": 0.96, + "objective/train/docs_used": 1062124, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.174569010734558, + "objective/train/theoretical_loss": 3.3343305395625515, + "objective/train/tokens_used": 3202232800, + "theoretical_loss": 3.3343305395625515, + "tokens_seen": 3181772800 + }, + { + "epoch": 0.96, + "learning_rate": 2.919157472417252e-05, + "loss": 1.1781, + "theoretical_loss": 3.3343305395625515, + "tokens_seen": 3181772800 + }, + { + "epoch": 0.96, + "learning_rate": 2.9175526579739222e-05, + "loss": 1.2275, + "theoretical_loss": 3.334325366587203, + "tokens_seen": 3181838336 + }, + { + "epoch": 0.96, + "learning_rate": 2.915947843530592e-05, + "loss": 1.178, + "theoretical_loss": 3.3343201937482325, + "tokens_seen": 3181903872 + }, + { + "epoch": 0.96, + "learning_rate": 2.914343029087262e-05, + "loss": 1.2947, + "theoretical_loss": 3.3343150210456347, + "tokens_seen": 3181969408 + }, + { + "epoch": 0.96, + "learning_rate": 2.912738214643932e-05, + "loss": 1.1584, + "theoretical_loss": 3.3343098484794025, + "tokens_seen": 3182034944 + }, + { + "epoch": 0.96, + "learning_rate": 2.9111334002006018e-05, + "loss": 1.2737, + "theoretical_loss": 3.3343046760495296, + "tokens_seen": 3182100480 + }, + { + "epoch": 0.96, + "learning_rate": 2.909528585757272e-05, + "loss": 1.2195, + "theoretical_loss": 3.3342995037560104, + "tokens_seen": 3182166016 + }, + { + "epoch": 0.96, + "learning_rate": 2.907923771313942e-05, + "loss": 1.2133, + "theoretical_loss": 3.334294331598837, + "tokens_seen": 3182231552 + }, + { + "epoch": 0.96, + "learning_rate": 2.9063189568706116e-05, + "loss": 1.1846, + "theoretical_loss": 3.3342891595780046, + "tokens_seen": 3182297088 + }, + { + "epoch": 0.96, + "learning_rate": 2.9047141424272824e-05, + "loss": 1.2361, + "theoretical_loss": 3.3342839876935058, + "tokens_seen": 3182362624 + }, + { + "epoch": 0.96, + "learning_rate": 2.903109327983952e-05, + "loss": 1.1904, + "theoretical_loss": 3.3342788159453347, + "tokens_seen": 3182428160 + }, + { + "epoch": 0.96, + "learning_rate": 2.9015045135406222e-05, + "loss": 1.251, + "theoretical_loss": 3.334273644333484, + "tokens_seen": 3182493696 + }, + { + "epoch": 0.96, + "learning_rate": 2.8998996990972922e-05, + "loss": 1.2458, + "theoretical_loss": 3.3342684728579486, + "tokens_seen": 3182559232 + }, + { + "epoch": 0.96, + "learning_rate": 2.898294884653962e-05, + "loss": 1.1985, + "theoretical_loss": 3.334263301518721, + "tokens_seen": 3182624768 + }, + { + "epoch": 0.96, + "learning_rate": 2.896690070210632e-05, + "loss": 1.1942, + "theoretical_loss": 3.3342581303157957, + "tokens_seen": 3182690304 + }, + { + "epoch": 0.96, + "learning_rate": 2.895085255767302e-05, + "loss": 1.2404, + "theoretical_loss": 3.3342529592491656, + "tokens_seen": 3182755840 + }, + { + "epoch": 0.96, + "learning_rate": 2.8934804413239718e-05, + "loss": 1.2692, + "theoretical_loss": 3.3342477883188244, + "tokens_seen": 3182821376 + }, + { + "epoch": 0.96, + "learning_rate": 2.891875626880642e-05, + "loss": 1.2189, + "theoretical_loss": 3.334242617524766, + "tokens_seen": 3182886912 + }, + { + "epoch": 0.96, + "learning_rate": 2.8902708124373123e-05, + "loss": 1.2347, + "theoretical_loss": 3.334237446866984, + "tokens_seen": 3182952448 + }, + { + "epoch": 0.96, + "learning_rate": 2.8886659979939823e-05, + "loss": 1.2196, + "theoretical_loss": 3.334232276345472, + "tokens_seen": 3183017984 + }, + { + "epoch": 0.96, + "learning_rate": 2.8870611835506524e-05, + "loss": 1.2164, + "theoretical_loss": 3.334227105960223, + "tokens_seen": 3183083520 + }, + { + "epoch": 0.96, + "learning_rate": 2.885456369107322e-05, + "loss": 1.178, + "theoretical_loss": 3.3342219357112315, + "tokens_seen": 3183149056 + }, + { + "epoch": 0.96, + "learning_rate": 2.8838515546639922e-05, + "loss": 1.2214, + "theoretical_loss": 3.3342167655984905, + "tokens_seen": 3183214592 + }, + { + "epoch": 0.96, + "learning_rate": 2.8822467402206623e-05, + "loss": 1.2847, + "theoretical_loss": 3.3342115956219933, + "tokens_seen": 3183280128 + }, + { + "epoch": 0.96, + "learning_rate": 2.880641925777332e-05, + "loss": 1.1935, + "theoretical_loss": 3.3342064257817348, + "tokens_seen": 3183345664 + }, + { + "epoch": 0.96, + "learning_rate": 2.879037111334002e-05, + "loss": 1.2899, + "theoretical_loss": 3.3342012560777072, + "tokens_seen": 3183411200 + }, + { + "epoch": 0.96, + "learning_rate": 2.877432296890672e-05, + "loss": 1.168, + "theoretical_loss": 3.334196086509905, + "tokens_seen": 3183476736 + }, + { + "epoch": 0.96, + "learning_rate": 2.875827482447342e-05, + "loss": 1.1701, + "theoretical_loss": 3.3341909170783213, + "tokens_seen": 3183542272 + }, + { + "epoch": 0.96, + "learning_rate": 2.8742226680040126e-05, + "loss": 1.2393, + "theoretical_loss": 3.33418574778295, + "tokens_seen": 3183607808 + }, + { + "epoch": 0.96, + "learning_rate": 2.8726178535606823e-05, + "loss": 1.1376, + "theoretical_loss": 3.334180578623785, + "tokens_seen": 3183673344 + }, + { + "epoch": 0.96, + "learning_rate": 2.8710130391173524e-05, + "loss": 1.2463, + "theoretical_loss": 3.334175409600819, + "tokens_seen": 3183738880 + }, + { + "epoch": 0.96, + "learning_rate": 2.8694082246740224e-05, + "loss": 1.2714, + "theoretical_loss": 3.3341702407140463, + "tokens_seen": 3183804416 + }, + { + "epoch": 0.96, + "learning_rate": 2.867803410230692e-05, + "loss": 1.1851, + "theoretical_loss": 3.3341650719634606, + "tokens_seen": 3183869952 + }, + { + "epoch": 0.96, + "learning_rate": 2.8661985957873622e-05, + "loss": 1.1684, + "theoretical_loss": 3.3341599033490548, + "tokens_seen": 3183935488 + }, + { + "epoch": 0.96, + "learning_rate": 2.8645937813440323e-05, + "loss": 1.2255, + "theoretical_loss": 3.334154734870823, + "tokens_seen": 3184001024 + }, + { + "epoch": 0.96, + "learning_rate": 2.862988966900702e-05, + "loss": 1.1522, + "theoretical_loss": 3.334149566528759, + "tokens_seen": 3184066560 + }, + { + "epoch": 0.96, + "learning_rate": 2.861384152457372e-05, + "loss": 1.1238, + "theoretical_loss": 3.334144398322856, + "tokens_seen": 3184132096 + }, + { + "epoch": 0.96, + "learning_rate": 2.8597793380140425e-05, + "loss": 1.2233, + "theoretical_loss": 3.3341392302531077, + "tokens_seen": 3184197632 + }, + { + "epoch": 0.96, + "learning_rate": 2.8581745235707125e-05, + "loss": 1.1524, + "theoretical_loss": 3.334134062319508, + "tokens_seen": 3184263168 + }, + { + "epoch": 0.96, + "learning_rate": 2.8565697091273823e-05, + "loss": 1.1498, + "theoretical_loss": 3.3341288945220504, + "tokens_seen": 3184328704 + }, + { + "epoch": 0.96, + "learning_rate": 2.8549648946840523e-05, + "loss": 1.1432, + "theoretical_loss": 3.334123726860728, + "tokens_seen": 3184394240 + }, + { + "epoch": 0.96, + "learning_rate": 2.8533600802407224e-05, + "loss": 1.1871, + "theoretical_loss": 3.3341185593355354, + "tokens_seen": 3184459776 + }, + { + "epoch": 0.97, + "learning_rate": 2.851755265797392e-05, + "loss": 1.201, + "theoretical_loss": 3.3341133919464654, + "tokens_seen": 3184525312 + }, + { + "epoch": 0.97, + "learning_rate": 2.850150451354062e-05, + "loss": 1.2069, + "theoretical_loss": 3.334108224693512, + "tokens_seen": 3184590848 + }, + { + "epoch": 0.97, + "learning_rate": 2.8485456369107322e-05, + "loss": 1.2836, + "theoretical_loss": 3.334103057576668, + "tokens_seen": 3184656384 + }, + { + "epoch": 0.97, + "learning_rate": 2.846940822467402e-05, + "loss": 1.2174, + "theoretical_loss": 3.3340978905959284, + "tokens_seen": 3184721920 + }, + { + "epoch": 0.97, + "learning_rate": 2.8453360080240727e-05, + "loss": 1.1753, + "theoretical_loss": 3.3340927237512856, + "tokens_seen": 3184787456 + }, + { + "epoch": 0.97, + "learning_rate": 2.8437311935807424e-05, + "loss": 1.242, + "theoretical_loss": 3.334087557042734, + "tokens_seen": 3184852992 + }, + { + "epoch": 0.97, + "learning_rate": 2.8421263791374125e-05, + "loss": 1.2406, + "theoretical_loss": 3.334082390470267, + "tokens_seen": 3184918528 + }, + { + "epoch": 0.97, + "learning_rate": 2.8405215646940825e-05, + "loss": 1.2107, + "theoretical_loss": 3.3340772240338783, + "tokens_seen": 3184984064 + }, + { + "epoch": 0.97, + "objective/train/docs_used": 1063056, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.189962387084961, + "objective/train/theoretical_loss": 3.334072057733561, + "objective/train/tokens_used": 3205509600, + "theoretical_loss": 3.334072057733561, + "tokens_seen": 3185049600 + }, + { + "epoch": 0.97, + "learning_rate": 2.8389167502507523e-05, + "loss": 1.1786, + "theoretical_loss": 3.334072057733561, + "tokens_seen": 3185049600 + }, + { + "epoch": 0.97, + "learning_rate": 2.8373119358074223e-05, + "loss": 1.2599, + "theoretical_loss": 3.334066891569309, + "tokens_seen": 3185115136 + }, + { + "epoch": 0.97, + "learning_rate": 2.8357071213640924e-05, + "loss": 1.18, + "theoretical_loss": 3.3340617255411162, + "tokens_seen": 3185180672 + }, + { + "epoch": 0.97, + "learning_rate": 2.834102306920762e-05, + "loss": 1.1471, + "theoretical_loss": 3.3340565596489764, + "tokens_seen": 3185246208 + }, + { + "epoch": 0.97, + "learning_rate": 2.8324974924774322e-05, + "loss": 1.1877, + "theoretical_loss": 3.3340513938928824, + "tokens_seen": 3185311744 + }, + { + "epoch": 0.97, + "learning_rate": 2.8308926780341023e-05, + "loss": 1.2852, + "theoretical_loss": 3.334046228272828, + "tokens_seen": 3185377280 + }, + { + "epoch": 0.97, + "learning_rate": 2.8292878635907727e-05, + "loss": 1.2539, + "theoretical_loss": 3.3340410627888075, + "tokens_seen": 3185442816 + }, + { + "epoch": 0.97, + "learning_rate": 2.8276830491474427e-05, + "loss": 1.2445, + "theoretical_loss": 3.3340358974408137, + "tokens_seen": 3185508352 + }, + { + "epoch": 0.97, + "learning_rate": 2.8260782347041124e-05, + "loss": 1.2826, + "theoretical_loss": 3.334030732228841, + "tokens_seen": 3185573888 + }, + { + "epoch": 0.97, + "learning_rate": 2.8244734202607825e-05, + "loss": 1.1706, + "theoretical_loss": 3.3340255671528825, + "tokens_seen": 3185639424 + }, + { + "epoch": 0.97, + "learning_rate": 2.8228686058174526e-05, + "loss": 1.2026, + "theoretical_loss": 3.334020402212932, + "tokens_seen": 3185704960 + }, + { + "epoch": 0.97, + "learning_rate": 2.8212637913741223e-05, + "loss": 1.1894, + "theoretical_loss": 3.3340152374089826, + "tokens_seen": 3185770496 + }, + { + "epoch": 0.97, + "learning_rate": 2.8196589769307924e-05, + "loss": 1.1953, + "theoretical_loss": 3.334010072741029, + "tokens_seen": 3185836032 + }, + { + "epoch": 0.97, + "learning_rate": 2.8180541624874624e-05, + "loss": 1.1946, + "theoretical_loss": 3.3340049082090637, + "tokens_seen": 3185901568 + }, + { + "epoch": 0.97, + "learning_rate": 2.816449348044132e-05, + "loss": 1.1136, + "theoretical_loss": 3.3339997438130813, + "tokens_seen": 3185967104 + }, + { + "epoch": 0.97, + "learning_rate": 2.814844533600803e-05, + "loss": 1.2214, + "theoretical_loss": 3.3339945795530745, + "tokens_seen": 3186032640 + }, + { + "epoch": 0.97, + "learning_rate": 2.8132397191574726e-05, + "loss": 1.1721, + "theoretical_loss": 3.3339894154290377, + "tokens_seen": 3186098176 + }, + { + "epoch": 0.97, + "learning_rate": 2.8116349047141427e-05, + "loss": 1.2596, + "theoretical_loss": 3.333984251440964, + "tokens_seen": 3186163712 + }, + { + "epoch": 0.97, + "learning_rate": 2.8100300902708127e-05, + "loss": 1.1053, + "theoretical_loss": 3.3339790875888475, + "tokens_seen": 3186229248 + }, + { + "epoch": 0.97, + "learning_rate": 2.8084252758274825e-05, + "loss": 1.2736, + "theoretical_loss": 3.3339739238726813, + "tokens_seen": 3186294784 + }, + { + "epoch": 0.97, + "learning_rate": 2.8068204613841525e-05, + "loss": 1.2123, + "theoretical_loss": 3.3339687602924597, + "tokens_seen": 3186360320 + }, + { + "epoch": 0.97, + "learning_rate": 2.8052156469408226e-05, + "loss": 1.1208, + "theoretical_loss": 3.3339635968481756, + "tokens_seen": 3186425856 + }, + { + "epoch": 0.97, + "learning_rate": 2.8036108324974923e-05, + "loss": 1.1279, + "theoretical_loss": 3.3339584335398227, + "tokens_seen": 3186491392 + }, + { + "epoch": 0.97, + "learning_rate": 2.8020060180541624e-05, + "loss": 1.1864, + "theoretical_loss": 3.333953270367395, + "tokens_seen": 3186556928 + }, + { + "epoch": 0.97, + "learning_rate": 2.8004012036108324e-05, + "loss": 1.2456, + "theoretical_loss": 3.333948107330886, + "tokens_seen": 3186622464 + }, + { + "epoch": 0.97, + "learning_rate": 2.798796389167503e-05, + "loss": 1.1949, + "theoretical_loss": 3.3339429444302895, + "tokens_seen": 3186688000 + }, + { + "epoch": 0.97, + "learning_rate": 2.797191574724173e-05, + "loss": 1.1552, + "theoretical_loss": 3.3339377816655986, + "tokens_seen": 3186753536 + }, + { + "epoch": 0.97, + "learning_rate": 2.7955867602808426e-05, + "loss": 1.2998, + "theoretical_loss": 3.3339326190368075, + "tokens_seen": 3186819072 + }, + { + "epoch": 0.97, + "learning_rate": 2.7939819458375127e-05, + "loss": 1.1796, + "theoretical_loss": 3.3339274565439094, + "tokens_seen": 3186884608 + }, + { + "epoch": 0.97, + "learning_rate": 2.7923771313941828e-05, + "loss": 1.228, + "theoretical_loss": 3.3339222941868982, + "tokens_seen": 3186950144 + }, + { + "epoch": 0.97, + "learning_rate": 2.7907723169508525e-05, + "loss": 1.1379, + "theoretical_loss": 3.3339171319657672, + "tokens_seen": 3187015680 + }, + { + "epoch": 0.97, + "learning_rate": 2.7891675025075225e-05, + "loss": 1.2871, + "theoretical_loss": 3.3339119698805106, + "tokens_seen": 3187081216 + }, + { + "epoch": 0.97, + "learning_rate": 2.7875626880641926e-05, + "loss": 1.1446, + "theoretical_loss": 3.333906807931122, + "tokens_seen": 3187146752 + }, + { + "epoch": 0.97, + "learning_rate": 2.7859578736208623e-05, + "loss": 1.2223, + "theoretical_loss": 3.333901646117594, + "tokens_seen": 3187212288 + }, + { + "epoch": 0.97, + "learning_rate": 2.784353059177533e-05, + "loss": 1.2427, + "theoretical_loss": 3.3338964844399213, + "tokens_seen": 3187277824 + }, + { + "epoch": 0.97, + "learning_rate": 2.7827482447342028e-05, + "loss": 1.2669, + "theoretical_loss": 3.333891322898097, + "tokens_seen": 3187343360 + }, + { + "epoch": 0.97, + "learning_rate": 2.781143430290873e-05, + "loss": 1.2462, + "theoretical_loss": 3.3338861614921154, + "tokens_seen": 3187408896 + }, + { + "epoch": 0.97, + "learning_rate": 2.779538615847543e-05, + "loss": 1.1493, + "theoretical_loss": 3.333881000221969, + "tokens_seen": 3187474432 + }, + { + "epoch": 0.97, + "learning_rate": 2.7779338014042127e-05, + "loss": 1.1996, + "theoretical_loss": 3.3338758390876526, + "tokens_seen": 3187539968 + }, + { + "epoch": 0.97, + "learning_rate": 2.7763289869608827e-05, + "loss": 1.2245, + "theoretical_loss": 3.333870678089159, + "tokens_seen": 3187605504 + }, + { + "epoch": 0.97, + "learning_rate": 2.7747241725175528e-05, + "loss": 1.2184, + "theoretical_loss": 3.3338655172264824, + "tokens_seen": 3187671040 + }, + { + "epoch": 0.97, + "learning_rate": 2.7731193580742225e-05, + "loss": 1.2334, + "theoretical_loss": 3.3338603564996165, + "tokens_seen": 3187736576 + }, + { + "epoch": 0.97, + "learning_rate": 2.7715145436308926e-05, + "loss": 1.188, + "theoretical_loss": 3.333855195908554, + "tokens_seen": 3187802112 + }, + { + "epoch": 0.97, + "learning_rate": 2.769909729187563e-05, + "loss": 1.2163, + "theoretical_loss": 3.3338500354532896, + "tokens_seen": 3187867648 + }, + { + "epoch": 0.97, + "learning_rate": 2.768304914744233e-05, + "loss": 1.243, + "theoretical_loss": 3.333844875133816, + "tokens_seen": 3187933184 + }, + { + "epoch": 0.97, + "learning_rate": 2.766700100300903e-05, + "loss": 1.2194, + "theoretical_loss": 3.3338397149501278, + "tokens_seen": 3187998720 + }, + { + "epoch": 0.97, + "learning_rate": 2.7650952858575728e-05, + "loss": 1.1609, + "theoretical_loss": 3.333834554902218, + "tokens_seen": 3188064256 + }, + { + "epoch": 0.97, + "learning_rate": 2.763490471414243e-05, + "loss": 1.3035, + "theoretical_loss": 3.3338293949900804, + "tokens_seen": 3188129792 + }, + { + "epoch": 0.97, + "learning_rate": 2.761885656970913e-05, + "loss": 1.2363, + "theoretical_loss": 3.333824235213709, + "tokens_seen": 3188195328 + }, + { + "epoch": 0.97, + "learning_rate": 2.7602808425275827e-05, + "loss": 1.1616, + "theoretical_loss": 3.3338190755730963, + "tokens_seen": 3188260864 + }, + { + "epoch": 0.97, + "objective/train/docs_used": 1064137, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.34062659740448, + "objective/train/theoretical_loss": 3.3338139160682374, + "objective/train/tokens_used": 3208786400, + "theoretical_loss": 3.3338139160682374, + "tokens_seen": 3188326400 + }, + { + "epoch": 0.97, + "learning_rate": 2.7586760280842527e-05, + "loss": 1.2885, + "theoretical_loss": 3.3338139160682374, + "tokens_seen": 3188326400 + }, + { + "epoch": 0.97, + "learning_rate": 2.7570712136409228e-05, + "loss": 1.2341, + "theoretical_loss": 3.3338087566991246, + "tokens_seen": 3188391936 + }, + { + "epoch": 0.97, + "learning_rate": 2.7554663991975925e-05, + "loss": 1.1896, + "theoretical_loss": 3.3338035974657525, + "tokens_seen": 3188457472 + }, + { + "epoch": 0.97, + "learning_rate": 2.7538615847542633e-05, + "loss": 1.1506, + "theoretical_loss": 3.3337984383681145, + "tokens_seen": 3188523008 + }, + { + "epoch": 0.97, + "learning_rate": 2.752256770310933e-05, + "loss": 1.1274, + "theoretical_loss": 3.333793279406204, + "tokens_seen": 3188588544 + }, + { + "epoch": 0.97, + "learning_rate": 2.750651955867603e-05, + "loss": 1.2161, + "theoretical_loss": 3.333788120580015, + "tokens_seen": 3188654080 + }, + { + "epoch": 0.97, + "learning_rate": 2.749047141424273e-05, + "loss": 1.1324, + "theoretical_loss": 3.333782961889541, + "tokens_seen": 3188719616 + }, + { + "epoch": 0.97, + "learning_rate": 2.747442326980943e-05, + "loss": 1.1973, + "theoretical_loss": 3.3337778033347756, + "tokens_seen": 3188785152 + }, + { + "epoch": 0.97, + "learning_rate": 2.745837512537613e-05, + "loss": 1.1949, + "theoretical_loss": 3.333772644915712, + "tokens_seen": 3188850688 + }, + { + "epoch": 0.97, + "learning_rate": 2.744232698094283e-05, + "loss": 1.1491, + "theoretical_loss": 3.333767486632345, + "tokens_seen": 3188916224 + }, + { + "epoch": 0.97, + "learning_rate": 2.7426278836509527e-05, + "loss": 1.2427, + "theoretical_loss": 3.333762328484667, + "tokens_seen": 3188981760 + }, + { + "epoch": 0.97, + "learning_rate": 2.7410230692076228e-05, + "loss": 1.2663, + "theoretical_loss": 3.3337571704726723, + "tokens_seen": 3189047296 + }, + { + "epoch": 0.97, + "learning_rate": 2.739418254764293e-05, + "loss": 1.107, + "theoretical_loss": 3.3337520125963542, + "tokens_seen": 3189112832 + }, + { + "epoch": 0.97, + "learning_rate": 2.7378134403209632e-05, + "loss": 1.2079, + "theoretical_loss": 3.3337468548557068, + "tokens_seen": 3189178368 + }, + { + "epoch": 0.97, + "learning_rate": 2.7362086258776333e-05, + "loss": 1.144, + "theoretical_loss": 3.3337416972507232, + "tokens_seen": 3189243904 + }, + { + "epoch": 0.97, + "learning_rate": 2.734603811434303e-05, + "loss": 1.1188, + "theoretical_loss": 3.3337365397813974, + "tokens_seen": 3189309440 + }, + { + "epoch": 0.97, + "learning_rate": 2.732998996990973e-05, + "loss": 1.1971, + "theoretical_loss": 3.333731382447723, + "tokens_seen": 3189374976 + }, + { + "epoch": 0.97, + "learning_rate": 2.731394182547643e-05, + "loss": 1.162, + "theoretical_loss": 3.333726225249694, + "tokens_seen": 3189440512 + }, + { + "epoch": 0.97, + "learning_rate": 2.729789368104313e-05, + "loss": 1.294, + "theoretical_loss": 3.333721068187303, + "tokens_seen": 3189506048 + }, + { + "epoch": 0.97, + "learning_rate": 2.728184553660983e-05, + "loss": 1.1521, + "theoretical_loss": 3.3337159112605446, + "tokens_seen": 3189571584 + }, + { + "epoch": 0.97, + "learning_rate": 2.726579739217653e-05, + "loss": 1.3224, + "theoretical_loss": 3.3337107544694122, + "tokens_seen": 3189637120 + }, + { + "epoch": 0.97, + "learning_rate": 2.7249749247743234e-05, + "loss": 1.2317, + "theoretical_loss": 3.3337055978138994, + "tokens_seen": 3189702656 + }, + { + "epoch": 0.97, + "learning_rate": 2.7233701103309935e-05, + "loss": 1.2072, + "theoretical_loss": 3.333700441294, + "tokens_seen": 3189768192 + }, + { + "epoch": 0.97, + "learning_rate": 2.7217652958876632e-05, + "loss": 1.1499, + "theoretical_loss": 3.333695284909707, + "tokens_seen": 3189833728 + }, + { + "epoch": 0.97, + "learning_rate": 2.7201604814443332e-05, + "loss": 1.2337, + "theoretical_loss": 3.333690128661015, + "tokens_seen": 3189899264 + }, + { + "epoch": 0.97, + "learning_rate": 2.7185556670010033e-05, + "loss": 1.1699, + "theoretical_loss": 3.3336849725479167, + "tokens_seen": 3189964800 + }, + { + "epoch": 0.97, + "learning_rate": 2.716950852557673e-05, + "loss": 1.2341, + "theoretical_loss": 3.333679816570407, + "tokens_seen": 3190030336 + }, + { + "epoch": 0.97, + "learning_rate": 2.715346038114343e-05, + "loss": 1.2426, + "theoretical_loss": 3.3336746607284784, + "tokens_seen": 3190095872 + }, + { + "epoch": 0.97, + "learning_rate": 2.713741223671013e-05, + "loss": 1.33, + "theoretical_loss": 3.333669505022125, + "tokens_seen": 3190161408 + }, + { + "epoch": 0.97, + "learning_rate": 2.712136409227683e-05, + "loss": 1.2381, + "theoretical_loss": 3.33366434945134, + "tokens_seen": 3190226944 + }, + { + "epoch": 0.97, + "learning_rate": 2.710531594784353e-05, + "loss": 1.232, + "theoretical_loss": 3.3336591940161178, + "tokens_seen": 3190292480 + }, + { + "epoch": 0.97, + "learning_rate": 2.7089267803410233e-05, + "loss": 1.1828, + "theoretical_loss": 3.3336540387164515, + "tokens_seen": 3190358016 + }, + { + "epoch": 0.97, + "learning_rate": 2.7073219658976934e-05, + "loss": 1.1632, + "theoretical_loss": 3.333648883552335, + "tokens_seen": 3190423552 + }, + { + "epoch": 0.97, + "learning_rate": 2.7057171514543635e-05, + "loss": 1.179, + "theoretical_loss": 3.333643728523762, + "tokens_seen": 3190489088 + }, + { + "epoch": 0.97, + "learning_rate": 2.7041123370110332e-05, + "loss": 1.1978, + "theoretical_loss": 3.333638573630726, + "tokens_seen": 3190554624 + }, + { + "epoch": 0.97, + "learning_rate": 2.7025075225677033e-05, + "loss": 1.1848, + "theoretical_loss": 3.333633418873221, + "tokens_seen": 3190620160 + }, + { + "epoch": 0.97, + "learning_rate": 2.7009027081243733e-05, + "loss": 1.1729, + "theoretical_loss": 3.3336282642512396, + "tokens_seen": 3190685696 + }, + { + "epoch": 0.97, + "learning_rate": 2.699297893681043e-05, + "loss": 1.1855, + "theoretical_loss": 3.3336231097647766, + "tokens_seen": 3190751232 + }, + { + "epoch": 0.97, + "learning_rate": 2.697693079237713e-05, + "loss": 1.2333, + "theoretical_loss": 3.3336179554138257, + "tokens_seen": 3190816768 + }, + { + "epoch": 0.97, + "learning_rate": 2.6960882647943832e-05, + "loss": 1.2172, + "theoretical_loss": 3.333612801198379, + "tokens_seen": 3190882304 + }, + { + "epoch": 0.97, + "learning_rate": 2.6944834503510536e-05, + "loss": 1.0981, + "theoretical_loss": 3.3336076471184324, + "tokens_seen": 3190947840 + }, + { + "epoch": 0.97, + "learning_rate": 2.6928786359077236e-05, + "loss": 1.1318, + "theoretical_loss": 3.333602493173978, + "tokens_seen": 3191013376 + }, + { + "epoch": 0.97, + "learning_rate": 2.6912738214643934e-05, + "loss": 1.2353, + "theoretical_loss": 3.33359733936501, + "tokens_seen": 3191078912 + }, + { + "epoch": 0.97, + "learning_rate": 2.6896690070210634e-05, + "loss": 1.1742, + "theoretical_loss": 3.3335921856915216, + "tokens_seen": 3191144448 + }, + { + "epoch": 0.97, + "learning_rate": 2.6880641925777335e-05, + "loss": 1.2462, + "theoretical_loss": 3.333587032153507, + "tokens_seen": 3191209984 + }, + { + "epoch": 0.97, + "learning_rate": 2.6864593781344032e-05, + "loss": 1.1674, + "theoretical_loss": 3.3335818787509597, + "tokens_seen": 3191275520 + }, + { + "epoch": 0.97, + "learning_rate": 2.6848545636910733e-05, + "loss": 1.1685, + "theoretical_loss": 3.333576725483873, + "tokens_seen": 3191341056 + }, + { + "epoch": 0.97, + "learning_rate": 2.6832497492477433e-05, + "loss": 1.3042, + "theoretical_loss": 3.333571572352241, + "tokens_seen": 3191406592 + }, + { + "epoch": 0.97, + "learning_rate": 2.681644934804413e-05, + "loss": 1.1566, + "theoretical_loss": 3.3335664193560572, + "tokens_seen": 3191472128 + }, + { + "epoch": 0.97, + "learning_rate": 2.6800401203610838e-05, + "loss": 1.1878, + "theoretical_loss": 3.3335612664953156, + "tokens_seen": 3191537664 + }, + { + "epoch": 0.97, + "objective/train/docs_used": 1065047, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2758793830871582, + "objective/train/theoretical_loss": 3.333556113770009, + "objective/train/tokens_used": 3212063200, + "theoretical_loss": 3.333556113770009, + "tokens_seen": 3191603200 + }, + { + "epoch": 0.97, + "learning_rate": 2.6784353059177535e-05, + "loss": 1.2738, + "theoretical_loss": 3.333556113770009, + "tokens_seen": 3191603200 + }, + { + "epoch": 0.97, + "learning_rate": 2.6768304914744236e-05, + "loss": 1.2408, + "theoretical_loss": 3.333550961180132, + "tokens_seen": 3191668736 + }, + { + "epoch": 0.97, + "learning_rate": 2.6752256770310937e-05, + "loss": 1.142, + "theoretical_loss": 3.333545808725678, + "tokens_seen": 3191734272 + }, + { + "epoch": 0.97, + "learning_rate": 2.6736208625877634e-05, + "loss": 1.1965, + "theoretical_loss": 3.33354065640664, + "tokens_seen": 3191799808 + }, + { + "epoch": 0.97, + "learning_rate": 2.6720160481444335e-05, + "loss": 1.1315, + "theoretical_loss": 3.3335355042230126, + "tokens_seen": 3191865344 + }, + { + "epoch": 0.97, + "learning_rate": 2.6704112337011035e-05, + "loss": 1.2216, + "theoretical_loss": 3.3335303521747885, + "tokens_seen": 3191930880 + }, + { + "epoch": 0.97, + "learning_rate": 2.6688064192577732e-05, + "loss": 1.1942, + "theoretical_loss": 3.3335252002619624, + "tokens_seen": 3191996416 + }, + { + "epoch": 0.97, + "learning_rate": 2.6672016048144433e-05, + "loss": 1.2404, + "theoretical_loss": 3.3335200484845275, + "tokens_seen": 3192061952 + }, + { + "epoch": 0.97, + "learning_rate": 2.6655967903711134e-05, + "loss": 1.1495, + "theoretical_loss": 3.3335148968424773, + "tokens_seen": 3192127488 + }, + { + "epoch": 0.97, + "learning_rate": 2.6639919759277838e-05, + "loss": 1.2747, + "theoretical_loss": 3.3335097453358054, + "tokens_seen": 3192193024 + }, + { + "epoch": 0.97, + "learning_rate": 2.6623871614844535e-05, + "loss": 1.2595, + "theoretical_loss": 3.3335045939645056, + "tokens_seen": 3192258560 + }, + { + "epoch": 0.97, + "learning_rate": 2.6607823470411236e-05, + "loss": 1.1931, + "theoretical_loss": 3.333499442728572, + "tokens_seen": 3192324096 + }, + { + "epoch": 0.97, + "learning_rate": 2.6591775325977936e-05, + "loss": 1.2571, + "theoretical_loss": 3.3334942916279977, + "tokens_seen": 3192389632 + }, + { + "epoch": 0.97, + "learning_rate": 2.6575727181544633e-05, + "loss": 1.1244, + "theoretical_loss": 3.3334891406627762, + "tokens_seen": 3192455168 + }, + { + "epoch": 0.97, + "learning_rate": 2.6559679037111334e-05, + "loss": 1.2744, + "theoretical_loss": 3.333483989832902, + "tokens_seen": 3192520704 + }, + { + "epoch": 0.97, + "learning_rate": 2.6543630892678035e-05, + "loss": 1.151, + "theoretical_loss": 3.333478839138368, + "tokens_seen": 3192586240 + }, + { + "epoch": 0.97, + "learning_rate": 2.6527582748244732e-05, + "loss": 1.2725, + "theoretical_loss": 3.3334736885791685, + "tokens_seen": 3192651776 + }, + { + "epoch": 0.97, + "learning_rate": 2.6511534603811433e-05, + "loss": 1.1537, + "theoretical_loss": 3.333468538155296, + "tokens_seen": 3192717312 + }, + { + "epoch": 0.97, + "learning_rate": 2.6495486459378137e-05, + "loss": 1.1941, + "theoretical_loss": 3.333463387866746, + "tokens_seen": 3192782848 + }, + { + "epoch": 0.97, + "learning_rate": 2.6479438314944837e-05, + "loss": 1.2497, + "theoretical_loss": 3.3334582377135105, + "tokens_seen": 3192848384 + }, + { + "epoch": 0.97, + "learning_rate": 2.6463390170511538e-05, + "loss": 1.2415, + "theoretical_loss": 3.3334530876955837, + "tokens_seen": 3192913920 + }, + { + "epoch": 0.97, + "learning_rate": 2.6447342026078235e-05, + "loss": 1.1771, + "theoretical_loss": 3.3334479378129593, + "tokens_seen": 3192979456 + }, + { + "epoch": 0.97, + "learning_rate": 2.6431293881644936e-05, + "loss": 1.2277, + "theoretical_loss": 3.3334427880656317, + "tokens_seen": 3193044992 + }, + { + "epoch": 0.97, + "learning_rate": 2.6415245737211636e-05, + "loss": 1.1849, + "theoretical_loss": 3.3334376384535935, + "tokens_seen": 3193110528 + }, + { + "epoch": 0.97, + "learning_rate": 2.6399197592778334e-05, + "loss": 1.1864, + "theoretical_loss": 3.3334324889768387, + "tokens_seen": 3193176064 + }, + { + "epoch": 0.97, + "learning_rate": 2.6383149448345034e-05, + "loss": 1.1934, + "theoretical_loss": 3.333427339635361, + "tokens_seen": 3193241600 + }, + { + "epoch": 0.97, + "learning_rate": 2.6367101303911735e-05, + "loss": 1.0611, + "theoretical_loss": 3.3334221904291543, + "tokens_seen": 3193307136 + }, + { + "epoch": 0.97, + "learning_rate": 2.635105315947844e-05, + "loss": 1.1202, + "theoretical_loss": 3.3334170413582123, + "tokens_seen": 3193372672 + }, + { + "epoch": 0.97, + "learning_rate": 2.633500501504514e-05, + "loss": 1.1589, + "theoretical_loss": 3.333411892422528, + "tokens_seen": 3193438208 + }, + { + "epoch": 0.97, + "learning_rate": 2.6318956870611837e-05, + "loss": 1.1332, + "theoretical_loss": 3.3334067436220955, + "tokens_seen": 3193503744 + }, + { + "epoch": 0.97, + "learning_rate": 2.6302908726178537e-05, + "loss": 1.1667, + "theoretical_loss": 3.333401594956909, + "tokens_seen": 3193569280 + }, + { + "epoch": 0.97, + "learning_rate": 2.6286860581745238e-05, + "loss": 1.2345, + "theoretical_loss": 3.3333964464269616, + "tokens_seen": 3193634816 + }, + { + "epoch": 0.97, + "learning_rate": 2.6270812437311935e-05, + "loss": 1.1712, + "theoretical_loss": 3.3333912980322467, + "tokens_seen": 3193700352 + }, + { + "epoch": 0.97, + "learning_rate": 2.6254764292878636e-05, + "loss": 1.1823, + "theoretical_loss": 3.3333861497727586, + "tokens_seen": 3193765888 + }, + { + "epoch": 0.97, + "learning_rate": 2.6238716148445337e-05, + "loss": 1.2169, + "theoretical_loss": 3.3333810016484904, + "tokens_seen": 3193831424 + }, + { + "epoch": 0.97, + "learning_rate": 2.6222668004012034e-05, + "loss": 1.1169, + "theoretical_loss": 3.3333758536594362, + "tokens_seen": 3193896960 + }, + { + "epoch": 0.97, + "learning_rate": 2.6206619859578735e-05, + "loss": 1.2228, + "theoretical_loss": 3.3333707058055895, + "tokens_seen": 3193962496 + }, + { + "epoch": 0.97, + "learning_rate": 2.619057171514544e-05, + "loss": 1.1804, + "theoretical_loss": 3.3333655580869443, + "tokens_seen": 3194028032 + }, + { + "epoch": 0.97, + "learning_rate": 2.617452357071214e-05, + "loss": 1.2379, + "theoretical_loss": 3.3333604105034937, + "tokens_seen": 3194093568 + }, + { + "epoch": 0.97, + "learning_rate": 2.615847542627884e-05, + "loss": 1.2461, + "theoretical_loss": 3.333355263055232, + "tokens_seen": 3194159104 + }, + { + "epoch": 0.97, + "learning_rate": 2.6142427281845537e-05, + "loss": 1.2114, + "theoretical_loss": 3.333350115742152, + "tokens_seen": 3194224640 + }, + { + "epoch": 0.97, + "learning_rate": 2.6126379137412238e-05, + "loss": 1.124, + "theoretical_loss": 3.3333449685642482, + "tokens_seen": 3194290176 + }, + { + "epoch": 0.97, + "learning_rate": 2.611033099297894e-05, + "loss": 1.1595, + "theoretical_loss": 3.333339821521514, + "tokens_seen": 3194355712 + }, + { + "epoch": 0.97, + "learning_rate": 2.6094282848545636e-05, + "loss": 1.2625, + "theoretical_loss": 3.333334674613943, + "tokens_seen": 3194421248 + }, + { + "epoch": 0.97, + "learning_rate": 2.6078234704112336e-05, + "loss": 1.1743, + "theoretical_loss": 3.333329527841529, + "tokens_seen": 3194486784 + }, + { + "epoch": 0.97, + "learning_rate": 2.6062186559679037e-05, + "loss": 1.1433, + "theoretical_loss": 3.3333243812042657, + "tokens_seen": 3194552320 + }, + { + "epoch": 0.97, + "learning_rate": 2.604613841524574e-05, + "loss": 1.2627, + "theoretical_loss": 3.3333192347021465, + "tokens_seen": 3194617856 + }, + { + "epoch": 0.97, + "learning_rate": 2.603009027081244e-05, + "loss": 1.2435, + "theoretical_loss": 3.3333140883351655, + "tokens_seen": 3194683392 + }, + { + "epoch": 0.97, + "learning_rate": 2.601404212637914e-05, + "loss": 1.174, + "theoretical_loss": 3.333308942103316, + "tokens_seen": 3194748928 + }, + { + "epoch": 0.97, + "learning_rate": 2.599799398194584e-05, + "loss": 1.1536, + "theoretical_loss": 3.333303796006592, + "tokens_seen": 3194814464 + }, + { + "epoch": 0.97, + "objective/train/docs_used": 1066315, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4155097007751465, + "objective/train/theoretical_loss": 3.333298650044987, + "objective/train/tokens_used": 3215340000, + "theoretical_loss": 3.333298650044987, + "tokens_seen": 3194880000 + }, + { + "epoch": 0.97, + "learning_rate": 2.598194583751254e-05, + "loss": 1.317, + "theoretical_loss": 3.333298650044987, + "tokens_seen": 3194880000 + }, + { + "epoch": 0.97, + "learning_rate": 2.5965897693079237e-05, + "loss": 1.2044, + "theoretical_loss": 3.333293504218495, + "tokens_seen": 3194945536 + }, + { + "epoch": 0.97, + "learning_rate": 2.5949849548645938e-05, + "loss": 1.1869, + "theoretical_loss": 3.333288358527109, + "tokens_seen": 3195011072 + }, + { + "epoch": 0.97, + "learning_rate": 2.593380140421264e-05, + "loss": 1.2237, + "theoretical_loss": 3.333283212970823, + "tokens_seen": 3195076608 + }, + { + "epoch": 0.97, + "learning_rate": 2.5917753259779336e-05, + "loss": 1.1964, + "theoretical_loss": 3.3332780675496307, + "tokens_seen": 3195142144 + }, + { + "epoch": 0.97, + "learning_rate": 2.5901705115346036e-05, + "loss": 1.2774, + "theoretical_loss": 3.333272922263526, + "tokens_seen": 3195207680 + }, + { + "epoch": 0.97, + "learning_rate": 2.588565697091274e-05, + "loss": 1.1535, + "theoretical_loss": 3.3332677771125026, + "tokens_seen": 3195273216 + }, + { + "epoch": 0.97, + "learning_rate": 2.586960882647944e-05, + "loss": 1.2594, + "theoretical_loss": 3.3332626320965537, + "tokens_seen": 3195338752 + }, + { + "epoch": 0.97, + "learning_rate": 2.585356068204614e-05, + "loss": 1.1764, + "theoretical_loss": 3.3332574872156733, + "tokens_seen": 3195404288 + }, + { + "epoch": 0.97, + "learning_rate": 2.583751253761284e-05, + "loss": 1.1958, + "theoretical_loss": 3.3332523424698555, + "tokens_seen": 3195469824 + }, + { + "epoch": 0.97, + "learning_rate": 2.582146439317954e-05, + "loss": 1.2515, + "theoretical_loss": 3.333247197859093, + "tokens_seen": 3195535360 + }, + { + "epoch": 0.97, + "learning_rate": 2.580541624874624e-05, + "loss": 1.1905, + "theoretical_loss": 3.33324205338338, + "tokens_seen": 3195600896 + }, + { + "epoch": 0.97, + "learning_rate": 2.5789368104312937e-05, + "loss": 1.2148, + "theoretical_loss": 3.3332369090427107, + "tokens_seen": 3195666432 + }, + { + "epoch": 0.97, + "learning_rate": 2.5773319959879638e-05, + "loss": 1.1689, + "theoretical_loss": 3.333231764837078, + "tokens_seen": 3195731968 + }, + { + "epoch": 0.97, + "learning_rate": 2.575727181544634e-05, + "loss": 1.2171, + "theoretical_loss": 3.3332266207664762, + "tokens_seen": 3195797504 + }, + { + "epoch": 0.97, + "learning_rate": 2.5741223671013043e-05, + "loss": 1.1889, + "theoretical_loss": 3.3332214768308983, + "tokens_seen": 3195863040 + }, + { + "epoch": 0.97, + "learning_rate": 2.5725175526579743e-05, + "loss": 1.2135, + "theoretical_loss": 3.3332163330303386, + "tokens_seen": 3195928576 + }, + { + "epoch": 0.97, + "learning_rate": 2.570912738214644e-05, + "loss": 1.2766, + "theoretical_loss": 3.3332111893647904, + "tokens_seen": 3195994112 + }, + { + "epoch": 0.97, + "learning_rate": 2.569307923771314e-05, + "loss": 1.2372, + "theoretical_loss": 3.3332060458342476, + "tokens_seen": 3196059648 + }, + { + "epoch": 0.97, + "learning_rate": 2.5677031093279842e-05, + "loss": 1.1049, + "theoretical_loss": 3.333200902438704, + "tokens_seen": 3196125184 + }, + { + "epoch": 0.97, + "learning_rate": 2.566098294884654e-05, + "loss": 1.2194, + "theoretical_loss": 3.333195759178153, + "tokens_seen": 3196190720 + }, + { + "epoch": 0.97, + "learning_rate": 2.564493480441324e-05, + "loss": 1.206, + "theoretical_loss": 3.333190616052588, + "tokens_seen": 3196256256 + }, + { + "epoch": 0.97, + "learning_rate": 2.562888665997994e-05, + "loss": 1.2305, + "theoretical_loss": 3.3331854730620036, + "tokens_seen": 3196321792 + }, + { + "epoch": 0.97, + "learning_rate": 2.5612838515546638e-05, + "loss": 1.1633, + "theoretical_loss": 3.333180330206393, + "tokens_seen": 3196387328 + }, + { + "epoch": 0.97, + "learning_rate": 2.5596790371113345e-05, + "loss": 1.1027, + "theoretical_loss": 3.33317518748575, + "tokens_seen": 3196452864 + }, + { + "epoch": 0.97, + "learning_rate": 2.5580742226680042e-05, + "loss": 1.2163, + "theoretical_loss": 3.3331700449000676, + "tokens_seen": 3196518400 + }, + { + "epoch": 0.97, + "learning_rate": 2.5564694082246743e-05, + "loss": 1.26, + "theoretical_loss": 3.33316490244934, + "tokens_seen": 3196583936 + }, + { + "epoch": 0.97, + "learning_rate": 2.5548645937813444e-05, + "loss": 1.1597, + "theoretical_loss": 3.3331597601335616, + "tokens_seen": 3196649472 + }, + { + "epoch": 0.97, + "learning_rate": 2.553259779338014e-05, + "loss": 1.1376, + "theoretical_loss": 3.3331546179527254, + "tokens_seen": 3196715008 + }, + { + "epoch": 0.97, + "learning_rate": 2.551654964894684e-05, + "loss": 1.1773, + "theoretical_loss": 3.3331494759068248, + "tokens_seen": 3196780544 + }, + { + "epoch": 0.97, + "learning_rate": 2.5500501504513542e-05, + "loss": 1.1727, + "theoretical_loss": 3.333144333995854, + "tokens_seen": 3196846080 + }, + { + "epoch": 0.97, + "learning_rate": 2.548445336008024e-05, + "loss": 1.1498, + "theoretical_loss": 3.3331391922198064, + "tokens_seen": 3196911616 + }, + { + "epoch": 0.97, + "learning_rate": 2.546840521564694e-05, + "loss": 1.2552, + "theoretical_loss": 3.3331340505786757, + "tokens_seen": 3196977152 + }, + { + "epoch": 0.97, + "learning_rate": 2.545235707121364e-05, + "loss": 1.1189, + "theoretical_loss": 3.333128909072456, + "tokens_seen": 3197042688 + }, + { + "epoch": 0.97, + "learning_rate": 2.5436308926780345e-05, + "loss": 1.2211, + "theoretical_loss": 3.3331237677011405, + "tokens_seen": 3197108224 + }, + { + "epoch": 0.97, + "learning_rate": 2.5420260782347045e-05, + "loss": 1.2728, + "theoretical_loss": 3.3331186264647235, + "tokens_seen": 3197173760 + }, + { + "epoch": 0.97, + "learning_rate": 2.5404212637913743e-05, + "loss": 1.1637, + "theoretical_loss": 3.3331134853631976, + "tokens_seen": 3197239296 + }, + { + "epoch": 0.97, + "learning_rate": 2.5388164493480443e-05, + "loss": 1.1656, + "theoretical_loss": 3.3331083443965577, + "tokens_seen": 3197304832 + }, + { + "epoch": 0.97, + "learning_rate": 2.5372116349047144e-05, + "loss": 1.1626, + "theoretical_loss": 3.333103203564797, + "tokens_seen": 3197370368 + }, + { + "epoch": 0.97, + "learning_rate": 2.535606820461384e-05, + "loss": 1.1948, + "theoretical_loss": 3.3330980628679088, + "tokens_seen": 3197435904 + }, + { + "epoch": 0.97, + "learning_rate": 2.534002006018054e-05, + "loss": 1.2843, + "theoretical_loss": 3.3330929223058874, + "tokens_seen": 3197501440 + }, + { + "epoch": 0.97, + "learning_rate": 2.5323971915747242e-05, + "loss": 1.1483, + "theoretical_loss": 3.3330877818787266, + "tokens_seen": 3197566976 + }, + { + "epoch": 0.97, + "learning_rate": 2.530792377131394e-05, + "loss": 1.1878, + "theoretical_loss": 3.333082641586419, + "tokens_seen": 3197632512 + }, + { + "epoch": 0.97, + "learning_rate": 2.5291875626880647e-05, + "loss": 1.1495, + "theoretical_loss": 3.33307750142896, + "tokens_seen": 3197698048 + }, + { + "epoch": 0.97, + "learning_rate": 2.5275827482447344e-05, + "loss": 1.1608, + "theoretical_loss": 3.3330723614063418, + "tokens_seen": 3197763584 + }, + { + "epoch": 0.97, + "learning_rate": 2.5259779338014045e-05, + "loss": 1.2192, + "theoretical_loss": 3.333067221518559, + "tokens_seen": 3197829120 + }, + { + "epoch": 0.97, + "learning_rate": 2.5243731193580745e-05, + "loss": 1.2283, + "theoretical_loss": 3.3330620817656045, + "tokens_seen": 3197894656 + }, + { + "epoch": 0.97, + "learning_rate": 2.5227683049147443e-05, + "loss": 1.1976, + "theoretical_loss": 3.333056942147473, + "tokens_seen": 3197960192 + }, + { + "epoch": 0.97, + "learning_rate": 2.5211634904714143e-05, + "loss": 1.2654, + "theoretical_loss": 3.333051802664157, + "tokens_seen": 3198025728 + }, + { + "epoch": 0.97, + "learning_rate": 2.5195586760280844e-05, + "loss": 1.1982, + "theoretical_loss": 3.3330466633156517, + "tokens_seen": 3198091264 + }, + { + "epoch": 0.97, + "objective/train/docs_used": 1067296, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2987068891525269, + "objective/train/theoretical_loss": 3.3330415241019495, + "objective/train/tokens_used": 3218616800, + "theoretical_loss": 3.3330415241019495, + "tokens_seen": 3198156800 + }, + { + "epoch": 0.97, + "learning_rate": 2.517953861584754e-05, + "loss": 1.2782, + "theoretical_loss": 3.3330415241019495, + "tokens_seen": 3198156800 + }, + { + "epoch": 0.97, + "learning_rate": 2.5163490471414242e-05, + "loss": 1.2211, + "theoretical_loss": 3.3330363850230444, + "tokens_seen": 3198222336 + }, + { + "epoch": 0.97, + "learning_rate": 2.5147442326980946e-05, + "loss": 1.1999, + "theoretical_loss": 3.3330312460789306, + "tokens_seen": 3198287872 + }, + { + "epoch": 0.97, + "learning_rate": 2.5131394182547647e-05, + "loss": 1.2124, + "theoretical_loss": 3.3330261072696015, + "tokens_seen": 3198353408 + }, + { + "epoch": 0.97, + "learning_rate": 2.5115346038114347e-05, + "loss": 1.1847, + "theoretical_loss": 3.3330209685950507, + "tokens_seen": 3198418944 + }, + { + "epoch": 0.97, + "learning_rate": 2.5099297893681044e-05, + "loss": 1.2704, + "theoretical_loss": 3.333015830055272, + "tokens_seen": 3198484480 + }, + { + "epoch": 0.97, + "learning_rate": 2.5083249749247745e-05, + "loss": 1.2304, + "theoretical_loss": 3.3330106916502595, + "tokens_seen": 3198550016 + }, + { + "epoch": 0.97, + "learning_rate": 2.5067201604814446e-05, + "loss": 1.2103, + "theoretical_loss": 3.333005553380006, + "tokens_seen": 3198615552 + }, + { + "epoch": 0.97, + "learning_rate": 2.5051153460381143e-05, + "loss": 1.1925, + "theoretical_loss": 3.3330004152445056, + "tokens_seen": 3198681088 + }, + { + "epoch": 0.97, + "learning_rate": 2.5035105315947844e-05, + "loss": 1.2492, + "theoretical_loss": 3.3329952772437528, + "tokens_seen": 3198746624 + }, + { + "epoch": 0.97, + "learning_rate": 2.5019057171514544e-05, + "loss": 1.0983, + "theoretical_loss": 3.33299013937774, + "tokens_seen": 3198812160 + }, + { + "epoch": 0.97, + "learning_rate": 2.500300902708124e-05, + "loss": 1.2715, + "theoretical_loss": 3.3329850016464615, + "tokens_seen": 3198877696 + }, + { + "epoch": 0.97, + "learning_rate": 2.4986960882647945e-05, + "loss": 1.1628, + "theoretical_loss": 3.332979864049911, + "tokens_seen": 3198943232 + }, + { + "epoch": 0.97, + "learning_rate": 2.4970912738214643e-05, + "loss": 1.2427, + "theoretical_loss": 3.3329747265880827, + "tokens_seen": 3199008768 + }, + { + "epoch": 0.97, + "learning_rate": 2.4954864593781347e-05, + "loss": 1.0851, + "theoretical_loss": 3.3329695892609696, + "tokens_seen": 3199074304 + }, + { + "epoch": 0.97, + "learning_rate": 2.4938816449348047e-05, + "loss": 1.2254, + "theoretical_loss": 3.3329644520685657, + "tokens_seen": 3199139840 + }, + { + "epoch": 0.97, + "learning_rate": 2.4922768304914745e-05, + "loss": 1.1634, + "theoretical_loss": 3.3329593150108643, + "tokens_seen": 3199205376 + }, + { + "epoch": 0.97, + "learning_rate": 2.4906720160481445e-05, + "loss": 1.1903, + "theoretical_loss": 3.33295417808786, + "tokens_seen": 3199270912 + }, + { + "epoch": 0.97, + "learning_rate": 2.4890672016048146e-05, + "loss": 1.2361, + "theoretical_loss": 3.3329490412995457, + "tokens_seen": 3199336448 + }, + { + "epoch": 0.97, + "learning_rate": 2.4874623871614847e-05, + "loss": 1.3532, + "theoretical_loss": 3.3329439046459153, + "tokens_seen": 3199401984 + }, + { + "epoch": 0.97, + "learning_rate": 2.4858575727181547e-05, + "loss": 1.2083, + "theoretical_loss": 3.332938768126963, + "tokens_seen": 3199467520 + }, + { + "epoch": 0.97, + "learning_rate": 2.4842527582748244e-05, + "loss": 1.2216, + "theoretical_loss": 3.332933631742682, + "tokens_seen": 3199533056 + }, + { + "epoch": 0.97, + "learning_rate": 2.4826479438314945e-05, + "loss": 1.1467, + "theoretical_loss": 3.3329284954930656, + "tokens_seen": 3199598592 + }, + { + "epoch": 0.97, + "learning_rate": 2.481043129388165e-05, + "loss": 1.273, + "theoretical_loss": 3.3329233593781082, + "tokens_seen": 3199664128 + }, + { + "epoch": 0.97, + "learning_rate": 2.4794383149448346e-05, + "loss": 1.2035, + "theoretical_loss": 3.3329182233978036, + "tokens_seen": 3199729664 + }, + { + "epoch": 0.97, + "learning_rate": 2.4778335005015047e-05, + "loss": 1.2136, + "theoretical_loss": 3.332913087552145, + "tokens_seen": 3199795200 + }, + { + "epoch": 0.97, + "learning_rate": 2.4762286860581748e-05, + "loss": 1.1417, + "theoretical_loss": 3.332907951841127, + "tokens_seen": 3199860736 + }, + { + "epoch": 0.97, + "learning_rate": 2.4746238716148445e-05, + "loss": 1.2335, + "theoretical_loss": 3.332902816264742, + "tokens_seen": 3199926272 + }, + { + "epoch": 0.97, + "learning_rate": 2.473019057171515e-05, + "loss": 1.2167, + "theoretical_loss": 3.3328976808229847, + "tokens_seen": 3199991808 + }, + { + "epoch": 0.97, + "learning_rate": 2.4714142427281846e-05, + "loss": 1.2953, + "theoretical_loss": 3.3328925455158482, + "tokens_seen": 3200057344 + }, + { + "epoch": 0.97, + "learning_rate": 2.4698094282848547e-05, + "loss": 1.116, + "theoretical_loss": 3.332887410343327, + "tokens_seen": 3200122880 + }, + { + "epoch": 0.97, + "learning_rate": 2.4682046138415247e-05, + "loss": 1.1392, + "theoretical_loss": 3.3328822753054137, + "tokens_seen": 3200188416 + }, + { + "epoch": 0.97, + "learning_rate": 2.4665997993981945e-05, + "loss": 1.1906, + "theoretical_loss": 3.332877140402103, + "tokens_seen": 3200253952 + }, + { + "epoch": 0.97, + "learning_rate": 2.464994984954865e-05, + "loss": 1.1324, + "theoretical_loss": 3.3328720056333885, + "tokens_seen": 3200319488 + }, + { + "epoch": 0.97, + "learning_rate": 2.4633901705115346e-05, + "loss": 1.1509, + "theoretical_loss": 3.3328668709992635, + "tokens_seen": 3200385024 + }, + { + "epoch": 0.97, + "learning_rate": 2.4617853560682047e-05, + "loss": 1.2228, + "theoretical_loss": 3.3328617364997215, + "tokens_seen": 3200450560 + }, + { + "epoch": 0.97, + "learning_rate": 2.4601805416248747e-05, + "loss": 1.2352, + "theoretical_loss": 3.332856602134757, + "tokens_seen": 3200516096 + }, + { + "epoch": 0.97, + "learning_rate": 2.4585757271815448e-05, + "loss": 1.1342, + "theoretical_loss": 3.3328514679043635, + "tokens_seen": 3200581632 + }, + { + "epoch": 0.97, + "learning_rate": 2.456970912738215e-05, + "loss": 1.2331, + "theoretical_loss": 3.332846333808534, + "tokens_seen": 3200647168 + }, + { + "epoch": 0.97, + "learning_rate": 2.455366098294885e-05, + "loss": 1.2395, + "theoretical_loss": 3.3328411998472633, + "tokens_seen": 3200712704 + }, + { + "epoch": 0.97, + "learning_rate": 2.4537612838515546e-05, + "loss": 1.248, + "theoretical_loss": 3.3328360660205445, + "tokens_seen": 3200778240 + }, + { + "epoch": 0.97, + "learning_rate": 2.4521564694082247e-05, + "loss": 1.1231, + "theoretical_loss": 3.3328309323283714, + "tokens_seen": 3200843776 + }, + { + "epoch": 0.97, + "learning_rate": 2.4505516549648948e-05, + "loss": 1.1279, + "theoretical_loss": 3.3328257987707373, + "tokens_seen": 3200909312 + }, + { + "epoch": 0.97, + "learning_rate": 2.4489468405215648e-05, + "loss": 1.1017, + "theoretical_loss": 3.332820665347637, + "tokens_seen": 3200974848 + }, + { + "epoch": 0.97, + "learning_rate": 2.447342026078235e-05, + "loss": 1.1125, + "theoretical_loss": 3.3328155320590627, + "tokens_seen": 3201040384 + }, + { + "epoch": 0.97, + "learning_rate": 2.4457372116349046e-05, + "loss": 1.194, + "theoretical_loss": 3.3328103989050097, + "tokens_seen": 3201105920 + }, + { + "epoch": 0.97, + "learning_rate": 2.4441323971915747e-05, + "loss": 1.2193, + "theoretical_loss": 3.332805265885471, + "tokens_seen": 3201171456 + }, + { + "epoch": 0.97, + "learning_rate": 2.442527582748245e-05, + "loss": 1.1094, + "theoretical_loss": 3.3328001330004398, + "tokens_seen": 3201236992 + }, + { + "epoch": 0.97, + "learning_rate": 2.4409227683049148e-05, + "loss": 1.149, + "theoretical_loss": 3.3327950002499107, + "tokens_seen": 3201302528 + }, + { + "epoch": 0.97, + "learning_rate": 2.439317953861585e-05, + "loss": 1.2551, + "theoretical_loss": 3.332789867633877, + "tokens_seen": 3201368064 + }, + { + "epoch": 0.97, + "objective/train/docs_used": 1068655, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.213536024093628, + "objective/train/theoretical_loss": 3.3327847351523325, + "objective/train/tokens_used": 3221893600, + "theoretical_loss": 3.3327847351523325, + "tokens_seen": 3201433600 + }, + { + "epoch": 0.97, + "learning_rate": 2.437713139418255e-05, + "loss": 1.1647, + "theoretical_loss": 3.3327847351523325, + "tokens_seen": 3201433600 + }, + { + "epoch": 0.97, + "learning_rate": 2.436108324974925e-05, + "loss": 1.1296, + "theoretical_loss": 3.332779602805271, + "tokens_seen": 3201499136 + }, + { + "epoch": 0.97, + "learning_rate": 2.434503510531595e-05, + "loss": 1.2297, + "theoretical_loss": 3.3327744705926863, + "tokens_seen": 3201564672 + }, + { + "epoch": 0.97, + "learning_rate": 2.4328986960882648e-05, + "loss": 1.1954, + "theoretical_loss": 3.3327693385145714, + "tokens_seen": 3201630208 + }, + { + "epoch": 0.97, + "learning_rate": 2.431293881644935e-05, + "loss": 1.274, + "theoretical_loss": 3.332764206570921, + "tokens_seen": 3201695744 + }, + { + "epoch": 0.97, + "learning_rate": 2.429689067201605e-05, + "loss": 1.2499, + "theoretical_loss": 3.332759074761728, + "tokens_seen": 3201761280 + }, + { + "epoch": 0.97, + "learning_rate": 2.428084252758275e-05, + "loss": 1.2068, + "theoretical_loss": 3.332753943086987, + "tokens_seen": 3201826816 + }, + { + "epoch": 0.97, + "learning_rate": 2.426479438314945e-05, + "loss": 1.2314, + "theoretical_loss": 3.3327488115466912, + "tokens_seen": 3201892352 + }, + { + "epoch": 0.97, + "learning_rate": 2.424874623871615e-05, + "loss": 1.1754, + "theoretical_loss": 3.3327436801408346, + "tokens_seen": 3201957888 + }, + { + "epoch": 0.97, + "learning_rate": 2.4232698094282848e-05, + "loss": 1.1604, + "theoretical_loss": 3.3327385488694103, + "tokens_seen": 3202023424 + }, + { + "epoch": 0.97, + "learning_rate": 2.421664994984955e-05, + "loss": 1.2161, + "theoretical_loss": 3.3327334177324124, + "tokens_seen": 3202088960 + }, + { + "epoch": 0.97, + "learning_rate": 2.420060180541625e-05, + "loss": 1.1833, + "theoretical_loss": 3.332728286729835, + "tokens_seen": 3202154496 + }, + { + "epoch": 0.97, + "learning_rate": 2.418455366098295e-05, + "loss": 1.2755, + "theoretical_loss": 3.3327231558616712, + "tokens_seen": 3202220032 + }, + { + "epoch": 0.97, + "learning_rate": 2.416850551654965e-05, + "loss": 1.1441, + "theoretical_loss": 3.332718025127915, + "tokens_seen": 3202285568 + }, + { + "epoch": 0.97, + "learning_rate": 2.4152457372116348e-05, + "loss": 1.1802, + "theoretical_loss": 3.3327128945285605, + "tokens_seen": 3202351104 + }, + { + "epoch": 0.97, + "learning_rate": 2.4136409227683052e-05, + "loss": 1.1898, + "theoretical_loss": 3.332707764063601, + "tokens_seen": 3202416640 + }, + { + "epoch": 0.97, + "learning_rate": 2.4120361083249753e-05, + "loss": 1.1204, + "theoretical_loss": 3.33270263373303, + "tokens_seen": 3202482176 + }, + { + "epoch": 0.97, + "learning_rate": 2.410431293881645e-05, + "loss": 1.0409, + "theoretical_loss": 3.3326975035368416, + "tokens_seen": 3202547712 + }, + { + "epoch": 0.97, + "learning_rate": 2.408826479438315e-05, + "loss": 1.255, + "theoretical_loss": 3.3326923734750293, + "tokens_seen": 3202613248 + }, + { + "epoch": 0.97, + "learning_rate": 2.407221664994985e-05, + "loss": 1.2408, + "theoretical_loss": 3.3326872435475874, + "tokens_seen": 3202678784 + }, + { + "epoch": 0.97, + "learning_rate": 2.4056168505516552e-05, + "loss": 1.0937, + "theoretical_loss": 3.332682113754509, + "tokens_seen": 3202744320 + }, + { + "epoch": 0.97, + "learning_rate": 2.4040120361083252e-05, + "loss": 1.1361, + "theoretical_loss": 3.3326769840957886, + "tokens_seen": 3202809856 + }, + { + "epoch": 0.97, + "learning_rate": 2.402407221664995e-05, + "loss": 1.2899, + "theoretical_loss": 3.332671854571419, + "tokens_seen": 3202875392 + }, + { + "epoch": 0.97, + "learning_rate": 2.400802407221665e-05, + "loss": 1.1647, + "theoretical_loss": 3.332666725181394, + "tokens_seen": 3202940928 + }, + { + "epoch": 0.97, + "learning_rate": 2.399197592778335e-05, + "loss": 1.1628, + "theoretical_loss": 3.332661595925708, + "tokens_seen": 3203006464 + }, + { + "epoch": 0.97, + "learning_rate": 2.397592778335005e-05, + "loss": 1.1965, + "theoretical_loss": 3.3326564668043543, + "tokens_seen": 3203072000 + }, + { + "epoch": 0.97, + "learning_rate": 2.3959879638916752e-05, + "loss": 1.3107, + "theoretical_loss": 3.332651337817327, + "tokens_seen": 3203137536 + }, + { + "epoch": 0.97, + "learning_rate": 2.3943831494483453e-05, + "loss": 1.2275, + "theoretical_loss": 3.3326462089646194, + "tokens_seen": 3203203072 + }, + { + "epoch": 0.97, + "learning_rate": 2.392778335005015e-05, + "loss": 1.2117, + "theoretical_loss": 3.332641080246225, + "tokens_seen": 3203268608 + }, + { + "epoch": 0.97, + "learning_rate": 2.391173520561685e-05, + "loss": 1.1806, + "theoretical_loss": 3.332635951662138, + "tokens_seen": 3203334144 + }, + { + "epoch": 0.97, + "learning_rate": 2.389568706118355e-05, + "loss": 1.2742, + "theoretical_loss": 3.3326308232123525, + "tokens_seen": 3203399680 + }, + { + "epoch": 0.97, + "learning_rate": 2.3879638916750252e-05, + "loss": 1.2151, + "theoretical_loss": 3.3326256948968616, + "tokens_seen": 3203465216 + }, + { + "epoch": 0.97, + "learning_rate": 2.3863590772316953e-05, + "loss": 1.1269, + "theoretical_loss": 3.332620566715659, + "tokens_seen": 3203530752 + }, + { + "epoch": 0.97, + "learning_rate": 2.384754262788365e-05, + "loss": 1.1148, + "theoretical_loss": 3.332615438668739, + "tokens_seen": 3203596288 + }, + { + "epoch": 0.97, + "learning_rate": 2.3831494483450354e-05, + "loss": 1.1866, + "theoretical_loss": 3.3326103107560954, + "tokens_seen": 3203661824 + }, + { + "epoch": 0.97, + "learning_rate": 2.3815446339017055e-05, + "loss": 1.1436, + "theoretical_loss": 3.332605182977721, + "tokens_seen": 3203727360 + }, + { + "epoch": 0.97, + "learning_rate": 2.3799398194583752e-05, + "loss": 1.1722, + "theoretical_loss": 3.33260005533361, + "tokens_seen": 3203792896 + }, + { + "epoch": 0.97, + "learning_rate": 2.3783350050150452e-05, + "loss": 1.1953, + "theoretical_loss": 3.3325949278237563, + "tokens_seen": 3203858432 + }, + { + "epoch": 0.97, + "learning_rate": 2.3767301905717153e-05, + "loss": 1.3152, + "theoretical_loss": 3.3325898004481536, + "tokens_seen": 3203923968 + }, + { + "epoch": 0.97, + "learning_rate": 2.3751253761283854e-05, + "loss": 1.2248, + "theoretical_loss": 3.332584673206796, + "tokens_seen": 3203989504 + }, + { + "epoch": 0.97, + "learning_rate": 2.3735205616850554e-05, + "loss": 1.0677, + "theoretical_loss": 3.332579546099676, + "tokens_seen": 3204055040 + }, + { + "epoch": 0.97, + "learning_rate": 2.371915747241725e-05, + "loss": 1.3037, + "theoretical_loss": 3.332574419126789, + "tokens_seen": 3204120576 + }, + { + "epoch": 0.97, + "learning_rate": 2.3703109327983952e-05, + "loss": 1.2461, + "theoretical_loss": 3.3325692922881274, + "tokens_seen": 3204186112 + }, + { + "epoch": 0.97, + "learning_rate": 2.3687061183550653e-05, + "loss": 1.2288, + "theoretical_loss": 3.332564165583686, + "tokens_seen": 3204251648 + }, + { + "epoch": 0.97, + "learning_rate": 2.3671013039117353e-05, + "loss": 1.3141, + "theoretical_loss": 3.332559039013457, + "tokens_seen": 3204317184 + }, + { + "epoch": 0.97, + "learning_rate": 2.3654964894684054e-05, + "loss": 1.2033, + "theoretical_loss": 3.332553912577436, + "tokens_seen": 3204382720 + }, + { + "epoch": 0.97, + "learning_rate": 2.3638916750250755e-05, + "loss": 1.1294, + "theoretical_loss": 3.332548786275616, + "tokens_seen": 3204448256 + }, + { + "epoch": 0.97, + "learning_rate": 2.3622868605817452e-05, + "loss": 1.167, + "theoretical_loss": 3.33254366010799, + "tokens_seen": 3204513792 + }, + { + "epoch": 0.97, + "learning_rate": 2.3606820461384156e-05, + "loss": 1.2247, + "theoretical_loss": 3.3325385340745526, + "tokens_seen": 3204579328 + }, + { + "epoch": 0.97, + "learning_rate": 2.3590772316950853e-05, + "loss": 1.1352, + "theoretical_loss": 3.3325334081752973, + "tokens_seen": 3204644864 + }, + { + "epoch": 0.97, + "objective/train/docs_used": 1069641, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2387337684631348, + "objective/train/theoretical_loss": 3.332528282410218, + "objective/train/tokens_used": 3225170400, + "theoretical_loss": 3.332528282410218, + "tokens_seen": 3204710400 + }, + { + "epoch": 0.97, + "learning_rate": 2.3574724172517554e-05, + "loss": 1.2011, + "theoretical_loss": 3.332528282410218, + "tokens_seen": 3204710400 + }, + { + "epoch": 0.97, + "learning_rate": 2.3558676028084255e-05, + "loss": 1.1575, + "theoretical_loss": 3.3325231567793083, + "tokens_seen": 3204775936 + }, + { + "epoch": 0.97, + "learning_rate": 2.3542627883650952e-05, + "loss": 1.1983, + "theoretical_loss": 3.3325180312825617, + "tokens_seen": 3204841472 + }, + { + "epoch": 0.97, + "learning_rate": 2.3526579739217656e-05, + "loss": 1.2349, + "theoretical_loss": 3.3325129059199723, + "tokens_seen": 3204907008 + }, + { + "epoch": 0.97, + "learning_rate": 2.3510531594784353e-05, + "loss": 1.2724, + "theoretical_loss": 3.332507780691534, + "tokens_seen": 3204972544 + }, + { + "epoch": 0.97, + "learning_rate": 2.3494483450351054e-05, + "loss": 1.1851, + "theoretical_loss": 3.33250265559724, + "tokens_seen": 3205038080 + }, + { + "epoch": 0.97, + "learning_rate": 2.3478435305917754e-05, + "loss": 1.1112, + "theoretical_loss": 3.3324975306370845, + "tokens_seen": 3205103616 + }, + { + "epoch": 0.97, + "learning_rate": 2.346238716148445e-05, + "loss": 1.1721, + "theoretical_loss": 3.3324924058110605, + "tokens_seen": 3205169152 + }, + { + "epoch": 0.97, + "learning_rate": 2.3446339017051156e-05, + "loss": 1.1787, + "theoretical_loss": 3.332487281119163, + "tokens_seen": 3205234688 + }, + { + "epoch": 0.97, + "learning_rate": 2.3430290872617856e-05, + "loss": 1.1833, + "theoretical_loss": 3.332482156561385, + "tokens_seen": 3205300224 + }, + { + "epoch": 0.97, + "learning_rate": 2.3414242728184553e-05, + "loss": 1.1917, + "theoretical_loss": 3.33247703213772, + "tokens_seen": 3205365760 + }, + { + "epoch": 0.97, + "learning_rate": 2.3398194583751254e-05, + "loss": 1.2054, + "theoretical_loss": 3.3324719078481624, + "tokens_seen": 3205431296 + }, + { + "epoch": 0.97, + "learning_rate": 2.3382146439317955e-05, + "loss": 1.2207, + "theoretical_loss": 3.3324667836927055, + "tokens_seen": 3205496832 + }, + { + "epoch": 0.97, + "learning_rate": 2.3366098294884655e-05, + "loss": 1.1765, + "theoretical_loss": 3.332461659671343, + "tokens_seen": 3205562368 + }, + { + "epoch": 0.97, + "learning_rate": 2.3350050150451356e-05, + "loss": 1.2078, + "theoretical_loss": 3.332456535784069, + "tokens_seen": 3205627904 + }, + { + "epoch": 0.97, + "learning_rate": 2.3334002006018053e-05, + "loss": 1.3195, + "theoretical_loss": 3.332451412030877, + "tokens_seen": 3205693440 + }, + { + "epoch": 0.97, + "learning_rate": 2.3317953861584754e-05, + "loss": 1.2184, + "theoretical_loss": 3.3324462884117607, + "tokens_seen": 3205758976 + }, + { + "epoch": 0.97, + "learning_rate": 2.3301905717151458e-05, + "loss": 1.1809, + "theoretical_loss": 3.332441164926714, + "tokens_seen": 3205824512 + }, + { + "epoch": 0.97, + "learning_rate": 2.3285857572718155e-05, + "loss": 1.2008, + "theoretical_loss": 3.332436041575731, + "tokens_seen": 3205890048 + }, + { + "epoch": 0.97, + "learning_rate": 2.3269809428284856e-05, + "loss": 1.2014, + "theoretical_loss": 3.3324309183588046, + "tokens_seen": 3205955584 + }, + { + "epoch": 0.97, + "learning_rate": 2.3253761283851556e-05, + "loss": 1.1594, + "theoretical_loss": 3.332425795275929, + "tokens_seen": 3206021120 + }, + { + "epoch": 0.97, + "learning_rate": 2.3237713139418254e-05, + "loss": 1.1793, + "theoretical_loss": 3.332420672327098, + "tokens_seen": 3206086656 + }, + { + "epoch": 0.97, + "learning_rate": 2.3221664994984958e-05, + "loss": 1.0995, + "theoretical_loss": 3.3324155495123056, + "tokens_seen": 3206152192 + }, + { + "epoch": 0.97, + "learning_rate": 2.3205616850551655e-05, + "loss": 1.0904, + "theoretical_loss": 3.332410426831545, + "tokens_seen": 3206217728 + }, + { + "epoch": 0.97, + "learning_rate": 2.3189568706118356e-05, + "loss": 1.1984, + "theoretical_loss": 3.3324053042848103, + "tokens_seen": 3206283264 + }, + { + "epoch": 0.97, + "learning_rate": 2.3173520561685056e-05, + "loss": 1.1671, + "theoretical_loss": 3.332400181872095, + "tokens_seen": 3206348800 + }, + { + "epoch": 0.97, + "learning_rate": 2.3157472417251757e-05, + "loss": 1.1913, + "theoretical_loss": 3.3323950595933933, + "tokens_seen": 3206414336 + }, + { + "epoch": 0.97, + "learning_rate": 2.3141424272818457e-05, + "loss": 1.1613, + "theoretical_loss": 3.3323899374486987, + "tokens_seen": 3206479872 + }, + { + "epoch": 0.97, + "learning_rate": 2.3125376128385158e-05, + "loss": 1.1866, + "theoretical_loss": 3.332384815438005, + "tokens_seen": 3206545408 + }, + { + "epoch": 0.97, + "learning_rate": 2.3109327983951855e-05, + "loss": 1.1936, + "theoretical_loss": 3.3323796935613057, + "tokens_seen": 3206610944 + }, + { + "epoch": 0.97, + "learning_rate": 2.3093279839518556e-05, + "loss": 1.2094, + "theoretical_loss": 3.332374571818595, + "tokens_seen": 3206676480 + }, + { + "epoch": 0.97, + "learning_rate": 2.3077231695085257e-05, + "loss": 1.2127, + "theoretical_loss": 3.332369450209866, + "tokens_seen": 3206742016 + }, + { + "epoch": 0.97, + "learning_rate": 2.3061183550651957e-05, + "loss": 1.1722, + "theoretical_loss": 3.332364328735113, + "tokens_seen": 3206807552 + }, + { + "epoch": 0.97, + "learning_rate": 2.3045135406218658e-05, + "loss": 1.1608, + "theoretical_loss": 3.33235920739433, + "tokens_seen": 3206873088 + }, + { + "epoch": 0.97, + "learning_rate": 2.3029087261785355e-05, + "loss": 1.2579, + "theoretical_loss": 3.33235408618751, + "tokens_seen": 3206938624 + }, + { + "epoch": 0.97, + "learning_rate": 2.3013039117352056e-05, + "loss": 1.1559, + "theoretical_loss": 3.3323489651146474, + "tokens_seen": 3207004160 + }, + { + "epoch": 0.97, + "learning_rate": 2.299699097291876e-05, + "loss": 1.1534, + "theoretical_loss": 3.332343844175736, + "tokens_seen": 3207069696 + }, + { + "epoch": 0.97, + "learning_rate": 2.2980942828485457e-05, + "loss": 1.2608, + "theoretical_loss": 3.3323387233707686, + "tokens_seen": 3207135232 + }, + { + "epoch": 0.97, + "learning_rate": 2.2964894684052158e-05, + "loss": 1.1811, + "theoretical_loss": 3.33233360269974, + "tokens_seen": 3207200768 + }, + { + "epoch": 0.97, + "learning_rate": 2.2948846539618858e-05, + "loss": 1.1058, + "theoretical_loss": 3.3323284821626435, + "tokens_seen": 3207266304 + }, + { + "epoch": 0.97, + "learning_rate": 2.293279839518556e-05, + "loss": 1.2401, + "theoretical_loss": 3.3323233617594727, + "tokens_seen": 3207331840 + }, + { + "epoch": 0.97, + "learning_rate": 2.291675025075226e-05, + "loss": 1.1198, + "theoretical_loss": 3.332318241490222, + "tokens_seen": 3207397376 + }, + { + "epoch": 0.97, + "learning_rate": 2.2900702106318957e-05, + "loss": 1.2307, + "theoretical_loss": 3.3323131213548844, + "tokens_seen": 3207462912 + }, + { + "epoch": 0.97, + "learning_rate": 2.2884653961885657e-05, + "loss": 1.1498, + "theoretical_loss": 3.3323080013534545, + "tokens_seen": 3207528448 + }, + { + "epoch": 0.97, + "learning_rate": 2.2868605817452358e-05, + "loss": 1.1743, + "theoretical_loss": 3.3323028814859255, + "tokens_seen": 3207593984 + }, + { + "epoch": 0.97, + "learning_rate": 2.285255767301906e-05, + "loss": 1.17, + "theoretical_loss": 3.3322977617522906, + "tokens_seen": 3207659520 + }, + { + "epoch": 0.97, + "learning_rate": 2.283650952858576e-05, + "loss": 1.2468, + "theoretical_loss": 3.3322926421525447, + "tokens_seen": 3207725056 + }, + { + "epoch": 0.97, + "learning_rate": 2.282046138415246e-05, + "loss": 1.1135, + "theoretical_loss": 3.332287522686681, + "tokens_seen": 3207790592 + }, + { + "epoch": 0.97, + "learning_rate": 2.2804413239719157e-05, + "loss": 1.1758, + "theoretical_loss": 3.332282403354694, + "tokens_seen": 3207856128 + }, + { + "epoch": 0.97, + "learning_rate": 2.2788365095285858e-05, + "loss": 1.1669, + "theoretical_loss": 3.332277284156576, + "tokens_seen": 3207921664 + }, + { + "epoch": 0.97, + "objective/train/docs_used": 1070878, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1322975158691406, + "objective/train/theoretical_loss": 3.3322721650923217, + "objective/train/tokens_used": 3228447200, + "theoretical_loss": 3.3322721650923217, + "tokens_seen": 3207987200 + }, + { + "epoch": 0.97, + "learning_rate": 2.277231695085256e-05, + "loss": 1.171, + "theoretical_loss": 3.3322721650923217, + "tokens_seen": 3207987200 + }, + { + "epoch": 0.97, + "learning_rate": 2.275626880641926e-05, + "loss": 1.1165, + "theoretical_loss": 3.332267046161925, + "tokens_seen": 3208052736 + }, + { + "epoch": 0.97, + "learning_rate": 2.274022066198596e-05, + "loss": 1.1723, + "theoretical_loss": 3.332261927365379, + "tokens_seen": 3208118272 + }, + { + "epoch": 0.97, + "learning_rate": 2.2724172517552657e-05, + "loss": 1.2488, + "theoretical_loss": 3.332256808702678, + "tokens_seen": 3208183808 + }, + { + "epoch": 0.97, + "learning_rate": 2.270812437311936e-05, + "loss": 1.2616, + "theoretical_loss": 3.3322516901738157, + "tokens_seen": 3208249344 + }, + { + "epoch": 0.97, + "learning_rate": 2.269207622868606e-05, + "loss": 1.2205, + "theoretical_loss": 3.3322465717787857, + "tokens_seen": 3208314880 + }, + { + "epoch": 0.97, + "learning_rate": 2.267602808425276e-05, + "loss": 1.2192, + "theoretical_loss": 3.332241453517582, + "tokens_seen": 3208380416 + }, + { + "epoch": 0.97, + "learning_rate": 2.265997993981946e-05, + "loss": 1.2895, + "theoretical_loss": 3.3322363353901983, + "tokens_seen": 3208445952 + }, + { + "epoch": 0.97, + "learning_rate": 2.264393179538616e-05, + "loss": 1.2011, + "theoretical_loss": 3.332231217396628, + "tokens_seen": 3208511488 + }, + { + "epoch": 0.97, + "learning_rate": 2.262788365095286e-05, + "loss": 1.2302, + "theoretical_loss": 3.3322260995368653, + "tokens_seen": 3208577024 + }, + { + "epoch": 0.97, + "learning_rate": 2.261183550651956e-05, + "loss": 1.2407, + "theoretical_loss": 3.332220981810904, + "tokens_seen": 3208642560 + }, + { + "epoch": 0.97, + "learning_rate": 2.259578736208626e-05, + "loss": 1.1919, + "theoretical_loss": 3.332215864218738, + "tokens_seen": 3208708096 + }, + { + "epoch": 0.97, + "learning_rate": 2.257973921765296e-05, + "loss": 1.2325, + "theoretical_loss": 3.33221074676036, + "tokens_seen": 3208773632 + }, + { + "epoch": 0.97, + "learning_rate": 2.256369107321966e-05, + "loss": 1.1273, + "theoretical_loss": 3.332205629435765, + "tokens_seen": 3208839168 + }, + { + "epoch": 0.97, + "learning_rate": 2.254764292878636e-05, + "loss": 1.1526, + "theoretical_loss": 3.3322005122449463, + "tokens_seen": 3208904704 + }, + { + "epoch": 0.97, + "learning_rate": 2.253159478435306e-05, + "loss": 1.1967, + "theoretical_loss": 3.332195395187898, + "tokens_seen": 3208970240 + }, + { + "epoch": 0.97, + "learning_rate": 2.251554663991976e-05, + "loss": 1.1475, + "theoretical_loss": 3.332190278264613, + "tokens_seen": 3209035776 + }, + { + "epoch": 0.97, + "learning_rate": 2.249949849548646e-05, + "loss": 1.1619, + "theoretical_loss": 3.3321851614750857, + "tokens_seen": 3209101312 + }, + { + "epoch": 0.97, + "learning_rate": 2.2483450351053163e-05, + "loss": 1.2094, + "theoretical_loss": 3.33218004481931, + "tokens_seen": 3209166848 + }, + { + "epoch": 0.97, + "learning_rate": 2.246740220661986e-05, + "loss": 1.196, + "theoretical_loss": 3.3321749282972797, + "tokens_seen": 3209232384 + }, + { + "epoch": 0.97, + "learning_rate": 2.245135406218656e-05, + "loss": 1.1933, + "theoretical_loss": 3.332169811908988, + "tokens_seen": 3209297920 + }, + { + "epoch": 0.97, + "learning_rate": 2.243530591775326e-05, + "loss": 1.1757, + "theoretical_loss": 3.332164695654429, + "tokens_seen": 3209363456 + }, + { + "epoch": 0.97, + "learning_rate": 2.241925777331996e-05, + "loss": 1.2572, + "theoretical_loss": 3.3321595795335965, + "tokens_seen": 3209428992 + }, + { + "epoch": 0.97, + "learning_rate": 2.2403209628886663e-05, + "loss": 1.1228, + "theoretical_loss": 3.332154463546485, + "tokens_seen": 3209494528 + }, + { + "epoch": 0.97, + "learning_rate": 2.238716148445336e-05, + "loss": 1.2436, + "theoretical_loss": 3.3321493476930866, + "tokens_seen": 3209560064 + }, + { + "epoch": 0.97, + "learning_rate": 2.237111334002006e-05, + "loss": 1.2491, + "theoretical_loss": 3.3321442319733965, + "tokens_seen": 3209625600 + }, + { + "epoch": 0.97, + "learning_rate": 2.235506519558676e-05, + "loss": 1.209, + "theoretical_loss": 3.3321391163874075, + "tokens_seen": 3209691136 + }, + { + "epoch": 0.97, + "learning_rate": 2.233901705115346e-05, + "loss": 1.1228, + "theoretical_loss": 3.332134000935114, + "tokens_seen": 3209756672 + }, + { + "epoch": 0.97, + "learning_rate": 2.2322968906720163e-05, + "loss": 1.2124, + "theoretical_loss": 3.33212888561651, + "tokens_seen": 3209822208 + }, + { + "epoch": 0.97, + "learning_rate": 2.2306920762286863e-05, + "loss": 1.2482, + "theoretical_loss": 3.332123770431589, + "tokens_seen": 3209887744 + }, + { + "epoch": 0.97, + "learning_rate": 2.229087261785356e-05, + "loss": 1.2319, + "theoretical_loss": 3.332118655380344, + "tokens_seen": 3209953280 + }, + { + "epoch": 0.97, + "learning_rate": 2.227482447342026e-05, + "loss": 1.1632, + "theoretical_loss": 3.33211354046277, + "tokens_seen": 3210018816 + }, + { + "epoch": 0.97, + "learning_rate": 2.2258776328986962e-05, + "loss": 1.2357, + "theoretical_loss": 3.3321084256788605, + "tokens_seen": 3210084352 + }, + { + "epoch": 0.97, + "learning_rate": 2.2242728184553662e-05, + "loss": 1.173, + "theoretical_loss": 3.3321033110286087, + "tokens_seen": 3210149888 + }, + { + "epoch": 0.97, + "learning_rate": 2.2226680040120363e-05, + "loss": 1.2298, + "theoretical_loss": 3.3320981965120087, + "tokens_seen": 3210215424 + }, + { + "epoch": 0.97, + "learning_rate": 2.221063189568706e-05, + "loss": 1.1734, + "theoretical_loss": 3.3320930821290546, + "tokens_seen": 3210280960 + }, + { + "epoch": 0.97, + "learning_rate": 2.219458375125376e-05, + "loss": 1.1335, + "theoretical_loss": 3.3320879678797395, + "tokens_seen": 3210346496 + }, + { + "epoch": 0.97, + "learning_rate": 2.2178535606820465e-05, + "loss": 1.304, + "theoretical_loss": 3.3320828537640574, + "tokens_seen": 3210412032 + }, + { + "epoch": 0.97, + "learning_rate": 2.2162487462387162e-05, + "loss": 1.2731, + "theoretical_loss": 3.3320777397820027, + "tokens_seen": 3210477568 + }, + { + "epoch": 0.97, + "learning_rate": 2.2146439317953863e-05, + "loss": 1.1779, + "theoretical_loss": 3.3320726259335687, + "tokens_seen": 3210543104 + }, + { + "epoch": 0.97, + "learning_rate": 2.2130391173520564e-05, + "loss": 1.1154, + "theoretical_loss": 3.332067512218749, + "tokens_seen": 3210608640 + }, + { + "epoch": 0.97, + "learning_rate": 2.211434302908726e-05, + "loss": 1.1799, + "theoretical_loss": 3.3320623986375377, + "tokens_seen": 3210674176 + }, + { + "epoch": 0.97, + "learning_rate": 2.2098294884653965e-05, + "loss": 1.2254, + "theoretical_loss": 3.3320572851899284, + "tokens_seen": 3210739712 + }, + { + "epoch": 0.97, + "learning_rate": 2.2082246740220662e-05, + "loss": 1.1895, + "theoretical_loss": 3.332052171875915, + "tokens_seen": 3210805248 + }, + { + "epoch": 0.97, + "learning_rate": 2.2066198595787363e-05, + "loss": 1.1514, + "theoretical_loss": 3.332047058695491, + "tokens_seen": 3210870784 + }, + { + "epoch": 0.97, + "learning_rate": 2.2050150451354063e-05, + "loss": 1.2761, + "theoretical_loss": 3.3320419456486503, + "tokens_seen": 3210936320 + }, + { + "epoch": 0.97, + "learning_rate": 2.2034102306920764e-05, + "loss": 1.2207, + "theoretical_loss": 3.332036832735387, + "tokens_seen": 3211001856 + }, + { + "epoch": 0.97, + "learning_rate": 2.2018054162487465e-05, + "loss": 1.2876, + "theoretical_loss": 3.332031719955695, + "tokens_seen": 3211067392 + }, + { + "epoch": 0.97, + "learning_rate": 2.2002006018054165e-05, + "loss": 1.1501, + "theoretical_loss": 3.332026607309567, + "tokens_seen": 3211132928 + }, + { + "epoch": 0.97, + "learning_rate": 2.1985957873620862e-05, + "loss": 1.2, + "theoretical_loss": 3.332021494796998, + "tokens_seen": 3211198464 + }, + { + "epoch": 0.97, + "objective/train/docs_used": 1071766, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1696527004241943, + "objective/train/theoretical_loss": 3.3320163824179816, + "objective/train/tokens_used": 3231724000, + "theoretical_loss": 3.3320163824179816, + "tokens_seen": 3211264000 + }, + { + "epoch": 0.97, + "learning_rate": 2.1969909729187563e-05, + "loss": 1.17, + "theoretical_loss": 3.3320163824179816, + "tokens_seen": 3211264000 + }, + { + "epoch": 0.97, + "learning_rate": 2.1953861584754264e-05, + "loss": 1.2567, + "theoretical_loss": 3.332011270172511, + "tokens_seen": 3211329536 + }, + { + "epoch": 0.97, + "learning_rate": 2.1937813440320964e-05, + "loss": 1.1872, + "theoretical_loss": 3.3320061580605804, + "tokens_seen": 3211395072 + }, + { + "epoch": 0.97, + "learning_rate": 2.1921765295887665e-05, + "loss": 1.1957, + "theoretical_loss": 3.3320010460821834, + "tokens_seen": 3211460608 + }, + { + "epoch": 0.97, + "learning_rate": 2.1905717151454362e-05, + "loss": 1.2242, + "theoretical_loss": 3.3319959342373138, + "tokens_seen": 3211526144 + }, + { + "epoch": 0.97, + "learning_rate": 2.1889669007021063e-05, + "loss": 1.22, + "theoretical_loss": 3.331990822525966, + "tokens_seen": 3211591680 + }, + { + "epoch": 0.97, + "learning_rate": 2.1873620862587767e-05, + "loss": 1.2578, + "theoretical_loss": 3.3319857109481323, + "tokens_seen": 3211657216 + }, + { + "epoch": 0.97, + "learning_rate": 2.1857572718154464e-05, + "loss": 1.1317, + "theoretical_loss": 3.331980599503808, + "tokens_seen": 3211722752 + }, + { + "epoch": 0.97, + "learning_rate": 2.1841524573721165e-05, + "loss": 1.1213, + "theoretical_loss": 3.3319754881929864, + "tokens_seen": 3211788288 + }, + { + "epoch": 0.97, + "learning_rate": 2.1825476429287865e-05, + "loss": 1.2155, + "theoretical_loss": 3.331970377015661, + "tokens_seen": 3211853824 + }, + { + "epoch": 0.97, + "learning_rate": 2.1809428284854563e-05, + "loss": 1.2691, + "theoretical_loss": 3.331965265971826, + "tokens_seen": 3211919360 + }, + { + "epoch": 0.97, + "learning_rate": 2.1793380140421267e-05, + "loss": 1.2821, + "theoretical_loss": 3.331960155061475, + "tokens_seen": 3211984896 + }, + { + "epoch": 0.97, + "learning_rate": 2.1777331995987964e-05, + "loss": 1.2106, + "theoretical_loss": 3.3319550442846015, + "tokens_seen": 3212050432 + }, + { + "epoch": 0.97, + "learning_rate": 2.1761283851554665e-05, + "loss": 1.2053, + "theoretical_loss": 3.3319499336411997, + "tokens_seen": 3212115968 + }, + { + "epoch": 0.97, + "learning_rate": 2.1745235707121365e-05, + "loss": 1.1134, + "theoretical_loss": 3.3319448231312636, + "tokens_seen": 3212181504 + }, + { + "epoch": 0.97, + "learning_rate": 2.1729187562688066e-05, + "loss": 1.3081, + "theoretical_loss": 3.3319397127547865, + "tokens_seen": 3212247040 + }, + { + "epoch": 0.97, + "learning_rate": 2.1713139418254766e-05, + "loss": 1.2425, + "theoretical_loss": 3.331934602511762, + "tokens_seen": 3212312576 + }, + { + "epoch": 0.97, + "learning_rate": 2.1697091273821467e-05, + "loss": 1.1387, + "theoretical_loss": 3.3319294924021845, + "tokens_seen": 3212378112 + }, + { + "epoch": 0.97, + "learning_rate": 2.1681043129388164e-05, + "loss": 1.2141, + "theoretical_loss": 3.3319243824260476, + "tokens_seen": 3212443648 + }, + { + "epoch": 0.97, + "learning_rate": 2.1664994984954865e-05, + "loss": 1.1416, + "theoretical_loss": 3.3319192725833453, + "tokens_seen": 3212509184 + }, + { + "epoch": 0.97, + "learning_rate": 2.1648946840521566e-05, + "loss": 1.176, + "theoretical_loss": 3.3319141628740705, + "tokens_seen": 3212574720 + }, + { + "epoch": 0.97, + "learning_rate": 2.1632898696088266e-05, + "loss": 1.1619, + "theoretical_loss": 3.331909053298218, + "tokens_seen": 3212640256 + }, + { + "epoch": 0.97, + "learning_rate": 2.1616850551654967e-05, + "loss": 1.11, + "theoretical_loss": 3.331903943855781, + "tokens_seen": 3212705792 + }, + { + "epoch": 0.97, + "learning_rate": 2.1600802407221664e-05, + "loss": 1.1427, + "theoretical_loss": 3.3318988345467537, + "tokens_seen": 3212771328 + }, + { + "epoch": 0.97, + "learning_rate": 2.1584754262788365e-05, + "loss": 1.2435, + "theoretical_loss": 3.33189372537113, + "tokens_seen": 3212836864 + }, + { + "epoch": 0.97, + "learning_rate": 2.1568706118355065e-05, + "loss": 1.1963, + "theoretical_loss": 3.3318886163289028, + "tokens_seen": 3212902400 + }, + { + "epoch": 0.97, + "learning_rate": 2.1552657973921766e-05, + "loss": 1.1003, + "theoretical_loss": 3.331883507420067, + "tokens_seen": 3212967936 + }, + { + "epoch": 0.97, + "learning_rate": 2.1536609829488467e-05, + "loss": 1.2642, + "theoretical_loss": 3.331878398644615, + "tokens_seen": 3213033472 + }, + { + "epoch": 0.97, + "learning_rate": 2.1520561685055164e-05, + "loss": 1.2124, + "theoretical_loss": 3.3318732900025427, + "tokens_seen": 3213099008 + }, + { + "epoch": 0.97, + "learning_rate": 2.1504513540621868e-05, + "loss": 1.1703, + "theoretical_loss": 3.331868181493842, + "tokens_seen": 3213164544 + }, + { + "epoch": 0.97, + "learning_rate": 2.148846539618857e-05, + "loss": 1.256, + "theoretical_loss": 3.331863073118507, + "tokens_seen": 3213230080 + }, + { + "epoch": 0.97, + "learning_rate": 2.1472417251755266e-05, + "loss": 1.0471, + "theoretical_loss": 3.3318579648765327, + "tokens_seen": 3213295616 + }, + { + "epoch": 0.97, + "learning_rate": 2.1456369107321966e-05, + "loss": 1.1623, + "theoretical_loss": 3.3318528567679118, + "tokens_seen": 3213361152 + }, + { + "epoch": 0.97, + "learning_rate": 2.1440320962888667e-05, + "loss": 1.1224, + "theoretical_loss": 3.3318477487926383, + "tokens_seen": 3213426688 + }, + { + "epoch": 0.97, + "learning_rate": 2.1424272818455368e-05, + "loss": 1.2278, + "theoretical_loss": 3.3318426409507063, + "tokens_seen": 3213492224 + }, + { + "epoch": 0.97, + "learning_rate": 2.140822467402207e-05, + "loss": 1.2014, + "theoretical_loss": 3.3318375332421093, + "tokens_seen": 3213557760 + }, + { + "epoch": 0.97, + "learning_rate": 2.1392176529588766e-05, + "loss": 1.2198, + "theoretical_loss": 3.331832425666841, + "tokens_seen": 3213623296 + }, + { + "epoch": 0.97, + "learning_rate": 2.1376128385155466e-05, + "loss": 1.1028, + "theoretical_loss": 3.3318273182248954, + "tokens_seen": 3213688832 + }, + { + "epoch": 0.97, + "learning_rate": 2.1360080240722167e-05, + "loss": 1.1249, + "theoretical_loss": 3.3318222109162665, + "tokens_seen": 3213754368 + }, + { + "epoch": 0.97, + "learning_rate": 2.1344032096288868e-05, + "loss": 1.1634, + "theoretical_loss": 3.3318171037409474, + "tokens_seen": 3213819904 + }, + { + "epoch": 0.97, + "learning_rate": 2.1327983951855568e-05, + "loss": 1.1495, + "theoretical_loss": 3.331811996698933, + "tokens_seen": 3213885440 + }, + { + "epoch": 0.97, + "learning_rate": 2.131193580742227e-05, + "loss": 1.0854, + "theoretical_loss": 3.331806889790216, + "tokens_seen": 3213950976 + }, + { + "epoch": 0.97, + "learning_rate": 2.1295887662988966e-05, + "loss": 1.1581, + "theoretical_loss": 3.331801783014791, + "tokens_seen": 3214016512 + }, + { + "epoch": 0.97, + "learning_rate": 2.127983951855567e-05, + "loss": 1.1219, + "theoretical_loss": 3.3317966763726514, + "tokens_seen": 3214082048 + }, + { + "epoch": 0.97, + "learning_rate": 2.1263791374122367e-05, + "loss": 1.0581, + "theoretical_loss": 3.3317915698637908, + "tokens_seen": 3214147584 + }, + { + "epoch": 0.97, + "learning_rate": 2.1247743229689068e-05, + "loss": 1.1224, + "theoretical_loss": 3.331786463488204, + "tokens_seen": 3214213120 + }, + { + "epoch": 0.97, + "learning_rate": 2.123169508525577e-05, + "loss": 1.1194, + "theoretical_loss": 3.3317813572458834, + "tokens_seen": 3214278656 + }, + { + "epoch": 0.97, + "learning_rate": 2.1215646940822466e-05, + "loss": 1.1841, + "theoretical_loss": 3.331776251136824, + "tokens_seen": 3214344192 + }, + { + "epoch": 0.97, + "learning_rate": 2.119959879638917e-05, + "loss": 1.2599, + "theoretical_loss": 3.331771145161019, + "tokens_seen": 3214409728 + }, + { + "epoch": 0.97, + "learning_rate": 2.118355065195587e-05, + "loss": 1.223, + "theoretical_loss": 3.3317660393184623, + "tokens_seen": 3214475264 + }, + { + "epoch": 0.97, + "objective/train/docs_used": 1072937, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2786359786987305, + "objective/train/theoretical_loss": 3.3317609336091474, + "objective/train/tokens_used": 3235000800, + "theoretical_loss": 3.3317609336091474, + "tokens_seen": 3214540800 + }, + { + "epoch": 0.97, + "learning_rate": 2.1167502507522568e-05, + "loss": 1.2948, + "theoretical_loss": 3.3317609336091474, + "tokens_seen": 3214540800 + }, + { + "epoch": 0.97, + "learning_rate": 2.115145436308927e-05, + "loss": 1.1923, + "theoretical_loss": 3.331755828033069, + "tokens_seen": 3214606336 + }, + { + "epoch": 0.97, + "learning_rate": 2.113540621865597e-05, + "loss": 1.23, + "theoretical_loss": 3.33175072259022, + "tokens_seen": 3214671872 + }, + { + "epoch": 0.97, + "learning_rate": 2.111935807422267e-05, + "loss": 1.2641, + "theoretical_loss": 3.3317456172805944, + "tokens_seen": 3214737408 + }, + { + "epoch": 0.97, + "learning_rate": 2.110330992978937e-05, + "loss": 1.1325, + "theoretical_loss": 3.3317405121041865, + "tokens_seen": 3214802944 + }, + { + "epoch": 0.97, + "learning_rate": 2.1087261785356068e-05, + "loss": 1.2146, + "theoretical_loss": 3.3317354070609895, + "tokens_seen": 3214868480 + }, + { + "epoch": 0.97, + "learning_rate": 2.1071213640922768e-05, + "loss": 1.1917, + "theoretical_loss": 3.331730302150998, + "tokens_seen": 3214934016 + }, + { + "epoch": 0.97, + "learning_rate": 2.1055165496489472e-05, + "loss": 1.2877, + "theoretical_loss": 3.3317251973742046, + "tokens_seen": 3214999552 + }, + { + "epoch": 0.97, + "learning_rate": 2.103911735205617e-05, + "loss": 1.2289, + "theoretical_loss": 3.3317200927306043, + "tokens_seen": 3215065088 + }, + { + "epoch": 0.97, + "learning_rate": 2.102306920762287e-05, + "loss": 1.2473, + "theoretical_loss": 3.33171498822019, + "tokens_seen": 3215130624 + }, + { + "epoch": 0.97, + "learning_rate": 2.100702106318957e-05, + "loss": 1.2045, + "theoretical_loss": 3.331709883842956, + "tokens_seen": 3215196160 + }, + { + "epoch": 0.97, + "learning_rate": 2.0990972918756268e-05, + "loss": 1.2496, + "theoretical_loss": 3.3317047795988963, + "tokens_seen": 3215261696 + }, + { + "epoch": 0.97, + "learning_rate": 2.0974924774322972e-05, + "loss": 1.2751, + "theoretical_loss": 3.331699675488004, + "tokens_seen": 3215327232 + }, + { + "epoch": 0.97, + "learning_rate": 2.095887662988967e-05, + "loss": 1.2485, + "theoretical_loss": 3.331694571510274, + "tokens_seen": 3215392768 + }, + { + "epoch": 0.97, + "learning_rate": 2.094282848545637e-05, + "loss": 1.1878, + "theoretical_loss": 3.331689467665699, + "tokens_seen": 3215458304 + }, + { + "epoch": 0.97, + "learning_rate": 2.092678034102307e-05, + "loss": 1.1873, + "theoretical_loss": 3.331684363954273, + "tokens_seen": 3215523840 + }, + { + "epoch": 0.97, + "learning_rate": 2.0910732196589768e-05, + "loss": 1.1549, + "theoretical_loss": 3.331679260375991, + "tokens_seen": 3215589376 + }, + { + "epoch": 0.97, + "learning_rate": 2.0894684052156472e-05, + "loss": 1.2303, + "theoretical_loss": 3.331674156930845, + "tokens_seen": 3215654912 + }, + { + "epoch": 0.97, + "learning_rate": 2.0878635907723172e-05, + "loss": 1.279, + "theoretical_loss": 3.33166905361883, + "tokens_seen": 3215720448 + }, + { + "epoch": 0.97, + "learning_rate": 2.086258776328987e-05, + "loss": 1.2548, + "theoretical_loss": 3.33166395043994, + "tokens_seen": 3215785984 + }, + { + "epoch": 0.97, + "learning_rate": 2.084653961885657e-05, + "loss": 1.305, + "theoretical_loss": 3.3316588473941673, + "tokens_seen": 3215851520 + }, + { + "epoch": 0.97, + "learning_rate": 2.083049147442327e-05, + "loss": 1.2064, + "theoretical_loss": 3.3316537444815078, + "tokens_seen": 3215917056 + }, + { + "epoch": 0.97, + "learning_rate": 2.081444332998997e-05, + "loss": 1.2274, + "theoretical_loss": 3.3316486417019537, + "tokens_seen": 3215982592 + }, + { + "epoch": 0.97, + "learning_rate": 2.0798395185556672e-05, + "loss": 1.1635, + "theoretical_loss": 3.3316435390554995, + "tokens_seen": 3216048128 + }, + { + "epoch": 0.97, + "learning_rate": 2.078234704112337e-05, + "loss": 1.1831, + "theoretical_loss": 3.3316384365421388, + "tokens_seen": 3216113664 + }, + { + "epoch": 0.97, + "learning_rate": 2.076629889669007e-05, + "loss": 1.2711, + "theoretical_loss": 3.3316333341618654, + "tokens_seen": 3216179200 + }, + { + "epoch": 0.97, + "learning_rate": 2.0750250752256774e-05, + "loss": 1.2608, + "theoretical_loss": 3.3316282319146735, + "tokens_seen": 3216244736 + }, + { + "epoch": 0.97, + "learning_rate": 2.073420260782347e-05, + "loss": 1.1555, + "theoretical_loss": 3.3316231298005565, + "tokens_seen": 3216310272 + }, + { + "epoch": 0.97, + "learning_rate": 2.0718154463390172e-05, + "loss": 1.2603, + "theoretical_loss": 3.3316180278195087, + "tokens_seen": 3216375808 + }, + { + "epoch": 0.97, + "learning_rate": 2.0702106318956873e-05, + "loss": 1.1802, + "theoretical_loss": 3.3316129259715233, + "tokens_seen": 3216441344 + }, + { + "epoch": 0.97, + "learning_rate": 2.068605817452357e-05, + "loss": 1.1591, + "theoretical_loss": 3.3316078242565945, + "tokens_seen": 3216506880 + }, + { + "epoch": 0.97, + "learning_rate": 2.0670010030090274e-05, + "loss": 1.2043, + "theoretical_loss": 3.331602722674716, + "tokens_seen": 3216572416 + }, + { + "epoch": 0.97, + "learning_rate": 2.065396188565697e-05, + "loss": 1.24, + "theoretical_loss": 3.3315976212258818, + "tokens_seen": 3216637952 + }, + { + "epoch": 0.97, + "learning_rate": 2.0637913741223672e-05, + "loss": 1.1091, + "theoretical_loss": 3.3315925199100853, + "tokens_seen": 3216703488 + }, + { + "epoch": 0.97, + "learning_rate": 2.0621865596790372e-05, + "loss": 1.1944, + "theoretical_loss": 3.3315874187273207, + "tokens_seen": 3216769024 + }, + { + "epoch": 0.97, + "learning_rate": 2.0605817452357073e-05, + "loss": 1.2719, + "theoretical_loss": 3.3315823176775816, + "tokens_seen": 3216834560 + }, + { + "epoch": 0.97, + "learning_rate": 2.0589769307923774e-05, + "loss": 1.2004, + "theoretical_loss": 3.331577216760862, + "tokens_seen": 3216900096 + }, + { + "epoch": 0.97, + "learning_rate": 2.057372116349047e-05, + "loss": 1.1747, + "theoretical_loss": 3.331572115977156, + "tokens_seen": 3216965632 + }, + { + "epoch": 0.97, + "learning_rate": 2.055767301905717e-05, + "loss": 1.1746, + "theoretical_loss": 3.3315670153264567, + "tokens_seen": 3217031168 + }, + { + "epoch": 0.97, + "learning_rate": 2.0541624874623872e-05, + "loss": 1.1847, + "theoretical_loss": 3.3315619148087583, + "tokens_seen": 3217096704 + }, + { + "epoch": 0.97, + "learning_rate": 2.0525576730190573e-05, + "loss": 1.2315, + "theoretical_loss": 3.3315568144240544, + "tokens_seen": 3217162240 + }, + { + "epoch": 0.97, + "learning_rate": 2.0509528585757273e-05, + "loss": 1.2511, + "theoretical_loss": 3.331551714172339, + "tokens_seen": 3217227776 + }, + { + "epoch": 0.97, + "learning_rate": 2.0493480441323974e-05, + "loss": 1.0978, + "theoretical_loss": 3.3315466140536065, + "tokens_seen": 3217293312 + }, + { + "epoch": 0.97, + "learning_rate": 2.047743229689067e-05, + "loss": 1.2256, + "theoretical_loss": 3.33154151406785, + "tokens_seen": 3217358848 + }, + { + "epoch": 0.97, + "learning_rate": 2.0461384152457372e-05, + "loss": 1.1474, + "theoretical_loss": 3.331536414215064, + "tokens_seen": 3217424384 + }, + { + "epoch": 0.97, + "learning_rate": 2.0445336008024073e-05, + "loss": 1.1305, + "theoretical_loss": 3.3315313144952405, + "tokens_seen": 3217489920 + }, + { + "epoch": 0.98, + "learning_rate": 2.0429287863590773e-05, + "loss": 1.2315, + "theoretical_loss": 3.3315262149083757, + "tokens_seen": 3217555456 + }, + { + "epoch": 0.98, + "learning_rate": 2.0413239719157474e-05, + "loss": 1.2123, + "theoretical_loss": 3.3315211154544624, + "tokens_seen": 3217620992 + }, + { + "epoch": 0.98, + "learning_rate": 2.039719157472417e-05, + "loss": 1.3174, + "theoretical_loss": 3.331516016133494, + "tokens_seen": 3217686528 + }, + { + "epoch": 0.98, + "learning_rate": 2.0381143430290875e-05, + "loss": 1.294, + "theoretical_loss": 3.3315109169454646, + "tokens_seen": 3217752064 + }, + { + "epoch": 0.98, + "objective/train/docs_used": 1074222, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1902265548706055, + "objective/train/theoretical_loss": 3.3315058178903687, + "objective/train/tokens_used": 3238277600, + "theoretical_loss": 3.3315058178903687, + "tokens_seen": 3217817600 + }, + { + "epoch": 0.98, + "learning_rate": 2.0365095285857576e-05, + "loss": 1.1259, + "theoretical_loss": 3.3315058178903687, + "tokens_seen": 3217817600 + }, + { + "epoch": 0.98, + "learning_rate": 2.0349047141424273e-05, + "loss": 1.1455, + "theoretical_loss": 3.3315007189681993, + "tokens_seen": 3217883136 + }, + { + "epoch": 0.98, + "learning_rate": 2.0332998996990974e-05, + "loss": 1.2159, + "theoretical_loss": 3.3314956201789503, + "tokens_seen": 3217948672 + }, + { + "epoch": 0.98, + "learning_rate": 2.0316950852557674e-05, + "loss": 1.2503, + "theoretical_loss": 3.3314905215226163, + "tokens_seen": 3218014208 + }, + { + "epoch": 0.98, + "learning_rate": 2.0300902708124375e-05, + "loss": 1.1857, + "theoretical_loss": 3.33148542299919, + "tokens_seen": 3218079744 + }, + { + "epoch": 0.98, + "learning_rate": 2.0284854563691076e-05, + "loss": 1.1547, + "theoretical_loss": 3.331480324608666, + "tokens_seen": 3218145280 + }, + { + "epoch": 0.98, + "learning_rate": 2.0268806419257773e-05, + "loss": 1.168, + "theoretical_loss": 3.331475226351038, + "tokens_seen": 3218210816 + }, + { + "epoch": 0.98, + "learning_rate": 2.0252758274824473e-05, + "loss": 1.0895, + "theoretical_loss": 3.3314701282263, + "tokens_seen": 3218276352 + }, + { + "epoch": 0.98, + "learning_rate": 2.0236710130391174e-05, + "loss": 1.2007, + "theoretical_loss": 3.3314650302344453, + "tokens_seen": 3218341888 + }, + { + "epoch": 0.98, + "learning_rate": 2.0220661985957875e-05, + "loss": 1.1651, + "theoretical_loss": 3.331459932375468, + "tokens_seen": 3218407424 + }, + { + "epoch": 0.98, + "learning_rate": 2.0204613841524575e-05, + "loss": 1.3124, + "theoretical_loss": 3.331454834649362, + "tokens_seen": 3218472960 + }, + { + "epoch": 0.98, + "learning_rate": 2.0188565697091276e-05, + "loss": 1.1462, + "theoretical_loss": 3.331449737056121, + "tokens_seen": 3218538496 + }, + { + "epoch": 0.98, + "learning_rate": 2.0172517552657973e-05, + "loss": 1.1966, + "theoretical_loss": 3.3314446395957393, + "tokens_seen": 3218604032 + }, + { + "epoch": 0.98, + "learning_rate": 2.0156469408224677e-05, + "loss": 1.2403, + "theoretical_loss": 3.3314395422682104, + "tokens_seen": 3218669568 + }, + { + "epoch": 0.98, + "learning_rate": 2.0140421263791374e-05, + "loss": 1.2184, + "theoretical_loss": 3.3314344450735276, + "tokens_seen": 3218735104 + }, + { + "epoch": 0.98, + "learning_rate": 2.0124373119358075e-05, + "loss": 1.1504, + "theoretical_loss": 3.331429348011685, + "tokens_seen": 3218800640 + }, + { + "epoch": 0.98, + "learning_rate": 2.0108324974924776e-05, + "loss": 1.1992, + "theoretical_loss": 3.3314242510826775, + "tokens_seen": 3218866176 + }, + { + "epoch": 0.98, + "learning_rate": 2.0092276830491473e-05, + "loss": 1.3014, + "theoretical_loss": 3.3314191542864973, + "tokens_seen": 3218931712 + }, + { + "epoch": 0.98, + "learning_rate": 2.0076228686058177e-05, + "loss": 1.1087, + "theoretical_loss": 3.3314140576231397, + "tokens_seen": 3218997248 + }, + { + "epoch": 0.98, + "learning_rate": 2.0060180541624878e-05, + "loss": 1.2688, + "theoretical_loss": 3.331408961092597, + "tokens_seen": 3219062784 + }, + { + "epoch": 0.98, + "learning_rate": 2.0044132397191575e-05, + "loss": 1.3502, + "theoretical_loss": 3.3314038646948645, + "tokens_seen": 3219128320 + }, + { + "epoch": 0.98, + "learning_rate": 2.0028084252758276e-05, + "loss": 1.1449, + "theoretical_loss": 3.3313987684299353, + "tokens_seen": 3219193856 + }, + { + "epoch": 0.98, + "learning_rate": 2.0012036108324976e-05, + "loss": 1.1951, + "theoretical_loss": 3.3313936722978035, + "tokens_seen": 3219259392 + }, + { + "epoch": 0.98, + "learning_rate": 1.9995987963891677e-05, + "loss": 1.2186, + "theoretical_loss": 3.3313885762984627, + "tokens_seen": 3219324928 + }, + { + "epoch": 0.98, + "learning_rate": 1.9979939819458377e-05, + "loss": 1.1856, + "theoretical_loss": 3.331383480431907, + "tokens_seen": 3219390464 + }, + { + "epoch": 0.98, + "learning_rate": 1.9963891675025075e-05, + "loss": 1.2188, + "theoretical_loss": 3.3313783846981297, + "tokens_seen": 3219456000 + }, + { + "epoch": 0.98, + "learning_rate": 1.9947843530591775e-05, + "loss": 1.1371, + "theoretical_loss": 3.331373289097125, + "tokens_seen": 3219521536 + }, + { + "epoch": 0.98, + "learning_rate": 1.993179538615848e-05, + "loss": 1.3036, + "theoretical_loss": 3.331368193628887, + "tokens_seen": 3219587072 + }, + { + "epoch": 0.98, + "learning_rate": 1.9915747241725177e-05, + "loss": 1.1725, + "theoretical_loss": 3.331363098293409, + "tokens_seen": 3219652608 + }, + { + "epoch": 0.98, + "learning_rate": 1.9899699097291877e-05, + "loss": 1.1595, + "theoretical_loss": 3.3313580030906853, + "tokens_seen": 3219718144 + }, + { + "epoch": 0.98, + "learning_rate": 1.9883650952858578e-05, + "loss": 1.2212, + "theoretical_loss": 3.3313529080207096, + "tokens_seen": 3219783680 + }, + { + "epoch": 0.98, + "learning_rate": 1.9867602808425275e-05, + "loss": 1.2717, + "theoretical_loss": 3.3313478130834757, + "tokens_seen": 3219849216 + }, + { + "epoch": 0.98, + "learning_rate": 1.985155466399198e-05, + "loss": 1.1744, + "theoretical_loss": 3.3313427182789774, + "tokens_seen": 3219914752 + }, + { + "epoch": 0.98, + "learning_rate": 1.9835506519558676e-05, + "loss": 1.1957, + "theoretical_loss": 3.3313376236072085, + "tokens_seen": 3219980288 + }, + { + "epoch": 0.98, + "learning_rate": 1.9819458375125377e-05, + "loss": 1.1553, + "theoretical_loss": 3.3313325290681632, + "tokens_seen": 3220045824 + }, + { + "epoch": 0.98, + "learning_rate": 1.9803410230692078e-05, + "loss": 1.1902, + "theoretical_loss": 3.331327434661835, + "tokens_seen": 3220111360 + }, + { + "epoch": 0.98, + "learning_rate": 1.9787362086258775e-05, + "loss": 1.1517, + "theoretical_loss": 3.3313223403882173, + "tokens_seen": 3220176896 + }, + { + "epoch": 0.98, + "learning_rate": 1.977131394182548e-05, + "loss": 1.0079, + "theoretical_loss": 3.331317246247305, + "tokens_seen": 3220242432 + }, + { + "epoch": 0.98, + "learning_rate": 1.975526579739218e-05, + "loss": 1.1446, + "theoretical_loss": 3.331312152239091, + "tokens_seen": 3220307968 + }, + { + "epoch": 0.98, + "learning_rate": 1.9739217652958877e-05, + "loss": 1.2303, + "theoretical_loss": 3.3313070583635698, + "tokens_seen": 3220373504 + }, + { + "epoch": 0.98, + "learning_rate": 1.9723169508525577e-05, + "loss": 1.2022, + "theoretical_loss": 3.331301964620735, + "tokens_seen": 3220439040 + }, + { + "epoch": 0.98, + "learning_rate": 1.9707121364092278e-05, + "loss": 1.3142, + "theoretical_loss": 3.33129687101058, + "tokens_seen": 3220504576 + }, + { + "epoch": 0.98, + "learning_rate": 1.969107321965898e-05, + "loss": 1.2291, + "theoretical_loss": 3.3312917775331, + "tokens_seen": 3220570112 + }, + { + "epoch": 0.98, + "learning_rate": 1.967502507522568e-05, + "loss": 1.1179, + "theoretical_loss": 3.331286684188287, + "tokens_seen": 3220635648 + }, + { + "epoch": 0.98, + "learning_rate": 1.9658976930792377e-05, + "loss": 1.2327, + "theoretical_loss": 3.331281590976136, + "tokens_seen": 3220701184 + }, + { + "epoch": 0.98, + "learning_rate": 1.9642928786359077e-05, + "loss": 1.1865, + "theoretical_loss": 3.3312764978966407, + "tokens_seen": 3220766720 + }, + { + "epoch": 0.98, + "learning_rate": 1.9626880641925778e-05, + "loss": 1.2041, + "theoretical_loss": 3.3312714049497947, + "tokens_seen": 3220832256 + }, + { + "epoch": 0.98, + "learning_rate": 1.961083249749248e-05, + "loss": 1.1163, + "theoretical_loss": 3.3312663121355923, + "tokens_seen": 3220897792 + }, + { + "epoch": 0.98, + "learning_rate": 1.959478435305918e-05, + "loss": 1.1572, + "theoretical_loss": 3.331261219454027, + "tokens_seen": 3220963328 + }, + { + "epoch": 0.98, + "learning_rate": 1.9578736208625876e-05, + "loss": 1.2774, + "theoretical_loss": 3.3312561269050924, + "tokens_seen": 3221028864 + }, + { + "epoch": 0.98, + "objective/train/docs_used": 1075115, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.273951530456543, + "objective/train/theoretical_loss": 3.3312510344887825, + "objective/train/tokens_used": 3241554400, + "theoretical_loss": 3.3312510344887825, + "tokens_seen": 3221094400 + }, + { + "epoch": 0.98, + "learning_rate": 1.9562688064192577e-05, + "loss": 1.2538, + "theoretical_loss": 3.3312510344887825, + "tokens_seen": 3221094400 + }, + { + "epoch": 0.98, + "learning_rate": 1.954663991975928e-05, + "loss": 1.2427, + "theoretical_loss": 3.3312459422050917, + "tokens_seen": 3221159936 + }, + { + "epoch": 0.98, + "learning_rate": 1.9530591775325978e-05, + "loss": 1.1874, + "theoretical_loss": 3.331240850054013, + "tokens_seen": 3221225472 + }, + { + "epoch": 0.98, + "learning_rate": 1.951454363089268e-05, + "loss": 1.1227, + "theoretical_loss": 3.331235758035541, + "tokens_seen": 3221291008 + }, + { + "epoch": 0.98, + "learning_rate": 1.949849548645938e-05, + "loss": 1.1727, + "theoretical_loss": 3.3312306661496693, + "tokens_seen": 3221356544 + }, + { + "epoch": 0.98, + "learning_rate": 1.9482447342026077e-05, + "loss": 1.1581, + "theoretical_loss": 3.331225574396391, + "tokens_seen": 3221422080 + }, + { + "epoch": 0.98, + "learning_rate": 1.946639919759278e-05, + "loss": 1.1455, + "theoretical_loss": 3.3312204827757013, + "tokens_seen": 3221487616 + }, + { + "epoch": 0.98, + "learning_rate": 1.9450351053159478e-05, + "loss": 1.1262, + "theoretical_loss": 3.3312153912875933, + "tokens_seen": 3221553152 + }, + { + "epoch": 0.98, + "learning_rate": 1.943430290872618e-05, + "loss": 1.1987, + "theoretical_loss": 3.331210299932061, + "tokens_seen": 3221618688 + }, + { + "epoch": 0.98, + "learning_rate": 1.941825476429288e-05, + "loss": 1.221, + "theoretical_loss": 3.3312052087090978, + "tokens_seen": 3221684224 + }, + { + "epoch": 0.98, + "learning_rate": 1.940220661985958e-05, + "loss": 1.2367, + "theoretical_loss": 3.3312001176186983, + "tokens_seen": 3221749760 + }, + { + "epoch": 0.98, + "learning_rate": 1.938615847542628e-05, + "loss": 1.1662, + "theoretical_loss": 3.331195026660856, + "tokens_seen": 3221815296 + }, + { + "epoch": 0.98, + "learning_rate": 1.937011033099298e-05, + "loss": 1.1366, + "theoretical_loss": 3.3311899358355643, + "tokens_seen": 3221880832 + }, + { + "epoch": 0.98, + "learning_rate": 1.935406218655968e-05, + "loss": 1.1054, + "theoretical_loss": 3.3311848451428174, + "tokens_seen": 3221946368 + }, + { + "epoch": 0.98, + "learning_rate": 1.933801404212638e-05, + "loss": 1.1878, + "theoretical_loss": 3.33117975458261, + "tokens_seen": 3222011904 + }, + { + "epoch": 0.98, + "learning_rate": 1.932196589769308e-05, + "loss": 1.1161, + "theoretical_loss": 3.3311746641549345, + "tokens_seen": 3222077440 + }, + { + "epoch": 0.98, + "learning_rate": 1.930591775325978e-05, + "loss": 1.2069, + "theoretical_loss": 3.3311695738597855, + "tokens_seen": 3222142976 + }, + { + "epoch": 0.98, + "learning_rate": 1.928986960882648e-05, + "loss": 1.2605, + "theoretical_loss": 3.3311644836971572, + "tokens_seen": 3222208512 + }, + { + "epoch": 0.98, + "learning_rate": 1.9273821464393178e-05, + "loss": 1.2424, + "theoretical_loss": 3.3311593936670425, + "tokens_seen": 3222274048 + }, + { + "epoch": 0.98, + "learning_rate": 1.925777331995988e-05, + "loss": 1.177, + "theoretical_loss": 3.3311543037694364, + "tokens_seen": 3222339584 + }, + { + "epoch": 0.98, + "learning_rate": 1.9241725175526583e-05, + "loss": 1.2447, + "theoretical_loss": 3.331149214004332, + "tokens_seen": 3222405120 + }, + { + "epoch": 0.98, + "learning_rate": 1.922567703109328e-05, + "loss": 1.1902, + "theoretical_loss": 3.3311441243717232, + "tokens_seen": 3222470656 + }, + { + "epoch": 0.98, + "learning_rate": 1.920962888665998e-05, + "loss": 1.1328, + "theoretical_loss": 3.3311390348716037, + "tokens_seen": 3222536192 + }, + { + "epoch": 0.98, + "learning_rate": 1.919358074222668e-05, + "loss": 1.2307, + "theoretical_loss": 3.331133945503968, + "tokens_seen": 3222601728 + }, + { + "epoch": 0.98, + "learning_rate": 1.9177532597793382e-05, + "loss": 1.2367, + "theoretical_loss": 3.3311288562688097, + "tokens_seen": 3222667264 + }, + { + "epoch": 0.98, + "learning_rate": 1.9161484453360083e-05, + "loss": 1.1227, + "theoretical_loss": 3.3311237671661225, + "tokens_seen": 3222732800 + }, + { + "epoch": 0.98, + "learning_rate": 1.914543630892678e-05, + "loss": 1.2922, + "theoretical_loss": 3.3311186781959, + "tokens_seen": 3222798336 + }, + { + "epoch": 0.98, + "learning_rate": 1.912938816449348e-05, + "loss": 1.2477, + "theoretical_loss": 3.3311135893581367, + "tokens_seen": 3222863872 + }, + { + "epoch": 0.98, + "learning_rate": 1.911334002006018e-05, + "loss": 1.1744, + "theoretical_loss": 3.331108500652826, + "tokens_seen": 3222929408 + }, + { + "epoch": 0.98, + "learning_rate": 1.9097291875626882e-05, + "loss": 1.185, + "theoretical_loss": 3.331103412079962, + "tokens_seen": 3222994944 + }, + { + "epoch": 0.98, + "learning_rate": 1.9081243731193582e-05, + "loss": 1.1924, + "theoretical_loss": 3.331098323639538, + "tokens_seen": 3223060480 + }, + { + "epoch": 0.98, + "learning_rate": 1.9065195586760283e-05, + "loss": 1.2146, + "theoretical_loss": 3.3310932353315486, + "tokens_seen": 3223126016 + }, + { + "epoch": 0.98, + "learning_rate": 1.904914744232698e-05, + "loss": 1.1449, + "theoretical_loss": 3.3310881471559872, + "tokens_seen": 3223191552 + }, + { + "epoch": 0.98, + "learning_rate": 1.903309929789368e-05, + "loss": 1.1546, + "theoretical_loss": 3.331083059112848, + "tokens_seen": 3223257088 + }, + { + "epoch": 0.98, + "learning_rate": 1.901705115346038e-05, + "loss": 1.2309, + "theoretical_loss": 3.3310779712021246, + "tokens_seen": 3223322624 + }, + { + "epoch": 0.98, + "learning_rate": 1.9001003009027082e-05, + "loss": 1.2538, + "theoretical_loss": 3.3310728834238112, + "tokens_seen": 3223388160 + }, + { + "epoch": 0.98, + "learning_rate": 1.8984954864593783e-05, + "loss": 1.2603, + "theoretical_loss": 3.3310677957779014, + "tokens_seen": 3223453696 + }, + { + "epoch": 0.98, + "learning_rate": 1.896890672016048e-05, + "loss": 1.1317, + "theoretical_loss": 3.3310627082643887, + "tokens_seen": 3223519232 + }, + { + "epoch": 0.98, + "learning_rate": 1.8952858575727184e-05, + "loss": 1.2656, + "theoretical_loss": 3.3310576208832674, + "tokens_seen": 3223584768 + }, + { + "epoch": 0.98, + "learning_rate": 1.8936810431293885e-05, + "loss": 1.1537, + "theoretical_loss": 3.3310525336345314, + "tokens_seen": 3223650304 + }, + { + "epoch": 0.98, + "learning_rate": 1.8920762286860582e-05, + "loss": 1.2234, + "theoretical_loss": 3.3310474465181743, + "tokens_seen": 3223715840 + }, + { + "epoch": 0.98, + "learning_rate": 1.8904714142427283e-05, + "loss": 1.1939, + "theoretical_loss": 3.3310423595341905, + "tokens_seen": 3223781376 + }, + { + "epoch": 0.98, + "learning_rate": 1.8888665997993983e-05, + "loss": 1.1923, + "theoretical_loss": 3.331037272682573, + "tokens_seen": 3223846912 + }, + { + "epoch": 0.98, + "learning_rate": 1.8872617853560684e-05, + "loss": 1.1436, + "theoretical_loss": 3.3310321859633167, + "tokens_seen": 3223912448 + }, + { + "epoch": 0.98, + "learning_rate": 1.8856569709127385e-05, + "loss": 1.2922, + "theoretical_loss": 3.331027099376415, + "tokens_seen": 3223977984 + }, + { + "epoch": 0.98, + "learning_rate": 1.8840521564694082e-05, + "loss": 1.1615, + "theoretical_loss": 3.3310220129218613, + "tokens_seen": 3224043520 + }, + { + "epoch": 0.98, + "learning_rate": 1.8824473420260782e-05, + "loss": 1.1735, + "theoretical_loss": 3.33101692659965, + "tokens_seen": 3224109056 + }, + { + "epoch": 0.98, + "learning_rate": 1.8808425275827483e-05, + "loss": 1.3439, + "theoretical_loss": 3.3310118404097744, + "tokens_seen": 3224174592 + }, + { + "epoch": 0.98, + "learning_rate": 1.8792377131394184e-05, + "loss": 1.2286, + "theoretical_loss": 3.331006754352229, + "tokens_seen": 3224240128 + }, + { + "epoch": 0.98, + "learning_rate": 1.8776328986960884e-05, + "loss": 1.1577, + "theoretical_loss": 3.3310016684270076, + "tokens_seen": 3224305664 + }, + { + "epoch": 0.98, + "objective/train/docs_used": 1076370, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3098498582839966, + "objective/train/theoretical_loss": 3.3309965826341044, + "objective/train/tokens_used": 3244831200, + "theoretical_loss": 3.3309965826341044, + "tokens_seen": 3224371200 + }, + { + "epoch": 0.98, + "learning_rate": 1.8760280842527585e-05, + "loss": 1.2365, + "theoretical_loss": 3.3309965826341044, + "tokens_seen": 3224371200 + }, + { + "epoch": 0.98, + "learning_rate": 1.8744232698094282e-05, + "loss": 1.1504, + "theoretical_loss": 3.330991496973512, + "tokens_seen": 3224436736 + }, + { + "epoch": 0.98, + "learning_rate": 1.8728184553660986e-05, + "loss": 1.1674, + "theoretical_loss": 3.3309864114452252, + "tokens_seen": 3224502272 + }, + { + "epoch": 0.98, + "learning_rate": 1.8712136409227684e-05, + "loss": 1.2795, + "theoretical_loss": 3.330981326049238, + "tokens_seen": 3224567808 + }, + { + "epoch": 0.98, + "learning_rate": 1.8696088264794384e-05, + "loss": 1.2972, + "theoretical_loss": 3.3309762407855437, + "tokens_seen": 3224633344 + }, + { + "epoch": 0.98, + "learning_rate": 1.8680040120361085e-05, + "loss": 1.2434, + "theoretical_loss": 3.3309711556541366, + "tokens_seen": 3224698880 + }, + { + "epoch": 0.98, + "learning_rate": 1.8663991975927782e-05, + "loss": 1.2502, + "theoretical_loss": 3.330966070655011, + "tokens_seen": 3224764416 + }, + { + "epoch": 0.98, + "learning_rate": 1.8647943831494486e-05, + "loss": 1.2129, + "theoretical_loss": 3.3309609857881592, + "tokens_seen": 3224829952 + }, + { + "epoch": 0.98, + "learning_rate": 1.8631895687061183e-05, + "loss": 1.1755, + "theoretical_loss": 3.330955901053577, + "tokens_seen": 3224895488 + }, + { + "epoch": 0.98, + "learning_rate": 1.8615847542627884e-05, + "loss": 1.1856, + "theoretical_loss": 3.3309508164512565, + "tokens_seen": 3224961024 + }, + { + "epoch": 0.98, + "learning_rate": 1.8599799398194585e-05, + "loss": 1.2047, + "theoretical_loss": 3.330945731981193, + "tokens_seen": 3225026560 + }, + { + "epoch": 0.98, + "learning_rate": 1.8583751253761282e-05, + "loss": 1.1851, + "theoretical_loss": 3.33094064764338, + "tokens_seen": 3225092096 + }, + { + "epoch": 0.98, + "learning_rate": 1.8567703109327986e-05, + "loss": 1.3102, + "theoretical_loss": 3.3309355634378104, + "tokens_seen": 3225157632 + }, + { + "epoch": 0.98, + "learning_rate": 1.8551654964894686e-05, + "loss": 1.2129, + "theoretical_loss": 3.330930479364479, + "tokens_seen": 3225223168 + }, + { + "epoch": 0.98, + "learning_rate": 1.8535606820461384e-05, + "loss": 1.1866, + "theoretical_loss": 3.33092539542338, + "tokens_seen": 3225288704 + }, + { + "epoch": 0.98, + "learning_rate": 1.8519558676028084e-05, + "loss": 1.1073, + "theoretical_loss": 3.3309203116145065, + "tokens_seen": 3225354240 + }, + { + "epoch": 0.98, + "learning_rate": 1.8503510531594785e-05, + "loss": 1.2133, + "theoretical_loss": 3.3309152279378527, + "tokens_seen": 3225419776 + }, + { + "epoch": 0.98, + "learning_rate": 1.8487462387161486e-05, + "loss": 1.1913, + "theoretical_loss": 3.3309101443934126, + "tokens_seen": 3225485312 + }, + { + "epoch": 0.98, + "learning_rate": 1.8471414242728186e-05, + "loss": 1.2655, + "theoretical_loss": 3.3309050609811797, + "tokens_seen": 3225550848 + }, + { + "epoch": 0.98, + "learning_rate": 1.8455366098294884e-05, + "loss": 1.1659, + "theoretical_loss": 3.330899977701148, + "tokens_seen": 3225616384 + }, + { + "epoch": 0.98, + "learning_rate": 1.8439317953861584e-05, + "loss": 1.2294, + "theoretical_loss": 3.3308948945533117, + "tokens_seen": 3225681920 + }, + { + "epoch": 0.98, + "learning_rate": 1.8423269809428288e-05, + "loss": 1.1327, + "theoretical_loss": 3.3308898115376646, + "tokens_seen": 3225747456 + }, + { + "epoch": 0.98, + "learning_rate": 1.8407221664994985e-05, + "loss": 1.2124, + "theoretical_loss": 3.3308847286542, + "tokens_seen": 3225812992 + }, + { + "epoch": 0.98, + "learning_rate": 1.8391173520561686e-05, + "loss": 1.2616, + "theoretical_loss": 3.3308796459029124, + "tokens_seen": 3225878528 + }, + { + "epoch": 0.98, + "learning_rate": 1.8375125376128387e-05, + "loss": 1.2084, + "theoretical_loss": 3.3308745632837953, + "tokens_seen": 3225944064 + }, + { + "epoch": 0.98, + "learning_rate": 1.8359077231695084e-05, + "loss": 1.0972, + "theoretical_loss": 3.330869480796843, + "tokens_seen": 3226009600 + }, + { + "epoch": 0.98, + "learning_rate": 1.8343029087261788e-05, + "loss": 1.2037, + "theoretical_loss": 3.330864398442049, + "tokens_seen": 3226075136 + }, + { + "epoch": 0.98, + "learning_rate": 1.8326980942828485e-05, + "loss": 1.1061, + "theoretical_loss": 3.3308593162194073, + "tokens_seen": 3226140672 + }, + { + "epoch": 0.98, + "learning_rate": 1.8310932798395186e-05, + "loss": 1.1907, + "theoretical_loss": 3.330854234128912, + "tokens_seen": 3226206208 + }, + { + "epoch": 0.98, + "learning_rate": 1.8294884653961886e-05, + "loss": 1.1958, + "theoretical_loss": 3.3308491521705563, + "tokens_seen": 3226271744 + }, + { + "epoch": 0.98, + "learning_rate": 1.8278836509528587e-05, + "loss": 1.2154, + "theoretical_loss": 3.3308440703443347, + "tokens_seen": 3226337280 + }, + { + "epoch": 0.98, + "learning_rate": 1.8262788365095288e-05, + "loss": 1.0917, + "theoretical_loss": 3.330838988650241, + "tokens_seen": 3226402816 + }, + { + "epoch": 0.98, + "learning_rate": 1.824674022066199e-05, + "loss": 1.1432, + "theoretical_loss": 3.3308339070882695, + "tokens_seen": 3226468352 + }, + { + "epoch": 0.98, + "learning_rate": 1.8230692076228686e-05, + "loss": 1.1469, + "theoretical_loss": 3.330828825658413, + "tokens_seen": 3226533888 + }, + { + "epoch": 0.98, + "learning_rate": 1.8214643931795386e-05, + "loss": 1.3128, + "theoretical_loss": 3.330823744360666, + "tokens_seen": 3226599424 + }, + { + "epoch": 0.98, + "learning_rate": 1.8198595787362087e-05, + "loss": 1.1507, + "theoretical_loss": 3.3308186631950227, + "tokens_seen": 3226664960 + }, + { + "epoch": 0.98, + "learning_rate": 1.8182547642928788e-05, + "loss": 1.156, + "theoretical_loss": 3.3308135821614764, + "tokens_seen": 3226730496 + }, + { + "epoch": 0.98, + "learning_rate": 1.8166499498495488e-05, + "loss": 1.1115, + "theoretical_loss": 3.3308085012600213, + "tokens_seen": 3226796032 + }, + { + "epoch": 0.98, + "learning_rate": 1.8150451354062185e-05, + "loss": 1.1647, + "theoretical_loss": 3.330803420490651, + "tokens_seen": 3226861568 + }, + { + "epoch": 0.98, + "learning_rate": 1.8134403209628886e-05, + "loss": 1.2215, + "theoretical_loss": 3.33079833985336, + "tokens_seen": 3226927104 + }, + { + "epoch": 0.98, + "learning_rate": 1.811835506519559e-05, + "loss": 1.236, + "theoretical_loss": 3.3307932593481415, + "tokens_seen": 3226992640 + }, + { + "epoch": 0.98, + "learning_rate": 1.8102306920762287e-05, + "loss": 1.2578, + "theoretical_loss": 3.3307881789749896, + "tokens_seen": 3227058176 + }, + { + "epoch": 0.98, + "learning_rate": 1.8086258776328988e-05, + "loss": 1.2312, + "theoretical_loss": 3.330783098733898, + "tokens_seen": 3227123712 + }, + { + "epoch": 0.98, + "learning_rate": 1.807021063189569e-05, + "loss": 1.1662, + "theoretical_loss": 3.3307780186248612, + "tokens_seen": 3227189248 + }, + { + "epoch": 0.98, + "learning_rate": 1.805416248746239e-05, + "loss": 1.1923, + "theoretical_loss": 3.330772938647873, + "tokens_seen": 3227254784 + }, + { + "epoch": 0.98, + "learning_rate": 1.803811434302909e-05, + "loss": 1.1828, + "theoretical_loss": 3.3307678588029264, + "tokens_seen": 3227320320 + }, + { + "epoch": 0.98, + "learning_rate": 1.8022066198595787e-05, + "loss": 1.1255, + "theoretical_loss": 3.3307627790900165, + "tokens_seen": 3227385856 + }, + { + "epoch": 0.98, + "learning_rate": 1.8006018054162488e-05, + "loss": 1.1461, + "theoretical_loss": 3.330757699509136, + "tokens_seen": 3227451392 + }, + { + "epoch": 0.98, + "learning_rate": 1.798996990972919e-05, + "loss": 1.0885, + "theoretical_loss": 3.3307526200602795, + "tokens_seen": 3227516928 + }, + { + "epoch": 0.98, + "learning_rate": 1.797392176529589e-05, + "loss": 1.1238, + "theoretical_loss": 3.330747540743441, + "tokens_seen": 3227582464 + }, + { + "epoch": 0.98, + "objective/train/docs_used": 1077352, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1114211082458496, + "objective/train/theoretical_loss": 3.330742461558614, + "objective/train/tokens_used": 3248108000, + "theoretical_loss": 3.330742461558614, + "tokens_seen": 3227648000 + }, + { + "epoch": 0.98, + "learning_rate": 1.795787362086259e-05, + "loss": 1.0898, + "theoretical_loss": 3.330742461558614, + "tokens_seen": 3227648000 + }, + { + "epoch": 0.98, + "learning_rate": 1.794182547642929e-05, + "loss": 1.1412, + "theoretical_loss": 3.3307373825057924, + "tokens_seen": 3227713536 + }, + { + "epoch": 0.98, + "learning_rate": 1.7925777331995988e-05, + "loss": 1.2244, + "theoretical_loss": 3.3307323035849703, + "tokens_seen": 3227779072 + }, + { + "epoch": 0.98, + "learning_rate": 1.7909729187562688e-05, + "loss": 1.1355, + "theoretical_loss": 3.3307272247961413, + "tokens_seen": 3227844608 + }, + { + "epoch": 0.98, + "learning_rate": 1.789368104312939e-05, + "loss": 1.2675, + "theoretical_loss": 3.3307221461393, + "tokens_seen": 3227910144 + }, + { + "epoch": 0.98, + "learning_rate": 1.787763289869609e-05, + "loss": 1.2949, + "theoretical_loss": 3.3307170676144393, + "tokens_seen": 3227975680 + }, + { + "epoch": 0.98, + "learning_rate": 1.786158475426279e-05, + "loss": 1.2669, + "theoretical_loss": 3.3307119892215535, + "tokens_seen": 3228041216 + }, + { + "epoch": 0.98, + "learning_rate": 1.7845536609829487e-05, + "loss": 1.3158, + "theoretical_loss": 3.330706910960637, + "tokens_seen": 3228106752 + }, + { + "epoch": 0.98, + "learning_rate": 1.782948846539619e-05, + "loss": 1.1761, + "theoretical_loss": 3.330701832831683, + "tokens_seen": 3228172288 + }, + { + "epoch": 0.98, + "learning_rate": 1.7813440320962892e-05, + "loss": 1.2283, + "theoretical_loss": 3.330696754834686, + "tokens_seen": 3228237824 + }, + { + "epoch": 0.98, + "learning_rate": 1.779739217652959e-05, + "loss": 1.2575, + "theoretical_loss": 3.3306916769696393, + "tokens_seen": 3228303360 + }, + { + "epoch": 0.98, + "learning_rate": 1.778134403209629e-05, + "loss": 1.1275, + "theoretical_loss": 3.330686599236537, + "tokens_seen": 3228368896 + }, + { + "epoch": 0.98, + "learning_rate": 1.776529588766299e-05, + "loss": 1.0972, + "theoretical_loss": 3.330681521635373, + "tokens_seen": 3228434432 + }, + { + "epoch": 0.98, + "learning_rate": 1.774924774322969e-05, + "loss": 1.1835, + "theoretical_loss": 3.330676444166141, + "tokens_seen": 3228499968 + }, + { + "epoch": 0.98, + "learning_rate": 1.7733199598796392e-05, + "loss": 1.1997, + "theoretical_loss": 3.3306713668288355, + "tokens_seen": 3228565504 + }, + { + "epoch": 0.98, + "learning_rate": 1.771715145436309e-05, + "loss": 1.1849, + "theoretical_loss": 3.3306662896234496, + "tokens_seen": 3228631040 + }, + { + "epoch": 0.98, + "learning_rate": 1.770110330992979e-05, + "loss": 1.2776, + "theoretical_loss": 3.330661212549978, + "tokens_seen": 3228696576 + }, + { + "epoch": 0.98, + "learning_rate": 1.768505516549649e-05, + "loss": 1.0703, + "theoretical_loss": 3.330656135608414, + "tokens_seen": 3228762112 + }, + { + "epoch": 0.98, + "learning_rate": 1.766900702106319e-05, + "loss": 1.1224, + "theoretical_loss": 3.330651058798752, + "tokens_seen": 3228827648 + }, + { + "epoch": 0.98, + "learning_rate": 1.765295887662989e-05, + "loss": 1.128, + "theoretical_loss": 3.330645982120985, + "tokens_seen": 3228893184 + }, + { + "epoch": 0.98, + "learning_rate": 1.763691073219659e-05, + "loss": 1.2318, + "theoretical_loss": 3.330640905575108, + "tokens_seen": 3228958720 + }, + { + "epoch": 0.98, + "learning_rate": 1.762086258776329e-05, + "loss": 1.1633, + "theoretical_loss": 3.3306358291611144, + "tokens_seen": 3229024256 + }, + { + "epoch": 0.98, + "learning_rate": 1.7604814443329993e-05, + "loss": 1.1157, + "theoretical_loss": 3.330630752878998, + "tokens_seen": 3229089792 + }, + { + "epoch": 0.98, + "learning_rate": 1.758876629889669e-05, + "loss": 1.1054, + "theoretical_loss": 3.3306256767287525, + "tokens_seen": 3229155328 + }, + { + "epoch": 0.98, + "learning_rate": 1.757271815446339e-05, + "loss": 1.0717, + "theoretical_loss": 3.3306206007103722, + "tokens_seen": 3229220864 + }, + { + "epoch": 0.98, + "learning_rate": 1.7556670010030092e-05, + "loss": 1.2618, + "theoretical_loss": 3.330615524823851, + "tokens_seen": 3229286400 + }, + { + "epoch": 0.98, + "learning_rate": 1.754062186559679e-05, + "loss": 1.2564, + "theoretical_loss": 3.330610449069183, + "tokens_seen": 3229351936 + }, + { + "epoch": 0.98, + "learning_rate": 1.7524573721163493e-05, + "loss": 1.119, + "theoretical_loss": 3.330605373446361, + "tokens_seen": 3229417472 + }, + { + "epoch": 0.98, + "learning_rate": 1.750852557673019e-05, + "loss": 1.133, + "theoretical_loss": 3.33060029795538, + "tokens_seen": 3229483008 + }, + { + "epoch": 0.98, + "learning_rate": 1.749247743229689e-05, + "loss": 1.2228, + "theoretical_loss": 3.3305952225962336, + "tokens_seen": 3229548544 + }, + { + "epoch": 0.98, + "learning_rate": 1.7476429287863592e-05, + "loss": 1.2492, + "theoretical_loss": 3.330590147368916, + "tokens_seen": 3229614080 + }, + { + "epoch": 0.98, + "learning_rate": 1.746038114343029e-05, + "loss": 1.2016, + "theoretical_loss": 3.3305850722734203, + "tokens_seen": 3229679616 + }, + { + "epoch": 0.98, + "learning_rate": 1.7444332998996993e-05, + "loss": 1.1662, + "theoretical_loss": 3.330579997309741, + "tokens_seen": 3229745152 + }, + { + "epoch": 0.98, + "learning_rate": 1.7428284854563694e-05, + "loss": 1.1673, + "theoretical_loss": 3.3305749224778722, + "tokens_seen": 3229810688 + }, + { + "epoch": 0.98, + "learning_rate": 1.741223671013039e-05, + "loss": 1.2008, + "theoretical_loss": 3.3305698477778067, + "tokens_seen": 3229876224 + }, + { + "epoch": 0.98, + "learning_rate": 1.739618856569709e-05, + "loss": 1.1857, + "theoretical_loss": 3.33056477320954, + "tokens_seen": 3229941760 + }, + { + "epoch": 0.98, + "learning_rate": 1.7380140421263792e-05, + "loss": 1.2852, + "theoretical_loss": 3.330559698773065, + "tokens_seen": 3230007296 + }, + { + "epoch": 0.98, + "learning_rate": 1.7364092276830493e-05, + "loss": 1.1784, + "theoretical_loss": 3.3305546244683755, + "tokens_seen": 3230072832 + }, + { + "epoch": 0.98, + "learning_rate": 1.7348044132397193e-05, + "loss": 1.2804, + "theoretical_loss": 3.330549550295466, + "tokens_seen": 3230138368 + }, + { + "epoch": 0.98, + "learning_rate": 1.733199598796389e-05, + "loss": 1.2225, + "theoretical_loss": 3.33054447625433, + "tokens_seen": 3230203904 + }, + { + "epoch": 0.98, + "learning_rate": 1.731594784353059e-05, + "loss": 1.3019, + "theoretical_loss": 3.3305394023449613, + "tokens_seen": 3230269440 + }, + { + "epoch": 0.98, + "learning_rate": 1.7299899699097295e-05, + "loss": 1.0283, + "theoretical_loss": 3.330534328567354, + "tokens_seen": 3230334976 + }, + { + "epoch": 0.98, + "learning_rate": 1.7283851554663993e-05, + "loss": 1.1484, + "theoretical_loss": 3.3305292549215024, + "tokens_seen": 3230400512 + }, + { + "epoch": 0.98, + "learning_rate": 1.7267803410230693e-05, + "loss": 1.2072, + "theoretical_loss": 3.3305241814074, + "tokens_seen": 3230466048 + }, + { + "epoch": 0.98, + "learning_rate": 1.7251755265797394e-05, + "loss": 1.2261, + "theoretical_loss": 3.3305191080250403, + "tokens_seen": 3230531584 + }, + { + "epoch": 0.98, + "learning_rate": 1.723570712136409e-05, + "loss": 1.3151, + "theoretical_loss": 3.3305140347744175, + "tokens_seen": 3230597120 + }, + { + "epoch": 0.98, + "learning_rate": 1.7219658976930795e-05, + "loss": 1.0863, + "theoretical_loss": 3.3305089616555263, + "tokens_seen": 3230662656 + }, + { + "epoch": 0.98, + "learning_rate": 1.7203610832497492e-05, + "loss": 1.2051, + "theoretical_loss": 3.3305038886683596, + "tokens_seen": 3230728192 + }, + { + "epoch": 0.98, + "learning_rate": 1.7187562688064193e-05, + "loss": 1.2704, + "theoretical_loss": 3.3304988158129114, + "tokens_seen": 3230793728 + }, + { + "epoch": 0.98, + "learning_rate": 1.7171514543630894e-05, + "loss": 1.2169, + "theoretical_loss": 3.3304937430891766, + "tokens_seen": 3230859264 + }, + { + "epoch": 0.98, + "objective/train/docs_used": 1078633, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1810296773910522, + "objective/train/theoretical_loss": 3.3304886704971475, + "objective/train/tokens_used": 3251384800, + "theoretical_loss": 3.3304886704971475, + "tokens_seen": 3230924800 + }, + { + "epoch": 0.98, + "learning_rate": 1.715546639919759e-05, + "loss": 1.1615, + "theoretical_loss": 3.3304886704971475, + "tokens_seen": 3230924800 + }, + { + "epoch": 0.98, + "learning_rate": 1.7139418254764295e-05, + "loss": 1.2627, + "theoretical_loss": 3.3304835980368193, + "tokens_seen": 3230990336 + }, + { + "epoch": 0.98, + "learning_rate": 1.7123370110330996e-05, + "loss": 1.2326, + "theoretical_loss": 3.3304785257081857, + "tokens_seen": 3231055872 + }, + { + "epoch": 0.98, + "learning_rate": 1.7107321965897693e-05, + "loss": 1.1132, + "theoretical_loss": 3.33047345351124, + "tokens_seen": 3231121408 + }, + { + "epoch": 0.98, + "learning_rate": 1.7091273821464393e-05, + "loss": 1.2143, + "theoretical_loss": 3.330468381445977, + "tokens_seen": 3231186944 + }, + { + "epoch": 0.98, + "learning_rate": 1.7075225677031094e-05, + "loss": 1.193, + "theoretical_loss": 3.3304633095123894, + "tokens_seen": 3231252480 + }, + { + "epoch": 0.98, + "learning_rate": 1.7059177532597795e-05, + "loss": 1.2501, + "theoretical_loss": 3.330458237710472, + "tokens_seen": 3231318016 + }, + { + "epoch": 0.98, + "learning_rate": 1.7043129388164495e-05, + "loss": 1.1734, + "theoretical_loss": 3.330453166040219, + "tokens_seen": 3231383552 + }, + { + "epoch": 0.98, + "learning_rate": 1.7027081243731193e-05, + "loss": 1.2192, + "theoretical_loss": 3.3304480945016235, + "tokens_seen": 3231449088 + }, + { + "epoch": 0.98, + "learning_rate": 1.7011033099297893e-05, + "loss": 1.1228, + "theoretical_loss": 3.3304430230946798, + "tokens_seen": 3231514624 + }, + { + "epoch": 0.98, + "learning_rate": 1.6994984954864597e-05, + "loss": 1.1601, + "theoretical_loss": 3.330437951819382, + "tokens_seen": 3231580160 + }, + { + "epoch": 0.98, + "learning_rate": 1.6978936810431294e-05, + "loss": 1.1944, + "theoretical_loss": 3.3304328806757235, + "tokens_seen": 3231645696 + }, + { + "epoch": 0.98, + "learning_rate": 1.6962888665997995e-05, + "loss": 1.1659, + "theoretical_loss": 3.3304278096636986, + "tokens_seen": 3231711232 + }, + { + "epoch": 0.98, + "learning_rate": 1.6946840521564696e-05, + "loss": 1.227, + "theoretical_loss": 3.330422738783301, + "tokens_seen": 3231776768 + }, + { + "epoch": 0.98, + "learning_rate": 1.6930792377131393e-05, + "loss": 1.1782, + "theoretical_loss": 3.330417668034525, + "tokens_seen": 3231842304 + }, + { + "epoch": 0.98, + "learning_rate": 1.6914744232698097e-05, + "loss": 1.236, + "theoretical_loss": 3.3304125974173644, + "tokens_seen": 3231907840 + }, + { + "epoch": 0.98, + "learning_rate": 1.6898696088264794e-05, + "loss": 1.2516, + "theoretical_loss": 3.3304075269318125, + "tokens_seen": 3231973376 + }, + { + "epoch": 0.98, + "learning_rate": 1.6882647943831495e-05, + "loss": 1.1801, + "theoretical_loss": 3.330402456577864, + "tokens_seen": 3232038912 + }, + { + "epoch": 0.98, + "learning_rate": 1.6866599799398196e-05, + "loss": 1.1733, + "theoretical_loss": 3.330397386355512, + "tokens_seen": 3232104448 + }, + { + "epoch": 0.98, + "learning_rate": 1.6850551654964896e-05, + "loss": 1.2493, + "theoretical_loss": 3.3303923162647515, + "tokens_seen": 3232169984 + }, + { + "epoch": 0.98, + "learning_rate": 1.6834503510531597e-05, + "loss": 1.176, + "theoretical_loss": 3.3303872463055755, + "tokens_seen": 3232235520 + }, + { + "epoch": 0.98, + "learning_rate": 1.6818455366098297e-05, + "loss": 1.0651, + "theoretical_loss": 3.330382176477978, + "tokens_seen": 3232301056 + }, + { + "epoch": 0.98, + "learning_rate": 1.6802407221664995e-05, + "loss": 1.2639, + "theoretical_loss": 3.3303771067819534, + "tokens_seen": 3232366592 + }, + { + "epoch": 0.98, + "learning_rate": 1.6786359077231695e-05, + "loss": 1.1823, + "theoretical_loss": 3.330372037217496, + "tokens_seen": 3232432128 + }, + { + "epoch": 0.98, + "learning_rate": 1.6770310932798396e-05, + "loss": 1.0497, + "theoretical_loss": 3.3303669677845984, + "tokens_seen": 3232497664 + }, + { + "epoch": 0.98, + "learning_rate": 1.6754262788365097e-05, + "loss": 1.0284, + "theoretical_loss": 3.3303618984832553, + "tokens_seen": 3232563200 + }, + { + "epoch": 0.98, + "learning_rate": 1.6738214643931797e-05, + "loss": 1.1924, + "theoretical_loss": 3.3303568293134607, + "tokens_seen": 3232628736 + }, + { + "epoch": 0.98, + "learning_rate": 1.6722166499498494e-05, + "loss": 1.2011, + "theoretical_loss": 3.3303517602752084, + "tokens_seen": 3232694272 + }, + { + "epoch": 0.98, + "learning_rate": 1.6706118355065195e-05, + "loss": 1.1739, + "theoretical_loss": 3.330346691368492, + "tokens_seen": 3232759808 + }, + { + "epoch": 0.98, + "learning_rate": 1.6690070210631896e-05, + "loss": 1.1662, + "theoretical_loss": 3.3303416225933056, + "tokens_seen": 3232825344 + }, + { + "epoch": 0.98, + "learning_rate": 1.6674022066198596e-05, + "loss": 1.1842, + "theoretical_loss": 3.3303365539496435, + "tokens_seen": 3232890880 + }, + { + "epoch": 0.98, + "learning_rate": 1.6657973921765297e-05, + "loss": 1.2001, + "theoretical_loss": 3.3303314854374992, + "tokens_seen": 3232956416 + }, + { + "epoch": 0.98, + "learning_rate": 1.6641925777331994e-05, + "loss": 1.0564, + "theoretical_loss": 3.3303264170568667, + "tokens_seen": 3233021952 + }, + { + "epoch": 0.98, + "learning_rate": 1.6625877632898698e-05, + "loss": 1.19, + "theoretical_loss": 3.33032134880774, + "tokens_seen": 3233087488 + }, + { + "epoch": 0.98, + "learning_rate": 1.66098294884654e-05, + "loss": 1.208, + "theoretical_loss": 3.330316280690113, + "tokens_seen": 3233153024 + }, + { + "epoch": 0.98, + "learning_rate": 1.6593781344032096e-05, + "loss": 1.1665, + "theoretical_loss": 3.3303112127039802, + "tokens_seen": 3233218560 + }, + { + "epoch": 0.98, + "learning_rate": 1.6577733199598797e-05, + "loss": 1.1237, + "theoretical_loss": 3.3303061448493345, + "tokens_seen": 3233284096 + }, + { + "epoch": 0.98, + "learning_rate": 1.6561685055165497e-05, + "loss": 1.2175, + "theoretical_loss": 3.33030107712617, + "tokens_seen": 3233349632 + }, + { + "epoch": 0.98, + "learning_rate": 1.6545636910732198e-05, + "loss": 1.076, + "theoretical_loss": 3.3302960095344813, + "tokens_seen": 3233415168 + }, + { + "epoch": 0.98, + "learning_rate": 1.65295887662989e-05, + "loss": 1.176, + "theoretical_loss": 3.330290942074262, + "tokens_seen": 3233480704 + }, + { + "epoch": 0.98, + "learning_rate": 1.6513540621865596e-05, + "loss": 1.1419, + "theoretical_loss": 3.3302858747455053, + "tokens_seen": 3233546240 + }, + { + "epoch": 0.98, + "learning_rate": 1.6497492477432297e-05, + "loss": 1.1566, + "theoretical_loss": 3.330280807548206, + "tokens_seen": 3233611776 + }, + { + "epoch": 0.98, + "learning_rate": 1.6481444332998997e-05, + "loss": 1.2241, + "theoretical_loss": 3.3302757404823584, + "tokens_seen": 3233677312 + }, + { + "epoch": 0.98, + "learning_rate": 1.6465396188565698e-05, + "loss": 1.1951, + "theoretical_loss": 3.330270673547955, + "tokens_seen": 3233742848 + }, + { + "epoch": 0.98, + "learning_rate": 1.64493480441324e-05, + "loss": 1.2143, + "theoretical_loss": 3.330265606744991, + "tokens_seen": 3233808384 + }, + { + "epoch": 0.98, + "learning_rate": 1.64332998996991e-05, + "loss": 1.2729, + "theoretical_loss": 3.3302605400734597, + "tokens_seen": 3233873920 + }, + { + "epoch": 0.98, + "learning_rate": 1.6417251755265796e-05, + "loss": 1.2157, + "theoretical_loss": 3.3302554735333554, + "tokens_seen": 3233939456 + }, + { + "epoch": 0.98, + "learning_rate": 1.64012036108325e-05, + "loss": 1.1604, + "theoretical_loss": 3.3302504071246717, + "tokens_seen": 3234004992 + }, + { + "epoch": 0.98, + "learning_rate": 1.6385155466399198e-05, + "loss": 1.1601, + "theoretical_loss": 3.330245340847403, + "tokens_seen": 3234070528 + }, + { + "epoch": 0.98, + "learning_rate": 1.6369107321965898e-05, + "loss": 1.18, + "theoretical_loss": 3.3302402747015423, + "tokens_seen": 3234136064 + }, + { + "epoch": 0.98, + "objective/train/docs_used": 1079431, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2391008138656616, + "objective/train/theoretical_loss": 3.3302352086870846, + "objective/train/tokens_used": 3254661600, + "theoretical_loss": 3.3302352086870846, + "tokens_seen": 3234201600 + }, + { + "epoch": 0.98, + "learning_rate": 1.63530591775326e-05, + "loss": 1.2381, + "theoretical_loss": 3.3302352086870846, + "tokens_seen": 3234201600 + }, + { + "epoch": 0.98, + "learning_rate": 1.6337011033099296e-05, + "loss": 1.2014, + "theoretical_loss": 3.330230142804023, + "tokens_seen": 3234267136 + }, + { + "epoch": 0.98, + "learning_rate": 1.6320962888666e-05, + "loss": 1.1495, + "theoretical_loss": 3.3302250770523516, + "tokens_seen": 3234332672 + }, + { + "epoch": 0.98, + "learning_rate": 1.63049147442327e-05, + "loss": 1.1776, + "theoretical_loss": 3.330220011432065, + "tokens_seen": 3234398208 + }, + { + "epoch": 0.98, + "learning_rate": 1.6288866599799398e-05, + "loss": 1.1957, + "theoretical_loss": 3.3302149459431565, + "tokens_seen": 3234463744 + }, + { + "epoch": 0.98, + "learning_rate": 1.62728184553661e-05, + "loss": 1.1162, + "theoretical_loss": 3.33020988058562, + "tokens_seen": 3234529280 + }, + { + "epoch": 0.98, + "learning_rate": 1.62567703109328e-05, + "loss": 1.0943, + "theoretical_loss": 3.3302048153594495, + "tokens_seen": 3234594816 + }, + { + "epoch": 0.98, + "learning_rate": 1.62407221664995e-05, + "loss": 1.2128, + "theoretical_loss": 3.330199750264639, + "tokens_seen": 3234660352 + }, + { + "epoch": 0.98, + "learning_rate": 1.62246740220662e-05, + "loss": 1.1237, + "theoretical_loss": 3.330194685301183, + "tokens_seen": 3234725888 + }, + { + "epoch": 0.98, + "learning_rate": 1.6208625877632898e-05, + "loss": 1.2192, + "theoretical_loss": 3.3301896204690746, + "tokens_seen": 3234791424 + }, + { + "epoch": 0.98, + "learning_rate": 1.61925777331996e-05, + "loss": 1.1338, + "theoretical_loss": 3.330184555768308, + "tokens_seen": 3234856960 + }, + { + "epoch": 0.98, + "learning_rate": 1.6176529588766302e-05, + "loss": 1.237, + "theoretical_loss": 3.330179491198877, + "tokens_seen": 3234922496 + }, + { + "epoch": 0.98, + "learning_rate": 1.6160481444333e-05, + "loss": 1.2209, + "theoretical_loss": 3.3301744267607756, + "tokens_seen": 3234988032 + }, + { + "epoch": 0.98, + "learning_rate": 1.61444332998997e-05, + "loss": 1.1291, + "theoretical_loss": 3.330169362453998, + "tokens_seen": 3235053568 + }, + { + "epoch": 0.98, + "learning_rate": 1.61283851554664e-05, + "loss": 1.2625, + "theoretical_loss": 3.330164298278538, + "tokens_seen": 3235119104 + }, + { + "epoch": 0.98, + "learning_rate": 1.6112337011033098e-05, + "loss": 1.2226, + "theoretical_loss": 3.3301592342343898, + "tokens_seen": 3235184640 + }, + { + "epoch": 0.98, + "learning_rate": 1.6096288866599802e-05, + "loss": 1.1736, + "theoretical_loss": 3.3301541703215465, + "tokens_seen": 3235250176 + }, + { + "epoch": 0.98, + "learning_rate": 1.60802407221665e-05, + "loss": 1.2402, + "theoretical_loss": 3.330149106540003, + "tokens_seen": 3235315712 + }, + { + "epoch": 0.98, + "learning_rate": 1.60641925777332e-05, + "loss": 1.1326, + "theoretical_loss": 3.3301440428897524, + "tokens_seen": 3235381248 + }, + { + "epoch": 0.98, + "learning_rate": 1.60481444332999e-05, + "loss": 1.2725, + "theoretical_loss": 3.3301389793707896, + "tokens_seen": 3235446784 + }, + { + "epoch": 0.98, + "learning_rate": 1.6032096288866598e-05, + "loss": 1.0616, + "theoretical_loss": 3.330133915983107, + "tokens_seen": 3235512320 + }, + { + "epoch": 0.98, + "learning_rate": 1.6016048144433302e-05, + "loss": 1.0584, + "theoretical_loss": 3.3301288527267006, + "tokens_seen": 3235577856 + }, + { + "epoch": 0.98, + "learning_rate": 1.6000000000000003e-05, + "loss": 1.2256, + "theoretical_loss": 3.3301237896015627, + "tokens_seen": 3235643392 + }, + { + "epoch": 0.98, + "learning_rate": 1.59839518555667e-05, + "loss": 1.1727, + "theoretical_loss": 3.3301187266076875, + "tokens_seen": 3235708928 + }, + { + "epoch": 0.98, + "learning_rate": 1.59679037111334e-05, + "loss": 1.2321, + "theoretical_loss": 3.33011366374507, + "tokens_seen": 3235774464 + }, + { + "epoch": 0.98, + "learning_rate": 1.59518555667001e-05, + "loss": 1.1535, + "theoretical_loss": 3.330108601013703, + "tokens_seen": 3235840000 + }, + { + "epoch": 0.98, + "learning_rate": 1.5935807422266802e-05, + "loss": 1.1925, + "theoretical_loss": 3.3301035384135806, + "tokens_seen": 3235905536 + }, + { + "epoch": 0.98, + "learning_rate": 1.5919759277833502e-05, + "loss": 1.283, + "theoretical_loss": 3.330098475944697, + "tokens_seen": 3235971072 + }, + { + "epoch": 0.98, + "learning_rate": 1.59037111334002e-05, + "loss": 1.3325, + "theoretical_loss": 3.330093413607046, + "tokens_seen": 3236036608 + }, + { + "epoch": 0.98, + "learning_rate": 1.58876629889669e-05, + "loss": 1.1167, + "theoretical_loss": 3.330088351400622, + "tokens_seen": 3236102144 + }, + { + "epoch": 0.98, + "learning_rate": 1.5871614844533604e-05, + "loss": 1.1679, + "theoretical_loss": 3.3300832893254184, + "tokens_seen": 3236167680 + }, + { + "epoch": 0.98, + "learning_rate": 1.58555667001003e-05, + "loss": 1.1456, + "theoretical_loss": 3.330078227381429, + "tokens_seen": 3236233216 + }, + { + "epoch": 0.98, + "learning_rate": 1.5839518555667002e-05, + "loss": 1.1403, + "theoretical_loss": 3.330073165568648, + "tokens_seen": 3236298752 + }, + { + "epoch": 0.98, + "learning_rate": 1.5823470411233703e-05, + "loss": 1.2511, + "theoretical_loss": 3.33006810388707, + "tokens_seen": 3236364288 + }, + { + "epoch": 0.98, + "learning_rate": 1.58074222668004e-05, + "loss": 1.1133, + "theoretical_loss": 3.330063042336688, + "tokens_seen": 3236429824 + }, + { + "epoch": 0.98, + "learning_rate": 1.5791374122367104e-05, + "loss": 1.0995, + "theoretical_loss": 3.3300579809174966, + "tokens_seen": 3236495360 + }, + { + "epoch": 0.98, + "learning_rate": 1.57753259779338e-05, + "loss": 1.2544, + "theoretical_loss": 3.330052919629489, + "tokens_seen": 3236560896 + }, + { + "epoch": 0.98, + "learning_rate": 1.5759277833500502e-05, + "loss": 1.2113, + "theoretical_loss": 3.3300478584726596, + "tokens_seen": 3236626432 + }, + { + "epoch": 0.98, + "learning_rate": 1.5743229689067203e-05, + "loss": 1.1202, + "theoretical_loss": 3.3300427974470024, + "tokens_seen": 3236691968 + }, + { + "epoch": 0.98, + "learning_rate": 1.5727181544633903e-05, + "loss": 1.2469, + "theoretical_loss": 3.3300377365525113, + "tokens_seen": 3236757504 + }, + { + "epoch": 0.98, + "learning_rate": 1.5711133400200604e-05, + "loss": 1.2133, + "theoretical_loss": 3.3300326757891803, + "tokens_seen": 3236823040 + }, + { + "epoch": 0.98, + "learning_rate": 1.56950852557673e-05, + "loss": 1.1938, + "theoretical_loss": 3.330027615157003, + "tokens_seen": 3236888576 + }, + { + "epoch": 0.98, + "learning_rate": 1.5679037111334002e-05, + "loss": 1.1765, + "theoretical_loss": 3.330022554655973, + "tokens_seen": 3236954112 + }, + { + "epoch": 0.98, + "learning_rate": 1.5662988966900702e-05, + "loss": 1.2282, + "theoretical_loss": 3.3300174942860856, + "tokens_seen": 3237019648 + }, + { + "epoch": 0.98, + "learning_rate": 1.5646940822467403e-05, + "loss": 1.1697, + "theoretical_loss": 3.330012434047334, + "tokens_seen": 3237085184 + }, + { + "epoch": 0.98, + "learning_rate": 1.5630892678034104e-05, + "loss": 1.1486, + "theoretical_loss": 3.3300073739397122, + "tokens_seen": 3237150720 + }, + { + "epoch": 0.98, + "learning_rate": 1.5614844533600804e-05, + "loss": 1.0827, + "theoretical_loss": 3.330002313963214, + "tokens_seen": 3237216256 + }, + { + "epoch": 0.98, + "learning_rate": 1.55987963891675e-05, + "loss": 1.1746, + "theoretical_loss": 3.329997254117833, + "tokens_seen": 3237281792 + }, + { + "epoch": 0.98, + "learning_rate": 1.5582748244734202e-05, + "loss": 1.166, + "theoretical_loss": 3.3299921944035638, + "tokens_seen": 3237347328 + }, + { + "epoch": 0.98, + "learning_rate": 1.5566700100300903e-05, + "loss": 1.2211, + "theoretical_loss": 3.3299871348204, + "tokens_seen": 3237412864 + }, + { + "epoch": 0.98, + "objective/train/docs_used": 1080699, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.143287181854248, + "objective/train/theoretical_loss": 3.3299820753683362, + "objective/train/tokens_used": 3257938400, + "theoretical_loss": 3.3299820753683362, + "tokens_seen": 3237478400 + }, + { + "epoch": 0.98, + "learning_rate": 1.5550651955867603e-05, + "loss": 1.1193, + "theoretical_loss": 3.3299820753683362, + "tokens_seen": 3237478400 + }, + { + "epoch": 0.98, + "learning_rate": 1.5534603811434304e-05, + "loss": 1.2241, + "theoretical_loss": 3.3299770160473656, + "tokens_seen": 3237543936 + }, + { + "epoch": 0.98, + "learning_rate": 1.5518555667001e-05, + "loss": 1.2458, + "theoretical_loss": 3.329971956857482, + "tokens_seen": 3237609472 + }, + { + "epoch": 0.98, + "learning_rate": 1.5502507522567705e-05, + "loss": 1.1649, + "theoretical_loss": 3.32996689779868, + "tokens_seen": 3237675008 + }, + { + "epoch": 0.98, + "learning_rate": 1.5486459378134406e-05, + "loss": 1.1278, + "theoretical_loss": 3.329961838870953, + "tokens_seen": 3237740544 + }, + { + "epoch": 0.98, + "learning_rate": 1.5470411233701103e-05, + "loss": 1.1891, + "theoretical_loss": 3.3299567800742955, + "tokens_seen": 3237806080 + }, + { + "epoch": 0.98, + "learning_rate": 1.5454363089267804e-05, + "loss": 1.1547, + "theoretical_loss": 3.329951721408701, + "tokens_seen": 3237871616 + }, + { + "epoch": 0.98, + "learning_rate": 1.5438314944834505e-05, + "loss": 1.1823, + "theoretical_loss": 3.329946662874164, + "tokens_seen": 3237937152 + }, + { + "epoch": 0.98, + "learning_rate": 1.5422266800401205e-05, + "loss": 1.2147, + "theoretical_loss": 3.3299416044706778, + "tokens_seen": 3238002688 + }, + { + "epoch": 0.98, + "learning_rate": 1.5406218655967906e-05, + "loss": 1.2097, + "theoretical_loss": 3.3299365461982364, + "tokens_seen": 3238068224 + }, + { + "epoch": 0.98, + "learning_rate": 1.5390170511534603e-05, + "loss": 1.1056, + "theoretical_loss": 3.3299314880568343, + "tokens_seen": 3238133760 + }, + { + "epoch": 0.98, + "learning_rate": 1.5374122367101304e-05, + "loss": 1.2497, + "theoretical_loss": 3.329926430046465, + "tokens_seen": 3238199296 + }, + { + "epoch": 0.98, + "learning_rate": 1.5358074222668004e-05, + "loss": 1.2341, + "theoretical_loss": 3.3299213721671226, + "tokens_seen": 3238264832 + }, + { + "epoch": 0.98, + "learning_rate": 1.5342026078234705e-05, + "loss": 1.1362, + "theoretical_loss": 3.329916314418801, + "tokens_seen": 3238330368 + }, + { + "epoch": 0.98, + "learning_rate": 1.5325977933801406e-05, + "loss": 1.2464, + "theoretical_loss": 3.329911256801494, + "tokens_seen": 3238395904 + }, + { + "epoch": 0.98, + "learning_rate": 1.5309929789368106e-05, + "loss": 1.1535, + "theoretical_loss": 3.329906199315196, + "tokens_seen": 3238461440 + }, + { + "epoch": 0.98, + "learning_rate": 1.5293881644934803e-05, + "loss": 1.2232, + "theoretical_loss": 3.3299011419599007, + "tokens_seen": 3238526976 + }, + { + "epoch": 0.98, + "learning_rate": 1.5277833500501504e-05, + "loss": 1.1939, + "theoretical_loss": 3.3298960847356023, + "tokens_seen": 3238592512 + }, + { + "epoch": 0.98, + "learning_rate": 1.5261785356068205e-05, + "loss": 1.2261, + "theoretical_loss": 3.329891027642294, + "tokens_seen": 3238658048 + }, + { + "epoch": 0.98, + "learning_rate": 1.5245737211634905e-05, + "loss": 1.1969, + "theoretical_loss": 3.3298859706799706, + "tokens_seen": 3238723584 + }, + { + "epoch": 0.98, + "learning_rate": 1.5229689067201604e-05, + "loss": 1.2457, + "theoretical_loss": 3.329880913848626, + "tokens_seen": 3238789120 + }, + { + "epoch": 0.98, + "learning_rate": 1.5213640922768305e-05, + "loss": 1.0771, + "theoretical_loss": 3.329875857148253, + "tokens_seen": 3238854656 + }, + { + "epoch": 0.98, + "learning_rate": 1.5197592778335007e-05, + "loss": 1.1948, + "theoretical_loss": 3.3298708005788473, + "tokens_seen": 3238920192 + }, + { + "epoch": 0.98, + "learning_rate": 1.5181544633901706e-05, + "loss": 1.1125, + "theoretical_loss": 3.3298657441404016, + "tokens_seen": 3238985728 + }, + { + "epoch": 0.98, + "learning_rate": 1.5165496489468405e-05, + "loss": 1.1652, + "theoretical_loss": 3.3298606878329107, + "tokens_seen": 3239051264 + }, + { + "epoch": 0.98, + "learning_rate": 1.5149448345035106e-05, + "loss": 1.2248, + "theoretical_loss": 3.3298556316563674, + "tokens_seen": 3239116800 + }, + { + "epoch": 0.98, + "learning_rate": 1.5133400200601805e-05, + "loss": 1.1443, + "theoretical_loss": 3.3298505756107666, + "tokens_seen": 3239182336 + }, + { + "epoch": 0.98, + "learning_rate": 1.5117352056168507e-05, + "loss": 1.2437, + "theoretical_loss": 3.3298455196961028, + "tokens_seen": 3239247872 + }, + { + "epoch": 0.98, + "learning_rate": 1.5101303911735206e-05, + "loss": 1.2747, + "theoretical_loss": 3.3298404639123684, + "tokens_seen": 3239313408 + }, + { + "epoch": 0.98, + "learning_rate": 1.5085255767301907e-05, + "loss": 1.1191, + "theoretical_loss": 3.3298354082595583, + "tokens_seen": 3239378944 + }, + { + "epoch": 0.98, + "learning_rate": 1.5069207622868606e-05, + "loss": 1.2318, + "theoretical_loss": 3.3298303527376665, + "tokens_seen": 3239444480 + }, + { + "epoch": 0.98, + "learning_rate": 1.5053159478435305e-05, + "loss": 1.2537, + "theoretical_loss": 3.3298252973466864, + "tokens_seen": 3239510016 + }, + { + "epoch": 0.98, + "learning_rate": 1.5037111334002007e-05, + "loss": 1.1439, + "theoretical_loss": 3.3298202420866128, + "tokens_seen": 3239575552 + }, + { + "epoch": 0.98, + "learning_rate": 1.5021063189568707e-05, + "loss": 1.2069, + "theoretical_loss": 3.329815186957439, + "tokens_seen": 3239641088 + }, + { + "epoch": 0.98, + "learning_rate": 1.5005015045135406e-05, + "loss": 1.2612, + "theoretical_loss": 3.3298101319591593, + "tokens_seen": 3239706624 + }, + { + "epoch": 0.98, + "learning_rate": 1.4988966900702105e-05, + "loss": 1.1707, + "theoretical_loss": 3.329805077091767, + "tokens_seen": 3239772160 + }, + { + "epoch": 0.98, + "learning_rate": 1.4972918756268808e-05, + "loss": 1.1754, + "theoretical_loss": 3.3298000223552573, + "tokens_seen": 3239837696 + }, + { + "epoch": 0.98, + "learning_rate": 1.4956870611835508e-05, + "loss": 1.1494, + "theoretical_loss": 3.3297949677496232, + "tokens_seen": 3239903232 + }, + { + "epoch": 0.98, + "learning_rate": 1.4940822467402207e-05, + "loss": 1.1514, + "theoretical_loss": 3.3297899132748587, + "tokens_seen": 3239968768 + }, + { + "epoch": 0.98, + "learning_rate": 1.4924774322968906e-05, + "loss": 1.167, + "theoretical_loss": 3.3297848589309584, + "tokens_seen": 3240034304 + }, + { + "epoch": 0.98, + "learning_rate": 1.4908726178535607e-05, + "loss": 1.2847, + "theoretical_loss": 3.3297798047179157, + "tokens_seen": 3240099840 + }, + { + "epoch": 0.98, + "learning_rate": 1.489267803410231e-05, + "loss": 1.1082, + "theoretical_loss": 3.3297747506357247, + "tokens_seen": 3240165376 + }, + { + "epoch": 0.98, + "learning_rate": 1.4876629889669008e-05, + "loss": 1.2316, + "theoretical_loss": 3.3297696966843793, + "tokens_seen": 3240230912 + }, + { + "epoch": 0.98, + "learning_rate": 1.4860581745235707e-05, + "loss": 1.2073, + "theoretical_loss": 3.329764642863873, + "tokens_seen": 3240296448 + }, + { + "epoch": 0.98, + "learning_rate": 1.4844533600802408e-05, + "loss": 1.1819, + "theoretical_loss": 3.329759589174201, + "tokens_seen": 3240361984 + }, + { + "epoch": 0.98, + "learning_rate": 1.4828485456369107e-05, + "loss": 1.1467, + "theoretical_loss": 3.329754535615357, + "tokens_seen": 3240427520 + }, + { + "epoch": 0.98, + "learning_rate": 1.4812437311935809e-05, + "loss": 1.2522, + "theoretical_loss": 3.3297494821873337, + "tokens_seen": 3240493056 + }, + { + "epoch": 0.98, + "learning_rate": 1.4796389167502508e-05, + "loss": 1.2239, + "theoretical_loss": 3.3297444288901263, + "tokens_seen": 3240558592 + }, + { + "epoch": 0.98, + "learning_rate": 1.4780341023069209e-05, + "loss": 1.1336, + "theoretical_loss": 3.3297393757237286, + "tokens_seen": 3240624128 + }, + { + "epoch": 0.98, + "learning_rate": 1.4764292878635907e-05, + "loss": 1.2148, + "theoretical_loss": 3.3297343226881337, + "tokens_seen": 3240689664 + }, + { + "epoch": 0.98, + "objective/train/docs_used": 1081612, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1601338386535645, + "objective/train/theoretical_loss": 3.329729269783337, + "objective/train/tokens_used": 3261215200, + "theoretical_loss": 3.329729269783337, + "tokens_seen": 3240755200 + }, + { + "epoch": 0.98, + "learning_rate": 1.474824473420261e-05, + "loss": 1.1883, + "theoretical_loss": 3.329729269783337, + "tokens_seen": 3240755200 + }, + { + "epoch": 0.98, + "learning_rate": 1.4732196589769309e-05, + "loss": 1.1632, + "theoretical_loss": 3.329724217009331, + "tokens_seen": 3240820736 + }, + { + "epoch": 0.98, + "learning_rate": 1.471614844533601e-05, + "loss": 1.1828, + "theoretical_loss": 3.329719164366111, + "tokens_seen": 3240886272 + }, + { + "epoch": 0.98, + "learning_rate": 1.4700100300902708e-05, + "loss": 1.1332, + "theoretical_loss": 3.32971411185367, + "tokens_seen": 3240951808 + }, + { + "epoch": 0.98, + "learning_rate": 1.4684052156469407e-05, + "loss": 1.2377, + "theoretical_loss": 3.3297090594720022, + "tokens_seen": 3241017344 + }, + { + "epoch": 0.98, + "learning_rate": 1.466800401203611e-05, + "loss": 1.1641, + "theoretical_loss": 3.3297040072211015, + "tokens_seen": 3241082880 + }, + { + "epoch": 0.98, + "learning_rate": 1.465195586760281e-05, + "loss": 1.1602, + "theoretical_loss": 3.3296989551009624, + "tokens_seen": 3241148416 + }, + { + "epoch": 0.98, + "learning_rate": 1.463590772316951e-05, + "loss": 1.2521, + "theoretical_loss": 3.3296939031115786, + "tokens_seen": 3241213952 + }, + { + "epoch": 0.98, + "learning_rate": 1.4619859578736208e-05, + "loss": 1.1434, + "theoretical_loss": 3.3296888512529437, + "tokens_seen": 3241279488 + }, + { + "epoch": 0.98, + "learning_rate": 1.4603811434302909e-05, + "loss": 1.216, + "theoretical_loss": 3.329683799525052, + "tokens_seen": 3241345024 + }, + { + "epoch": 0.98, + "learning_rate": 1.4587763289869611e-05, + "loss": 1.2221, + "theoretical_loss": 3.3296787479278978, + "tokens_seen": 3241410560 + }, + { + "epoch": 0.98, + "learning_rate": 1.457171514543631e-05, + "loss": 1.2389, + "theoretical_loss": 3.3296736964614744, + "tokens_seen": 3241476096 + }, + { + "epoch": 0.98, + "learning_rate": 1.4555667001003009e-05, + "loss": 1.1412, + "theoretical_loss": 3.329668645125776, + "tokens_seen": 3241541632 + }, + { + "epoch": 0.98, + "learning_rate": 1.453961885656971e-05, + "loss": 1.3452, + "theoretical_loss": 3.329663593920797, + "tokens_seen": 3241607168 + }, + { + "epoch": 0.98, + "learning_rate": 1.4523570712136412e-05, + "loss": 1.2292, + "theoretical_loss": 3.3296585428465306, + "tokens_seen": 3241672704 + }, + { + "epoch": 0.98, + "learning_rate": 1.4507522567703111e-05, + "loss": 1.2149, + "theoretical_loss": 3.3296534919029717, + "tokens_seen": 3241738240 + }, + { + "epoch": 0.98, + "learning_rate": 1.449147442326981e-05, + "loss": 1.1181, + "theoretical_loss": 3.3296484410901135, + "tokens_seen": 3241803776 + }, + { + "epoch": 0.98, + "learning_rate": 1.447542627883651e-05, + "loss": 1.1616, + "theoretical_loss": 3.3296433904079503, + "tokens_seen": 3241869312 + }, + { + "epoch": 0.98, + "learning_rate": 1.445937813440321e-05, + "loss": 1.1542, + "theoretical_loss": 3.329638339856476, + "tokens_seen": 3241934848 + }, + { + "epoch": 0.98, + "learning_rate": 1.4443329989969912e-05, + "loss": 1.2235, + "theoretical_loss": 3.3296332894356846, + "tokens_seen": 3242000384 + }, + { + "epoch": 0.98, + "learning_rate": 1.442728184553661e-05, + "loss": 1.1549, + "theoretical_loss": 3.32962823914557, + "tokens_seen": 3242065920 + }, + { + "epoch": 0.98, + "learning_rate": 1.4411233701103311e-05, + "loss": 1.2399, + "theoretical_loss": 3.3296231889861265, + "tokens_seen": 3242131456 + }, + { + "epoch": 0.98, + "learning_rate": 1.439518555667001e-05, + "loss": 1.1006, + "theoretical_loss": 3.329618138957348, + "tokens_seen": 3242196992 + }, + { + "epoch": 0.98, + "learning_rate": 1.437913741223671e-05, + "loss": 1.1284, + "theoretical_loss": 3.3296130890592277, + "tokens_seen": 3242262528 + }, + { + "epoch": 0.98, + "learning_rate": 1.4363089267803411e-05, + "loss": 1.1592, + "theoretical_loss": 3.3296080392917604, + "tokens_seen": 3242328064 + }, + { + "epoch": 0.98, + "learning_rate": 1.4347041123370112e-05, + "loss": 1.1542, + "theoretical_loss": 3.32960298965494, + "tokens_seen": 3242393600 + }, + { + "epoch": 0.98, + "learning_rate": 1.4330992978936811e-05, + "loss": 1.0883, + "theoretical_loss": 3.3295979401487608, + "tokens_seen": 3242459136 + }, + { + "epoch": 0.98, + "learning_rate": 1.431494483450351e-05, + "loss": 1.1687, + "theoretical_loss": 3.329592890773216, + "tokens_seen": 3242524672 + }, + { + "epoch": 0.98, + "learning_rate": 1.4298896690070212e-05, + "loss": 1.1999, + "theoretical_loss": 3.3295878415282996, + "tokens_seen": 3242590208 + }, + { + "epoch": 0.98, + "learning_rate": 1.4282848545636911e-05, + "loss": 1.1277, + "theoretical_loss": 3.3295827924140062, + "tokens_seen": 3242655744 + }, + { + "epoch": 0.98, + "learning_rate": 1.4266800401203612e-05, + "loss": 1.1316, + "theoretical_loss": 3.3295777434303293, + "tokens_seen": 3242721280 + }, + { + "epoch": 0.98, + "learning_rate": 1.425075225677031e-05, + "loss": 1.1556, + "theoretical_loss": 3.329572694577263, + "tokens_seen": 3242786816 + }, + { + "epoch": 0.98, + "learning_rate": 1.423470411233701e-05, + "loss": 1.1725, + "theoretical_loss": 3.3295676458548016, + "tokens_seen": 3242852352 + }, + { + "epoch": 0.98, + "learning_rate": 1.4218655967903712e-05, + "loss": 1.2945, + "theoretical_loss": 3.329562597262939, + "tokens_seen": 3242917888 + }, + { + "epoch": 0.98, + "learning_rate": 1.4202607823470413e-05, + "loss": 1.1386, + "theoretical_loss": 3.3295575488016684, + "tokens_seen": 3242983424 + }, + { + "epoch": 0.98, + "learning_rate": 1.4186559679037112e-05, + "loss": 1.1521, + "theoretical_loss": 3.329552500470985, + "tokens_seen": 3243048960 + }, + { + "epoch": 0.98, + "learning_rate": 1.417051153460381e-05, + "loss": 1.1514, + "theoretical_loss": 3.329547452270882, + "tokens_seen": 3243114496 + }, + { + "epoch": 0.98, + "learning_rate": 1.4154463390170511e-05, + "loss": 1.1939, + "theoretical_loss": 3.3295424042013533, + "tokens_seen": 3243180032 + }, + { + "epoch": 0.98, + "learning_rate": 1.4138415245737214e-05, + "loss": 1.124, + "theoretical_loss": 3.329537356262393, + "tokens_seen": 3243245568 + }, + { + "epoch": 0.98, + "learning_rate": 1.4122367101303913e-05, + "loss": 1.3108, + "theoretical_loss": 3.329532308453996, + "tokens_seen": 3243311104 + }, + { + "epoch": 0.98, + "learning_rate": 1.4106318956870611e-05, + "loss": 1.1568, + "theoretical_loss": 3.329527260776155, + "tokens_seen": 3243376640 + }, + { + "epoch": 0.98, + "learning_rate": 1.4090270812437312e-05, + "loss": 1.1385, + "theoretical_loss": 3.3295222132288647, + "tokens_seen": 3243442176 + }, + { + "epoch": 0.98, + "learning_rate": 1.4074222668004014e-05, + "loss": 1.2607, + "theoretical_loss": 3.3295171658121188, + "tokens_seen": 3243507712 + }, + { + "epoch": 0.98, + "learning_rate": 1.4058174523570713e-05, + "loss": 1.175, + "theoretical_loss": 3.329512118525911, + "tokens_seen": 3243573248 + }, + { + "epoch": 0.98, + "learning_rate": 1.4042126379137412e-05, + "loss": 1.2568, + "theoretical_loss": 3.329507071370236, + "tokens_seen": 3243638784 + }, + { + "epoch": 0.98, + "learning_rate": 1.4026078234704113e-05, + "loss": 1.2686, + "theoretical_loss": 3.3295020243450875, + "tokens_seen": 3243704320 + }, + { + "epoch": 0.98, + "learning_rate": 1.4010030090270812e-05, + "loss": 1.1749, + "theoretical_loss": 3.329496977450459, + "tokens_seen": 3243769856 + }, + { + "epoch": 0.98, + "learning_rate": 1.3993981945837514e-05, + "loss": 1.1753, + "theoretical_loss": 3.3294919306863457, + "tokens_seen": 3243835392 + }, + { + "epoch": 0.98, + "learning_rate": 1.3977933801404213e-05, + "loss": 1.2269, + "theoretical_loss": 3.32948688405274, + "tokens_seen": 3243900928 + }, + { + "epoch": 0.98, + "learning_rate": 1.3961885656970914e-05, + "loss": 1.2136, + "theoretical_loss": 3.329481837549637, + "tokens_seen": 3243966464 + }, + { + "epoch": 0.98, + "objective/train/docs_used": 1082941, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1789045333862305, + "objective/train/theoretical_loss": 3.3294767911770307, + "objective/train/tokens_used": 3264492000, + "theoretical_loss": 3.3294767911770307, + "tokens_seen": 3244032000 + }, + { + "epoch": 0.98, + "learning_rate": 1.3945837512537613e-05, + "loss": 1.2108, + "theoretical_loss": 3.3294767911770307, + "tokens_seen": 3244032000 + }, + { + "epoch": 0.98, + "learning_rate": 1.3929789368104312e-05, + "loss": 1.133, + "theoretical_loss": 3.3294717449349145, + "tokens_seen": 3244097536 + }, + { + "epoch": 0.98, + "learning_rate": 1.3913741223671014e-05, + "loss": 1.2215, + "theoretical_loss": 3.329466698823283, + "tokens_seen": 3244163072 + }, + { + "epoch": 0.98, + "learning_rate": 1.3897693079237715e-05, + "loss": 1.2144, + "theoretical_loss": 3.329461652842129, + "tokens_seen": 3244228608 + }, + { + "epoch": 0.98, + "learning_rate": 1.3881644934804414e-05, + "loss": 1.1175, + "theoretical_loss": 3.3294566069914477, + "tokens_seen": 3244294144 + }, + { + "epoch": 0.98, + "learning_rate": 1.3865596790371113e-05, + "loss": 1.2214, + "theoretical_loss": 3.329451561271233, + "tokens_seen": 3244359680 + }, + { + "epoch": 0.98, + "learning_rate": 1.3849548645937815e-05, + "loss": 1.178, + "theoretical_loss": 3.3294465156814788, + "tokens_seen": 3244425216 + }, + { + "epoch": 0.98, + "learning_rate": 1.3833500501504515e-05, + "loss": 1.2108, + "theoretical_loss": 3.3294414702221786, + "tokens_seen": 3244490752 + }, + { + "epoch": 0.98, + "learning_rate": 1.3817452357071214e-05, + "loss": 1.2737, + "theoretical_loss": 3.3294364248933266, + "tokens_seen": 3244556288 + }, + { + "epoch": 0.98, + "learning_rate": 1.3801404212637913e-05, + "loss": 1.1672, + "theoretical_loss": 3.3294313796949173, + "tokens_seen": 3244621824 + }, + { + "epoch": 0.98, + "learning_rate": 1.3785356068204614e-05, + "loss": 1.1414, + "theoretical_loss": 3.3294263346269437, + "tokens_seen": 3244687360 + }, + { + "epoch": 0.98, + "learning_rate": 1.3769307923771316e-05, + "loss": 1.1769, + "theoretical_loss": 3.329421289689401, + "tokens_seen": 3244752896 + }, + { + "epoch": 0.98, + "learning_rate": 1.3753259779338015e-05, + "loss": 1.1681, + "theoretical_loss": 3.329416244882282, + "tokens_seen": 3244818432 + }, + { + "epoch": 0.98, + "learning_rate": 1.3737211634904714e-05, + "loss": 1.1872, + "theoretical_loss": 3.329411200205582, + "tokens_seen": 3244883968 + }, + { + "epoch": 0.98, + "learning_rate": 1.3721163490471415e-05, + "loss": 1.1263, + "theoretical_loss": 3.3294061556592935, + "tokens_seen": 3244949504 + }, + { + "epoch": 0.98, + "learning_rate": 1.3705115346038114e-05, + "loss": 1.1309, + "theoretical_loss": 3.3294011112434116, + "tokens_seen": 3245015040 + }, + { + "epoch": 0.98, + "learning_rate": 1.3689067201604816e-05, + "loss": 1.2197, + "theoretical_loss": 3.32939606695793, + "tokens_seen": 3245080576 + }, + { + "epoch": 0.98, + "learning_rate": 1.3673019057171515e-05, + "loss": 1.1834, + "theoretical_loss": 3.3293910228028425, + "tokens_seen": 3245146112 + }, + { + "epoch": 0.98, + "learning_rate": 1.3656970912738216e-05, + "loss": 1.1589, + "theoretical_loss": 3.3293859787781437, + "tokens_seen": 3245211648 + }, + { + "epoch": 0.98, + "learning_rate": 1.3640922768304915e-05, + "loss": 1.1967, + "theoretical_loss": 3.329380934883827, + "tokens_seen": 3245277184 + }, + { + "epoch": 0.98, + "learning_rate": 1.3624874623871617e-05, + "loss": 1.1559, + "theoretical_loss": 3.329375891119886, + "tokens_seen": 3245342720 + }, + { + "epoch": 0.98, + "learning_rate": 1.3608826479438316e-05, + "loss": 1.2141, + "theoretical_loss": 3.3293708474863157, + "tokens_seen": 3245408256 + }, + { + "epoch": 0.98, + "learning_rate": 1.3592778335005017e-05, + "loss": 1.2128, + "theoretical_loss": 3.3293658039831096, + "tokens_seen": 3245473792 + }, + { + "epoch": 0.98, + "learning_rate": 1.3576730190571715e-05, + "loss": 1.2361, + "theoretical_loss": 3.3293607606102618, + "tokens_seen": 3245539328 + }, + { + "epoch": 0.98, + "learning_rate": 1.3560682046138414e-05, + "loss": 1.1831, + "theoretical_loss": 3.329355717367766, + "tokens_seen": 3245604864 + }, + { + "epoch": 0.98, + "learning_rate": 1.3544633901705117e-05, + "loss": 1.2224, + "theoretical_loss": 3.3293506742556165, + "tokens_seen": 3245670400 + }, + { + "epoch": 0.98, + "learning_rate": 1.3528585757271817e-05, + "loss": 1.0614, + "theoretical_loss": 3.3293456312738074, + "tokens_seen": 3245735936 + }, + { + "epoch": 0.98, + "learning_rate": 1.3512537612838516e-05, + "loss": 1.2452, + "theoretical_loss": 3.3293405884223324, + "tokens_seen": 3245801472 + }, + { + "epoch": 0.98, + "learning_rate": 1.3496489468405215e-05, + "loss": 1.2127, + "theoretical_loss": 3.329335545701186, + "tokens_seen": 3245867008 + }, + { + "epoch": 0.98, + "learning_rate": 1.3480441323971916e-05, + "loss": 1.2023, + "theoretical_loss": 3.329330503110361, + "tokens_seen": 3245932544 + }, + { + "epoch": 0.98, + "learning_rate": 1.3464393179538618e-05, + "loss": 1.1682, + "theoretical_loss": 3.3293254606498532, + "tokens_seen": 3245998080 + }, + { + "epoch": 0.98, + "learning_rate": 1.3448345035105317e-05, + "loss": 1.1186, + "theoretical_loss": 3.329320418319655, + "tokens_seen": 3246063616 + }, + { + "epoch": 0.98, + "learning_rate": 1.3432296890672016e-05, + "loss": 1.1416, + "theoretical_loss": 3.3293153761197614, + "tokens_seen": 3246129152 + }, + { + "epoch": 0.98, + "learning_rate": 1.3416248746238717e-05, + "loss": 1.0742, + "theoretical_loss": 3.3293103340501657, + "tokens_seen": 3246194688 + }, + { + "epoch": 0.98, + "learning_rate": 1.3400200601805419e-05, + "loss": 1.2247, + "theoretical_loss": 3.3293052921108623, + "tokens_seen": 3246260224 + }, + { + "epoch": 0.98, + "learning_rate": 1.3384152457372118e-05, + "loss": 1.2656, + "theoretical_loss": 3.3293002503018454, + "tokens_seen": 3246325760 + }, + { + "epoch": 0.98, + "learning_rate": 1.3368104312938817e-05, + "loss": 1.2151, + "theoretical_loss": 3.3292952086231087, + "tokens_seen": 3246391296 + }, + { + "epoch": 0.98, + "learning_rate": 1.3352056168505518e-05, + "loss": 1.2028, + "theoretical_loss": 3.329290167074646, + "tokens_seen": 3246456832 + }, + { + "epoch": 0.98, + "learning_rate": 1.3336008024072217e-05, + "loss": 1.1704, + "theoretical_loss": 3.329285125656452, + "tokens_seen": 3246522368 + }, + { + "epoch": 0.98, + "learning_rate": 1.3319959879638919e-05, + "loss": 1.2191, + "theoretical_loss": 3.3292800843685195, + "tokens_seen": 3246587904 + }, + { + "epoch": 0.98, + "learning_rate": 1.3303911735205618e-05, + "loss": 1.2069, + "theoretical_loss": 3.3292750432108438, + "tokens_seen": 3246653440 + }, + { + "epoch": 0.98, + "learning_rate": 1.3287863590772317e-05, + "loss": 1.2048, + "theoretical_loss": 3.3292700021834185, + "tokens_seen": 3246718976 + }, + { + "epoch": 0.98, + "learning_rate": 1.3271815446339017e-05, + "loss": 1.2352, + "theoretical_loss": 3.329264961286237, + "tokens_seen": 3246784512 + }, + { + "epoch": 0.98, + "learning_rate": 1.3255767301905716e-05, + "loss": 1.0531, + "theoretical_loss": 3.329259920519294, + "tokens_seen": 3246850048 + }, + { + "epoch": 0.98, + "learning_rate": 1.3239719157472419e-05, + "loss": 1.1456, + "theoretical_loss": 3.3292548798825834, + "tokens_seen": 3246915584 + }, + { + "epoch": 0.98, + "learning_rate": 1.3223671013039118e-05, + "loss": 1.2585, + "theoretical_loss": 3.329249839376099, + "tokens_seen": 3246981120 + }, + { + "epoch": 0.98, + "learning_rate": 1.3207622868605818e-05, + "loss": 1.301, + "theoretical_loss": 3.3292447989998353, + "tokens_seen": 3247046656 + }, + { + "epoch": 0.98, + "learning_rate": 1.3191574724172517e-05, + "loss": 1.0864, + "theoretical_loss": 3.3292397587537854, + "tokens_seen": 3247112192 + }, + { + "epoch": 0.98, + "learning_rate": 1.317552657973922e-05, + "loss": 1.1143, + "theoretical_loss": 3.3292347186379443, + "tokens_seen": 3247177728 + }, + { + "epoch": 0.98, + "learning_rate": 1.3159478435305918e-05, + "loss": 1.2381, + "theoretical_loss": 3.329229678652305, + "tokens_seen": 3247243264 + }, + { + "epoch": 0.98, + "objective/train/docs_used": 1084182, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1916065216064453, + "objective/train/theoretical_loss": 3.3292246387968625, + "objective/train/tokens_used": 3267768800, + "theoretical_loss": 3.3292246387968625, + "tokens_seen": 3247308800 + }, + { + "epoch": 0.98, + "learning_rate": 1.3143430290872619e-05, + "loss": 1.2144, + "theoretical_loss": 3.3292246387968625, + "tokens_seen": 3247308800 + }, + { + "epoch": 0.98, + "learning_rate": 1.3127382146439318e-05, + "loss": 1.1754, + "theoretical_loss": 3.32921959907161, + "tokens_seen": 3247374336 + }, + { + "epoch": 0.98, + "learning_rate": 1.3111334002006017e-05, + "loss": 1.1925, + "theoretical_loss": 3.329214559476542, + "tokens_seen": 3247439872 + }, + { + "epoch": 0.98, + "learning_rate": 1.309528585757272e-05, + "loss": 1.2968, + "theoretical_loss": 3.329209520011652, + "tokens_seen": 3247505408 + }, + { + "epoch": 0.98, + "learning_rate": 1.307923771313942e-05, + "loss": 1.2049, + "theoretical_loss": 3.329204480676935, + "tokens_seen": 3247570944 + }, + { + "epoch": 0.98, + "learning_rate": 1.3063189568706119e-05, + "loss": 1.2016, + "theoretical_loss": 3.329199441472384, + "tokens_seen": 3247636480 + }, + { + "epoch": 0.98, + "learning_rate": 1.3047141424272818e-05, + "loss": 1.1443, + "theoretical_loss": 3.3291944023979934, + "tokens_seen": 3247702016 + }, + { + "epoch": 0.98, + "learning_rate": 1.3031093279839518e-05, + "loss": 1.1849, + "theoretical_loss": 3.3291893634537573, + "tokens_seen": 3247767552 + }, + { + "epoch": 0.98, + "learning_rate": 1.301504513540622e-05, + "loss": 1.219, + "theoretical_loss": 3.3291843246396695, + "tokens_seen": 3247833088 + }, + { + "epoch": 0.98, + "learning_rate": 1.299899699097292e-05, + "loss": 1.2319, + "theoretical_loss": 3.3291792859557248, + "tokens_seen": 3247898624 + }, + { + "epoch": 0.98, + "learning_rate": 1.2982948846539619e-05, + "loss": 1.0988, + "theoretical_loss": 3.329174247401916, + "tokens_seen": 3247964160 + }, + { + "epoch": 0.98, + "learning_rate": 1.296690070210632e-05, + "loss": 1.1922, + "theoretical_loss": 3.329169208978237, + "tokens_seen": 3248029696 + }, + { + "epoch": 0.98, + "learning_rate": 1.2950852557673018e-05, + "loss": 1.2181, + "theoretical_loss": 3.3291641706846837, + "tokens_seen": 3248095232 + }, + { + "epoch": 0.98, + "learning_rate": 1.293480441323972e-05, + "loss": 1.144, + "theoretical_loss": 3.329159132521248, + "tokens_seen": 3248160768 + }, + { + "epoch": 0.98, + "learning_rate": 1.291875626880642e-05, + "loss": 1.1279, + "theoretical_loss": 3.329154094487925, + "tokens_seen": 3248226304 + }, + { + "epoch": 0.98, + "learning_rate": 1.290270812437312e-05, + "loss": 1.1728, + "theoretical_loss": 3.3291490565847086, + "tokens_seen": 3248291840 + }, + { + "epoch": 0.98, + "learning_rate": 1.2886659979939819e-05, + "loss": 1.239, + "theoretical_loss": 3.329144018811593, + "tokens_seen": 3248357376 + }, + { + "epoch": 0.98, + "learning_rate": 1.2870611835506521e-05, + "loss": 1.2179, + "theoretical_loss": 3.3291389811685717, + "tokens_seen": 3248422912 + }, + { + "epoch": 0.98, + "learning_rate": 1.285456369107322e-05, + "loss": 1.2004, + "theoretical_loss": 3.329133943655639, + "tokens_seen": 3248488448 + }, + { + "epoch": 0.98, + "learning_rate": 1.2838515546639921e-05, + "loss": 1.2475, + "theoretical_loss": 3.329128906272789, + "tokens_seen": 3248553984 + }, + { + "epoch": 0.98, + "learning_rate": 1.282246740220662e-05, + "loss": 1.1451, + "theoretical_loss": 3.329123869020015, + "tokens_seen": 3248619520 + }, + { + "epoch": 0.98, + "learning_rate": 1.2806419257773319e-05, + "loss": 1.1936, + "theoretical_loss": 3.3291188318973126, + "tokens_seen": 3248685056 + }, + { + "epoch": 0.98, + "learning_rate": 1.2790371113340021e-05, + "loss": 1.1388, + "theoretical_loss": 3.329113794904674, + "tokens_seen": 3248750592 + }, + { + "epoch": 0.98, + "learning_rate": 1.2774322968906722e-05, + "loss": 1.2039, + "theoretical_loss": 3.3291087580420946, + "tokens_seen": 3248816128 + }, + { + "epoch": 0.98, + "learning_rate": 1.275827482447342e-05, + "loss": 1.1641, + "theoretical_loss": 3.3291037213095676, + "tokens_seen": 3248881664 + }, + { + "epoch": 0.98, + "learning_rate": 1.274222668004012e-05, + "loss": 1.2497, + "theoretical_loss": 3.329098684707087, + "tokens_seen": 3248947200 + }, + { + "epoch": 0.98, + "learning_rate": 1.272617853560682e-05, + "loss": 1.2432, + "theoretical_loss": 3.3290936482346476, + "tokens_seen": 3249012736 + }, + { + "epoch": 0.98, + "learning_rate": 1.2710130391173523e-05, + "loss": 1.2571, + "theoretical_loss": 3.3290886118922427, + "tokens_seen": 3249078272 + }, + { + "epoch": 0.98, + "learning_rate": 1.2694082246740222e-05, + "loss": 1.266, + "theoretical_loss": 3.3290835756798667, + "tokens_seen": 3249143808 + }, + { + "epoch": 0.98, + "learning_rate": 1.267803410230692e-05, + "loss": 1.2819, + "theoretical_loss": 3.3290785395975133, + "tokens_seen": 3249209344 + }, + { + "epoch": 0.98, + "learning_rate": 1.2661985957873621e-05, + "loss": 1.1954, + "theoretical_loss": 3.3290735036451764, + "tokens_seen": 3249274880 + }, + { + "epoch": 0.98, + "learning_rate": 1.2645937813440323e-05, + "loss": 1.1751, + "theoretical_loss": 3.329068467822851, + "tokens_seen": 3249340416 + }, + { + "epoch": 0.98, + "learning_rate": 1.2629889669007022e-05, + "loss": 1.0756, + "theoretical_loss": 3.32906343213053, + "tokens_seen": 3249405952 + }, + { + "epoch": 0.98, + "learning_rate": 1.2613841524573721e-05, + "loss": 1.1494, + "theoretical_loss": 3.329058396568208, + "tokens_seen": 3249471488 + }, + { + "epoch": 0.98, + "learning_rate": 1.2597793380140422e-05, + "loss": 1.1214, + "theoretical_loss": 3.3290533611358786, + "tokens_seen": 3249537024 + }, + { + "epoch": 0.98, + "learning_rate": 1.2581745235707121e-05, + "loss": 1.1935, + "theoretical_loss": 3.3290483258335364, + "tokens_seen": 3249602560 + }, + { + "epoch": 0.98, + "learning_rate": 1.2565697091273823e-05, + "loss": 1.1819, + "theoretical_loss": 3.3290432906611747, + "tokens_seen": 3249668096 + }, + { + "epoch": 0.98, + "learning_rate": 1.2549648946840522e-05, + "loss": 1.1583, + "theoretical_loss": 3.3290382556187885, + "tokens_seen": 3249733632 + }, + { + "epoch": 0.98, + "learning_rate": 1.2533600802407223e-05, + "loss": 1.1779, + "theoretical_loss": 3.329033220706371, + "tokens_seen": 3249799168 + }, + { + "epoch": 0.98, + "learning_rate": 1.2517552657973922e-05, + "loss": 1.1787, + "theoretical_loss": 3.3290281859239164, + "tokens_seen": 3249864704 + }, + { + "epoch": 0.98, + "learning_rate": 1.250150451354062e-05, + "loss": 1.199, + "theoretical_loss": 3.329023151271419, + "tokens_seen": 3249930240 + }, + { + "epoch": 0.98, + "learning_rate": 1.2485456369107321e-05, + "loss": 1.1977, + "theoretical_loss": 3.3290181167488724, + "tokens_seen": 3249995776 + }, + { + "epoch": 0.98, + "learning_rate": 1.2469408224674024e-05, + "loss": 1.1931, + "theoretical_loss": 3.329013082356271, + "tokens_seen": 3250061312 + }, + { + "epoch": 0.98, + "learning_rate": 1.2453360080240723e-05, + "loss": 1.0525, + "theoretical_loss": 3.329008048093609, + "tokens_seen": 3250126848 + }, + { + "epoch": 0.98, + "learning_rate": 1.2437311935807423e-05, + "loss": 1.1658, + "theoretical_loss": 3.3290030139608797, + "tokens_seen": 3250192384 + }, + { + "epoch": 0.98, + "learning_rate": 1.2421263791374122e-05, + "loss": 1.0973, + "theoretical_loss": 3.3289979799580776, + "tokens_seen": 3250257920 + }, + { + "epoch": 0.98, + "learning_rate": 1.2405215646940825e-05, + "loss": 1.2189, + "theoretical_loss": 3.328992946085197, + "tokens_seen": 3250323456 + }, + { + "epoch": 0.98, + "learning_rate": 1.2389167502507523e-05, + "loss": 1.2535, + "theoretical_loss": 3.328987912342231, + "tokens_seen": 3250388992 + }, + { + "epoch": 0.98, + "learning_rate": 1.2373119358074222e-05, + "loss": 1.2101, + "theoretical_loss": 3.3289828787291746, + "tokens_seen": 3250454528 + }, + { + "epoch": 0.99, + "learning_rate": 1.2357071213640923e-05, + "loss": 1.1481, + "theoretical_loss": 3.3289778452460217, + "tokens_seen": 3250520064 + }, + { + "epoch": 0.99, + "objective/train/docs_used": 1085018, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2695276737213135, + "objective/train/theoretical_loss": 3.3289728118927657, + "objective/train/tokens_used": 3271045600, + "theoretical_loss": 3.3289728118927657, + "tokens_seen": 3250585600 + }, + { + "epoch": 0.99, + "learning_rate": 1.2341023069207624e-05, + "loss": 1.2379, + "theoretical_loss": 3.3289728118927657, + "tokens_seen": 3250585600 + }, + { + "epoch": 0.99, + "learning_rate": 1.2324974924774324e-05, + "loss": 1.2293, + "theoretical_loss": 3.328967778669401, + "tokens_seen": 3250651136 + }, + { + "epoch": 0.99, + "learning_rate": 1.2308926780341023e-05, + "loss": 1.1576, + "theoretical_loss": 3.3289627455759216, + "tokens_seen": 3250716672 + }, + { + "epoch": 0.99, + "learning_rate": 1.2292878635907724e-05, + "loss": 1.1626, + "theoretical_loss": 3.328957712612322, + "tokens_seen": 3250782208 + }, + { + "epoch": 0.99, + "learning_rate": 1.2276830491474425e-05, + "loss": 1.1615, + "theoretical_loss": 3.3289526797785953, + "tokens_seen": 3250847744 + }, + { + "epoch": 0.99, + "learning_rate": 1.2260782347041123e-05, + "loss": 1.0901, + "theoretical_loss": 3.3289476470747363, + "tokens_seen": 3250913280 + }, + { + "epoch": 0.99, + "learning_rate": 1.2244734202607824e-05, + "loss": 1.3722, + "theoretical_loss": 3.328942614500739, + "tokens_seen": 3250978816 + }, + { + "epoch": 0.99, + "learning_rate": 1.2228686058174523e-05, + "loss": 1.2198, + "theoretical_loss": 3.3289375820565965, + "tokens_seen": 3251044352 + }, + { + "epoch": 0.99, + "learning_rate": 1.2212637913741225e-05, + "loss": 1.2481, + "theoretical_loss": 3.328932549742304, + "tokens_seen": 3251109888 + }, + { + "epoch": 0.99, + "learning_rate": 1.2196589769307924e-05, + "loss": 1.1293, + "theoretical_loss": 3.328927517557855, + "tokens_seen": 3251175424 + }, + { + "epoch": 0.99, + "learning_rate": 1.2180541624874625e-05, + "loss": 1.1408, + "theoretical_loss": 3.3289224855032433, + "tokens_seen": 3251240960 + }, + { + "epoch": 0.99, + "learning_rate": 1.2164493480441324e-05, + "loss": 1.2177, + "theoretical_loss": 3.328917453578464, + "tokens_seen": 3251306496 + }, + { + "epoch": 0.99, + "learning_rate": 1.2148445336008025e-05, + "loss": 1.144, + "theoretical_loss": 3.3289124217835093, + "tokens_seen": 3251372032 + }, + { + "epoch": 0.99, + "learning_rate": 1.2132397191574725e-05, + "loss": 1.2306, + "theoretical_loss": 3.328907390118375, + "tokens_seen": 3251437568 + }, + { + "epoch": 0.99, + "learning_rate": 1.2116349047141424e-05, + "loss": 1.1958, + "theoretical_loss": 3.328902358583054, + "tokens_seen": 3251503104 + }, + { + "epoch": 0.99, + "learning_rate": 1.2100300902708125e-05, + "loss": 1.0914, + "theoretical_loss": 3.3288973271775415, + "tokens_seen": 3251568640 + }, + { + "epoch": 0.99, + "learning_rate": 1.2084252758274825e-05, + "loss": 1.0814, + "theoretical_loss": 3.3288922959018303, + "tokens_seen": 3251634176 + }, + { + "epoch": 0.99, + "learning_rate": 1.2068204613841526e-05, + "loss": 1.1949, + "theoretical_loss": 3.328887264755915, + "tokens_seen": 3251699712 + }, + { + "epoch": 0.99, + "learning_rate": 1.2052156469408225e-05, + "loss": 1.1018, + "theoretical_loss": 3.3288822337397894, + "tokens_seen": 3251765248 + }, + { + "epoch": 0.99, + "learning_rate": 1.2036108324974926e-05, + "loss": 1.209, + "theoretical_loss": 3.3288772028534477, + "tokens_seen": 3251830784 + }, + { + "epoch": 0.99, + "learning_rate": 1.2020060180541626e-05, + "loss": 1.2341, + "theoretical_loss": 3.328872172096884, + "tokens_seen": 3251896320 + }, + { + "epoch": 0.99, + "learning_rate": 1.2004012036108325e-05, + "loss": 1.2456, + "theoretical_loss": 3.3288671414700923, + "tokens_seen": 3251961856 + }, + { + "epoch": 0.99, + "learning_rate": 1.1987963891675026e-05, + "loss": 1.1416, + "theoretical_loss": 3.3288621109730667, + "tokens_seen": 3252027392 + }, + { + "epoch": 0.99, + "learning_rate": 1.1971915747241726e-05, + "loss": 1.1078, + "theoretical_loss": 3.3288570806058013, + "tokens_seen": 3252092928 + }, + { + "epoch": 0.99, + "learning_rate": 1.1955867602808425e-05, + "loss": 1.1107, + "theoretical_loss": 3.3288520503682895, + "tokens_seen": 3252158464 + }, + { + "epoch": 0.99, + "learning_rate": 1.1939819458375126e-05, + "loss": 1.2406, + "theoretical_loss": 3.3288470202605263, + "tokens_seen": 3252224000 + }, + { + "epoch": 0.99, + "learning_rate": 1.1923771313941825e-05, + "loss": 1.0753, + "theoretical_loss": 3.328841990282505, + "tokens_seen": 3252289536 + }, + { + "epoch": 0.99, + "learning_rate": 1.1907723169508527e-05, + "loss": 1.2691, + "theoretical_loss": 3.32883696043422, + "tokens_seen": 3252355072 + }, + { + "epoch": 0.99, + "learning_rate": 1.1891675025075226e-05, + "loss": 1.1077, + "theoretical_loss": 3.328831930715665, + "tokens_seen": 3252420608 + }, + { + "epoch": 0.99, + "learning_rate": 1.1875626880641927e-05, + "loss": 1.176, + "theoretical_loss": 3.3288269011268348, + "tokens_seen": 3252486144 + }, + { + "epoch": 0.99, + "learning_rate": 1.1859578736208626e-05, + "loss": 1.171, + "theoretical_loss": 3.3288218716677225, + "tokens_seen": 3252551680 + }, + { + "epoch": 0.99, + "learning_rate": 1.1843530591775326e-05, + "loss": 1.0486, + "theoretical_loss": 3.3288168423383224, + "tokens_seen": 3252617216 + }, + { + "epoch": 0.99, + "learning_rate": 1.1827482447342027e-05, + "loss": 1.0288, + "theoretical_loss": 3.3288118131386293, + "tokens_seen": 3252682752 + }, + { + "epoch": 0.99, + "learning_rate": 1.1811434302908726e-05, + "loss": 1.2243, + "theoretical_loss": 3.328806784068636, + "tokens_seen": 3252748288 + }, + { + "epoch": 0.99, + "learning_rate": 1.1795386158475427e-05, + "loss": 1.1916, + "theoretical_loss": 3.328801755128338, + "tokens_seen": 3252813824 + }, + { + "epoch": 0.99, + "learning_rate": 1.1779338014042127e-05, + "loss": 1.1344, + "theoretical_loss": 3.328796726317728, + "tokens_seen": 3252879360 + }, + { + "epoch": 0.99, + "learning_rate": 1.1763289869608828e-05, + "loss": 1.1051, + "theoretical_loss": 3.3287916976368006, + "tokens_seen": 3252944896 + }, + { + "epoch": 0.99, + "learning_rate": 1.1747241725175527e-05, + "loss": 1.1755, + "theoretical_loss": 3.3287866690855497, + "tokens_seen": 3253010432 + }, + { + "epoch": 0.99, + "learning_rate": 1.1731193580742226e-05, + "loss": 1.2561, + "theoretical_loss": 3.3287816406639696, + "tokens_seen": 3253075968 + }, + { + "epoch": 0.99, + "learning_rate": 1.1715145436308928e-05, + "loss": 1.136, + "theoretical_loss": 3.328776612372054, + "tokens_seen": 3253141504 + }, + { + "epoch": 0.99, + "learning_rate": 1.1699097291875627e-05, + "loss": 1.1949, + "theoretical_loss": 3.3287715842097976, + "tokens_seen": 3253207040 + }, + { + "epoch": 0.99, + "learning_rate": 1.1683049147442328e-05, + "loss": 1.148, + "theoretical_loss": 3.3287665561771935, + "tokens_seen": 3253272576 + }, + { + "epoch": 0.99, + "learning_rate": 1.1667001003009027e-05, + "loss": 1.0767, + "theoretical_loss": 3.328761528274237, + "tokens_seen": 3253338112 + }, + { + "epoch": 0.99, + "learning_rate": 1.1650952858575729e-05, + "loss": 1.1744, + "theoretical_loss": 3.3287565005009205, + "tokens_seen": 3253403648 + }, + { + "epoch": 0.99, + "learning_rate": 1.1634904714142428e-05, + "loss": 1.1547, + "theoretical_loss": 3.328751472857239, + "tokens_seen": 3253469184 + }, + { + "epoch": 0.99, + "learning_rate": 1.1618856569709127e-05, + "loss": 1.1571, + "theoretical_loss": 3.328746445343187, + "tokens_seen": 3253534720 + }, + { + "epoch": 0.99, + "learning_rate": 1.1602808425275827e-05, + "loss": 1.1819, + "theoretical_loss": 3.3287414179587578, + "tokens_seen": 3253600256 + }, + { + "epoch": 0.99, + "learning_rate": 1.1586760280842528e-05, + "loss": 1.2498, + "theoretical_loss": 3.328736390703946, + "tokens_seen": 3253665792 + }, + { + "epoch": 0.99, + "learning_rate": 1.1570712136409229e-05, + "loss": 1.1434, + "theoretical_loss": 3.3287313635787448, + "tokens_seen": 3253731328 + }, + { + "epoch": 0.99, + "learning_rate": 1.1554663991975928e-05, + "loss": 1.1918, + "theoretical_loss": 3.328726336583149, + "tokens_seen": 3253796864 + }, + { + "epoch": 0.99, + "objective/train/docs_used": 1086203, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.122308611869812, + "objective/train/theoretical_loss": 3.328721309717152, + "objective/train/tokens_used": 3274322400, + "theoretical_loss": 3.328721309717152, + "tokens_seen": 3253862400 + }, + { + "epoch": 0.99, + "learning_rate": 1.1538615847542628e-05, + "loss": 1.1492, + "theoretical_loss": 3.328721309717152, + "tokens_seen": 3253862400 + }, + { + "epoch": 0.99, + "learning_rate": 1.1522567703109329e-05, + "loss": 1.2293, + "theoretical_loss": 3.328716282980749, + "tokens_seen": 3253927936 + }, + { + "epoch": 0.99, + "learning_rate": 1.1506519558676028e-05, + "loss": 1.2493, + "theoretical_loss": 3.328711256373933, + "tokens_seen": 3253993472 + }, + { + "epoch": 0.99, + "learning_rate": 1.1490471414242729e-05, + "loss": 1.2265, + "theoretical_loss": 3.328706229896698, + "tokens_seen": 3254059008 + }, + { + "epoch": 0.99, + "learning_rate": 1.1474423269809429e-05, + "loss": 1.174, + "theoretical_loss": 3.328701203549039, + "tokens_seen": 3254124544 + }, + { + "epoch": 0.99, + "learning_rate": 1.145837512537613e-05, + "loss": 1.1632, + "theoretical_loss": 3.328696177330949, + "tokens_seen": 3254190080 + }, + { + "epoch": 0.99, + "learning_rate": 1.1442326980942829e-05, + "loss": 1.1737, + "theoretical_loss": 3.328691151242423, + "tokens_seen": 3254255616 + }, + { + "epoch": 0.99, + "learning_rate": 1.142627883650953e-05, + "loss": 1.1538, + "theoretical_loss": 3.328686125283454, + "tokens_seen": 3254321152 + }, + { + "epoch": 0.99, + "learning_rate": 1.141023069207623e-05, + "loss": 1.1611, + "theoretical_loss": 3.328681099454037, + "tokens_seen": 3254386688 + }, + { + "epoch": 0.99, + "learning_rate": 1.1394182547642929e-05, + "loss": 1.243, + "theoretical_loss": 3.3286760737541656, + "tokens_seen": 3254452224 + }, + { + "epoch": 0.99, + "learning_rate": 1.137813440320963e-05, + "loss": 1.3247, + "theoretical_loss": 3.3286710481838337, + "tokens_seen": 3254517760 + }, + { + "epoch": 0.99, + "learning_rate": 1.1362086258776329e-05, + "loss": 1.1386, + "theoretical_loss": 3.328666022743036, + "tokens_seen": 3254583296 + }, + { + "epoch": 0.99, + "learning_rate": 1.134603811434303e-05, + "loss": 1.2075, + "theoretical_loss": 3.328660997431766, + "tokens_seen": 3254648832 + }, + { + "epoch": 0.99, + "learning_rate": 1.132998996990973e-05, + "loss": 1.2138, + "theoretical_loss": 3.3286559722500177, + "tokens_seen": 3254714368 + }, + { + "epoch": 0.99, + "learning_rate": 1.131394182547643e-05, + "loss": 1.1816, + "theoretical_loss": 3.328650947197785, + "tokens_seen": 3254779904 + }, + { + "epoch": 0.99, + "learning_rate": 1.129789368104313e-05, + "loss": 1.1885, + "theoretical_loss": 3.328645922275063, + "tokens_seen": 3254845440 + }, + { + "epoch": 0.99, + "learning_rate": 1.128184553660983e-05, + "loss": 1.1118, + "theoretical_loss": 3.3286408974818444, + "tokens_seen": 3254910976 + }, + { + "epoch": 0.99, + "learning_rate": 1.126579739217653e-05, + "loss": 1.101, + "theoretical_loss": 3.3286358728181247, + "tokens_seen": 3254976512 + }, + { + "epoch": 0.99, + "learning_rate": 1.124974924774323e-05, + "loss": 1.186, + "theoretical_loss": 3.3286308482838964, + "tokens_seen": 3255042048 + }, + { + "epoch": 0.99, + "learning_rate": 1.123370110330993e-05, + "loss": 1.1323, + "theoretical_loss": 3.3286258238791544, + "tokens_seen": 3255107584 + }, + { + "epoch": 0.99, + "learning_rate": 1.121765295887663e-05, + "loss": 1.149, + "theoretical_loss": 3.3286207996038932, + "tokens_seen": 3255173120 + }, + { + "epoch": 0.99, + "learning_rate": 1.1201604814443331e-05, + "loss": 1.1834, + "theoretical_loss": 3.328615775458106, + "tokens_seen": 3255238656 + }, + { + "epoch": 0.99, + "learning_rate": 1.118555667001003e-05, + "loss": 1.233, + "theoretical_loss": 3.328610751441787, + "tokens_seen": 3255304192 + }, + { + "epoch": 0.99, + "learning_rate": 1.116950852557673e-05, + "loss": 1.2892, + "theoretical_loss": 3.3286057275549306, + "tokens_seen": 3255369728 + }, + { + "epoch": 0.99, + "learning_rate": 1.1153460381143432e-05, + "loss": 1.1937, + "theoretical_loss": 3.3286007037975303, + "tokens_seen": 3255435264 + }, + { + "epoch": 0.99, + "learning_rate": 1.113741223671013e-05, + "loss": 1.1399, + "theoretical_loss": 3.328595680169581, + "tokens_seen": 3255500800 + }, + { + "epoch": 0.99, + "learning_rate": 1.1121364092276831e-05, + "loss": 1.2408, + "theoretical_loss": 3.3285906566710763, + "tokens_seen": 3255566336 + }, + { + "epoch": 0.99, + "learning_rate": 1.110531594784353e-05, + "loss": 1.1877, + "theoretical_loss": 3.3285856333020103, + "tokens_seen": 3255631872 + }, + { + "epoch": 0.99, + "learning_rate": 1.1089267803410233e-05, + "loss": 1.1349, + "theoretical_loss": 3.3285806100623767, + "tokens_seen": 3255697408 + }, + { + "epoch": 0.99, + "learning_rate": 1.1073219658976931e-05, + "loss": 1.1897, + "theoretical_loss": 3.3285755869521703, + "tokens_seen": 3255762944 + }, + { + "epoch": 0.99, + "learning_rate": 1.105717151454363e-05, + "loss": 1.1472, + "theoretical_loss": 3.3285705639713843, + "tokens_seen": 3255828480 + }, + { + "epoch": 0.99, + "learning_rate": 1.1041123370110331e-05, + "loss": 1.2497, + "theoretical_loss": 3.3285655411200135, + "tokens_seen": 3255894016 + }, + { + "epoch": 0.99, + "learning_rate": 1.1025075225677032e-05, + "loss": 1.279, + "theoretical_loss": 3.3285605183980516, + "tokens_seen": 3255959552 + }, + { + "epoch": 0.99, + "learning_rate": 1.1009027081243732e-05, + "loss": 1.1975, + "theoretical_loss": 3.328555495805493, + "tokens_seen": 3256025088 + }, + { + "epoch": 0.99, + "learning_rate": 1.0992978936810431e-05, + "loss": 1.0964, + "theoretical_loss": 3.328550473342331, + "tokens_seen": 3256090624 + }, + { + "epoch": 0.99, + "learning_rate": 1.0976930792377132e-05, + "loss": 1.1426, + "theoretical_loss": 3.3285454510085604, + "tokens_seen": 3256156160 + }, + { + "epoch": 0.99, + "learning_rate": 1.0960882647943833e-05, + "loss": 1.1793, + "theoretical_loss": 3.328540428804175, + "tokens_seen": 3256221696 + }, + { + "epoch": 0.99, + "learning_rate": 1.0944834503510531e-05, + "loss": 1.1963, + "theoretical_loss": 3.328535406729169, + "tokens_seen": 3256287232 + }, + { + "epoch": 0.99, + "learning_rate": 1.0928786359077232e-05, + "loss": 1.1856, + "theoretical_loss": 3.328530384783536, + "tokens_seen": 3256352768 + }, + { + "epoch": 0.99, + "learning_rate": 1.0912738214643933e-05, + "loss": 1.2092, + "theoretical_loss": 3.328525362967271, + "tokens_seen": 3256418304 + }, + { + "epoch": 0.99, + "learning_rate": 1.0896690070210633e-05, + "loss": 1.11, + "theoretical_loss": 3.328520341280367, + "tokens_seen": 3256483840 + }, + { + "epoch": 0.99, + "learning_rate": 1.0880641925777332e-05, + "loss": 1.1961, + "theoretical_loss": 3.3285153197228183, + "tokens_seen": 3256549376 + }, + { + "epoch": 0.99, + "learning_rate": 1.0864593781344033e-05, + "loss": 1.1199, + "theoretical_loss": 3.3285102982946198, + "tokens_seen": 3256614912 + }, + { + "epoch": 0.99, + "learning_rate": 1.0848545636910734e-05, + "loss": 1.178, + "theoretical_loss": 3.3285052769957644, + "tokens_seen": 3256680448 + }, + { + "epoch": 0.99, + "learning_rate": 1.0832497492477433e-05, + "loss": 1.1643, + "theoretical_loss": 3.3285002558262473, + "tokens_seen": 3256745984 + }, + { + "epoch": 0.99, + "learning_rate": 1.0816449348044133e-05, + "loss": 1.0929, + "theoretical_loss": 3.3284952347860615, + "tokens_seen": 3256811520 + }, + { + "epoch": 0.99, + "learning_rate": 1.0800401203610832e-05, + "loss": 1.203, + "theoretical_loss": 3.3284902138752015, + "tokens_seen": 3256877056 + }, + { + "epoch": 0.99, + "learning_rate": 1.0784353059177533e-05, + "loss": 1.1971, + "theoretical_loss": 3.328485193093662, + "tokens_seen": 3256942592 + }, + { + "epoch": 0.99, + "learning_rate": 1.0768304914744233e-05, + "loss": 1.2412, + "theoretical_loss": 3.328480172441436, + "tokens_seen": 3257008128 + }, + { + "epoch": 0.99, + "learning_rate": 1.0752256770310934e-05, + "loss": 1.2456, + "theoretical_loss": 3.3284751519185183, + "tokens_seen": 3257073664 + }, + { + "epoch": 0.99, + "objective/train/docs_used": 1087135, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1532700061798096, + "objective/train/theoretical_loss": 3.328470131524903, + "objective/train/tokens_used": 3277599200, + "theoretical_loss": 3.328470131524903, + "tokens_seen": 3257139200 + }, + { + "epoch": 0.99, + "learning_rate": 1.0736208625877633e-05, + "loss": 1.1639, + "theoretical_loss": 3.328470131524903, + "tokens_seen": 3257139200 + }, + { + "epoch": 0.99, + "learning_rate": 1.0720160481444334e-05, + "loss": 1.2054, + "theoretical_loss": 3.3284651112605834, + "tokens_seen": 3257204736 + }, + { + "epoch": 0.99, + "learning_rate": 1.0704112337011034e-05, + "loss": 1.1009, + "theoretical_loss": 3.328460091125554, + "tokens_seen": 3257270272 + }, + { + "epoch": 0.99, + "learning_rate": 1.0688064192577733e-05, + "loss": 1.2292, + "theoretical_loss": 3.3284550711198095, + "tokens_seen": 3257335808 + }, + { + "epoch": 0.99, + "learning_rate": 1.0672016048144434e-05, + "loss": 1.1864, + "theoretical_loss": 3.328450051243343, + "tokens_seen": 3257401344 + }, + { + "epoch": 0.99, + "learning_rate": 1.0655967903711134e-05, + "loss": 1.1013, + "theoretical_loss": 3.328445031496149, + "tokens_seen": 3257466880 + }, + { + "epoch": 0.99, + "learning_rate": 1.0639919759277835e-05, + "loss": 1.1327, + "theoretical_loss": 3.3284400118782216, + "tokens_seen": 3257532416 + }, + { + "epoch": 0.99, + "learning_rate": 1.0623871614844534e-05, + "loss": 1.2017, + "theoretical_loss": 3.328434992389555, + "tokens_seen": 3257597952 + }, + { + "epoch": 0.99, + "learning_rate": 1.0607823470411233e-05, + "loss": 1.1874, + "theoretical_loss": 3.328429973030143, + "tokens_seen": 3257663488 + }, + { + "epoch": 0.99, + "learning_rate": 1.0591775325977935e-05, + "loss": 1.1974, + "theoretical_loss": 3.3284249537999795, + "tokens_seen": 3257729024 + }, + { + "epoch": 0.99, + "learning_rate": 1.0575727181544634e-05, + "loss": 1.2158, + "theoretical_loss": 3.328419934699059, + "tokens_seen": 3257794560 + }, + { + "epoch": 0.99, + "learning_rate": 1.0559679037111335e-05, + "loss": 1.1592, + "theoretical_loss": 3.3284149157273752, + "tokens_seen": 3257860096 + }, + { + "epoch": 0.99, + "learning_rate": 1.0543630892678034e-05, + "loss": 1.2004, + "theoretical_loss": 3.328409896884923, + "tokens_seen": 3257925632 + }, + { + "epoch": 0.99, + "learning_rate": 1.0527582748244736e-05, + "loss": 1.2326, + "theoretical_loss": 3.3284048781716953, + "tokens_seen": 3257991168 + }, + { + "epoch": 0.99, + "learning_rate": 1.0511534603811435e-05, + "loss": 1.3235, + "theoretical_loss": 3.328399859587687, + "tokens_seen": 3258056704 + }, + { + "epoch": 0.99, + "learning_rate": 1.0495486459378134e-05, + "loss": 1.1437, + "theoretical_loss": 3.3283948411328916, + "tokens_seen": 3258122240 + }, + { + "epoch": 0.99, + "learning_rate": 1.0479438314944835e-05, + "loss": 1.2311, + "theoretical_loss": 3.3283898228073037, + "tokens_seen": 3258187776 + }, + { + "epoch": 0.99, + "learning_rate": 1.0463390170511535e-05, + "loss": 1.1895, + "theoretical_loss": 3.3283848046109172, + "tokens_seen": 3258253312 + }, + { + "epoch": 0.99, + "learning_rate": 1.0447342026078236e-05, + "loss": 1.2, + "theoretical_loss": 3.3283797865437257, + "tokens_seen": 3258318848 + }, + { + "epoch": 0.99, + "learning_rate": 1.0431293881644935e-05, + "loss": 1.1147, + "theoretical_loss": 3.328374768605724, + "tokens_seen": 3258384384 + }, + { + "epoch": 0.99, + "learning_rate": 1.0415245737211635e-05, + "loss": 1.1238, + "theoretical_loss": 3.328369750796906, + "tokens_seen": 3258449920 + }, + { + "epoch": 0.99, + "learning_rate": 1.0399197592778336e-05, + "loss": 1.2453, + "theoretical_loss": 3.328364733117265, + "tokens_seen": 3258515456 + }, + { + "epoch": 0.99, + "learning_rate": 1.0383149448345035e-05, + "loss": 1.2586, + "theoretical_loss": 3.3283597155667968, + "tokens_seen": 3258580992 + }, + { + "epoch": 0.99, + "learning_rate": 1.0367101303911736e-05, + "loss": 1.2443, + "theoretical_loss": 3.328354698145494, + "tokens_seen": 3258646528 + }, + { + "epoch": 0.99, + "learning_rate": 1.0351053159478436e-05, + "loss": 1.1198, + "theoretical_loss": 3.328349680853351, + "tokens_seen": 3258712064 + }, + { + "epoch": 0.99, + "learning_rate": 1.0335005015045137e-05, + "loss": 1.1142, + "theoretical_loss": 3.3283446636903617, + "tokens_seen": 3258777600 + }, + { + "epoch": 0.99, + "learning_rate": 1.0318956870611836e-05, + "loss": 1.19, + "theoretical_loss": 3.3283396466565205, + "tokens_seen": 3258843136 + }, + { + "epoch": 0.99, + "learning_rate": 1.0302908726178537e-05, + "loss": 1.1837, + "theoretical_loss": 3.3283346297518213, + "tokens_seen": 3258908672 + }, + { + "epoch": 0.99, + "learning_rate": 1.0286860581745235e-05, + "loss": 1.1549, + "theoretical_loss": 3.3283296129762587, + "tokens_seen": 3258974208 + }, + { + "epoch": 0.99, + "learning_rate": 1.0270812437311936e-05, + "loss": 1.1198, + "theoretical_loss": 3.3283245963298262, + "tokens_seen": 3259039744 + }, + { + "epoch": 0.99, + "learning_rate": 1.0254764292878637e-05, + "loss": 1.2601, + "theoretical_loss": 3.328319579812518, + "tokens_seen": 3259105280 + }, + { + "epoch": 0.99, + "learning_rate": 1.0238716148445336e-05, + "loss": 1.1002, + "theoretical_loss": 3.3283145634243283, + "tokens_seen": 3259170816 + }, + { + "epoch": 0.99, + "learning_rate": 1.0222668004012036e-05, + "loss": 1.1675, + "theoretical_loss": 3.328309547165251, + "tokens_seen": 3259236352 + }, + { + "epoch": 0.99, + "learning_rate": 1.0206619859578737e-05, + "loss": 1.2194, + "theoretical_loss": 3.3283045310352803, + "tokens_seen": 3259301888 + }, + { + "epoch": 0.99, + "learning_rate": 1.0190571715145438e-05, + "loss": 1.1659, + "theoretical_loss": 3.32829951503441, + "tokens_seen": 3259367424 + }, + { + "epoch": 0.99, + "learning_rate": 1.0174523570712136e-05, + "loss": 1.1646, + "theoretical_loss": 3.3282944991626353, + "tokens_seen": 3259432960 + }, + { + "epoch": 0.99, + "learning_rate": 1.0158475426278837e-05, + "loss": 1.1958, + "theoretical_loss": 3.328289483419949, + "tokens_seen": 3259498496 + }, + { + "epoch": 0.99, + "learning_rate": 1.0142427281845538e-05, + "loss": 1.149, + "theoretical_loss": 3.3282844678063452, + "tokens_seen": 3259564032 + }, + { + "epoch": 0.99, + "learning_rate": 1.0126379137412237e-05, + "loss": 1.1459, + "theoretical_loss": 3.328279452321819, + "tokens_seen": 3259629568 + }, + { + "epoch": 0.99, + "learning_rate": 1.0110330992978937e-05, + "loss": 1.1995, + "theoretical_loss": 3.3282744369663635, + "tokens_seen": 3259695104 + }, + { + "epoch": 0.99, + "learning_rate": 1.0094282848545638e-05, + "loss": 1.1323, + "theoretical_loss": 3.3282694217399733, + "tokens_seen": 3259760640 + }, + { + "epoch": 0.99, + "learning_rate": 1.0078234704112339e-05, + "loss": 1.1398, + "theoretical_loss": 3.328264406642642, + "tokens_seen": 3259826176 + }, + { + "epoch": 0.99, + "learning_rate": 1.0062186559679038e-05, + "loss": 1.111, + "theoretical_loss": 3.3282593916743646, + "tokens_seen": 3259891712 + }, + { + "epoch": 0.99, + "learning_rate": 1.0046138415245736e-05, + "loss": 1.1236, + "theoretical_loss": 3.3282543768351345, + "tokens_seen": 3259957248 + }, + { + "epoch": 0.99, + "learning_rate": 1.0030090270812439e-05, + "loss": 1.2917, + "theoretical_loss": 3.3282493621249456, + "tokens_seen": 3260022784 + }, + { + "epoch": 0.99, + "learning_rate": 1.0014042126379138e-05, + "loss": 1.1529, + "theoretical_loss": 3.3282443475437926, + "tokens_seen": 3260088320 + }, + { + "epoch": 0.99, + "learning_rate": 9.997993981945838e-06, + "loss": 1.1151, + "theoretical_loss": 3.3282393330916693, + "tokens_seen": 3260153856 + }, + { + "epoch": 0.99, + "learning_rate": 9.981945837512537e-06, + "loss": 1.1645, + "theoretical_loss": 3.3282343187685695, + "tokens_seen": 3260219392 + }, + { + "epoch": 0.99, + "learning_rate": 9.96589769307924e-06, + "loss": 1.2559, + "theoretical_loss": 3.3282293045744877, + "tokens_seen": 3260284928 + }, + { + "epoch": 0.99, + "learning_rate": 9.949849548645939e-06, + "loss": 1.1776, + "theoretical_loss": 3.328224290509418, + "tokens_seen": 3260350464 + }, + { + "epoch": 0.99, + "objective/train/docs_used": 1088365, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2857568264007568, + "objective/train/theoretical_loss": 3.328219276573354, + "objective/train/tokens_used": 3280876000, + "theoretical_loss": 3.328219276573354, + "tokens_seen": 3260416000 + }, + { + "epoch": 0.99, + "learning_rate": 9.933801404212638e-06, + "loss": 1.2289, + "theoretical_loss": 3.328219276573354, + "tokens_seen": 3260416000 + }, + { + "epoch": 0.99, + "learning_rate": 9.917753259779338e-06, + "loss": 1.1968, + "theoretical_loss": 3.32821426276629, + "tokens_seen": 3260481536 + }, + { + "epoch": 0.99, + "learning_rate": 9.901705115346039e-06, + "loss": 1.1845, + "theoretical_loss": 3.328209249088221, + "tokens_seen": 3260547072 + }, + { + "epoch": 0.99, + "learning_rate": 9.88565697091274e-06, + "loss": 1.0766, + "theoretical_loss": 3.3282042355391397, + "tokens_seen": 3260612608 + }, + { + "epoch": 0.99, + "learning_rate": 9.869608826479438e-06, + "loss": 1.2332, + "theoretical_loss": 3.3281992221190406, + "tokens_seen": 3260678144 + }, + { + "epoch": 0.99, + "learning_rate": 9.853560682046139e-06, + "loss": 1.1931, + "theoretical_loss": 3.3281942088279184, + "tokens_seen": 3260743680 + }, + { + "epoch": 0.99, + "learning_rate": 9.83751253761284e-06, + "loss": 1.1156, + "theoretical_loss": 3.3281891956657663, + "tokens_seen": 3260809216 + }, + { + "epoch": 0.99, + "learning_rate": 9.821464393179539e-06, + "loss": 1.115, + "theoretical_loss": 3.328184182632579, + "tokens_seen": 3260874752 + }, + { + "epoch": 0.99, + "learning_rate": 9.80541624874624e-06, + "loss": 1.1739, + "theoretical_loss": 3.328179169728351, + "tokens_seen": 3260940288 + }, + { + "epoch": 0.99, + "learning_rate": 9.789368104312938e-06, + "loss": 1.2704, + "theoretical_loss": 3.328174156953075, + "tokens_seen": 3261005824 + }, + { + "epoch": 0.99, + "learning_rate": 9.77331995987964e-06, + "loss": 1.2405, + "theoretical_loss": 3.3281691443067465, + "tokens_seen": 3261071360 + }, + { + "epoch": 0.99, + "learning_rate": 9.75727181544634e-06, + "loss": 1.2183, + "theoretical_loss": 3.3281641317893587, + "tokens_seen": 3261136896 + }, + { + "epoch": 0.99, + "learning_rate": 9.741223671013038e-06, + "loss": 1.0697, + "theoretical_loss": 3.328159119400906, + "tokens_seen": 3261202432 + }, + { + "epoch": 0.99, + "learning_rate": 9.725175526579739e-06, + "loss": 1.1683, + "theoretical_loss": 3.3281541071413825, + "tokens_seen": 3261267968 + }, + { + "epoch": 0.99, + "learning_rate": 9.70912738214644e-06, + "loss": 1.1992, + "theoretical_loss": 3.3281490950107826, + "tokens_seen": 3261333504 + }, + { + "epoch": 0.99, + "learning_rate": 9.69307923771314e-06, + "loss": 1.1759, + "theoretical_loss": 3.3281440830091, + "tokens_seen": 3261399040 + }, + { + "epoch": 0.99, + "learning_rate": 9.67703109327984e-06, + "loss": 1.166, + "theoretical_loss": 3.3281390711363286, + "tokens_seen": 3261464576 + }, + { + "epoch": 0.99, + "learning_rate": 9.66098294884654e-06, + "loss": 1.1381, + "theoretical_loss": 3.328134059392463, + "tokens_seen": 3261530112 + }, + { + "epoch": 0.99, + "learning_rate": 9.64493480441324e-06, + "loss": 1.2067, + "theoretical_loss": 3.3281290477774967, + "tokens_seen": 3261595648 + }, + { + "epoch": 0.99, + "learning_rate": 9.62888665997994e-06, + "loss": 1.2447, + "theoretical_loss": 3.328124036291425, + "tokens_seen": 3261661184 + }, + { + "epoch": 0.99, + "learning_rate": 9.61283851554664e-06, + "loss": 1.1181, + "theoretical_loss": 3.3281190249342405, + "tokens_seen": 3261726720 + }, + { + "epoch": 0.99, + "learning_rate": 9.59679037111334e-06, + "loss": 1.1663, + "theoretical_loss": 3.328114013705938, + "tokens_seen": 3261792256 + }, + { + "epoch": 0.99, + "learning_rate": 9.580742226680041e-06, + "loss": 1.0962, + "theoretical_loss": 3.3281090026065114, + "tokens_seen": 3261857792 + }, + { + "epoch": 0.99, + "learning_rate": 9.56469408224674e-06, + "loss": 1.2015, + "theoretical_loss": 3.328103991635955, + "tokens_seen": 3261923328 + }, + { + "epoch": 0.99, + "learning_rate": 9.548645937813441e-06, + "loss": 1.1025, + "theoretical_loss": 3.3280989807942634, + "tokens_seen": 3261988864 + }, + { + "epoch": 0.99, + "learning_rate": 9.532597793380142e-06, + "loss": 1.2462, + "theoretical_loss": 3.3280939700814294, + "tokens_seen": 3262054400 + }, + { + "epoch": 0.99, + "learning_rate": 9.51654964894684e-06, + "loss": 1.1274, + "theoretical_loss": 3.3280889594974483, + "tokens_seen": 3262119936 + }, + { + "epoch": 0.99, + "learning_rate": 9.500501504513541e-06, + "loss": 1.2164, + "theoretical_loss": 3.3280839490423135, + "tokens_seen": 3262185472 + }, + { + "epoch": 0.99, + "learning_rate": 9.48445336008024e-06, + "loss": 1.2415, + "theoretical_loss": 3.3280789387160197, + "tokens_seen": 3262251008 + }, + { + "epoch": 0.99, + "learning_rate": 9.468405215646942e-06, + "loss": 1.061, + "theoretical_loss": 3.32807392851856, + "tokens_seen": 3262316544 + }, + { + "epoch": 0.99, + "learning_rate": 9.452357071213641e-06, + "loss": 1.1933, + "theoretical_loss": 3.3280689184499295, + "tokens_seen": 3262382080 + }, + { + "epoch": 0.99, + "learning_rate": 9.436308926780342e-06, + "loss": 1.1583, + "theoretical_loss": 3.328063908510122, + "tokens_seen": 3262447616 + }, + { + "epoch": 0.99, + "learning_rate": 9.420260782347041e-06, + "loss": 1.1086, + "theoretical_loss": 3.3280588986991315, + "tokens_seen": 3262513152 + }, + { + "epoch": 0.99, + "learning_rate": 9.404212637913742e-06, + "loss": 1.249, + "theoretical_loss": 3.328053889016952, + "tokens_seen": 3262578688 + }, + { + "epoch": 0.99, + "learning_rate": 9.388164493480442e-06, + "loss": 1.1435, + "theoretical_loss": 3.328048879463578, + "tokens_seen": 3262644224 + }, + { + "epoch": 0.99, + "learning_rate": 9.372116349047141e-06, + "loss": 1.1857, + "theoretical_loss": 3.3280438700390027, + "tokens_seen": 3262709760 + }, + { + "epoch": 0.99, + "learning_rate": 9.356068204613842e-06, + "loss": 1.1911, + "theoretical_loss": 3.328038860743222, + "tokens_seen": 3262775296 + }, + { + "epoch": 0.99, + "learning_rate": 9.340020060180542e-06, + "loss": 1.2125, + "theoretical_loss": 3.3280338515762278, + "tokens_seen": 3262840832 + }, + { + "epoch": 0.99, + "learning_rate": 9.323971915747243e-06, + "loss": 1.2032, + "theoretical_loss": 3.3280288425380156, + "tokens_seen": 3262906368 + }, + { + "epoch": 0.99, + "learning_rate": 9.307923771313942e-06, + "loss": 1.1873, + "theoretical_loss": 3.328023833628579, + "tokens_seen": 3262971904 + }, + { + "epoch": 0.99, + "learning_rate": 9.291875626880641e-06, + "loss": 1.1028, + "theoretical_loss": 3.3280188248479123, + "tokens_seen": 3263037440 + }, + { + "epoch": 0.99, + "learning_rate": 9.275827482447343e-06, + "loss": 1.3103, + "theoretical_loss": 3.3280138161960093, + "tokens_seen": 3263102976 + }, + { + "epoch": 0.99, + "learning_rate": 9.259779338014042e-06, + "loss": 1.1866, + "theoretical_loss": 3.3280088076728647, + "tokens_seen": 3263168512 + }, + { + "epoch": 0.99, + "learning_rate": 9.243731193580743e-06, + "loss": 1.1502, + "theoretical_loss": 3.328003799278472, + "tokens_seen": 3263234048 + }, + { + "epoch": 0.99, + "learning_rate": 9.227683049147442e-06, + "loss": 1.1505, + "theoretical_loss": 3.327998791012826, + "tokens_seen": 3263299584 + }, + { + "epoch": 0.99, + "learning_rate": 9.211634904714144e-06, + "loss": 1.3008, + "theoretical_loss": 3.32799378287592, + "tokens_seen": 3263365120 + }, + { + "epoch": 0.99, + "learning_rate": 9.195586760280843e-06, + "loss": 1.1588, + "theoretical_loss": 3.3279887748677486, + "tokens_seen": 3263430656 + }, + { + "epoch": 0.99, + "learning_rate": 9.179538615847542e-06, + "loss": 1.2238, + "theoretical_loss": 3.327983766988306, + "tokens_seen": 3263496192 + }, + { + "epoch": 0.99, + "learning_rate": 9.163490471414243e-06, + "loss": 1.1499, + "theoretical_loss": 3.3279787592375856, + "tokens_seen": 3263561728 + }, + { + "epoch": 0.99, + "learning_rate": 9.147442326980943e-06, + "loss": 1.1626, + "theoretical_loss": 3.3279737516155823, + "tokens_seen": 3263627264 + }, + { + "epoch": 0.99, + "objective/train/docs_used": 1089160, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 0.9532618522644043, + "objective/train/theoretical_loss": 3.3279687441222894, + "objective/train/tokens_used": 3284152800, + "theoretical_loss": 3.3279687441222894, + "tokens_seen": 3263692800 + }, + { + "epoch": 0.99, + "learning_rate": 9.131394182547644e-06, + "loss": 1.0665, + "theoretical_loss": 3.3279687441222894, + "tokens_seen": 3263692800 + }, + { + "epoch": 0.99, + "learning_rate": 9.115346038114343e-06, + "loss": 1.1547, + "theoretical_loss": 3.327963736757702, + "tokens_seen": 3263758336 + }, + { + "epoch": 0.99, + "learning_rate": 9.099297893681043e-06, + "loss": 1.24, + "theoretical_loss": 3.3279587295218134, + "tokens_seen": 3263823872 + }, + { + "epoch": 0.99, + "learning_rate": 9.083249749247744e-06, + "loss": 1.2949, + "theoretical_loss": 3.3279537224146187, + "tokens_seen": 3263889408 + }, + { + "epoch": 0.99, + "learning_rate": 9.067201604814443e-06, + "loss": 1.2337, + "theoretical_loss": 3.3279487154361105, + "tokens_seen": 3263954944 + }, + { + "epoch": 0.99, + "learning_rate": 9.051153460381144e-06, + "loss": 1.195, + "theoretical_loss": 3.327943708586284, + "tokens_seen": 3264020480 + }, + { + "epoch": 0.99, + "learning_rate": 9.035105315947844e-06, + "loss": 1.0615, + "theoretical_loss": 3.3279387018651336, + "tokens_seen": 3264086016 + }, + { + "epoch": 0.99, + "learning_rate": 9.019057171514545e-06, + "loss": 1.0991, + "theoretical_loss": 3.327933695272652, + "tokens_seen": 3264151552 + }, + { + "epoch": 0.99, + "learning_rate": 9.003009027081244e-06, + "loss": 1.0414, + "theoretical_loss": 3.327928688808835, + "tokens_seen": 3264217088 + }, + { + "epoch": 0.99, + "learning_rate": 8.986960882647944e-06, + "loss": 1.2062, + "theoretical_loss": 3.327923682473675, + "tokens_seen": 3264282624 + }, + { + "epoch": 0.99, + "learning_rate": 8.970912738214645e-06, + "loss": 1.1425, + "theoretical_loss": 3.3279186762671675, + "tokens_seen": 3264348160 + }, + { + "epoch": 0.99, + "learning_rate": 8.954864593781344e-06, + "loss": 1.187, + "theoretical_loss": 3.327913670189306, + "tokens_seen": 3264413696 + }, + { + "epoch": 0.99, + "learning_rate": 8.938816449348045e-06, + "loss": 1.1346, + "theoretical_loss": 3.3279086642400846, + "tokens_seen": 3264479232 + }, + { + "epoch": 0.99, + "learning_rate": 8.922768304914744e-06, + "loss": 1.184, + "theoretical_loss": 3.3279036584194976, + "tokens_seen": 3264544768 + }, + { + "epoch": 0.99, + "learning_rate": 8.906720160481446e-06, + "loss": 1.2491, + "theoretical_loss": 3.327898652727539, + "tokens_seen": 3264610304 + }, + { + "epoch": 0.99, + "learning_rate": 8.890672016048145e-06, + "loss": 1.1639, + "theoretical_loss": 3.327893647164203, + "tokens_seen": 3264675840 + }, + { + "epoch": 0.99, + "learning_rate": 8.874623871614846e-06, + "loss": 1.1815, + "theoretical_loss": 3.3278886417294835, + "tokens_seen": 3264741376 + }, + { + "epoch": 0.99, + "learning_rate": 8.858575727181544e-06, + "loss": 1.2428, + "theoretical_loss": 3.3278836364233753, + "tokens_seen": 3264806912 + }, + { + "epoch": 0.99, + "learning_rate": 8.842527582748245e-06, + "loss": 1.1659, + "theoretical_loss": 3.3278786312458717, + "tokens_seen": 3264872448 + }, + { + "epoch": 0.99, + "learning_rate": 8.826479438314946e-06, + "loss": 1.1401, + "theoretical_loss": 3.3278736261969666, + "tokens_seen": 3264937984 + }, + { + "epoch": 0.99, + "learning_rate": 8.810431293881645e-06, + "loss": 1.2293, + "theoretical_loss": 3.327868621276655, + "tokens_seen": 3265003520 + }, + { + "epoch": 0.99, + "learning_rate": 8.794383149448345e-06, + "loss": 1.1603, + "theoretical_loss": 3.327863616484931, + "tokens_seen": 3265069056 + }, + { + "epoch": 0.99, + "learning_rate": 8.778335005015046e-06, + "loss": 1.1937, + "theoretical_loss": 3.327858611821788, + "tokens_seen": 3265134592 + }, + { + "epoch": 0.99, + "learning_rate": 8.762286860581747e-06, + "loss": 1.2216, + "theoretical_loss": 3.3278536072872202, + "tokens_seen": 3265200128 + }, + { + "epoch": 0.99, + "learning_rate": 8.746238716148446e-06, + "loss": 1.2528, + "theoretical_loss": 3.327848602881222, + "tokens_seen": 3265265664 + }, + { + "epoch": 0.99, + "learning_rate": 8.730190571715144e-06, + "loss": 1.2165, + "theoretical_loss": 3.327843598603788, + "tokens_seen": 3265331200 + }, + { + "epoch": 0.99, + "learning_rate": 8.714142427281847e-06, + "loss": 1.2101, + "theoretical_loss": 3.3278385944549114, + "tokens_seen": 3265396736 + }, + { + "epoch": 0.99, + "learning_rate": 8.698094282848546e-06, + "loss": 1.2278, + "theoretical_loss": 3.3278335904345866, + "tokens_seen": 3265462272 + }, + { + "epoch": 0.99, + "learning_rate": 8.682046138415246e-06, + "loss": 1.1723, + "theoretical_loss": 3.3278285865428083, + "tokens_seen": 3265527808 + }, + { + "epoch": 0.99, + "learning_rate": 8.665997993981945e-06, + "loss": 1.1442, + "theoretical_loss": 3.32782358277957, + "tokens_seen": 3265593344 + }, + { + "epoch": 0.99, + "learning_rate": 8.649949849548648e-06, + "loss": 1.1321, + "theoretical_loss": 3.327818579144866, + "tokens_seen": 3265658880 + }, + { + "epoch": 0.99, + "learning_rate": 8.633901705115347e-06, + "loss": 1.2121, + "theoretical_loss": 3.3278135756386904, + "tokens_seen": 3265724416 + }, + { + "epoch": 0.99, + "learning_rate": 8.617853560682046e-06, + "loss": 1.1922, + "theoretical_loss": 3.327808572261037, + "tokens_seen": 3265789952 + }, + { + "epoch": 0.99, + "learning_rate": 8.601805416248746e-06, + "loss": 1.1688, + "theoretical_loss": 3.3278035690119006, + "tokens_seen": 3265855488 + }, + { + "epoch": 0.99, + "learning_rate": 8.585757271815447e-06, + "loss": 1.1016, + "theoretical_loss": 3.3277985658912748, + "tokens_seen": 3265921024 + }, + { + "epoch": 0.99, + "learning_rate": 8.569709127382147e-06, + "loss": 1.3015, + "theoretical_loss": 3.3277935628991537, + "tokens_seen": 3265986560 + }, + { + "epoch": 0.99, + "learning_rate": 8.553660982948846e-06, + "loss": 1.1846, + "theoretical_loss": 3.327788560035532, + "tokens_seen": 3266052096 + }, + { + "epoch": 0.99, + "learning_rate": 8.537612838515547e-06, + "loss": 1.1909, + "theoretical_loss": 3.3277835573004033, + "tokens_seen": 3266117632 + }, + { + "epoch": 0.99, + "learning_rate": 8.521564694082248e-06, + "loss": 1.1624, + "theoretical_loss": 3.327778554693762, + "tokens_seen": 3266183168 + }, + { + "epoch": 0.99, + "learning_rate": 8.505516549648947e-06, + "loss": 1.0903, + "theoretical_loss": 3.327773552215602, + "tokens_seen": 3266248704 + }, + { + "epoch": 0.99, + "learning_rate": 8.489468405215647e-06, + "loss": 1.1611, + "theoretical_loss": 3.327768549865917, + "tokens_seen": 3266314240 + }, + { + "epoch": 0.99, + "learning_rate": 8.473420260782348e-06, + "loss": 1.0838, + "theoretical_loss": 3.327763547644702, + "tokens_seen": 3266379776 + }, + { + "epoch": 0.99, + "learning_rate": 8.457372116349048e-06, + "loss": 1.2113, + "theoretical_loss": 3.3277585455519505, + "tokens_seen": 3266445312 + }, + { + "epoch": 0.99, + "learning_rate": 8.441323971915747e-06, + "loss": 1.1395, + "theoretical_loss": 3.327753543587657, + "tokens_seen": 3266510848 + }, + { + "epoch": 0.99, + "learning_rate": 8.425275827482448e-06, + "loss": 1.23, + "theoretical_loss": 3.3277485417518156, + "tokens_seen": 3266576384 + }, + { + "epoch": 0.99, + "learning_rate": 8.409227683049149e-06, + "loss": 1.106, + "theoretical_loss": 3.32774354004442, + "tokens_seen": 3266641920 + }, + { + "epoch": 0.99, + "learning_rate": 8.393179538615848e-06, + "loss": 1.2799, + "theoretical_loss": 3.327738538465465, + "tokens_seen": 3266707456 + }, + { + "epoch": 0.99, + "learning_rate": 8.377131394182548e-06, + "loss": 1.1636, + "theoretical_loss": 3.3277335370149443, + "tokens_seen": 3266772992 + }, + { + "epoch": 0.99, + "learning_rate": 8.361083249749247e-06, + "loss": 1.2068, + "theoretical_loss": 3.3277285356928523, + "tokens_seen": 3266838528 + }, + { + "epoch": 0.99, + "learning_rate": 8.345035105315948e-06, + "loss": 1.174, + "theoretical_loss": 3.3277235344991825, + "tokens_seen": 3266904064 + }, + { + "epoch": 0.99, + "objective/train/docs_used": 1090471, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2105252742767334, + "objective/train/theoretical_loss": 3.3277185334339294, + "objective/train/tokens_used": 3287429600, + "theoretical_loss": 3.3277185334339294, + "tokens_seen": 3266969600 + }, + { + "epoch": 0.99, + "learning_rate": 8.328986960882648e-06, + "loss": 1.2132, + "theoretical_loss": 3.3277185334339294, + "tokens_seen": 3266969600 + }, + { + "epoch": 0.99, + "learning_rate": 8.312938816449349e-06, + "loss": 1.1861, + "theoretical_loss": 3.3277135324970875, + "tokens_seen": 3267035136 + }, + { + "epoch": 0.99, + "learning_rate": 8.296890672016048e-06, + "loss": 1.2001, + "theoretical_loss": 3.3277085316886503, + "tokens_seen": 3267100672 + }, + { + "epoch": 0.99, + "learning_rate": 8.280842527582749e-06, + "loss": 1.1465, + "theoretical_loss": 3.3277035310086123, + "tokens_seen": 3267166208 + }, + { + "epoch": 0.99, + "learning_rate": 8.26479438314945e-06, + "loss": 1.2985, + "theoretical_loss": 3.3276985304569675, + "tokens_seen": 3267231744 + }, + { + "epoch": 0.99, + "learning_rate": 8.248746238716148e-06, + "loss": 1.2745, + "theoretical_loss": 3.3276935300337103, + "tokens_seen": 3267297280 + }, + { + "epoch": 0.99, + "learning_rate": 8.232698094282849e-06, + "loss": 1.2204, + "theoretical_loss": 3.3276885297388343, + "tokens_seen": 3267362816 + }, + { + "epoch": 0.99, + "learning_rate": 8.21664994984955e-06, + "loss": 1.2278, + "theoretical_loss": 3.327683529572334, + "tokens_seen": 3267428352 + }, + { + "epoch": 0.99, + "learning_rate": 8.20060180541625e-06, + "loss": 1.0662, + "theoretical_loss": 3.3276785295342033, + "tokens_seen": 3267493888 + }, + { + "epoch": 0.99, + "learning_rate": 8.184553660982949e-06, + "loss": 1.2202, + "theoretical_loss": 3.3276735296244366, + "tokens_seen": 3267559424 + }, + { + "epoch": 0.99, + "learning_rate": 8.168505516549648e-06, + "loss": 1.1869, + "theoretical_loss": 3.327668529843028, + "tokens_seen": 3267624960 + }, + { + "epoch": 0.99, + "learning_rate": 8.15245737211635e-06, + "loss": 1.2231, + "theoretical_loss": 3.3276635301899717, + "tokens_seen": 3267690496 + }, + { + "epoch": 0.99, + "learning_rate": 8.13640922768305e-06, + "loss": 1.1867, + "theoretical_loss": 3.3276585306652615, + "tokens_seen": 3267756032 + }, + { + "epoch": 0.99, + "learning_rate": 8.12036108324975e-06, + "loss": 1.2093, + "theoretical_loss": 3.3276535312688917, + "tokens_seen": 3267821568 + }, + { + "epoch": 0.99, + "learning_rate": 8.104312938816449e-06, + "loss": 1.2725, + "theoretical_loss": 3.3276485320008566, + "tokens_seen": 3267887104 + }, + { + "epoch": 0.99, + "learning_rate": 8.088264794383151e-06, + "loss": 1.2145, + "theoretical_loss": 3.32764353286115, + "tokens_seen": 3267952640 + }, + { + "epoch": 0.99, + "learning_rate": 8.07221664994985e-06, + "loss": 1.154, + "theoretical_loss": 3.327638533849766, + "tokens_seen": 3268018176 + }, + { + "epoch": 0.99, + "learning_rate": 8.056168505516549e-06, + "loss": 1.1536, + "theoretical_loss": 3.3276335349666994, + "tokens_seen": 3268083712 + }, + { + "epoch": 0.99, + "learning_rate": 8.04012036108325e-06, + "loss": 1.1806, + "theoretical_loss": 3.3276285362119435, + "tokens_seen": 3268149248 + }, + { + "epoch": 0.99, + "learning_rate": 8.02407221664995e-06, + "loss": 1.2029, + "theoretical_loss": 3.327623537585493, + "tokens_seen": 3268214784 + }, + { + "epoch": 0.99, + "learning_rate": 8.008024072216651e-06, + "loss": 1.1354, + "theoretical_loss": 3.327618539087342, + "tokens_seen": 3268280320 + }, + { + "epoch": 0.99, + "learning_rate": 7.99197592778335e-06, + "loss": 1.1628, + "theoretical_loss": 3.3276135407174845, + "tokens_seen": 3268345856 + }, + { + "epoch": 0.99, + "learning_rate": 7.97592778335005e-06, + "loss": 1.2175, + "theoretical_loss": 3.327608542475914, + "tokens_seen": 3268411392 + }, + { + "epoch": 0.99, + "learning_rate": 7.959879638916751e-06, + "loss": 1.1883, + "theoretical_loss": 3.327603544362626, + "tokens_seen": 3268476928 + }, + { + "epoch": 0.99, + "learning_rate": 7.94383149448345e-06, + "loss": 1.1809, + "theoretical_loss": 3.3275985463776134, + "tokens_seen": 3268542464 + }, + { + "epoch": 0.99, + "learning_rate": 7.92778335005015e-06, + "loss": 1.2426, + "theoretical_loss": 3.3275935485208707, + "tokens_seen": 3268608000 + }, + { + "epoch": 0.99, + "learning_rate": 7.911735205616851e-06, + "loss": 1.1423, + "theoretical_loss": 3.3275885507923926, + "tokens_seen": 3268673536 + }, + { + "epoch": 0.99, + "learning_rate": 7.895687061183552e-06, + "loss": 1.1826, + "theoretical_loss": 3.3275835531921722, + "tokens_seen": 3268739072 + }, + { + "epoch": 0.99, + "learning_rate": 7.879638916750251e-06, + "loss": 1.1094, + "theoretical_loss": 3.327578555720205, + "tokens_seen": 3268804608 + }, + { + "epoch": 0.99, + "learning_rate": 7.863590772316952e-06, + "loss": 1.1673, + "theoretical_loss": 3.327573558376484, + "tokens_seen": 3268870144 + }, + { + "epoch": 0.99, + "learning_rate": 7.84754262788365e-06, + "loss": 1.19, + "theoretical_loss": 3.3275685611610033, + "tokens_seen": 3268935680 + }, + { + "epoch": 0.99, + "learning_rate": 7.831494483450351e-06, + "loss": 1.1271, + "theoretical_loss": 3.327563564073758, + "tokens_seen": 3269001216 + }, + { + "epoch": 0.99, + "learning_rate": 7.815446339017052e-06, + "loss": 1.178, + "theoretical_loss": 3.327558567114741, + "tokens_seen": 3269066752 + }, + { + "epoch": 0.99, + "learning_rate": 7.79939819458375e-06, + "loss": 1.0763, + "theoretical_loss": 3.327553570283948, + "tokens_seen": 3269132288 + }, + { + "epoch": 0.99, + "learning_rate": 7.783350050150451e-06, + "loss": 1.1583, + "theoretical_loss": 3.327548573581372, + "tokens_seen": 3269197824 + }, + { + "epoch": 0.99, + "learning_rate": 7.767301905717152e-06, + "loss": 1.1032, + "theoretical_loss": 3.327543577007007, + "tokens_seen": 3269263360 + }, + { + "epoch": 0.99, + "learning_rate": 7.751253761283853e-06, + "loss": 1.2321, + "theoretical_loss": 3.3275385805608475, + "tokens_seen": 3269328896 + }, + { + "epoch": 0.99, + "learning_rate": 7.735205616850552e-06, + "loss": 1.2912, + "theoretical_loss": 3.327533584242888, + "tokens_seen": 3269394432 + }, + { + "epoch": 0.99, + "learning_rate": 7.719157472417252e-06, + "loss": 1.1983, + "theoretical_loss": 3.3275285880531227, + "tokens_seen": 3269459968 + }, + { + "epoch": 0.99, + "learning_rate": 7.703109327983953e-06, + "loss": 1.1012, + "theoretical_loss": 3.3275235919915445, + "tokens_seen": 3269525504 + }, + { + "epoch": 0.99, + "learning_rate": 7.687061183550652e-06, + "loss": 1.1139, + "theoretical_loss": 3.327518596058149, + "tokens_seen": 3269591040 + }, + { + "epoch": 0.99, + "learning_rate": 7.671013039117352e-06, + "loss": 1.1306, + "theoretical_loss": 3.3275136002529297, + "tokens_seen": 3269656576 + }, + { + "epoch": 0.99, + "learning_rate": 7.654964894684053e-06, + "loss": 1.1713, + "theoretical_loss": 3.3275086045758804, + "tokens_seen": 3269722112 + }, + { + "epoch": 0.99, + "learning_rate": 7.638916750250752e-06, + "loss": 1.223, + "theoretical_loss": 3.327503609026996, + "tokens_seen": 3269787648 + }, + { + "epoch": 0.99, + "learning_rate": 7.622868605817453e-06, + "loss": 1.197, + "theoretical_loss": 3.32749861360627, + "tokens_seen": 3269853184 + }, + { + "epoch": 0.99, + "learning_rate": 7.6068204613841525e-06, + "loss": 1.2995, + "theoretical_loss": 3.327493618313697, + "tokens_seen": 3269918720 + }, + { + "epoch": 0.99, + "learning_rate": 7.590772316950853e-06, + "loss": 1.1858, + "theoretical_loss": 3.3274886231492706, + "tokens_seen": 3269984256 + }, + { + "epoch": 0.99, + "learning_rate": 7.574724172517553e-06, + "loss": 1.1747, + "theoretical_loss": 3.327483628112986, + "tokens_seen": 3270049792 + }, + { + "epoch": 0.99, + "learning_rate": 7.5586760280842535e-06, + "loss": 1.2351, + "theoretical_loss": 3.3274786332048363, + "tokens_seen": 3270115328 + }, + { + "epoch": 0.99, + "learning_rate": 7.542627883650953e-06, + "loss": 1.1606, + "theoretical_loss": 3.3274736384248156, + "tokens_seen": 3270180864 + }, + { + "epoch": 0.99, + "objective/train/docs_used": 1091376, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.110980749130249, + "objective/train/theoretical_loss": 3.327468643772919, + "objective/train/tokens_used": 3290706400, + "theoretical_loss": 3.327468643772919, + "tokens_seen": 3270246400 + }, + { + "epoch": 0.99, + "learning_rate": 7.526579739217652e-06, + "loss": 1.1203, + "theoretical_loss": 3.327468643772919, + "tokens_seen": 3270246400 + }, + { + "epoch": 0.99, + "learning_rate": 7.510531594784354e-06, + "loss": 1.1587, + "theoretical_loss": 3.32746364924914, + "tokens_seen": 3270311936 + }, + { + "epoch": 0.99, + "learning_rate": 7.494483450351053e-06, + "loss": 1.2184, + "theoretical_loss": 3.327458654853473, + "tokens_seen": 3270377472 + }, + { + "epoch": 0.99, + "learning_rate": 7.478435305917754e-06, + "loss": 1.147, + "theoretical_loss": 3.3274536605859115, + "tokens_seen": 3270443008 + }, + { + "epoch": 0.99, + "learning_rate": 7.462387161484453e-06, + "loss": 1.2255, + "theoretical_loss": 3.32744866644645, + "tokens_seen": 3270508544 + }, + { + "epoch": 0.99, + "learning_rate": 7.446339017051155e-06, + "loss": 1.0937, + "theoretical_loss": 3.3274436724350833, + "tokens_seen": 3270574080 + }, + { + "epoch": 0.99, + "learning_rate": 7.4302908726178535e-06, + "loss": 1.1448, + "theoretical_loss": 3.327438678551805, + "tokens_seen": 3270639616 + }, + { + "epoch": 0.99, + "learning_rate": 7.414242728184553e-06, + "loss": 1.1995, + "theoretical_loss": 3.327433684796609, + "tokens_seen": 3270705152 + }, + { + "epoch": 0.99, + "learning_rate": 7.398194583751254e-06, + "loss": 1.2644, + "theoretical_loss": 3.3274286911694895, + "tokens_seen": 3270770688 + }, + { + "epoch": 0.99, + "learning_rate": 7.382146439317954e-06, + "loss": 1.1335, + "theoretical_loss": 3.327423697670441, + "tokens_seen": 3270836224 + }, + { + "epoch": 0.99, + "learning_rate": 7.366098294884654e-06, + "loss": 1.117, + "theoretical_loss": 3.3274187042994576, + "tokens_seen": 3270901760 + }, + { + "epoch": 0.99, + "learning_rate": 7.350050150451354e-06, + "loss": 1.1486, + "theoretical_loss": 3.3274137110565336, + "tokens_seen": 3270967296 + }, + { + "epoch": 0.99, + "learning_rate": 7.334002006018055e-06, + "loss": 1.1588, + "theoretical_loss": 3.3274087179416627, + "tokens_seen": 3271032832 + }, + { + "epoch": 0.99, + "learning_rate": 7.317953861584755e-06, + "loss": 1.1915, + "theoretical_loss": 3.3274037249548396, + "tokens_seen": 3271098368 + }, + { + "epoch": 0.99, + "learning_rate": 7.301905717151454e-06, + "loss": 1.1832, + "theoretical_loss": 3.3273987320960576, + "tokens_seen": 3271163904 + }, + { + "epoch": 0.99, + "learning_rate": 7.285857572718155e-06, + "loss": 1.2084, + "theoretical_loss": 3.3273937393653115, + "tokens_seen": 3271229440 + }, + { + "epoch": 0.99, + "learning_rate": 7.269809428284855e-06, + "loss": 1.1975, + "theoretical_loss": 3.3273887467625958, + "tokens_seen": 3271294976 + }, + { + "epoch": 0.99, + "learning_rate": 7.2537612838515554e-06, + "loss": 1.266, + "theoretical_loss": 3.3273837542879035, + "tokens_seen": 3271360512 + }, + { + "epoch": 0.99, + "learning_rate": 7.237713139418255e-06, + "loss": 1.2278, + "theoretical_loss": 3.3273787619412296, + "tokens_seen": 3271426048 + }, + { + "epoch": 0.99, + "learning_rate": 7.221664994984956e-06, + "loss": 1.208, + "theoretical_loss": 3.327373769722568, + "tokens_seen": 3271491584 + }, + { + "epoch": 0.99, + "learning_rate": 7.205616850551656e-06, + "loss": 1.2087, + "theoretical_loss": 3.3273687776319134, + "tokens_seen": 3271557120 + }, + { + "epoch": 0.99, + "learning_rate": 7.189568706118355e-06, + "loss": 1.1839, + "theoretical_loss": 3.327363785669259, + "tokens_seen": 3271622656 + }, + { + "epoch": 0.99, + "learning_rate": 7.173520561685056e-06, + "loss": 1.1009, + "theoretical_loss": 3.3273587938346, + "tokens_seen": 3271688192 + }, + { + "epoch": 0.99, + "learning_rate": 7.157472417251755e-06, + "loss": 1.2318, + "theoretical_loss": 3.3273538021279294, + "tokens_seen": 3271753728 + }, + { + "epoch": 0.99, + "learning_rate": 7.141424272818456e-06, + "loss": 1.1318, + "theoretical_loss": 3.327348810549242, + "tokens_seen": 3271819264 + }, + { + "epoch": 0.99, + "learning_rate": 7.125376128385155e-06, + "loss": 1.1788, + "theoretical_loss": 3.327343819098532, + "tokens_seen": 3271884800 + }, + { + "epoch": 0.99, + "learning_rate": 7.109327983951856e-06, + "loss": 1.1866, + "theoretical_loss": 3.327338827775794, + "tokens_seen": 3271950336 + }, + { + "epoch": 0.99, + "learning_rate": 7.093279839518556e-06, + "loss": 1.1079, + "theoretical_loss": 3.3273338365810208, + "tokens_seen": 3272015872 + }, + { + "epoch": 0.99, + "learning_rate": 7.077231695085256e-06, + "loss": 1.2035, + "theoretical_loss": 3.327328845514208, + "tokens_seen": 3272081408 + }, + { + "epoch": 0.99, + "learning_rate": 7.061183550651956e-06, + "loss": 1.2245, + "theoretical_loss": 3.327323854575349, + "tokens_seen": 3272146944 + }, + { + "epoch": 0.99, + "learning_rate": 7.045135406218656e-06, + "loss": 1.1214, + "theoretical_loss": 3.3273188637644378, + "tokens_seen": 3272212480 + }, + { + "epoch": 0.99, + "learning_rate": 7.029087261785357e-06, + "loss": 1.1671, + "theoretical_loss": 3.3273138730814686, + "tokens_seen": 3272278016 + }, + { + "epoch": 0.99, + "learning_rate": 7.0130391173520565e-06, + "loss": 1.1144, + "theoretical_loss": 3.3273088825264363, + "tokens_seen": 3272343552 + }, + { + "epoch": 0.99, + "learning_rate": 6.996990972918757e-06, + "loss": 1.1544, + "theoretical_loss": 3.3273038920993345, + "tokens_seen": 3272409088 + }, + { + "epoch": 0.99, + "learning_rate": 6.980942828485457e-06, + "loss": 1.1521, + "theoretical_loss": 3.3272989018001575, + "tokens_seen": 3272474624 + }, + { + "epoch": 0.99, + "learning_rate": 6.964894684052156e-06, + "loss": 1.1833, + "theoretical_loss": 3.327293911628899, + "tokens_seen": 3272540160 + }, + { + "epoch": 0.99, + "learning_rate": 6.948846539618857e-06, + "loss": 1.1437, + "theoretical_loss": 3.327288921585554, + "tokens_seen": 3272605696 + }, + { + "epoch": 0.99, + "learning_rate": 6.932798395185556e-06, + "loss": 1.1769, + "theoretical_loss": 3.3272839316701157, + "tokens_seen": 3272671232 + }, + { + "epoch": 0.99, + "learning_rate": 6.916750250752258e-06, + "loss": 1.1642, + "theoretical_loss": 3.3272789418825788, + "tokens_seen": 3272736768 + }, + { + "epoch": 0.99, + "learning_rate": 6.900702106318957e-06, + "loss": 1.2056, + "theoretical_loss": 3.3272739522229378, + "tokens_seen": 3272802304 + }, + { + "epoch": 0.99, + "learning_rate": 6.884653961885658e-06, + "loss": 1.2394, + "theoretical_loss": 3.3272689626911864, + "tokens_seen": 3272867840 + }, + { + "epoch": 0.99, + "learning_rate": 6.868605817452357e-06, + "loss": 1.0317, + "theoretical_loss": 3.3272639732873186, + "tokens_seen": 3272933376 + }, + { + "epoch": 0.99, + "learning_rate": 6.852557673019057e-06, + "loss": 1.2324, + "theoretical_loss": 3.327258984011329, + "tokens_seen": 3272998912 + }, + { + "epoch": 0.99, + "learning_rate": 6.8365095285857575e-06, + "loss": 1.2342, + "theoretical_loss": 3.3272539948632116, + "tokens_seen": 3273064448 + }, + { + "epoch": 0.99, + "learning_rate": 6.820461384152457e-06, + "loss": 1.1444, + "theoretical_loss": 3.32724900584296, + "tokens_seen": 3273129984 + }, + { + "epoch": 0.99, + "learning_rate": 6.804413239719158e-06, + "loss": 1.2276, + "theoretical_loss": 3.3272440169505697, + "tokens_seen": 3273195520 + }, + { + "epoch": 0.99, + "learning_rate": 6.788365095285858e-06, + "loss": 1.1047, + "theoretical_loss": 3.3272390281860336, + "tokens_seen": 3273261056 + }, + { + "epoch": 0.99, + "learning_rate": 6.772316950852558e-06, + "loss": 1.1755, + "theoretical_loss": 3.3272340395493463, + "tokens_seen": 3273326592 + }, + { + "epoch": 0.99, + "learning_rate": 6.756268806419258e-06, + "loss": 1.157, + "theoretical_loss": 3.327229051040502, + "tokens_seen": 3273392128 + }, + { + "epoch": 0.99, + "learning_rate": 6.740220661985958e-06, + "loss": 1.12, + "theoretical_loss": 3.3272240626594947, + "tokens_seen": 3273457664 + }, + { + "epoch": 0.99, + "objective/train/docs_used": 1092629, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1520678997039795, + "objective/train/theoretical_loss": 3.327219074406319, + "objective/train/tokens_used": 3293983200, + "theoretical_loss": 3.327219074406319, + "tokens_seen": 3273523200 + }, + { + "epoch": 0.99, + "learning_rate": 6.724172517552659e-06, + "loss": 1.1968, + "theoretical_loss": 3.327219074406319, + "tokens_seen": 3273523200 + }, + { + "epoch": 0.99, + "learning_rate": 6.708124373119358e-06, + "loss": 1.1577, + "theoretical_loss": 3.3272140862809687, + "tokens_seen": 3273588736 + }, + { + "epoch": 0.99, + "learning_rate": 6.692076228686059e-06, + "loss": 1.0812, + "theoretical_loss": 3.327209098283438, + "tokens_seen": 3273654272 + }, + { + "epoch": 0.99, + "learning_rate": 6.676028084252759e-06, + "loss": 1.1937, + "theoretical_loss": 3.327204110413721, + "tokens_seen": 3273719808 + }, + { + "epoch": 0.99, + "learning_rate": 6.6599799398194594e-06, + "loss": 1.1556, + "theoretical_loss": 3.327199122671812, + "tokens_seen": 3273785344 + }, + { + "epoch": 0.99, + "learning_rate": 6.643931795386158e-06, + "loss": 1.2099, + "theoretical_loss": 3.327194135057705, + "tokens_seen": 3273850880 + }, + { + "epoch": 0.99, + "learning_rate": 6.627883650952858e-06, + "loss": 1.1049, + "theoretical_loss": 3.327189147571395, + "tokens_seen": 3273916416 + }, + { + "epoch": 0.99, + "learning_rate": 6.611835506519559e-06, + "loss": 1.1179, + "theoretical_loss": 3.3271841602128744, + "tokens_seen": 3273981952 + }, + { + "epoch": 0.99, + "learning_rate": 6.5957873620862586e-06, + "loss": 1.1987, + "theoretical_loss": 3.3271791729821394, + "tokens_seen": 3274047488 + }, + { + "epoch": 0.99, + "learning_rate": 6.579739217652959e-06, + "loss": 1.0473, + "theoretical_loss": 3.3271741858791826, + "tokens_seen": 3274113024 + }, + { + "epoch": 0.99, + "learning_rate": 6.563691073219659e-06, + "loss": 1.1652, + "theoretical_loss": 3.327169198903999, + "tokens_seen": 3274178560 + }, + { + "epoch": 0.99, + "learning_rate": 6.54764292878636e-06, + "loss": 1.2208, + "theoretical_loss": 3.3271642120565827, + "tokens_seen": 3274244096 + }, + { + "epoch": 0.99, + "learning_rate": 6.531594784353059e-06, + "loss": 1.2773, + "theoretical_loss": 3.3271592253369273, + "tokens_seen": 3274309632 + }, + { + "epoch": 0.99, + "learning_rate": 6.515546639919759e-06, + "loss": 1.2071, + "theoretical_loss": 3.3271542387450275, + "tokens_seen": 3274375168 + }, + { + "epoch": 0.99, + "learning_rate": 6.49949849548646e-06, + "loss": 1.1455, + "theoretical_loss": 3.3271492522808774, + "tokens_seen": 3274440704 + }, + { + "epoch": 0.99, + "learning_rate": 6.48345035105316e-06, + "loss": 1.1523, + "theoretical_loss": 3.3271442659444714, + "tokens_seen": 3274506240 + }, + { + "epoch": 0.99, + "learning_rate": 6.46740220661986e-06, + "loss": 1.1404, + "theoretical_loss": 3.3271392797358033, + "tokens_seen": 3274571776 + }, + { + "epoch": 0.99, + "learning_rate": 6.45135406218656e-06, + "loss": 1.2113, + "theoretical_loss": 3.327134293654867, + "tokens_seen": 3274637312 + }, + { + "epoch": 0.99, + "learning_rate": 6.435305917753261e-06, + "loss": 1.093, + "theoretical_loss": 3.3271293077016573, + "tokens_seen": 3274702848 + }, + { + "epoch": 0.99, + "learning_rate": 6.4192577733199605e-06, + "loss": 1.1096, + "theoretical_loss": 3.327124321876168, + "tokens_seen": 3274768384 + }, + { + "epoch": 0.99, + "learning_rate": 6.403209628886659e-06, + "loss": 1.203, + "theoretical_loss": 3.3271193361783937, + "tokens_seen": 3274833920 + }, + { + "epoch": 0.99, + "learning_rate": 6.387161484453361e-06, + "loss": 1.1718, + "theoretical_loss": 3.327114350608328, + "tokens_seen": 3274899456 + }, + { + "epoch": 0.99, + "learning_rate": 6.37111334002006e-06, + "loss": 1.101, + "theoretical_loss": 3.327109365165965, + "tokens_seen": 3274964992 + }, + { + "epoch": 0.99, + "learning_rate": 6.355065195586761e-06, + "loss": 1.1772, + "theoretical_loss": 3.3271043798513, + "tokens_seen": 3275030528 + }, + { + "epoch": 0.99, + "learning_rate": 6.33901705115346e-06, + "loss": 1.2785, + "theoretical_loss": 3.3270993946643257, + "tokens_seen": 3275096064 + }, + { + "epoch": 0.99, + "learning_rate": 6.322968906720162e-06, + "loss": 1.1962, + "theoretical_loss": 3.327094409605037, + "tokens_seen": 3275161600 + }, + { + "epoch": 0.99, + "learning_rate": 6.306920762286861e-06, + "loss": 1.0833, + "theoretical_loss": 3.3270894246734284, + "tokens_seen": 3275227136 + }, + { + "epoch": 0.99, + "learning_rate": 6.2908726178535605e-06, + "loss": 1.2104, + "theoretical_loss": 3.3270844398694934, + "tokens_seen": 3275292672 + }, + { + "epoch": 0.99, + "learning_rate": 6.274824473420261e-06, + "loss": 1.0191, + "theoretical_loss": 3.3270794551932266, + "tokens_seen": 3275358208 + }, + { + "epoch": 0.99, + "learning_rate": 6.258776328986961e-06, + "loss": 1.2048, + "theoretical_loss": 3.327074470644622, + "tokens_seen": 3275423744 + }, + { + "epoch": 0.99, + "learning_rate": 6.242728184553661e-06, + "loss": 1.2, + "theoretical_loss": 3.3270694862236736, + "tokens_seen": 3275489280 + }, + { + "epoch": 0.99, + "learning_rate": 6.226680040120361e-06, + "loss": 1.116, + "theoretical_loss": 3.3270645019303764, + "tokens_seen": 3275554816 + }, + { + "epoch": 0.99, + "learning_rate": 6.210631895687061e-06, + "loss": 1.2251, + "theoretical_loss": 3.3270595177647238, + "tokens_seen": 3275620352 + }, + { + "epoch": 0.99, + "learning_rate": 6.194583751253762e-06, + "loss": 1.2453, + "theoretical_loss": 3.32705453372671, + "tokens_seen": 3275685888 + }, + { + "epoch": 0.99, + "learning_rate": 6.1785356068204615e-06, + "loss": 1.1743, + "theoretical_loss": 3.327049549816329, + "tokens_seen": 3275751424 + }, + { + "epoch": 0.99, + "learning_rate": 6.162487462387162e-06, + "loss": 1.2037, + "theoretical_loss": 3.3270445660335755, + "tokens_seen": 3275816960 + }, + { + "epoch": 0.99, + "learning_rate": 6.146439317953862e-06, + "loss": 1.2911, + "theoretical_loss": 3.3270395823784438, + "tokens_seen": 3275882496 + }, + { + "epoch": 0.99, + "learning_rate": 6.130391173520562e-06, + "loss": 1.1371, + "theoretical_loss": 3.3270345988509273, + "tokens_seen": 3275948032 + }, + { + "epoch": 0.99, + "learning_rate": 6.1143430290872615e-06, + "loss": 1.2286, + "theoretical_loss": 3.3270296154510213, + "tokens_seen": 3276013568 + }, + { + "epoch": 0.99, + "learning_rate": 6.098294884653962e-06, + "loss": 1.1642, + "theoretical_loss": 3.327024632178719, + "tokens_seen": 3276079104 + }, + { + "epoch": 0.99, + "learning_rate": 6.082246740220662e-06, + "loss": 1.1444, + "theoretical_loss": 3.327019649034015, + "tokens_seen": 3276144640 + }, + { + "epoch": 0.99, + "learning_rate": 6.0661985957873626e-06, + "loss": 1.1625, + "theoretical_loss": 3.327014666016903, + "tokens_seen": 3276210176 + }, + { + "epoch": 0.99, + "learning_rate": 6.050150451354062e-06, + "loss": 1.1749, + "theoretical_loss": 3.3270096831273777, + "tokens_seen": 3276275712 + }, + { + "epoch": 0.99, + "learning_rate": 6.034102306920763e-06, + "loss": 1.2034, + "theoretical_loss": 3.3270047003654337, + "tokens_seen": 3276341248 + }, + { + "epoch": 0.99, + "learning_rate": 6.018054162487463e-06, + "loss": 1.1848, + "theoretical_loss": 3.326999717731064, + "tokens_seen": 3276406784 + }, + { + "epoch": 0.99, + "learning_rate": 6.0020060180541626e-06, + "loss": 1.2072, + "theoretical_loss": 3.3269947352242637, + "tokens_seen": 3276472320 + }, + { + "epoch": 0.99, + "learning_rate": 5.985957873620863e-06, + "loss": 1.1502, + "theoretical_loss": 3.3269897528450265, + "tokens_seen": 3276537856 + }, + { + "epoch": 0.99, + "learning_rate": 5.969909729187563e-06, + "loss": 1.0734, + "theoretical_loss": 3.326984770593347, + "tokens_seen": 3276603392 + }, + { + "epoch": 0.99, + "learning_rate": 5.953861584754264e-06, + "loss": 1.1109, + "theoretical_loss": 3.326979788469219, + "tokens_seen": 3276668928 + }, + { + "epoch": 0.99, + "learning_rate": 5.937813440320963e-06, + "loss": 1.1734, + "theoretical_loss": 3.326974806472637, + "tokens_seen": 3276734464 + }, + { + "debugging/Compilability": 0.0, + "debugging/distinct-1-grams": 0.6789873647402326, + "debugging/entropy-1-grams": 4.704768879036057, + "debugging/length": 528.3333333333334, + "debugging/num_segments": 6, + "debugging/score": 0.06076800725343776, + "debugging/score_std": 0.01756025935850065, + "epoch": 0.99, + "objective/train/docs_used": 1093820, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.1526380777359009, + "objective/train/theoretical_loss": 3.326969824603595, + "objective/train/tokens_used": 3297260000, + "theoretical_loss": 3.326969824603595, + "tokens_seen": 3276800000 + }, + { + "epoch": 0.99, + "learning_rate": 5.921765295887663e-06, + "loss": 1.2075, + "theoretical_loss": 3.326969824603595, + "tokens_seen": 3276800000 + } + ], + "max_steps": 50354, + "num_train_epochs": 9223372036854775807, + "total_flos": 1.6722690048e+18, + "trial_name": null, + "trial_params": null +}