diff --git "a/checkpoint-10000/trainer_state.json" "b/checkpoint-10000/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-10000/trainer_state.json" @@ -0,0 +1,72230 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.19859395480001588, + "global_step": 10000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 1.5873015873015873e-06, + "loss": 10.5641, + "theoretical_loss": 20.84128112621979, + "tokens_seen": 65536 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031746e-06, + "loss": 10.5513, + "theoretical_loss": 17.594664429001284, + "tokens_seen": 131072 + }, + { + "epoch": 0.0, + "learning_rate": 4.7619047619047615e-06, + "loss": 10.4631, + "theoretical_loss": 15.967940417509208, + "tokens_seen": 196608 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-06, + "loss": 10.2495, + "theoretical_loss": 14.92078008066121, + "tokens_seen": 262144 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507936e-06, + "loss": 10.0077, + "theoretical_loss": 14.164680262435617, + "tokens_seen": 327680 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809523e-06, + "loss": 9.7256, + "theoretical_loss": 13.581024797222863, + "tokens_seen": 393216 + }, + { + "epoch": 0.0, + "learning_rate": 1.1111111111111112e-05, + "loss": 9.5124, + "theoretical_loss": 13.11027232607383, + "tokens_seen": 458752 + }, + { + "epoch": 0.0, + "learning_rate": 1.2698412698412699e-05, + "loss": 9.3381, + "theoretical_loss": 12.718592950155966, + "tokens_seen": 524288 + }, + { + "epoch": 0.0, + "learning_rate": 1.4285714285714285e-05, + "loss": 9.1669, + "theoretical_loss": 12.385055788546264, + "tokens_seen": 589824 + }, + { + "epoch": 0.0, + "learning_rate": 1.5873015873015872e-05, + "loss": 8.9763, + "theoretical_loss": 12.09587593170772, + "tokens_seen": 655360 + }, + { + "epoch": 0.0, + "learning_rate": 1.746031746031746e-05, + "loss": 8.8295, + "theoretical_loss": 11.841529555774319, + "tokens_seen": 720896 + }, + { + "epoch": 0.0, + "learning_rate": 1.9047619047619046e-05, + "loss": 8.8699, + "theoretical_loss": 11.615182533379372, + "tokens_seen": 786432 + }, + { + "epoch": 0.0, + "learning_rate": 2.0634920634920636e-05, + "loss": 8.7295, + "theoretical_loss": 11.41177746485943, + "tokens_seen": 851968 + }, + { + "epoch": 0.0, + "learning_rate": 2.2222222222222223e-05, + "loss": 8.6618, + "theoretical_loss": 11.227475026784514, + "tokens_seen": 917504 + }, + { + "epoch": 0.0, + "learning_rate": 2.380952380952381e-05, + "loss": 8.6244, + "theoretical_loss": 11.059297242650555, + "tokens_seen": 983040 + }, + { + "epoch": 0.0, + "learning_rate": 2.5396825396825397e-05, + "loss": 8.5977, + "theoretical_loss": 10.90489141112959, + "tokens_seen": 1048576 + }, + { + "epoch": 0.0, + "learning_rate": 2.6984126984126984e-05, + "loss": 8.4756, + "theoretical_loss": 10.762369064735367, + "tokens_seen": 1114112 + }, + { + "epoch": 0.0, + "learning_rate": 2.857142857142857e-05, + "loss": 8.4981, + "theoretical_loss": 10.63019320090292, + "tokens_seen": 1179648 + }, + { + "epoch": 0.0, + "learning_rate": 3.0158730158730158e-05, + "loss": 8.3172, + "theoretical_loss": 10.507097497029282, + "tokens_seen": 1245184 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031745e-05, + "loss": 8.176, + "theoretical_loss": 10.392027268435971, + "tokens_seen": 1310720 + }, + { + "epoch": 0.0, + "learning_rate": 3.3333333333333335e-05, + "loss": 8.1964, + "theoretical_loss": 10.284095545569386, + "tokens_seen": 1376256 + }, + { + "epoch": 0.0, + "learning_rate": 3.492063492063492e-05, + "loss": 8.1379, + "theoretical_loss": 10.182549877942659, + "tokens_seen": 1441792 + }, + { + "epoch": 0.0, + "learning_rate": 3.650793650793651e-05, + "loss": 7.9969, + "theoretical_loss": 10.08674688640524, + "tokens_seen": 1507328 + }, + { + "epoch": 0.0, + "learning_rate": 3.809523809523809e-05, + "loss": 8.1059, + "theoretical_loss": 9.99613250351292, + "tokens_seen": 1572864 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-05, + "loss": 7.9094, + "theoretical_loss": 9.91022645106575, + "tokens_seen": 1638400 + }, + { + "epoch": 0.0, + "learning_rate": 4.126984126984127e-05, + "loss": 7.7211, + "theoretical_loss": 9.828609916213201, + "tokens_seen": 1703936 + }, + { + "epoch": 0.0, + "learning_rate": 4.2857142857142856e-05, + "loss": 7.7885, + "theoretical_loss": 9.750915671580087, + "tokens_seen": 1769472 + }, + { + "epoch": 0.0, + "learning_rate": 4.4444444444444447e-05, + "loss": 7.7711, + "theoretical_loss": 9.676820083754187, + "tokens_seen": 1835008 + }, + { + "epoch": 0.0, + "learning_rate": 4.603174603174603e-05, + "loss": 7.6633, + "theoretical_loss": 9.606036595818281, + "tokens_seen": 1900544 + }, + { + "epoch": 0.0, + "learning_rate": 4.761904761904762e-05, + "loss": 7.5596, + "theoretical_loss": 9.538310371437493, + "tokens_seen": 1966080 + }, + { + "epoch": 0.0, + "learning_rate": 4.9206349206349204e-05, + "loss": 7.5956, + "theoretical_loss": 9.473413862304575, + "tokens_seen": 2031616 + }, + { + "epoch": 0.0, + "learning_rate": 5.0793650793650794e-05, + "loss": 7.4655, + "theoretical_loss": 9.411143115583098, + "tokens_seen": 2097152 + }, + { + "epoch": 0.0, + "learning_rate": 5.2380952380952384e-05, + "loss": 7.5066, + "theoretical_loss": 9.351314678906622, + "tokens_seen": 2162688 + }, + { + "epoch": 0.0, + "learning_rate": 5.396825396825397e-05, + "loss": 7.3266, + "theoretical_loss": 9.293762991332915, + "tokens_seen": 2228224 + }, + { + "epoch": 0.0, + "learning_rate": 5.555555555555556e-05, + "loss": 7.2918, + "theoretical_loss": 9.238338172117398, + "tokens_seen": 2293760 + }, + { + "epoch": 0.0, + "learning_rate": 5.714285714285714e-05, + "loss": 7.3421, + "theoretical_loss": 9.184904137180933, + "tokens_seen": 2359296 + }, + { + "epoch": 0.0, + "learning_rate": 5.873015873015873e-05, + "loss": 7.2943, + "theoretical_loss": 9.133336987085992, + "tokens_seen": 2424832 + }, + { + "epoch": 0.0, + "learning_rate": 6.0317460317460316e-05, + "loss": 7.277, + "theoretical_loss": 9.083523621206375, + "tokens_seen": 2490368 + }, + { + "epoch": 0.0, + "learning_rate": 6.19047619047619e-05, + "loss": 7.1754, + "theoretical_loss": 9.03536054131647, + "tokens_seen": 2555904 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-05, + "loss": 7.1903, + "theoretical_loss": 8.988752814581996, + "tokens_seen": 2621440 + }, + { + "epoch": 0.0, + "learning_rate": 6.507936507936509e-05, + "loss": 7.1207, + "theoretical_loss": 8.943613171313253, + "tokens_seen": 2686976 + }, + { + "epoch": 0.0, + "learning_rate": 6.666666666666667e-05, + "loss": 7.0332, + "theoretical_loss": 8.899861217150864, + "tokens_seen": 2752512 + }, + { + "epoch": 0.0, + "learning_rate": 6.825396825396825e-05, + "loss": 6.8774, + "theoretical_loss": 8.85742274282614, + "tokens_seen": 2818048 + }, + { + "epoch": 0.0, + "learning_rate": 6.984126984126984e-05, + "loss": 6.9871, + "theoretical_loss": 8.816229117451055, + "tokens_seen": 2883584 + }, + { + "epoch": 0.0, + "learning_rate": 7.142857142857143e-05, + "loss": 6.7777, + "theoretical_loss": 8.776216753584098, + "tokens_seen": 2949120 + }, + { + "epoch": 0.0, + "learning_rate": 7.301587301587302e-05, + "loss": 6.8747, + "theoretical_loss": 8.737326634193474, + "tokens_seen": 3014656 + }, + { + "epoch": 0.0, + "learning_rate": 7.460317460317461e-05, + "loss": 6.6371, + "theoretical_loss": 8.699503893181488, + "tokens_seen": 3080192 + }, + { + "epoch": 0.0, + "learning_rate": 7.619047619047618e-05, + "loss": 6.5967, + "theoretical_loss": 8.662697442408115, + "tokens_seen": 3145728 + }, + { + "epoch": 0.0, + "learning_rate": 7.777777777777778e-05, + "loss": 6.7174, + "theoretical_loss": 8.626859639208966, + "tokens_seen": 3211264 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 7703, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 6.697691917419434, + "objective/train/theoretical_loss": 8.591945989283708, + "objective/train/tokens_used": 23736800, + "theoretical_loss": 8.591945989283708, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507937e-05, + "loss": 6.7105, + "theoretical_loss": 8.591945989283708, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 8.095238095238096e-05, + "loss": 6.599, + "theoretical_loss": 8.557914880567758, + "tokens_seen": 3342336 + }, + { + "epoch": 0.0, + "learning_rate": 8.253968253968255e-05, + "loss": 6.4888, + "theoretical_loss": 8.524727344318643, + "tokens_seen": 3407872 + }, + { + "epoch": 0.0, + "learning_rate": 8.412698412698413e-05, + "loss": 6.4092, + "theoretical_loss": 8.492346840169828, + "tokens_seen": 3473408 + }, + { + "epoch": 0.0, + "learning_rate": 8.571428571428571e-05, + "loss": 6.5305, + "theoretical_loss": 8.460739062345422, + "tokens_seen": 3538944 + }, + { + "epoch": 0.0, + "learning_rate": 8.730158730158731e-05, + "loss": 6.3927, + "theoretical_loss": 8.429871764603359, + "tokens_seen": 3604480 + }, + { + "epoch": 0.0, + "learning_rate": 8.888888888888889e-05, + "loss": 6.2062, + "theoretical_loss": 8.399714601792851, + "tokens_seen": 3670016 + }, + { + "epoch": 0.0, + "learning_rate": 9.047619047619048e-05, + "loss": 6.2861, + "theoretical_loss": 8.370238986183718, + "tokens_seen": 3735552 + }, + { + "epoch": 0.0, + "learning_rate": 9.206349206349206e-05, + "loss": 6.2594, + "theoretical_loss": 8.34141795695797, + "tokens_seen": 3801088 + }, + { + "epoch": 0.0, + "learning_rate": 9.365079365079366e-05, + "loss": 6.0057, + "theoretical_loss": 8.313226061453822, + "tokens_seen": 3866624 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809524e-05, + "loss": 6.1933, + "theoretical_loss": 8.285639246924504, + "tokens_seen": 3932160 + }, + { + "epoch": 0.0, + "learning_rate": 9.682539682539682e-05, + "loss": 6.1101, + "theoretical_loss": 8.258634761722755, + "tokens_seen": 3997696 + }, + { + "epoch": 0.0, + "learning_rate": 9.841269841269841e-05, + "loss": 6.0988, + "theoretical_loss": 8.232191064950612, + "tokens_seen": 4063232 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001, + "loss": 6.0056, + "theoretical_loss": 8.206287743725687, + "tokens_seen": 4128768 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010158730158730159, + "loss": 6.0406, + "theoretical_loss": 8.180905437312258, + "tokens_seen": 4194304 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010317460317460319, + "loss": 5.9078, + "theoretical_loss": 8.15602576745014, + "tokens_seen": 4259840 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010476190476190477, + "loss": 5.7249, + "theoretical_loss": 8.13163127428835, + "tokens_seen": 4325376 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010634920634920635, + "loss": 5.847, + "theoretical_loss": 8.107705357395282, + "tokens_seen": 4390912 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010793650793650794, + "loss": 5.7916, + "theoretical_loss": 8.084232221374057, + "tokens_seen": 4456448 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010952380952380953, + "loss": 5.7336, + "theoretical_loss": 8.06119682566165, + "tokens_seen": 4521984 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011111111111111112, + "loss": 5.7012, + "theoretical_loss": 8.038584838134476, + "tokens_seen": 4587520 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001126984126984127, + "loss": 5.6223, + "theoretical_loss": 8.016382592182019, + "tokens_seen": 4653056 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011428571428571428, + "loss": 5.6187, + "theoretical_loss": 7.994577046944442, + "tokens_seen": 4718592 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001158730158730159, + "loss": 5.5594, + "theoretical_loss": 7.973155750440629, + "tokens_seen": 4784128 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011746031746031746, + "loss": 5.5114, + "theoretical_loss": 7.95210680534016, + "tokens_seen": 4849664 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011904761904761905, + "loss": 5.7208, + "theoretical_loss": 7.931418837156708, + "tokens_seen": 4915200 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012063492063492063, + "loss": 5.6176, + "theoretical_loss": 7.911080964661844, + "tokens_seen": 4980736 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012222222222222224, + "loss": 5.4612, + "theoretical_loss": 7.891082772337269, + "tokens_seen": 5046272 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001238095238095238, + "loss": 5.4758, + "theoretical_loss": 7.871414284700578, + "tokens_seen": 5111808 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001253968253968254, + "loss": 5.4206, + "theoretical_loss": 7.852065942354954, + "tokens_seen": 5177344 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012698412698412698, + "loss": 5.4236, + "theoretical_loss": 7.833028579626806, + "tokens_seen": 5242880 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012857142857142858, + "loss": 5.526, + "theoretical_loss": 7.814293403667714, + "tokens_seen": 5308416 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013015873015873017, + "loss": 5.4883, + "theoretical_loss": 7.795851974908005, + "tokens_seen": 5373952 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013174603174603174, + "loss": 5.3321, + "theoretical_loss": 7.77769618875926, + "tokens_seen": 5439488 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013333333333333334, + "loss": 5.3773, + "theoretical_loss": 7.7598182584719595, + "tokens_seen": 5505024 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013492063492063494, + "loss": 5.3975, + "theoretical_loss": 7.742210699062561, + "tokens_seen": 5570560 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001365079365079365, + "loss": 5.301, + "theoretical_loss": 7.724866312231575, + "tokens_seen": 5636096 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001380952380952381, + "loss": 5.3225, + "theoretical_loss": 7.707778172200822, + "tokens_seen": 5701632 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013968253968253967, + "loss": 5.4532, + "theoretical_loss": 7.6909396124039695, + "tokens_seen": 5767168 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001412698412698413, + "loss": 5.3501, + "theoretical_loss": 7.67434421296992, + "tokens_seen": 5832704 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014285714285714287, + "loss": 5.3752, + "theoretical_loss": 7.657985788943502, + "tokens_seen": 5898240 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014444444444444444, + "loss": 5.1601, + "theoretical_loss": 7.641858379192337, + "tokens_seen": 5963776 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014603174603174603, + "loss": 5.1855, + "theoretical_loss": 7.625956235952921, + "tokens_seen": 6029312 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014761904761904763, + "loss": 5.2364, + "theoretical_loss": 7.610273814972529, + "tokens_seen": 6094848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014920634920634923, + "loss": 5.1903, + "theoretical_loss": 7.594805766207012, + "tokens_seen": 6160384 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001507936507936508, + "loss": 5.2898, + "theoretical_loss": 7.57954692503759, + "tokens_seen": 6225920 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015238095238095237, + "loss": 5.1998, + "theoretical_loss": 7.5644923039725604, + "tokens_seen": 6291456 + }, + { + "epoch": 0.0, + "learning_rate": 0.000153968253968254, + "loss": 4.9998, + "theoretical_loss": 7.549637084802403, + "tokens_seen": 6356992 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015555555555555556, + "loss": 5.2391, + "theoretical_loss": 7.534976611179138, + "tokens_seen": 6422528 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015714285714285716, + "loss": 5.1035, + "theoretical_loss": 7.52050638159289, + "tokens_seen": 6488064 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 8901, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 4.883451461791992, + "objective/train/theoretical_loss": 7.506222042720672, + "objective/train/tokens_used": 27013600, + "theoretical_loss": 7.506222042720672, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015873015873015873, + "loss": 5.0335, + "theoretical_loss": 7.506222042720672, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016031746031746033, + "loss": 5.1059, + "theoretical_loss": 7.492119383124103, + "tokens_seen": 6619136 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016190476190476192, + "loss": 5.2313, + "theoretical_loss": 7.478194327274557, + "tokens_seen": 6684672 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001634920634920635, + "loss": 5.0888, + "theoretical_loss": 7.46444292988566, + "tokens_seen": 6750208 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001650793650793651, + "loss": 4.9131, + "theoretical_loss": 7.450861370534546, + "tokens_seen": 6815744 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001666666666666667, + "loss": 5.1414, + "theoretical_loss": 7.437445948554516, + "tokens_seen": 6881280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016825396825396826, + "loss": 5.0188, + "theoretical_loss": 7.424193078182986, + "tokens_seen": 6946816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016984126984126986, + "loss": 5.0327, + "theoretical_loss": 7.411099283949674, + "tokens_seen": 7012352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017142857142857143, + "loss": 5.1783, + "theoretical_loss": 7.3981611962910145, + "tokens_seen": 7077888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017301587301587302, + "loss": 5.1053, + "theoretical_loss": 7.38537554737773, + "tokens_seen": 7143424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017460317460317462, + "loss": 4.9697, + "theoretical_loss": 7.372739167143329, + "tokens_seen": 7208960 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001761904761904762, + "loss": 5.009, + "theoretical_loss": 7.36024897950216, + "tokens_seen": 7274496 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017777777777777779, + "loss": 5.0185, + "theoretical_loss": 7.347901998746318, + "tokens_seen": 7340032 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017936507936507938, + "loss": 5.0807, + "theoretical_loss": 7.335695326111465, + "tokens_seen": 7405568 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018095238095238095, + "loss": 4.9311, + "theoretical_loss": 7.323626146502205, + "tokens_seen": 7471104 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018253968253968255, + "loss": 5.051, + "theoretical_loss": 7.311691725368293, + "tokens_seen": 7536640 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018412698412698412, + "loss": 5.0423, + "theoretical_loss": 7.299889405723473, + "tokens_seen": 7602176 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018571428571428572, + "loss": 4.9575, + "theoretical_loss": 7.288216605299264, + "tokens_seen": 7667712 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018730158730158731, + "loss": 4.9026, + "theoretical_loss": 7.276670813826511, + "tokens_seen": 7733248 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018888888888888888, + "loss": 4.9363, + "theoretical_loss": 7.265249590437903, + "tokens_seen": 7798784 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019047619047619048, + "loss": 4.9141, + "theoretical_loss": 7.253950561185138, + "tokens_seen": 7864320 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019206349206349208, + "loss": 4.8576, + "theoretical_loss": 7.242771416664761, + "tokens_seen": 7929856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019365079365079365, + "loss": 4.8336, + "theoretical_loss": 7.231709909747042, + "tokens_seen": 7995392 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019523809523809525, + "loss": 4.9158, + "theoretical_loss": 7.220763853402656, + "tokens_seen": 8060928 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019682539682539682, + "loss": 4.8969, + "theoretical_loss": 7.209931118622157, + "tokens_seen": 8126464 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019841269841269844, + "loss": 5.0328, + "theoretical_loss": 7.199209632423606, + "tokens_seen": 8192000 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002, + "loss": 4.7254, + "theoretical_loss": 7.188597375943913, + "tokens_seen": 8257536 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020158730158730158, + "loss": 4.8566, + "theoretical_loss": 7.178092382609779, + "tokens_seen": 8323072 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020317460317460318, + "loss": 4.9601, + "theoretical_loss": 7.1676927363842795, + "tokens_seen": 8388608 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020476190476190475, + "loss": 4.8234, + "theoretical_loss": 7.157396570085435, + "tokens_seen": 8454144 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020634920634920637, + "loss": 4.9181, + "theoretical_loss": 7.147202063773257, + "tokens_seen": 8519680 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020793650793650797, + "loss": 4.8418, + "theoretical_loss": 7.137107443201986, + "tokens_seen": 8585216 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020952380952380954, + "loss": 4.819, + "theoretical_loss": 7.127110978334411, + "tokens_seen": 8650752 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021111111111111113, + "loss": 4.9044, + "theoretical_loss": 7.117210981915334, + "tokens_seen": 8716288 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002126984126984127, + "loss": 4.8086, + "theoretical_loss": 7.10740580810139, + "tokens_seen": 8781824 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002142857142857143, + "loss": 4.7956, + "theoretical_loss": 7.097693851144607, + "tokens_seen": 8847360 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021587301587301587, + "loss": 4.8153, + "theoretical_loss": 7.088073544127209, + "tokens_seen": 8912896 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021746031746031744, + "loss": 4.8548, + "theoretical_loss": 7.07854335774531, + "tokens_seen": 8978432 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021904761904761907, + "loss": 4.6907, + "theoretical_loss": 7.069101799139266, + "tokens_seen": 9043968 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022063492063492066, + "loss": 4.7805, + "theoretical_loss": 7.0597474107685745, + "tokens_seen": 9109504 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022222222222222223, + "loss": 4.7233, + "theoretical_loss": 7.050478769329319, + "tokens_seen": 9175040 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022380952380952383, + "loss": 4.8346, + "theoretical_loss": 7.0412944847122505, + "tokens_seen": 9240576 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002253968253968254, + "loss": 4.7503, + "theoretical_loss": 7.032193198999712, + "tokens_seen": 9306112 + }, + { + "epoch": 0.0, + "learning_rate": 0.000226984126984127, + "loss": 4.6828, + "theoretical_loss": 7.023173585499698, + "tokens_seen": 9371648 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022857142857142857, + "loss": 4.6974, + "theoretical_loss": 7.014234347815421, + "tokens_seen": 9437184 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023015873015873016, + "loss": 4.8072, + "theoretical_loss": 7.00537421894883, + "tokens_seen": 9502720 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002317460317460318, + "loss": 4.692, + "theoretical_loss": 6.996591960436656, + "tokens_seen": 9568256 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023333333333333336, + "loss": 4.7647, + "theoretical_loss": 6.9878863615175355, + "tokens_seen": 9633792 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023492063492063493, + "loss": 4.7072, + "theoretical_loss": 6.97925623832895, + "tokens_seen": 9699328 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023650793650793653, + "loss": 4.8046, + "theoretical_loss": 6.97070043313267, + "tokens_seen": 9764864 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 9988, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 4.5934672355651855, + "objective/train/theoretical_loss": 6.962217813567541, + "objective/train/tokens_used": 30290400, + "theoretical_loss": 6.962217813567541, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002380952380952381, + "loss": 4.7083, + "theoretical_loss": 6.962217813567541, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002396825396825397, + "loss": 4.6527, + "theoretical_loss": 6.953807271928458, + "tokens_seen": 9895936 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024126984126984126, + "loss": 4.4851, + "theoretical_loss": 6.945467724470439, + "tokens_seen": 9961472 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024285714285714286, + "loss": 4.4885, + "theoretical_loss": 6.937198110736788, + "tokens_seen": 10027008 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002444444444444445, + "loss": 4.5112, + "theoretical_loss": 6.928997392910329, + "tokens_seen": 10092544 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024603174603174605, + "loss": 4.7377, + "theoretical_loss": 6.920864555186801, + "tokens_seen": 10158080 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002476190476190476, + "loss": 4.6674, + "theoretical_loss": 6.912798603169512, + "tokens_seen": 10223616 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024920634920634925, + "loss": 4.5936, + "theoretical_loss": 6.904798563284391, + "tokens_seen": 10289152 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002507936507936508, + "loss": 4.8009, + "theoretical_loss": 6.896863482214638, + "tokens_seen": 10354688 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002523809523809524, + "loss": 4.7057, + "theoretical_loss": 6.888992426354192, + "tokens_seen": 10420224 + }, + { + "epoch": 0.0, + "learning_rate": 0.00025396825396825396, + "loss": 4.7217, + "theoretical_loss": 6.881184481279271, + "tokens_seen": 10485760 + }, + { + "epoch": 0.0, + "learning_rate": 0.00025555555555555553, + "loss": 4.5575, + "theoretical_loss": 6.873438751237283, + "tokens_seen": 10551296 + }, + { + "epoch": 0.0, + "learning_rate": 0.00025714285714285715, + "loss": 4.6313, + "theoretical_loss": 6.865754358652436, + "tokens_seen": 10616832 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002587301587301588, + "loss": 4.5677, + "theoretical_loss": 6.85813044364739, + "tokens_seen": 10682368 + }, + { + "epoch": 0.0, + "learning_rate": 0.00026031746031746035, + "loss": 4.5783, + "theoretical_loss": 6.85056616358036, + "tokens_seen": 10747904 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002619047619047619, + "loss": 4.7549, + "theoretical_loss": 6.843060692597036, + "tokens_seen": 10813440 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002634920634920635, + "loss": 4.5659, + "theoretical_loss": 6.835613221196822, + "tokens_seen": 10878976 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002650793650793651, + "loss": 4.6755, + "theoretical_loss": 6.8282229558127865, + "tokens_seen": 10944512 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002666666666666667, + "loss": 4.5563, + "theoretical_loss": 6.820889118404866, + "tokens_seen": 11010048 + }, + { + "epoch": 0.0, + "learning_rate": 0.00026825396825396825, + "loss": 4.5433, + "theoretical_loss": 6.813610946065789, + "tokens_seen": 11075584 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002698412698412699, + "loss": 4.5981, + "theoretical_loss": 6.806387690639279, + "tokens_seen": 11141120 + }, + { + "epoch": 0.0, + "learning_rate": 0.00027142857142857144, + "loss": 4.5952, + "theoretical_loss": 6.799218618350064, + "tokens_seen": 11206656 + }, + { + "epoch": 0.0, + "learning_rate": 0.000273015873015873, + "loss": 4.6503, + "theoretical_loss": 6.792103009445266, + "tokens_seen": 11272192 + }, + { + "epoch": 0.0, + "learning_rate": 0.00027460317460317464, + "loss": 4.4218, + "theoretical_loss": 6.7850401578467645, + "tokens_seen": 11337728 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002761904761904762, + "loss": 4.5615, + "theoretical_loss": 6.778029370814127, + "tokens_seen": 11403264 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002777777777777778, + "loss": 4.535, + "theoretical_loss": 6.771069968617732, + "tokens_seen": 11468800 + }, + { + "epoch": 0.0, + "learning_rate": 0.00027936507936507935, + "loss": 4.6074, + "theoretical_loss": 6.764161284221716, + "tokens_seen": 11534336 + }, + { + "epoch": 0.0, + "learning_rate": 0.00028095238095238097, + "loss": 4.7694, + "theoretical_loss": 6.757302662976402, + "tokens_seen": 11599872 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002825396825396826, + "loss": 4.5586, + "theoretical_loss": 6.750493462319867, + "tokens_seen": 11665408 + }, + { + "epoch": 0.0, + "learning_rate": 0.00028412698412698417, + "loss": 4.4784, + "theoretical_loss": 6.74373305148833, + "tokens_seen": 11730944 + }, + { + "epoch": 0.0, + "learning_rate": 0.00028571428571428574, + "loss": 4.4651, + "theoretical_loss": 6.737020811235054, + "tokens_seen": 11796480 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002873015873015873, + "loss": 4.3919, + "theoretical_loss": 6.730356133557465, + "tokens_seen": 11862016 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002888888888888889, + "loss": 4.5578, + "theoretical_loss": 6.7237384214322, + "tokens_seen": 11927552 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002904761904761905, + "loss": 4.4157, + "theoretical_loss": 6.717167088557821, + "tokens_seen": 11993088 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029206349206349207, + "loss": 4.534, + "theoretical_loss": 6.710641559104915, + "tokens_seen": 12058624 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029365079365079364, + "loss": 4.5115, + "theoretical_loss": 6.704161267473356, + "tokens_seen": 12124160 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029523809523809526, + "loss": 4.5263, + "theoretical_loss": 6.697725658056456, + "tokens_seen": 12189696 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029682539682539683, + "loss": 4.623, + "theoretical_loss": 6.691334185011799, + "tokens_seen": 12255232 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029841269841269846, + "loss": 4.4268, + "theoretical_loss": 6.68498631203852, + "tokens_seen": 12320768 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030000000000000003, + "loss": 4.4161, + "theoretical_loss": 6.678681512160812, + "tokens_seen": 12386304 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003015873015873016, + "loss": 4.3853, + "theoretical_loss": 6.672419267517472, + "tokens_seen": 12451840 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030317460317460317, + "loss": 4.4433, + "theoretical_loss": 6.666199069157267, + "tokens_seen": 12517376 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030476190476190474, + "loss": 4.5607, + "theoretical_loss": 6.660020416839945, + "tokens_seen": 12582912 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030634920634920636, + "loss": 4.4346, + "theoretical_loss": 6.65388281884269, + "tokens_seen": 12648448 + }, + { + "epoch": 0.0, + "learning_rate": 0.000307936507936508, + "loss": 4.4253, + "theoretical_loss": 6.647785791771864, + "tokens_seen": 12713984 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030952380952380956, + "loss": 4.423, + "theoretical_loss": 6.641728860379852, + "tokens_seen": 12779520 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003111111111111111, + "loss": 4.5214, + "theoretical_loss": 6.635711557386854, + "tokens_seen": 12845056 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003126984126984127, + "loss": 4.5018, + "theoretical_loss": 6.629733423307451, + "tokens_seen": 12910592 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003142857142857143, + "loss": 4.5074, + "theoretical_loss": 6.6237940062818215, + "tokens_seen": 12976128 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003158730158730159, + "loss": 4.5088, + "theoretical_loss": 6.617892861911426, + "tokens_seen": 13041664 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 11319, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 4.34834098815918, + "objective/train/theoretical_loss": 6.612029553099055, + "objective/train/tokens_used": 33567200, + "theoretical_loss": 6.612029553099055, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.00031746031746031746, + "loss": 4.4598, + "theoretical_loss": 6.612029553099055, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003190476190476191, + "loss": 4.5264, + "theoretical_loss": 6.606203649893072, + "tokens_seen": 13172736 + }, + { + "epoch": 0.0, + "learning_rate": 0.00032063492063492065, + "loss": 4.4953, + "theoretical_loss": 6.600414729335739, + "tokens_seen": 13238272 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003222222222222222, + "loss": 4.5654, + "theoretical_loss": 6.5946623753155045, + "tokens_seen": 13303808 + }, + { + "epoch": 0.0, + "learning_rate": 0.00032380952380952385, + "loss": 4.4299, + "theoretical_loss": 6.588946178423104, + "tokens_seen": 13369344 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003253968253968254, + "loss": 4.5707, + "theoretical_loss": 6.583265735811386, + "tokens_seen": 13434880 + }, + { + "epoch": 0.0, + "learning_rate": 0.000326984126984127, + "loss": 4.4346, + "theoretical_loss": 6.577620651058732, + "tokens_seen": 13500416 + }, + { + "epoch": 0.0, + "learning_rate": 0.00032857142857142856, + "loss": 4.4908, + "theoretical_loss": 6.572010534035971, + "tokens_seen": 13565952 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003301587301587302, + "loss": 4.5556, + "theoretical_loss": 6.566435000776667, + "tokens_seen": 13631488 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003317460317460318, + "loss": 4.5845, + "theoretical_loss": 6.5608936733507095, + "tokens_seen": 13697024 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003333333333333334, + "loss": 4.5322, + "theoretical_loss": 6.555386179741061, + "tokens_seen": 13762560 + }, + { + "epoch": 0.0, + "learning_rate": 0.00033492063492063495, + "loss": 4.4791, + "theoretical_loss": 6.549912153723619, + "tokens_seen": 13828096 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003365079365079365, + "loss": 4.4018, + "theoretical_loss": 6.544471234750057, + "tokens_seen": 13893632 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003380952380952381, + "loss": 4.451, + "theoretical_loss": 6.539063067833591, + "tokens_seen": 13959168 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003396825396825397, + "loss": 4.2116, + "theoretical_loss": 6.533687303437551, + "tokens_seen": 14024704 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003412698412698413, + "loss": 4.462, + "theoretical_loss": 6.528343597366709, + "tokens_seen": 14090240 + }, + { + "epoch": 0.0, + "learning_rate": 0.00034285714285714285, + "loss": 4.405, + "theoretical_loss": 6.523031610661259, + "tokens_seen": 14155776 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003444444444444445, + "loss": 4.5102, + "theoretical_loss": 6.517751009493386, + "tokens_seen": 14221312 + }, + { + "epoch": 0.0, + "learning_rate": 0.00034603174603174604, + "loss": 4.3705, + "theoretical_loss": 6.51250146506634, + "tokens_seen": 14286848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00034761904761904767, + "loss": 4.4277, + "theoretical_loss": 6.507282653515952, + "tokens_seen": 14352384 + }, + { + "epoch": 0.0, + "learning_rate": 0.00034920634920634924, + "loss": 4.384, + "theoretical_loss": 6.5020942558145105, + "tokens_seen": 14417920 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003507936507936508, + "loss": 4.4511, + "theoretical_loss": 6.496935957676949, + "tokens_seen": 14483456 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003523809523809524, + "loss": 4.4827, + "theoretical_loss": 6.491807449469257, + "tokens_seen": 14548992 + }, + { + "epoch": 0.0, + "learning_rate": 0.00035396825396825395, + "loss": 4.4038, + "theoretical_loss": 6.486708426119071, + "tokens_seen": 14614528 + }, + { + "epoch": 0.0, + "learning_rate": 0.00035555555555555557, + "loss": 4.4651, + "theoretical_loss": 6.481638587028371, + "tokens_seen": 14680064 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003571428571428572, + "loss": 4.2654, + "theoretical_loss": 6.4765976359882345, + "tokens_seen": 14745600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00035873015873015877, + "loss": 4.3412, + "theoretical_loss": 6.47158528109558, + "tokens_seen": 14811136 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036031746031746034, + "loss": 4.4479, + "theoretical_loss": 6.466601234671855, + "tokens_seen": 14876672 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003619047619047619, + "loss": 4.2494, + "theoretical_loss": 6.461645213183605, + "tokens_seen": 14942208 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036349206349206353, + "loss": 4.3764, + "theoretical_loss": 6.45671693716488, + "tokens_seen": 15007744 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003650793650793651, + "loss": 4.2567, + "theoretical_loss": 6.451816131141427, + "tokens_seen": 15073280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036666666666666667, + "loss": 4.3653, + "theoretical_loss": 6.4469425235566105, + "tokens_seen": 15138816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036825396825396824, + "loss": 4.462, + "theoretical_loss": 6.442095846699032, + "tokens_seen": 15204352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036984126984126986, + "loss": 4.4221, + "theoretical_loss": 6.437275836631781, + "tokens_seen": 15269888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00037142857142857143, + "loss": 4.2723, + "theoretical_loss": 6.432482233123297, + "tokens_seen": 15335424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00037301587301587306, + "loss": 4.2973, + "theoretical_loss": 6.427714779579773, + "tokens_seen": 15400960 + }, + { + "epoch": 0.0, + "learning_rate": 0.00037460317460317463, + "loss": 4.2562, + "theoretical_loss": 6.422973222979093, + "tokens_seen": 15466496 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003761904761904762, + "loss": 4.3652, + "theoretical_loss": 6.418257313806215, + "tokens_seen": 15532032 + }, + { + "epoch": 0.0, + "learning_rate": 0.00037777777777777777, + "loss": 4.3504, + "theoretical_loss": 6.413566805990028, + "tokens_seen": 15597568 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003793650793650794, + "loss": 4.2707, + "theoretical_loss": 6.408901456841564, + "tokens_seen": 15663104 + }, + { + "epoch": 0.0, + "learning_rate": 0.00038095238095238096, + "loss": 4.2359, + "theoretical_loss": 6.404261026993607, + "tokens_seen": 15728640 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003825396825396826, + "loss": 4.3195, + "theoretical_loss": 6.399645280341607, + "tokens_seen": 15794176 + }, + { + "epoch": 0.0, + "learning_rate": 0.00038412698412698416, + "loss": 4.3046, + "theoretical_loss": 6.395053983985889, + "tokens_seen": 15859712 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003857142857142857, + "loss": 4.1676, + "theoretical_loss": 6.390486908175121, + "tokens_seen": 15925248 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003873015873015873, + "loss": 4.3473, + "theoretical_loss": 6.385943826251012, + "tokens_seen": 15990784 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003888888888888889, + "loss": 4.3593, + "theoretical_loss": 6.381424514594194, + "tokens_seen": 16056320 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003904761904761905, + "loss": 4.2621, + "theoretical_loss": 6.376928752571271, + "tokens_seen": 16121856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00039206349206349206, + "loss": 4.2769, + "theoretical_loss": 6.372456322483007, + "tokens_seen": 16187392 + }, + { + "epoch": 0.0, + "learning_rate": 0.00039365079365079363, + "loss": 4.3993, + "theoretical_loss": 6.368007009513611, + "tokens_seen": 16252928 + }, + { + "epoch": 0.0, + "learning_rate": 0.00039523809523809526, + "loss": 4.3462, + "theoretical_loss": 6.363580601681101, + "tokens_seen": 16318464 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 12210, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 4.425621032714844, + "objective/train/theoretical_loss": 6.359176889788724, + "objective/train/tokens_used": 36844000, + "theoretical_loss": 6.359176889788724, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003968253968253969, + "loss": 4.3752, + "theoretical_loss": 6.359176889788724, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.00039841269841269845, + "loss": 4.2801, + "theoretical_loss": 6.354795667377392, + "tokens_seen": 16449536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004, + "loss": 4.282, + "theoretical_loss": 6.350436730679111, + "tokens_seen": 16515072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004015873015873016, + "loss": 4.3637, + "theoretical_loss": 6.346099878571405, + "tokens_seen": 16580608 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040317460317460316, + "loss": 4.2939, + "theoretical_loss": 6.341784912532662, + "tokens_seen": 16646144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004047619047619048, + "loss": 4.3405, + "theoretical_loss": 6.337491636598426, + "tokens_seen": 16711680 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040634920634920635, + "loss": 4.1746, + "theoretical_loss": 6.33321985731858, + "tokens_seen": 16777216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004079365079365079, + "loss": 4.2457, + "theoretical_loss": 6.328969383715416, + "tokens_seen": 16842752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004095238095238095, + "loss": 4.254, + "theoretical_loss": 6.324740027242559, + "tokens_seen": 16908288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004111111111111111, + "loss": 4.1043, + "theoretical_loss": 6.32053160174474, + "tokens_seen": 16973824 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041269841269841274, + "loss": 4.1842, + "theoretical_loss": 6.316343923418371, + "tokens_seen": 17039360 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041428571428571437, + "loss": 4.0588, + "theoretical_loss": 6.312176810772938, + "tokens_seen": 17104896 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041587301587301594, + "loss": 4.1163, + "theoretical_loss": 6.308030084593148, + "tokens_seen": 17170432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004174603174603175, + "loss": 4.2286, + "theoretical_loss": 6.30390356790187, + "tokens_seen": 17235968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004190476190476191, + "loss": 4.2368, + "theoretical_loss": 6.2997970859237835, + "tokens_seen": 17301504 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042063492063492065, + "loss": 4.2891, + "theoretical_loss": 6.295710466049778, + "tokens_seen": 17367040 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042222222222222227, + "loss": 4.3256, + "theoretical_loss": 6.29164353780205, + "tokens_seen": 17432576 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042380952380952384, + "loss": 4.2051, + "theoretical_loss": 6.287596132799887, + "tokens_seen": 17498112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004253968253968254, + "loss": 4.2521, + "theoretical_loss": 6.283568084726138, + "tokens_seen": 17563648 + }, + { + "epoch": 0.01, + "learning_rate": 0.000426984126984127, + "loss": 4.214, + "theoretical_loss": 6.27955922929433, + "tokens_seen": 17629184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004285714285714286, + "loss": 4.1344, + "theoretical_loss": 6.2755694042164425, + "tokens_seen": 17694720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004301587301587302, + "loss": 4.2665, + "theoretical_loss": 6.271598449171293, + "tokens_seen": 17760256 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043174603174603174, + "loss": 4.1745, + "theoretical_loss": 6.267646205773557, + "tokens_seen": 17825792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004333333333333333, + "loss": 4.1578, + "theoretical_loss": 6.263712517543373, + "tokens_seen": 17891328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004349206349206349, + "loss": 4.0887, + "theoretical_loss": 6.25979722987654, + "tokens_seen": 17956864 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004365079365079365, + "loss": 4.1966, + "theoretical_loss": 6.255900190015284, + "tokens_seen": 18022400 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043809523809523813, + "loss": 4.2064, + "theoretical_loss": 6.252021247019596, + "tokens_seen": 18087936 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043968253968253976, + "loss": 4.153, + "theoretical_loss": 6.2481602517390975, + "tokens_seen": 18153472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004412698412698413, + "loss": 4.1275, + "theoretical_loss": 6.244317056785462, + "tokens_seen": 18219008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004428571428571429, + "loss": 4.1322, + "theoretical_loss": 6.240491516505343, + "tokens_seen": 18284544 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044444444444444447, + "loss": 4.1679, + "theoretical_loss": 6.236683486953819, + "tokens_seen": 18350080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004460317460317461, + "loss": 4.0966, + "theoretical_loss": 6.232892825868342, + "tokens_seen": 18415616 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044761904761904766, + "loss": 4.1138, + "theoretical_loss": 6.229119392643171, + "tokens_seen": 18481152 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044920634920634923, + "loss": 4.1271, + "theoretical_loss": 6.225363048304281, + "tokens_seen": 18546688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004507936507936508, + "loss": 4.2432, + "theoretical_loss": 6.221623655484739, + "tokens_seen": 18612224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045238095238095237, + "loss": 4.2264, + "theoretical_loss": 6.217901078400542, + "tokens_seen": 18677760 + }, + { + "epoch": 0.01, + "learning_rate": 0.000453968253968254, + "loss": 4.2129, + "theoretical_loss": 6.214195182826902, + "tokens_seen": 18743296 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045555555555555556, + "loss": 4.105, + "theoretical_loss": 6.2105058360749545, + "tokens_seen": 18808832 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045714285714285713, + "loss": 4.0347, + "theoretical_loss": 6.206832906968911, + "tokens_seen": 18874368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004587301587301587, + "loss": 4.0975, + "theoretical_loss": 6.203176265823624, + "tokens_seen": 18939904 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046031746031746033, + "loss": 4.2128, + "theoretical_loss": 6.1995357844225545, + "tokens_seen": 19005440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004619047619047619, + "loss": 4.0438, + "theoretical_loss": 6.195911335996151, + "tokens_seen": 19070976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004634920634920636, + "loss": 4.2097, + "theoretical_loss": 6.192302795200606, + "tokens_seen": 19136512 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046507936507936515, + "loss": 4.1158, + "theoretical_loss": 6.188710038097013, + "tokens_seen": 19202048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004666666666666667, + "loss": 4.1666, + "theoretical_loss": 6.185132942130884, + "tokens_seen": 19267584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004682539682539683, + "loss": 4.1723, + "theoretical_loss": 6.181571386112038, + "tokens_seen": 19333120 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046984126984126986, + "loss": 4.1159, + "theoretical_loss": 6.178025250194852, + "tokens_seen": 19398656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004714285714285715, + "loss": 4.2241, + "theoretical_loss": 6.174494415858863, + "tokens_seen": 19464192 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047301587301587305, + "loss": 4.0897, + "theoretical_loss": 6.170978765889705, + "tokens_seen": 19529728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004746031746031746, + "loss": 4.1264, + "theoretical_loss": 6.167478184360402, + "tokens_seen": 19595264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 13501, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 4.086112976074219, + "objective/train/theoretical_loss": 6.163992556612967, + "objective/train/tokens_used": 40120800, + "theoretical_loss": 6.163992556612967, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004761904761904762, + "loss": 4.1434, + "theoretical_loss": 6.163992556612967, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004777777777777778, + "loss": 4.0815, + "theoretical_loss": 6.160521769240346, + "tokens_seen": 19726336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004793650793650794, + "loss": 4.1549, + "theoretical_loss": 6.157065710068663, + "tokens_seen": 19791872 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048095238095238095, + "loss": 4.0909, + "theoretical_loss": 6.1536242681397795, + "tokens_seen": 19857408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004825396825396825, + "loss": 3.918, + "theoretical_loss": 6.150197333694162, + "tokens_seen": 19922944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004841269841269841, + "loss": 4.0747, + "theoretical_loss": 6.146784798154039, + "tokens_seen": 19988480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004857142857142857, + "loss": 4.1139, + "theoretical_loss": 6.143386554106854, + "tokens_seen": 20054016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004873015873015873, + "loss": 3.9063, + "theoretical_loss": 6.140002495289, + "tokens_seen": 20119552 + }, + { + "epoch": 0.01, + "learning_rate": 0.000488888888888889, + "loss": 4.0389, + "theoretical_loss": 6.136632516569836, + "tokens_seen": 20185088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004904761904761905, + "loss": 4.0393, + "theoretical_loss": 6.133276513935977, + "tokens_seen": 20250624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004920634920634921, + "loss": 4.0666, + "theoretical_loss": 6.129934384475845, + "tokens_seen": 20316160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004936507936507937, + "loss": 4.0097, + "theoretical_loss": 6.126606026364497, + "tokens_seen": 20381696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004952380952380952, + "loss": 4.0515, + "theoretical_loss": 6.123291338848694, + "tokens_seen": 20447232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004968253968253969, + "loss": 4.0653, + "theoretical_loss": 6.119990222232234, + "tokens_seen": 20512768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984126984126985, + "loss": 4.0558, + "theoretical_loss": 6.116702577861526, + "tokens_seen": 20578304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 4.1177, + "theoretical_loss": 6.113428308111404, + "tokens_seen": 20643840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005015873015873016, + "loss": 4.1428, + "theoretical_loss": 6.110167316371188, + "tokens_seen": 20709376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005031746031746032, + "loss": 3.9055, + "theoretical_loss": 6.106919507030966, + "tokens_seen": 20774912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005047619047619048, + "loss": 4.1163, + "theoretical_loss": 6.103684785468113, + "tokens_seen": 20840448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005063492063492064, + "loss": 3.929, + "theoretical_loss": 6.100463058034027, + "tokens_seen": 20905984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005079365079365079, + "loss": 4.0137, + "theoretical_loss": 6.097254232041089, + "tokens_seen": 20971520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005095238095238095, + "loss": 3.9822, + "theoretical_loss": 6.094058215749833, + "tokens_seen": 21037056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005111111111111111, + "loss": 4.1239, + "theoretical_loss": 6.090874918356328, + "tokens_seen": 21102592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005126984126984128, + "loss": 3.888, + "theoretical_loss": 6.087704249979775, + "tokens_seen": 21168128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005142857142857143, + "loss": 3.9305, + "theoretical_loss": 6.084546121650289, + "tokens_seen": 21233664 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005158730158730159, + "loss": 4.0623, + "theoretical_loss": 6.081400445296899, + "tokens_seen": 21299200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005174603174603176, + "loss": 4.0524, + "theoretical_loss": 6.078267133735724, + "tokens_seen": 21364736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005190476190476191, + "loss": 4.0179, + "theoretical_loss": 6.075146100658356, + "tokens_seen": 21430272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005206349206349207, + "loss": 3.8274, + "theoretical_loss": 6.072037260620421, + "tokens_seen": 21495808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005222222222222222, + "loss": 3.9728, + "theoretical_loss": 6.0689405290303275, + "tokens_seen": 21561344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005238095238095238, + "loss": 4.0168, + "theoretical_loss": 6.065855822138193, + "tokens_seen": 21626880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005253968253968255, + "loss": 3.9993, + "theoretical_loss": 6.06278305702495, + "tokens_seen": 21692416 + }, + { + "epoch": 0.01, + "learning_rate": 0.000526984126984127, + "loss": 3.9977, + "theoretical_loss": 6.05972215159162, + "tokens_seen": 21757952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005285714285714286, + "loss": 4.0765, + "theoretical_loss": 6.056673024548766, + "tokens_seen": 21823488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005301587301587302, + "loss": 3.9494, + "theoretical_loss": 6.053635595406105, + "tokens_seen": 21889024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005317460317460317, + "loss": 3.8884, + "theoretical_loss": 6.050609784462289, + "tokens_seen": 21954560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005333333333333334, + "loss": 3.9952, + "theoretical_loss": 6.047595512794841, + "tokens_seen": 22020096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005349206349206349, + "loss": 3.8465, + "theoretical_loss": 6.044592702250255, + "tokens_seen": 22085632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005365079365079365, + "loss": 3.8355, + "theoretical_loss": 6.041601275434251, + "tokens_seen": 22151168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005380952380952381, + "loss": 3.9425, + "theoretical_loss": 6.038621155702174, + "tokens_seen": 22216704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005396825396825397, + "loss": 3.9988, + "theoretical_loss": 6.0356522671495485, + "tokens_seen": 22282240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005412698412698414, + "loss": 3.9923, + "theoretical_loss": 6.032694534602784, + "tokens_seen": 22347776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005428571428571429, + "loss": 3.8449, + "theoretical_loss": 6.029747883610009, + "tokens_seen": 22413312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005444444444444445, + "loss": 3.8843, + "theoretical_loss": 6.026812240432067, + "tokens_seen": 22478848 + }, + { + "epoch": 0.01, + "learning_rate": 0.000546031746031746, + "loss": 3.9525, + "theoretical_loss": 6.023887532033628, + "tokens_seen": 22544384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005476190476190477, + "loss": 3.9851, + "theoretical_loss": 6.020973686074459, + "tokens_seen": 22609920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005492063492063493, + "loss": 3.8682, + "theoretical_loss": 6.018070630900809, + "tokens_seen": 22675456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005507936507936508, + "loss": 3.9546, + "theoretical_loss": 6.015178295536933, + "tokens_seen": 22740992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005523809523809524, + "loss": 3.8417, + "theoretical_loss": 6.012296609676753, + "tokens_seen": 22806528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005539682539682539, + "loss": 3.9796, + "theoretical_loss": 6.009425503675624, + "tokens_seen": 22872064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 14491, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.740633487701416, + "objective/train/theoretical_loss": 6.006564908542249, + "objective/train/tokens_used": 43397600, + "theoretical_loss": 6.006564908542249, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005555555555555556, + "loss": 3.8125, + "theoretical_loss": 6.006564908542249, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005571428571428572, + "loss": 4.0247, + "theoretical_loss": 6.003714755930698, + "tokens_seen": 23003136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005587301587301587, + "loss": 3.8499, + "theoretical_loss": 6.000874978132554, + "tokens_seen": 23068672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005603174603174603, + "loss": 3.7935, + "theoretical_loss": 5.998045508069175, + "tokens_seen": 23134208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005619047619047619, + "loss": 3.6955, + "theoretical_loss": 5.995226279284073, + "tokens_seen": 23199744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005634920634920636, + "loss": 3.848, + "theoretical_loss": 5.992417225935405, + "tokens_seen": 23265280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005650793650793652, + "loss": 4.0086, + "theoretical_loss": 5.989618282788578, + "tokens_seen": 23330816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005666666666666667, + "loss": 4.0299, + "theoretical_loss": 5.9868293852089565, + "tokens_seen": 23396352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005682539682539683, + "loss": 3.7135, + "theoretical_loss": 5.984050469154685, + "tokens_seen": 23461888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005698412698412698, + "loss": 3.9052, + "theoretical_loss": 5.981281471169619, + "tokens_seen": 23527424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005714285714285715, + "loss": 3.916, + "theoretical_loss": 5.978522328376346, + "tokens_seen": 23592960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005730158730158731, + "loss": 3.9566, + "theoretical_loss": 5.975772978469321, + "tokens_seen": 23658496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005746031746031746, + "loss": 3.861, + "theoretical_loss": 5.973033359708099, + "tokens_seen": 23724032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005761904761904762, + "loss": 4.0453, + "theoretical_loss": 5.9703034109106685, + "tokens_seen": 23789568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005777777777777778, + "loss": 3.8675, + "theoretical_loss": 5.967583071446869, + "tokens_seen": 23855104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005793650793650794, + "loss": 3.8817, + "theoretical_loss": 5.9648722812319255, + "tokens_seen": 23920640 + }, + { + "epoch": 0.01, + "learning_rate": 0.000580952380952381, + "loss": 3.9538, + "theoretical_loss": 5.962170980720055, + "tokens_seen": 23986176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005825396825396825, + "loss": 3.8526, + "theoretical_loss": 5.959479110898181, + "tokens_seen": 24051712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005841269841269841, + "loss": 3.8755, + "theoretical_loss": 5.956796613279727, + "tokens_seen": 24117248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005857142857142857, + "loss": 3.9102, + "theoretical_loss": 5.954123429898508, + "tokens_seen": 24182784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005873015873015873, + "loss": 3.721, + "theoretical_loss": 5.951459503302701, + "tokens_seen": 24248320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005888888888888889, + "loss": 3.8645, + "theoretical_loss": 5.948804776548908, + "tokens_seen": 24313856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005904761904761905, + "loss": 4.0393, + "theoretical_loss": 5.9461591931963, + "tokens_seen": 24379392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005920634920634922, + "loss": 3.9327, + "theoretical_loss": 5.943522697300843, + "tokens_seen": 24444928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005936507936507937, + "loss": 3.6637, + "theoretical_loss": 5.940895233409613, + "tokens_seen": 24510464 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005952380952380953, + "loss": 3.8462, + "theoretical_loss": 5.938276746555175, + "tokens_seen": 24576000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005968253968253969, + "loss": 3.8563, + "theoretical_loss": 5.9356671822500635, + "tokens_seen": 24641536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005984126984126984, + "loss": 3.6734, + "theoretical_loss": 5.933066486481318, + "tokens_seen": 24707072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006000000000000001, + "loss": 3.9587, + "theoretical_loss": 5.930474605705113, + "tokens_seen": 24772608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006015873015873016, + "loss": 3.8129, + "theoretical_loss": 5.927891486841449, + "tokens_seen": 24838144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006031746031746032, + "loss": 3.8233, + "theoretical_loss": 5.925317077268927, + "tokens_seen": 24903680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006047619047619048, + "loss": 3.8697, + "theoretical_loss": 5.92275132481959, + "tokens_seen": 24969216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006063492063492063, + "loss": 3.7977, + "theoretical_loss": 5.9201941777738405, + "tokens_seen": 25034752 + }, + { + "epoch": 0.01, + "learning_rate": 0.000607936507936508, + "loss": 3.8119, + "theoretical_loss": 5.917645584855421, + "tokens_seen": 25100288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006095238095238095, + "loss": 3.8489, + "theoretical_loss": 5.915105495226474, + "tokens_seen": 25165824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006111111111111111, + "loss": 3.7896, + "theoretical_loss": 5.912573858482657, + "tokens_seen": 25231360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006126984126984127, + "loss": 3.7893, + "theoretical_loss": 5.910050624648335, + "tokens_seen": 25296896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006142857142857143, + "loss": 3.6833, + "theoretical_loss": 5.907535744171835, + "tokens_seen": 25362432 + }, + { + "epoch": 0.01, + "learning_rate": 0.000615873015873016, + "loss": 3.7628, + "theoretical_loss": 5.905029167920765, + "tokens_seen": 25427968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006174603174603175, + "loss": 3.7026, + "theoretical_loss": 5.902530847177394, + "tokens_seen": 25493504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006190476190476191, + "loss": 3.8071, + "theoretical_loss": 5.900040733634098, + "tokens_seen": 25559040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006206349206349206, + "loss": 3.8134, + "theoretical_loss": 5.897558779388872, + "tokens_seen": 25624576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006222222222222223, + "loss": 3.6805, + "theoretical_loss": 5.895084936940887, + "tokens_seen": 25690112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006238095238095239, + "loss": 3.7538, + "theoretical_loss": 5.892619159186127, + "tokens_seen": 25755648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006253968253968254, + "loss": 3.7874, + "theoretical_loss": 5.890161399413071, + "tokens_seen": 25821184 + }, + { + "epoch": 0.01, + "learning_rate": 0.000626984126984127, + "loss": 3.7056, + "theoretical_loss": 5.887711611298428, + "tokens_seen": 25886720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006285714285714286, + "loss": 3.7162, + "theoretical_loss": 5.885269748902955, + "tokens_seen": 25952256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006301587301587302, + "loss": 3.6735, + "theoretical_loss": 5.882835766667295, + "tokens_seen": 26017792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006317460317460318, + "loss": 3.721, + "theoretical_loss": 5.880409619407899, + "tokens_seen": 26083328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006333333333333333, + "loss": 3.618, + "theoretical_loss": 5.877991262312986, + "tokens_seen": 26148864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 15706, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.5033531188964844, + "objective/train/theoretical_loss": 5.875580650938565, + "objective/train/tokens_used": 46674400, + "theoretical_loss": 5.875580650938565, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006349206349206349, + "loss": 3.6113, + "theoretical_loss": 5.875580650938565, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006365079365079365, + "loss": 3.7019, + "theoretical_loss": 5.873177741204501, + "tokens_seen": 26279936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006380952380952382, + "loss": 3.644, + "theoretical_loss": 5.8707824893906455, + "tokens_seen": 26345472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006396825396825398, + "loss": 3.6657, + "theoretical_loss": 5.868394852133001, + "tokens_seen": 26411008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006412698412698413, + "loss": 3.7231, + "theoretical_loss": 5.866014786419951, + "tokens_seen": 26476544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006428571428571429, + "loss": 3.8293, + "theoretical_loss": 5.86364224958853, + "tokens_seen": 26542080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006444444444444444, + "loss": 3.7392, + "theoretical_loss": 5.861277199320743, + "tokens_seen": 26607616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006460317460317461, + "loss": 3.7438, + "theoretical_loss": 5.858919593639941, + "tokens_seen": 26673152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006476190476190477, + "loss": 3.7997, + "theoretical_loss": 5.856569390907229, + "tokens_seen": 26738688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006492063492063492, + "loss": 3.8363, + "theoretical_loss": 5.854226549817936, + "tokens_seen": 26804224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006507936507936508, + "loss": 3.6657, + "theoretical_loss": 5.851891029398114, + "tokens_seen": 26869760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006523809523809524, + "loss": 3.7011, + "theoretical_loss": 5.8495627890011015, + "tokens_seen": 26935296 + }, + { + "epoch": 0.01, + "learning_rate": 0.000653968253968254, + "loss": 3.7339, + "theoretical_loss": 5.847241788304114, + "tokens_seen": 27000832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006555555555555556, + "loss": 3.77, + "theoretical_loss": 5.844927987304884, + "tokens_seen": 27066368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006571428571428571, + "loss": 3.5868, + "theoretical_loss": 5.842621346318348, + "tokens_seen": 27131904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006587301587301587, + "loss": 3.709, + "theoretical_loss": 5.84032182597337, + "tokens_seen": 27197440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006603174603174604, + "loss": 3.6199, + "theoretical_loss": 5.8380293872095095, + "tokens_seen": 27262976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006619047619047619, + "loss": 3.6197, + "theoretical_loss": 5.835743991273832, + "tokens_seen": 27328512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006634920634920636, + "loss": 3.507, + "theoretical_loss": 5.833465599717752, + "tokens_seen": 27394048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006650793650793651, + "loss": 3.7224, + "theoretical_loss": 5.831194174393929, + "tokens_seen": 27459584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006666666666666668, + "loss": 3.5851, + "theoretical_loss": 5.828929677453187, + "tokens_seen": 27525120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006682539682539683, + "loss": 3.5854, + "theoretical_loss": 5.826672071341488, + "tokens_seen": 27590656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006698412698412699, + "loss": 3.6313, + "theoretical_loss": 5.824421318796932, + "tokens_seen": 27656192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006714285714285715, + "loss": 3.6213, + "theoretical_loss": 5.822177382846799, + "tokens_seen": 27721728 + }, + { + "epoch": 0.01, + "learning_rate": 0.000673015873015873, + "loss": 3.6204, + "theoretical_loss": 5.8199402268046265, + "tokens_seen": 27787264 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006746031746031747, + "loss": 3.6112, + "theoretical_loss": 5.817709814267329, + "tokens_seen": 27852800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006761904761904762, + "loss": 3.7146, + "theoretical_loss": 5.815486109112342, + "tokens_seen": 27918336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006777777777777778, + "loss": 3.5925, + "theoretical_loss": 5.813269075494812, + "tokens_seen": 27983872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006793650793650794, + "loss": 3.5826, + "theoretical_loss": 5.811058677844817, + "tokens_seen": 28049408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006809523809523809, + "loss": 3.6308, + "theoretical_loss": 5.808854880864618, + "tokens_seen": 28114944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006825396825396826, + "loss": 3.5636, + "theoretical_loss": 5.8066576495259525, + "tokens_seen": 28180480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006841269841269841, + "loss": 3.5684, + "theoretical_loss": 5.804466949067352, + "tokens_seen": 28246016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006857142857142857, + "loss": 3.6715, + "theoretical_loss": 5.8022827449914995, + "tokens_seen": 28311552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006873015873015873, + "loss": 3.6888, + "theoretical_loss": 5.800105003062612, + "tokens_seen": 28377088 + }, + { + "epoch": 0.01, + "learning_rate": 0.000688888888888889, + "loss": 3.5941, + "theoretical_loss": 5.797933689303866, + "tokens_seen": 28442624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006904761904761906, + "loss": 3.6386, + "theoretical_loss": 5.795768769994842, + "tokens_seen": 28508160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006920634920634921, + "loss": 3.5843, + "theoretical_loss": 5.793610211669004, + "tokens_seen": 28573696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006936507936507937, + "loss": 3.5697, + "theoretical_loss": 5.79145798111122, + "tokens_seen": 28639232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006952380952380953, + "loss": 3.6252, + "theoretical_loss": 5.789312045355292, + "tokens_seen": 28704768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006968253968253969, + "loss": 3.6311, + "theoretical_loss": 5.787172371681535, + "tokens_seen": 28770304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006984126984126985, + "loss": 3.7606, + "theoretical_loss": 5.785038927614373, + "tokens_seen": 28835840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007, + "loss": 3.3781, + "theoretical_loss": 5.782911680919971, + "tokens_seen": 28901376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007015873015873016, + "loss": 3.6269, + "theoretical_loss": 5.78079059960389, + "tokens_seen": 28966912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007031746031746032, + "loss": 3.5331, + "theoretical_loss": 5.77867565190877, + "tokens_seen": 29032448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007047619047619048, + "loss": 3.5922, + "theoretical_loss": 5.776566806312052, + "tokens_seen": 29097984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007063492063492064, + "loss": 3.572, + "theoretical_loss": 5.774464031523705, + "tokens_seen": 29163520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007079365079365079, + "loss": 3.5933, + "theoretical_loss": 5.772367296484001, + "tokens_seen": 29229056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007095238095238095, + "loss": 3.624, + "theoretical_loss": 5.770276570361309, + "tokens_seen": 29294592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007111111111111111, + "loss": 3.5538, + "theoretical_loss": 5.768191822549908, + "tokens_seen": 29360128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007126984126984128, + "loss": 3.5156, + "theoretical_loss": 5.766113022667838, + "tokens_seen": 29425664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 16666, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.571195363998413, + "objective/train/theoretical_loss": 5.764040140554766, + "objective/train/tokens_used": 49951200, + "theoretical_loss": 5.764040140554766, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007142857142857144, + "loss": 3.5886, + "theoretical_loss": 5.764040140554766, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007158730158730159, + "loss": 3.6281, + "theoretical_loss": 5.761973146269881, + "tokens_seen": 29556736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007174603174603175, + "loss": 3.3317, + "theoretical_loss": 5.759912010089819, + "tokens_seen": 29622272 + }, + { + "epoch": 0.01, + "learning_rate": 0.000719047619047619, + "loss": 3.5542, + "theoretical_loss": 5.757856702506597, + "tokens_seen": 29687808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007206349206349207, + "loss": 3.584, + "theoretical_loss": 5.755807194225595, + "tokens_seen": 29753344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007222222222222223, + "loss": 3.4624, + "theoretical_loss": 5.753763456163529, + "tokens_seen": 29818880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007238095238095238, + "loss": 3.5575, + "theoretical_loss": 5.751725459446487, + "tokens_seen": 29884416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007253968253968254, + "loss": 3.5754, + "theoretical_loss": 5.7496931754079466, + "tokens_seen": 29949952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007269841269841271, + "loss": 3.6041, + "theoretical_loss": 5.747666575586849, + "tokens_seen": 30015488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007285714285714286, + "loss": 3.5614, + "theoretical_loss": 5.7456456317256785, + "tokens_seen": 30081024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007301587301587302, + "loss": 3.68, + "theoretical_loss": 5.743630315768563, + "tokens_seen": 30146560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007317460317460317, + "loss": 3.5685, + "theoretical_loss": 5.741620599859402, + "tokens_seen": 30212096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007333333333333333, + "loss": 3.646, + "theoretical_loss": 5.739616456340021, + "tokens_seen": 30277632 + }, + { + "epoch": 0.01, + "learning_rate": 0.000734920634920635, + "loss": 3.5125, + "theoretical_loss": 5.737617857748329, + "tokens_seen": 30343168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007365079365079365, + "loss": 3.5059, + "theoretical_loss": 5.73562477681652, + "tokens_seen": 30408704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007380952380952382, + "loss": 3.6117, + "theoretical_loss": 5.733637186469274, + "tokens_seen": 30474240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007396825396825397, + "loss": 3.5285, + "theoretical_loss": 5.731655059821993, + "tokens_seen": 30539776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007412698412698414, + "loss": 3.5112, + "theoretical_loss": 5.729678370179052, + "tokens_seen": 30605312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007428571428571429, + "loss": 3.3423, + "theoretical_loss": 5.727707091032062, + "tokens_seen": 30670848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007444444444444445, + "loss": 3.5549, + "theoretical_loss": 5.725741196058175, + "tokens_seen": 30736384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007460317460317461, + "loss": 3.5069, + "theoretical_loss": 5.723780659118377, + "tokens_seen": 30801920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007476190476190476, + "loss": 3.5296, + "theoretical_loss": 5.721825454255825, + "tokens_seen": 30867456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007492063492063493, + "loss": 3.6073, + "theoretical_loss": 5.71987555569419, + "tokens_seen": 30932992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007507936507936508, + "loss": 3.4977, + "theoretical_loss": 5.717930937836029, + "tokens_seen": 30998528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007523809523809524, + "loss": 3.5041, + "theoretical_loss": 5.715991575261159, + "tokens_seen": 31064064 + }, + { + "epoch": 0.01, + "learning_rate": 0.000753968253968254, + "loss": 3.4418, + "theoretical_loss": 5.714057442725068, + "tokens_seen": 31129600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007555555555555555, + "loss": 3.5898, + "theoretical_loss": 5.712128515157324, + "tokens_seen": 31195136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007571428571428572, + "loss": 3.5238, + "theoretical_loss": 5.710204767660022, + "tokens_seen": 31260672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007587301587301588, + "loss": 3.4737, + "theoretical_loss": 5.708286175506232, + "tokens_seen": 31326208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007603174603174603, + "loss": 3.5078, + "theoretical_loss": 5.706372714138472, + "tokens_seen": 31391744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007619047619047619, + "loss": 3.3246, + "theoretical_loss": 5.704464359167195, + "tokens_seen": 31457280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007634920634920635, + "loss": 3.5155, + "theoretical_loss": 5.702561086369294, + "tokens_seen": 31522816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007650793650793652, + "loss": 3.6084, + "theoretical_loss": 5.7006628716866246, + "tokens_seen": 31588352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007666666666666667, + "loss": 3.54, + "theoretical_loss": 5.698769691224539, + "tokens_seen": 31653888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007682539682539683, + "loss": 3.5811, + "theoretical_loss": 5.696881521250441, + "tokens_seen": 31719424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007698412698412699, + "loss": 3.4774, + "theoretical_loss": 5.694998338192356, + "tokens_seen": 31784960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007714285714285715, + "loss": 3.3984, + "theoretical_loss": 5.693120118637513, + "tokens_seen": 31850496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007730158730158731, + "loss": 3.495, + "theoretical_loss": 5.691246839330946, + "tokens_seen": 31916032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007746031746031746, + "loss": 3.3938, + "theoretical_loss": 5.689378477174106, + "tokens_seen": 31981568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007761904761904762, + "loss": 3.4542, + "theoretical_loss": 5.687515009223498, + "tokens_seen": 32047104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007777777777777778, + "loss": 3.5173, + "theoretical_loss": 5.685656412689318, + "tokens_seen": 32112640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007793650793650794, + "loss": 3.4633, + "theoretical_loss": 5.683802664934115, + "tokens_seen": 32178176 + }, + { + "epoch": 0.01, + "learning_rate": 0.000780952380952381, + "loss": 3.3623, + "theoretical_loss": 5.681953743471466, + "tokens_seen": 32243712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007825396825396825, + "loss": 3.5352, + "theoretical_loss": 5.680109625964663, + "tokens_seen": 32309248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007841269841269841, + "loss": 3.5366, + "theoretical_loss": 5.678270290225415, + "tokens_seen": 32374784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007857142857142857, + "loss": 3.4648, + "theoretical_loss": 5.676435714212562, + "tokens_seen": 32440320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007873015873015873, + "loss": 3.3286, + "theoretical_loss": 5.674605876030808, + "tokens_seen": 32505856 + }, + { + "epoch": 0.01, + "learning_rate": 0.000788888888888889, + "loss": 3.4303, + "theoretical_loss": 5.672780753929461, + "tokens_seen": 32571392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007904761904761905, + "loss": 3.4569, + "theoretical_loss": 5.67096032630119, + "tokens_seen": 32636928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007920634920634921, + "loss": 3.4918, + "theoretical_loss": 5.669144571680797, + "tokens_seen": 32702464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 17795, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.261939525604248, + "objective/train/theoretical_loss": 5.667333468743994, + "objective/train/tokens_used": 53228000, + "theoretical_loss": 5.667333468743994, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007936507936507938, + "loss": 3.3689, + "theoretical_loss": 5.667333468743994, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007952380952380953, + "loss": 3.3711, + "theoretical_loss": 5.665526996306205, + "tokens_seen": 32833536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968253968253969, + "loss": 3.5329, + "theoretical_loss": 5.663725133321369, + "tokens_seen": 32899072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984126984126984, + "loss": 3.5592, + "theoretical_loss": 5.661927858880761, + "tokens_seen": 32964608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0008, + "loss": 3.3173, + "theoretical_loss": 5.6601351522118275, + "tokens_seen": 33030144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999839518555668, + "loss": 3.5531, + "theoretical_loss": 5.65834699267703, + "tokens_seen": 33095680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999679037111334, + "loss": 3.2348, + "theoretical_loss": 5.6565633597727025, + "tokens_seen": 33161216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999518555667002, + "loss": 3.4544, + "theoretical_loss": 5.65478423312792, + "tokens_seen": 33226752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999358074222668, + "loss": 3.4522, + "theoretical_loss": 5.65300959250338, + "tokens_seen": 33292288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999197592778336, + "loss": 3.4293, + "theoretical_loss": 5.651239417790293, + "tokens_seen": 33357824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999037111334002, + "loss": 3.3407, + "theoretical_loss": 5.649473689009292, + "tokens_seen": 33423360 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799887662988967, + "loss": 3.3672, + "theoretical_loss": 5.6477123863093395, + "tokens_seen": 33488896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007998716148445336, + "loss": 3.403, + "theoretical_loss": 5.645955489966661, + "tokens_seen": 33554432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007998555667001004, + "loss": 3.3867, + "theoretical_loss": 5.644202980383678, + "tokens_seen": 33619968 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799839518555667, + "loss": 3.351, + "theoretical_loss": 5.6424548380879616, + "tokens_seen": 33685504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007998234704112338, + "loss": 3.4848, + "theoretical_loss": 5.640711043731184, + "tokens_seen": 33751040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007998074222668004, + "loss": 3.4619, + "theoretical_loss": 5.6389715780880945, + "tokens_seen": 33816576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997913741223672, + "loss": 3.4355, + "theoretical_loss": 5.6372364220555, + "tokens_seen": 33882112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997753259779338, + "loss": 3.4325, + "theoretical_loss": 5.635505556651251, + "tokens_seen": 33947648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997592778335006, + "loss": 3.4346, + "theoretical_loss": 5.633778963013247, + "tokens_seen": 34013184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997432296890672, + "loss": 3.3384, + "theoretical_loss": 5.632056622398448, + "tokens_seen": 34078720 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799727181544634, + "loss": 3.4516, + "theoretical_loss": 5.63033851618189, + "tokens_seen": 34144256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997111334002006, + "loss": 3.2575, + "theoretical_loss": 5.628624625855725, + "tokens_seen": 34209792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996950852557674, + "loss": 3.3118, + "theoretical_loss": 5.626914933028255, + "tokens_seen": 34275328 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799679037111334, + "loss": 3.3996, + "theoretical_loss": 5.625209419422983, + "tokens_seen": 34340864 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996629889669008, + "loss": 3.4081, + "theoretical_loss": 5.623508066877678, + "tokens_seen": 34406400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996469408224675, + "loss": 3.3567, + "theoretical_loss": 5.621810857343441, + "tokens_seen": 34471936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996308926780342, + "loss": 3.2694, + "theoretical_loss": 5.620117772883784, + "tokens_seen": 34537472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996148445336009, + "loss": 3.4846, + "theoretical_loss": 5.618428795673718, + "tokens_seen": 34603008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995987963891676, + "loss": 3.4082, + "theoretical_loss": 5.616743907998853, + "tokens_seen": 34668544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995827482447343, + "loss": 3.3935, + "theoretical_loss": 5.6150630922545, + "tokens_seen": 34734080 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799566700100301, + "loss": 3.3234, + "theoretical_loss": 5.613386330944792, + "tokens_seen": 34799616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995506519558677, + "loss": 3.3006, + "theoretical_loss": 5.6117136066818, + "tokens_seen": 34865152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995346038114344, + "loss": 3.3978, + "theoretical_loss": 5.610044902184672, + "tokens_seen": 34930688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995185556670011, + "loss": 3.3917, + "theoretical_loss": 5.608380200278775, + "tokens_seen": 34996224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995025075225678, + "loss": 3.2776, + "theoretical_loss": 5.606719483894837, + "tokens_seen": 35061760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994864593781345, + "loss": 3.429, + "theoretical_loss": 5.605062736068113, + "tokens_seen": 35127296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994704112337012, + "loss": 3.3658, + "theoretical_loss": 5.603409939937547, + "tokens_seen": 35192832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994543630892679, + "loss": 3.3409, + "theoretical_loss": 5.601761078744952, + "tokens_seen": 35258368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994383149448346, + "loss": 3.3312, + "theoretical_loss": 5.600116135834183, + "tokens_seen": 35323904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994222668004013, + "loss": 3.1932, + "theoretical_loss": 5.598475094650334, + "tokens_seen": 35389440 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799406218655968, + "loss": 3.2632, + "theoretical_loss": 5.59683793873894, + "tokens_seen": 35454976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993901705115347, + "loss": 3.3286, + "theoretical_loss": 5.595204651745176, + "tokens_seen": 35520512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993741223671014, + "loss": 3.1714, + "theoretical_loss": 5.593575217413074, + "tokens_seen": 35586048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993580742226681, + "loss": 3.2249, + "theoretical_loss": 5.591949619584746, + "tokens_seen": 35651584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993420260782347, + "loss": 3.1943, + "theoretical_loss": 5.590327842199611, + "tokens_seen": 35717120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993259779338015, + "loss": 3.261, + "theoretical_loss": 5.588709869293634, + "tokens_seen": 35782656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993099297893681, + "loss": 3.3474, + "theoretical_loss": 5.5870956849985705, + "tokens_seen": 35848192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992938816449349, + "loss": 3.4015, + "theoretical_loss": 5.585485273541212, + "tokens_seen": 35913728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992778335005015, + "loss": 3.3678, + "theoretical_loss": 5.583878619242652, + "tokens_seen": 35979264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 18676, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 3.2103042602539062, + "objective/train/theoretical_loss": 5.582275706517549, + "objective/train/tokens_used": 56504800, + "theoretical_loss": 5.582275706517549, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992617853560683, + "loss": 3.3616, + "theoretical_loss": 5.582275706517549, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992457372116349, + "loss": 3.3332, + "theoretical_loss": 5.580676519873396, + "tokens_seen": 36110336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992296890672017, + "loss": 3.5113, + "theoretical_loss": 5.579081043909808, + "tokens_seen": 36175872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992136409227683, + "loss": 3.4334, + "theoretical_loss": 5.577489263317796, + "tokens_seen": 36241408 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799197592778335, + "loss": 3.3125, + "theoretical_loss": 5.5759011628790764, + "tokens_seen": 36306944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991815446339017, + "loss": 3.4013, + "theoretical_loss": 5.574316727465358, + "tokens_seen": 36372480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991654964894684, + "loss": 3.2732, + "theoretical_loss": 5.572735942037655, + "tokens_seen": 36438016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991494483450351, + "loss": 3.1625, + "theoretical_loss": 5.571158791645597, + "tokens_seen": 36503552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991334002006018, + "loss": 3.284, + "theoretical_loss": 5.569585261426752, + "tokens_seen": 36569088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991173520561685, + "loss": 3.4507, + "theoretical_loss": 5.568015336605953, + "tokens_seen": 36634624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991013039117352, + "loss": 3.3153, + "theoretical_loss": 5.566449002494627, + "tokens_seen": 36700160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990852557673019, + "loss": 3.135, + "theoretical_loss": 5.564886244490136, + "tokens_seen": 36765696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990692076228686, + "loss": 3.3133, + "theoretical_loss": 5.563327048075124, + "tokens_seen": 36831232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990531594784353, + "loss": 3.3034, + "theoretical_loss": 5.5617713988168616, + "tokens_seen": 36896768 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799037111334002, + "loss": 3.2984, + "theoretical_loss": 5.560219282366614, + "tokens_seen": 36962304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990210631895687, + "loss": 3.3318, + "theoretical_loss": 5.558670684458994, + "tokens_seen": 37027840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990050150451354, + "loss": 3.3465, + "theoretical_loss": 5.557125590911339, + "tokens_seen": 37093376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989889669007021, + "loss": 3.3319, + "theoretical_loss": 5.5555839876230815, + "tokens_seen": 37158912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989729187562688, + "loss": 3.2583, + "theoretical_loss": 5.55404586057513, + "tokens_seen": 37224448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989568706118355, + "loss": 3.1394, + "theoretical_loss": 5.5525111958292594, + "tokens_seen": 37289984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989408224674022, + "loss": 3.3485, + "theoretical_loss": 5.550979979527501, + "tokens_seen": 37355520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989247743229689, + "loss": 3.3191, + "theoretical_loss": 5.549452197891538, + "tokens_seen": 37421056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989087261785356, + "loss": 3.243, + "theoretical_loss": 5.547927837222119, + "tokens_seen": 37486592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988926780341023, + "loss": 3.128, + "theoretical_loss": 5.546406883898451, + "tokens_seen": 37552128 + }, + { + "epoch": 0.01, + "learning_rate": 0.000798876629889669, + "loss": 3.2572, + "theoretical_loss": 5.544889324377628, + "tokens_seen": 37617664 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988605817452357, + "loss": 3.1857, + "theoretical_loss": 5.543375145194048, + "tokens_seen": 37683200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988445336008024, + "loss": 3.1529, + "theoretical_loss": 5.541864332958831, + "tokens_seen": 37748736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988284854563691, + "loss": 3.2037, + "theoretical_loss": 5.540356874359257, + "tokens_seen": 37814272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988124373119358, + "loss": 3.2591, + "theoretical_loss": 5.538852756158201, + "tokens_seen": 37879808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987963891675025, + "loss": 3.1802, + "theoretical_loss": 5.537351965193569, + "tokens_seen": 37945344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987803410230693, + "loss": 3.1964, + "theoretical_loss": 5.535854488377751, + "tokens_seen": 38010880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987642928786359, + "loss": 3.346, + "theoretical_loss": 5.534360312697064, + "tokens_seen": 38076416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987482447342027, + "loss": 3.1631, + "theoretical_loss": 5.532869425211214, + "tokens_seen": 38141952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987321965897693, + "loss": 3.2718, + "theoretical_loss": 5.531381813052757, + "tokens_seen": 38207488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987161484453361, + "loss": 3.17, + "theoretical_loss": 5.529897463426559, + "tokens_seen": 38273024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987001003009027, + "loss": 3.2378, + "theoretical_loss": 5.528416363609276, + "tokens_seen": 38338560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986840521564695, + "loss": 3.1696, + "theoretical_loss": 5.526938500948817, + "tokens_seen": 38404096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986680040120361, + "loss": 3.2103, + "theoretical_loss": 5.525463862863836, + "tokens_seen": 38469632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986519558676029, + "loss": 3.0849, + "theoretical_loss": 5.523992436843209, + "tokens_seen": 38535168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986359077231695, + "loss": 3.2743, + "theoretical_loss": 5.522524210445528, + "tokens_seen": 38600704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986198595787363, + "loss": 3.272, + "theoretical_loss": 5.521059171298592, + "tokens_seen": 38666240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986038114343029, + "loss": 3.3046, + "theoretical_loss": 5.519597307098907, + "tokens_seen": 38731776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985877632898697, + "loss": 3.2986, + "theoretical_loss": 5.518138605611188, + "tokens_seen": 38797312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985717151454363, + "loss": 3.2102, + "theoretical_loss": 5.516683054667868, + "tokens_seen": 38862848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985556670010031, + "loss": 3.3232, + "theoretical_loss": 5.515230642168607, + "tokens_seen": 38928384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985396188565697, + "loss": 3.2207, + "theoretical_loss": 5.513781356079811, + "tokens_seen": 38993920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985235707121365, + "loss": 3.1524, + "theoretical_loss": 5.512335184434152, + "tokens_seen": 39059456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985075225677031, + "loss": 3.3457, + "theoretical_loss": 5.5108921153300905, + "tokens_seen": 39124992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984914744232699, + "loss": 3.1989, + "theoretical_loss": 5.5094521369314045, + "tokens_seen": 39190528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984754262788365, + "loss": 3.0462, + "theoretical_loss": 5.5080152374667275, + "tokens_seen": 39256064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 19810, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.971580982208252, + "objective/train/theoretical_loss": 5.506581405229079, + "objective/train/tokens_used": 59781600, + "theoretical_loss": 5.506581405229079, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984593781344033, + "loss": 3.0874, + "theoretical_loss": 5.506581405229079, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984433299899699, + "loss": 3.2052, + "theoretical_loss": 5.505150628575409, + "tokens_seen": 39387136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984272818455367, + "loss": 3.164, + "theoretical_loss": 5.503722895926143, + "tokens_seen": 39452672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984112337011033, + "loss": 3.0422, + "theoretical_loss": 5.50229819576473, + "tokens_seen": 39518208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983951855566701, + "loss": 3.0343, + "theoretical_loss": 5.500876516637199, + "tokens_seen": 39583744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983791374122367, + "loss": 3.1188, + "theoretical_loss": 5.49945784715171, + "tokens_seen": 39649280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983630892678035, + "loss": 3.0288, + "theoretical_loss": 5.498042175978123, + "tokens_seen": 39714816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983470411233701, + "loss": 3.2065, + "theoretical_loss": 5.496629491847553, + "tokens_seen": 39780352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983309929789369, + "loss": 3.1265, + "theoretical_loss": 5.495219783551947, + "tokens_seen": 39845888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983149448345036, + "loss": 3.1179, + "theoretical_loss": 5.493813039943654, + "tokens_seen": 39911424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982988966900703, + "loss": 2.9897, + "theoretical_loss": 5.492409249934996, + "tokens_seen": 39976960 + }, + { + "epoch": 0.01, + "learning_rate": 0.000798282848545637, + "loss": 3.1839, + "theoretical_loss": 5.491008402497855, + "tokens_seen": 40042496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982668004012037, + "loss": 3.1042, + "theoretical_loss": 5.489610486663252, + "tokens_seen": 40108032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982507522567704, + "loss": 3.0255, + "theoretical_loss": 5.488215491520933, + "tokens_seen": 40173568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982347041123371, + "loss": 3.0889, + "theoretical_loss": 5.486823406218962, + "tokens_seen": 40239104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982186559679038, + "loss": 3.2075, + "theoretical_loss": 5.485434219963315, + "tokens_seen": 40304640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982026078234705, + "loss": 3.0804, + "theoretical_loss": 5.484047922017476, + "tokens_seen": 40370176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981865596790372, + "loss": 3.199, + "theoretical_loss": 5.482664501702036, + "tokens_seen": 40435712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981705115346039, + "loss": 3.0804, + "theoretical_loss": 5.481283948394303, + "tokens_seen": 40501248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981544633901706, + "loss": 3.1914, + "theoretical_loss": 5.479906251527901, + "tokens_seen": 40566784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981384152457373, + "loss": 3.1122, + "theoretical_loss": 5.478531400592393, + "tokens_seen": 40632320 + }, + { + "epoch": 0.01, + "learning_rate": 0.000798122367101304, + "loss": 3.0153, + "theoretical_loss": 5.4771593851328815, + "tokens_seen": 40697856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981063189568707, + "loss": 3.0745, + "theoretical_loss": 5.475790194749637, + "tokens_seen": 40763392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980902708124374, + "loss": 3.2291, + "theoretical_loss": 5.474423819097713, + "tokens_seen": 40828928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980742226680041, + "loss": 3.072, + "theoretical_loss": 5.473060247886572, + "tokens_seen": 40894464 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980581745235708, + "loss": 3.0078, + "theoretical_loss": 5.471699470879715, + "tokens_seen": 40960000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980421263791375, + "loss": 3.0983, + "theoretical_loss": 5.470341477894303, + "tokens_seen": 41025536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980260782347042, + "loss": 3.1138, + "theoretical_loss": 5.468986258800805, + "tokens_seen": 41091072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980100300902709, + "loss": 3.0204, + "theoretical_loss": 5.4676338035226175, + "tokens_seen": 41156608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007979939819458376, + "loss": 3.2892, + "theoretical_loss": 5.466284102035719, + "tokens_seen": 41222144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007979779338014043, + "loss": 3.1621, + "theoretical_loss": 5.464937144368307, + "tokens_seen": 41287680 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797961885656971, + "loss": 3.0173, + "theoretical_loss": 5.463592920600435, + "tokens_seen": 41353216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007979458375125376, + "loss": 3.0774, + "theoretical_loss": 5.46225142086368, + "tokens_seen": 41418752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007979297893681044, + "loss": 3.1238, + "theoretical_loss": 5.460912635340775, + "tokens_seen": 41484288 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797913741223671, + "loss": 3.0618, + "theoretical_loss": 5.459576554265276, + "tokens_seen": 41549824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978976930792378, + "loss": 3.1313, + "theoretical_loss": 5.458243167921212, + "tokens_seen": 41615360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978816449348044, + "loss": 3.2165, + "theoretical_loss": 5.456912466642748, + "tokens_seen": 41680896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978655967903712, + "loss": 2.9085, + "theoretical_loss": 5.455584440813853, + "tokens_seen": 41746432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978495486459378, + "loss": 2.9859, + "theoretical_loss": 5.454259080867951, + "tokens_seen": 41811968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978335005015046, + "loss": 2.9411, + "theoretical_loss": 5.4529363772876085, + "tokens_seen": 41877504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978174523570712, + "loss": 3.0094, + "theoretical_loss": 5.451616320604193, + "tokens_seen": 41943040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978014042126379, + "loss": 2.9767, + "theoretical_loss": 5.450298901397547, + "tokens_seen": 42008576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977853560682046, + "loss": 3.1042, + "theoretical_loss": 5.448984110295672, + "tokens_seen": 42074112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977693079237713, + "loss": 3.0762, + "theoretical_loss": 5.447671937974404, + "tokens_seen": 42139648 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797753259779338, + "loss": 2.9742, + "theoretical_loss": 5.446362375157092, + "tokens_seen": 42205184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977372116349047, + "loss": 3.2779, + "theoretical_loss": 5.445055412614287, + "tokens_seen": 42270720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977211634904714, + "loss": 2.9849, + "theoretical_loss": 5.443751041163428, + "tokens_seen": 42336256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977051153460381, + "loss": 3.0884, + "theoretical_loss": 5.442449251668533, + "tokens_seen": 42401792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976890672016048, + "loss": 3.0079, + "theoretical_loss": 5.441150035039886, + "tokens_seen": 42467328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976730190571715, + "loss": 3.0901, + "theoretical_loss": 5.439853382233738, + "tokens_seen": 42532864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 20768, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.992009401321411, + "objective/train/theoretical_loss": 5.438559284252003, + "objective/train/tokens_used": 63058400, + "theoretical_loss": 5.438559284252003, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976569709127382, + "loss": 2.9284, + "theoretical_loss": 5.438559284252003, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976409227683049, + "loss": 2.9027, + "theoretical_loss": 5.437267732141949, + "tokens_seen": 42663936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976248746238716, + "loss": 2.9383, + "theoretical_loss": 5.435978716995917, + "tokens_seen": 42729472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976088264794383, + "loss": 3.0424, + "theoretical_loss": 5.434692229951009, + "tokens_seen": 42795008 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797592778335005, + "loss": 3.0061, + "theoretical_loss": 5.433408262188802, + "tokens_seen": 42860544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975767301905718, + "loss": 3.0298, + "theoretical_loss": 5.432126804935059, + "tokens_seen": 42926080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975606820461384, + "loss": 3.1409, + "theoretical_loss": 5.430847849459438, + "tokens_seen": 42991616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975446339017052, + "loss": 3.0722, + "theoretical_loss": 5.429571387075203, + "tokens_seen": 43057152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975285857572718, + "loss": 3.055, + "theoretical_loss": 5.428297409138947, + "tokens_seen": 43122688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975125376128386, + "loss": 2.936, + "theoretical_loss": 5.427025907050307, + "tokens_seen": 43188224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974964894684052, + "loss": 3.0608, + "theoretical_loss": 5.425756872251682, + "tokens_seen": 43253760 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797480441323972, + "loss": 2.969, + "theoretical_loss": 5.42449029622796, + "tokens_seen": 43319296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974643931795386, + "loss": 2.982, + "theoretical_loss": 5.423226170506243, + "tokens_seen": 43384832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974483450351054, + "loss": 2.959, + "theoretical_loss": 5.421964486655567, + "tokens_seen": 43450368 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797432296890672, + "loss": 3.1053, + "theoretical_loss": 5.420705236286646, + "tokens_seen": 43515904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974162487462388, + "loss": 3.1943, + "theoretical_loss": 5.41944841105159, + "tokens_seen": 43581440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974002006018054, + "loss": 2.9255, + "theoretical_loss": 5.418194002643643, + "tokens_seen": 43646976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973841524573722, + "loss": 3.0381, + "theoretical_loss": 5.416942002796926, + "tokens_seen": 43712512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973681043129388, + "loss": 2.9989, + "theoretical_loss": 5.415692403286162, + "tokens_seen": 43778048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973520561685056, + "loss": 3.0323, + "theoretical_loss": 5.414445195926426, + "tokens_seen": 43843584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973360080240722, + "loss": 2.9815, + "theoretical_loss": 5.413200372572884, + "tokens_seen": 43909120 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797319959879639, + "loss": 3.0141, + "theoretical_loss": 5.411957925120538, + "tokens_seen": 43974656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973039117352056, + "loss": 3.0708, + "theoretical_loss": 5.410717845503969, + "tokens_seen": 44040192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972878635907724, + "loss": 2.8975, + "theoretical_loss": 5.40948012569709, + "tokens_seen": 44105728 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797271815446339, + "loss": 3.0037, + "theoretical_loss": 5.408244757712893, + "tokens_seen": 44171264 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972557673019058, + "loss": 2.9777, + "theoretical_loss": 5.407011733603199, + "tokens_seen": 44236800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972397191574724, + "loss": 2.885, + "theoretical_loss": 5.4057810454584185, + "tokens_seen": 44302336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972236710130392, + "loss": 2.9414, + "theoretical_loss": 5.404552685407301, + "tokens_seen": 44367872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972076228686058, + "loss": 3.097, + "theoretical_loss": 5.403326645616698, + "tokens_seen": 44433408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007971915747241726, + "loss": 2.9061, + "theoretical_loss": 5.402102918291316, + "tokens_seen": 44498944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007971755265797392, + "loss": 3.0093, + "theoretical_loss": 5.400881495673488, + "tokens_seen": 44564480 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797159478435306, + "loss": 3.103, + "theoretical_loss": 5.3996623700429245, + "tokens_seen": 44630016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007971434302908726, + "loss": 2.9274, + "theoretical_loss": 5.398445533716492, + "tokens_seen": 44695552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007971273821464394, + "loss": 3.0244, + "theoretical_loss": 5.3972309790479684, + "tokens_seen": 44761088 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797111334002006, + "loss": 2.9874, + "theoretical_loss": 5.39601869842782, + "tokens_seen": 44826624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970952858575728, + "loss": 2.893, + "theoretical_loss": 5.394808684282968, + "tokens_seen": 44892160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970792377131394, + "loss": 2.9119, + "theoretical_loss": 5.393600929076564, + "tokens_seen": 44957696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970631895687062, + "loss": 2.845, + "theoretical_loss": 5.392395425307758, + "tokens_seen": 45023232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970471414242728, + "loss": 2.8557, + "theoretical_loss": 5.391192165511484, + "tokens_seen": 45088768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970310932798396, + "loss": 2.9916, + "theoretical_loss": 5.389991142258227, + "tokens_seen": 45154304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970150451354062, + "loss": 3.0289, + "theoretical_loss": 5.38879234815381, + "tokens_seen": 45219840 + }, + { + "epoch": 0.01, + "learning_rate": 0.000796998996990973, + "loss": 2.857, + "theoretical_loss": 5.387595775839175, + "tokens_seen": 45285376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969829488465396, + "loss": 2.9597, + "theoretical_loss": 5.386401417990159, + "tokens_seen": 45350912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969669007021064, + "loss": 2.9621, + "theoretical_loss": 5.385209267317286, + "tokens_seen": 45416448 + }, + { + "epoch": 0.01, + "learning_rate": 0.000796950852557673, + "loss": 2.9259, + "theoretical_loss": 5.3840193165655466, + "tokens_seen": 45481984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969348044132398, + "loss": 2.8076, + "theoretical_loss": 5.382831558514191, + "tokens_seen": 45547520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969187562688065, + "loss": 2.9423, + "theoretical_loss": 5.3816459859765144, + "tokens_seen": 45613056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969027081243732, + "loss": 2.7761, + "theoretical_loss": 5.380462591799651, + "tokens_seen": 45678592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968866599799399, + "loss": 2.8484, + "theoretical_loss": 5.379281368864362, + "tokens_seen": 45744128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968706118355066, + "loss": 2.8477, + "theoretical_loss": 5.378102310084834, + "tokens_seen": 45809664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 22096, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.943756580352783, + "objective/train/theoretical_loss": 5.376925408408475, + "objective/train/tokens_used": 66335200, + "theoretical_loss": 5.376925408408475, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968545636910733, + "loss": 2.9656, + "theoretical_loss": 5.376925408408475, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00079683851554664, + "loss": 2.8387, + "theoretical_loss": 5.375750656815705, + "tokens_seen": 45940736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968224674022067, + "loss": 2.9168, + "theoretical_loss": 5.37457804831976, + "tokens_seen": 46006272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968064192577734, + "loss": 2.8562, + "theoretical_loss": 5.373407575966493, + "tokens_seen": 46071808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967903711133401, + "loss": 2.8091, + "theoretical_loss": 5.372239232834173, + "tokens_seen": 46137344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967743229689068, + "loss": 2.9054, + "theoretical_loss": 5.3710730120332855, + "tokens_seen": 46202880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967582748244735, + "loss": 2.9022, + "theoretical_loss": 5.369908906706346, + "tokens_seen": 46268416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967422266800402, + "loss": 2.9656, + "theoretical_loss": 5.368746910027696, + "tokens_seen": 46333952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967261785356069, + "loss": 2.7845, + "theoretical_loss": 5.3675870152033145, + "tokens_seen": 46399488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967101303911736, + "loss": 2.8521, + "theoretical_loss": 5.36642921547063, + "tokens_seen": 46465024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966940822467403, + "loss": 2.8249, + "theoretical_loss": 5.365273504098326, + "tokens_seen": 46530560 + }, + { + "epoch": 0.01, + "learning_rate": 0.000796678034102307, + "loss": 2.7694, + "theoretical_loss": 5.364119874386157, + "tokens_seen": 46596096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966619859578737, + "loss": 2.9288, + "theoretical_loss": 5.362968319664756, + "tokens_seen": 46661632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966459378134404, + "loss": 2.9025, + "theoretical_loss": 5.361818833295455, + "tokens_seen": 46727168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966298896690071, + "loss": 2.9219, + "theoretical_loss": 5.3606714086701, + "tokens_seen": 46792704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966138415245738, + "loss": 2.9099, + "theoretical_loss": 5.359526039210861, + "tokens_seen": 46858240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965977933801405, + "loss": 2.88, + "theoretical_loss": 5.358382718370063, + "tokens_seen": 46923776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965817452357072, + "loss": 2.8903, + "theoretical_loss": 5.357241439629992, + "tokens_seen": 46989312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965656970912739, + "loss": 2.7691, + "theoretical_loss": 5.3561021965027305, + "tokens_seen": 47054848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965496489468405, + "loss": 2.8294, + "theoretical_loss": 5.354964982529967, + "tokens_seen": 47120384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965336008024073, + "loss": 2.8102, + "theoretical_loss": 5.353829791282831, + "tokens_seen": 47185920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965175526579739, + "loss": 2.7822, + "theoretical_loss": 5.35269661636171, + "tokens_seen": 47251456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965015045135407, + "loss": 2.8876, + "theoretical_loss": 5.35156545139608, + "tokens_seen": 47316992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964854563691073, + "loss": 2.7635, + "theoretical_loss": 5.350436290044332, + "tokens_seen": 47382528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964694082246741, + "loss": 2.8377, + "theoretical_loss": 5.3493091259936, + "tokens_seen": 47448064 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964533600802407, + "loss": 2.8324, + "theoretical_loss": 5.348183952959596, + "tokens_seen": 47513600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964373119358075, + "loss": 2.7908, + "theoretical_loss": 5.347060764686436, + "tokens_seen": 47579136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964212637913741, + "loss": 2.8496, + "theoretical_loss": 5.345939554946472, + "tokens_seen": 47644672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964052156469408, + "loss": 2.7488, + "theoretical_loss": 5.344820317540132, + "tokens_seen": 47710208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963891675025075, + "loss": 2.8114, + "theoretical_loss": 5.343703046295751, + "tokens_seen": 47775744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963731193580743, + "loss": 2.8671, + "theoretical_loss": 5.342587735069408, + "tokens_seen": 47841280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963570712136409, + "loss": 2.9205, + "theoretical_loss": 5.3414743777447615, + "tokens_seen": 47906816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963410230692077, + "loss": 2.7384, + "theoretical_loss": 5.340362968232892, + "tokens_seen": 47972352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963249749247743, + "loss": 2.9322, + "theoretical_loss": 5.339253500472138, + "tokens_seen": 48037888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963089267803411, + "loss": 2.6983, + "theoretical_loss": 5.338145968427941, + "tokens_seen": 48103424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962928786359077, + "loss": 2.8289, + "theoretical_loss": 5.337040366092687, + "tokens_seen": 48168960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962768304914745, + "loss": 2.9222, + "theoretical_loss": 5.335936687485542, + "tokens_seen": 48234496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962607823470411, + "loss": 2.7881, + "theoretical_loss": 5.334834926652308, + "tokens_seen": 48300032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962447342026079, + "loss": 2.7958, + "theoretical_loss": 5.33373507766526, + "tokens_seen": 48365568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962286860581745, + "loss": 2.8646, + "theoretical_loss": 5.332637134622999, + "tokens_seen": 48431104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962126379137413, + "loss": 2.7199, + "theoretical_loss": 5.331541091650294, + "tokens_seen": 48496640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961965897693079, + "loss": 2.8311, + "theoretical_loss": 5.330446942897931, + "tokens_seen": 48562176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961805416248747, + "loss": 2.8684, + "theoretical_loss": 5.32935468254257, + "tokens_seen": 48627712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961644934804413, + "loss": 2.822, + "theoretical_loss": 5.328264304786586, + "tokens_seen": 48693248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961484453360081, + "loss": 2.7969, + "theoretical_loss": 5.327175803857929, + "tokens_seen": 48758784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961323971915747, + "loss": 2.7726, + "theoretical_loss": 5.326089174009972, + "tokens_seen": 48824320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961163490471415, + "loss": 2.7235, + "theoretical_loss": 5.325004409521368, + "tokens_seen": 48889856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961003009027081, + "loss": 2.7985, + "theoretical_loss": 5.323921504695903, + "tokens_seen": 48955392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960842527582749, + "loss": 2.9327, + "theoretical_loss": 5.322840453862356, + "tokens_seen": 49020928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960682046138415, + "loss": 2.7259, + "theoretical_loss": 5.3217612513743475, + "tokens_seen": 49086464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 22959, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.8861923217773438, + "objective/train/theoretical_loss": 5.320683891610208, + "objective/train/tokens_used": 69612000, + "theoretical_loss": 5.320683891610208, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960521564694083, + "loss": 2.8363, + "theoretical_loss": 5.320683891610208, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960361083249749, + "loss": 2.8106, + "theoretical_loss": 5.3196083689728315, + "tokens_seen": 49217536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960200601805417, + "loss": 2.8124, + "theoretical_loss": 5.318534677889536, + "tokens_seen": 49283072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960040120361083, + "loss": 2.6993, + "theoretical_loss": 5.317462812811925, + "tokens_seen": 49348608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007959879638916751, + "loss": 2.7378, + "theoretical_loss": 5.316392768215751, + "tokens_seen": 49414144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007959719157472417, + "loss": 2.7691, + "theoretical_loss": 5.315324538600778, + "tokens_seen": 49479680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007959558676028085, + "loss": 2.9151, + "theoretical_loss": 5.31425811849064, + "tokens_seen": 49545216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007959398194583751, + "loss": 2.616, + "theoretical_loss": 5.313193502432718, + "tokens_seen": 49610752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007959237713139419, + "loss": 2.7306, + "theoretical_loss": 5.312130684997992, + "tokens_seen": 49676288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007959077231695085, + "loss": 2.801, + "theoretical_loss": 5.31106966078092, + "tokens_seen": 49741824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958916750250753, + "loss": 2.736, + "theoretical_loss": 5.310010424399295, + "tokens_seen": 49807360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958756268806419, + "loss": 2.7149, + "theoretical_loss": 5.308952970494124, + "tokens_seen": 49872896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958595787362087, + "loss": 2.7643, + "theoretical_loss": 5.307897293729489, + "tokens_seen": 49938432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958435305917753, + "loss": 2.7688, + "theoretical_loss": 5.306843388792423, + "tokens_seen": 50003968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958274824473421, + "loss": 2.6344, + "theoretical_loss": 5.305791250392778, + "tokens_seen": 50069504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958114343029087, + "loss": 2.7253, + "theoretical_loss": 5.304740873263098, + "tokens_seen": 50135040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957953861584755, + "loss": 2.7176, + "theoretical_loss": 5.303692252158496, + "tokens_seen": 50200576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957793380140421, + "loss": 2.7815, + "theoretical_loss": 5.302645381856522, + "tokens_seen": 50266112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957632898696089, + "loss": 2.7198, + "theoretical_loss": 5.30160025715704, + "tokens_seen": 50331648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957472417251755, + "loss": 2.5469, + "theoretical_loss": 5.300556872882108, + "tokens_seen": 50397184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957311935807423, + "loss": 2.6499, + "theoretical_loss": 5.299515223875848, + "tokens_seen": 50462720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957151454363089, + "loss": 2.7165, + "theoretical_loss": 5.2984753050043265, + "tokens_seen": 50528256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956990972918757, + "loss": 2.6517, + "theoretical_loss": 5.2974371111554355, + "tokens_seen": 50593792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956830491474423, + "loss": 2.6449, + "theoretical_loss": 5.296400637238764, + "tokens_seen": 50659328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956670010030091, + "loss": 2.5609, + "theoretical_loss": 5.295365878185488, + "tokens_seen": 50724864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956509528585757, + "loss": 2.4886, + "theoretical_loss": 5.294332828948243, + "tokens_seen": 50790400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956349047141425, + "loss": 2.6212, + "theoretical_loss": 5.2933014845010105, + "tokens_seen": 50855936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956188565697091, + "loss": 2.6007, + "theoretical_loss": 5.292271839838996, + "tokens_seen": 50921472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956028084252759, + "loss": 2.7466, + "theoretical_loss": 5.291243889978516, + "tokens_seen": 50987008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955867602808426, + "loss": 2.7354, + "theoretical_loss": 5.290217629956882, + "tokens_seen": 51052544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955707121364093, + "loss": 2.8293, + "theoretical_loss": 5.289193054832282, + "tokens_seen": 51118080 + }, + { + "epoch": 0.02, + "learning_rate": 0.000795554663991976, + "loss": 2.7158, + "theoretical_loss": 5.288170159683668, + "tokens_seen": 51183616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955386158475427, + "loss": 2.6579, + "theoretical_loss": 5.287148939610645, + "tokens_seen": 51249152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955225677031094, + "loss": 2.5181, + "theoretical_loss": 5.28612938973335, + "tokens_seen": 51314688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955065195586761, + "loss": 2.7695, + "theoretical_loss": 5.285111505192349, + "tokens_seen": 51380224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954904714142428, + "loss": 2.6868, + "theoretical_loss": 5.284095281148522, + "tokens_seen": 51445760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954744232698095, + "loss": 2.747, + "theoretical_loss": 5.283080712782951, + "tokens_seen": 51511296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954583751253762, + "loss": 2.6446, + "theoretical_loss": 5.282067795296811, + "tokens_seen": 51576832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954423269809429, + "loss": 2.6414, + "theoretical_loss": 5.28105652391126, + "tokens_seen": 51642368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954262788365096, + "loss": 2.6543, + "theoretical_loss": 5.280046893867333, + "tokens_seen": 51707904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954102306920763, + "loss": 2.5674, + "theoretical_loss": 5.279038900425832, + "tokens_seen": 51773440 + }, + { + "epoch": 0.02, + "learning_rate": 0.000795394182547643, + "loss": 2.6305, + "theoretical_loss": 5.278032538867221, + "tokens_seen": 51838976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953781344032097, + "loss": 2.5573, + "theoretical_loss": 5.277027804491517, + "tokens_seen": 51904512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953620862587764, + "loss": 2.6, + "theoretical_loss": 5.2760246926181855, + "tokens_seen": 51970048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953460381143431, + "loss": 2.6884, + "theoretical_loss": 5.275023198586037, + "tokens_seen": 52035584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953299899699098, + "loss": 2.6203, + "theoretical_loss": 5.274023317753125, + "tokens_seen": 52101120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953139418254765, + "loss": 2.6784, + "theoretical_loss": 5.273025045496631, + "tokens_seen": 52166656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952978936810432, + "loss": 2.5281, + "theoretical_loss": 5.272028377212777, + "tokens_seen": 52232192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952818455366099, + "loss": 2.5807, + "theoretical_loss": 5.271033308316715, + "tokens_seen": 52297728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952657973921766, + "loss": 2.6348, + "theoretical_loss": 5.270039834242425, + "tokens_seen": 52363264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 24243, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.5289902687072754, + "objective/train/theoretical_loss": 5.269047950442619, + "objective/train/tokens_used": 72888800, + "theoretical_loss": 5.269047950442619, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952497492477433, + "loss": 2.485, + "theoretical_loss": 5.269047950442619, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.00079523370110331, + "loss": 2.6376, + "theoretical_loss": 5.2680576523886335, + "tokens_seen": 52494336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952176529588767, + "loss": 2.7194, + "theoretical_loss": 5.2670689355703395, + "tokens_seen": 52559872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952016048144434, + "loss": 2.6341, + "theoretical_loss": 5.266081795496035, + "tokens_seen": 52625408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951855566700101, + "loss": 2.5662, + "theoretical_loss": 5.265096227692354, + "tokens_seen": 52690944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951695085255768, + "loss": 2.6624, + "theoretical_loss": 5.264112227704162, + "tokens_seen": 52756480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951534603811434, + "loss": 2.7557, + "theoretical_loss": 5.263129791094466, + "tokens_seen": 52822016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951374122367102, + "loss": 2.6062, + "theoretical_loss": 5.26214891344431, + "tokens_seen": 52887552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951213640922768, + "loss": 2.5699, + "theoretical_loss": 5.26116959035269, + "tokens_seen": 52953088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951053159478436, + "loss": 2.506, + "theoretical_loss": 5.260191817436452, + "tokens_seen": 53018624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007950892678034102, + "loss": 2.6012, + "theoretical_loss": 5.259215590330195, + "tokens_seen": 53084160 + }, + { + "epoch": 0.02, + "learning_rate": 0.000795073219658977, + "loss": 2.5899, + "theoretical_loss": 5.258240904686186, + "tokens_seen": 53149696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007950571715145436, + "loss": 2.5064, + "theoretical_loss": 5.25726775617426, + "tokens_seen": 53215232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007950411233701104, + "loss": 2.4435, + "theoretical_loss": 5.256296140481728, + "tokens_seen": 53280768 + }, + { + "epoch": 0.02, + "learning_rate": 0.000795025075225677, + "loss": 2.5262, + "theoretical_loss": 5.255326053313292, + "tokens_seen": 53346304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007950090270812438, + "loss": 2.5646, + "theoretical_loss": 5.254357490390941, + "tokens_seen": 53411840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949929789368104, + "loss": 2.4133, + "theoretical_loss": 5.253390447453873, + "tokens_seen": 53477376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949769307923772, + "loss": 2.6187, + "theoretical_loss": 5.252424920258396, + "tokens_seen": 53542912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949608826479438, + "loss": 2.5912, + "theoretical_loss": 5.251460904577845, + "tokens_seen": 53608448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949448345035106, + "loss": 2.5508, + "theoretical_loss": 5.250498396202486, + "tokens_seen": 53673984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949287863590772, + "loss": 2.5112, + "theoretical_loss": 5.249537390939436, + "tokens_seen": 53739520 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794912738214644, + "loss": 2.4452, + "theoretical_loss": 5.248577884612564, + "tokens_seen": 53805056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948966900702106, + "loss": 2.4712, + "theoretical_loss": 5.247619873062416, + "tokens_seen": 53870592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948806419257774, + "loss": 2.5985, + "theoretical_loss": 5.246663352146122, + "tokens_seen": 53936128 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794864593781344, + "loss": 2.6442, + "theoretical_loss": 5.245708317737305, + "tokens_seen": 54001664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948485456369108, + "loss": 2.4876, + "theoretical_loss": 5.244754765726007, + "tokens_seen": 54067200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948324974924774, + "loss": 2.5376, + "theoretical_loss": 5.243802692018596, + "tokens_seen": 54132736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948164493480442, + "loss": 2.4553, + "theoretical_loss": 5.242852092537677, + "tokens_seen": 54198272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948004012036108, + "loss": 2.6929, + "theoretical_loss": 5.241902963222023, + "tokens_seen": 54263808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007947843530591776, + "loss": 2.5035, + "theoretical_loss": 5.240955300026476, + "tokens_seen": 54329344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007947683049147442, + "loss": 2.5704, + "theoretical_loss": 5.240009098921874, + "tokens_seen": 54394880 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794752256770311, + "loss": 2.4209, + "theoretical_loss": 5.239064355894965, + "tokens_seen": 54460416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007947362086258776, + "loss": 2.5877, + "theoretical_loss": 5.238121066948324, + "tokens_seen": 54525952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007947201604814444, + "loss": 2.4356, + "theoretical_loss": 5.237179228100272, + "tokens_seen": 54591488 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794704112337011, + "loss": 2.498, + "theoretical_loss": 5.2362388353848, + "tokens_seen": 54657024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946880641925778, + "loss": 2.426, + "theoretical_loss": 5.23529988485148, + "tokens_seen": 54722560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946720160481444, + "loss": 2.527, + "theoretical_loss": 5.2343623725653945, + "tokens_seen": 54788096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946559679037112, + "loss": 2.5915, + "theoretical_loss": 5.233426294607049, + "tokens_seen": 54853632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946399197592778, + "loss": 2.5706, + "theoretical_loss": 5.232491647072299, + "tokens_seen": 54919168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946238716148446, + "loss": 2.5656, + "theoretical_loss": 5.231558426072269, + "tokens_seen": 54984704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946078234704112, + "loss": 2.4501, + "theoretical_loss": 5.230626627733274, + "tokens_seen": 55050240 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794591775325978, + "loss": 2.5487, + "theoretical_loss": 5.2296962481967455, + "tokens_seen": 55115776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007945757271815446, + "loss": 2.3679, + "theoretical_loss": 5.228767283619151, + "tokens_seen": 55181312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007945596790371114, + "loss": 2.6326, + "theoretical_loss": 5.227839730171918, + "tokens_seen": 55246848 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794543630892678, + "loss": 2.4476, + "theoretical_loss": 5.2269135840413625, + "tokens_seen": 55312384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007945275827482448, + "loss": 2.309, + "theoretical_loss": 5.225988841428606, + "tokens_seen": 55377920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007945115346038114, + "loss": 2.2936, + "theoretical_loss": 5.225065498549508, + "tokens_seen": 55443456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944954864593782, + "loss": 2.4656, + "theoretical_loss": 5.224143551634588, + "tokens_seen": 55508992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944794383149448, + "loss": 2.428, + "theoretical_loss": 5.22322299692895, + "tokens_seen": 55574528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944633901705116, + "loss": 2.4442, + "theoretical_loss": 5.222303830692211, + "tokens_seen": 55640064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 25141, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.6088738441467285, + "objective/train/theoretical_loss": 5.22138604919843, + "objective/train/tokens_used": 76165600, + "theoretical_loss": 5.22138604919843, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944473420260782, + "loss": 2.5252, + "theoretical_loss": 5.22138604919843, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794431293881645, + "loss": 2.6352, + "theoretical_loss": 5.220469648736031, + "tokens_seen": 55771136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944152457372116, + "loss": 2.4382, + "theoretical_loss": 5.219554625607731, + "tokens_seen": 55836672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943991975927784, + "loss": 2.5073, + "theoretical_loss": 5.218640976130474, + "tokens_seen": 55902208 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794383149448345, + "loss": 2.502, + "theoretical_loss": 5.217728696635353, + "tokens_seen": 55967744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943671013039118, + "loss": 2.3945, + "theoretical_loss": 5.216817783467543, + "tokens_seen": 56033280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943510531594784, + "loss": 2.6394, + "theoretical_loss": 5.215908232986227, + "tokens_seen": 56098816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943350050150452, + "loss": 2.4114, + "theoretical_loss": 5.215000041564533, + "tokens_seen": 56164352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943189568706118, + "loss": 2.47, + "theoretical_loss": 5.214093205589455, + "tokens_seen": 56229888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943029087261786, + "loss": 2.4864, + "theoretical_loss": 5.213187721461791, + "tokens_seen": 56295424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007942868605817452, + "loss": 2.5074, + "theoretical_loss": 5.212283585596071, + "tokens_seen": 56360960 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794270812437312, + "loss": 2.3196, + "theoretical_loss": 5.211380794420492, + "tokens_seen": 56426496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007942547642928786, + "loss": 2.3877, + "theoretical_loss": 5.210479344376845, + "tokens_seen": 56492032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007942387161484454, + "loss": 2.5349, + "theoretical_loss": 5.209579231920451, + "tokens_seen": 56557568 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794222668004012, + "loss": 2.4561, + "theoretical_loss": 5.208680453520092, + "tokens_seen": 56623104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007942066198595788, + "loss": 2.4743, + "theoretical_loss": 5.2077830056579515, + "tokens_seen": 56688640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941905717151455, + "loss": 2.5124, + "theoretical_loss": 5.206886884829535, + "tokens_seen": 56754176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941745235707122, + "loss": 2.3724, + "theoretical_loss": 5.205992087543614, + "tokens_seen": 56819712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941584754262789, + "loss": 2.5328, + "theoretical_loss": 5.205098610322162, + "tokens_seen": 56885248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941424272818456, + "loss": 2.6109, + "theoretical_loss": 5.204206449700279, + "tokens_seen": 56950784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941263791374123, + "loss": 2.4425, + "theoretical_loss": 5.203315602226139, + "tokens_seen": 57016320 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794110330992979, + "loss": 2.353, + "theoretical_loss": 5.2024260644609175, + "tokens_seen": 57081856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940942828485457, + "loss": 2.4324, + "theoretical_loss": 5.201537832978731, + "tokens_seen": 57147392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940782347041124, + "loss": 2.3614, + "theoretical_loss": 5.200650904366574, + "tokens_seen": 57212928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940621865596791, + "loss": 2.4856, + "theoretical_loss": 5.199765275224252, + "tokens_seen": 57278464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940461384152458, + "loss": 2.4181, + "theoretical_loss": 5.198880942164323, + "tokens_seen": 57344000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940300902708125, + "loss": 2.351, + "theoretical_loss": 5.197997901812036, + "tokens_seen": 57409536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940140421263792, + "loss": 2.5241, + "theoretical_loss": 5.197116150805264, + "tokens_seen": 57475072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939979939819459, + "loss": 2.3707, + "theoretical_loss": 5.196235685794446, + "tokens_seen": 57540608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939819458375126, + "loss": 2.4242, + "theoretical_loss": 5.195356503442525, + "tokens_seen": 57606144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939658976930794, + "loss": 2.4264, + "theoretical_loss": 5.194478600424889, + "tokens_seen": 57671680 + }, + { + "epoch": 0.02, + "learning_rate": 0.000793949849548646, + "loss": 2.4905, + "theoretical_loss": 5.193601973429306, + "tokens_seen": 57737216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939338014042128, + "loss": 2.3264, + "theoretical_loss": 5.192726619155867, + "tokens_seen": 57802752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939177532597794, + "loss": 2.2947, + "theoretical_loss": 5.191852534316928, + "tokens_seen": 57868288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939017051153462, + "loss": 2.5367, + "theoretical_loss": 5.190979715637049, + "tokens_seen": 57933824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938856569709128, + "loss": 2.4345, + "theoretical_loss": 5.190108159852929, + "tokens_seen": 57999360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938696088264795, + "loss": 2.48, + "theoretical_loss": 5.189237863713357, + "tokens_seen": 58064896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938535606820462, + "loss": 2.5036, + "theoretical_loss": 5.188368823979152, + "tokens_seen": 58130432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938375125376129, + "loss": 2.2739, + "theoretical_loss": 5.187501037423096, + "tokens_seen": 58195968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938214643931796, + "loss": 2.3866, + "theoretical_loss": 5.186634500829886, + "tokens_seen": 58261504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938054162487463, + "loss": 2.4664, + "theoretical_loss": 5.185769210996076, + "tokens_seen": 58327040 + }, + { + "epoch": 0.02, + "learning_rate": 0.000793789368104313, + "loss": 2.2739, + "theoretical_loss": 5.184905164730012, + "tokens_seen": 58392576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937733199598797, + "loss": 2.4244, + "theoretical_loss": 5.184042358851785, + "tokens_seen": 58458112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937572718154463, + "loss": 2.3718, + "theoretical_loss": 5.1831807901931715, + "tokens_seen": 58523648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937412236710131, + "loss": 2.2701, + "theoretical_loss": 5.18232045559757, + "tokens_seen": 58589184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937251755265797, + "loss": 2.3716, + "theoretical_loss": 5.181461351919959, + "tokens_seen": 58654720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937091273821465, + "loss": 2.5091, + "theoretical_loss": 5.180603476026832, + "tokens_seen": 58720256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936930792377131, + "loss": 2.3163, + "theoretical_loss": 5.179746824796146, + "tokens_seen": 58785792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936770310932799, + "loss": 2.2918, + "theoretical_loss": 5.178891395117262, + "tokens_seen": 58851328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936609829488465, + "loss": 2.4097, + "theoretical_loss": 5.178037183890902, + "tokens_seen": 58916864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 26367, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.4226865768432617, + "objective/train/theoretical_loss": 5.177184188029082, + "objective/train/tokens_used": 79442400, + "theoretical_loss": 5.177184188029082, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936449348044133, + "loss": 2.3572, + "theoretical_loss": 5.177184188029082, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936288866599799, + "loss": 2.452, + "theoretical_loss": 5.1763324044550645, + "tokens_seen": 59047936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936128385155467, + "loss": 2.3822, + "theoretical_loss": 5.175481830103307, + "tokens_seen": 59113472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935967903711133, + "loss": 2.376, + "theoretical_loss": 5.174632461919406, + "tokens_seen": 59179008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935807422266801, + "loss": 2.3096, + "theoretical_loss": 5.173784296860043, + "tokens_seen": 59244544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935646940822467, + "loss": 2.4012, + "theoretical_loss": 5.172937331892934, + "tokens_seen": 59310080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935486459378135, + "loss": 2.2779, + "theoretical_loss": 5.17209156399678, + "tokens_seen": 59375616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935325977933801, + "loss": 2.4276, + "theoretical_loss": 5.171246990161208, + "tokens_seen": 59441152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935165496489469, + "loss": 2.4266, + "theoretical_loss": 5.17040360738673, + "tokens_seen": 59506688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935005015045135, + "loss": 2.2993, + "theoretical_loss": 5.169561412684677, + "tokens_seen": 59572224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934844533600803, + "loss": 2.4694, + "theoretical_loss": 5.168720403077165, + "tokens_seen": 59637760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934684052156469, + "loss": 2.2188, + "theoretical_loss": 5.167880575597031, + "tokens_seen": 59703296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934523570712137, + "loss": 2.2279, + "theoretical_loss": 5.167041927287787, + "tokens_seen": 59768832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934363089267803, + "loss": 2.4385, + "theoretical_loss": 5.166204455203575, + "tokens_seen": 59834368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934202607823471, + "loss": 2.2816, + "theoretical_loss": 5.16536815640911, + "tokens_seen": 59899904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934042126379137, + "loss": 2.4746, + "theoretical_loss": 5.1645330279796315, + "tokens_seen": 59965440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933881644934805, + "loss": 2.3473, + "theoretical_loss": 5.1636990670008585, + "tokens_seen": 60030976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933721163490471, + "loss": 2.4463, + "theoretical_loss": 5.162866270568938, + "tokens_seen": 60096512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933560682046139, + "loss": 2.3849, + "theoretical_loss": 5.162034635790397, + "tokens_seen": 60162048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933400200601805, + "loss": 2.384, + "theoretical_loss": 5.1612041597820895, + "tokens_seen": 60227584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933239719157473, + "loss": 2.4535, + "theoretical_loss": 5.160374839671159, + "tokens_seen": 60293120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933079237713139, + "loss": 2.3911, + "theoretical_loss": 5.1595466725949795, + "tokens_seen": 60358656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932918756268807, + "loss": 2.3502, + "theoretical_loss": 5.158719655701115, + "tokens_seen": 60424192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932758274824473, + "loss": 2.4282, + "theoretical_loss": 5.157893786147269, + "tokens_seen": 60489728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932597793380141, + "loss": 2.3445, + "theoretical_loss": 5.157069061101238, + "tokens_seen": 60555264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932437311935807, + "loss": 2.3545, + "theoretical_loss": 5.156245477740866, + "tokens_seen": 60620800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932276830491475, + "loss": 2.2794, + "theoretical_loss": 5.155423033253997, + "tokens_seen": 60686336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932116349047141, + "loss": 2.2316, + "theoretical_loss": 5.154601724838431, + "tokens_seen": 60751872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931955867602809, + "loss": 2.3536, + "theoretical_loss": 5.153781549701872, + "tokens_seen": 60817408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931795386158475, + "loss": 2.4524, + "theoretical_loss": 5.15296250506189, + "tokens_seen": 60882944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931634904714143, + "loss": 2.3219, + "theoretical_loss": 5.152144588145871, + "tokens_seen": 60948480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931474423269809, + "loss": 2.2977, + "theoretical_loss": 5.151327796190975, + "tokens_seen": 61014016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931313941825477, + "loss": 2.2728, + "theoretical_loss": 5.1505121264440845, + "tokens_seen": 61079552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931153460381143, + "loss": 2.4291, + "theoretical_loss": 5.149697576161769, + "tokens_seen": 61145088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930992978936811, + "loss": 2.258, + "theoretical_loss": 5.148884142610232, + "tokens_seen": 61210624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930832497492477, + "loss": 2.3553, + "theoretical_loss": 5.148071823065274, + "tokens_seen": 61276160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930672016048145, + "loss": 2.3682, + "theoretical_loss": 5.147260614812246, + "tokens_seen": 61341696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930511534603811, + "loss": 2.3699, + "theoretical_loss": 5.146450515146002, + "tokens_seen": 61407232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930351053159479, + "loss": 2.2222, + "theoretical_loss": 5.145641521370861, + "tokens_seen": 61472768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930190571715145, + "loss": 2.3013, + "theoretical_loss": 5.144833630800562, + "tokens_seen": 61538304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930030090270813, + "loss": 2.3248, + "theoretical_loss": 5.1440268407582215, + "tokens_seen": 61603840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929869608826479, + "loss": 2.3713, + "theoretical_loss": 5.143221148576288, + "tokens_seen": 61669376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929709127382147, + "loss": 2.1181, + "theoretical_loss": 5.142416551596505, + "tokens_seen": 61734912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929548645937813, + "loss": 2.1712, + "theoretical_loss": 5.141613047169862, + "tokens_seen": 61800448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929388164493481, + "loss": 2.3425, + "theoretical_loss": 5.14081063265656, + "tokens_seen": 61865984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929227683049147, + "loss": 2.2708, + "theoretical_loss": 5.140009305425963, + "tokens_seen": 61931520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929067201604815, + "loss": 2.205, + "theoretical_loss": 5.13920906285656, + "tokens_seen": 61997056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928906720160481, + "loss": 2.3632, + "theoretical_loss": 5.138409902335925, + "tokens_seen": 62062592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928746238716149, + "loss": 2.2693, + "theoretical_loss": 5.137611821260673, + "tokens_seen": 62128128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928585757271815, + "loss": 2.2251, + "theoretical_loss": 5.13681481703642, + "tokens_seen": 62193664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 27626, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.3975930213928223, + "objective/train/theoretical_loss": 5.136018887077743, + "objective/train/tokens_used": 82719200, + "theoretical_loss": 5.136018887077743, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928425275827483, + "loss": 2.3101, + "theoretical_loss": 5.136018887077743, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792826479438315, + "loss": 2.446, + "theoretical_loss": 5.13522402880814, + "tokens_seen": 62324736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928104312938817, + "loss": 2.4389, + "theoretical_loss": 5.13443023965999, + "tokens_seen": 62390272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927943831494484, + "loss": 2.3297, + "theoretical_loss": 5.13363751707451, + "tokens_seen": 62455808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927783350050151, + "loss": 2.2738, + "theoretical_loss": 5.132845858501721, + "tokens_seen": 62521344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927622868605819, + "loss": 2.2318, + "theoretical_loss": 5.132055261400403, + "tokens_seen": 62586880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927462387161485, + "loss": 2.193, + "theoretical_loss": 5.131265723238059, + "tokens_seen": 62652416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927301905717153, + "loss": 2.2944, + "theoretical_loss": 5.130477241490876, + "tokens_seen": 62717952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927141424272819, + "loss": 2.2733, + "theoretical_loss": 5.129689813643683, + "tokens_seen": 62783488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926980942828487, + "loss": 2.2248, + "theoretical_loss": 5.128903437189917, + "tokens_seen": 62849024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926820461384153, + "loss": 2.2887, + "theoretical_loss": 5.128118109631581, + "tokens_seen": 62914560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926659979939821, + "loss": 2.31, + "theoretical_loss": 5.127333828479208, + "tokens_seen": 62980096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926499498495487, + "loss": 2.2063, + "theoretical_loss": 5.126550591251821, + "tokens_seen": 63045632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926339017051155, + "loss": 2.3662, + "theoretical_loss": 5.125768395476896, + "tokens_seen": 63111168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926178535606821, + "loss": 2.2167, + "theoretical_loss": 5.124987238690329, + "tokens_seen": 63176704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926018054162489, + "loss": 2.3044, + "theoretical_loss": 5.124207118436388, + "tokens_seen": 63242240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925857572718155, + "loss": 2.3506, + "theoretical_loss": 5.123428032267688, + "tokens_seen": 63307776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925697091273823, + "loss": 2.2806, + "theoretical_loss": 5.122649977745146, + "tokens_seen": 63373312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925536609829489, + "loss": 2.3118, + "theoretical_loss": 5.121872952437947, + "tokens_seen": 63438848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925376128385157, + "loss": 2.3615, + "theoretical_loss": 5.121096953923507, + "tokens_seen": 63504384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925215646940823, + "loss": 2.3037, + "theoretical_loss": 5.1203219797874375, + "tokens_seen": 63569920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925055165496491, + "loss": 2.3262, + "theoretical_loss": 5.119548027623509, + "tokens_seen": 63635456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924894684052157, + "loss": 2.3319, + "theoretical_loss": 5.118775095033612, + "tokens_seen": 63700992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924734202607824, + "loss": 2.2909, + "theoretical_loss": 5.118003179627729, + "tokens_seen": 63766528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924573721163491, + "loss": 2.24, + "theoretical_loss": 5.11723227902389, + "tokens_seen": 63832064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924413239719158, + "loss": 2.3249, + "theoretical_loss": 5.1164623908481435, + "tokens_seen": 63897600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924252758274825, + "loss": 2.2101, + "theoretical_loss": 5.115693512734515, + "tokens_seen": 63963136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924092276830492, + "loss": 2.1202, + "theoretical_loss": 5.114925642324982, + "tokens_seen": 64028672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923931795386159, + "loss": 2.267, + "theoretical_loss": 5.114158777269429, + "tokens_seen": 64094208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923771313941826, + "loss": 2.2321, + "theoretical_loss": 5.113392915225617, + "tokens_seen": 64159744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923610832497493, + "loss": 2.35, + "theoretical_loss": 5.112628053859153, + "tokens_seen": 64225280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792345035105316, + "loss": 2.0774, + "theoretical_loss": 5.1118641908434475, + "tokens_seen": 64290816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923289869608826, + "loss": 2.2412, + "theoretical_loss": 5.111101323859687, + "tokens_seen": 64356352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923129388164494, + "loss": 2.3277, + "theoretical_loss": 5.1103394505968005, + "tokens_seen": 64421888 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792296890672016, + "loss": 2.315, + "theoretical_loss": 5.109578568751417, + "tokens_seen": 64487424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922808425275828, + "loss": 2.413, + "theoretical_loss": 5.108818676027845, + "tokens_seen": 64552960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922647943831494, + "loss": 2.1771, + "theoretical_loss": 5.108059770138027, + "tokens_seen": 64618496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922487462387162, + "loss": 2.3026, + "theoretical_loss": 5.107301848801519, + "tokens_seen": 64684032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922326980942828, + "loss": 2.253, + "theoretical_loss": 5.106544909745443, + "tokens_seen": 64749568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922166499498496, + "loss": 2.1783, + "theoretical_loss": 5.105788950704467, + "tokens_seen": 64815104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922006018054162, + "loss": 2.2324, + "theoretical_loss": 5.105033969420764, + "tokens_seen": 64880640 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792184553660983, + "loss": 2.2459, + "theoretical_loss": 5.104279963643982, + "tokens_seen": 64946176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007921685055165496, + "loss": 2.1426, + "theoretical_loss": 5.103526931131217, + "tokens_seen": 65011712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007921524573721164, + "loss": 2.2505, + "theoretical_loss": 5.102774869646972, + "tokens_seen": 65077248 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792136409227683, + "loss": 2.0972, + "theoretical_loss": 5.102023776963131, + "tokens_seen": 65142784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007921203610832498, + "loss": 2.1683, + "theoretical_loss": 5.101273650858926, + "tokens_seen": 65208320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007921043129388164, + "loss": 2.2065, + "theoretical_loss": 5.100524489120902, + "tokens_seen": 65273856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920882647943832, + "loss": 2.2527, + "theoretical_loss": 5.099776289542893, + "tokens_seen": 65339392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920722166499498, + "loss": 2.1725, + "theoretical_loss": 5.099029049925985, + "tokens_seen": 65404928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920561685055166, + "loss": 2.2228, + "theoretical_loss": 5.098282768078483, + "tokens_seen": 65470464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 28581, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.2630820274353027, + "objective/train/theoretical_loss": 5.097537441815887, + "objective/train/tokens_used": 85996000, + "theoretical_loss": 5.097537441815887, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920401203610832, + "loss": 2.244, + "theoretical_loss": 5.097537441815887, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.00079202407221665, + "loss": 2.2649, + "theoretical_loss": 5.096793068960857, + "tokens_seen": 65601536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920080240722166, + "loss": 2.2591, + "theoretical_loss": 5.096049647343182, + "tokens_seen": 65667072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919919759277834, + "loss": 2.256, + "theoretical_loss": 5.095307174799752, + "tokens_seen": 65732608 + }, + { + "epoch": 0.02, + "learning_rate": 0.00079197592778335, + "loss": 2.2502, + "theoretical_loss": 5.094565649174524, + "tokens_seen": 65798144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919598796389168, + "loss": 2.1983, + "theoretical_loss": 5.0938250683184965, + "tokens_seen": 65863680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919438314944834, + "loss": 2.261, + "theoretical_loss": 5.093085430089674, + "tokens_seen": 65929216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919277833500502, + "loss": 2.3129, + "theoretical_loss": 5.092346732353046, + "tokens_seen": 65994752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919117352056168, + "loss": 2.2523, + "theoretical_loss": 5.091608972980545, + "tokens_seen": 66060288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007918956870611836, + "loss": 2.2646, + "theoretical_loss": 5.090872149851029, + "tokens_seen": 66125824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007918796389167502, + "loss": 2.2916, + "theoretical_loss": 5.090136260850245, + "tokens_seen": 66191360 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791863590772317, + "loss": 2.1602, + "theoretical_loss": 5.089401303870801, + "tokens_seen": 66256896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007918475426278836, + "loss": 2.2429, + "theoretical_loss": 5.088667276812138, + "tokens_seen": 66322432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007918314944834504, + "loss": 2.1725, + "theoretical_loss": 5.087934177580502, + "tokens_seen": 66387968 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791815446339017, + "loss": 2.2774, + "theoretical_loss": 5.087202004088914, + "tokens_seen": 66453504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917993981945838, + "loss": 2.1667, + "theoretical_loss": 5.086470754257139, + "tokens_seen": 66519040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917833500501504, + "loss": 2.2032, + "theoretical_loss": 5.0857404260116645, + "tokens_seen": 66584576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917673019057172, + "loss": 2.148, + "theoretical_loss": 5.085011017285662, + "tokens_seen": 66650112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917512537612838, + "loss": 2.2979, + "theoretical_loss": 5.084282526018972, + "tokens_seen": 66715648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917352056168506, + "loss": 2.2236, + "theoretical_loss": 5.083554950158062, + "tokens_seen": 66781184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917191574724172, + "loss": 2.1649, + "theoretical_loss": 5.0828282876560085, + "tokens_seen": 66846720 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791703109327984, + "loss": 2.2443, + "theoretical_loss": 5.0821025364724655, + "tokens_seen": 66912256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916870611835506, + "loss": 2.3334, + "theoretical_loss": 5.081377694573638, + "tokens_seen": 66977792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916710130391174, + "loss": 2.2181, + "theoretical_loss": 5.080653759932255, + "tokens_seen": 67043328 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791654964894684, + "loss": 2.1852, + "theoretical_loss": 5.079930730527538, + "tokens_seen": 67108864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916389167502508, + "loss": 2.1556, + "theoretical_loss": 5.079208604345181, + "tokens_seen": 67174400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916228686058174, + "loss": 2.2345, + "theoretical_loss": 5.078487379377318, + "tokens_seen": 67239936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916068204613842, + "loss": 2.1317, + "theoretical_loss": 5.0777670536225, + "tokens_seen": 67305472 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791590772316951, + "loss": 2.1145, + "theoretical_loss": 5.07704762508566, + "tokens_seen": 67371008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007915747241725176, + "loss": 2.1057, + "theoretical_loss": 5.076329091778101, + "tokens_seen": 67436544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007915586760280844, + "loss": 2.1766, + "theoretical_loss": 5.075611451717457, + "tokens_seen": 67502080 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791542627883651, + "loss": 2.1238, + "theoretical_loss": 5.0748947029276685, + "tokens_seen": 67567616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007915265797392178, + "loss": 2.2196, + "theoretical_loss": 5.074178843438963, + "tokens_seen": 67633152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007915105315947844, + "loss": 2.2516, + "theoretical_loss": 5.073463871287823, + "tokens_seen": 67698688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914944834503512, + "loss": 2.2305, + "theoretical_loss": 5.07274978451696, + "tokens_seen": 67764224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914784353059178, + "loss": 2.2993, + "theoretical_loss": 5.072036581175293, + "tokens_seen": 67829760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914623871614846, + "loss": 2.2447, + "theoretical_loss": 5.071324259317921, + "tokens_seen": 67895296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914463390170512, + "loss": 2.1236, + "theoretical_loss": 5.070612817006094, + "tokens_seen": 67960832 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791430290872618, + "loss": 2.3058, + "theoretical_loss": 5.06990225230719, + "tokens_seen": 68026368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914142427281846, + "loss": 2.1259, + "theoretical_loss": 5.069192563294694, + "tokens_seen": 68091904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913981945837514, + "loss": 2.3627, + "theoretical_loss": 5.068483748048166, + "tokens_seen": 68157440 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791382146439318, + "loss": 2.3496, + "theoretical_loss": 5.06777580465322, + "tokens_seen": 68222976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913660982948848, + "loss": 2.315, + "theoretical_loss": 5.067068731201498, + "tokens_seen": 68288512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913500501504514, + "loss": 2.1493, + "theoretical_loss": 5.066362525790646, + "tokens_seen": 68354048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913340020060182, + "loss": 2.113, + "theoretical_loss": 5.065657186524287, + "tokens_seen": 68419584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913179538615848, + "loss": 2.2397, + "theoretical_loss": 5.064952711512001, + "tokens_seen": 68485120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913019057171516, + "loss": 2.2762, + "theoretical_loss": 5.064249098869295, + "tokens_seen": 68550656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007912858575727182, + "loss": 2.2793, + "theoretical_loss": 5.063546346717585, + "tokens_seen": 68616192 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791269809428285, + "loss": 2.2232, + "theoretical_loss": 5.062844453184166, + "tokens_seen": 68681728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007912537612838516, + "loss": 2.2233, + "theoretical_loss": 5.062143416402192, + "tokens_seen": 68747264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 29892, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.302198648452759, + "objective/train/theoretical_loss": 5.061443234510648, + "objective/train/tokens_used": 89272800, + "theoretical_loss": 5.061443234510648, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007912377131394184, + "loss": 2.2377, + "theoretical_loss": 5.061443234510648, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791221664994985, + "loss": 2.2414, + "theoretical_loss": 5.060743905654332, + "tokens_seen": 68878336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007912056168505518, + "loss": 2.2892, + "theoretical_loss": 5.0600454279838285, + "tokens_seen": 68943872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911895687061184, + "loss": 2.1666, + "theoretical_loss": 5.059347799655483, + "tokens_seen": 69009408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911735205616852, + "loss": 2.1416, + "theoretical_loss": 5.05865101883138, + "tokens_seen": 69074944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911574724172518, + "loss": 2.1452, + "theoretical_loss": 5.057955083679321, + "tokens_seen": 69140480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911414242728186, + "loss": 2.2087, + "theoretical_loss": 5.0572599923728, + "tokens_seen": 69206016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911253761283852, + "loss": 2.2024, + "theoretical_loss": 5.056565743090982, + "tokens_seen": 69271552 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791109327983952, + "loss": 2.0612, + "theoretical_loss": 5.055872334018677, + "tokens_seen": 69337088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910932798395186, + "loss": 2.2113, + "theoretical_loss": 5.0551797633463185, + "tokens_seen": 69402624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910772316950853, + "loss": 2.1887, + "theoretical_loss": 5.054488029269946, + "tokens_seen": 69468160 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791061183550652, + "loss": 2.2918, + "theoretical_loss": 5.053797129991171, + "tokens_seen": 69533696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910451354062187, + "loss": 2.2494, + "theoretical_loss": 5.053107063717167, + "tokens_seen": 69599232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910290872617854, + "loss": 2.2314, + "theoretical_loss": 5.052417828660639, + "tokens_seen": 69664768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910130391173521, + "loss": 2.2445, + "theoretical_loss": 5.051729423039802, + "tokens_seen": 69730304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909969909729188, + "loss": 2.2076, + "theoretical_loss": 5.0510418450783625, + "tokens_seen": 69795840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909809428284855, + "loss": 2.2421, + "theoretical_loss": 5.050355093005495, + "tokens_seen": 69861376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909648946840522, + "loss": 2.0975, + "theoretical_loss": 5.049669165055816, + "tokens_seen": 69926912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909488465396189, + "loss": 2.1393, + "theoretical_loss": 5.048984059469369, + "tokens_seen": 69992448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909327983951855, + "loss": 2.1042, + "theoretical_loss": 5.048299774491598, + "tokens_seen": 70057984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909167502507523, + "loss": 2.1535, + "theoretical_loss": 5.047616308373328, + "tokens_seen": 70123520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909007021063189, + "loss": 2.2246, + "theoretical_loss": 5.046933659370742, + "tokens_seen": 70189056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908846539618857, + "loss": 2.1902, + "theoretical_loss": 5.046251825745361, + "tokens_seen": 70254592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908686058174523, + "loss": 2.247, + "theoretical_loss": 5.0455708057640205, + "tokens_seen": 70320128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908525576730191, + "loss": 2.0948, + "theoretical_loss": 5.044890597698853, + "tokens_seen": 70385664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908365095285857, + "loss": 2.2894, + "theoretical_loss": 5.044211199827263, + "tokens_seen": 70451200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908204613841525, + "loss": 2.1606, + "theoretical_loss": 5.04353261043191, + "tokens_seen": 70516736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908044132397191, + "loss": 2.2046, + "theoretical_loss": 5.0428548278006815, + "tokens_seen": 70582272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907883650952859, + "loss": 2.0641, + "theoretical_loss": 5.042177850226682, + "tokens_seen": 70647808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907723169508525, + "loss": 2.1669, + "theoretical_loss": 5.0415016760082, + "tokens_seen": 70713344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907562688064193, + "loss": 2.1651, + "theoretical_loss": 5.040826303448698, + "tokens_seen": 70778880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907402206619859, + "loss": 2.1432, + "theoretical_loss": 5.040151730856788, + "tokens_seen": 70844416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907241725175527, + "loss": 2.1211, + "theoretical_loss": 5.039477956546207, + "tokens_seen": 70909952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907081243731193, + "loss": 2.2094, + "theoretical_loss": 5.038804978835808, + "tokens_seen": 70975488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906920762286861, + "loss": 2.2419, + "theoretical_loss": 5.038132796049523, + "tokens_seen": 71041024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906760280842527, + "loss": 2.1796, + "theoretical_loss": 5.037461406516362, + "tokens_seen": 71106560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906599799398195, + "loss": 2.1457, + "theoretical_loss": 5.036790808570377, + "tokens_seen": 71172096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906439317953861, + "loss": 2.1222, + "theoretical_loss": 5.036121000550651, + "tokens_seen": 71237632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906278836509529, + "loss": 2.2514, + "theoretical_loss": 5.035451980801276, + "tokens_seen": 71303168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906118355065195, + "loss": 2.1864, + "theoretical_loss": 5.034783747671332, + "tokens_seen": 71368704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905957873620863, + "loss": 2.148, + "theoretical_loss": 5.034116299514871, + "tokens_seen": 71434240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905797392176529, + "loss": 2.1176, + "theoretical_loss": 5.033449634690893, + "tokens_seen": 71499776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905636910732197, + "loss": 2.0967, + "theoretical_loss": 5.032783751563327, + "tokens_seen": 71565312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905476429287863, + "loss": 2.1888, + "theoretical_loss": 5.032118648501017, + "tokens_seen": 71630848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905315947843531, + "loss": 2.3203, + "theoretical_loss": 5.031454323877698, + "tokens_seen": 71696384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905155466399197, + "loss": 2.1714, + "theoretical_loss": 5.030790776071976, + "tokens_seen": 71761920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904994984954865, + "loss": 2.0794, + "theoretical_loss": 5.030128003467312, + "tokens_seen": 71827456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904834503510531, + "loss": 2.1015, + "theoretical_loss": 5.029466004452004, + "tokens_seen": 71892992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904674022066199, + "loss": 2.0396, + "theoretical_loss": 5.028804777419165, + "tokens_seen": 71958528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904513540621865, + "loss": 2.1914, + "theoretical_loss": 5.028144320766701, + "tokens_seen": 72024064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 30813, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.0049593448638916, + "objective/train/theoretical_loss": 5.0274846328973055, + "objective/train/tokens_used": 92549600, + "theoretical_loss": 5.0274846328973055, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904353059177533, + "loss": 2.1007, + "theoretical_loss": 5.0274846328973055, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904192577733199, + "loss": 2.2275, + "theoretical_loss": 5.0268257122184234, + "tokens_seen": 72155136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904032096288867, + "loss": 2.1329, + "theoretical_loss": 5.026167557142247, + "tokens_seen": 72220672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903871614844534, + "loss": 2.098, + "theoretical_loss": 5.02551016608569, + "tokens_seen": 72286208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903711133400201, + "loss": 2.2127, + "theoretical_loss": 5.024853537470369, + "tokens_seen": 72351744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903550651955868, + "loss": 2.0479, + "theoretical_loss": 5.02419766972259, + "tokens_seen": 72417280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903390170511535, + "loss": 2.2583, + "theoretical_loss": 5.023542561273327, + "tokens_seen": 72482816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903229689067202, + "loss": 2.2263, + "theoretical_loss": 5.022888210558206, + "tokens_seen": 72548352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903069207622869, + "loss": 2.2046, + "theoretical_loss": 5.02223461601748, + "tokens_seen": 72613888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902908726178537, + "loss": 2.3318, + "theoretical_loss": 5.021581776096022, + "tokens_seen": 72679424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902748244734203, + "loss": 2.131, + "theoretical_loss": 5.0209296892433, + "tokens_seen": 72744960 + }, + { + "epoch": 0.02, + "learning_rate": 0.000790258776328987, + "loss": 2.2132, + "theoretical_loss": 5.020278353913362, + "tokens_seen": 72810496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902427281845537, + "loss": 2.0379, + "theoretical_loss": 5.019627768564816, + "tokens_seen": 72876032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902266800401205, + "loss": 2.1585, + "theoretical_loss": 5.018977931660814, + "tokens_seen": 72941568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902106318956871, + "loss": 2.1721, + "theoretical_loss": 5.018328841669037, + "tokens_seen": 73007104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901945837512539, + "loss": 2.3544, + "theoretical_loss": 5.017680497061671, + "tokens_seen": 73072640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901785356068205, + "loss": 2.175, + "theoretical_loss": 5.017032896315397, + "tokens_seen": 73138176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901624874623873, + "loss": 2.0713, + "theoretical_loss": 5.016386037911371, + "tokens_seen": 73203712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901464393179539, + "loss": 2.2291, + "theoretical_loss": 5.015739920335203, + "tokens_seen": 73269248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901303911735207, + "loss": 2.185, + "theoretical_loss": 5.015094542076945, + "tokens_seen": 73334784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901143430290873, + "loss": 2.09, + "theoretical_loss": 5.014449901631075, + "tokens_seen": 73400320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900982948846541, + "loss": 2.1841, + "theoretical_loss": 5.013805997496473, + "tokens_seen": 73465856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900822467402207, + "loss": 2.1307, + "theoretical_loss": 5.013162828176412, + "tokens_seen": 73531392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900661985957875, + "loss": 2.2505, + "theoretical_loss": 5.012520392178539, + "tokens_seen": 73596928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900501504513541, + "loss": 2.1488, + "theoretical_loss": 5.011878688014852, + "tokens_seen": 73662464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900341023069209, + "loss": 2.1883, + "theoretical_loss": 5.011237714201694, + "tokens_seen": 73728000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900180541624875, + "loss": 2.0638, + "theoretical_loss": 5.010597469259729, + "tokens_seen": 73793536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900020060180543, + "loss": 2.1461, + "theoretical_loss": 5.009957951713927, + "tokens_seen": 73859072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899859578736209, + "loss": 2.1418, + "theoretical_loss": 5.009319160093551, + "tokens_seen": 73924608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899699097291877, + "loss": 2.1104, + "theoretical_loss": 5.008681092932137, + "tokens_seen": 73990144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899538615847543, + "loss": 2.2886, + "theoretical_loss": 5.008043748767477, + "tokens_seen": 74055680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899378134403211, + "loss": 2.1436, + "theoretical_loss": 5.007407126141608, + "tokens_seen": 74121216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899217652958877, + "loss": 2.171, + "theoretical_loss": 5.00677122360079, + "tokens_seen": 74186752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899057171514545, + "loss": 2.189, + "theoretical_loss": 5.006136039695496, + "tokens_seen": 74252288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898896690070211, + "loss": 2.1575, + "theoretical_loss": 5.005501572980391, + "tokens_seen": 74317824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898736208625879, + "loss": 2.0729, + "theoretical_loss": 5.004867822014317, + "tokens_seen": 74383360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898575727181545, + "loss": 2.144, + "theoretical_loss": 5.004234785360282, + "tokens_seen": 74448896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898415245737213, + "loss": 2.1052, + "theoretical_loss": 5.0036024615854355, + "tokens_seen": 74514432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898254764292879, + "loss": 2.113, + "theoretical_loss": 5.002970849261065, + "tokens_seen": 74579968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898094282848547, + "loss": 2.1806, + "theoretical_loss": 5.002339946962568, + "tokens_seen": 74645504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897933801404213, + "loss": 2.1274, + "theoretical_loss": 5.001709753269443, + "tokens_seen": 74711040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897773319959881, + "loss": 2.1426, + "theoretical_loss": 5.001080266765275, + "tokens_seen": 74776576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897612838515547, + "loss": 2.2028, + "theoretical_loss": 5.0004514860377185, + "tokens_seen": 74842112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897452357071215, + "loss": 2.1624, + "theoretical_loss": 4.999823409678481, + "tokens_seen": 74907648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897291875626881, + "loss": 2.1102, + "theoretical_loss": 4.999196036283308, + "tokens_seen": 74973184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897131394182549, + "loss": 2.196, + "theoretical_loss": 4.9985693644519715, + "tokens_seen": 75038720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896970912738215, + "loss": 2.1068, + "theoretical_loss": 4.997943392788251, + "tokens_seen": 75104256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896810431293882, + "loss": 2.0444, + "theoretical_loss": 4.997318119899922, + "tokens_seen": 75169792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896649949849549, + "loss": 1.96, + "theoretical_loss": 4.996693544398734, + "tokens_seen": 75235328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896489468405216, + "loss": 2.133, + "theoretical_loss": 4.996069664900406, + "tokens_seen": 75300864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 32032, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.0214736461639404, + "objective/train/theoretical_loss": 4.9954464800246035, + "objective/train/tokens_used": 95826400, + "theoretical_loss": 4.9954464800246035, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896328986960883, + "loss": 2.0347, + "theoretical_loss": 4.9954464800246035, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789616850551655, + "loss": 2.1474, + "theoretical_loss": 4.994823988394928, + "tokens_seen": 75431936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896008024072217, + "loss": 2.1111, + "theoretical_loss": 4.994202188638901, + "tokens_seen": 75497472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895847542627884, + "loss": 2.1961, + "theoretical_loss": 4.9935810793879485, + "tokens_seen": 75563008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895687061183551, + "loss": 2.1599, + "theoretical_loss": 4.992960659277389, + "tokens_seen": 75628544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895526579739218, + "loss": 2.1578, + "theoretical_loss": 4.992340926946417, + "tokens_seen": 75694080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895366098294884, + "loss": 2.041, + "theoretical_loss": 4.991721881038091, + "tokens_seen": 75759616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895205616850552, + "loss": 2.1556, + "theoretical_loss": 4.991103520199314, + "tokens_seen": 75825152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895045135406218, + "loss": 2.0935, + "theoretical_loss": 4.990485843080824, + "tokens_seen": 75890688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007894884653961886, + "loss": 2.153, + "theoretical_loss": 4.989868848337181, + "tokens_seen": 75956224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007894724172517552, + "loss": 2.1098, + "theoretical_loss": 4.989252534626749, + "tokens_seen": 76021760 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789456369107322, + "loss": 2.2057, + "theoretical_loss": 4.988636900611683, + "tokens_seen": 76087296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007894403209628886, + "loss": 2.1796, + "theoretical_loss": 4.988021944957915, + "tokens_seen": 76152832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007894242728184554, + "loss": 2.1348, + "theoretical_loss": 4.987407666335143, + "tokens_seen": 76218368 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789408224674022, + "loss": 2.003, + "theoretical_loss": 4.986794063416813, + "tokens_seen": 76283904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893921765295888, + "loss": 2.2363, + "theoretical_loss": 4.986181134880107, + "tokens_seen": 76349440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893761283851554, + "loss": 2.1553, + "theoretical_loss": 4.98556887940593, + "tokens_seen": 76414976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893600802407222, + "loss": 2.0457, + "theoretical_loss": 4.984957295678894, + "tokens_seen": 76480512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893440320962888, + "loss": 2.0834, + "theoretical_loss": 4.98434638238731, + "tokens_seen": 76546048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893279839518556, + "loss": 2.1583, + "theoretical_loss": 4.983736138223165, + "tokens_seen": 76611584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893119358074222, + "loss": 2.1141, + "theoretical_loss": 4.983126561882118, + "tokens_seen": 76677120 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789295887662989, + "loss": 2.1146, + "theoretical_loss": 4.982517652063482, + "tokens_seen": 76742656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007892798395185556, + "loss": 2.0532, + "theoretical_loss": 4.981909407470209, + "tokens_seen": 76808192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007892637913741224, + "loss": 2.1857, + "theoretical_loss": 4.981301826808882, + "tokens_seen": 76873728 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789247743229689, + "loss": 2.1357, + "theoretical_loss": 4.9806949087896975, + "tokens_seen": 76939264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007892316950852558, + "loss": 1.9885, + "theoretical_loss": 4.980088652126453, + "tokens_seen": 77004800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007892156469408224, + "loss": 2.1321, + "theoretical_loss": 4.9794830555365355, + "tokens_seen": 77070336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891995987963892, + "loss": 2.0872, + "theoretical_loss": 4.978878117740907, + "tokens_seen": 77135872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891835506519559, + "loss": 2.1593, + "theoretical_loss": 4.978273837464091, + "tokens_seen": 77201408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891675025075226, + "loss": 2.0579, + "theoretical_loss": 4.977670213434161, + "tokens_seen": 77266944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891514543630893, + "loss": 2.2943, + "theoretical_loss": 4.977067244382729, + "tokens_seen": 77332480 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789135406218656, + "loss": 1.9335, + "theoretical_loss": 4.976464929044928, + "tokens_seen": 77398016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891193580742227, + "loss": 2.0717, + "theoretical_loss": 4.975863266159403, + "tokens_seen": 77463552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891033099297894, + "loss": 2.0471, + "theoretical_loss": 4.975262254468299, + "tokens_seen": 77529088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890872617853561, + "loss": 2.07, + "theoretical_loss": 4.9746618927172435, + "tokens_seen": 77594624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890712136409228, + "loss": 2.1022, + "theoretical_loss": 4.9740621796553395, + "tokens_seen": 77660160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890551654964895, + "loss": 2.191, + "theoretical_loss": 4.973463114035149, + "tokens_seen": 77725696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890391173520562, + "loss": 2.0829, + "theoretical_loss": 4.972864694612684, + "tokens_seen": 77791232 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789023069207623, + "loss": 2.0328, + "theoretical_loss": 4.972266920147391, + "tokens_seen": 77856768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890070210631896, + "loss": 2.1021, + "theoretical_loss": 4.971669789402139, + "tokens_seen": 77922304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889909729187563, + "loss": 2.045, + "theoretical_loss": 4.971073301143207, + "tokens_seen": 77987840 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788974924774323, + "loss": 2.1521, + "theoretical_loss": 4.970477454140278, + "tokens_seen": 78053376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889588766298897, + "loss": 2.1709, + "theoretical_loss": 4.9698822471664155, + "tokens_seen": 78118912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889428284854564, + "loss": 2.0816, + "theoretical_loss": 4.96928767899806, + "tokens_seen": 78184448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889267803410232, + "loss": 2.1294, + "theoretical_loss": 4.9686937484150135, + "tokens_seen": 78249984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889107321965898, + "loss": 2.0653, + "theoretical_loss": 4.96810045420043, + "tokens_seen": 78315520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007888946840521566, + "loss": 2.0877, + "theoretical_loss": 4.967507795140797, + "tokens_seen": 78381056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007888786359077232, + "loss": 2.17, + "theoretical_loss": 4.966915770025935, + "tokens_seen": 78446592 + }, + { + "epoch": 0.02, + "learning_rate": 0.00078886258776329, + "loss": 2.1473, + "theoretical_loss": 4.966324377648973, + "tokens_seen": 78512128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007888465396188566, + "loss": 2.1472, + "theoretical_loss": 4.965733616806345, + "tokens_seen": 78577664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 32979, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.1013200283050537, + "objective/train/theoretical_loss": 4.965143486297777, + "objective/train/tokens_used": 99103200, + "theoretical_loss": 4.965143486297777, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007888304914744234, + "loss": 2.0807, + "theoretical_loss": 4.965143486297777, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.00078881444332999, + "loss": 2.1694, + "theoretical_loss": 4.964553984926271, + "tokens_seen": 78708736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887983951855568, + "loss": 2.0942, + "theoretical_loss": 4.963965111498098, + "tokens_seen": 78774272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887823470411234, + "loss": 2.1027, + "theoretical_loss": 4.963376864822784, + "tokens_seen": 78839808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887662988966902, + "loss": 2.1018, + "theoretical_loss": 4.962789243713102, + "tokens_seen": 78905344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887502507522568, + "loss": 2.1184, + "theoretical_loss": 4.962202246985054, + "tokens_seen": 78970880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887342026078236, + "loss": 2.0848, + "theoretical_loss": 4.9616158734578635, + "tokens_seen": 79036416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887181544633902, + "loss": 2.0948, + "theoretical_loss": 4.961030121953965, + "tokens_seen": 79101952 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788702106318957, + "loss": 2.1181, + "theoretical_loss": 4.960444991298992, + "tokens_seen": 79167488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886860581745236, + "loss": 2.1234, + "theoretical_loss": 4.959860480321762, + "tokens_seen": 79233024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886700100300904, + "loss": 2.112, + "theoretical_loss": 4.959276587854272, + "tokens_seen": 79298560 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788653961885657, + "loss": 2.1383, + "theoretical_loss": 4.958693312731681, + "tokens_seen": 79364096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886379137412238, + "loss": 2.1764, + "theoretical_loss": 4.9581106537923, + "tokens_seen": 79429632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886218655967904, + "loss": 2.0779, + "theoretical_loss": 4.957528609877587, + "tokens_seen": 79495168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886058174523572, + "loss": 2.0655, + "theoretical_loss": 4.9569471798321265, + "tokens_seen": 79560704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885897693079238, + "loss": 2.013, + "theoretical_loss": 4.9563663625036245, + "tokens_seen": 79626240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885737211634906, + "loss": 2.0405, + "theoretical_loss": 4.955786156742898, + "tokens_seen": 79691776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885576730190572, + "loss": 2.0635, + "theoretical_loss": 4.955206561403859, + "tokens_seen": 79757312 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788541624874624, + "loss": 2.1037, + "theoretical_loss": 4.954627575343509, + "tokens_seen": 79822848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885255767301906, + "loss": 1.9574, + "theoretical_loss": 4.954049197421925, + "tokens_seen": 79888384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885095285857574, + "loss": 2.1531, + "theoretical_loss": 4.953471426502249, + "tokens_seen": 79953920 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788493480441324, + "loss": 2.157, + "theoretical_loss": 4.952894261450679, + "tokens_seen": 80019456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884774322968908, + "loss": 2.044, + "theoretical_loss": 4.952317701136457, + "tokens_seen": 80084992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884613841524574, + "loss": 2.1789, + "theoretical_loss": 4.9517417444318585, + "tokens_seen": 80150528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884453360080242, + "loss": 2.1454, + "theoretical_loss": 4.951166390212181, + "tokens_seen": 80216064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884292878635908, + "loss": 2.0481, + "theoretical_loss": 4.950591637355737, + "tokens_seen": 80281600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884132397191576, + "loss": 2.0555, + "theoretical_loss": 4.950017484743835, + "tokens_seen": 80347136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883971915747242, + "loss": 2.0603, + "theoretical_loss": 4.949443931260783, + "tokens_seen": 80412672 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788381143430291, + "loss": 2.0551, + "theoretical_loss": 4.948870975793863, + "tokens_seen": 80478208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883650952858576, + "loss": 2.1272, + "theoretical_loss": 4.9482986172333305, + "tokens_seen": 80543744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883490471414244, + "loss": 2.035, + "theoretical_loss": 4.9477268544724, + "tokens_seen": 80609280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788332998996991, + "loss": 2.0542, + "theoretical_loss": 4.947155686407239, + "tokens_seen": 80674816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883169508525578, + "loss": 2.1393, + "theoretical_loss": 4.946585111936951, + "tokens_seen": 80740352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883009027081244, + "loss": 2.0737, + "theoretical_loss": 4.94601512996357, + "tokens_seen": 80805888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882848545636911, + "loss": 1.9408, + "theoretical_loss": 4.945445739392049, + "tokens_seen": 80871424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882688064192578, + "loss": 1.8858, + "theoretical_loss": 4.944876939130252, + "tokens_seen": 80936960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882527582748245, + "loss": 2.0729, + "theoretical_loss": 4.94430872808894, + "tokens_seen": 81002496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882367101303912, + "loss": 2.04, + "theoretical_loss": 4.943741105181765, + "tokens_seen": 81068032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882206619859579, + "loss": 2.0131, + "theoretical_loss": 4.943174069325255, + "tokens_seen": 81133568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882046138415246, + "loss": 1.9603, + "theoretical_loss": 4.94260761943881, + "tokens_seen": 81199104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881885656970913, + "loss": 1.985, + "theoretical_loss": 4.942041754444688, + "tokens_seen": 81264640 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788172517552658, + "loss": 2.039, + "theoretical_loss": 4.941476473267996, + "tokens_seen": 81330176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881564694082247, + "loss": 2.1917, + "theoretical_loss": 4.940911774836682, + "tokens_seen": 81395712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881404212637913, + "loss": 2.155, + "theoretical_loss": 4.940347658081521, + "tokens_seen": 81461248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881243731193581, + "loss": 2.0186, + "theoretical_loss": 4.939784121936112, + "tokens_seen": 81526784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881083249749247, + "loss": 2.0964, + "theoretical_loss": 4.9392211653368605, + "tokens_seen": 81592320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880922768304915, + "loss": 1.9986, + "theoretical_loss": 4.938658787222975, + "tokens_seen": 81657856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880762286860581, + "loss": 2.078, + "theoretical_loss": 4.938096986536452, + "tokens_seen": 81723392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880601805416249, + "loss": 2.0657, + "theoretical_loss": 4.937535762222075, + "tokens_seen": 81788928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880441323971915, + "loss": 2.111, + "theoretical_loss": 4.936975113227394, + "tokens_seen": 81854464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 34316, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.1313977241516113, + "objective/train/theoretical_loss": 4.936415038502723, + "objective/train/tokens_used": 102380000, + "theoretical_loss": 4.936415038502723, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880280842527583, + "loss": 2.1647, + "theoretical_loss": 4.936415038502723, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880120361083249, + "loss": 2.0975, + "theoretical_loss": 4.93585553700113, + "tokens_seen": 81985536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879959879638917, + "loss": 2.0226, + "theoretical_loss": 4.935296607678426, + "tokens_seen": 82051072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879799398194584, + "loss": 2.0912, + "theoretical_loss": 4.934738249493156, + "tokens_seen": 82116608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879638916750251, + "loss": 2.0794, + "theoretical_loss": 4.934180461406591, + "tokens_seen": 82182144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879478435305918, + "loss": 2.0681, + "theoretical_loss": 4.933623242382714, + "tokens_seen": 82247680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879317953861585, + "loss": 2.0694, + "theoretical_loss": 4.9330665913882195, + "tokens_seen": 82313216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879157472417252, + "loss": 2.0263, + "theoretical_loss": 4.932510507392495, + "tokens_seen": 82378752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007878996990972919, + "loss": 2.0941, + "theoretical_loss": 4.931954989367618, + "tokens_seen": 82444288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878836509528586, + "loss": 2.1159, + "theoretical_loss": 4.931400036288343, + "tokens_seen": 82509824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878676028084253, + "loss": 1.965, + "theoretical_loss": 4.930845647132097, + "tokens_seen": 82575360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000787851554663992, + "loss": 2.0173, + "theoretical_loss": 4.9302918208789634, + "tokens_seen": 82640896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878355065195587, + "loss": 2.0566, + "theoretical_loss": 4.929738556511681, + "tokens_seen": 82706432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878194583751254, + "loss": 2.0425, + "theoretical_loss": 4.929185853015629, + "tokens_seen": 82771968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878034102306921, + "loss": 2.0891, + "theoretical_loss": 4.928633709378822, + "tokens_seen": 82837504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877873620862588, + "loss": 2.1192, + "theoretical_loss": 4.9280821245918975, + "tokens_seen": 82903040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877713139418255, + "loss": 1.8945, + "theoretical_loss": 4.92753109764811, + "tokens_seen": 82968576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877552657973922, + "loss": 2.1443, + "theoretical_loss": 4.92698062754332, + "tokens_seen": 83034112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877392176529589, + "loss": 2.263, + "theoretical_loss": 4.9264307132759875, + "tokens_seen": 83099648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877231695085256, + "loss": 2.0305, + "theoretical_loss": 4.92588135384716, + "tokens_seen": 83165184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877071213640923, + "loss": 2.0397, + "theoretical_loss": 4.925332548260469, + "tokens_seen": 83230720 + }, + { + "epoch": 0.03, + "learning_rate": 0.000787691073219659, + "loss": 1.9951, + "theoretical_loss": 4.924784295522113, + "tokens_seen": 83296256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876750250752257, + "loss": 2.014, + "theoretical_loss": 4.92423659464086, + "tokens_seen": 83361792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876589769307924, + "loss": 2.0954, + "theoretical_loss": 4.923689444628027, + "tokens_seen": 83427328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876429287863591, + "loss": 1.9732, + "theoretical_loss": 4.923142844497482, + "tokens_seen": 83492864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876268806419258, + "loss": 2.0625, + "theoretical_loss": 4.922596793265625, + "tokens_seen": 83558400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876108324974925, + "loss": 2.0099, + "theoretical_loss": 4.922051289951391, + "tokens_seen": 83623936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875947843530592, + "loss": 1.9741, + "theoretical_loss": 4.921506333576232, + "tokens_seen": 83689472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875787362086259, + "loss": 2.0757, + "theoretical_loss": 4.920961923164114, + "tokens_seen": 83755008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875626880641927, + "loss": 2.0747, + "theoretical_loss": 4.920418057741504, + "tokens_seen": 83820544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875466399197593, + "loss": 2.1795, + "theoretical_loss": 4.919874736337368, + "tokens_seen": 83886080 + }, + { + "epoch": 0.03, + "learning_rate": 0.000787530591775326, + "loss": 2.0879, + "theoretical_loss": 4.9193319579831565, + "tokens_seen": 83951616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875145436308927, + "loss": 2.0928, + "theoretical_loss": 4.918789721712799, + "tokens_seen": 84017152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874984954864595, + "loss": 2.1112, + "theoretical_loss": 4.918248026562697, + "tokens_seen": 84082688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874824473420261, + "loss": 2.0547, + "theoretical_loss": 4.917706871571712, + "tokens_seen": 84148224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874663991975929, + "loss": 2.1856, + "theoretical_loss": 4.9171662557811615, + "tokens_seen": 84213760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874503510531595, + "loss": 2.1379, + "theoretical_loss": 4.916626178234807, + "tokens_seen": 84279296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874343029087263, + "loss": 2.0925, + "theoretical_loss": 4.916086637978851, + "tokens_seen": 84344832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874182547642929, + "loss": 2.1199, + "theoretical_loss": 4.915547634061921, + "tokens_seen": 84410368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874022066198597, + "loss": 2.206, + "theoretical_loss": 4.91500916553507, + "tokens_seen": 84475904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873861584754263, + "loss": 2.1473, + "theoretical_loss": 4.914471231451762, + "tokens_seen": 84541440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873701103309931, + "loss": 2.1729, + "theoretical_loss": 4.913933830867868, + "tokens_seen": 84606976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873540621865597, + "loss": 2.0473, + "theoretical_loss": 4.913396962841655, + "tokens_seen": 84672512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873380140421265, + "loss": 2.1081, + "theoretical_loss": 4.9128606264337815, + "tokens_seen": 84738048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873219658976931, + "loss": 2.2149, + "theoretical_loss": 4.912324820707285, + "tokens_seen": 84803584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873059177532599, + "loss": 2.1657, + "theoretical_loss": 4.911789544727581, + "tokens_seen": 84869120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872898696088265, + "loss": 2.0486, + "theoretical_loss": 4.911254797562448, + "tokens_seen": 84934656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872738214643933, + "loss": 2.1763, + "theoretical_loss": 4.910720578282021, + "tokens_seen": 85000192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872577733199599, + "loss": 2.025, + "theoretical_loss": 4.91018688595879, + "tokens_seen": 85065728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872417251755267, + "loss": 2.0928, + "theoretical_loss": 4.909653719667584, + "tokens_seen": 85131264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 35098, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.9277470111846924, + "objective/train/theoretical_loss": 4.909121078485567, + "objective/train/tokens_used": 105656800, + "theoretical_loss": 4.909121078485567, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872256770310933, + "loss": 2.1105, + "theoretical_loss": 4.909121078485567, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872096288866601, + "loss": 2.0636, + "theoretical_loss": 4.908588961492235, + "tokens_seen": 85262336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871935807422267, + "loss": 2.1548, + "theoretical_loss": 4.908057367769396, + "tokens_seen": 85327872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871775325977935, + "loss": 2.146, + "theoretical_loss": 4.907526296401175, + "tokens_seen": 85393408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871614844533601, + "loss": 2.0305, + "theoretical_loss": 4.906995746474001, + "tokens_seen": 85458944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871454363089269, + "loss": 2.1158, + "theoretical_loss": 4.9064657170766, + "tokens_seen": 85524480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871293881644935, + "loss": 2.1002, + "theoretical_loss": 4.905936207299984, + "tokens_seen": 85590016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871133400200603, + "loss": 2.0993, + "theoretical_loss": 4.90540721623745, + "tokens_seen": 85655552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870972918756269, + "loss": 1.978, + "theoretical_loss": 4.904878742984569, + "tokens_seen": 85721088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870812437311937, + "loss": 2.0845, + "theoretical_loss": 4.904350786639178, + "tokens_seen": 85786624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870651955867603, + "loss": 2.1174, + "theoretical_loss": 4.903823346301374, + "tokens_seen": 85852160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870491474423271, + "loss": 1.9148, + "theoretical_loss": 4.903296421073506, + "tokens_seen": 85917696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870330992978937, + "loss": 2.203, + "theoretical_loss": 4.902770010060166, + "tokens_seen": 85983232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870170511534605, + "loss": 2.1191, + "theoretical_loss": 4.902244112368188, + "tokens_seen": 86048768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870010030090271, + "loss": 2.1491, + "theoretical_loss": 4.901718727106631, + "tokens_seen": 86114304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869849548645939, + "loss": 2.1635, + "theoretical_loss": 4.90119385338678, + "tokens_seen": 86179840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869689067201605, + "loss": 1.9647, + "theoretical_loss": 4.900669490322134, + "tokens_seen": 86245376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869528585757273, + "loss": 2.0817, + "theoretical_loss": 4.900145637028402, + "tokens_seen": 86310912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869368104312939, + "loss": 2.1156, + "theoretical_loss": 4.899622292623493, + "tokens_seen": 86376448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869207622868607, + "loss": 1.9936, + "theoretical_loss": 4.8990994562275105, + "tokens_seen": 86441984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869047141424273, + "loss": 2.1465, + "theoretical_loss": 4.898577126962746, + "tokens_seen": 86507520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868886659979941, + "loss": 2.1501, + "theoretical_loss": 4.898055303953669, + "tokens_seen": 86573056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868726178535607, + "loss": 2.0958, + "theoretical_loss": 4.897533986326922, + "tokens_seen": 86638592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868565697091274, + "loss": 2.0493, + "theoretical_loss": 4.897013173211316, + "tokens_seen": 86704128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868405215646941, + "loss": 2.061, + "theoretical_loss": 4.896492863737818, + "tokens_seen": 86769664 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868244734202608, + "loss": 2.1131, + "theoretical_loss": 4.895973057039548, + "tokens_seen": 86835200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868084252758275, + "loss": 2.0576, + "theoretical_loss": 4.89545375225177, + "tokens_seen": 86900736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867923771313942, + "loss": 2.0952, + "theoretical_loss": 4.894934948511889, + "tokens_seen": 86966272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867763289869609, + "loss": 2.0713, + "theoretical_loss": 4.894416644959437, + "tokens_seen": 87031808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867602808425276, + "loss": 2.1069, + "theoretical_loss": 4.893898840736071, + "tokens_seen": 87097344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867442326980943, + "loss": 2.0183, + "theoretical_loss": 4.8933815349855685, + "tokens_seen": 87162880 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786728184553661, + "loss": 2.1052, + "theoretical_loss": 4.892864726853814, + "tokens_seen": 87228416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867121364092277, + "loss": 2.0933, + "theoretical_loss": 4.892348415488799, + "tokens_seen": 87293952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866960882647944, + "loss": 2.0157, + "theoretical_loss": 4.891832600040609, + "tokens_seen": 87359488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866800401203611, + "loss": 2.1374, + "theoretical_loss": 4.891317279661422, + "tokens_seen": 87425024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866639919759278, + "loss": 2.0133, + "theoretical_loss": 4.890802453505498, + "tokens_seen": 87490560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866479438314945, + "loss": 1.901, + "theoretical_loss": 4.890288120729174, + "tokens_seen": 87556096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866318956870612, + "loss": 2.1402, + "theoretical_loss": 4.88977428049086, + "tokens_seen": 87621632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866158475426279, + "loss": 1.9311, + "theoretical_loss": 4.889260931951026, + "tokens_seen": 87687168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865997993981946, + "loss": 2.1391, + "theoretical_loss": 4.888748074272201, + "tokens_seen": 87752704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865837512537613, + "loss": 2.1003, + "theoretical_loss": 4.888235706618964, + "tokens_seen": 87818240 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786567703109328, + "loss": 2.0694, + "theoretical_loss": 4.887723828157939, + "tokens_seen": 87883776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865516549648947, + "loss": 2.0203, + "theoretical_loss": 4.8872124380577855, + "tokens_seen": 87949312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865356068204614, + "loss": 2.0587, + "theoretical_loss": 4.886701535489195, + "tokens_seen": 88014848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865195586760281, + "loss": 2.1354, + "theoretical_loss": 4.886191119624883, + "tokens_seen": 88080384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865035105315948, + "loss": 2.034, + "theoretical_loss": 4.885681189639584, + "tokens_seen": 88145920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864874623871615, + "loss": 2.1003, + "theoretical_loss": 4.885171744710044, + "tokens_seen": 88211456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864714142427282, + "loss": 2.1345, + "theoretical_loss": 4.884662784015012, + "tokens_seen": 88276992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864553660982949, + "loss": 2.0893, + "theoretical_loss": 4.884154306735239, + "tokens_seen": 88342528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864393179538616, + "loss": 2.1243, + "theoretical_loss": 4.8836463120534646, + "tokens_seen": 88408064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 36417, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8406845331192017, + "objective/train/theoretical_loss": 4.88313879915442, + "objective/train/tokens_used": 108933600, + "theoretical_loss": 4.88313879915442, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864232698094283, + "loss": 1.8913, + "theoretical_loss": 4.88313879915442, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786407221664995, + "loss": 2.1626, + "theoretical_loss": 4.882631767224812, + "tokens_seen": 88539136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863911735205617, + "loss": 2.0407, + "theoretical_loss": 4.882125215453321, + "tokens_seen": 88604672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863751253761284, + "loss": 2.0725, + "theoretical_loss": 4.881619143030598, + "tokens_seen": 88670208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863590772316951, + "loss": 2.007, + "theoretical_loss": 4.8811135491492506, + "tokens_seen": 88735744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863430290872618, + "loss": 2.0586, + "theoretical_loss": 4.880608433003846, + "tokens_seen": 88801280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863269809428285, + "loss": 2.0653, + "theoretical_loss": 4.880103793790896, + "tokens_seen": 88866816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863109327983952, + "loss": 2.0433, + "theoretical_loss": 4.87959963070886, + "tokens_seen": 88932352 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786294884653962, + "loss": 2.0326, + "theoretical_loss": 4.879095942958127, + "tokens_seen": 88997888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007862788365095286, + "loss": 2.0379, + "theoretical_loss": 4.878592729741024, + "tokens_seen": 89063424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007862627883650953, + "loss": 2.1523, + "theoretical_loss": 4.878089990261797, + "tokens_seen": 89128960 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786246740220662, + "loss": 1.9848, + "theoretical_loss": 4.87758772372661, + "tokens_seen": 89194496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007862306920762287, + "loss": 1.9817, + "theoretical_loss": 4.877085929343544, + "tokens_seen": 89260032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007862146439317954, + "loss": 2.1303, + "theoretical_loss": 4.876584606322583, + "tokens_seen": 89325568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861985957873622, + "loss": 2.0567, + "theoretical_loss": 4.8760837538756086, + "tokens_seen": 89391104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861825476429288, + "loss": 1.9851, + "theoretical_loss": 4.875583371216404, + "tokens_seen": 89456640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861664994984956, + "loss": 2.0458, + "theoretical_loss": 4.875083457560633, + "tokens_seen": 89522176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861504513540622, + "loss": 1.9532, + "theoretical_loss": 4.874584012125847, + "tokens_seen": 89587712 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786134403209629, + "loss": 2.0171, + "theoretical_loss": 4.874085034131472, + "tokens_seen": 89653248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861183550651956, + "loss": 2.016, + "theoretical_loss": 4.873586522798803, + "tokens_seen": 89718784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861023069207624, + "loss": 2.0034, + "theoretical_loss": 4.873088477351005, + "tokens_seen": 89784320 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786086258776329, + "loss": 2.0347, + "theoretical_loss": 4.8725908970130964, + "tokens_seen": 89849856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860702106318958, + "loss": 2.0416, + "theoretical_loss": 4.872093781011952, + "tokens_seen": 89915392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860541624874624, + "loss": 1.9762, + "theoretical_loss": 4.871597128576292, + "tokens_seen": 89980928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860381143430292, + "loss": 1.9752, + "theoretical_loss": 4.8711009389366815, + "tokens_seen": 90046464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860220661985958, + "loss": 2.0416, + "theoretical_loss": 4.870605211325517, + "tokens_seen": 90112000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860060180541626, + "loss": 1.9874, + "theoretical_loss": 4.870109944977029, + "tokens_seen": 90177536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007859899699097292, + "loss": 2.0183, + "theoretical_loss": 4.869615139127273, + "tokens_seen": 90243072 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785973921765296, + "loss": 2.079, + "theoretical_loss": 4.869120793014117, + "tokens_seen": 90308608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007859578736208626, + "loss": 2.039, + "theoretical_loss": 4.868626905877251, + "tokens_seen": 90374144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007859418254764294, + "loss": 2.0082, + "theoretical_loss": 4.8681334769581674, + "tokens_seen": 90439680 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785925777331996, + "loss": 1.88, + "theoretical_loss": 4.867640505500161, + "tokens_seen": 90505216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007859097291875628, + "loss": 2.0596, + "theoretical_loss": 4.8671479907483235, + "tokens_seen": 90570752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858936810431294, + "loss": 2.0777, + "theoretical_loss": 4.866655931949537, + "tokens_seen": 90636288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858776328986962, + "loss": 2.0637, + "theoretical_loss": 4.86616432835247, + "tokens_seen": 90701824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858615847542628, + "loss": 1.9995, + "theoretical_loss": 4.865673179207572, + "tokens_seen": 90767360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858455366098296, + "loss": 2.1029, + "theoretical_loss": 4.865182483767063, + "tokens_seen": 90832896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858294884653962, + "loss": 2.0009, + "theoretical_loss": 4.864692241284933, + "tokens_seen": 90898432 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785813440320963, + "loss": 2.0887, + "theoretical_loss": 4.864202451016939, + "tokens_seen": 90963968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857973921765296, + "loss": 2.0239, + "theoretical_loss": 4.863713112220592, + "tokens_seen": 91029504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857813440320964, + "loss": 2.0429, + "theoretical_loss": 4.8632242241551555, + "tokens_seen": 91095040 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785765295887663, + "loss": 2.0759, + "theoretical_loss": 4.862735786081642, + "tokens_seen": 91160576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857492477432298, + "loss": 2.014, + "theoretical_loss": 4.862247797262806, + "tokens_seen": 91226112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857331995987964, + "loss": 2.0947, + "theoretical_loss": 4.861760256963136, + "tokens_seen": 91291648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857171514543632, + "loss": 2.0521, + "theoretical_loss": 4.861273164448854, + "tokens_seen": 91357184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857011033099298, + "loss": 1.9563, + "theoretical_loss": 4.860786518987906, + "tokens_seen": 91422720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007856850551654966, + "loss": 2.0038, + "theoretical_loss": 4.86030031984996, + "tokens_seen": 91488256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007856690070210632, + "loss": 1.9585, + "theoretical_loss": 4.859814566306397, + "tokens_seen": 91553792 + }, + { + "epoch": 0.03, + "learning_rate": 0.00078565295887663, + "loss": 1.9885, + "theoretical_loss": 4.859329257630311, + "tokens_seen": 91619328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007856369107321966, + "loss": 2.0648, + "theoretical_loss": 4.858844393096497, + "tokens_seen": 91684864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 37720, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.0330617427825928, + "objective/train/theoretical_loss": 4.858359971981454, + "objective/train/tokens_used": 112210400, + "theoretical_loss": 4.858359971981454, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007856208625877634, + "loss": 2.0313, + "theoretical_loss": 4.858359971981454, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.00078560481444333, + "loss": 2.1479, + "theoretical_loss": 4.85787599356337, + "tokens_seen": 91815936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855887662988968, + "loss": 2.0328, + "theoretical_loss": 4.857392457122128, + "tokens_seen": 91881472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855727181544634, + "loss": 2.1289, + "theoretical_loss": 4.856909361939288, + "tokens_seen": 91947008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855566700100302, + "loss": 1.9571, + "theoretical_loss": 4.856426707298096, + "tokens_seen": 92012544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855406218655968, + "loss": 1.9227, + "theoretical_loss": 4.8559444924834665, + "tokens_seen": 92078080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855245737211636, + "loss": 1.9564, + "theoretical_loss": 4.855462716781985, + "tokens_seen": 92143616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855085255767302, + "loss": 2.0934, + "theoretical_loss": 4.854981379481901, + "tokens_seen": 92209152 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785492477432297, + "loss": 2.0322, + "theoretical_loss": 4.854500479873119, + "tokens_seen": 92274688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007854764292878636, + "loss": 1.9538, + "theoretical_loss": 4.854020017247203, + "tokens_seen": 92340224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007854603811434303, + "loss": 1.9776, + "theoretical_loss": 4.853539990897358, + "tokens_seen": 92405760 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785444332998997, + "loss": 2.0646, + "theoretical_loss": 4.85306040011844, + "tokens_seen": 92471296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007854282848545637, + "loss": 1.9781, + "theoretical_loss": 4.852581244206938, + "tokens_seen": 92536832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007854122367101304, + "loss": 2.0651, + "theoretical_loss": 4.852102522460975, + "tokens_seen": 92602368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853961885656971, + "loss": 1.9221, + "theoretical_loss": 4.851624234180306, + "tokens_seen": 92667904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853801404212638, + "loss": 2.1273, + "theoretical_loss": 4.851146378666305, + "tokens_seen": 92733440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853640922768305, + "loss": 1.9866, + "theoretical_loss": 4.85066895522197, + "tokens_seen": 92798976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853480441323972, + "loss": 2.0728, + "theoretical_loss": 4.850191963151908, + "tokens_seen": 92864512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853319959879639, + "loss": 2.1431, + "theoretical_loss": 4.849715401762337, + "tokens_seen": 92930048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853159478435306, + "loss": 2.0067, + "theoretical_loss": 4.849239270361078, + "tokens_seen": 92995584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852998996990973, + "loss": 1.9941, + "theoretical_loss": 4.848763568257554, + "tokens_seen": 93061120 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785283851554664, + "loss": 2.0881, + "theoretical_loss": 4.848288294762779, + "tokens_seen": 93126656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852678034102307, + "loss": 2.0293, + "theoretical_loss": 4.847813449189358, + "tokens_seen": 93192192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852517552657974, + "loss": 1.958, + "theoretical_loss": 4.847339030851482, + "tokens_seen": 93257728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852357071213641, + "loss": 1.9812, + "theoretical_loss": 4.84686503906492, + "tokens_seen": 93323264 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852196589769308, + "loss": 2.1079, + "theoretical_loss": 4.846391473147019, + "tokens_seen": 93388800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852036108324975, + "loss": 1.9527, + "theoretical_loss": 4.8459183324166935, + "tokens_seen": 93454336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851875626880642, + "loss": 1.9455, + "theoretical_loss": 4.845445616194426, + "tokens_seen": 93519872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851715145436309, + "loss": 2.1299, + "theoretical_loss": 4.844973323802259, + "tokens_seen": 93585408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851554663991976, + "loss": 2.0971, + "theoretical_loss": 4.844501454563792, + "tokens_seen": 93650944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851394182547643, + "loss": 1.9184, + "theoretical_loss": 4.844030007804177, + "tokens_seen": 93716480 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785123370110331, + "loss": 1.9424, + "theoretical_loss": 4.843558982850113, + "tokens_seen": 93782016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851073219658977, + "loss": 2.0786, + "theoretical_loss": 4.84308837902984, + "tokens_seen": 93847552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850912738214644, + "loss": 1.9767, + "theoretical_loss": 4.842618195673138, + "tokens_seen": 93913088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850752256770311, + "loss": 2.0386, + "theoretical_loss": 4.84214843211132, + "tokens_seen": 93978624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850591775325978, + "loss": 1.9345, + "theoretical_loss": 4.841679087677226, + "tokens_seen": 94044160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850431293881645, + "loss": 2.1009, + "theoretical_loss": 4.841210161705223, + "tokens_seen": 94109696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850270812437312, + "loss": 2.0208, + "theoretical_loss": 4.840741653531195, + "tokens_seen": 94175232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850110330992979, + "loss": 1.8831, + "theoretical_loss": 4.840273562492545, + "tokens_seen": 94240768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849949849548646, + "loss": 2.0515, + "theoretical_loss": 4.839805887928181, + "tokens_seen": 94306304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849789368104313, + "loss": 1.9214, + "theoretical_loss": 4.839338629178522, + "tokens_seen": 94371840 + }, + { + "epoch": 0.03, + "learning_rate": 0.000784962888665998, + "loss": 1.9477, + "theoretical_loss": 4.8388717855854875, + "tokens_seen": 94437376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849468405215647, + "loss": 1.927, + "theoretical_loss": 4.838405356492494, + "tokens_seen": 94502912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849307923771314, + "loss": 1.9606, + "theoretical_loss": 4.83793934124445, + "tokens_seen": 94568448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849147442326981, + "loss": 1.9475, + "theoretical_loss": 4.837473739187754, + "tokens_seen": 94633984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848986960882648, + "loss": 2.0574, + "theoretical_loss": 4.837008549670285, + "tokens_seen": 94699520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848826479438315, + "loss": 2.0748, + "theoretical_loss": 4.8365437720414075, + "tokens_seen": 94765056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848665997993982, + "loss": 2.0284, + "theoretical_loss": 4.836079405651956, + "tokens_seen": 94830592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848505516549649, + "loss": 2.039, + "theoretical_loss": 4.835615449854238, + "tokens_seen": 94896128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848345035105317, + "loss": 1.9871, + "theoretical_loss": 4.8351519040020285, + "tokens_seen": 94961664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 38548, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.0551602840423584, + "objective/train/theoretical_loss": 4.834688767450562, + "objective/train/tokens_used": 115487200, + "theoretical_loss": 4.834688767450562, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848184553660983, + "loss": 2.0169, + "theoretical_loss": 4.834688767450562, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.000784802407221665, + "loss": 1.9129, + "theoretical_loss": 4.8342260395565315, + "tokens_seen": 95092736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847863590772317, + "loss": 1.9872, + "theoretical_loss": 4.833763719678085, + "tokens_seen": 95158272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847703109327985, + "loss": 1.954, + "theoretical_loss": 4.833301807174822, + "tokens_seen": 95223808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847542627883651, + "loss": 2.0831, + "theoretical_loss": 4.83284030140778, + "tokens_seen": 95289344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847382146439319, + "loss": 1.894, + "theoretical_loss": 4.832379201739442, + "tokens_seen": 95354880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847221664994985, + "loss": 1.9736, + "theoretical_loss": 4.831918507533728, + "tokens_seen": 95420416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847061183550653, + "loss": 2.0363, + "theoretical_loss": 4.831458218155989, + "tokens_seen": 95485952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846900702106319, + "loss": 2.0462, + "theoretical_loss": 4.830998332973005, + "tokens_seen": 95551488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846740220661987, + "loss": 1.8864, + "theoretical_loss": 4.830538851352976, + "tokens_seen": 95617024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846579739217653, + "loss": 1.9409, + "theoretical_loss": 4.830079772665529, + "tokens_seen": 95682560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846419257773321, + "loss": 1.9883, + "theoretical_loss": 4.829621096281702, + "tokens_seen": 95748096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846258776328987, + "loss": 1.8937, + "theoretical_loss": 4.8291628215739415, + "tokens_seen": 95813632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846098294884655, + "loss": 1.9911, + "theoretical_loss": 4.828704947916108, + "tokens_seen": 95879168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845937813440321, + "loss": 1.9991, + "theoretical_loss": 4.82824747468346, + "tokens_seen": 95944704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845777331995989, + "loss": 1.9115, + "theoretical_loss": 4.827790401252658, + "tokens_seen": 96010240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845616850551655, + "loss": 1.9385, + "theoretical_loss": 4.827333727001756, + "tokens_seen": 96075776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845456369107323, + "loss": 1.9522, + "theoretical_loss": 4.8268774513101995, + "tokens_seen": 96141312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845295887662989, + "loss": 1.8942, + "theoretical_loss": 4.82642157355882, + "tokens_seen": 96206848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845135406218657, + "loss": 1.9496, + "theoretical_loss": 4.825966093129834, + "tokens_seen": 96272384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844974924774323, + "loss": 1.9805, + "theoretical_loss": 4.825511009406833, + "tokens_seen": 96337920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844814443329991, + "loss": 1.9984, + "theoretical_loss": 4.825056321774788, + "tokens_seen": 96403456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844653961885657, + "loss": 1.9623, + "theoretical_loss": 4.8246020296200385, + "tokens_seen": 96468992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844493480441325, + "loss": 1.9433, + "theoretical_loss": 4.82414813233029, + "tokens_seen": 96534528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844332998996991, + "loss": 2.0465, + "theoretical_loss": 4.823694629294609, + "tokens_seen": 96600064 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844172517552659, + "loss": 2.0518, + "theoretical_loss": 4.823241519903428, + "tokens_seen": 96665600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844012036108325, + "loss": 1.9394, + "theoretical_loss": 4.8227888035485265, + "tokens_seen": 96731136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843851554663993, + "loss": 1.8898, + "theoretical_loss": 4.82233647962304, + "tokens_seen": 96796672 + }, + { + "epoch": 0.03, + "learning_rate": 0.000784369107321966, + "loss": 1.9072, + "theoretical_loss": 4.821884547521449, + "tokens_seen": 96862208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843530591775327, + "loss": 2.0767, + "theoretical_loss": 4.821433006639578, + "tokens_seen": 96927744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843370110330994, + "loss": 1.9428, + "theoretical_loss": 4.820981856374589, + "tokens_seen": 96993280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843209628886661, + "loss": 1.9907, + "theoretical_loss": 4.8205310961249825, + "tokens_seen": 97058816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843049147442328, + "loss": 1.9943, + "theoretical_loss": 4.820080725290589, + "tokens_seen": 97124352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842888665997995, + "loss": 1.9588, + "theoretical_loss": 4.819630743272566, + "tokens_seen": 97189888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842728184553662, + "loss": 2.0534, + "theoretical_loss": 4.819181149473396, + "tokens_seen": 97255424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842567703109329, + "loss": 2.0432, + "theoretical_loss": 4.81873194329688, + "tokens_seen": 97320960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842407221664996, + "loss": 2.0024, + "theoretical_loss": 4.81828312414814, + "tokens_seen": 97386496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842246740220663, + "loss": 2.0566, + "theoretical_loss": 4.817834691433602, + "tokens_seen": 97452032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842086258776329, + "loss": 1.9204, + "theoretical_loss": 4.81738664456101, + "tokens_seen": 97517568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841925777331997, + "loss": 1.8443, + "theoretical_loss": 4.816938982939407, + "tokens_seen": 97583104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841765295887663, + "loss": 2.0024, + "theoretical_loss": 4.816491705979138, + "tokens_seen": 97648640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841604814443331, + "loss": 2.0781, + "theoretical_loss": 4.816044813091848, + "tokens_seen": 97714176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841444332998997, + "loss": 2.0516, + "theoretical_loss": 4.815598303690473, + "tokens_seen": 97779712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841283851554665, + "loss": 1.8457, + "theoretical_loss": 4.8151521771892405, + "tokens_seen": 97845248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841123370110331, + "loss": 1.9794, + "theoretical_loss": 4.814706433003665, + "tokens_seen": 97910784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840962888665999, + "loss": 1.988, + "theoretical_loss": 4.8142610705505415, + "tokens_seen": 97976320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840802407221665, + "loss": 1.9506, + "theoretical_loss": 4.813816089247945, + "tokens_seen": 98041856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840641925777332, + "loss": 2.0677, + "theoretical_loss": 4.813371488515227, + "tokens_seen": 98107392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840481444332999, + "loss": 1.8749, + "theoretical_loss": 4.812927267773008, + "tokens_seen": 98172928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840320962888666, + "loss": 2.0423, + "theoretical_loss": 4.812483426443181, + "tokens_seen": 98238464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 39774, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.065314292907715, + "objective/train/theoretical_loss": 4.812039963948898, + "objective/train/tokens_used": 118764000, + "theoretical_loss": 4.812039963948898, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840160481444333, + "loss": 1.9389, + "theoretical_loss": 4.812039963948898, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.000784, + "loss": 2.0512, + "theoretical_loss": 4.811596879714575, + "tokens_seen": 98369536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839839518555667, + "loss": 2.0692, + "theoretical_loss": 4.811154173165886, + "tokens_seen": 98435072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839679037111334, + "loss": 2.0087, + "theoretical_loss": 4.810711843729758, + "tokens_seen": 98500608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839518555667001, + "loss": 1.9157, + "theoretical_loss": 4.810269890834364, + "tokens_seen": 98566144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839358074222668, + "loss": 1.8269, + "theoretical_loss": 4.809828313909129, + "tokens_seen": 98631680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839197592778335, + "loss": 1.9956, + "theoretical_loss": 4.809387112384721, + "tokens_seen": 98697216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839037111334002, + "loss": 1.9715, + "theoretical_loss": 4.808946285693043, + "tokens_seen": 98762752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838876629889669, + "loss": 1.8838, + "theoretical_loss": 4.808505833267237, + "tokens_seen": 98828288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838716148445336, + "loss": 1.8904, + "theoretical_loss": 4.808065754541676, + "tokens_seen": 98893824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838555667001003, + "loss": 1.89, + "theoretical_loss": 4.807626048951965, + "tokens_seen": 98959360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783839518555667, + "loss": 1.9809, + "theoretical_loss": 4.807186715934931, + "tokens_seen": 99024896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838234704112337, + "loss": 1.8864, + "theoretical_loss": 4.806747754928622, + "tokens_seen": 99090432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838074222668004, + "loss": 1.8412, + "theoretical_loss": 4.806309165372311, + "tokens_seen": 99155968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837913741223671, + "loss": 1.8538, + "theoretical_loss": 4.805870946706479, + "tokens_seen": 99221504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837753259779338, + "loss": 2.0107, + "theoretical_loss": 4.805433098372822, + "tokens_seen": 99287040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837592778335005, + "loss": 2.0084, + "theoretical_loss": 4.8049956198142425, + "tokens_seen": 99352576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837432296890672, + "loss": 1.9323, + "theoretical_loss": 4.804558510474852, + "tokens_seen": 99418112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837271815446339, + "loss": 1.9652, + "theoretical_loss": 4.804121769799959, + "tokens_seen": 99483648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837111334002006, + "loss": 1.8399, + "theoretical_loss": 4.8036853972360705, + "tokens_seen": 99549184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836950852557673, + "loss": 1.9356, + "theoretical_loss": 4.80324939223089, + "tokens_seen": 99614720 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783679037111334, + "loss": 2.0503, + "theoretical_loss": 4.802813754233311, + "tokens_seen": 99680256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836629889669007, + "loss": 1.9646, + "theoretical_loss": 4.802378482693417, + "tokens_seen": 99745792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836469408224674, + "loss": 1.9134, + "theoretical_loss": 4.801943577062472, + "tokens_seen": 99811328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836308926780341, + "loss": 1.9831, + "theoretical_loss": 4.801509036792925, + "tokens_seen": 99876864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836148445336008, + "loss": 2.1101, + "theoretical_loss": 4.801074861338401, + "tokens_seen": 99942400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835987963891675, + "loss": 1.9731, + "theoretical_loss": 4.800641050153703, + "tokens_seen": 100007936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835827482447342, + "loss": 2.0708, + "theoretical_loss": 4.800207602694801, + "tokens_seen": 100073472 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783566700100301, + "loss": 1.9291, + "theoretical_loss": 4.7997745184188325, + "tokens_seen": 100139008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835506519558676, + "loss": 2.0162, + "theoretical_loss": 4.799341796784106, + "tokens_seen": 100204544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835346038114343, + "loss": 2.0161, + "theoretical_loss": 4.798909437250086, + "tokens_seen": 100270080 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783518555667001, + "loss": 1.872, + "theoretical_loss": 4.798477439277397, + "tokens_seen": 100335616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835025075225677, + "loss": 1.9359, + "theoretical_loss": 4.7980458023278185, + "tokens_seen": 100401152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834864593781344, + "loss": 2.046, + "theoretical_loss": 4.797614525864282, + "tokens_seen": 100466688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834704112337011, + "loss": 2.0471, + "theoretical_loss": 4.797183609350869, + "tokens_seen": 100532224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834543630892678, + "loss": 1.9752, + "theoretical_loss": 4.796753052252802, + "tokens_seen": 100597760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834383149448346, + "loss": 1.8555, + "theoretical_loss": 4.79632285403645, + "tokens_seen": 100663296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834222668004012, + "loss": 1.9888, + "theoretical_loss": 4.795893014169323, + "tokens_seen": 100728832 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783406218655968, + "loss": 1.9027, + "theoretical_loss": 4.795463532120058, + "tokens_seen": 100794368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833901705115346, + "loss": 1.9115, + "theoretical_loss": 4.795034407358435, + "tokens_seen": 100859904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833741223671014, + "loss": 1.9345, + "theoretical_loss": 4.794605639355357, + "tokens_seen": 100925440 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783358074222668, + "loss": 1.9462, + "theoretical_loss": 4.794177227582855, + "tokens_seen": 100990976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833420260782348, + "loss": 1.9011, + "theoretical_loss": 4.793749171514085, + "tokens_seen": 101056512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833259779338014, + "loss": 1.8762, + "theoretical_loss": 4.793321470623321, + "tokens_seen": 101122048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833099297893682, + "loss": 1.9244, + "theoretical_loss": 4.792894124385955, + "tokens_seen": 101187584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832938816449348, + "loss": 1.9741, + "theoretical_loss": 4.792467132278494, + "tokens_seen": 101253120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832778335005016, + "loss": 1.9348, + "theoretical_loss": 4.792040493778553, + "tokens_seen": 101318656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832617853560682, + "loss": 2.0157, + "theoretical_loss": 4.791614208364859, + "tokens_seen": 101384192 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783245737211635, + "loss": 1.9452, + "theoretical_loss": 4.791188275517239, + "tokens_seen": 101449728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832296890672016, + "loss": 1.9476, + "theoretical_loss": 4.7907626947166255, + "tokens_seen": 101515264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 40787, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.9380860328674316, + "objective/train/theoretical_loss": 4.790337465445049, + "objective/train/tokens_used": 122040800, + "theoretical_loss": 4.790337465445049, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832136409227684, + "loss": 1.9874, + "theoretical_loss": 4.790337465445049, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783197592778335, + "loss": 2.06, + "theoretical_loss": 4.7899125871856345, + "tokens_seen": 101646336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831815446339018, + "loss": 1.8105, + "theoretical_loss": 4.789488059422599, + "tokens_seen": 101711872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831654964894685, + "loss": 2.0042, + "theoretical_loss": 4.789063881641251, + "tokens_seen": 101777408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831494483450352, + "loss": 2.0481, + "theoretical_loss": 4.7886400533279865, + "tokens_seen": 101842944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831334002006019, + "loss": 1.9965, + "theoretical_loss": 4.788216573970281, + "tokens_seen": 101908480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831173520561686, + "loss": 1.9324, + "theoretical_loss": 4.787793443056694, + "tokens_seen": 101974016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831013039117353, + "loss": 1.8745, + "theoretical_loss": 4.787370660076862, + "tokens_seen": 102039552 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783085255767302, + "loss": 2.0481, + "theoretical_loss": 4.786948224521495, + "tokens_seen": 102105088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830692076228687, + "loss": 1.9234, + "theoretical_loss": 4.786526135882378, + "tokens_seen": 102170624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830531594784354, + "loss": 2.0611, + "theoretical_loss": 4.786104393652359, + "tokens_seen": 102236160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830371113340021, + "loss": 1.9584, + "theoretical_loss": 4.785682997325358, + "tokens_seen": 102301696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830210631895688, + "loss": 1.9192, + "theoretical_loss": 4.7852619463963535, + "tokens_seen": 102367232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830050150451355, + "loss": 1.8914, + "theoretical_loss": 4.784841240361389, + "tokens_seen": 102432768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829889669007022, + "loss": 1.9203, + "theoretical_loss": 4.784420878717558, + "tokens_seen": 102498304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829729187562689, + "loss": 1.9652, + "theoretical_loss": 4.7840008609630145, + "tokens_seen": 102563840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829568706118356, + "loss": 1.9889, + "theoretical_loss": 4.783581186596962, + "tokens_seen": 102629376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829408224674023, + "loss": 1.8846, + "theoretical_loss": 4.78316185511965, + "tokens_seen": 102694912 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782924774322969, + "loss": 2.0007, + "theoretical_loss": 4.782742866032379, + "tokens_seen": 102760448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829087261785357, + "loss": 1.8957, + "theoretical_loss": 4.782324218837486, + "tokens_seen": 102825984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828926780341024, + "loss": 1.9462, + "theoretical_loss": 4.781905913038351, + "tokens_seen": 102891520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828766298896691, + "loss": 1.9478, + "theoretical_loss": 4.781487948139393, + "tokens_seen": 102957056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828605817452358, + "loss": 1.9648, + "theoretical_loss": 4.7810703236460625, + "tokens_seen": 103022592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828445336008025, + "loss": 1.9738, + "theoretical_loss": 4.780653039064843, + "tokens_seen": 103088128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828284854563692, + "loss": 1.9539, + "theoretical_loss": 4.780236093903245, + "tokens_seen": 103153664 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828124373119358, + "loss": 1.8898, + "theoretical_loss": 4.779819487669805, + "tokens_seen": 103219200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827963891675026, + "loss": 1.8978, + "theoretical_loss": 4.779403219874085, + "tokens_seen": 103284736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827803410230692, + "loss": 1.97, + "theoretical_loss": 4.778987290026665, + "tokens_seen": 103350272 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782764292878636, + "loss": 1.9551, + "theoretical_loss": 4.778571697639142, + "tokens_seen": 103415808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827482447342026, + "loss": 1.8563, + "theoretical_loss": 4.778156442224132, + "tokens_seen": 103481344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827321965897694, + "loss": 2.0819, + "theoretical_loss": 4.7777415232952585, + "tokens_seen": 103546880 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782716148445336, + "loss": 1.9497, + "theoretical_loss": 4.777326940367155, + "tokens_seen": 103612416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827001003009028, + "loss": 2.0545, + "theoretical_loss": 4.776912692955463, + "tokens_seen": 103677952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826840521564694, + "loss": 1.9213, + "theoretical_loss": 4.776498780576826, + "tokens_seen": 103743488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826680040120361, + "loss": 1.9726, + "theoretical_loss": 4.776085202748893, + "tokens_seen": 103809024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826519558676028, + "loss": 1.8898, + "theoretical_loss": 4.775671958990305, + "tokens_seen": 103874560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826359077231695, + "loss": 2.004, + "theoretical_loss": 4.775259048820702, + "tokens_seen": 103940096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826198595787362, + "loss": 1.9216, + "theoretical_loss": 4.7748464717607195, + "tokens_seen": 104005632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826038114343029, + "loss": 1.8736, + "theoretical_loss": 4.774434227331979, + "tokens_seen": 104071168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825877632898696, + "loss": 1.9748, + "theoretical_loss": 4.774022315057092, + "tokens_seen": 104136704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825717151454363, + "loss": 1.9511, + "theoretical_loss": 4.773610734459654, + "tokens_seen": 104202240 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782555667001003, + "loss": 1.8595, + "theoretical_loss": 4.773199485064243, + "tokens_seen": 104267776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825396188565697, + "loss": 1.8144, + "theoretical_loss": 4.772788566396418, + "tokens_seen": 104333312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825235707121364, + "loss": 1.9711, + "theoretical_loss": 4.772377977982714, + "tokens_seen": 104398848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825075225677031, + "loss": 1.8884, + "theoretical_loss": 4.771967719350641, + "tokens_seen": 104464384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824914744232698, + "loss": 1.9437, + "theoretical_loss": 4.77155779002868, + "tokens_seen": 104529920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824754262788365, + "loss": 1.8921, + "theoretical_loss": 4.771148189546282, + "tokens_seen": 104595456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824593781344032, + "loss": 2.0112, + "theoretical_loss": 4.770738917433864, + "tokens_seen": 104660992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824433299899699, + "loss": 1.9376, + "theoretical_loss": 4.770329973222809, + "tokens_seen": 104726528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824272818455366, + "loss": 1.966, + "theoretical_loss": 4.769921356445458, + "tokens_seen": 104792064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 42078, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.965983510017395, + "objective/train/theoretical_loss": 4.769513066635114, + "objective/train/tokens_used": 125317600, + "theoretical_loss": 4.769513066635114, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824112337011033, + "loss": 1.9, + "theoretical_loss": 4.769513066635114, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.00078239518555667, + "loss": 1.9222, + "theoretical_loss": 4.769105103326036, + "tokens_seen": 104923136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823791374122367, + "loss": 2.0369, + "theoretical_loss": 4.768697466053435, + "tokens_seen": 104988672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823630892678034, + "loss": 1.9214, + "theoretical_loss": 4.768290154353474, + "tokens_seen": 105054208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823470411233701, + "loss": 1.9252, + "theoretical_loss": 4.767883167763267, + "tokens_seen": 105119744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823309929789368, + "loss": 1.9184, + "theoretical_loss": 4.76747650582087, + "tokens_seen": 105185280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823149448345035, + "loss": 1.8375, + "theoretical_loss": 4.767070168065285, + "tokens_seen": 105250816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822988966900702, + "loss": 1.8323, + "theoretical_loss": 4.766664154036456, + "tokens_seen": 105316352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822828485456369, + "loss": 1.7604, + "theoretical_loss": 4.766258463275265, + "tokens_seen": 105381888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822668004012036, + "loss": 1.9705, + "theoretical_loss": 4.7658530953235285, + "tokens_seen": 105447424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822507522567703, + "loss": 1.9077, + "theoretical_loss": 4.765448049723998, + "tokens_seen": 105512960 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782234704112337, + "loss": 1.7903, + "theoretical_loss": 4.765043326020358, + "tokens_seen": 105578496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822186559679037, + "loss": 1.9913, + "theoretical_loss": 4.764638923757217, + "tokens_seen": 105644032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822026078234704, + "loss": 1.8003, + "theoretical_loss": 4.764234842480114, + "tokens_seen": 105709568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821865596790371, + "loss": 1.8746, + "theoretical_loss": 4.7638310817355105, + "tokens_seen": 105775104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821705115346038, + "loss": 1.9363, + "theoretical_loss": 4.763427641070789, + "tokens_seen": 105840640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821544633901705, + "loss": 1.8989, + "theoretical_loss": 4.76302452003425, + "tokens_seen": 105906176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821384152457372, + "loss": 1.9954, + "theoretical_loss": 4.7626217181751125, + "tokens_seen": 105971712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821223671013039, + "loss": 1.9583, + "theoretical_loss": 4.762219235043508, + "tokens_seen": 106037248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821063189568706, + "loss": 1.9582, + "theoretical_loss": 4.761817070190481, + "tokens_seen": 106102784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820902708124373, + "loss": 2.018, + "theoretical_loss": 4.761415223167982, + "tokens_seen": 106168320 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782074222668004, + "loss": 2.0643, + "theoretical_loss": 4.761013693528874, + "tokens_seen": 106233856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820581745235707, + "loss": 1.9534, + "theoretical_loss": 4.760612480826917, + "tokens_seen": 106299392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820421263791375, + "loss": 1.9656, + "theoretical_loss": 4.76021158461678, + "tokens_seen": 106364928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820260782347041, + "loss": 1.9398, + "theoretical_loss": 4.759811004454027, + "tokens_seen": 106430464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820100300902709, + "loss": 1.9891, + "theoretical_loss": 4.759410739895122, + "tokens_seen": 106496000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819939819458375, + "loss": 1.9332, + "theoretical_loss": 4.759010790497422, + "tokens_seen": 106561536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819779338014043, + "loss": 1.9756, + "theoretical_loss": 4.758611155819178, + "tokens_seen": 106627072 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781961885656971, + "loss": 1.9532, + "theoretical_loss": 4.758211835419531, + "tokens_seen": 106692608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819458375125377, + "loss": 1.8169, + "theoretical_loss": 4.757812828858508, + "tokens_seen": 106758144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819297893681044, + "loss": 1.8261, + "theoretical_loss": 4.757414135697024, + "tokens_seen": 106823680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819137412236711, + "loss": 1.9218, + "theoretical_loss": 4.757015755496877, + "tokens_seen": 106889216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818976930792378, + "loss": 1.7847, + "theoretical_loss": 4.756617687820743, + "tokens_seen": 106954752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818816449348045, + "loss": 1.9165, + "theoretical_loss": 4.756219932232181, + "tokens_seen": 107020288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818655967903712, + "loss": 1.9031, + "theoretical_loss": 4.755822488295621, + "tokens_seen": 107085824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818495486459379, + "loss": 2.0325, + "theoretical_loss": 4.755425355576373, + "tokens_seen": 107151360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818335005015046, + "loss": 1.9744, + "theoretical_loss": 4.755028533640614, + "tokens_seen": 107216896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818174523570713, + "loss": 1.9273, + "theoretical_loss": 4.754632022055392, + "tokens_seen": 107282432 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781801404212638, + "loss": 1.9457, + "theoretical_loss": 4.754235820388624, + "tokens_seen": 107347968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817853560682047, + "loss": 1.7709, + "theoretical_loss": 4.7538399282090875, + "tokens_seen": 107413504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817693079237714, + "loss": 1.9709, + "theoretical_loss": 4.753444345086428, + "tokens_seen": 107479040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817532597793381, + "loss": 1.8569, + "theoretical_loss": 4.753049070591146, + "tokens_seen": 107544576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817372116349048, + "loss": 1.8751, + "theoretical_loss": 4.752654104294605, + "tokens_seen": 107610112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817211634904715, + "loss": 1.9358, + "theoretical_loss": 4.752259445769022, + "tokens_seen": 107675648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817051153460382, + "loss": 1.8664, + "theoretical_loss": 4.751865094587465, + "tokens_seen": 107741184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816890672016049, + "loss": 1.8228, + "theoretical_loss": 4.75147105032386, + "tokens_seen": 107806720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816730190571716, + "loss": 1.8293, + "theoretical_loss": 4.751077312552976, + "tokens_seen": 107872256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816569709127383, + "loss": 1.8754, + "theoretical_loss": 4.750683880850433, + "tokens_seen": 107937792 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781640922768305, + "loss": 1.9043, + "theoretical_loss": 4.750290754792694, + "tokens_seen": 108003328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816248746238717, + "loss": 1.8329, + "theoretical_loss": 4.749897933957065, + "tokens_seen": 108068864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 42949, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.9812370538711548, + "objective/train/theoretical_loss": 4.749505417921692, + "objective/train/tokens_used": 128594400, + "theoretical_loss": 4.749505417921692, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816088264794384, + "loss": 1.989, + "theoretical_loss": 4.749505417921692, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815927783350051, + "loss": 1.9188, + "theoretical_loss": 4.7491132062655605, + "tokens_seen": 108199936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815767301905718, + "loss": 1.8283, + "theoretical_loss": 4.748721298568491, + "tokens_seen": 108265472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815606820461385, + "loss": 2.0035, + "theoretical_loss": 4.748329694411137, + "tokens_seen": 108331008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815446339017052, + "loss": 1.9027, + "theoretical_loss": 4.747938393374987, + "tokens_seen": 108396544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815285857572719, + "loss": 1.8085, + "theoretical_loss": 4.747547395042355, + "tokens_seen": 108462080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815125376128386, + "loss": 1.8566, + "theoretical_loss": 4.747156698996388, + "tokens_seen": 108527616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814964894684053, + "loss": 1.8354, + "theoretical_loss": 4.746766304821053, + "tokens_seen": 108593152 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781480441323972, + "loss": 1.98, + "theoretical_loss": 4.746376212101142, + "tokens_seen": 108658688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814643931795387, + "loss": 1.98, + "theoretical_loss": 4.745986420422267, + "tokens_seen": 108724224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814483450351054, + "loss": 1.8918, + "theoretical_loss": 4.745596929370863, + "tokens_seen": 108789760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814322968906721, + "loss": 1.9303, + "theoretical_loss": 4.745207738534177, + "tokens_seen": 108855296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814162487462388, + "loss": 1.8991, + "theoretical_loss": 4.744818847500274, + "tokens_seen": 108920832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814002006018055, + "loss": 1.9216, + "theoretical_loss": 4.744430255858029, + "tokens_seen": 108986368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813841524573721, + "loss": 1.8368, + "theoretical_loss": 4.744041963197132, + "tokens_seen": 109051904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813681043129389, + "loss": 1.9503, + "theoretical_loss": 4.743653969108076, + "tokens_seen": 109117440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813520561685055, + "loss": 1.8052, + "theoretical_loss": 4.743266273182163, + "tokens_seen": 109182976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813360080240723, + "loss": 1.8667, + "theoretical_loss": 4.742878875011501, + "tokens_seen": 109248512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813199598796389, + "loss": 1.8948, + "theoretical_loss": 4.742491774188997, + "tokens_seen": 109314048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813039117352057, + "loss": 2.0147, + "theoretical_loss": 4.742104970308359, + "tokens_seen": 109379584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812878635907723, + "loss": 1.8349, + "theoretical_loss": 4.741718462964095, + "tokens_seen": 109445120 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781271815446339, + "loss": 2.0293, + "theoretical_loss": 4.741332251751509, + "tokens_seen": 109510656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812557673019057, + "loss": 1.9101, + "theoretical_loss": 4.740946336266697, + "tokens_seen": 109576192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812397191574724, + "loss": 1.9097, + "theoretical_loss": 4.740560716106549, + "tokens_seen": 109641728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812236710130391, + "loss": 1.9329, + "theoretical_loss": 4.740175390868742, + "tokens_seen": 109707264 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812076228686058, + "loss": 1.8663, + "theoretical_loss": 4.739790360151744, + "tokens_seen": 109772800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811915747241725, + "loss": 1.9846, + "theoretical_loss": 4.73940562355481, + "tokens_seen": 109838336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811755265797392, + "loss": 1.8642, + "theoretical_loss": 4.739021180677976, + "tokens_seen": 109903872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811594784353059, + "loss": 1.8875, + "theoretical_loss": 4.73863703112206, + "tokens_seen": 109969408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811434302908726, + "loss": 1.8295, + "theoretical_loss": 4.738253174488662, + "tokens_seen": 110034944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811273821464393, + "loss": 1.8563, + "theoretical_loss": 4.73786961038016, + "tokens_seen": 110100480 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781111334002006, + "loss": 1.9474, + "theoretical_loss": 4.737486338399704, + "tokens_seen": 110166016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810952858575727, + "loss": 2.0176, + "theoretical_loss": 4.737103358151225, + "tokens_seen": 110231552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810792377131394, + "loss": 1.8906, + "theoretical_loss": 4.7367206692394195, + "tokens_seen": 110297088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810631895687061, + "loss": 1.8809, + "theoretical_loss": 4.736338271269759, + "tokens_seen": 110362624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810471414242728, + "loss": 1.931, + "theoretical_loss": 4.735956163848478, + "tokens_seen": 110428160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810310932798395, + "loss": 1.968, + "theoretical_loss": 4.735574346582582, + "tokens_seen": 110493696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810150451354062, + "loss": 1.9077, + "theoretical_loss": 4.735192819079838, + "tokens_seen": 110559232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809989969909729, + "loss": 1.8817, + "theoretical_loss": 4.734811580948779, + "tokens_seen": 110624768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809829488465396, + "loss": 1.8448, + "theoretical_loss": 4.734430631798692, + "tokens_seen": 110690304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809669007021063, + "loss": 1.9445, + "theoretical_loss": 4.734049971239628, + "tokens_seen": 110755840 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780950852557673, + "loss": 1.9162, + "theoretical_loss": 4.7336695988823925, + "tokens_seen": 110821376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809348044132397, + "loss": 1.9709, + "theoretical_loss": 4.733289514338546, + "tokens_seen": 110886912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809187562688064, + "loss": 1.8657, + "theoretical_loss": 4.7329097172204, + "tokens_seen": 110952448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809027081243731, + "loss": 1.7885, + "theoretical_loss": 4.73253020714102, + "tokens_seen": 111017984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808866599799398, + "loss": 1.9659, + "theoretical_loss": 4.732150983714217, + "tokens_seen": 111083520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808706118355065, + "loss": 1.8298, + "theoretical_loss": 4.731772046554552, + "tokens_seen": 111149056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808545636910732, + "loss": 1.9417, + "theoretical_loss": 4.731393395277331, + "tokens_seen": 111214592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808385155466399, + "loss": 1.8618, + "theoretical_loss": 4.731015029498598, + "tokens_seen": 111280128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808224674022066, + "loss": 1.9002, + "theoretical_loss": 4.730636948835148, + "tokens_seen": 111345664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 44130, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.9857747554779053, + "objective/train/theoretical_loss": 4.730259152904507, + "objective/train/tokens_used": 131871200, + "theoretical_loss": 4.730259152904507, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808064192577733, + "loss": 1.9456, + "theoretical_loss": 4.730259152904507, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.00078079037111334, + "loss": 1.7301, + "theoretical_loss": 4.7298816413249405, + "tokens_seen": 111476736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807743229689067, + "loss": 1.7904, + "theoretical_loss": 4.729504413715454, + "tokens_seen": 111542272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807582748244735, + "loss": 1.8232, + "theoretical_loss": 4.729127469695783, + "tokens_seen": 111607808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807422266800401, + "loss": 1.8824, + "theoretical_loss": 4.728750808886394, + "tokens_seen": 111673344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807261785356069, + "loss": 1.9541, + "theoretical_loss": 4.72837443090849, + "tokens_seen": 111738880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807101303911736, + "loss": 1.9042, + "theoretical_loss": 4.727998335383996, + "tokens_seen": 111804416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806940822467403, + "loss": 1.8814, + "theoretical_loss": 4.727622521935563, + "tokens_seen": 111869952 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780678034102307, + "loss": 1.8453, + "theoretical_loss": 4.7272469901865755, + "tokens_seen": 111935488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806619859578737, + "loss": 1.8953, + "theoretical_loss": 4.726871739761132, + "tokens_seen": 112001024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806459378134404, + "loss": 1.8651, + "theoretical_loss": 4.726496770284056, + "tokens_seen": 112066560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806298896690071, + "loss": 1.9383, + "theoretical_loss": 4.726122081380891, + "tokens_seen": 112132096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806138415245738, + "loss": 1.8434, + "theoretical_loss": 4.725747672677894, + "tokens_seen": 112197632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805977933801405, + "loss": 1.9197, + "theoretical_loss": 4.725373543802043, + "tokens_seen": 112263168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805817452357072, + "loss": 1.9461, + "theoretical_loss": 4.724999694381027, + "tokens_seen": 112328704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805656970912739, + "loss": 1.858, + "theoretical_loss": 4.724626124043247, + "tokens_seen": 112394240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805496489468406, + "loss": 1.9208, + "theoretical_loss": 4.724252832417817, + "tokens_seen": 112459776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805336008024073, + "loss": 1.7769, + "theoretical_loss": 4.723879819134558, + "tokens_seen": 112525312 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780517552657974, + "loss": 1.807, + "theoretical_loss": 4.7235070838239945, + "tokens_seen": 112590848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805015045135407, + "loss": 1.9201, + "theoretical_loss": 4.723134626117363, + "tokens_seen": 112656384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804854563691074, + "loss": 1.8909, + "theoretical_loss": 4.722762445646598, + "tokens_seen": 112721920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804694082246741, + "loss": 1.9641, + "theoretical_loss": 4.722390542044339, + "tokens_seen": 112787456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804533600802408, + "loss": 1.8542, + "theoretical_loss": 4.722018914943924, + "tokens_seen": 112852992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804373119358075, + "loss": 1.7947, + "theoretical_loss": 4.721647563979385, + "tokens_seen": 112918528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804212637913742, + "loss": 1.7628, + "theoretical_loss": 4.72127648878546, + "tokens_seen": 112984064 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804052156469409, + "loss": 2.0437, + "theoretical_loss": 4.720905688997572, + "tokens_seen": 113049600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803891675025076, + "loss": 1.8696, + "theoretical_loss": 4.720535164251842, + "tokens_seen": 113115136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803731193580743, + "loss": 2.0169, + "theoretical_loss": 4.720164914185084, + "tokens_seen": 113180672 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780357071213641, + "loss": 1.9562, + "theoretical_loss": 4.719794938434794, + "tokens_seen": 113246208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803410230692077, + "loss": 1.8761, + "theoretical_loss": 4.7194252366391645, + "tokens_seen": 113311744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803249749247744, + "loss": 1.8482, + "theoretical_loss": 4.719055808437068, + "tokens_seen": 113377280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803089267803411, + "loss": 1.8214, + "theoretical_loss": 4.718686653468065, + "tokens_seen": 113442816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802928786359078, + "loss": 1.89, + "theoretical_loss": 4.718317771372398, + "tokens_seen": 113508352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802768304914745, + "loss": 2.0138, + "theoretical_loss": 4.7179491617909886, + "tokens_seen": 113573888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802607823470412, + "loss": 1.8962, + "theoretical_loss": 4.717580824365439, + "tokens_seen": 113639424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802447342026079, + "loss": 1.9767, + "theoretical_loss": 4.71721275873803, + "tokens_seen": 113704960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802286860581746, + "loss": 1.8079, + "theoretical_loss": 4.716844964551717, + "tokens_seen": 113770496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802126379137413, + "loss": 1.9601, + "theoretical_loss": 4.716477441450131, + "tokens_seen": 113836032 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780196589769308, + "loss": 1.8804, + "theoretical_loss": 4.716110189077575, + "tokens_seen": 113901568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801805416248747, + "loss": 1.9204, + "theoretical_loss": 4.715743207079022, + "tokens_seen": 113967104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801644934804414, + "loss": 1.8374, + "theoretical_loss": 4.715376495100118, + "tokens_seen": 114032640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801484453360081, + "loss": 1.9237, + "theoretical_loss": 4.7150100527871714, + "tokens_seen": 114098176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801323971915748, + "loss": 1.8361, + "theoretical_loss": 4.714643879787161, + "tokens_seen": 114163712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801163490471415, + "loss": 1.9508, + "theoretical_loss": 4.714277975747729, + "tokens_seen": 114229248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801003009027082, + "loss": 1.8377, + "theoretical_loss": 4.7139123403171785, + "tokens_seen": 114294784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800842527582749, + "loss": 1.771, + "theoretical_loss": 4.713546973144476, + "tokens_seen": 114360320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800682046138416, + "loss": 1.7296, + "theoretical_loss": 4.713181873879248, + "tokens_seen": 114425856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800521564694083, + "loss": 1.9301, + "theoretical_loss": 4.712817042171776, + "tokens_seen": 114491392 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780036108324975, + "loss": 1.804, + "theoretical_loss": 4.712452477673001, + "tokens_seen": 114556928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800200601805417, + "loss": 1.7899, + "theoretical_loss": 4.712088180034517, + "tokens_seen": 114622464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 45100, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7656515836715698, + "objective/train/theoretical_loss": 4.711724148908571, + "objective/train/tokens_used": 135148000, + "theoretical_loss": 4.711724148908571, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800040120361084, + "loss": 1.7773, + "theoretical_loss": 4.711724148908571, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.000779987963891675, + "loss": 1.8698, + "theoretical_loss": 4.711360383948064, + "tokens_seen": 114753536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799719157472418, + "loss": 1.81, + "theoretical_loss": 4.710996884806542, + "tokens_seen": 114819072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799558676028084, + "loss": 1.8963, + "theoretical_loss": 4.710633651138206, + "tokens_seen": 114884608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799398194583752, + "loss": 2.0256, + "theoretical_loss": 4.710270682597898, + "tokens_seen": 114950144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799237713139418, + "loss": 1.9627, + "theoretical_loss": 4.709907978841109, + "tokens_seen": 115015680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799077231695086, + "loss": 1.9976, + "theoretical_loss": 4.709545539523971, + "tokens_seen": 115081216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007798916750250752, + "loss": 1.7934, + "theoretical_loss": 4.709183364303258, + "tokens_seen": 115146752 + }, + { + "epoch": 0.03, + "learning_rate": 0.000779875626880642, + "loss": 1.7876, + "theoretical_loss": 4.708821452836388, + "tokens_seen": 115212288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007798595787362086, + "loss": 1.8766, + "theoretical_loss": 4.708459804781414, + "tokens_seen": 115277824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007798435305917753, + "loss": 1.8795, + "theoretical_loss": 4.708098419797031, + "tokens_seen": 115343360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000779827482447342, + "loss": 1.8992, + "theoretical_loss": 4.707737297542563, + "tokens_seen": 115408896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007798114343029087, + "loss": 1.7714, + "theoretical_loss": 4.707376437677974, + "tokens_seen": 115474432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797953861584754, + "loss": 1.8703, + "theoretical_loss": 4.707015839863859, + "tokens_seen": 115539968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797793380140421, + "loss": 1.8995, + "theoretical_loss": 4.706655503761443, + "tokens_seen": 115605504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797632898696088, + "loss": 1.8673, + "theoretical_loss": 4.706295429032584, + "tokens_seen": 115671040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797472417251755, + "loss": 1.8975, + "theoretical_loss": 4.705935615339763, + "tokens_seen": 115736576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797311935807422, + "loss": 1.964, + "theoretical_loss": 4.705576062346094, + "tokens_seen": 115802112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797151454363089, + "loss": 1.7302, + "theoretical_loss": 4.70521676971531, + "tokens_seen": 115867648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796990972918756, + "loss": 1.8957, + "theoretical_loss": 4.704857737111773, + "tokens_seen": 115933184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796830491474423, + "loss": 1.8279, + "theoretical_loss": 4.704498964200463, + "tokens_seen": 115998720 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779667001003009, + "loss": 1.9176, + "theoretical_loss": 4.70414045064698, + "tokens_seen": 116064256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796509528585757, + "loss": 1.9368, + "theoretical_loss": 4.703782196117549, + "tokens_seen": 116129792 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796349047141424, + "loss": 1.9024, + "theoretical_loss": 4.703424200279007, + "tokens_seen": 116195328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796188565697091, + "loss": 1.7334, + "theoretical_loss": 4.703066462798809, + "tokens_seen": 116260864 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796028084252758, + "loss": 2.0384, + "theoretical_loss": 4.7027089833450235, + "tokens_seen": 116326400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007795867602808425, + "loss": 1.8372, + "theoretical_loss": 4.7023517615863355, + "tokens_seen": 116391936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007795707121364092, + "loss": 1.7742, + "theoretical_loss": 4.701994797192038, + "tokens_seen": 116457472 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779554663991976, + "loss": 1.977, + "theoretical_loss": 4.701638089832034, + "tokens_seen": 116523008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007795386158475426, + "loss": 1.9297, + "theoretical_loss": 4.701281639176839, + "tokens_seen": 116588544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007795225677031094, + "loss": 1.8724, + "theoretical_loss": 4.700925444897575, + "tokens_seen": 116654080 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779506519558676, + "loss": 1.7618, + "theoretical_loss": 4.700569506665965, + "tokens_seen": 116719616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794904714142428, + "loss": 1.8056, + "theoretical_loss": 4.700213824154341, + "tokens_seen": 116785152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794744232698094, + "loss": 1.8923, + "theoretical_loss": 4.699858397035637, + "tokens_seen": 116850688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794583751253762, + "loss": 1.9056, + "theoretical_loss": 4.699503224983388, + "tokens_seen": 116916224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794423269809428, + "loss": 1.9701, + "theoretical_loss": 4.699148307671729, + "tokens_seen": 116981760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794262788365096, + "loss": 1.8863, + "theoretical_loss": 4.698793644775394, + "tokens_seen": 117047296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794102306920762, + "loss": 1.9355, + "theoretical_loss": 4.698439235969713, + "tokens_seen": 117112832 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779394182547643, + "loss": 1.9582, + "theoretical_loss": 4.698085080930612, + "tokens_seen": 117178368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007793781344032096, + "loss": 1.8616, + "theoretical_loss": 4.697731179334614, + "tokens_seen": 117243904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007793620862587764, + "loss": 1.7266, + "theoretical_loss": 4.69737753085883, + "tokens_seen": 117309440 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779346038114343, + "loss": 1.9112, + "theoretical_loss": 4.697024135180968, + "tokens_seen": 117374976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007793299899699098, + "loss": 1.8236, + "theoretical_loss": 4.696670991979321, + "tokens_seen": 117440512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007793139418254765, + "loss": 1.8411, + "theoretical_loss": 4.696318100932773, + "tokens_seen": 117506048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792978936810432, + "loss": 1.7059, + "theoretical_loss": 4.695965461720796, + "tokens_seen": 117571584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792818455366099, + "loss": 1.822, + "theoretical_loss": 4.695613074023446, + "tokens_seen": 117637120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792657973921766, + "loss": 1.894, + "theoretical_loss": 4.695260937521364, + "tokens_seen": 117702656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792497492477433, + "loss": 1.7513, + "theoretical_loss": 4.694909051895775, + "tokens_seen": 117768192 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077923370110331, + "loss": 1.8627, + "theoretical_loss": 4.694557416828484, + "tokens_seen": 117833728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792176529588767, + "loss": 1.7953, + "theoretical_loss": 4.694206032001879, + "tokens_seen": 117899264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 46257, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.9019111394882202, + "objective/train/theoretical_loss": 4.693854897098923, + "objective/train/tokens_used": 138424800, + "theoretical_loss": 4.693854897098923, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792016048144434, + "loss": 1.9417, + "theoretical_loss": 4.693854897098923, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791855566700101, + "loss": 1.8928, + "theoretical_loss": 4.693504011803159, + "tokens_seen": 118030336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791695085255768, + "loss": 1.8751, + "theoretical_loss": 4.6931533757987065, + "tokens_seen": 118095872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791534603811435, + "loss": 1.9013, + "theoretical_loss": 4.692802988770258, + "tokens_seen": 118161408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791374122367102, + "loss": 1.8634, + "theoretical_loss": 4.692452850403081, + "tokens_seen": 118226944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791213640922769, + "loss": 2.0152, + "theoretical_loss": 4.692102960383014, + "tokens_seen": 118292480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791053159478436, + "loss": 1.7348, + "theoretical_loss": 4.691753318396467, + "tokens_seen": 118358016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790892678034103, + "loss": 1.8863, + "theoretical_loss": 4.691403924130418, + "tokens_seen": 118423552 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779073219658977, + "loss": 1.9038, + "theoretical_loss": 4.691054777272415, + "tokens_seen": 118489088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790571715145437, + "loss": 1.8282, + "theoretical_loss": 4.69070587751057, + "tokens_seen": 118554624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790411233701104, + "loss": 1.9085, + "theoretical_loss": 4.690357224533564, + "tokens_seen": 118620160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790250752256771, + "loss": 1.8251, + "theoretical_loss": 4.690008818030638, + "tokens_seen": 118685696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790090270812438, + "loss": 1.6978, + "theoretical_loss": 4.689660657691598, + "tokens_seen": 118751232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789929789368105, + "loss": 1.9915, + "theoretical_loss": 4.689312743206811, + "tokens_seen": 118816768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789769307923772, + "loss": 1.9805, + "theoretical_loss": 4.688965074267203, + "tokens_seen": 118882304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789608826479439, + "loss": 1.8472, + "theoretical_loss": 4.688617650564259, + "tokens_seen": 118947840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789448345035106, + "loss": 1.9364, + "theoretical_loss": 4.688270471790023, + "tokens_seen": 119013376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789287863590773, + "loss": 1.8973, + "theoretical_loss": 4.687923537637094, + "tokens_seen": 119078912 + }, + { + "epoch": 0.04, + "learning_rate": 0.000778912738214644, + "loss": 1.8099, + "theoretical_loss": 4.687576847798622, + "tokens_seen": 119144448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788966900702107, + "loss": 1.8415, + "theoretical_loss": 4.687230401968319, + "tokens_seen": 119209984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788806419257774, + "loss": 1.9467, + "theoretical_loss": 4.68688419984044, + "tokens_seen": 119275520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788645937813441, + "loss": 1.8803, + "theoretical_loss": 4.686538241109796, + "tokens_seen": 119341056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788485456369108, + "loss": 1.8097, + "theoretical_loss": 4.686192525471746, + "tokens_seen": 119406592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788324974924775, + "loss": 1.8324, + "theoretical_loss": 4.685847052622199, + "tokens_seen": 119472128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788164493480442, + "loss": 1.8055, + "theoretical_loss": 4.685501822257608, + "tokens_seen": 119537664 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788004012036109, + "loss": 1.7789, + "theoretical_loss": 4.685156834074972, + "tokens_seen": 119603200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787843530591776, + "loss": 1.763, + "theoretical_loss": 4.684812087771839, + "tokens_seen": 119668736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787683049147443, + "loss": 1.8904, + "theoretical_loss": 4.684467583046292, + "tokens_seen": 119734272 + }, + { + "epoch": 0.04, + "learning_rate": 0.000778752256770311, + "loss": 1.84, + "theoretical_loss": 4.6841233195969645, + "tokens_seen": 119799808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787362086258777, + "loss": 1.9018, + "theoretical_loss": 4.683779297123024, + "tokens_seen": 119865344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787201604814444, + "loss": 1.804, + "theoretical_loss": 4.683435515324181, + "tokens_seen": 119930880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787041123370111, + "loss": 1.9663, + "theoretical_loss": 4.683091973900682, + "tokens_seen": 119996416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786880641925778, + "loss": 1.849, + "theoretical_loss": 4.682748672553313, + "tokens_seen": 120061952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786720160481445, + "loss": 1.8176, + "theoretical_loss": 4.68240561098339, + "tokens_seen": 120127488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786559679037112, + "loss": 1.7518, + "theoretical_loss": 4.68206278889277, + "tokens_seen": 120193024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786399197592779, + "loss": 1.8349, + "theoretical_loss": 4.681720205983839, + "tokens_seen": 120258560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786238716148446, + "loss": 1.8489, + "theoretical_loss": 4.681377861959516, + "tokens_seen": 120324096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786078234704113, + "loss": 1.8931, + "theoretical_loss": 4.681035756523249, + "tokens_seen": 120389632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785917753259779, + "loss": 1.9067, + "theoretical_loss": 4.680693889379018, + "tokens_seen": 120455168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785757271815447, + "loss": 1.855, + "theoretical_loss": 4.680352260231329, + "tokens_seen": 120520704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785596790371113, + "loss": 1.8698, + "theoretical_loss": 4.680010868785216, + "tokens_seen": 120586240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785436308926781, + "loss": 1.83, + "theoretical_loss": 4.679669714746238, + "tokens_seen": 120651776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785275827482447, + "loss": 1.9867, + "theoretical_loss": 4.679328797820478, + "tokens_seen": 120717312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785115346038115, + "loss": 1.944, + "theoretical_loss": 4.678988117714544, + "tokens_seen": 120782848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784954864593781, + "loss": 1.7626, + "theoretical_loss": 4.678647674135563, + "tokens_seen": 120848384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784794383149449, + "loss": 1.8657, + "theoretical_loss": 4.6783074667911855, + "tokens_seen": 120913920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784633901705115, + "loss": 1.8642, + "theoretical_loss": 4.677967495389581, + "tokens_seen": 120979456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784473420260782, + "loss": 1.7655, + "theoretical_loss": 4.677627759639435, + "tokens_seen": 121044992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784312938816449, + "loss": 1.8806, + "theoretical_loss": 4.677288259249954, + "tokens_seen": 121110528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784152457372116, + "loss": 1.8441, + "theoretical_loss": 4.676948993930857, + "tokens_seen": 121176064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 47440, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.9147517681121826, + "objective/train/theoretical_loss": 4.67660996339238, + "objective/train/tokens_used": 141701600, + "theoretical_loss": 4.67660996339238, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783991975927783, + "loss": 1.9058, + "theoretical_loss": 4.67660996339238, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.000778383149448345, + "loss": 1.9118, + "theoretical_loss": 4.676271167345272, + "tokens_seen": 121307136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783671013039117, + "loss": 1.9903, + "theoretical_loss": 4.675932605500794, + "tokens_seen": 121372672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783510531594785, + "loss": 1.846, + "theoretical_loss": 4.675594277570718, + "tokens_seen": 121438208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783350050150451, + "loss": 1.7874, + "theoretical_loss": 4.675256183267327, + "tokens_seen": 121503744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783189568706119, + "loss": 1.8911, + "theoretical_loss": 4.674918322303411, + "tokens_seen": 121569280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783029087261785, + "loss": 1.7944, + "theoretical_loss": 4.6745806943922705, + "tokens_seen": 121634816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782868605817453, + "loss": 1.7367, + "theoretical_loss": 4.67424329924771, + "tokens_seen": 121700352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782708124373119, + "loss": 1.8909, + "theoretical_loss": 4.673906136584041, + "tokens_seen": 121765888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782547642928787, + "loss": 1.9456, + "theoretical_loss": 4.673569206116078, + "tokens_seen": 121831424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782387161484453, + "loss": 1.8463, + "theoretical_loss": 4.6732325075591366, + "tokens_seen": 121896960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782226680040121, + "loss": 1.9333, + "theoretical_loss": 4.67289604062904, + "tokens_seen": 121962496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782066198595787, + "loss": 1.7753, + "theoretical_loss": 4.672559805042107, + "tokens_seen": 122028032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781905717151455, + "loss": 1.8157, + "theoretical_loss": 4.672223800515159, + "tokens_seen": 122093568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781745235707121, + "loss": 1.8541, + "theoretical_loss": 4.671888026765512, + "tokens_seen": 122159104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781584754262789, + "loss": 1.7951, + "theoretical_loss": 4.6715524835109825, + "tokens_seen": 122224640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781424272818455, + "loss": 1.8092, + "theoretical_loss": 4.671217170469884, + "tokens_seen": 122290176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781263791374123, + "loss": 1.7207, + "theoretical_loss": 4.67088208736102, + "tokens_seen": 122355712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781103309929789, + "loss": 1.9078, + "theoretical_loss": 4.670547233903694, + "tokens_seen": 122421248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780942828485457, + "loss": 1.8607, + "theoretical_loss": 4.670212609817698, + "tokens_seen": 122486784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780782347041123, + "loss": 1.8305, + "theoretical_loss": 4.669878214823316, + "tokens_seen": 122552320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780621865596791, + "loss": 1.9033, + "theoretical_loss": 4.669544048641325, + "tokens_seen": 122617856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780461384152457, + "loss": 1.9115, + "theoretical_loss": 4.669210110992989, + "tokens_seen": 122683392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780300902708125, + "loss": 1.9634, + "theoretical_loss": 4.668876401600059, + "tokens_seen": 122748928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780140421263791, + "loss": 1.7661, + "theoretical_loss": 4.668542920184779, + "tokens_seen": 122814464 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779979939819459, + "loss": 1.8726, + "theoretical_loss": 4.6682096664698705, + "tokens_seen": 122880000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779819458375126, + "loss": 1.9475, + "theoretical_loss": 4.667876640178546, + "tokens_seen": 122945536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779658976930793, + "loss": 1.9562, + "theoretical_loss": 4.667543841034502, + "tokens_seen": 123011072 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777949849548646, + "loss": 1.9686, + "theoretical_loss": 4.667211268761914, + "tokens_seen": 123076608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779338014042127, + "loss": 1.8776, + "theoretical_loss": 4.66687892308544, + "tokens_seen": 123142144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779177532597794, + "loss": 1.9679, + "theoretical_loss": 4.6665468037302205, + "tokens_seen": 123207680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779017051153461, + "loss": 1.804, + "theoretical_loss": 4.666214910421875, + "tokens_seen": 123273216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778856569709128, + "loss": 1.8628, + "theoretical_loss": 4.665883242886498, + "tokens_seen": 123338752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778696088264795, + "loss": 1.8482, + "theoretical_loss": 4.6655518008506665, + "tokens_seen": 123404288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778535606820462, + "loss": 1.8696, + "theoretical_loss": 4.665220584041428, + "tokens_seen": 123469824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778375125376129, + "loss": 1.9316, + "theoretical_loss": 4.664889592186309, + "tokens_seen": 123535360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778214643931796, + "loss": 1.9424, + "theoretical_loss": 4.6645588250133105, + "tokens_seen": 123600896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778054162487463, + "loss": 1.9075, + "theoretical_loss": 4.664228282250902, + "tokens_seen": 123666432 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777789368104313, + "loss": 1.8398, + "theoretical_loss": 4.663897963628029, + "tokens_seen": 123731968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777733199598797, + "loss": 1.8985, + "theoretical_loss": 4.6635678688741065, + "tokens_seen": 123797504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777572718154464, + "loss": 1.9671, + "theoretical_loss": 4.663237997719017, + "tokens_seen": 123863040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777412236710131, + "loss": 1.8071, + "theoretical_loss": 4.662908349893116, + "tokens_seen": 123928576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777251755265798, + "loss": 1.9107, + "theoretical_loss": 4.662578925127223, + "tokens_seen": 123994112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777091273821465, + "loss": 1.7927, + "theoretical_loss": 4.662249723152624, + "tokens_seen": 124059648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776930792377132, + "loss": 1.9306, + "theoretical_loss": 4.661920743701073, + "tokens_seen": 124125184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776770310932799, + "loss": 2.021, + "theoretical_loss": 4.661591986504786, + "tokens_seen": 124190720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776609829488466, + "loss": 1.9438, + "theoretical_loss": 4.661263451296444, + "tokens_seen": 124256256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776449348044133, + "loss": 1.8106, + "theoretical_loss": 4.660935137809188, + "tokens_seen": 124321792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077762888665998, + "loss": 1.8744, + "theoretical_loss": 4.660607045776624, + "tokens_seen": 124387328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776128385155467, + "loss": 1.7867, + "theoretical_loss": 4.660279174932814, + "tokens_seen": 124452864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 48338, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7055823802947998, + "objective/train/theoretical_loss": 4.659951525012284, + "objective/train/tokens_used": 144978400, + "theoretical_loss": 4.659951525012284, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775967903711134, + "loss": 1.8029, + "theoretical_loss": 4.659951525012284, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775807422266801, + "loss": 1.8085, + "theoretical_loss": 4.6596240957500115, + "tokens_seen": 124583936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775646940822468, + "loss": 1.8204, + "theoretical_loss": 4.659296886881438, + "tokens_seen": 124649472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775486459378135, + "loss": 1.8372, + "theoretical_loss": 4.658969898142456, + "tokens_seen": 124715008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775325977933802, + "loss": 1.7415, + "theoretical_loss": 4.658643129269416, + "tokens_seen": 124780544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775165496489469, + "loss": 1.7994, + "theoretical_loss": 4.658316579999122, + "tokens_seen": 124846080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775005015045136, + "loss": 1.8632, + "theoretical_loss": 4.65799025006883, + "tokens_seen": 124911616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774844533600803, + "loss": 1.8766, + "theoretical_loss": 4.65766413921625, + "tokens_seen": 124977152 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777468405215647, + "loss": 1.8277, + "theoretical_loss": 4.657338247179538, + "tokens_seen": 125042688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774523570712137, + "loss": 1.8551, + "theoretical_loss": 4.657012573697307, + "tokens_seen": 125108224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774363089267804, + "loss": 1.8749, + "theoretical_loss": 4.656687118508616, + "tokens_seen": 125173760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774202607823471, + "loss": 1.7936, + "theoretical_loss": 4.656361881352968, + "tokens_seen": 125239296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774042126379138, + "loss": 1.7392, + "theoretical_loss": 4.656036861970319, + "tokens_seen": 125304832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773881644934805, + "loss": 1.8527, + "theoretical_loss": 4.655712060101068, + "tokens_seen": 125370368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773721163490472, + "loss": 1.854, + "theoretical_loss": 4.65538747548606, + "tokens_seen": 125435904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773560682046139, + "loss": 1.8442, + "theoretical_loss": 4.655063107866582, + "tokens_seen": 125501440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773400200601806, + "loss": 2.0244, + "theoretical_loss": 4.654738956984366, + "tokens_seen": 125566976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773239719157473, + "loss": 1.7517, + "theoretical_loss": 4.654415022581585, + "tokens_seen": 125632512 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777307923771314, + "loss": 1.8149, + "theoretical_loss": 4.654091304400853, + "tokens_seen": 125698048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772918756268807, + "loss": 1.9215, + "theoretical_loss": 4.653767802185225, + "tokens_seen": 125763584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772758274824474, + "loss": 1.8353, + "theoretical_loss": 4.653444515678195, + "tokens_seen": 125829120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772597793380141, + "loss": 1.8363, + "theoretical_loss": 4.653121444623693, + "tokens_seen": 125894656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772437311935808, + "loss": 1.7447, + "theoretical_loss": 4.652798588766088, + "tokens_seen": 125960192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772276830491475, + "loss": 1.7671, + "theoretical_loss": 4.652475947850185, + "tokens_seen": 126025728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772116349047142, + "loss": 1.753, + "theoretical_loss": 4.652153521621223, + "tokens_seen": 126091264 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771955867602808, + "loss": 1.9541, + "theoretical_loss": 4.651831309824875, + "tokens_seen": 126156800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771795386158476, + "loss": 1.7825, + "theoretical_loss": 4.65150931220725, + "tokens_seen": 126222336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771634904714142, + "loss": 1.9339, + "theoretical_loss": 4.651187528514887, + "tokens_seen": 126287872 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777147442326981, + "loss": 1.8252, + "theoretical_loss": 4.650865958494755, + "tokens_seen": 126353408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771313941825476, + "loss": 1.8909, + "theoretical_loss": 4.650544601894257, + "tokens_seen": 126418944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771153460381144, + "loss": 1.9024, + "theoretical_loss": 4.650223458461221, + "tokens_seen": 126484480 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777099297893681, + "loss": 1.862, + "theoretical_loss": 4.649902527943908, + "tokens_seen": 126550016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770832497492478, + "loss": 1.8003, + "theoretical_loss": 4.649581810091003, + "tokens_seen": 126615552 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770672016048144, + "loss": 2.0104, + "theoretical_loss": 4.649261304651619, + "tokens_seen": 126681088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770511534603812, + "loss": 1.7893, + "theoretical_loss": 4.6489410113752925, + "tokens_seen": 126746624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770351053159478, + "loss": 1.814, + "theoretical_loss": 4.6486209300119885, + "tokens_seen": 126812160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770190571715146, + "loss": 1.9046, + "theoretical_loss": 4.648301060312093, + "tokens_seen": 126877696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770030090270812, + "loss": 1.8817, + "theoretical_loss": 4.647981402026417, + "tokens_seen": 126943232 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776986960882648, + "loss": 1.8367, + "theoretical_loss": 4.647661954906188, + "tokens_seen": 127008768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007769709127382146, + "loss": 1.7926, + "theoretical_loss": 4.647342718703062, + "tokens_seen": 127074304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007769548645937814, + "loss": 1.9298, + "theoretical_loss": 4.647023693169107, + "tokens_seen": 127139840 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776938816449348, + "loss": 1.7827, + "theoretical_loss": 4.646704878056816, + "tokens_seen": 127205376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007769227683049148, + "loss": 1.7387, + "theoretical_loss": 4.6463862731191, + "tokens_seen": 127270912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007769067201604814, + "loss": 1.91, + "theoretical_loss": 4.6460678781092835, + "tokens_seen": 127336448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768906720160482, + "loss": 1.993, + "theoretical_loss": 4.6457496927811075, + "tokens_seen": 127401984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768746238716148, + "loss": 1.6914, + "theoretical_loss": 4.645431716888734, + "tokens_seen": 127467520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768585757271816, + "loss": 1.8668, + "theoretical_loss": 4.645113950186732, + "tokens_seen": 127533056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768425275827482, + "loss": 1.7944, + "theoretical_loss": 4.64479639243009, + "tokens_seen": 127598592 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776826479438315, + "loss": 1.7889, + "theoretical_loss": 4.644479043374204, + "tokens_seen": 127664128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768104312938816, + "loss": 1.8859, + "theoretical_loss": 4.644161902774886, + "tokens_seen": 127729664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 49581, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.94822096824646, + "objective/train/theoretical_loss": 4.643844970388358, + "objective/train/tokens_used": 148255200, + "theoretical_loss": 4.643844970388358, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767943831494484, + "loss": 1.9133, + "theoretical_loss": 4.643844970388358, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776778335005015, + "loss": 1.8609, + "theoretical_loss": 4.6435282459712495, + "tokens_seen": 127860736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767622868605818, + "loss": 1.8257, + "theoretical_loss": 4.643211729280601, + "tokens_seen": 127926272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767462387161484, + "loss": 1.7747, + "theoretical_loss": 4.64289542007386, + "tokens_seen": 127991808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767301905717152, + "loss": 1.8617, + "theoretical_loss": 4.642579318108883, + "tokens_seen": 128057344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767141424272818, + "loss": 1.8137, + "theoretical_loss": 4.642263423143932, + "tokens_seen": 128122880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766980942828486, + "loss": 1.8099, + "theoretical_loss": 4.641947734937673, + "tokens_seen": 128188416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766820461384152, + "loss": 1.8716, + "theoretical_loss": 4.641632253249179, + "tokens_seen": 128253952 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776665997993982, + "loss": 2.0769, + "theoretical_loss": 4.641316977837924, + "tokens_seen": 128319488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766499498495486, + "loss": 1.89, + "theoretical_loss": 4.641001908463787, + "tokens_seen": 128385024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766339017051154, + "loss": 1.8734, + "theoretical_loss": 4.6406870448870485, + "tokens_seen": 128450560 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776617853560682, + "loss": 1.8353, + "theoretical_loss": 4.64037238686839, + "tokens_seen": 128516096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766018054162488, + "loss": 1.8572, + "theoretical_loss": 4.640057934168891, + "tokens_seen": 128581632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765857572718155, + "loss": 1.7848, + "theoretical_loss": 4.639743686550034, + "tokens_seen": 128647168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765697091273822, + "loss": 1.8396, + "theoretical_loss": 4.639429643773697, + "tokens_seen": 128712704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765536609829489, + "loss": 1.8095, + "theoretical_loss": 4.639115805602156, + "tokens_seen": 128778240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765376128385156, + "loss": 1.8191, + "theoretical_loss": 4.638802171798086, + "tokens_seen": 128843776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765215646940823, + "loss": 1.9008, + "theoretical_loss": 4.638488742124554, + "tokens_seen": 128909312 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776505516549649, + "loss": 1.8786, + "theoretical_loss": 4.638175516345027, + "tokens_seen": 128974848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764894684052157, + "loss": 1.8323, + "theoretical_loss": 4.637862494223359, + "tokens_seen": 129040384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764734202607824, + "loss": 1.7773, + "theoretical_loss": 4.637549675523806, + "tokens_seen": 129105920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764573721163491, + "loss": 1.8804, + "theoretical_loss": 4.637237060011007, + "tokens_seen": 129171456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764413239719158, + "loss": 1.847, + "theoretical_loss": 4.636924647450002, + "tokens_seen": 129236992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764252758274825, + "loss": 1.8726, + "theoretical_loss": 4.636612437606212, + "tokens_seen": 129302528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764092276830492, + "loss": 1.8728, + "theoretical_loss": 4.636300430245457, + "tokens_seen": 129368064 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763931795386159, + "loss": 1.7916, + "theoretical_loss": 4.63598862513394, + "tokens_seen": 129433600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763771313941826, + "loss": 1.8813, + "theoretical_loss": 4.635677022038254, + "tokens_seen": 129499136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763610832497493, + "loss": 1.8075, + "theoretical_loss": 4.63536562072538, + "tokens_seen": 129564672 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776345035105316, + "loss": 1.8598, + "theoretical_loss": 4.635054420962684, + "tokens_seen": 129630208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763289869608827, + "loss": 1.8148, + "theoretical_loss": 4.634743422517918, + "tokens_seen": 129695744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763129388164494, + "loss": 1.7992, + "theoretical_loss": 4.634432625159221, + "tokens_seen": 129761280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762968906720161, + "loss": 1.8888, + "theoretical_loss": 4.634122028655112, + "tokens_seen": 129826816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762808425275828, + "loss": 1.7781, + "theoretical_loss": 4.633811632774496, + "tokens_seen": 129892352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762647943831495, + "loss": 1.8929, + "theoretical_loss": 4.633501437286659, + "tokens_seen": 129957888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762487462387162, + "loss": 1.9065, + "theoretical_loss": 4.633191441961272, + "tokens_seen": 130023424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762326980942829, + "loss": 1.7713, + "theoretical_loss": 4.63288164656838, + "tokens_seen": 130088960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762166499498496, + "loss": 1.7753, + "theoretical_loss": 4.632572050878415, + "tokens_seen": 130154496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762006018054163, + "loss": 1.8997, + "theoretical_loss": 4.632262654662181, + "tokens_seen": 130220032 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776184553660983, + "loss": 1.8561, + "theoretical_loss": 4.631953457690868, + "tokens_seen": 130285568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761685055165497, + "loss": 1.9013, + "theoretical_loss": 4.631644459736036, + "tokens_seen": 130351104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761524573721164, + "loss": 1.9393, + "theoretical_loss": 4.631335660569626, + "tokens_seen": 130416640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761364092276831, + "loss": 1.9441, + "theoretical_loss": 4.631027059963953, + "tokens_seen": 130482176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761203610832498, + "loss": 1.8176, + "theoretical_loss": 4.630718657691709, + "tokens_seen": 130547712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761043129388165, + "loss": 1.7216, + "theoretical_loss": 4.630410453525958, + "tokens_seen": 130613248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760882647943832, + "loss": 1.766, + "theoretical_loss": 4.630102447240138, + "tokens_seen": 130678784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760722166499499, + "loss": 1.8036, + "theoretical_loss": 4.629794638608059, + "tokens_seen": 130744320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760561685055166, + "loss": 1.851, + "theoretical_loss": 4.629487027403905, + "tokens_seen": 130809856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760401203610833, + "loss": 1.8026, + "theoretical_loss": 4.629179613402228, + "tokens_seen": 130875392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077602407221665, + "loss": 1.8168, + "theoretical_loss": 4.628872396377953, + "tokens_seen": 130940928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760080240722167, + "loss": 1.872, + "theoretical_loss": 4.628565376106371, + "tokens_seen": 131006464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 50603, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7769169807434082, + "objective/train/theoretical_loss": 4.628258552363146, + "objective/train/tokens_used": 151532000, + "theoretical_loss": 4.628258552363146, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759919759277834, + "loss": 1.8162, + "theoretical_loss": 4.628258552363146, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759759277833502, + "loss": 1.8492, + "theoretical_loss": 4.627951924924305, + "tokens_seen": 131137536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759598796389168, + "loss": 1.8724, + "theoretical_loss": 4.627645493566245, + "tokens_seen": 131203072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759438314944836, + "loss": 1.8344, + "theoretical_loss": 4.62733925806573, + "tokens_seen": 131268608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759277833500502, + "loss": 1.8899, + "theoretical_loss": 4.627033218199886, + "tokens_seen": 131334144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759117352056169, + "loss": 1.8562, + "theoretical_loss": 4.626727373746206, + "tokens_seen": 131399680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758956870611836, + "loss": 1.8066, + "theoretical_loss": 4.626421724482549, + "tokens_seen": 131465216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758796389167503, + "loss": 1.8461, + "theoretical_loss": 4.626116270187131, + "tokens_seen": 131530752 + }, + { + "epoch": 0.04, + "learning_rate": 0.000775863590772317, + "loss": 1.736, + "theoretical_loss": 4.625811010638538, + "tokens_seen": 131596288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758475426278837, + "loss": 1.8001, + "theoretical_loss": 4.62550594561571, + "tokens_seen": 131661824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758314944834504, + "loss": 1.823, + "theoretical_loss": 4.625201074897952, + "tokens_seen": 131727360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758154463390171, + "loss": 1.9525, + "theoretical_loss": 4.62489639826493, + "tokens_seen": 131792896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757993981945837, + "loss": 1.8137, + "theoretical_loss": 4.6245919154966675, + "tokens_seen": 131858432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757833500501505, + "loss": 1.8273, + "theoretical_loss": 4.624287626373545, + "tokens_seen": 131923968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757673019057171, + "loss": 1.7471, + "theoretical_loss": 4.623983530676304, + "tokens_seen": 131989504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757512537612839, + "loss": 1.8519, + "theoretical_loss": 4.62367962818604, + "tokens_seen": 132055040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757352056168505, + "loss": 1.944, + "theoretical_loss": 4.623375918684207, + "tokens_seen": 132120576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757191574724173, + "loss": 1.8725, + "theoretical_loss": 4.623072401952614, + "tokens_seen": 132186112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757031093279839, + "loss": 1.8953, + "theoretical_loss": 4.622769077773423, + "tokens_seen": 132251648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756870611835507, + "loss": 1.879, + "theoretical_loss": 4.622465945929152, + "tokens_seen": 132317184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756710130391173, + "loss": 1.7999, + "theoretical_loss": 4.622163006202671, + "tokens_seen": 132382720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756549648946841, + "loss": 1.7466, + "theoretical_loss": 4.621860258377203, + "tokens_seen": 132448256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756389167502507, + "loss": 1.8657, + "theoretical_loss": 4.621557702236323, + "tokens_seen": 132513792 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756228686058175, + "loss": 1.806, + "theoretical_loss": 4.621255337563955, + "tokens_seen": 132579328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756068204613841, + "loss": 1.8109, + "theoretical_loss": 4.620953164144376, + "tokens_seen": 132644864 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755907723169509, + "loss": 1.8321, + "theoretical_loss": 4.620651181762212, + "tokens_seen": 132710400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755747241725175, + "loss": 1.8282, + "theoretical_loss": 4.620349390202436, + "tokens_seen": 132775936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755586760280843, + "loss": 1.8138, + "theoretical_loss": 4.6200477892503695, + "tokens_seen": 132841472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755426278836509, + "loss": 1.8533, + "theoretical_loss": 4.619746378691682, + "tokens_seen": 132907008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755265797392177, + "loss": 1.7597, + "theoretical_loss": 4.61944515831239, + "tokens_seen": 132972544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755105315947843, + "loss": 1.8646, + "theoretical_loss": 4.6191441278988545, + "tokens_seen": 133038080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754944834503511, + "loss": 1.7799, + "theoretical_loss": 4.618843287237782, + "tokens_seen": 133103616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754784353059177, + "loss": 1.8689, + "theoretical_loss": 4.618542636116224, + "tokens_seen": 133169152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754623871614845, + "loss": 1.833, + "theoretical_loss": 4.618242174321575, + "tokens_seen": 133234688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754463390170511, + "loss": 1.8088, + "theoretical_loss": 4.617941901641573, + "tokens_seen": 133300224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754302908726179, + "loss": 1.8624, + "theoretical_loss": 4.617641817864296, + "tokens_seen": 133365760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754142427281845, + "loss": 1.9235, + "theoretical_loss": 4.617341922778168, + "tokens_seen": 133431296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753981945837513, + "loss": 1.701, + "theoretical_loss": 4.617042216171949, + "tokens_seen": 133496832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753821464393179, + "loss": 1.7201, + "theoretical_loss": 4.616742697834742, + "tokens_seen": 133562368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753660982948847, + "loss": 1.8286, + "theoretical_loss": 4.616443367555988, + "tokens_seen": 133627904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753500501504513, + "loss": 1.8104, + "theoretical_loss": 4.616144225125465, + "tokens_seen": 133693440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753340020060181, + "loss": 1.8212, + "theoretical_loss": 4.615845270333294, + "tokens_seen": 133758976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753179538615847, + "loss": 1.7838, + "theoretical_loss": 4.615546502969929, + "tokens_seen": 133824512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753019057171515, + "loss": 1.8179, + "theoretical_loss": 4.61524792282616, + "tokens_seen": 133890048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752858575727181, + "loss": 1.8488, + "theoretical_loss": 4.614949529693115, + "tokens_seen": 133955584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752698094282849, + "loss": 1.7209, + "theoretical_loss": 4.614651323362257, + "tokens_seen": 134021120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752537612838515, + "loss": 1.6942, + "theoretical_loss": 4.614353303625382, + "tokens_seen": 134086656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752377131394183, + "loss": 1.819, + "theoretical_loss": 4.6140554702746215, + "tokens_seen": 134152192 + }, + { + "epoch": 0.04, + "learning_rate": 0.000775221664994985, + "loss": 1.7107, + "theoretical_loss": 4.613757823102437, + "tokens_seen": 134217728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752056168505517, + "loss": 1.782, + "theoretical_loss": 4.613460361901627, + "tokens_seen": 134283264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 51744, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.0655648708343506, + "objective/train/theoretical_loss": 4.613163086465315, + "objective/train/tokens_used": 154808800, + "theoretical_loss": 4.613163086465315, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751895687061184, + "loss": 1.9222, + "theoretical_loss": 4.613163086465315, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751735205616851, + "loss": 1.7955, + "theoretical_loss": 4.612865996586962, + "tokens_seen": 134414336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751574724172518, + "loss": 1.7857, + "theoretical_loss": 4.612569092060355, + "tokens_seen": 134479872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751414242728185, + "loss": 1.8141, + "theoretical_loss": 4.612272372679611, + "tokens_seen": 134545408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751253761283852, + "loss": 1.9289, + "theoretical_loss": 4.611975838239179, + "tokens_seen": 134610944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751093279839519, + "loss": 1.6775, + "theoretical_loss": 4.611679488533832, + "tokens_seen": 134676480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750932798395186, + "loss": 1.8499, + "theoretical_loss": 4.611383323358671, + "tokens_seen": 134742016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750772316950853, + "loss": 1.8098, + "theoretical_loss": 4.61108734250913, + "tokens_seen": 134807552 + }, + { + "epoch": 0.04, + "learning_rate": 0.000775061183550652, + "loss": 1.8215, + "theoretical_loss": 4.610791545780959, + "tokens_seen": 134873088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750451354062187, + "loss": 1.6756, + "theoretical_loss": 4.610495932970241, + "tokens_seen": 134938624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750290872617854, + "loss": 1.7515, + "theoretical_loss": 4.61020050387338, + "tokens_seen": 135004160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750130391173521, + "loss": 1.7896, + "theoretical_loss": 4.609905258287107, + "tokens_seen": 135069696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749969909729188, + "loss": 1.8716, + "theoretical_loss": 4.609610196008473, + "tokens_seen": 135135232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749809428284855, + "loss": 1.7909, + "theoretical_loss": 4.609315316834855, + "tokens_seen": 135200768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749648946840522, + "loss": 1.7184, + "theoretical_loss": 4.60902062056395, + "tokens_seen": 135266304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749488465396189, + "loss": 1.8287, + "theoretical_loss": 4.608726106993776, + "tokens_seen": 135331840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749327983951856, + "loss": 1.9142, + "theoretical_loss": 4.608431775922673, + "tokens_seen": 135397376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749167502507523, + "loss": 1.7733, + "theoretical_loss": 4.608137627149302, + "tokens_seen": 135462912 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774900702106319, + "loss": 1.8085, + "theoretical_loss": 4.607843660472642, + "tokens_seen": 135528448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748846539618857, + "loss": 1.8051, + "theoretical_loss": 4.607549875691989, + "tokens_seen": 135593984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748686058174524, + "loss": 1.8313, + "theoretical_loss": 4.607256272606961, + "tokens_seen": 135659520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748525576730191, + "loss": 1.8189, + "theoretical_loss": 4.606962851017489, + "tokens_seen": 135725056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748365095285858, + "loss": 1.905, + "theoretical_loss": 4.606669610723825, + "tokens_seen": 135790592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748204613841525, + "loss": 1.8924, + "theoretical_loss": 4.606376551526536, + "tokens_seen": 135856128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748044132397192, + "loss": 1.7682, + "theoretical_loss": 4.6060836732265, + "tokens_seen": 135921664 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747883650952859, + "loss": 1.7632, + "theoretical_loss": 4.605790975624917, + "tokens_seen": 135987200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747723169508527, + "loss": 1.8681, + "theoretical_loss": 4.605498458523296, + "tokens_seen": 136052736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747562688064193, + "loss": 1.7845, + "theoretical_loss": 4.605206121723461, + "tokens_seen": 136118272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747402206619861, + "loss": 1.7568, + "theoretical_loss": 4.604913965027551, + "tokens_seen": 136183808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747241725175527, + "loss": 1.8752, + "theoretical_loss": 4.604621988238012, + "tokens_seen": 136249344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747081243731195, + "loss": 1.9572, + "theoretical_loss": 4.604330191157608, + "tokens_seen": 136314880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746920762286861, + "loss": 1.7565, + "theoretical_loss": 4.604038573589408, + "tokens_seen": 136380416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746760280842529, + "loss": 1.783, + "theoretical_loss": 4.603747135336796, + "tokens_seen": 136445952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746599799398195, + "loss": 1.8388, + "theoretical_loss": 4.603455876203463, + "tokens_seen": 136511488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746439317953863, + "loss": 1.7053, + "theoretical_loss": 4.603164795993409, + "tokens_seen": 136577024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746278836509529, + "loss": 1.7839, + "theoretical_loss": 4.602873894510945, + "tokens_seen": 136642560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746118355065197, + "loss": 1.7043, + "theoretical_loss": 4.602583171560686, + "tokens_seen": 136708096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745957873620863, + "loss": 1.7794, + "theoretical_loss": 4.602292626947556, + "tokens_seen": 136773632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745797392176531, + "loss": 1.8616, + "theoretical_loss": 4.602002260476787, + "tokens_seen": 136839168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745636910732197, + "loss": 1.783, + "theoretical_loss": 4.601712071953914, + "tokens_seen": 136904704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745476429287865, + "loss": 1.7862, + "theoretical_loss": 4.601422061184778, + "tokens_seen": 136970240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745315947843531, + "loss": 1.6995, + "theoretical_loss": 4.601132227975527, + "tokens_seen": 137035776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745155466399198, + "loss": 2.0007, + "theoretical_loss": 4.60084257213261, + "tokens_seen": 137101312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744994984954865, + "loss": 1.7742, + "theoretical_loss": 4.600553093462781, + "tokens_seen": 137166848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744834503510532, + "loss": 1.9237, + "theoretical_loss": 4.600263791773097, + "tokens_seen": 137232384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744674022066199, + "loss": 1.8399, + "theoretical_loss": 4.599974666870914, + "tokens_seen": 137297920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744513540621866, + "loss": 1.7832, + "theoretical_loss": 4.599685718563894, + "tokens_seen": 137363456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744353059177533, + "loss": 1.801, + "theoretical_loss": 4.599396946659997, + "tokens_seen": 137428992 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077441925777332, + "loss": 1.7621, + "theoretical_loss": 4.5991083509674855, + "tokens_seen": 137494528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744032096288867, + "loss": 1.7486, + "theoretical_loss": 4.598819931294919, + "tokens_seen": 137560064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 52717, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7819393873214722, + "objective/train/theoretical_loss": 4.598531687451157, + "objective/train/tokens_used": 158085600, + "theoretical_loss": 4.598531687451157, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743871614844534, + "loss": 1.8069, + "theoretical_loss": 4.598531687451157, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077437111334002, + "loss": 1.8632, + "theoretical_loss": 4.598243619245359, + "tokens_seen": 137691136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743550651955868, + "loss": 1.8544, + "theoretical_loss": 4.59795572648698, + "tokens_seen": 137756672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743390170511534, + "loss": 1.8115, + "theoretical_loss": 4.597668008985774, + "tokens_seen": 137822208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743229689067202, + "loss": 1.7225, + "theoretical_loss": 4.59738046655179, + "tokens_seen": 137887744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743069207622868, + "loss": 1.7712, + "theoretical_loss": 4.597093098995377, + "tokens_seen": 137953280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007742908726178536, + "loss": 1.735, + "theoretical_loss": 4.596805906127173, + "tokens_seen": 138018816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007742748244734202, + "loss": 1.7213, + "theoretical_loss": 4.596518887758117, + "tokens_seen": 138084352 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774258776328987, + "loss": 1.867, + "theoretical_loss": 4.596232043699438, + "tokens_seen": 138149888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007742427281845536, + "loss": 1.842, + "theoretical_loss": 4.59594537376266, + "tokens_seen": 138215424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007742266800401204, + "loss": 1.8787, + "theoretical_loss": 4.595658877759602, + "tokens_seen": 138280960 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774210631895687, + "loss": 1.9223, + "theoretical_loss": 4.595372555502371, + "tokens_seen": 138346496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741945837512538, + "loss": 1.9055, + "theoretical_loss": 4.59508640680337, + "tokens_seen": 138412032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741785356068204, + "loss": 1.8217, + "theoretical_loss": 4.594800431475292, + "tokens_seen": 138477568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741624874623872, + "loss": 1.7421, + "theoretical_loss": 4.594514629331121, + "tokens_seen": 138543104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741464393179538, + "loss": 1.7933, + "theoretical_loss": 4.594229000184128, + "tokens_seen": 138608640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741303911735206, + "loss": 1.8861, + "theoretical_loss": 4.593943543847878, + "tokens_seen": 138674176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741143430290872, + "loss": 1.8354, + "theoretical_loss": 4.5936582601362215, + "tokens_seen": 138739712 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774098294884654, + "loss": 1.8875, + "theoretical_loss": 4.593373148863301, + "tokens_seen": 138805248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740822467402206, + "loss": 1.8701, + "theoretical_loss": 4.593088209843543, + "tokens_seen": 138870784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740661985957874, + "loss": 1.8651, + "theoretical_loss": 4.592803442891663, + "tokens_seen": 138936320 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774050150451354, + "loss": 1.8731, + "theoretical_loss": 4.592518847822661, + "tokens_seen": 139001856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740341023069208, + "loss": 1.7787, + "theoretical_loss": 4.592234424451828, + "tokens_seen": 139067392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740180541624874, + "loss": 1.8654, + "theoretical_loss": 4.591950172594734, + "tokens_seen": 139132928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740020060180542, + "loss": 1.728, + "theoretical_loss": 4.591666092067238, + "tokens_seen": 139198464 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739859578736208, + "loss": 1.8389, + "theoretical_loss": 4.591382182685484, + "tokens_seen": 139264000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739699097291876, + "loss": 1.7957, + "theoretical_loss": 4.591098444265896, + "tokens_seen": 139329536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739538615847542, + "loss": 1.7924, + "theoretical_loss": 4.590814876625183, + "tokens_seen": 139395072 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773937813440321, + "loss": 1.8632, + "theoretical_loss": 4.590531479580338, + "tokens_seen": 139460608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739217652958876, + "loss": 1.747, + "theoretical_loss": 4.5902482529486335, + "tokens_seen": 139526144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739057171514544, + "loss": 1.8658, + "theoretical_loss": 4.589965196547625, + "tokens_seen": 139591680 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773889669007021, + "loss": 1.904, + "theoretical_loss": 4.5896823101951485, + "tokens_seen": 139657216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738736208625878, + "loss": 1.8037, + "theoretical_loss": 4.58939959370932, + "tokens_seen": 139722752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738575727181545, + "loss": 1.7745, + "theoretical_loss": 4.5891170469085365, + "tokens_seen": 139788288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738415245737212, + "loss": 1.7752, + "theoretical_loss": 4.588834669611472, + "tokens_seen": 139853824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738254764292879, + "loss": 1.8438, + "theoretical_loss": 4.5885524616370805, + "tokens_seen": 139919360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738094282848546, + "loss": 1.8438, + "theoretical_loss": 4.5882704228045945, + "tokens_seen": 139984896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737933801404213, + "loss": 1.916, + "theoretical_loss": 4.587988552933523, + "tokens_seen": 140050432 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773777331995988, + "loss": 1.7058, + "theoretical_loss": 4.587706851843652, + "tokens_seen": 140115968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737612838515547, + "loss": 1.8099, + "theoretical_loss": 4.587425319355044, + "tokens_seen": 140181504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737452357071214, + "loss": 1.7592, + "theoretical_loss": 4.587143955288038, + "tokens_seen": 140247040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737291875626881, + "loss": 1.8013, + "theoretical_loss": 4.586862759463248, + "tokens_seen": 140312576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737131394182548, + "loss": 1.8653, + "theoretical_loss": 4.586581731701562, + "tokens_seen": 140378112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736970912738215, + "loss": 1.8956, + "theoretical_loss": 4.586300871824143, + "tokens_seen": 140443648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736810431293882, + "loss": 1.8632, + "theoretical_loss": 4.586020179652427, + "tokens_seen": 140509184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736649949849549, + "loss": 1.7402, + "theoretical_loss": 4.585739655008123, + "tokens_seen": 140574720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736489468405216, + "loss": 1.8236, + "theoretical_loss": 4.585459297713215, + "tokens_seen": 140640256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736328986960883, + "loss": 1.8042, + "theoretical_loss": 4.585179107589955, + "tokens_seen": 140705792 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773616850551655, + "loss": 1.8166, + "theoretical_loss": 4.58489908446087, + "tokens_seen": 140771328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736008024072217, + "loss": 1.7575, + "theoretical_loss": 4.584619228148755, + "tokens_seen": 140836864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 53917, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8665790557861328, + "objective/train/theoretical_loss": 4.584339538476678, + "objective/train/tokens_used": 161362400, + "theoretical_loss": 4.584339538476678, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735847542627884, + "loss": 1.8941, + "theoretical_loss": 4.584339538476678, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735687061183552, + "loss": 1.6903, + "theoretical_loss": 4.5840600152679745, + "tokens_seen": 140967936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735526579739218, + "loss": 1.8307, + "theoretical_loss": 4.583780658346252, + "tokens_seen": 141033472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735366098294886, + "loss": 1.7627, + "theoretical_loss": 4.583501467535383, + "tokens_seen": 141099008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735205616850552, + "loss": 1.6755, + "theoretical_loss": 4.583222442659514, + "tokens_seen": 141164544 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773504513540622, + "loss": 1.8388, + "theoretical_loss": 4.5829435835430505, + "tokens_seen": 141230080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734884653961886, + "loss": 1.833, + "theoretical_loss": 4.582664890010674, + "tokens_seen": 141295616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734724172517554, + "loss": 1.9159, + "theoretical_loss": 4.582386361887329, + "tokens_seen": 141361152 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773456369107322, + "loss": 1.7725, + "theoretical_loss": 4.582107998998225, + "tokens_seen": 141426688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734403209628888, + "loss": 1.8281, + "theoretical_loss": 4.581829801168839, + "tokens_seen": 141492224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734242728184554, + "loss": 1.7452, + "theoretical_loss": 4.5815517682249105, + "tokens_seen": 141557760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734082246740222, + "loss": 1.8201, + "theoretical_loss": 4.581273899992447, + "tokens_seen": 141623296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733921765295888, + "loss": 1.8788, + "theoretical_loss": 4.580996196297718, + "tokens_seen": 141688832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733761283851556, + "loss": 1.8429, + "theoretical_loss": 4.580718656967257, + "tokens_seen": 141754368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733600802407222, + "loss": 1.8612, + "theoretical_loss": 4.580441281827858, + "tokens_seen": 141819904 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773344032096289, + "loss": 1.7208, + "theoretical_loss": 4.580164070706584, + "tokens_seen": 141885440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733279839518556, + "loss": 1.7845, + "theoretical_loss": 4.579887023430752, + "tokens_seen": 141950976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733119358074224, + "loss": 1.7119, + "theoretical_loss": 4.579610139827945, + "tokens_seen": 142016512 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773295887662989, + "loss": 1.7464, + "theoretical_loss": 4.579333419726007, + "tokens_seen": 142082048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732798395185558, + "loss": 1.8061, + "theoretical_loss": 4.579056862953039, + "tokens_seen": 142147584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732637913741224, + "loss": 1.8128, + "theoretical_loss": 4.578780469337407, + "tokens_seen": 142213120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732477432296892, + "loss": 1.77, + "theoretical_loss": 4.5785042387077315, + "tokens_seen": 142278656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732316950852558, + "loss": 1.8202, + "theoretical_loss": 4.578228170892896, + "tokens_seen": 142344192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732156469408226, + "loss": 1.9073, + "theoretical_loss": 4.577952265722038, + "tokens_seen": 142409728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731995987963892, + "loss": 1.8577, + "theoretical_loss": 4.5776765230245555, + "tokens_seen": 142475264 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773183550651956, + "loss": 1.8928, + "theoretical_loss": 4.5774009426301046, + "tokens_seen": 142540800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731675025075226, + "loss": 1.8294, + "theoretical_loss": 4.577125524368595, + "tokens_seen": 142606336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731514543630894, + "loss": 1.7549, + "theoretical_loss": 4.5768502680701975, + "tokens_seen": 142671872 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773135406218656, + "loss": 1.8273, + "theoretical_loss": 4.576575173565334, + "tokens_seen": 142737408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731193580742227, + "loss": 1.8096, + "theoretical_loss": 4.5763002406846836, + "tokens_seen": 142802944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731033099297894, + "loss": 1.8756, + "theoretical_loss": 4.57602546925918, + "tokens_seen": 142868480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730872617853561, + "loss": 1.6733, + "theoretical_loss": 4.575750859120014, + "tokens_seen": 142934016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730712136409228, + "loss": 1.8921, + "theoretical_loss": 4.575476410098624, + "tokens_seen": 142999552 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730551654964895, + "loss": 1.7704, + "theoretical_loss": 4.575202122026706, + "tokens_seen": 143065088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730391173520562, + "loss": 1.773, + "theoretical_loss": 4.5749279947362105, + "tokens_seen": 143130624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730230692076229, + "loss": 1.8325, + "theoretical_loss": 4.574654028059336, + "tokens_seen": 143196160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730070210631896, + "loss": 1.8279, + "theoretical_loss": 4.574380221828535, + "tokens_seen": 143261696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729909729187563, + "loss": 1.7535, + "theoretical_loss": 4.574106575876511, + "tokens_seen": 143327232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729749247743229, + "loss": 1.8514, + "theoretical_loss": 4.573833090036219, + "tokens_seen": 143392768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729588766298897, + "loss": 1.8637, + "theoretical_loss": 4.573559764140864, + "tokens_seen": 143458304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729428284854563, + "loss": 1.6962, + "theoretical_loss": 4.573286598023898, + "tokens_seen": 143523840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729267803410231, + "loss": 1.8597, + "theoretical_loss": 4.573013591519029, + "tokens_seen": 143589376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729107321965897, + "loss": 1.9061, + "theoretical_loss": 4.572740744460207, + "tokens_seen": 143654912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728946840521565, + "loss": 1.9639, + "theoretical_loss": 4.572468056681634, + "tokens_seen": 143720448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728786359077231, + "loss": 1.8062, + "theoretical_loss": 4.57219552801776, + "tokens_seen": 143785984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728625877632899, + "loss": 1.791, + "theoretical_loss": 4.57192315830328, + "tokens_seen": 143851520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728465396188565, + "loss": 1.7563, + "theoretical_loss": 4.571650947373138, + "tokens_seen": 143917056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728304914744233, + "loss": 1.8727, + "theoretical_loss": 4.571378895062525, + "tokens_seen": 143982592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728144433299899, + "loss": 1.7582, + "theoretical_loss": 4.571107001206875, + "tokens_seen": 144048128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727983951855567, + "loss": 1.885, + "theoretical_loss": 4.570835265641872, + "tokens_seen": 144113664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 54793, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8981997966766357, + "objective/train/theoretical_loss": 4.57056368820344, + "objective/train/tokens_used": 164639200, + "theoretical_loss": 4.57056368820344, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727823470411233, + "loss": 1.7713, + "theoretical_loss": 4.57056368820344, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727662988966901, + "loss": 1.722, + "theoretical_loss": 4.570292268727751, + "tokens_seen": 144244736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727502507522567, + "loss": 1.7943, + "theoretical_loss": 4.570021007051221, + "tokens_seen": 144310272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727342026078235, + "loss": 1.7481, + "theoretical_loss": 4.569749903010507, + "tokens_seen": 144375808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727181544633901, + "loss": 1.7384, + "theoretical_loss": 4.569478956442513, + "tokens_seen": 144441344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727021063189569, + "loss": 2.0361, + "theoretical_loss": 4.569208167184382, + "tokens_seen": 144506880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726860581745235, + "loss": 1.7303, + "theoretical_loss": 4.5689375350735, + "tokens_seen": 144572416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726700100300903, + "loss": 1.7676, + "theoretical_loss": 4.568667059947496, + "tokens_seen": 144637952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726539618856569, + "loss": 1.8165, + "theoretical_loss": 4.568396741644241, + "tokens_seen": 144703488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726379137412237, + "loss": 1.8475, + "theoretical_loss": 4.568126580001843, + "tokens_seen": 144769024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726218655967903, + "loss": 1.6271, + "theoretical_loss": 4.567856574858656, + "tokens_seen": 144834560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726058174523571, + "loss": 1.9387, + "theoretical_loss": 4.567586726053266, + "tokens_seen": 144900096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725897693079237, + "loss": 1.7635, + "theoretical_loss": 4.567317033424507, + "tokens_seen": 144965632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725737211634905, + "loss": 1.7297, + "theoretical_loss": 4.567047496811445, + "tokens_seen": 145031168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725576730190571, + "loss": 1.8948, + "theoretical_loss": 4.566778116053389, + "tokens_seen": 145096704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725416248746239, + "loss": 1.9421, + "theoretical_loss": 4.566508890989883, + "tokens_seen": 145162240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725255767301905, + "loss": 1.8752, + "theoretical_loss": 4.566239821460712, + "tokens_seen": 145227776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725095285857573, + "loss": 1.7984, + "theoretical_loss": 4.5659709073058945, + "tokens_seen": 145293312 + }, + { + "epoch": 0.04, + "learning_rate": 0.000772493480441324, + "loss": 1.8638, + "theoretical_loss": 4.565702148365686, + "tokens_seen": 145358848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724774322968907, + "loss": 1.7664, + "theoretical_loss": 4.56543354448058, + "tokens_seen": 145424384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724613841524574, + "loss": 1.842, + "theoretical_loss": 4.565165095491306, + "tokens_seen": 145489920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724453360080241, + "loss": 1.7626, + "theoretical_loss": 4.5648968012388265, + "tokens_seen": 145555456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724292878635908, + "loss": 1.7205, + "theoretical_loss": 4.56462866156434, + "tokens_seen": 145620992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724132397191575, + "loss": 1.7857, + "theoretical_loss": 4.564360676309279, + "tokens_seen": 145686528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723971915747242, + "loss": 1.8482, + "theoretical_loss": 4.564092845315312, + "tokens_seen": 145752064 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723811434302909, + "loss": 1.7422, + "theoretical_loss": 4.563825168424337, + "tokens_seen": 145817600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723650952858577, + "loss": 1.7175, + "theoretical_loss": 4.563557645478488, + "tokens_seen": 145883136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723490471414243, + "loss": 1.8653, + "theoretical_loss": 4.56329027632013, + "tokens_seen": 145948672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723329989969911, + "loss": 1.7809, + "theoretical_loss": 4.563023060791862, + "tokens_seen": 146014208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723169508525577, + "loss": 1.729, + "theoretical_loss": 4.562755998736511, + "tokens_seen": 146079744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723009027081245, + "loss": 1.7853, + "theoretical_loss": 4.562489089997141, + "tokens_seen": 146145280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722848545636911, + "loss": 1.7846, + "theoretical_loss": 4.562222334417041, + "tokens_seen": 146210816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722688064192579, + "loss": 1.7191, + "theoretical_loss": 4.561955731839734, + "tokens_seen": 146276352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722527582748245, + "loss": 1.8433, + "theoretical_loss": 4.561689282108971, + "tokens_seen": 146341888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722367101303913, + "loss": 1.818, + "theoretical_loss": 4.561422985068733, + "tokens_seen": 146407424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722206619859579, + "loss": 1.7675, + "theoretical_loss": 4.561156840563231, + "tokens_seen": 146472960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722046138415247, + "loss": 1.836, + "theoretical_loss": 4.560890848436902, + "tokens_seen": 146538496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721885656970913, + "loss": 1.8614, + "theoretical_loss": 4.560625008534414, + "tokens_seen": 146604032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721725175526581, + "loss": 1.824, + "theoretical_loss": 4.560359320700662, + "tokens_seen": 146669568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721564694082247, + "loss": 1.7436, + "theoretical_loss": 4.560093784780766, + "tokens_seen": 146735104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721404212637915, + "loss": 1.8323, + "theoretical_loss": 4.559828400620075, + "tokens_seen": 146800640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721243731193581, + "loss": 1.89, + "theoretical_loss": 4.559563168064165, + "tokens_seen": 146866176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721083249749249, + "loss": 1.9487, + "theoretical_loss": 4.559298086958837, + "tokens_seen": 146931712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720922768304915, + "loss": 1.6885, + "theoretical_loss": 4.559033157150115, + "tokens_seen": 146997248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720762286860583, + "loss": 1.8212, + "theoretical_loss": 4.5587683784842525, + "tokens_seen": 147062784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720601805416249, + "loss": 1.7993, + "theoretical_loss": 4.558503750807725, + "tokens_seen": 147128320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720441323971917, + "loss": 1.7828, + "theoretical_loss": 4.558239273967234, + "tokens_seen": 147193856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720280842527583, + "loss": 1.8936, + "theoretical_loss": 4.5579749478097025, + "tokens_seen": 147259392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720120361083251, + "loss": 1.8044, + "theoretical_loss": 4.557710772182278, + "tokens_seen": 147324928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719959879638917, + "loss": 1.8211, + "theoretical_loss": 4.5574467469323325, + "tokens_seen": 147390464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 56056, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8968627452850342, + "objective/train/theoretical_loss": 4.557182871907457, + "objective/train/tokens_used": 167916000, + "theoretical_loss": 4.557182871907457, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719799398194585, + "loss": 1.8249, + "theoretical_loss": 4.557182871907457, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719638916750251, + "loss": 1.7435, + "theoretical_loss": 4.556919146955469, + "tokens_seen": 147521536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719478435305919, + "loss": 1.7664, + "theoretical_loss": 4.556655571924404, + "tokens_seen": 147587072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719317953861585, + "loss": 1.8627, + "theoretical_loss": 4.556392146662521, + "tokens_seen": 147652608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719157472417253, + "loss": 1.8259, + "theoretical_loss": 4.556128871018297, + "tokens_seen": 147718144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718996990972919, + "loss": 2.0033, + "theoretical_loss": 4.555865744840434, + "tokens_seen": 147783680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718836509528587, + "loss": 1.698, + "theoretical_loss": 4.55560276797785, + "tokens_seen": 147849216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718676028084253, + "loss": 1.7972, + "theoretical_loss": 4.5553399402796835, + "tokens_seen": 147914752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718515546639921, + "loss": 1.7489, + "theoretical_loss": 4.5550772615952955, + "tokens_seen": 147980288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718355065195587, + "loss": 1.8099, + "theoretical_loss": 4.554814731774259, + "tokens_seen": 148045824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718194583751255, + "loss": 1.8885, + "theoretical_loss": 4.554552350666373, + "tokens_seen": 148111360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718034102306921, + "loss": 1.8405, + "theoretical_loss": 4.554290118121649, + "tokens_seen": 148176896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007717873620862589, + "loss": 1.8405, + "theoretical_loss": 4.554028033990319, + "tokens_seen": 148242432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007717713139418255, + "loss": 1.81, + "theoretical_loss": 4.5537660981228285, + "tokens_seen": 148307968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007717552657973923, + "loss": 1.8125, + "theoretical_loss": 4.553504310369844, + "tokens_seen": 148373504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007717392176529589, + "loss": 1.8753, + "theoretical_loss": 4.553242670582245, + "tokens_seen": 148439040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007717231695085256, + "loss": 1.9341, + "theoretical_loss": 4.55298117861113, + "tokens_seen": 148504576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007717071213640923, + "loss": 1.7652, + "theoretical_loss": 4.55271983430781, + "tokens_seen": 148570112 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771691073219659, + "loss": 1.9011, + "theoretical_loss": 4.552458637523815, + "tokens_seen": 148635648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716750250752257, + "loss": 1.7245, + "theoretical_loss": 4.552197588110882, + "tokens_seen": 148701184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716589769307924, + "loss": 1.736, + "theoretical_loss": 4.551936685920971, + "tokens_seen": 148766720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716429287863591, + "loss": 1.6701, + "theoretical_loss": 4.551675930806251, + "tokens_seen": 148832256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716268806419258, + "loss": 1.8453, + "theoretical_loss": 4.551415322619107, + "tokens_seen": 148897792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716108324974925, + "loss": 1.8299, + "theoretical_loss": 4.551154861212133, + "tokens_seen": 148963328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715947843530592, + "loss": 1.97, + "theoretical_loss": 4.550894546438139, + "tokens_seen": 149028864 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715787362086258, + "loss": 1.7303, + "theoretical_loss": 4.550634378150148, + "tokens_seen": 149094400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715626880641926, + "loss": 1.8074, + "theoretical_loss": 4.550374356201392, + "tokens_seen": 149159936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715466399197592, + "loss": 1.8377, + "theoretical_loss": 4.550114480445316, + "tokens_seen": 149225472 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771530591775326, + "loss": 1.8604, + "theoretical_loss": 4.549854750735575, + "tokens_seen": 149291008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715145436308926, + "loss": 1.9376, + "theoretical_loss": 4.5495951669260375, + "tokens_seen": 149356544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714984954864594, + "loss": 1.8481, + "theoretical_loss": 4.54933572887078, + "tokens_seen": 149422080 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771482447342026, + "loss": 1.8748, + "theoretical_loss": 4.5490764364240865, + "tokens_seen": 149487616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714663991975928, + "loss": 1.7571, + "theoretical_loss": 4.548817289440455, + "tokens_seen": 149553152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714503510531594, + "loss": 1.7867, + "theoretical_loss": 4.548558287774592, + "tokens_seen": 149618688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714343029087262, + "loss": 1.857, + "theoretical_loss": 4.548299431281409, + "tokens_seen": 149684224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714182547642928, + "loss": 1.773, + "theoretical_loss": 4.548040719816029, + "tokens_seen": 149749760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714022066198596, + "loss": 1.8777, + "theoretical_loss": 4.547782153233783, + "tokens_seen": 149815296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007713861584754262, + "loss": 1.7381, + "theoretical_loss": 4.547523731390206, + "tokens_seen": 149880832 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771370110330993, + "loss": 1.7556, + "theoretical_loss": 4.547265454141046, + "tokens_seen": 149946368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007713540621865596, + "loss": 1.7892, + "theoretical_loss": 4.547007321342251, + "tokens_seen": 150011904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007713380140421264, + "loss": 1.8294, + "theoretical_loss": 4.546749332849982, + "tokens_seen": 150077440 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771321965897693, + "loss": 1.7976, + "theoretical_loss": 4.546491488520601, + "tokens_seen": 150142976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007713059177532598, + "loss": 1.7639, + "theoretical_loss": 4.546233788210679, + "tokens_seen": 150208512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712898696088264, + "loss": 1.7777, + "theoretical_loss": 4.545976231776988, + "tokens_seen": 150274048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712738214643932, + "loss": 1.8345, + "theoretical_loss": 4.545718819076509, + "tokens_seen": 150339584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712577733199598, + "loss": 1.8569, + "theoretical_loss": 4.545461549966426, + "tokens_seen": 150405120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712417251755266, + "loss": 1.7723, + "theoretical_loss": 4.545204424304126, + "tokens_seen": 150470656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712256770310932, + "loss": 1.7852, + "theoretical_loss": 4.544947441947201, + "tokens_seen": 150536192 + }, + { + "epoch": 0.05, + "learning_rate": 0.00077120962888666, + "loss": 1.7586, + "theoretical_loss": 4.544690602753445, + "tokens_seen": 150601728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711935807422266, + "loss": 1.8189, + "theoretical_loss": 4.544433906580857, + "tokens_seen": 150667264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 56915, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7465159893035889, + "objective/train/theoretical_loss": 4.544177353287637, + "objective/train/tokens_used": 171192800, + "theoretical_loss": 4.544177353287637, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711775325977934, + "loss": 1.7764, + "theoretical_loss": 4.544177353287637, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711614844533602, + "loss": 1.8765, + "theoretical_loss": 4.543920942732187, + "tokens_seen": 150798336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711454363089268, + "loss": 1.7625, + "theoretical_loss": 4.543664674773112, + "tokens_seen": 150863872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711293881644936, + "loss": 1.8125, + "theoretical_loss": 4.543408549269218, + "tokens_seen": 150929408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711133400200602, + "loss": 1.7914, + "theoretical_loss": 4.54315256607951, + "tokens_seen": 150994944 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771097291875627, + "loss": 1.8573, + "theoretical_loss": 4.542896725063195, + "tokens_seen": 151060480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710812437311936, + "loss": 1.6647, + "theoretical_loss": 4.542641026079683, + "tokens_seen": 151126016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710651955867604, + "loss": 1.8338, + "theoretical_loss": 4.542385468988581, + "tokens_seen": 151191552 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771049147442327, + "loss": 1.8271, + "theoretical_loss": 4.542130053649695, + "tokens_seen": 151257088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710330992978938, + "loss": 1.6977, + "theoretical_loss": 4.541874779923031, + "tokens_seen": 151322624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710170511534604, + "loss": 1.7345, + "theoretical_loss": 4.541619647668796, + "tokens_seen": 151388160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710010030090272, + "loss": 1.9003, + "theoretical_loss": 4.541364656747392, + "tokens_seen": 151453696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709849548645938, + "loss": 1.7489, + "theoretical_loss": 4.541109807019422, + "tokens_seen": 151519232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709689067201606, + "loss": 1.7527, + "theoretical_loss": 4.540855098345683, + "tokens_seen": 151584768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709528585757272, + "loss": 1.784, + "theoretical_loss": 4.540600530587175, + "tokens_seen": 151650304 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770936810431294, + "loss": 1.7235, + "theoretical_loss": 4.5403461036050885, + "tokens_seen": 151715840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709207622868606, + "loss": 1.838, + "theoretical_loss": 4.540091817260815, + "tokens_seen": 151781376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709047141424274, + "loss": 1.8029, + "theoretical_loss": 4.539837671415941, + "tokens_seen": 151846912 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770888665997994, + "loss": 1.7071, + "theoretical_loss": 4.539583665932249, + "tokens_seen": 151912448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708726178535608, + "loss": 1.7128, + "theoretical_loss": 4.539329800671718, + "tokens_seen": 151977984 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708565697091274, + "loss": 1.7997, + "theoretical_loss": 4.539076075496519, + "tokens_seen": 152043520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708405215646942, + "loss": 1.9835, + "theoretical_loss": 4.538822490269021, + "tokens_seen": 152109056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708244734202608, + "loss": 1.7025, + "theoretical_loss": 4.538569044851785, + "tokens_seen": 152174592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708084252758276, + "loss": 1.7776, + "theoretical_loss": 4.538315739107571, + "tokens_seen": 152240128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007707923771313942, + "loss": 1.63, + "theoretical_loss": 4.5380625728993245, + "tokens_seen": 152305664 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770776328986961, + "loss": 1.736, + "theoretical_loss": 4.537809546090193, + "tokens_seen": 152371200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007707602808425276, + "loss": 1.8733, + "theoretical_loss": 4.53755665854351, + "tokens_seen": 152436736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007707442326980944, + "loss": 1.73, + "theoretical_loss": 4.537303910122807, + "tokens_seen": 152502272 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770728184553661, + "loss": 1.753, + "theoretical_loss": 4.537051300691805, + "tokens_seen": 152567808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007707121364092278, + "loss": 1.7725, + "theoretical_loss": 4.5367988301144155, + "tokens_seen": 152633344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706960882647944, + "loss": 1.7438, + "theoretical_loss": 4.536546498254746, + "tokens_seen": 152698880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706800401203612, + "loss": 1.8448, + "theoretical_loss": 4.536294304977092, + "tokens_seen": 152764416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706639919759278, + "loss": 1.76, + "theoretical_loss": 4.536042250145941, + "tokens_seen": 152829952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706479438314946, + "loss": 1.5725, + "theoretical_loss": 4.53579033362597, + "tokens_seen": 152895488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706318956870612, + "loss": 1.8505, + "theoretical_loss": 4.535538555282048, + "tokens_seen": 152961024 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770615847542628, + "loss": 1.9077, + "theoretical_loss": 4.535286914979232, + "tokens_seen": 153026560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705997993981946, + "loss": 1.7547, + "theoretical_loss": 4.5350354125827685, + "tokens_seen": 153092096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705837512537614, + "loss": 1.747, + "theoretical_loss": 4.534784047958096, + "tokens_seen": 153157632 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770567703109328, + "loss": 1.7225, + "theoretical_loss": 4.534532820970839, + "tokens_seen": 153223168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705516549648948, + "loss": 1.7695, + "theoretical_loss": 4.534281731486812, + "tokens_seen": 153288704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705356068204614, + "loss": 1.7913, + "theoretical_loss": 4.5340307793720145, + "tokens_seen": 153354240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705195586760282, + "loss": 1.703, + "theoretical_loss": 4.533779964492638, + "tokens_seen": 153419776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705035105315948, + "loss": 1.695, + "theoretical_loss": 4.533529286715059, + "tokens_seen": 153485312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007704874623871616, + "loss": 1.7663, + "theoretical_loss": 4.533278745905843, + "tokens_seen": 153550848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007704714142427282, + "loss": 1.6136, + "theoretical_loss": 4.533028341931738, + "tokens_seen": 153616384 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770455366098295, + "loss": 1.9402, + "theoretical_loss": 4.532778074659685, + "tokens_seen": 153681920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007704393179538616, + "loss": 1.8058, + "theoretical_loss": 4.532527943956804, + "tokens_seen": 153747456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007704232698094284, + "loss": 1.8964, + "theoretical_loss": 4.532277949690407, + "tokens_seen": 153812992 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770407221664995, + "loss": 1.6986, + "theoretical_loss": 4.532028091727986, + "tokens_seen": 153878528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703911735205618, + "loss": 1.6689, + "theoretical_loss": 4.531778369937222, + "tokens_seen": 153944064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 58041, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8966615200042725, + "objective/train/theoretical_loss": 4.53152878418598, + "objective/train/tokens_used": 174469600, + "theoretical_loss": 4.53152878418598, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703751253761284, + "loss": 1.7864, + "theoretical_loss": 4.53152878418598, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703590772316952, + "loss": 1.724, + "theoretical_loss": 4.531279334342306, + "tokens_seen": 154075136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703430290872618, + "loss": 1.7493, + "theoretical_loss": 4.531030020274436, + "tokens_seen": 154140672 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703269809428285, + "loss": 1.7452, + "theoretical_loss": 4.530780841850783, + "tokens_seen": 154206208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703109327983952, + "loss": 1.7928, + "theoretical_loss": 4.5305317989399505, + "tokens_seen": 154271744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702948846539619, + "loss": 1.7797, + "theoretical_loss": 4.530282891410717, + "tokens_seen": 154337280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702788365095286, + "loss": 1.7927, + "theoretical_loss": 4.530034119132052, + "tokens_seen": 154402816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702627883650953, + "loss": 1.7336, + "theoretical_loss": 4.529785481973102, + "tokens_seen": 154468352 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770246740220662, + "loss": 1.7642, + "theoretical_loss": 4.529536979803195, + "tokens_seen": 154533888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702306920762287, + "loss": 1.9063, + "theoretical_loss": 4.529288612491846, + "tokens_seen": 154599424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702146439317954, + "loss": 1.6956, + "theoretical_loss": 4.5290403799087455, + "tokens_seen": 154664960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701985957873621, + "loss": 1.9776, + "theoretical_loss": 4.528792281923769, + "tokens_seen": 154730496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701825476429287, + "loss": 1.8414, + "theoretical_loss": 4.52854431840697, + "tokens_seen": 154796032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701664994984955, + "loss": 1.8142, + "theoretical_loss": 4.528296489228585, + "tokens_seen": 154861568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701504513540621, + "loss": 1.6601, + "theoretical_loss": 4.528048794259028, + "tokens_seen": 154927104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701344032096289, + "loss": 1.8983, + "theoretical_loss": 4.527801233368896, + "tokens_seen": 154992640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701183550651955, + "loss": 1.7475, + "theoretical_loss": 4.527553806428962, + "tokens_seen": 155058176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701023069207623, + "loss": 1.8392, + "theoretical_loss": 4.5273065133101795, + "tokens_seen": 155123712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700862587763289, + "loss": 1.7844, + "theoretical_loss": 4.527059353883682, + "tokens_seen": 155189248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700702106318957, + "loss": 1.8887, + "theoretical_loss": 4.526812328020778, + "tokens_seen": 155254784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700541624874623, + "loss": 1.736, + "theoretical_loss": 4.52656543559296, + "tokens_seen": 155320320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700381143430291, + "loss": 1.7517, + "theoretical_loss": 4.526318676471891, + "tokens_seen": 155385856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700220661985957, + "loss": 1.7906, + "theoretical_loss": 4.526072050529417, + "tokens_seen": 155451392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700060180541625, + "loss": 1.8339, + "theoretical_loss": 4.52582555763756, + "tokens_seen": 155516928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699899699097291, + "loss": 1.8402, + "theoretical_loss": 4.525579197668517, + "tokens_seen": 155582464 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699739217652959, + "loss": 1.8406, + "theoretical_loss": 4.525332970494663, + "tokens_seen": 155648000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699578736208627, + "loss": 1.793, + "theoretical_loss": 4.525086875988549, + "tokens_seen": 155713536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699418254764293, + "loss": 1.8599, + "theoretical_loss": 4.524840914022902, + "tokens_seen": 155779072 + }, + { + "epoch": 0.05, + "learning_rate": 0.000769925777331996, + "loss": 1.7589, + "theoretical_loss": 4.524595084470625, + "tokens_seen": 155844608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699097291875627, + "loss": 1.8455, + "theoretical_loss": 4.524349387204794, + "tokens_seen": 155910144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698936810431295, + "loss": 1.6923, + "theoretical_loss": 4.524103822098664, + "tokens_seen": 155975680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698776328986961, + "loss": 1.7622, + "theoretical_loss": 4.52385838902566, + "tokens_seen": 156041216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698615847542629, + "loss": 1.6998, + "theoretical_loss": 4.523613087859386, + "tokens_seen": 156106752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698455366098295, + "loss": 1.7923, + "theoretical_loss": 4.523367918473614, + "tokens_seen": 156172288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698294884653963, + "loss": 1.814, + "theoretical_loss": 4.523122880742298, + "tokens_seen": 156237824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698134403209629, + "loss": 1.7696, + "theoretical_loss": 4.522877974539556, + "tokens_seen": 156303360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697973921765297, + "loss": 1.8467, + "theoretical_loss": 4.522633199739687, + "tokens_seen": 156368896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697813440320963, + "loss": 1.7256, + "theoretical_loss": 4.522388556217157, + "tokens_seen": 156434432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697652958876631, + "loss": 1.7527, + "theoretical_loss": 4.522144043846609, + "tokens_seen": 156499968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697492477432297, + "loss": 1.784, + "theoretical_loss": 4.521899662502856, + "tokens_seen": 156565504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697331995987965, + "loss": 1.7029, + "theoretical_loss": 4.5216554120608805, + "tokens_seen": 156631040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697171514543631, + "loss": 1.7115, + "theoretical_loss": 4.521411292395841, + "tokens_seen": 156696576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697011033099299, + "loss": 1.7872, + "theoretical_loss": 4.521167303383065, + "tokens_seen": 156762112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696850551654965, + "loss": 1.6923, + "theoretical_loss": 4.520923444898051, + "tokens_seen": 156827648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696690070210633, + "loss": 1.7266, + "theoretical_loss": 4.520679716816467, + "tokens_seen": 156893184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696529588766299, + "loss": 1.7961, + "theoretical_loss": 4.520436119014155, + "tokens_seen": 156958720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696369107321967, + "loss": 1.7192, + "theoretical_loss": 4.520192651367121, + "tokens_seen": 157024256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696208625877633, + "loss": 1.5909, + "theoretical_loss": 4.519949313751547, + "tokens_seen": 157089792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696048144433301, + "loss": 1.8818, + "theoretical_loss": 4.519706106043779, + "tokens_seen": 157155328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695887662988967, + "loss": 1.7198, + "theoretical_loss": 4.519463028120338, + "tokens_seen": 157220864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 59100, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.76185941696167, + "objective/train/theoretical_loss": 4.519220079857908, + "objective/train/tokens_used": 177746400, + "theoretical_loss": 4.519220079857908, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695727181544635, + "loss": 1.7576, + "theoretical_loss": 4.519220079857908, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695566700100301, + "loss": 1.7981, + "theoretical_loss": 4.518977261133343, + "tokens_seen": 157351936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695406218655969, + "loss": 1.6934, + "theoretical_loss": 4.518734571823668, + "tokens_seen": 157417472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695245737211635, + "loss": 1.7905, + "theoretical_loss": 4.518492011806073, + "tokens_seen": 157483008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695085255767303, + "loss": 1.7677, + "theoretical_loss": 4.518249580957917, + "tokens_seen": 157548544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694924774322969, + "loss": 1.8178, + "theoretical_loss": 4.518007279156725, + "tokens_seen": 157614080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694764292878637, + "loss": 1.8327, + "theoretical_loss": 4.5177651062801925, + "tokens_seen": 157679616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694603811434303, + "loss": 1.8089, + "theoretical_loss": 4.517523062206175, + "tokens_seen": 157745152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694443329989971, + "loss": 1.7748, + "theoretical_loss": 4.517281146812701, + "tokens_seen": 157810688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694282848545637, + "loss": 1.8415, + "theoretical_loss": 4.517039359977961, + "tokens_seen": 157876224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694122367101305, + "loss": 1.8614, + "theoretical_loss": 4.516797701580314, + "tokens_seen": 157941760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693961885656971, + "loss": 1.6887, + "theoretical_loss": 4.516556171498284, + "tokens_seen": 158007296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693801404212639, + "loss": 1.6983, + "theoretical_loss": 4.516314769610557, + "tokens_seen": 158072832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693640922768305, + "loss": 1.693, + "theoretical_loss": 4.516073495795988, + "tokens_seen": 158138368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693480441323973, + "loss": 1.8146, + "theoretical_loss": 4.5158323499335955, + "tokens_seen": 158203904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693319959879639, + "loss": 1.7686, + "theoretical_loss": 4.515591331902561, + "tokens_seen": 158269440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693159478435307, + "loss": 1.6949, + "theoretical_loss": 4.515350441582231, + "tokens_seen": 158334976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692998996990973, + "loss": 1.8541, + "theoretical_loss": 4.515109678852117, + "tokens_seen": 158400512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692838515546641, + "loss": 1.9401, + "theoretical_loss": 4.514869043591891, + "tokens_seen": 158466048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692678034102307, + "loss": 1.7639, + "theoretical_loss": 4.514628535681391, + "tokens_seen": 158531584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692517552657975, + "loss": 1.7224, + "theoretical_loss": 4.514388155000615, + "tokens_seen": 158597120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692357071213641, + "loss": 1.76, + "theoretical_loss": 4.5141479014297285, + "tokens_seen": 158662656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692196589769309, + "loss": 1.8276, + "theoretical_loss": 4.513907774849054, + "tokens_seen": 158728192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692036108324975, + "loss": 1.8055, + "theoretical_loss": 4.513667775139078, + "tokens_seen": 158793728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691875626880643, + "loss": 1.7416, + "theoretical_loss": 4.5134279021804495, + "tokens_seen": 158859264 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691715145436309, + "loss": 1.675, + "theoretical_loss": 4.513188155853978, + "tokens_seen": 158924800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691554663991977, + "loss": 1.8391, + "theoretical_loss": 4.512948536040636, + "tokens_seen": 158990336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691394182547643, + "loss": 1.8972, + "theoretical_loss": 4.512709042621554, + "tokens_seen": 159055872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691233701103311, + "loss": 1.7549, + "theoretical_loss": 4.512469675478026, + "tokens_seen": 159121408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691073219658977, + "loss": 1.7728, + "theoretical_loss": 4.512230434491503, + "tokens_seen": 159186944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690912738214645, + "loss": 1.8985, + "theoretical_loss": 4.511991319543599, + "tokens_seen": 159252480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690752256770311, + "loss": 1.7964, + "theoretical_loss": 4.511752330516086, + "tokens_seen": 159318016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690591775325979, + "loss": 1.658, + "theoretical_loss": 4.5115134672908965, + "tokens_seen": 159383552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690431293881645, + "loss": 1.9224, + "theoretical_loss": 4.511274729750123, + "tokens_seen": 159449088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690270812437313, + "loss": 1.7319, + "theoretical_loss": 4.5110361177760145, + "tokens_seen": 159514624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690110330992979, + "loss": 1.7229, + "theoretical_loss": 4.5107976312509805, + "tokens_seen": 159580160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689949849548647, + "loss": 1.7498, + "theoretical_loss": 4.510559270057587, + "tokens_seen": 159645696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689789368104313, + "loss": 1.7295, + "theoretical_loss": 4.51032103407856, + "tokens_seen": 159711232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689628886659981, + "loss": 1.7314, + "theoretical_loss": 4.510082923196784, + "tokens_seen": 159776768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689468405215647, + "loss": 1.7616, + "theoretical_loss": 4.5098449372952985, + "tokens_seen": 159842304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689307923771315, + "loss": 1.7165, + "theoretical_loss": 4.5096070762573, + "tokens_seen": 159907840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689147442326981, + "loss": 1.7341, + "theoretical_loss": 4.509369339966146, + "tokens_seen": 159973376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688986960882648, + "loss": 1.7801, + "theoretical_loss": 4.509131728305347, + "tokens_seen": 160038912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688826479438315, + "loss": 1.9064, + "theoretical_loss": 4.50889424115857, + "tokens_seen": 160104448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688665997993982, + "loss": 1.6784, + "theoretical_loss": 4.50865687840964, + "tokens_seen": 160169984 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688505516549649, + "loss": 1.7526, + "theoretical_loss": 4.508419639942538, + "tokens_seen": 160235520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688345035105316, + "loss": 1.7459, + "theoretical_loss": 4.5081825256413985, + "tokens_seen": 160301056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688184553660983, + "loss": 1.8808, + "theoretical_loss": 4.507945535390512, + "tokens_seen": 160366592 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768802407221665, + "loss": 1.816, + "theoretical_loss": 4.507708669074326, + "tokens_seen": 160432128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687863590772316, + "loss": 1.7083, + "theoretical_loss": 4.5074719265774394, + "tokens_seen": 160497664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 60314, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.1015968322753906, + "objective/train/theoretical_loss": 4.507235307784608, + "objective/train/tokens_used": 181023200, + "theoretical_loss": 4.507235307784608, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687703109327984, + "loss": 1.9357, + "theoretical_loss": 4.507235307784608, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687542627883651, + "loss": 1.6547, + "theoretical_loss": 4.5069988125807425, + "tokens_seen": 160628736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687382146439318, + "loss": 1.7709, + "theoretical_loss": 4.506762440850906, + "tokens_seen": 160694272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687221664994985, + "loss": 1.7327, + "theoretical_loss": 4.506526192480315, + "tokens_seen": 160759808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687061183550652, + "loss": 1.7753, + "theoretical_loss": 4.506290067354341, + "tokens_seen": 160825344 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768690070210632, + "loss": 1.7335, + "theoretical_loss": 4.506054065358508, + "tokens_seen": 160890880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007686740220661986, + "loss": 1.7166, + "theoretical_loss": 4.505818186378491, + "tokens_seen": 160956416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007686579739217653, + "loss": 1.6904, + "theoretical_loss": 4.505582430300121, + "tokens_seen": 161021952 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768641925777332, + "loss": 1.8413, + "theoretical_loss": 4.505346797009381, + "tokens_seen": 161087488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007686258776328987, + "loss": 1.7868, + "theoretical_loss": 4.505111286392402, + "tokens_seen": 161153024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007686098294884654, + "loss": 1.6634, + "theoretical_loss": 4.50487589833547, + "tokens_seen": 161218560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685937813440322, + "loss": 1.6353, + "theoretical_loss": 4.504640632725025, + "tokens_seen": 161284096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685777331995988, + "loss": 1.7892, + "theoretical_loss": 4.504405489447654, + "tokens_seen": 161349632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685616850551656, + "loss": 1.7234, + "theoretical_loss": 4.504170468390096, + "tokens_seen": 161415168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685456369107322, + "loss": 1.6762, + "theoretical_loss": 4.503935569439242, + "tokens_seen": 161480704 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768529588766299, + "loss": 1.8172, + "theoretical_loss": 4.5037007924821335, + "tokens_seen": 161546240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685135406218656, + "loss": 1.7786, + "theoretical_loss": 4.503466137405961, + "tokens_seen": 161611776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684974924774324, + "loss": 1.732, + "theoretical_loss": 4.503231604098065, + "tokens_seen": 161677312 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768481444332999, + "loss": 1.8209, + "theoretical_loss": 4.50299719244594, + "tokens_seen": 161742848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684653961885658, + "loss": 1.8043, + "theoretical_loss": 4.502762902337222, + "tokens_seen": 161808384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684493480441324, + "loss": 1.8455, + "theoretical_loss": 4.502528733659704, + "tokens_seen": 161873920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684332998996992, + "loss": 1.7825, + "theoretical_loss": 4.502294686301323, + "tokens_seen": 161939456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684172517552658, + "loss": 1.6838, + "theoretical_loss": 4.502060760150167, + "tokens_seen": 162004992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684012036108326, + "loss": 1.8061, + "theoretical_loss": 4.501826955094472, + "tokens_seen": 162070528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007683851554663992, + "loss": 1.6355, + "theoretical_loss": 4.501593271022623, + "tokens_seen": 162136064 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768369107321966, + "loss": 1.7908, + "theoretical_loss": 4.501359707823149, + "tokens_seen": 162201600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007683530591775326, + "loss": 1.7182, + "theoretical_loss": 4.501126265384732, + "tokens_seen": 162267136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007683370110330994, + "loss": 1.9053, + "theoretical_loss": 4.500892943596199, + "tokens_seen": 162332672 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768320962888666, + "loss": 1.8499, + "theoretical_loss": 4.500659742346524, + "tokens_seen": 162398208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007683049147442328, + "loss": 1.7459, + "theoretical_loss": 4.500426661524828, + "tokens_seen": 162463744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682888665997994, + "loss": 1.774, + "theoretical_loss": 4.500193701020379, + "tokens_seen": 162529280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682728184553662, + "loss": 1.8263, + "theoretical_loss": 4.499960860722592, + "tokens_seen": 162594816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682567703109328, + "loss": 1.7356, + "theoretical_loss": 4.499728140521025, + "tokens_seen": 162660352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682407221664996, + "loss": 1.8387, + "theoretical_loss": 4.499495540305388, + "tokens_seen": 162725888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682246740220662, + "loss": 1.6331, + "theoretical_loss": 4.49926305996553, + "tokens_seen": 162791424 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768208625877633, + "loss": 1.7426, + "theoretical_loss": 4.499030699391449, + "tokens_seen": 162856960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681925777331996, + "loss": 1.617, + "theoretical_loss": 4.49879845847329, + "tokens_seen": 162922496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681765295887664, + "loss": 1.7124, + "theoretical_loss": 4.498566337101337, + "tokens_seen": 162988032 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768160481444333, + "loss": 1.7712, + "theoretical_loss": 4.4983343351660245, + "tokens_seen": 163053568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681444332998998, + "loss": 1.9108, + "theoretical_loss": 4.498102452557928, + "tokens_seen": 163119104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681283851554664, + "loss": 1.6287, + "theoretical_loss": 4.497870689167771, + "tokens_seen": 163184640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681123370110332, + "loss": 1.8195, + "theoretical_loss": 4.497639044886412, + "tokens_seen": 163250176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680962888665998, + "loss": 1.7055, + "theoretical_loss": 4.497407519604866, + "tokens_seen": 163315712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680802407221666, + "loss": 1.8599, + "theoretical_loss": 4.49717611321428, + "tokens_seen": 163381248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680641925777332, + "loss": 1.6922, + "theoretical_loss": 4.496944825605951, + "tokens_seen": 163446784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680481444333, + "loss": 1.6333, + "theoretical_loss": 4.496713656671317, + "tokens_seen": 163512320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680320962888666, + "loss": 1.8145, + "theoretical_loss": 4.496482606301957, + "tokens_seen": 163577856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680160481444334, + "loss": 1.7066, + "theoretical_loss": 4.496251674389596, + "tokens_seen": 163643392 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768, + "loss": 1.7063, + "theoretical_loss": 4.496020860826098, + "tokens_seen": 163708928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679839518555668, + "loss": 1.7411, + "theoretical_loss": 4.49579016550347, + "tokens_seen": 163774464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 61072, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7547411918640137, + "objective/train/theoretical_loss": 4.49555958831386, + "objective/train/tokens_used": 184300000, + "theoretical_loss": 4.49555958831386, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679679037111334, + "loss": 1.7758, + "theoretical_loss": 4.49555958831386, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679518555667002, + "loss": 1.7571, + "theoretical_loss": 4.49532912914956, + "tokens_seen": 163905536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679358074222668, + "loss": 1.7834, + "theoretical_loss": 4.495098787903, + "tokens_seen": 163971072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679197592778336, + "loss": 1.7879, + "theoretical_loss": 4.4948685644667545, + "tokens_seen": 164036608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679037111334002, + "loss": 1.6408, + "theoretical_loss": 4.494638458733535, + "tokens_seen": 164102144 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767887662988967, + "loss": 1.6375, + "theoretical_loss": 4.494408470596195, + "tokens_seen": 164167680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007678716148445336, + "loss": 1.7134, + "theoretical_loss": 4.494178599947729, + "tokens_seen": 164233216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007678555667001004, + "loss": 1.8468, + "theoretical_loss": 4.4939488466812705, + "tokens_seen": 164298752 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767839518555667, + "loss": 1.8571, + "theoretical_loss": 4.493719210690093, + "tokens_seen": 164364288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007678234704112338, + "loss": 1.711, + "theoretical_loss": 4.493489691867612, + "tokens_seen": 164429824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007678074222668004, + "loss": 1.6744, + "theoretical_loss": 4.493260290107379, + "tokens_seen": 164495360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677913741223672, + "loss": 1.6627, + "theoretical_loss": 4.493031005303084, + "tokens_seen": 164560896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677753259779338, + "loss": 1.7317, + "theoretical_loss": 4.49280183734856, + "tokens_seen": 164626432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677592778335006, + "loss": 1.8705, + "theoretical_loss": 4.492572786137774, + "tokens_seen": 164691968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677432296890672, + "loss": 1.7851, + "theoretical_loss": 4.492343851564835, + "tokens_seen": 164757504 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767727181544634, + "loss": 1.8575, + "theoretical_loss": 4.492115033523987, + "tokens_seen": 164823040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677111334002006, + "loss": 1.7289, + "theoretical_loss": 4.491886331909615, + "tokens_seen": 164888576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676950852557674, + "loss": 1.7442, + "theoretical_loss": 4.491657746616241, + "tokens_seen": 164954112 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767679037111334, + "loss": 1.6633, + "theoretical_loss": 4.49142927753852, + "tokens_seen": 165019648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676629889669008, + "loss": 1.6377, + "theoretical_loss": 4.491200924571251, + "tokens_seen": 165085184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676469408224674, + "loss": 1.5454, + "theoretical_loss": 4.490972687609364, + "tokens_seen": 165150720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676308926780342, + "loss": 1.7009, + "theoretical_loss": 4.4907445665479315, + "tokens_seen": 165216256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676148445336008, + "loss": 1.7974, + "theoretical_loss": 4.490516561282157, + "tokens_seen": 165281792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675987963891676, + "loss": 1.6644, + "theoretical_loss": 4.4902886717073835, + "tokens_seen": 165347328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675827482447342, + "loss": 1.7853, + "theoretical_loss": 4.49006089771909, + "tokens_seen": 165412864 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767566700100301, + "loss": 1.5877, + "theoretical_loss": 4.489833239212888, + "tokens_seen": 165478400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675506519558676, + "loss": 1.6106, + "theoretical_loss": 4.48960569608453, + "tokens_seen": 165543936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675346038114344, + "loss": 1.7594, + "theoretical_loss": 4.489378268229897, + "tokens_seen": 165609472 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767518555667001, + "loss": 1.8262, + "theoretical_loss": 4.489150955545012, + "tokens_seen": 165675008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675025075225677, + "loss": 1.7828, + "theoretical_loss": 4.488923757926031, + "tokens_seen": 165740544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674864593781344, + "loss": 1.6994, + "theoretical_loss": 4.4886966752692405, + "tokens_seen": 165806080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674704112337011, + "loss": 1.6913, + "theoretical_loss": 4.488469707471066, + "tokens_seen": 165871616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674543630892678, + "loss": 1.6736, + "theoretical_loss": 4.488242854428064, + "tokens_seen": 165937152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674383149448345, + "loss": 1.6912, + "theoretical_loss": 4.488016116036929, + "tokens_seen": 166002688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674222668004012, + "loss": 1.9221, + "theoretical_loss": 4.487789492194485, + "tokens_seen": 166068224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674062186559679, + "loss": 1.7546, + "theoretical_loss": 4.487562982797693, + "tokens_seen": 166133760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673901705115346, + "loss": 1.7376, + "theoretical_loss": 4.487336587743643, + "tokens_seen": 166199296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673741223671013, + "loss": 1.7413, + "theoretical_loss": 4.487110306929562, + "tokens_seen": 166264832 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767358074222668, + "loss": 1.7142, + "theoretical_loss": 4.48688414025281, + "tokens_seen": 166330368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673420260782347, + "loss": 1.8665, + "theoretical_loss": 4.486658087610875, + "tokens_seen": 166395904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673259779338014, + "loss": 1.6341, + "theoretical_loss": 4.486432148901384, + "tokens_seen": 166461440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673099297893681, + "loss": 1.7621, + "theoretical_loss": 4.48620632402209, + "tokens_seen": 166526976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672938816449348, + "loss": 1.7723, + "theoretical_loss": 4.485980612870883, + "tokens_seen": 166592512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672778335005015, + "loss": 1.7573, + "theoretical_loss": 4.48575501534578, + "tokens_seen": 166658048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672617853560682, + "loss": 1.7641, + "theoretical_loss": 4.485529531344932, + "tokens_seen": 166723584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672457372116349, + "loss": 1.8136, + "theoretical_loss": 4.485304160766624, + "tokens_seen": 166789120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672296890672016, + "loss": 1.7361, + "theoretical_loss": 4.485078903509266, + "tokens_seen": 166854656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672136409227683, + "loss": 1.8344, + "theoretical_loss": 4.484853759471403, + "tokens_seen": 166920192 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767197592778335, + "loss": 1.833, + "theoretical_loss": 4.484628728551711, + "tokens_seen": 166985728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671815446339017, + "loss": 1.7914, + "theoretical_loss": 4.484403810648994, + "tokens_seen": 167051264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 62380, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6307178735733032, + "objective/train/theoretical_loss": 4.484179005662186, + "objective/train/tokens_used": 187576800, + "theoretical_loss": 4.484179005662186, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671654964894685, + "loss": 1.7346, + "theoretical_loss": 4.484179005662186, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671494483450351, + "loss": 1.651, + "theoretical_loss": 4.483954313490354, + "tokens_seen": 167182336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671334002006019, + "loss": 1.7858, + "theoretical_loss": 4.483729734032694, + "tokens_seen": 167247872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671173520561685, + "loss": 1.7515, + "theoretical_loss": 4.483505267188528, + "tokens_seen": 167313408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671013039117353, + "loss": 1.7517, + "theoretical_loss": 4.48328091285731, + "tokens_seen": 167378944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670852557673019, + "loss": 1.8486, + "theoretical_loss": 4.483056670938625, + "tokens_seen": 167444480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670692076228687, + "loss": 1.796, + "theoretical_loss": 4.482832541332183, + "tokens_seen": 167510016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670531594784353, + "loss": 1.7272, + "theoretical_loss": 4.482608523937824, + "tokens_seen": 167575552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670371113340021, + "loss": 1.6783, + "theoretical_loss": 4.4823846186555185, + "tokens_seen": 167641088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670210631895687, + "loss": 1.658, + "theoretical_loss": 4.482160825385363, + "tokens_seen": 167706624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670050150451355, + "loss": 1.6597, + "theoretical_loss": 4.481937144027583, + "tokens_seen": 167772160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669889669007021, + "loss": 1.7993, + "theoretical_loss": 4.48171357448253, + "tokens_seen": 167837696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669729187562689, + "loss": 1.5537, + "theoretical_loss": 4.481490116650686, + "tokens_seen": 167903232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669568706118355, + "loss": 1.6828, + "theoretical_loss": 4.48126677043266, + "tokens_seen": 167968768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669408224674023, + "loss": 1.7082, + "theoretical_loss": 4.481043535729183, + "tokens_seen": 168034304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669247743229689, + "loss": 1.75, + "theoretical_loss": 4.480820412441122, + "tokens_seen": 168099840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669087261785357, + "loss": 1.8383, + "theoretical_loss": 4.4805974004694615, + "tokens_seen": 168165376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668926780341023, + "loss": 1.7588, + "theoretical_loss": 4.48037449971532, + "tokens_seen": 168230912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668766298896691, + "loss": 1.7177, + "theoretical_loss": 4.480151710079937, + "tokens_seen": 168296448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668605817452357, + "loss": 1.722, + "theoretical_loss": 4.479929031464682, + "tokens_seen": 168361984 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668445336008025, + "loss": 1.7248, + "theoretical_loss": 4.479706463771047, + "tokens_seen": 168427520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668284854563691, + "loss": 1.6552, + "theoretical_loss": 4.479484006900652, + "tokens_seen": 168493056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668124373119359, + "loss": 1.6428, + "theoretical_loss": 4.479261660755242, + "tokens_seen": 168558592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667963891675025, + "loss": 1.8342, + "theoretical_loss": 4.479039425236687, + "tokens_seen": 168624128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667803410230693, + "loss": 1.7043, + "theoretical_loss": 4.478817300246982, + "tokens_seen": 168689664 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667642928786359, + "loss": 1.7463, + "theoretical_loss": 4.478595285688247, + "tokens_seen": 168755200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667482447342027, + "loss": 1.6112, + "theoretical_loss": 4.478373381462728, + "tokens_seen": 168820736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667321965897693, + "loss": 1.689, + "theoretical_loss": 4.478151587472791, + "tokens_seen": 168886272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667161484453361, + "loss": 1.7633, + "theoretical_loss": 4.477929903620931, + "tokens_seen": 168951808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667001003009027, + "loss": 1.7455, + "theoretical_loss": 4.477708329809766, + "tokens_seen": 169017344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666840521564695, + "loss": 1.7587, + "theoretical_loss": 4.477486865942036, + "tokens_seen": 169082880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666680040120361, + "loss": 1.8086, + "theoretical_loss": 4.477265511920607, + "tokens_seen": 169148416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666519558676029, + "loss": 1.7175, + "theoretical_loss": 4.477044267648466, + "tokens_seen": 169213952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666359077231695, + "loss": 1.8273, + "theoretical_loss": 4.476823133028725, + "tokens_seen": 169279488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666198595787363, + "loss": 1.7291, + "theoretical_loss": 4.47660210796462, + "tokens_seen": 169345024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666038114343029, + "loss": 1.7855, + "theoretical_loss": 4.476381192359504, + "tokens_seen": 169410560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665877632898697, + "loss": 1.8167, + "theoretical_loss": 4.476160386116861, + "tokens_seen": 169476096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665717151454363, + "loss": 1.8135, + "theoretical_loss": 4.475939689140291, + "tokens_seen": 169541632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665556670010031, + "loss": 1.6252, + "theoretical_loss": 4.475719101333519, + "tokens_seen": 169607168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665396188565697, + "loss": 1.851, + "theoretical_loss": 4.475498622600391, + "tokens_seen": 169672704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665235707121365, + "loss": 1.642, + "theoretical_loss": 4.475278252844875, + "tokens_seen": 169738240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665075225677031, + "loss": 1.7247, + "theoretical_loss": 4.475057991971062, + "tokens_seen": 169803776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664914744232699, + "loss": 1.729, + "theoretical_loss": 4.474837839883162, + "tokens_seen": 169869312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664754262788365, + "loss": 1.7534, + "theoretical_loss": 4.474617796485507, + "tokens_seen": 169934848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664593781344033, + "loss": 1.7936, + "theoretical_loss": 4.474397861682552, + "tokens_seen": 170000384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664433299899699, + "loss": 1.747, + "theoretical_loss": 4.474178035378869, + "tokens_seen": 170065920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664272818455367, + "loss": 1.8412, + "theoretical_loss": 4.473958317479154, + "tokens_seen": 170131456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664112337011033, + "loss": 1.5907, + "theoretical_loss": 4.473738707888221, + "tokens_seen": 170196992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663951855566701, + "loss": 1.778, + "theoretical_loss": 4.473519206511006, + "tokens_seen": 170262528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663791374122367, + "loss": 1.8488, + "theoretical_loss": 4.473299813252565, + "tokens_seen": 170328064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 63171, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6936179399490356, + "objective/train/theoretical_loss": 4.473080528018071, + "objective/train/tokens_used": 190853600, + "theoretical_loss": 4.473080528018071, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663630892678035, + "loss": 1.7195, + "theoretical_loss": 4.473080528018071, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663470411233702, + "loss": 1.7942, + "theoretical_loss": 4.4728613507128205, + "tokens_seen": 170459136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663309929789369, + "loss": 1.673, + "theoretical_loss": 4.472642281242226, + "tokens_seen": 170524672 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663149448345036, + "loss": 1.6945, + "theoretical_loss": 4.472423319511822, + "tokens_seen": 170590208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662988966900703, + "loss": 1.7537, + "theoretical_loss": 4.472204465427261, + "tokens_seen": 170655744 + }, + { + "epoch": 0.05, + "learning_rate": 0.000766282848545637, + "loss": 1.7052, + "theoretical_loss": 4.471985718894312, + "tokens_seen": 170721280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662668004012037, + "loss": 1.7989, + "theoretical_loss": 4.471767079818866, + "tokens_seen": 170786816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662507522567703, + "loss": 1.7626, + "theoretical_loss": 4.471548548106932, + "tokens_seen": 170852352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662347041123371, + "loss": 1.7287, + "theoretical_loss": 4.471330123664634, + "tokens_seen": 170917888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662186559679037, + "loss": 1.5974, + "theoretical_loss": 4.471111806398218, + "tokens_seen": 170983424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662026078234705, + "loss": 1.7094, + "theoretical_loss": 4.4708935962140455, + "tokens_seen": 171048960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661865596790371, + "loss": 1.7946, + "theoretical_loss": 4.4706754930185975, + "tokens_seen": 171114496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661705115346039, + "loss": 1.677, + "theoretical_loss": 4.47045749671847, + "tokens_seen": 171180032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661544633901705, + "loss": 1.6607, + "theoretical_loss": 4.470239607220377, + "tokens_seen": 171245568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661384152457373, + "loss": 1.7783, + "theoretical_loss": 4.470021824431152, + "tokens_seen": 171311104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661223671013039, + "loss": 1.7274, + "theoretical_loss": 4.469804148257742, + "tokens_seen": 171376640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661063189568706, + "loss": 1.8638, + "theoretical_loss": 4.469586578607213, + "tokens_seen": 171442176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660902708124373, + "loss": 1.7533, + "theoretical_loss": 4.469369115386746, + "tokens_seen": 171507712 + }, + { + "epoch": 0.05, + "learning_rate": 0.000766074222668004, + "loss": 1.7417, + "theoretical_loss": 4.46915175850364, + "tokens_seen": 171573248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660581745235707, + "loss": 1.6826, + "theoretical_loss": 4.468934507865308, + "tokens_seen": 171638784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660421263791374, + "loss": 1.7231, + "theoretical_loss": 4.46871736337928, + "tokens_seen": 171704320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660260782347041, + "loss": 1.7589, + "theoretical_loss": 4.468500324953202, + "tokens_seen": 171769856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660100300902708, + "loss": 1.8372, + "theoretical_loss": 4.468283392494836, + "tokens_seen": 171835392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007659939819458375, + "loss": 1.8463, + "theoretical_loss": 4.468066565912057, + "tokens_seen": 171900928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007659779338014042, + "loss": 1.7143, + "theoretical_loss": 4.467849845112859, + "tokens_seen": 171966464 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765961885656971, + "loss": 1.7111, + "theoretical_loss": 4.467633230005347, + "tokens_seen": 172032000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007659458375125376, + "loss": 1.7905, + "theoretical_loss": 4.467416720497743, + "tokens_seen": 172097536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007659297893681043, + "loss": 1.7305, + "theoretical_loss": 4.467200316498385, + "tokens_seen": 172163072 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765913741223671, + "loss": 1.6837, + "theoretical_loss": 4.4669840179157205, + "tokens_seen": 172228608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658976930792377, + "loss": 1.6952, + "theoretical_loss": 4.466767824658317, + "tokens_seen": 172294144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658816449348044, + "loss": 1.6608, + "theoretical_loss": 4.466551736634852, + "tokens_seen": 172359680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658655967903711, + "loss": 1.7433, + "theoretical_loss": 4.466335753754118, + "tokens_seen": 172425216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658495486459378, + "loss": 1.7105, + "theoretical_loss": 4.466119875925022, + "tokens_seen": 172490752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658335005015046, + "loss": 1.6972, + "theoretical_loss": 4.465904103056586, + "tokens_seen": 172556288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658174523570712, + "loss": 1.7239, + "theoretical_loss": 4.465688435057938, + "tokens_seen": 172621824 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765801404212638, + "loss": 1.6552, + "theoretical_loss": 4.46547287183833, + "tokens_seen": 172687360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657853560682046, + "loss": 1.5746, + "theoretical_loss": 4.465257413307119, + "tokens_seen": 172752896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657693079237714, + "loss": 1.7696, + "theoretical_loss": 4.465042059373776, + "tokens_seen": 172818432 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765753259779338, + "loss": 1.7059, + "theoretical_loss": 4.464826809947887, + "tokens_seen": 172883968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657372116349048, + "loss": 1.7048, + "theoretical_loss": 4.464611664939149, + "tokens_seen": 172949504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657211634904714, + "loss": 1.6989, + "theoretical_loss": 4.46439662425737, + "tokens_seen": 173015040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657051153460382, + "loss": 1.6436, + "theoretical_loss": 4.464181687812474, + "tokens_seen": 173080576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656890672016048, + "loss": 1.7972, + "theoretical_loss": 4.4639668555144905, + "tokens_seen": 173146112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656730190571716, + "loss": 1.7853, + "theoretical_loss": 4.463752127273567, + "tokens_seen": 173211648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656569709127382, + "loss": 1.795, + "theoretical_loss": 4.463537502999959, + "tokens_seen": 173277184 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765640922768305, + "loss": 1.6484, + "theoretical_loss": 4.4633229826040335, + "tokens_seen": 173342720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656248746238716, + "loss": 1.6078, + "theoretical_loss": 4.46310856599627, + "tokens_seen": 173408256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656088264794384, + "loss": 1.7552, + "theoretical_loss": 4.462894253087258, + "tokens_seen": 173473792 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765592778335005, + "loss": 1.8263, + "theoretical_loss": 4.462680043787699, + "tokens_seen": 173539328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655767301905718, + "loss": 1.794, + "theoretical_loss": 4.4624659380084015, + "tokens_seen": 173604864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 64355, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7965178489685059, + "objective/train/theoretical_loss": 4.462251935660287, + "objective/train/tokens_used": 194130400, + "theoretical_loss": 4.462251935660287, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655606820461384, + "loss": 1.6888, + "theoretical_loss": 4.462251935660287, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655446339017052, + "loss": 1.722, + "theoretical_loss": 4.46203803665439, + "tokens_seen": 173735936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655285857572718, + "loss": 1.8336, + "theoretical_loss": 4.4618242409018505, + "tokens_seen": 173801472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655125376128386, + "loss": 1.7571, + "theoretical_loss": 4.461610548313919, + "tokens_seen": 173867008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654964894684052, + "loss": 1.7694, + "theoretical_loss": 4.461396958801959, + "tokens_seen": 173932544 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765480441323972, + "loss": 1.7376, + "theoretical_loss": 4.461183472277439, + "tokens_seen": 173998080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654643931795386, + "loss": 1.6581, + "theoretical_loss": 4.460970088651941, + "tokens_seen": 174063616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654483450351054, + "loss": 1.6625, + "theoretical_loss": 4.460756807837154, + "tokens_seen": 174129152 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765432296890672, + "loss": 1.6933, + "theoretical_loss": 4.460543629744874, + "tokens_seen": 174194688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654162487462388, + "loss": 1.4361, + "theoretical_loss": 4.460330554287011, + "tokens_seen": 174260224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654002006018054, + "loss": 1.7742, + "theoretical_loss": 4.460117581375579, + "tokens_seen": 174325760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653841524573722, + "loss": 1.6683, + "theoretical_loss": 4.459904710922704, + "tokens_seen": 174391296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653681043129388, + "loss": 1.7873, + "theoretical_loss": 4.459691942840616, + "tokens_seen": 174456832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653520561685056, + "loss": 1.7836, + "theoretical_loss": 4.459479277041657, + "tokens_seen": 174522368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653360080240722, + "loss": 1.616, + "theoretical_loss": 4.459266713438277, + "tokens_seen": 174587904 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765319959879639, + "loss": 1.7366, + "theoretical_loss": 4.459054251943029, + "tokens_seen": 174653440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653039117352056, + "loss": 1.7021, + "theoretical_loss": 4.4588418924685795, + "tokens_seen": 174718976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652878635907724, + "loss": 1.9371, + "theoretical_loss": 4.458629634927698, + "tokens_seen": 174784512 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765271815446339, + "loss": 1.6323, + "theoretical_loss": 4.458417479233265, + "tokens_seen": 174850048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652557673019058, + "loss": 1.7393, + "theoretical_loss": 4.4582054252982655, + "tokens_seen": 174915584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652397191574724, + "loss": 1.7396, + "theoretical_loss": 4.457993473035792, + "tokens_seen": 174981120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652236710130392, + "loss": 1.6274, + "theoretical_loss": 4.457781622359043, + "tokens_seen": 175046656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652076228686058, + "loss": 1.6795, + "theoretical_loss": 4.457569873181326, + "tokens_seen": 175112192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651915747241726, + "loss": 1.7442, + "theoretical_loss": 4.457358225416052, + "tokens_seen": 175177728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651755265797392, + "loss": 1.6656, + "theoretical_loss": 4.4571466789767396, + "tokens_seen": 175243264 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765159478435306, + "loss": 1.6225, + "theoretical_loss": 4.456935233777013, + "tokens_seen": 175308800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651434302908727, + "loss": 1.7749, + "theoretical_loss": 4.456723889730605, + "tokens_seen": 175374336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651273821464394, + "loss": 1.7267, + "theoretical_loss": 4.456512646751349, + "tokens_seen": 175439872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651113340020061, + "loss": 1.7734, + "theoretical_loss": 4.456301504753188, + "tokens_seen": 175505408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650952858575728, + "loss": 1.7111, + "theoretical_loss": 4.456090463650169, + "tokens_seen": 175570944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650792377131395, + "loss": 1.6561, + "theoretical_loss": 4.455879523356444, + "tokens_seen": 175636480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650631895687062, + "loss": 1.77, + "theoretical_loss": 4.4556686837862705, + "tokens_seen": 175702016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650471414242729, + "loss": 1.6699, + "theoretical_loss": 4.455457944854011, + "tokens_seen": 175767552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650310932798396, + "loss": 1.7107, + "theoretical_loss": 4.455247306474131, + "tokens_seen": 175833088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650150451354063, + "loss": 1.8197, + "theoretical_loss": 4.455036768561204, + "tokens_seen": 175898624 + }, + { + "epoch": 0.05, + "learning_rate": 0.000764998996990973, + "loss": 1.7647, + "theoretical_loss": 4.4548263310299046, + "tokens_seen": 175964160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649829488465397, + "loss": 1.7439, + "theoretical_loss": 4.454615993795015, + "tokens_seen": 176029696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649669007021064, + "loss": 1.6774, + "theoretical_loss": 4.454405756771415, + "tokens_seen": 176095232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649508525576731, + "loss": 1.822, + "theoretical_loss": 4.454195619874096, + "tokens_seen": 176160768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649348044132398, + "loss": 1.7231, + "theoretical_loss": 4.453985583018149, + "tokens_seen": 176226304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649187562688065, + "loss": 1.7635, + "theoretical_loss": 4.4537756461187685, + "tokens_seen": 176291840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649027081243732, + "loss": 1.7327, + "theoretical_loss": 4.453565809091254, + "tokens_seen": 176357376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648866599799399, + "loss": 1.7449, + "theoretical_loss": 4.453356071851006, + "tokens_seen": 176422912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648706118355066, + "loss": 1.7211, + "theoretical_loss": 4.453146434313531, + "tokens_seen": 176488448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648545636910732, + "loss": 1.851, + "theoretical_loss": 4.452936896394435, + "tokens_seen": 176553984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00076483851554664, + "loss": 1.7872, + "theoretical_loss": 4.452727458009431, + "tokens_seen": 176619520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648224674022066, + "loss": 1.7427, + "theoretical_loss": 4.45251811907433, + "tokens_seen": 176685056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648064192577734, + "loss": 1.6532, + "theoretical_loss": 4.452308879505047, + "tokens_seen": 176750592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00076479037111334, + "loss": 1.8032, + "theoretical_loss": 4.452099739217601, + "tokens_seen": 176816128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647743229689068, + "loss": 1.6829, + "theoretical_loss": 4.451890698128112, + "tokens_seen": 176881664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 65665, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5994759798049927, + "objective/train/theoretical_loss": 4.4516817561528015, + "objective/train/tokens_used": 197407200, + "theoretical_loss": 4.4516817561528015, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647582748244734, + "loss": 1.6761, + "theoretical_loss": 4.4516817561528015, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647422266800402, + "loss": 1.6801, + "theoretical_loss": 4.451472913207992, + "tokens_seen": 177012736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647261785356068, + "loss": 1.7034, + "theoretical_loss": 4.451264169210107, + "tokens_seen": 177078272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647101303911735, + "loss": 1.651, + "theoretical_loss": 4.451055524075677, + "tokens_seen": 177143808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646940822467402, + "loss": 1.6878, + "theoretical_loss": 4.450846977721326, + "tokens_seen": 177209344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646780341023069, + "loss": 1.7719, + "theoretical_loss": 4.450638530063786, + "tokens_seen": 177274880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646619859578736, + "loss": 1.7237, + "theoretical_loss": 4.450430181019884, + "tokens_seen": 177340416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646459378134403, + "loss": 1.7421, + "theoretical_loss": 4.450221930506553, + "tokens_seen": 177405952 + }, + { + "epoch": 0.05, + "learning_rate": 0.000764629889669007, + "loss": 1.6159, + "theoretical_loss": 4.450013778440822, + "tokens_seen": 177471488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646138415245737, + "loss": 1.6349, + "theoretical_loss": 4.449805724739824, + "tokens_seen": 177537024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645977933801404, + "loss": 1.6987, + "theoretical_loss": 4.449597769320791, + "tokens_seen": 177602560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645817452357071, + "loss": 1.7311, + "theoretical_loss": 4.4493899121010525, + "tokens_seen": 177668096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645656970912738, + "loss": 1.7738, + "theoretical_loss": 4.449182152998044, + "tokens_seen": 177733632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645496489468405, + "loss": 1.6177, + "theoretical_loss": 4.448974491929297, + "tokens_seen": 177799168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645336008024072, + "loss": 1.8036, + "theoretical_loss": 4.448766928812441, + "tokens_seen": 177864704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645175526579739, + "loss": 1.7037, + "theoretical_loss": 4.448559463565209, + "tokens_seen": 177930240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645015045135406, + "loss": 1.6341, + "theoretical_loss": 4.44835209610543, + "tokens_seen": 177995776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644854563691073, + "loss": 1.7024, + "theoretical_loss": 4.448144826351036, + "tokens_seen": 178061312 + }, + { + "epoch": 0.05, + "learning_rate": 0.000764469408224674, + "loss": 1.6771, + "theoretical_loss": 4.447937654220054, + "tokens_seen": 178126848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644533600802407, + "loss": 1.7449, + "theoretical_loss": 4.447730579630612, + "tokens_seen": 178192384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644373119358075, + "loss": 1.5651, + "theoretical_loss": 4.447523602500936, + "tokens_seen": 178257920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644212637913741, + "loss": 1.6465, + "theoretical_loss": 4.447316722749351, + "tokens_seen": 178323456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644052156469409, + "loss": 1.6778, + "theoretical_loss": 4.447109940294283, + "tokens_seen": 178388992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643891675025075, + "loss": 1.7214, + "theoretical_loss": 4.44690325505425, + "tokens_seen": 178454528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643731193580743, + "loss": 1.6484, + "theoretical_loss": 4.446696666947874, + "tokens_seen": 178520064 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643570712136409, + "loss": 1.733, + "theoretical_loss": 4.446490175893872, + "tokens_seen": 178585600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643410230692077, + "loss": 1.6846, + "theoretical_loss": 4.44628378181106, + "tokens_seen": 178651136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643249749247743, + "loss": 1.7986, + "theoretical_loss": 4.4460774846183515, + "tokens_seen": 178716672 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643089267803411, + "loss": 1.6765, + "theoretical_loss": 4.445871284234757, + "tokens_seen": 178782208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642928786359077, + "loss": 1.8968, + "theoretical_loss": 4.445665180579383, + "tokens_seen": 178847744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642768304914745, + "loss": 1.6697, + "theoretical_loss": 4.445459173571438, + "tokens_seen": 178913280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642607823470411, + "loss": 1.731, + "theoretical_loss": 4.445253263130223, + "tokens_seen": 178978816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642447342026079, + "loss": 1.668, + "theoretical_loss": 4.445047449175136, + "tokens_seen": 179044352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642286860581745, + "loss": 1.6233, + "theoretical_loss": 4.444841731625674, + "tokens_seen": 179109888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642126379137413, + "loss": 1.8112, + "theoretical_loss": 4.444636110401431, + "tokens_seen": 179175424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641965897693079, + "loss": 1.7487, + "theoretical_loss": 4.444430585422094, + "tokens_seen": 179240960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641805416248747, + "loss": 1.5881, + "theoretical_loss": 4.444225156607449, + "tokens_seen": 179306496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641644934804413, + "loss": 1.7401, + "theoretical_loss": 4.444019823877378, + "tokens_seen": 179372032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641484453360081, + "loss": 1.6739, + "theoretical_loss": 4.4438145871518575, + "tokens_seen": 179437568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641323971915747, + "loss": 1.7123, + "theoretical_loss": 4.443609446350962, + "tokens_seen": 179503104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641163490471415, + "loss": 1.697, + "theoretical_loss": 4.44340440139486, + "tokens_seen": 179568640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641003009027081, + "loss": 1.7086, + "theoretical_loss": 4.443199452203816, + "tokens_seen": 179634176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640842527582749, + "loss": 1.6203, + "theoretical_loss": 4.442994598698189, + "tokens_seen": 179699712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640682046138415, + "loss": 1.7007, + "theoretical_loss": 4.4427898407984365, + "tokens_seen": 179765248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640521564694083, + "loss": 1.6982, + "theoretical_loss": 4.442585178425107, + "tokens_seen": 179830784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640361083249749, + "loss": 1.6712, + "theoretical_loss": 4.442380611498846, + "tokens_seen": 179896320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640200601805417, + "loss": 1.7155, + "theoretical_loss": 4.442176139940393, + "tokens_seen": 179961856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640040120361083, + "loss": 1.8174, + "theoretical_loss": 4.441971763670585, + "tokens_seen": 180027392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639879638916751, + "loss": 1.8372, + "theoretical_loss": 4.441767482610349, + "tokens_seen": 180092928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639719157472417, + "loss": 1.7136, + "theoretical_loss": 4.441563296680709, + "tokens_seen": 180158464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 66635, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7017289400100708, + "objective/train/theoretical_loss": 4.441359205802783, + "objective/train/tokens_used": 200684000, + "theoretical_loss": 4.441359205802783, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639558676028085, + "loss": 1.6671, + "theoretical_loss": 4.441359205802783, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639398194583752, + "loss": 1.74, + "theoretical_loss": 4.441155209897782, + "tokens_seen": 180289536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639237713139419, + "loss": 1.7344, + "theoretical_loss": 4.440951308887014, + "tokens_seen": 180355072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639077231695086, + "loss": 1.7618, + "theoretical_loss": 4.440747502691876, + "tokens_seen": 180420608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638916750250753, + "loss": 1.7888, + "theoretical_loss": 4.440543791233862, + "tokens_seen": 180486144 + }, + { + "epoch": 0.05, + "learning_rate": 0.000763875626880642, + "loss": 1.6206, + "theoretical_loss": 4.440340174434558, + "tokens_seen": 180551680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638595787362087, + "loss": 1.7611, + "theoretical_loss": 4.440136652215646, + "tokens_seen": 180617216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638435305917754, + "loss": 1.7102, + "theoretical_loss": 4.439933224498898, + "tokens_seen": 180682752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638274824473421, + "loss": 1.7108, + "theoretical_loss": 4.43972989120618, + "tokens_seen": 180748288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638114343029088, + "loss": 1.8015, + "theoretical_loss": 4.439526652259451, + "tokens_seen": 180813824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637953861584755, + "loss": 1.6381, + "theoretical_loss": 4.439323507580761, + "tokens_seen": 180879360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637793380140422, + "loss": 1.633, + "theoretical_loss": 4.439120457092258, + "tokens_seen": 180944896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637632898696089, + "loss": 1.6394, + "theoretical_loss": 4.438917500716178, + "tokens_seen": 181010432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637472417251756, + "loss": 1.658, + "theoretical_loss": 4.438714638374849, + "tokens_seen": 181075968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637311935807423, + "loss": 1.7647, + "theoretical_loss": 4.438511869990693, + "tokens_seen": 181141504 + }, + { + "epoch": 0.05, + "learning_rate": 0.000763715145436309, + "loss": 1.7654, + "theoretical_loss": 4.438309195486223, + "tokens_seen": 181207040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007636990972918757, + "loss": 1.7345, + "theoretical_loss": 4.438106614784045, + "tokens_seen": 181272576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007636830491474424, + "loss": 1.7755, + "theoretical_loss": 4.437904127806856, + "tokens_seen": 181338112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007636670010030091, + "loss": 1.7887, + "theoretical_loss": 4.437701734477445, + "tokens_seen": 181403648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007636509528585758, + "loss": 1.7479, + "theoretical_loss": 4.437499434718692, + "tokens_seen": 181469184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007636349047141425, + "loss": 1.8136, + "theoretical_loss": 4.437297228453567, + "tokens_seen": 181534720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007636188565697092, + "loss": 1.6026, + "theoretical_loss": 4.4370951156051355, + "tokens_seen": 181600256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007636028084252759, + "loss": 1.7126, + "theoretical_loss": 4.43689309609655, + "tokens_seen": 181665792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635867602808426, + "loss": 1.7019, + "theoretical_loss": 4.436691169851055, + "tokens_seen": 181731328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635707121364093, + "loss": 1.5814, + "theoretical_loss": 4.436489336791985, + "tokens_seen": 181796864 + }, + { + "epoch": 0.06, + "learning_rate": 0.000763554663991976, + "loss": 1.7454, + "theoretical_loss": 4.436287596842769, + "tokens_seen": 181862400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635386158475427, + "loss": 1.6315, + "theoretical_loss": 4.4360859499269205, + "tokens_seen": 181927936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635225677031094, + "loss": 1.7988, + "theoretical_loss": 4.435884395968047, + "tokens_seen": 181993472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635065195586761, + "loss": 1.763, + "theoretical_loss": 4.435682934889847, + "tokens_seen": 182059008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634904714142428, + "loss": 1.7406, + "theoretical_loss": 4.435481566616108, + "tokens_seen": 182124544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634744232698095, + "loss": 1.605, + "theoretical_loss": 4.435280291070705, + "tokens_seen": 182190080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634583751253762, + "loss": 1.7259, + "theoretical_loss": 4.435079108177606, + "tokens_seen": 182255616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634423269809429, + "loss": 1.7148, + "theoretical_loss": 4.434878017860869, + "tokens_seen": 182321152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634262788365095, + "loss": 1.752, + "theoretical_loss": 4.434677020044637, + "tokens_seen": 182386688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634102306920763, + "loss": 1.7357, + "theoretical_loss": 4.4344761146531475, + "tokens_seen": 182452224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633941825476429, + "loss": 1.712, + "theoretical_loss": 4.434275301610725, + "tokens_seen": 182517760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633781344032097, + "loss": 1.7025, + "theoretical_loss": 4.434074580841784, + "tokens_seen": 182583296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633620862587763, + "loss": 1.6831, + "theoretical_loss": 4.433873952270826, + "tokens_seen": 182648832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633460381143431, + "loss": 1.6822, + "theoretical_loss": 4.433673415822443, + "tokens_seen": 182714368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633299899699097, + "loss": 1.6375, + "theoretical_loss": 4.433472971421315, + "tokens_seen": 182779904 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633139418254764, + "loss": 1.7446, + "theoretical_loss": 4.433272618992213, + "tokens_seen": 182845440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632978936810431, + "loss": 1.6624, + "theoretical_loss": 4.433072358459991, + "tokens_seen": 182910976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632818455366098, + "loss": 1.667, + "theoretical_loss": 4.432872189749596, + "tokens_seen": 182976512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632657973921765, + "loss": 1.6365, + "theoretical_loss": 4.432672112786063, + "tokens_seen": 183042048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632497492477432, + "loss": 1.7046, + "theoretical_loss": 4.4324721274945125, + "tokens_seen": 183107584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632337011033099, + "loss": 1.6919, + "theoretical_loss": 4.432272233800154, + "tokens_seen": 183173120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632176529588766, + "loss": 1.6129, + "theoretical_loss": 4.432072431628286, + "tokens_seen": 183238656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632016048144433, + "loss": 1.6262, + "theoretical_loss": 4.431872720904292, + "tokens_seen": 183304192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00076318555667001, + "loss": 1.6003, + "theoretical_loss": 4.4316731015536455, + "tokens_seen": 183369728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631695085255767, + "loss": 1.6435, + "theoretical_loss": 4.431473573501906, + "tokens_seen": 183435264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 67570, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8887503147125244, + "objective/train/theoretical_loss": 4.431274136674721, + "objective/train/tokens_used": 203960800, + "theoretical_loss": 4.431274136674721, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631534603811434, + "loss": 1.8266, + "theoretical_loss": 4.431274136674721, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631374122367101, + "loss": 1.6985, + "theoretical_loss": 4.431074790997823, + "tokens_seen": 183566336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631213640922768, + "loss": 1.7259, + "theoretical_loss": 4.430875536397036, + "tokens_seen": 183631872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631053159478436, + "loss": 1.7227, + "theoretical_loss": 4.430676372798265, + "tokens_seen": 183697408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007630892678034102, + "loss": 1.8484, + "theoretical_loss": 4.430477300127507, + "tokens_seen": 183762944 + }, + { + "epoch": 0.06, + "learning_rate": 0.000763073219658977, + "loss": 1.7766, + "theoretical_loss": 4.43027831831084, + "tokens_seen": 183828480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007630571715145436, + "loss": 1.7096, + "theoretical_loss": 4.430079427274435, + "tokens_seen": 183894016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007630411233701104, + "loss": 1.6482, + "theoretical_loss": 4.429880626944543, + "tokens_seen": 183959552 + }, + { + "epoch": 0.06, + "learning_rate": 0.000763025075225677, + "loss": 1.7298, + "theoretical_loss": 4.429681917247506, + "tokens_seen": 184025088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007630090270812438, + "loss": 1.7268, + "theoretical_loss": 4.429483298109749, + "tokens_seen": 184090624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629929789368104, + "loss": 1.6922, + "theoretical_loss": 4.429284769457784, + "tokens_seen": 184156160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629769307923772, + "loss": 1.5606, + "theoretical_loss": 4.429086331218208, + "tokens_seen": 184221696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629608826479438, + "loss": 1.6067, + "theoretical_loss": 4.428887983317706, + "tokens_seen": 184287232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629448345035106, + "loss": 1.7639, + "theoretical_loss": 4.4286897256830455, + "tokens_seen": 184352768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629287863590772, + "loss": 1.7258, + "theoretical_loss": 4.428491558241081, + "tokens_seen": 184418304 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762912738214644, + "loss": 1.6622, + "theoretical_loss": 4.428293480918751, + "tokens_seen": 184483840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628966900702106, + "loss": 1.7358, + "theoretical_loss": 4.428095493643081, + "tokens_seen": 184549376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628806419257774, + "loss": 1.6846, + "theoretical_loss": 4.427897596341181, + "tokens_seen": 184614912 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762864593781344, + "loss": 1.6989, + "theoretical_loss": 4.427699788940243, + "tokens_seen": 184680448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628485456369108, + "loss": 1.6336, + "theoretical_loss": 4.427502071367549, + "tokens_seen": 184745984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628324974924774, + "loss": 1.684, + "theoretical_loss": 4.42730444355046, + "tokens_seen": 184811520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628164493480442, + "loss": 1.7098, + "theoretical_loss": 4.4271069054164265, + "tokens_seen": 184877056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628004012036108, + "loss": 1.7049, + "theoretical_loss": 4.426909456892979, + "tokens_seen": 184942592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627843530591776, + "loss": 1.6428, + "theoretical_loss": 4.426712097907735, + "tokens_seen": 185008128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627683049147442, + "loss": 1.6611, + "theoretical_loss": 4.426514828388394, + "tokens_seen": 185073664 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762752256770311, + "loss": 1.5817, + "theoretical_loss": 4.426317648262743, + "tokens_seen": 185139200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627362086258777, + "loss": 1.6968, + "theoretical_loss": 4.426120557458649, + "tokens_seen": 185204736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627201604814444, + "loss": 1.6527, + "theoretical_loss": 4.425923555904065, + "tokens_seen": 185270272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627041123370111, + "loss": 1.6712, + "theoretical_loss": 4.4257266435270255, + "tokens_seen": 185335808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626880641925778, + "loss": 1.622, + "theoretical_loss": 4.425529820255651, + "tokens_seen": 185401344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626720160481445, + "loss": 1.7908, + "theoretical_loss": 4.425333086018144, + "tokens_seen": 185466880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626559679037112, + "loss": 1.7091, + "theoretical_loss": 4.425136440742788, + "tokens_seen": 185532416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626399197592779, + "loss": 1.6781, + "theoretical_loss": 4.424939884357956, + "tokens_seen": 185597952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626238716148446, + "loss": 1.697, + "theoretical_loss": 4.424743416792096, + "tokens_seen": 185663488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626078234704113, + "loss": 1.6791, + "theoretical_loss": 4.424547037973745, + "tokens_seen": 185729024 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762591775325978, + "loss": 1.699, + "theoretical_loss": 4.424350747831518, + "tokens_seen": 185794560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625757271815447, + "loss": 1.7917, + "theoretical_loss": 4.424154546294117, + "tokens_seen": 185860096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625596790371114, + "loss": 1.707, + "theoretical_loss": 4.4239584332903235, + "tokens_seen": 185925632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625436308926781, + "loss": 1.6794, + "theoretical_loss": 4.423762408749001, + "tokens_seen": 185991168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625275827482448, + "loss": 1.6936, + "theoretical_loss": 4.423566472599099, + "tokens_seen": 186056704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625115346038115, + "loss": 1.8014, + "theoretical_loss": 4.423370624769643, + "tokens_seen": 186122240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624954864593782, + "loss": 1.7071, + "theoretical_loss": 4.423174865189747, + "tokens_seen": 186187776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624794383149449, + "loss": 1.6442, + "theoretical_loss": 4.422979193788601, + "tokens_seen": 186253312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624633901705116, + "loss": 1.5889, + "theoretical_loss": 4.422783610495481, + "tokens_seen": 186318848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624473420260783, + "loss": 1.6439, + "theoretical_loss": 4.422588115239742, + "tokens_seen": 186384384 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762431293881645, + "loss": 1.7129, + "theoretical_loss": 4.422392707950824, + "tokens_seen": 186449920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624152457372117, + "loss": 1.7146, + "theoretical_loss": 4.42219738855824, + "tokens_seen": 186515456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623991975927784, + "loss": 1.663, + "theoretical_loss": 4.422002156991597, + "tokens_seen": 186580992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623831494483451, + "loss": 1.6474, + "theoretical_loss": 4.421807013180571, + "tokens_seen": 186646528 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623671013039118, + "loss": 1.7353, + "theoretical_loss": 4.421611957054925, + "tokens_seen": 186712064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 68808, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6501104831695557, + "objective/train/theoretical_loss": 4.421416988544503, + "objective/train/tokens_used": 207237600, + "theoretical_loss": 4.421416988544503, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623510531594785, + "loss": 1.7163, + "theoretical_loss": 4.421416988544503, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623350050150452, + "loss": 1.7814, + "theoretical_loss": 4.4212221075792275, + "tokens_seen": 186843136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623189568706119, + "loss": 1.7138, + "theoretical_loss": 4.421027314089103, + "tokens_seen": 186908672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623029087261786, + "loss": 1.6188, + "theoretical_loss": 4.420832608004215, + "tokens_seen": 186974208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622868605817453, + "loss": 1.6911, + "theoretical_loss": 4.420637989254726, + "tokens_seen": 187039744 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762270812437312, + "loss": 1.6176, + "theoretical_loss": 4.420443457770885, + "tokens_seen": 187105280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622547642928787, + "loss": 1.7182, + "theoretical_loss": 4.420249013483014, + "tokens_seen": 187170816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622387161484454, + "loss": 1.7003, + "theoretical_loss": 4.42005465632152, + "tokens_seen": 187236352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622226680040121, + "loss": 1.7183, + "theoretical_loss": 4.419860386216888, + "tokens_seen": 187301888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622066198595788, + "loss": 1.737, + "theoretical_loss": 4.419666203099683, + "tokens_seen": 187367424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621905717151455, + "loss": 1.7014, + "theoretical_loss": 4.41947210690055, + "tokens_seen": 187432960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621745235707122, + "loss": 1.7964, + "theoretical_loss": 4.419278097550212, + "tokens_seen": 187498496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621584754262789, + "loss": 1.6903, + "theoretical_loss": 4.419084174979473, + "tokens_seen": 187564032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621424272818456, + "loss": 1.7771, + "theoretical_loss": 4.4188903391192165, + "tokens_seen": 187629568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621263791374123, + "loss": 1.7877, + "theoretical_loss": 4.418696589900405, + "tokens_seen": 187695104 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762110330992979, + "loss": 1.7553, + "theoretical_loss": 4.418502927254078, + "tokens_seen": 187760640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620942828485457, + "loss": 1.7333, + "theoretical_loss": 4.418309351111356, + "tokens_seen": 187826176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620782347041124, + "loss": 1.6975, + "theoretical_loss": 4.418115861403439, + "tokens_seen": 187891712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620621865596791, + "loss": 1.5968, + "theoretical_loss": 4.417922458061603, + "tokens_seen": 187957248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620461384152458, + "loss": 1.6263, + "theoretical_loss": 4.417729141017205, + "tokens_seen": 188022784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620300902708124, + "loss": 1.7959, + "theoretical_loss": 4.4175359102016785, + "tokens_seen": 188088320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620140421263792, + "loss": 1.7892, + "theoretical_loss": 4.417342765546538, + "tokens_seen": 188153856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619979939819458, + "loss": 1.75, + "theoretical_loss": 4.417149706983373, + "tokens_seen": 188219392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619819458375126, + "loss": 1.6113, + "theoretical_loss": 4.416956734443854, + "tokens_seen": 188284928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619658976930792, + "loss": 1.5337, + "theoretical_loss": 4.416763847859728, + "tokens_seen": 188350464 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761949849548646, + "loss": 1.6184, + "theoretical_loss": 4.41657104716282, + "tokens_seen": 188416000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619338014042126, + "loss": 1.7843, + "theoretical_loss": 4.416378332285031, + "tokens_seen": 188481536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619177532597794, + "loss": 1.6608, + "theoretical_loss": 4.4161857031583445, + "tokens_seen": 188547072 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761901705115346, + "loss": 1.7039, + "theoretical_loss": 4.415993159714818, + "tokens_seen": 188612608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618856569709127, + "loss": 1.8617, + "theoretical_loss": 4.415800701886584, + "tokens_seen": 188678144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618696088264794, + "loss": 1.813, + "theoretical_loss": 4.415608329605859, + "tokens_seen": 188743680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618535606820461, + "loss": 1.5579, + "theoretical_loss": 4.415416042804931, + "tokens_seen": 188809216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618375125376128, + "loss": 1.6717, + "theoretical_loss": 4.415223841416167, + "tokens_seen": 188874752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618214643931795, + "loss": 1.7537, + "theoretical_loss": 4.415031725372011, + "tokens_seen": 188940288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618054162487462, + "loss": 1.6978, + "theoretical_loss": 4.414839694604985, + "tokens_seen": 189005824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617893681043129, + "loss": 1.6423, + "theoretical_loss": 4.414647749047685, + "tokens_seen": 189071360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617733199598796, + "loss": 1.6646, + "theoretical_loss": 4.414455888632786, + "tokens_seen": 189136896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617572718154463, + "loss": 1.7303, + "theoretical_loss": 4.414264113293038, + "tokens_seen": 189202432 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761741223671013, + "loss": 1.7068, + "theoretical_loss": 4.41407242296127, + "tokens_seen": 189267968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617251755265797, + "loss": 1.6995, + "theoretical_loss": 4.413880817570382, + "tokens_seen": 189333504 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617091273821465, + "loss": 1.6645, + "theoretical_loss": 4.413689297053356, + "tokens_seen": 189399040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616930792377131, + "loss": 1.6716, + "theoretical_loss": 4.413497861343247, + "tokens_seen": 189464576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616770310932799, + "loss": 1.6816, + "theoretical_loss": 4.413306510373185, + "tokens_seen": 189530112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616609829488465, + "loss": 1.62, + "theoretical_loss": 4.41311524407638, + "tokens_seen": 189595648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616449348044133, + "loss": 1.715, + "theoretical_loss": 4.412924062386112, + "tokens_seen": 189661184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616288866599799, + "loss": 1.8238, + "theoretical_loss": 4.412732965235742, + "tokens_seen": 189726720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616128385155467, + "loss": 1.8357, + "theoretical_loss": 4.412541952558703, + "tokens_seen": 189792256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615967903711133, + "loss": 1.6682, + "theoretical_loss": 4.412351024288504, + "tokens_seen": 189857792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615807422266801, + "loss": 1.6815, + "theoretical_loss": 4.412160180358731, + "tokens_seen": 189923328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615646940822467, + "loss": 1.6328, + "theoretical_loss": 4.411969420703042, + "tokens_seen": 189988864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 70105, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.612324595451355, + "objective/train/theoretical_loss": 4.411778745255174, + "objective/train/tokens_used": 210514400, + "theoretical_loss": 4.411778745255174, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615486459378135, + "loss": 1.6841, + "theoretical_loss": 4.411778745255174, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615325977933802, + "loss": 1.7375, + "theoretical_loss": 4.411588153948935, + "tokens_seen": 190119936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615165496489469, + "loss": 1.8672, + "theoretical_loss": 4.411397646718211, + "tokens_seen": 190185472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615005015045136, + "loss": 1.6434, + "theoretical_loss": 4.411207223496962, + "tokens_seen": 190251008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614844533600803, + "loss": 1.6167, + "theoretical_loss": 4.41101688421922, + "tokens_seen": 190316544 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761468405215647, + "loss": 1.7425, + "theoretical_loss": 4.410826628819094, + "tokens_seen": 190382080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614523570712137, + "loss": 1.6488, + "theoretical_loss": 4.410636457230767, + "tokens_seen": 190447616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614363089267804, + "loss": 1.6163, + "theoretical_loss": 4.410446369388497, + "tokens_seen": 190513152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614202607823471, + "loss": 1.7284, + "theoretical_loss": 4.410256365226614, + "tokens_seen": 190578688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614042126379138, + "loss": 1.7268, + "theoretical_loss": 4.410066444679524, + "tokens_seen": 190644224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613881644934805, + "loss": 1.7091, + "theoretical_loss": 4.409876607681706, + "tokens_seen": 190709760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613721163490472, + "loss": 1.5607, + "theoretical_loss": 4.409686854167714, + "tokens_seen": 190775296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613560682046139, + "loss": 1.6649, + "theoretical_loss": 4.4094971840721735, + "tokens_seen": 190840832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613400200601806, + "loss": 1.7259, + "theoretical_loss": 4.409307597329786, + "tokens_seen": 190906368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613239719157473, + "loss": 1.8349, + "theoretical_loss": 4.409118093875324, + "tokens_seen": 190971904 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761307923771314, + "loss": 1.7556, + "theoretical_loss": 4.408928673643636, + "tokens_seen": 191037440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612918756268807, + "loss": 1.8219, + "theoretical_loss": 4.408739336569644, + "tokens_seen": 191102976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612758274824474, + "loss": 1.5849, + "theoretical_loss": 4.408550082588339, + "tokens_seen": 191168512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612597793380141, + "loss": 1.716, + "theoretical_loss": 4.408360911634789, + "tokens_seen": 191234048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612437311935808, + "loss": 1.6816, + "theoretical_loss": 4.408171823644135, + "tokens_seen": 191299584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612276830491475, + "loss": 1.7805, + "theoretical_loss": 4.407982818551589, + "tokens_seen": 191365120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612116349047142, + "loss": 1.822, + "theoretical_loss": 4.407793896292437, + "tokens_seen": 191430656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611955867602809, + "loss": 1.7945, + "theoretical_loss": 4.407605056802035, + "tokens_seen": 191496192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611795386158476, + "loss": 1.7431, + "theoretical_loss": 4.407416300015816, + "tokens_seen": 191561728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611634904714143, + "loss": 1.562, + "theoretical_loss": 4.407227625869283, + "tokens_seen": 191627264 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761147442326981, + "loss": 1.7065, + "theoretical_loss": 4.407039034298011, + "tokens_seen": 191692800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611313941825477, + "loss": 1.55, + "theoretical_loss": 4.4068505252376475, + "tokens_seen": 191758336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611153460381144, + "loss": 1.8253, + "theoretical_loss": 4.406662098623913, + "tokens_seen": 191823872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610992978936811, + "loss": 1.5841, + "theoretical_loss": 4.406473754392598, + "tokens_seen": 191889408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610832497492478, + "loss": 1.7049, + "theoretical_loss": 4.406285492479569, + "tokens_seen": 191954944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610672016048145, + "loss": 1.7289, + "theoretical_loss": 4.406097312820759, + "tokens_seen": 192020480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610511534603812, + "loss": 1.5932, + "theoretical_loss": 4.4059092153521755, + "tokens_seen": 192086016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610351053159479, + "loss": 1.7163, + "theoretical_loss": 4.405721200009898, + "tokens_seen": 192151552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610190571715146, + "loss": 1.7145, + "theoretical_loss": 4.405533266730077, + "tokens_seen": 192217088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610030090270813, + "loss": 1.7585, + "theoretical_loss": 4.405345415448934, + "tokens_seen": 192282624 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760986960882648, + "loss": 1.809, + "theoretical_loss": 4.405157646102762, + "tokens_seen": 192348160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609709127382147, + "loss": 1.6969, + "theoretical_loss": 4.404969958627927, + "tokens_seen": 192413696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609548645937814, + "loss": 1.6836, + "theoretical_loss": 4.4047823529608605, + "tokens_seen": 192479232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609388164493481, + "loss": 1.6517, + "theoretical_loss": 4.404594829038071, + "tokens_seen": 192544768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609227683049148, + "loss": 1.8073, + "theoretical_loss": 4.404407386796137, + "tokens_seen": 192610304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609067201604815, + "loss": 1.6976, + "theoretical_loss": 4.404220026171704, + "tokens_seen": 192675840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608906720160482, + "loss": 1.8556, + "theoretical_loss": 4.404032747101493, + "tokens_seen": 192741376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608746238716149, + "loss": 1.6249, + "theoretical_loss": 4.4038455495222895, + "tokens_seen": 192806912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608585757271816, + "loss": 1.7009, + "theoretical_loss": 4.403658433370957, + "tokens_seen": 192872448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608425275827483, + "loss": 1.6373, + "theoretical_loss": 4.403471398584422, + "tokens_seen": 192937984 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760826479438315, + "loss": 1.6357, + "theoretical_loss": 4.403284445099688, + "tokens_seen": 193003520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608104312938817, + "loss": 1.7354, + "theoretical_loss": 4.403097572853823, + "tokens_seen": 193069056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607943831494484, + "loss": 1.7761, + "theoretical_loss": 4.402910781783968, + "tokens_seen": 193134592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607783350050151, + "loss": 1.7687, + "theoretical_loss": 4.4027240718273335, + "tokens_seen": 193200128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607622868605818, + "loss": 1.6265, + "theoretical_loss": 4.4025374429212, + "tokens_seen": 193265664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 71032, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6292566061019897, + "objective/train/theoretical_loss": 4.402350895002916, + "objective/train/tokens_used": 213791200, + "theoretical_loss": 4.402350895002916, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607462387161485, + "loss": 1.6732, + "theoretical_loss": 4.402350895002916, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607301905717152, + "loss": 1.7352, + "theoretical_loss": 4.402164428009904, + "tokens_seen": 193396736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607141424272819, + "loss": 1.7918, + "theoretical_loss": 4.40197804187965, + "tokens_seen": 193462272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606980942828486, + "loss": 1.6467, + "theoretical_loss": 4.401791736549714, + "tokens_seen": 193527808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606820461384153, + "loss": 1.7642, + "theoretical_loss": 4.401605511957724, + "tokens_seen": 193593344 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760665997993982, + "loss": 1.6585, + "theoretical_loss": 4.401419368041377, + "tokens_seen": 193658880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606499498495487, + "loss": 1.6439, + "theoretical_loss": 4.401233304738438, + "tokens_seen": 193724416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606339017051153, + "loss": 1.7171, + "theoretical_loss": 4.401047321986745, + "tokens_seen": 193789952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606178535606821, + "loss": 1.7524, + "theoretical_loss": 4.4008614197242, + "tokens_seen": 193855488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606018054162487, + "loss": 1.7401, + "theoretical_loss": 4.400675597888777, + "tokens_seen": 193921024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605857572718155, + "loss": 1.6179, + "theoretical_loss": 4.400489856418518, + "tokens_seen": 193986560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605697091273821, + "loss": 1.6743, + "theoretical_loss": 4.4003041952515325, + "tokens_seen": 194052096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605536609829489, + "loss": 1.7264, + "theoretical_loss": 4.400118614326001, + "tokens_seen": 194117632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605376128385155, + "loss": 1.6531, + "theoretical_loss": 4.39993311358017, + "tokens_seen": 194183168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605215646940823, + "loss": 1.5623, + "theoretical_loss": 4.399747692952355, + "tokens_seen": 194248704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605055165496489, + "loss": 1.7761, + "theoretical_loss": 4.399562352380941, + "tokens_seen": 194314240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604894684052156, + "loss": 1.7038, + "theoretical_loss": 4.399377091804379, + "tokens_seen": 194379776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604734202607823, + "loss": 1.7686, + "theoretical_loss": 4.399191911161191, + "tokens_seen": 194445312 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760457372116349, + "loss": 1.6672, + "theoretical_loss": 4.399006810389963, + "tokens_seen": 194510848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604413239719157, + "loss": 1.6675, + "theoretical_loss": 4.398821789429354, + "tokens_seen": 194576384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604252758274824, + "loss": 1.844, + "theoretical_loss": 4.398636848218084, + "tokens_seen": 194641920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604092276830491, + "loss": 1.7601, + "theoretical_loss": 4.398451986694948, + "tokens_seen": 194707456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603931795386158, + "loss": 1.7763, + "theoretical_loss": 4.398267204798803, + "tokens_seen": 194772992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603771313941825, + "loss": 1.6735, + "theoretical_loss": 4.398082502468577, + "tokens_seen": 194838528 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603610832497492, + "loss": 1.5872, + "theoretical_loss": 4.397897879643262, + "tokens_seen": 194904064 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760345035105316, + "loss": 1.6719, + "theoretical_loss": 4.39771333626192, + "tokens_seen": 194969600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603289869608827, + "loss": 1.7731, + "theoretical_loss": 4.397528872263679, + "tokens_seen": 195035136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603129388164494, + "loss": 1.6565, + "theoretical_loss": 4.397344487587736, + "tokens_seen": 195100672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602968906720161, + "loss": 1.7334, + "theoretical_loss": 4.3971601821733515, + "tokens_seen": 195166208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602808425275828, + "loss": 1.6964, + "theoretical_loss": 4.396975955959856, + "tokens_seen": 195231744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602647943831495, + "loss": 1.7469, + "theoretical_loss": 4.396791808886643, + "tokens_seen": 195297280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602487462387162, + "loss": 1.8126, + "theoretical_loss": 4.396607740893179, + "tokens_seen": 195362816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602326980942829, + "loss": 1.6569, + "theoretical_loss": 4.396423751918991, + "tokens_seen": 195428352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602166499498496, + "loss": 1.5221, + "theoretical_loss": 4.396239841903674, + "tokens_seen": 195493888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602006018054163, + "loss": 1.6202, + "theoretical_loss": 4.396056010786893, + "tokens_seen": 195559424 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760184553660983, + "loss": 1.8545, + "theoretical_loss": 4.3958722585083745, + "tokens_seen": 195624960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601685055165497, + "loss": 1.7108, + "theoretical_loss": 4.395688585007913, + "tokens_seen": 195690496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601524573721164, + "loss": 1.7769, + "theoretical_loss": 4.395504990225371, + "tokens_seen": 195756032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601364092276831, + "loss": 1.8607, + "theoretical_loss": 4.395321474100673, + "tokens_seen": 195821568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601203610832498, + "loss": 1.7631, + "theoretical_loss": 4.395138036573814, + "tokens_seen": 195887104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601043129388165, + "loss": 1.7287, + "theoretical_loss": 4.3949546775848525, + "tokens_seen": 195952640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600882647943832, + "loss": 1.6529, + "theoretical_loss": 4.394771397073912, + "tokens_seen": 196018176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600722166499499, + "loss": 1.7395, + "theoretical_loss": 4.394588194981182, + "tokens_seen": 196083712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600561685055166, + "loss": 1.8479, + "theoretical_loss": 4.394405071246919, + "tokens_seen": 196149248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600401203610833, + "loss": 1.733, + "theoretical_loss": 4.394222025811445, + "tokens_seen": 196214784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00076002407221665, + "loss": 1.6186, + "theoretical_loss": 4.394039058615144, + "tokens_seen": 196280320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600080240722167, + "loss": 1.6977, + "theoretical_loss": 4.39385616959847, + "tokens_seen": 196345856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599919759277834, + "loss": 1.7501, + "theoretical_loss": 4.393673358701939, + "tokens_seen": 196411392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599759277833501, + "loss": 1.5986, + "theoretical_loss": 4.393490625866132, + "tokens_seen": 196476928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599598796389168, + "loss": 1.777, + "theoretical_loss": 4.393307971031697, + "tokens_seen": 196542464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 71932, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4793413877487183, + "objective/train/theoretical_loss": 4.3931253941393456, + "objective/train/tokens_used": 217068000, + "theoretical_loss": 4.3931253941393456, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599438314944835, + "loss": 1.5309, + "theoretical_loss": 4.3931253941393456, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599277833500502, + "loss": 1.7435, + "theoretical_loss": 4.392942895129854, + "tokens_seen": 196673536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599117352056169, + "loss": 1.6521, + "theoretical_loss": 4.392760473944064, + "tokens_seen": 196739072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598956870611836, + "loss": 1.708, + "theoretical_loss": 4.392578130522883, + "tokens_seen": 196804608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598796389167503, + "loss": 1.7905, + "theoretical_loss": 4.392395864807279, + "tokens_seen": 196870144 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759863590772317, + "loss": 1.7152, + "theoretical_loss": 4.392213676738288, + "tokens_seen": 196935680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598475426278837, + "loss": 1.8214, + "theoretical_loss": 4.3920315662570095, + "tokens_seen": 197001216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598314944834504, + "loss": 1.7028, + "theoretical_loss": 4.391849533304607, + "tokens_seen": 197066752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598154463390171, + "loss": 1.6381, + "theoretical_loss": 4.391667577822308, + "tokens_seen": 197132288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597993981945838, + "loss": 1.7244, + "theoretical_loss": 4.391485699751406, + "tokens_seen": 197197824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597833500501505, + "loss": 1.7536, + "theoretical_loss": 4.391303899033253, + "tokens_seen": 197263360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597673019057172, + "loss": 1.6795, + "theoretical_loss": 4.391122175609272, + "tokens_seen": 197328896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597512537612839, + "loss": 1.6457, + "theoretical_loss": 4.390940529420946, + "tokens_seen": 197394432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597352056168506, + "loss": 1.6911, + "theoretical_loss": 4.39075896040982, + "tokens_seen": 197459968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597191574724173, + "loss": 1.5381, + "theoretical_loss": 4.390577468517508, + "tokens_seen": 197525504 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759703109327984, + "loss": 1.7558, + "theoretical_loss": 4.390396053685682, + "tokens_seen": 197591040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596870611835507, + "loss": 1.7281, + "theoretical_loss": 4.39021471585608, + "tokens_seen": 197656576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596710130391174, + "loss": 1.6057, + "theoretical_loss": 4.390033454970504, + "tokens_seen": 197722112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596549648946841, + "loss": 1.6983, + "theoretical_loss": 4.389852270970818, + "tokens_seen": 197787648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596389167502508, + "loss": 1.7131, + "theoretical_loss": 4.389671163798949, + "tokens_seen": 197853184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596228686058175, + "loss": 1.7804, + "theoretical_loss": 4.3894901333968885, + "tokens_seen": 197918720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596068204613842, + "loss": 1.7999, + "theoretical_loss": 4.38930917970669, + "tokens_seen": 197984256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595907723169509, + "loss": 1.7063, + "theoretical_loss": 4.3891283026704695, + "tokens_seen": 198049792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595747241725176, + "loss": 1.7568, + "theoretical_loss": 4.388947502230407, + "tokens_seen": 198115328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595586760280843, + "loss": 1.6771, + "theoretical_loss": 4.388766778328744, + "tokens_seen": 198180864 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759542627883651, + "loss": 1.6316, + "theoretical_loss": 4.388586130907786, + "tokens_seen": 198246400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595265797392177, + "loss": 1.7721, + "theoretical_loss": 4.388405559909899, + "tokens_seen": 198311936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595105315947844, + "loss": 1.672, + "theoretical_loss": 4.3882250652775125, + "tokens_seen": 198377472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594944834503511, + "loss": 1.6478, + "theoretical_loss": 4.38804464695312, + "tokens_seen": 198443008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594784353059178, + "loss": 1.648, + "theoretical_loss": 4.387864304879275, + "tokens_seen": 198508544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594623871614845, + "loss": 1.7571, + "theoretical_loss": 4.387684038998595, + "tokens_seen": 198574080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594463390170512, + "loss": 1.7198, + "theoretical_loss": 4.387503849253757, + "tokens_seen": 198639616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594302908726179, + "loss": 1.7731, + "theoretical_loss": 4.387323735587502, + "tokens_seen": 198705152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594142427281846, + "loss": 1.7331, + "theoretical_loss": 4.387143697942634, + "tokens_seen": 198770688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593981945837513, + "loss": 1.8219, + "theoretical_loss": 4.386963736262016, + "tokens_seen": 198836224 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759382146439318, + "loss": 1.685, + "theoretical_loss": 4.386783850488575, + "tokens_seen": 198901760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593660982948847, + "loss": 1.7198, + "theoretical_loss": 4.386604040565299, + "tokens_seen": 198967296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593500501504514, + "loss": 1.7957, + "theoretical_loss": 4.386424306435236, + "tokens_seen": 199032832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593340020060181, + "loss": 1.7063, + "theoretical_loss": 4.386244648041498, + "tokens_seen": 199098368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593179538615848, + "loss": 1.6459, + "theoretical_loss": 4.386065065327258, + "tokens_seen": 199163904 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593019057171515, + "loss": 1.6441, + "theoretical_loss": 4.385885558235747, + "tokens_seen": 199229440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007592858575727182, + "loss": 1.7275, + "theoretical_loss": 4.385706126710264, + "tokens_seen": 199294976 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759269809428285, + "loss": 1.7563, + "theoretical_loss": 4.385526770694161, + "tokens_seen": 199360512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007592537612838516, + "loss": 1.6168, + "theoretical_loss": 4.385347490130858, + "tokens_seen": 199426048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007592377131394182, + "loss": 1.678, + "theoretical_loss": 4.385168284963832, + "tokens_seen": 199491584 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759221664994985, + "loss": 1.7211, + "theoretical_loss": 4.384989155136621, + "tokens_seen": 199557120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007592056168505516, + "loss": 1.6404, + "theoretical_loss": 4.384810100592826, + "tokens_seen": 199622656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591895687061184, + "loss": 1.7634, + "theoretical_loss": 4.384631121276108, + "tokens_seen": 199688192 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759173520561685, + "loss": 1.649, + "theoretical_loss": 4.384452217130187, + "tokens_seen": 199753728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591574724172518, + "loss": 1.6701, + "theoretical_loss": 4.384273388098845, + "tokens_seen": 199819264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 73160, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7228140830993652, + "objective/train/theoretical_loss": 4.384094634125924, + "objective/train/tokens_used": 220344800, + "theoretical_loss": 4.384094634125924, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591414242728184, + "loss": 1.6614, + "theoretical_loss": 4.384094634125924, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591253761283852, + "loss": 1.6742, + "theoretical_loss": 4.383915955155326, + "tokens_seen": 199950336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591093279839518, + "loss": 1.6602, + "theoretical_loss": 4.383737351131016, + "tokens_seen": 200015872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007590932798395186, + "loss": 1.7915, + "theoretical_loss": 4.383558821997014, + "tokens_seen": 200081408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007590772316950852, + "loss": 1.7205, + "theoretical_loss": 4.383380367697406, + "tokens_seen": 200146944 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759061183550652, + "loss": 1.7392, + "theoretical_loss": 4.383201988176333, + "tokens_seen": 200212480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007590451354062186, + "loss": 1.735, + "theoretical_loss": 4.383023683377999, + "tokens_seen": 200278016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007590290872617854, + "loss": 1.796, + "theoretical_loss": 4.382845453246668, + "tokens_seen": 200343552 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759013039117352, + "loss": 1.705, + "theoretical_loss": 4.382667297726661, + "tokens_seen": 200409088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589969909729188, + "loss": 1.6234, + "theoretical_loss": 4.3824892167623615, + "tokens_seen": 200474624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589809428284855, + "loss": 1.725, + "theoretical_loss": 4.382311210298212, + "tokens_seen": 200540160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589648946840522, + "loss": 1.7048, + "theoretical_loss": 4.3821332782787135, + "tokens_seen": 200605696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589488465396189, + "loss": 1.718, + "theoretical_loss": 4.381955420648428, + "tokens_seen": 200671232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589327983951856, + "loss": 1.7411, + "theoretical_loss": 4.381777637351974, + "tokens_seen": 200736768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589167502507523, + "loss": 1.8566, + "theoretical_loss": 4.381599928334033, + "tokens_seen": 200802304 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758900702106319, + "loss": 1.7141, + "theoretical_loss": 4.381422293539345, + "tokens_seen": 200867840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588846539618857, + "loss": 1.695, + "theoretical_loss": 4.381244732912706, + "tokens_seen": 200933376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588686058174524, + "loss": 1.6169, + "theoretical_loss": 4.381067246398975, + "tokens_seen": 200998912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588525576730191, + "loss": 1.6962, + "theoretical_loss": 4.380889833943066, + "tokens_seen": 201064448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588365095285858, + "loss": 1.6986, + "theoretical_loss": 4.380712495489957, + "tokens_seen": 201129984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588204613841525, + "loss": 1.7617, + "theoretical_loss": 4.380535230984679, + "tokens_seen": 201195520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588044132397192, + "loss": 1.6525, + "theoretical_loss": 4.380358040372325, + "tokens_seen": 201261056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587883650952859, + "loss": 1.7725, + "theoretical_loss": 4.380180923598047, + "tokens_seen": 201326592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587723169508526, + "loss": 1.7453, + "theoretical_loss": 4.380003880607055, + "tokens_seen": 201392128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587562688064193, + "loss": 1.787, + "theoretical_loss": 4.379826911344617, + "tokens_seen": 201457664 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758740220661986, + "loss": 1.7013, + "theoretical_loss": 4.3796500157560585, + "tokens_seen": 201523200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587241725175527, + "loss": 1.7672, + "theoretical_loss": 4.379473193786765, + "tokens_seen": 201588736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587081243731194, + "loss": 1.7227, + "theoretical_loss": 4.37929644538218, + "tokens_seen": 201654272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586920762286861, + "loss": 1.7301, + "theoretical_loss": 4.379119770487805, + "tokens_seen": 201719808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586760280842528, + "loss": 1.7752, + "theoretical_loss": 4.378943169049198, + "tokens_seen": 201785344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586599799398195, + "loss": 1.7993, + "theoretical_loss": 4.378766641011978, + "tokens_seen": 201850880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586439317953862, + "loss": 1.6976, + "theoretical_loss": 4.378590186321819, + "tokens_seen": 201916416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586278836509529, + "loss": 1.73, + "theoretical_loss": 4.378413804924454, + "tokens_seen": 201981952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586118355065196, + "loss": 1.6686, + "theoretical_loss": 4.378237496765674, + "tokens_seen": 202047488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585957873620863, + "loss": 1.6747, + "theoretical_loss": 4.378061261791328, + "tokens_seen": 202113024 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758579739217653, + "loss": 1.7438, + "theoretical_loss": 4.377885099947321, + "tokens_seen": 202178560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585636910732197, + "loss": 1.6094, + "theoretical_loss": 4.3777090111796175, + "tokens_seen": 202244096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585476429287864, + "loss": 1.7414, + "theoretical_loss": 4.377532995434238, + "tokens_seen": 202309632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585315947843531, + "loss": 1.7977, + "theoretical_loss": 4.37735705265726, + "tokens_seen": 202375168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585155466399198, + "loss": 1.6395, + "theoretical_loss": 4.3771811827948195, + "tokens_seen": 202440704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584994984954865, + "loss": 1.7897, + "theoretical_loss": 4.37700538579311, + "tokens_seen": 202506240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584834503510532, + "loss": 1.6584, + "theoretical_loss": 4.37682966159838, + "tokens_seen": 202571776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584674022066199, + "loss": 1.6479, + "theoretical_loss": 4.376654010156939, + "tokens_seen": 202637312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584513540621866, + "loss": 1.6911, + "theoretical_loss": 4.376478431415148, + "tokens_seen": 202702848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584353059177533, + "loss": 1.6819, + "theoretical_loss": 4.376302925319427, + "tokens_seen": 202768384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00075841925777332, + "loss": 1.7279, + "theoretical_loss": 4.3761274918162565, + "tokens_seen": 202833920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584032096288867, + "loss": 1.5278, + "theoretical_loss": 4.375952130852169, + "tokens_seen": 202899456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583871614844534, + "loss": 1.5301, + "theoretical_loss": 4.375776842373755, + "tokens_seen": 202964992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583711133400201, + "loss": 1.7144, + "theoretical_loss": 4.375601626327662, + "tokens_seen": 203030528 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583550651955868, + "loss": 1.6743, + "theoretical_loss": 4.375426482660595, + "tokens_seen": 203096064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 74031, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6785008907318115, + "objective/train/theoretical_loss": 4.375251411319312, + "objective/train/tokens_used": 223621600, + "theoretical_loss": 4.375251411319312, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583390170511535, + "loss": 1.7139, + "theoretical_loss": 4.375251411319312, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583229689067202, + "loss": 1.6417, + "theoretical_loss": 4.375076412250632, + "tokens_seen": 203227136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583069207622869, + "loss": 1.6601, + "theoretical_loss": 4.374901485401426, + "tokens_seen": 203292672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582908726178536, + "loss": 1.6632, + "theoretical_loss": 4.374726630718624, + "tokens_seen": 203358208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582748244734203, + "loss": 1.6369, + "theoretical_loss": 4.37455184814921, + "tokens_seen": 203423744 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758258776328987, + "loss": 1.5944, + "theoretical_loss": 4.374377137640225, + "tokens_seen": 203489280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582427281845537, + "loss": 1.633, + "theoretical_loss": 4.374202499138768, + "tokens_seen": 203554816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582266800401204, + "loss": 1.6212, + "theoretical_loss": 4.374027932591989, + "tokens_seen": 203620352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582106318956871, + "loss": 1.6938, + "theoretical_loss": 4.373853437947099, + "tokens_seen": 203685888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581945837512538, + "loss": 1.7528, + "theoretical_loss": 4.37367901515136, + "tokens_seen": 203751424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581785356068205, + "loss": 1.7577, + "theoretical_loss": 4.373504664152094, + "tokens_seen": 203816960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581624874623872, + "loss": 1.5288, + "theoretical_loss": 4.373330384896676, + "tokens_seen": 203882496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581464393179539, + "loss": 1.6905, + "theoretical_loss": 4.373156177332536, + "tokens_seen": 203948032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581303911735206, + "loss": 1.7408, + "theoretical_loss": 4.372982041407161, + "tokens_seen": 204013568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581143430290873, + "loss": 1.64, + "theoretical_loss": 4.372807977068092, + "tokens_seen": 204079104 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758098294884654, + "loss": 1.6473, + "theoretical_loss": 4.372633984262928, + "tokens_seen": 204144640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580822467402207, + "loss": 1.6681, + "theoretical_loss": 4.372460062939318, + "tokens_seen": 204210176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580661985957874, + "loss": 1.6786, + "theoretical_loss": 4.372286213044972, + "tokens_seen": 204275712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580501504513541, + "loss": 1.7793, + "theoretical_loss": 4.37211243452765, + "tokens_seen": 204341248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580341023069208, + "loss": 1.5185, + "theoretical_loss": 4.37193872733517, + "tokens_seen": 204406784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580180541624875, + "loss": 1.7019, + "theoretical_loss": 4.371765091415404, + "tokens_seen": 204472320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580020060180542, + "loss": 1.7595, + "theoretical_loss": 4.371591526716279, + "tokens_seen": 204537856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579859578736209, + "loss": 1.7805, + "theoretical_loss": 4.371418033185777, + "tokens_seen": 204603392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579699097291876, + "loss": 1.5919, + "theoretical_loss": 4.3712446107719325, + "tokens_seen": 204668928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579538615847543, + "loss": 1.7362, + "theoretical_loss": 4.371071259422838, + "tokens_seen": 204734464 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757937813440321, + "loss": 1.6875, + "theoretical_loss": 4.370897979086636, + "tokens_seen": 204800000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579217652958877, + "loss": 1.8177, + "theoretical_loss": 4.370724769711529, + "tokens_seen": 204865536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579057171514544, + "loss": 1.7856, + "theoretical_loss": 4.370551631245769, + "tokens_seen": 204931072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578896690070211, + "loss": 1.6602, + "theoretical_loss": 4.370378563637667, + "tokens_seen": 204996608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578736208625878, + "loss": 1.7391, + "theoretical_loss": 4.3702055668355815, + "tokens_seen": 205062144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578575727181545, + "loss": 1.7766, + "theoretical_loss": 4.370032640787931, + "tokens_seen": 205127680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578415245737211, + "loss": 1.7982, + "theoretical_loss": 4.369859785443188, + "tokens_seen": 205193216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578254764292879, + "loss": 1.6197, + "theoretical_loss": 4.369687000749873, + "tokens_seen": 205258752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578094282848545, + "loss": 1.7441, + "theoretical_loss": 4.3695142866565675, + "tokens_seen": 205324288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577933801404213, + "loss": 1.6629, + "theoretical_loss": 4.369341643111902, + "tokens_seen": 205389824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577773319959879, + "loss": 1.7171, + "theoretical_loss": 4.369169070064563, + "tokens_seen": 205455360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577612838515547, + "loss": 1.6443, + "theoretical_loss": 4.368996567463292, + "tokens_seen": 205520896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577452357071213, + "loss": 1.6624, + "theoretical_loss": 4.3688241352568795, + "tokens_seen": 205586432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577291875626881, + "loss": 1.8175, + "theoretical_loss": 4.368651773394173, + "tokens_seen": 205651968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577131394182547, + "loss": 1.5977, + "theoretical_loss": 4.368479481824075, + "tokens_seen": 205717504 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576970912738215, + "loss": 1.6515, + "theoretical_loss": 4.368307260495536, + "tokens_seen": 205783040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576810431293881, + "loss": 1.6509, + "theoretical_loss": 4.368135109357564, + "tokens_seen": 205848576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576649949849549, + "loss": 1.7278, + "theoretical_loss": 4.36796302835922, + "tokens_seen": 205914112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576489468405215, + "loss": 1.5491, + "theoretical_loss": 4.367791017449618, + "tokens_seen": 205979648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576328986960883, + "loss": 1.555, + "theoretical_loss": 4.367619076577923, + "tokens_seen": 206045184 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757616850551655, + "loss": 1.6893, + "theoretical_loss": 4.367447205693354, + "tokens_seen": 206110720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576008024072217, + "loss": 1.7705, + "theoretical_loss": 4.367275404745186, + "tokens_seen": 206176256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575847542627884, + "loss": 1.7363, + "theoretical_loss": 4.367103673682742, + "tokens_seen": 206241792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575687061183551, + "loss": 1.7842, + "theoretical_loss": 4.3669320124554, + "tokens_seen": 206307328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575526579739218, + "loss": 1.6683, + "theoretical_loss": 4.366760421012592, + "tokens_seen": 206372864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 75241, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.8373074531555176, + "objective/train/theoretical_loss": 4.366588899303801, + "objective/train/tokens_used": 226898400, + "theoretical_loss": 4.366588899303801, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575366098294885, + "loss": 1.7905, + "theoretical_loss": 4.366588899303801, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575205616850552, + "loss": 1.7056, + "theoretical_loss": 4.366417447278565, + "tokens_seen": 206503936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575045135406219, + "loss": 1.6869, + "theoretical_loss": 4.366246064886468, + "tokens_seen": 206569472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574884653961886, + "loss": 1.7188, + "theoretical_loss": 4.3660747520771555, + "tokens_seen": 206635008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574724172517553, + "loss": 1.6279, + "theoretical_loss": 4.36590350880032, + "tokens_seen": 206700544 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757456369107322, + "loss": 1.6978, + "theoretical_loss": 4.365732335005706, + "tokens_seen": 206766080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574403209628887, + "loss": 1.647, + "theoretical_loss": 4.365561230643112, + "tokens_seen": 206831616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574242728184554, + "loss": 1.691, + "theoretical_loss": 4.365390195662389, + "tokens_seen": 206897152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574082246740221, + "loss": 1.7532, + "theoretical_loss": 4.365219230013438, + "tokens_seen": 206962688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573921765295888, + "loss": 1.7129, + "theoretical_loss": 4.3650483336462145, + "tokens_seen": 207028224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573761283851555, + "loss": 1.7066, + "theoretical_loss": 4.364877506510723, + "tokens_seen": 207093760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573600802407222, + "loss": 1.6376, + "theoretical_loss": 4.3647067485570235, + "tokens_seen": 207159296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573440320962889, + "loss": 1.5943, + "theoretical_loss": 4.364536059735224, + "tokens_seen": 207224832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573279839518556, + "loss": 1.7438, + "theoretical_loss": 4.364365439995488, + "tokens_seen": 207290368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573119358074223, + "loss": 1.7198, + "theoretical_loss": 4.3641948892880285, + "tokens_seen": 207355904 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757295887662989, + "loss": 1.7217, + "theoretical_loss": 4.364024407563109, + "tokens_seen": 207421440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572798395185557, + "loss": 1.6305, + "theoretical_loss": 4.363853994771048, + "tokens_seen": 207486976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572637913741224, + "loss": 1.713, + "theoretical_loss": 4.363683650862212, + "tokens_seen": 207552512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572477432296891, + "loss": 1.6457, + "theoretical_loss": 4.36351337578702, + "tokens_seen": 207618048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572316950852558, + "loss": 1.6328, + "theoretical_loss": 4.363343169495945, + "tokens_seen": 207683584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572156469408225, + "loss": 1.717, + "theoretical_loss": 4.363173031939506, + "tokens_seen": 207749120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571995987963892, + "loss": 1.6263, + "theoretical_loss": 4.363002963068277, + "tokens_seen": 207814656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571835506519559, + "loss": 1.7193, + "theoretical_loss": 4.362832962832883, + "tokens_seen": 207880192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571675025075226, + "loss": 1.7143, + "theoretical_loss": 4.362663031183999, + "tokens_seen": 207945728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571514543630893, + "loss": 1.6003, + "theoretical_loss": 4.36249316807235, + "tokens_seen": 208011264 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757135406218656, + "loss": 1.7562, + "theoretical_loss": 4.3623233734487155, + "tokens_seen": 208076800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571193580742227, + "loss": 1.7425, + "theoretical_loss": 4.362153647263921, + "tokens_seen": 208142336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571033099297894, + "loss": 1.6691, + "theoretical_loss": 4.361983989468847, + "tokens_seen": 208207872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570872617853561, + "loss": 1.5959, + "theoretical_loss": 4.3618144000144214, + "tokens_seen": 208273408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570712136409228, + "loss": 1.6334, + "theoretical_loss": 4.361644878851626, + "tokens_seen": 208338944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570551654964895, + "loss": 1.832, + "theoretical_loss": 4.36147542593149, + "tokens_seen": 208404480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570391173520562, + "loss": 1.7518, + "theoretical_loss": 4.361306041205095, + "tokens_seen": 208470016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570230692076229, + "loss": 1.7394, + "theoretical_loss": 4.361136724623573, + "tokens_seen": 208535552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570070210631896, + "loss": 1.6212, + "theoretical_loss": 4.360967476138105, + "tokens_seen": 208601088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569909729187563, + "loss": 1.732, + "theoretical_loss": 4.360798295699925, + "tokens_seen": 208666624 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756974924774323, + "loss": 1.7454, + "theoretical_loss": 4.360629183260313, + "tokens_seen": 208732160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569588766298897, + "loss": 1.6911, + "theoretical_loss": 4.360460138770604, + "tokens_seen": 208797696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569428284854564, + "loss": 1.5386, + "theoretical_loss": 4.360291162182179, + "tokens_seen": 208863232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569267803410231, + "loss": 1.6819, + "theoretical_loss": 4.360122253446472, + "tokens_seen": 208928768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569107321965898, + "loss": 1.7132, + "theoretical_loss": 4.359953412514965, + "tokens_seen": 208994304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568946840521565, + "loss": 1.6954, + "theoretical_loss": 4.359784639339191, + "tokens_seen": 209059840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568786359077232, + "loss": 1.5804, + "theoretical_loss": 4.359615933870732, + "tokens_seen": 209125376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568625877632899, + "loss": 1.826, + "theoretical_loss": 4.359447296061219, + "tokens_seen": 209190912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568465396188566, + "loss": 1.6614, + "theoretical_loss": 4.359278725862336, + "tokens_seen": 209256448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568304914744233, + "loss": 1.7834, + "theoretical_loss": 4.359110223225814, + "tokens_seen": 209321984 + }, + { + "epoch": 0.06, + "learning_rate": 0.00075681444332999, + "loss": 1.7298, + "theoretical_loss": 4.358941788103433, + "tokens_seen": 209387520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567983951855567, + "loss": 1.6115, + "theoretical_loss": 4.358773420447024, + "tokens_seen": 209453056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567823470411234, + "loss": 1.7582, + "theoretical_loss": 4.358605120208469, + "tokens_seen": 209518592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567662988966901, + "loss": 1.6202, + "theoretical_loss": 4.358436887339694, + "tokens_seen": 209584128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567502507522569, + "loss": 1.6705, + "theoretical_loss": 4.35826872179268, + "tokens_seen": 209649664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 76142, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.782550573348999, + "objective/train/theoretical_loss": 4.358100623519453, + "objective/train/tokens_used": 230175200, + "theoretical_loss": 4.358100623519453, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567342026078235, + "loss": 1.6927, + "theoretical_loss": 4.358100623519453, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567181544633903, + "loss": 1.6178, + "theoretical_loss": 4.357932592472092, + "tokens_seen": 209780736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567021063189569, + "loss": 1.6931, + "theoretical_loss": 4.357764628602721, + "tokens_seen": 209846272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566860581745237, + "loss": 1.6771, + "theoretical_loss": 4.357596731863517, + "tokens_seen": 209911808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566700100300903, + "loss": 1.7497, + "theoretical_loss": 4.357428902206704, + "tokens_seen": 209977344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566539618856571, + "loss": 1.792, + "theoretical_loss": 4.357261139584553, + "tokens_seen": 210042880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566379137412237, + "loss": 1.7182, + "theoretical_loss": 4.357093443949388, + "tokens_seen": 210108416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566218655967905, + "loss": 1.7378, + "theoretical_loss": 4.356925815253579, + "tokens_seen": 210173952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566058174523571, + "loss": 1.623, + "theoretical_loss": 4.356758253449544, + "tokens_seen": 210239488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565897693079239, + "loss": 1.6864, + "theoretical_loss": 4.356590758489753, + "tokens_seen": 210305024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565737211634905, + "loss": 1.6433, + "theoretical_loss": 4.356423330326722, + "tokens_seen": 210370560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565576730190572, + "loss": 1.6874, + "theoretical_loss": 4.356255968913015, + "tokens_seen": 210436096 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756541624874624, + "loss": 1.6989, + "theoretical_loss": 4.356088674201246, + "tokens_seen": 210501632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565255767301906, + "loss": 1.6619, + "theoretical_loss": 4.355921446144077, + "tokens_seen": 210567168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565095285857573, + "loss": 1.6612, + "theoretical_loss": 4.355754284694218, + "tokens_seen": 210632704 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756493480441324, + "loss": 1.8118, + "theoretical_loss": 4.355587189804427, + "tokens_seen": 210698240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564774322968907, + "loss": 1.6183, + "theoretical_loss": 4.355420161427512, + "tokens_seen": 210763776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564613841524574, + "loss": 1.6552, + "theoretical_loss": 4.355253199516326, + "tokens_seen": 210829312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564453360080242, + "loss": 1.6956, + "theoretical_loss": 4.355086304023773, + "tokens_seen": 210894848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564292878635908, + "loss": 1.6164, + "theoretical_loss": 4.354919474902804, + "tokens_seen": 210960384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564132397191574, + "loss": 1.6407, + "theoretical_loss": 4.354752712106417, + "tokens_seen": 211025920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563971915747242, + "loss": 1.6663, + "theoretical_loss": 4.354586015587659, + "tokens_seen": 211091456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563811434302908, + "loss": 1.7245, + "theoretical_loss": 4.354419385299623, + "tokens_seen": 211156992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563650952858576, + "loss": 1.6848, + "theoretical_loss": 4.354252821195453, + "tokens_seen": 211222528 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563490471414242, + "loss": 1.7278, + "theoretical_loss": 4.354086323228338, + "tokens_seen": 211288064 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756332998996991, + "loss": 1.7467, + "theoretical_loss": 4.353919891351516, + "tokens_seen": 211353600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563169508525576, + "loss": 1.6958, + "theoretical_loss": 4.35375352551827, + "tokens_seen": 211419136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563009027081244, + "loss": 1.7661, + "theoretical_loss": 4.353587225681935, + "tokens_seen": 211484672 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756284854563691, + "loss": 1.7008, + "theoretical_loss": 4.35342099179589, + "tokens_seen": 211550208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562688064192578, + "loss": 1.6198, + "theoretical_loss": 4.353254823813561, + "tokens_seen": 211615744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562527582748245, + "loss": 1.6464, + "theoretical_loss": 4.353088721688424, + "tokens_seen": 211681280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562367101303912, + "loss": 1.6186, + "theoretical_loss": 4.352922685374001, + "tokens_seen": 211746816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562206619859579, + "loss": 1.7021, + "theoretical_loss": 4.352756714823859, + "tokens_seen": 211812352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562046138415246, + "loss": 1.6447, + "theoretical_loss": 4.352590809991616, + "tokens_seen": 211877888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561885656970913, + "loss": 1.7092, + "theoretical_loss": 4.352424970830935, + "tokens_seen": 211943424 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756172517552658, + "loss": 1.6352, + "theoretical_loss": 4.352259197295525, + "tokens_seen": 212008960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561564694082247, + "loss": 1.6442, + "theoretical_loss": 4.352093489339143, + "tokens_seen": 212074496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561404212637914, + "loss": 1.6247, + "theoretical_loss": 4.351927846915595, + "tokens_seen": 212140032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561243731193581, + "loss": 1.7138, + "theoretical_loss": 4.35176226997873, + "tokens_seen": 212205568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561083249749248, + "loss": 1.7305, + "theoretical_loss": 4.351596758482445, + "tokens_seen": 212271104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560922768304915, + "loss": 1.6121, + "theoretical_loss": 4.351431312380684, + "tokens_seen": 212336640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560762286860582, + "loss": 1.6804, + "theoretical_loss": 4.351265931627439, + "tokens_seen": 212402176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560601805416249, + "loss": 1.664, + "theoretical_loss": 4.351100616176747, + "tokens_seen": 212467712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560441323971916, + "loss": 1.6836, + "theoretical_loss": 4.350935365982691, + "tokens_seen": 212533248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560280842527583, + "loss": 1.7276, + "theoretical_loss": 4.350770180999402, + "tokens_seen": 212598784 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756012036108325, + "loss": 1.6969, + "theoretical_loss": 4.3506050611810565, + "tokens_seen": 212664320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559959879638917, + "loss": 1.5556, + "theoretical_loss": 4.350440006481877, + "tokens_seen": 212729856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559799398194584, + "loss": 1.7229, + "theoretical_loss": 4.350275016856132, + "tokens_seen": 212795392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559638916750251, + "loss": 1.6752, + "theoretical_loss": 4.350110092258138, + "tokens_seen": 212860928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559478435305918, + "loss": 1.6807, + "theoretical_loss": 4.349945232642256, + "tokens_seen": 212926464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 77362, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6674554347991943, + "objective/train/theoretical_loss": 4.3497804379628935, + "objective/train/tokens_used": 233452000, + "theoretical_loss": 4.3497804379628935, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559317953861585, + "loss": 1.7286, + "theoretical_loss": 4.3497804379628935, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559157472417252, + "loss": 1.6309, + "theoretical_loss": 4.349615708174504, + "tokens_seen": 213057536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558996990972919, + "loss": 1.589, + "theoretical_loss": 4.3494510432315865, + "tokens_seen": 213123072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558836509528586, + "loss": 1.6323, + "theoretical_loss": 4.349286443088687, + "tokens_seen": 213188608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558676028084253, + "loss": 1.6095, + "theoretical_loss": 4.349121907700396, + "tokens_seen": 213254144 + }, + { + "epoch": 0.06, + "learning_rate": 0.000755851554663992, + "loss": 1.7698, + "theoretical_loss": 4.348957437021351, + "tokens_seen": 213319680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558355065195587, + "loss": 1.7637, + "theoretical_loss": 4.348793031006235, + "tokens_seen": 213385216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558194583751254, + "loss": 1.636, + "theoretical_loss": 4.348628689609775, + "tokens_seen": 213450752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558034102306921, + "loss": 1.6531, + "theoretical_loss": 4.348464412786746, + "tokens_seen": 213516288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557873620862588, + "loss": 1.7114, + "theoretical_loss": 4.348300200491966, + "tokens_seen": 213581824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557713139418255, + "loss": 1.6954, + "theoretical_loss": 4.348136052680301, + "tokens_seen": 213647360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557552657973922, + "loss": 1.7568, + "theoretical_loss": 4.347971969306662, + "tokens_seen": 213712896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557392176529589, + "loss": 1.5466, + "theoretical_loss": 4.347807950326002, + "tokens_seen": 213778432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557231695085256, + "loss": 1.7092, + "theoretical_loss": 4.347643995693325, + "tokens_seen": 213843968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557071213640923, + "loss": 1.6906, + "theoretical_loss": 4.347480105363674, + "tokens_seen": 213909504 + }, + { + "epoch": 0.06, + "learning_rate": 0.000755691073219659, + "loss": 1.767, + "theoretical_loss": 4.3473162792921425, + "tokens_seen": 213975040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556750250752257, + "loss": 1.5911, + "theoretical_loss": 4.347152517433866, + "tokens_seen": 214040576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556589769307924, + "loss": 1.5156, + "theoretical_loss": 4.346988819744026, + "tokens_seen": 214106112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556429287863591, + "loss": 1.6893, + "theoretical_loss": 4.346825186177848, + "tokens_seen": 214171648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556268806419258, + "loss": 1.6864, + "theoretical_loss": 4.346661616690605, + "tokens_seen": 214237184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556108324974925, + "loss": 1.7354, + "theoretical_loss": 4.346498111237612, + "tokens_seen": 214302720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007555947843530592, + "loss": 1.6528, + "theoretical_loss": 4.346334669774231, + "tokens_seen": 214368256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007555787362086259, + "loss": 1.5685, + "theoretical_loss": 4.346171292255867, + "tokens_seen": 214433792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007555626880641926, + "loss": 1.6147, + "theoretical_loss": 4.34600797863797, + "tokens_seen": 214499328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007555466399197594, + "loss": 1.6934, + "theoretical_loss": 4.345844728876036, + "tokens_seen": 214564864 + }, + { + "epoch": 0.07, + "learning_rate": 0.000755530591775326, + "loss": 1.7624, + "theoretical_loss": 4.345681542925604, + "tokens_seen": 214630400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007555145436308928, + "loss": 1.5814, + "theoretical_loss": 4.345518420742259, + "tokens_seen": 214695936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554984954864594, + "loss": 1.6765, + "theoretical_loss": 4.345355362281628, + "tokens_seen": 214761472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554824473420262, + "loss": 1.6003, + "theoretical_loss": 4.345192367499386, + "tokens_seen": 214827008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554663991975928, + "loss": 1.7416, + "theoretical_loss": 4.3450294363512505, + "tokens_seen": 214892544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554503510531596, + "loss": 1.7229, + "theoretical_loss": 4.34486656879298, + "tokens_seen": 214958080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554343029087262, + "loss": 1.557, + "theoretical_loss": 4.344703764780386, + "tokens_seen": 215023616 + }, + { + "epoch": 0.07, + "learning_rate": 0.000755418254764293, + "loss": 1.7031, + "theoretical_loss": 4.344541024269312, + "tokens_seen": 215089152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554022066198596, + "loss": 1.6627, + "theoretical_loss": 4.344378347215657, + "tokens_seen": 215154688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553861584754264, + "loss": 1.6637, + "theoretical_loss": 4.344215733575357, + "tokens_seen": 215220224 + }, + { + "epoch": 0.07, + "learning_rate": 0.000755370110330993, + "loss": 1.5809, + "theoretical_loss": 4.344053183304396, + "tokens_seen": 215285760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553540621865598, + "loss": 1.7869, + "theoretical_loss": 4.343890696358798, + "tokens_seen": 215351296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553380140421264, + "loss": 1.4857, + "theoretical_loss": 4.343728272694635, + "tokens_seen": 215416832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553219658976932, + "loss": 1.4274, + "theoretical_loss": 4.34356591226802, + "tokens_seen": 215482368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553059177532598, + "loss": 1.7362, + "theoretical_loss": 4.343403615035111, + "tokens_seen": 215547904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007552898696088266, + "loss": 1.6923, + "theoretical_loss": 4.343241380952109, + "tokens_seen": 215613440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007552738214643932, + "loss": 1.7153, + "theoretical_loss": 4.34307920997526, + "tokens_seen": 215678976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075525777331996, + "loss": 1.7254, + "theoretical_loss": 4.34291710206085, + "tokens_seen": 215744512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007552417251755266, + "loss": 1.769, + "theoretical_loss": 4.342755057165214, + "tokens_seen": 215810048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007552256770310934, + "loss": 1.6395, + "theoretical_loss": 4.342593075244728, + "tokens_seen": 215875584 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075520962888666, + "loss": 1.6217, + "theoretical_loss": 4.342431156255809, + "tokens_seen": 215941120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551935807422268, + "loss": 1.5778, + "theoretical_loss": 4.3422693001549195, + "tokens_seen": 216006656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551775325977934, + "loss": 1.654, + "theoretical_loss": 4.342107506898567, + "tokens_seen": 216072192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551614844533601, + "loss": 1.6215, + "theoretical_loss": 4.3419457764433, + "tokens_seen": 216137728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551454363089268, + "loss": 1.5418, + "theoretical_loss": 4.34178410874571, + "tokens_seen": 216203264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 78526, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7171001434326172, + "objective/train/theoretical_loss": 4.341622503762434, + "objective/train/tokens_used": 236728800, + "theoretical_loss": 4.341622503762434, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551293881644935, + "loss": 1.6569, + "theoretical_loss": 4.341622503762434, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551133400200602, + "loss": 1.6461, + "theoretical_loss": 4.341460961450148, + "tokens_seen": 216334336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550972918756269, + "loss": 1.5453, + "theoretical_loss": 4.341299481765575, + "tokens_seen": 216399872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550812437311937, + "loss": 1.6849, + "theoretical_loss": 4.34113806466548, + "tokens_seen": 216465408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550651955867603, + "loss": 1.7779, + "theoretical_loss": 4.340976710106671, + "tokens_seen": 216530944 + }, + { + "epoch": 0.07, + "learning_rate": 0.000755049147442327, + "loss": 1.6228, + "theoretical_loss": 4.340815418045995, + "tokens_seen": 216596480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550330992978937, + "loss": 1.7109, + "theoretical_loss": 4.340654188440349, + "tokens_seen": 216662016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550170511534603, + "loss": 1.6088, + "theoretical_loss": 4.340493021246668, + "tokens_seen": 216727552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550010030090271, + "loss": 1.63, + "theoretical_loss": 4.3403319164219285, + "tokens_seen": 216793088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549849548645937, + "loss": 1.7283, + "theoretical_loss": 4.340170873923155, + "tokens_seen": 216858624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549689067201605, + "loss": 1.6471, + "theoretical_loss": 4.340009893707408, + "tokens_seen": 216924160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549528585757271, + "loss": 1.6254, + "theoretical_loss": 4.339848975731796, + "tokens_seen": 216989696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549368104312939, + "loss": 1.6024, + "theoretical_loss": 4.3396881199534665, + "tokens_seen": 217055232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549207622868605, + "loss": 1.6612, + "theoretical_loss": 4.339527326329613, + "tokens_seen": 217120768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549047141424273, + "loss": 1.6202, + "theoretical_loss": 4.3393665948174664, + "tokens_seen": 217186304 + }, + { + "epoch": 0.07, + "learning_rate": 0.000754888665997994, + "loss": 1.6312, + "theoretical_loss": 4.339205925374305, + "tokens_seen": 217251840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548726178535607, + "loss": 1.5936, + "theoretical_loss": 4.339045317957446, + "tokens_seen": 217317376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548565697091274, + "loss": 1.6884, + "theoretical_loss": 4.33888477252425, + "tokens_seen": 217382912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548405215646941, + "loss": 1.6815, + "theoretical_loss": 4.33872428903212, + "tokens_seen": 217448448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548244734202608, + "loss": 1.6695, + "theoretical_loss": 4.338563867438499, + "tokens_seen": 217513984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548084252758275, + "loss": 1.6838, + "theoretical_loss": 4.338403507700877, + "tokens_seen": 217579520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547923771313942, + "loss": 1.4434, + "theoretical_loss": 4.33824320977678, + "tokens_seen": 217645056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547763289869609, + "loss": 1.6037, + "theoretical_loss": 4.33808297362378, + "tokens_seen": 217710592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547602808425276, + "loss": 1.6591, + "theoretical_loss": 4.337922799199489, + "tokens_seen": 217776128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547442326980943, + "loss": 1.6701, + "theoretical_loss": 4.337762686461561, + "tokens_seen": 217841664 + }, + { + "epoch": 0.07, + "learning_rate": 0.000754728184553661, + "loss": 1.6801, + "theoretical_loss": 4.337602635367694, + "tokens_seen": 217907200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547121364092277, + "loss": 1.642, + "theoretical_loss": 4.337442645875624, + "tokens_seen": 217972736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546960882647944, + "loss": 1.695, + "theoretical_loss": 4.337282717943132, + "tokens_seen": 218038272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546800401203611, + "loss": 1.7297, + "theoretical_loss": 4.337122851528037, + "tokens_seen": 218103808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546639919759278, + "loss": 1.5796, + "theoretical_loss": 4.336963046588204, + "tokens_seen": 218169344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546479438314945, + "loss": 1.6636, + "theoretical_loss": 4.336803303081538, + "tokens_seen": 218234880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546318956870612, + "loss": 1.608, + "theoretical_loss": 4.336643620965982, + "tokens_seen": 218300416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546158475426279, + "loss": 1.6229, + "theoretical_loss": 4.336484000199524, + "tokens_seen": 218365952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545997993981946, + "loss": 1.5969, + "theoretical_loss": 4.336324440740195, + "tokens_seen": 218431488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545837512537613, + "loss": 1.6826, + "theoretical_loss": 4.336164942546061, + "tokens_seen": 218497024 + }, + { + "epoch": 0.07, + "learning_rate": 0.000754567703109328, + "loss": 1.5707, + "theoretical_loss": 4.336005505575235, + "tokens_seen": 218562560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545516549648947, + "loss": 1.7505, + "theoretical_loss": 4.335846129785869, + "tokens_seen": 218628096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545356068204614, + "loss": 1.554, + "theoretical_loss": 4.335686815136157, + "tokens_seen": 218693632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545195586760281, + "loss": 1.6179, + "theoretical_loss": 4.335527561584333, + "tokens_seen": 218759168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545035105315948, + "loss": 1.6906, + "theoretical_loss": 4.3353683690886715, + "tokens_seen": 218824704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544874623871615, + "loss": 1.7096, + "theoretical_loss": 4.33520923760749, + "tokens_seen": 218890240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544714142427282, + "loss": 1.6438, + "theoretical_loss": 4.335050167099145, + "tokens_seen": 218955776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544553660982949, + "loss": 1.7231, + "theoretical_loss": 4.334891157522035, + "tokens_seen": 219021312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544393179538616, + "loss": 1.6563, + "theoretical_loss": 4.3347322088346, + "tokens_seen": 219086848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544232698094283, + "loss": 1.5714, + "theoretical_loss": 4.33457332099532, + "tokens_seen": 219152384 + }, + { + "epoch": 0.07, + "learning_rate": 0.000754407221664995, + "loss": 1.5984, + "theoretical_loss": 4.334414493962714, + "tokens_seen": 219217920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543911735205617, + "loss": 1.6528, + "theoretical_loss": 4.334255727695344, + "tokens_seen": 219283456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543751253761284, + "loss": 1.6906, + "theoretical_loss": 4.334097022151812, + "tokens_seen": 219348992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543590772316951, + "loss": 1.71, + "theoretical_loss": 4.33393837729076, + "tokens_seen": 219414528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543430290872619, + "loss": 1.7512, + "theoretical_loss": 4.333779793070871, + "tokens_seen": 219480064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 79445, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6968114376068115, + "objective/train/theoretical_loss": 4.33362126945087, + "objective/train/tokens_used": 240005600, + "theoretical_loss": 4.33362126945087, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543269809428285, + "loss": 1.7406, + "theoretical_loss": 4.33362126945087, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543109327983953, + "loss": 1.6159, + "theoretical_loss": 4.333462806389517, + "tokens_seen": 219611136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542948846539619, + "loss": 1.6682, + "theoretical_loss": 4.33330440384562, + "tokens_seen": 219676672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542788365095287, + "loss": 1.6814, + "theoretical_loss": 4.333146061778021, + "tokens_seen": 219742208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542627883650953, + "loss": 1.6632, + "theoretical_loss": 4.332987780145606, + "tokens_seen": 219807744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542467402206621, + "loss": 1.5992, + "theoretical_loss": 4.3328295589073, + "tokens_seen": 219873280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542306920762287, + "loss": 1.6456, + "theoretical_loss": 4.332671398022067, + "tokens_seen": 219938816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542146439317955, + "loss": 1.632, + "theoretical_loss": 4.332513297448912, + "tokens_seen": 220004352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541985957873621, + "loss": 1.7385, + "theoretical_loss": 4.332355257146881, + "tokens_seen": 220069888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541825476429289, + "loss": 1.5406, + "theoretical_loss": 4.3321972770750605, + "tokens_seen": 220135424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541664994984955, + "loss": 1.6117, + "theoretical_loss": 4.332039357192572, + "tokens_seen": 220200960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541504513540623, + "loss": 1.6715, + "theoretical_loss": 4.331881497458584, + "tokens_seen": 220266496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541344032096289, + "loss": 1.6171, + "theoretical_loss": 4.3317236978323, + "tokens_seen": 220332032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541183550651957, + "loss": 1.6819, + "theoretical_loss": 4.3315659582729635, + "tokens_seen": 220397568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541023069207623, + "loss": 1.5839, + "theoretical_loss": 4.331408278739861, + "tokens_seen": 220463104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540862587763291, + "loss": 1.6511, + "theoretical_loss": 4.331250659192316, + "tokens_seen": 220528640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540702106318957, + "loss": 1.7196, + "theoretical_loss": 4.33109309958969, + "tokens_seen": 220594176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540541624874625, + "loss": 1.6633, + "theoretical_loss": 4.330935599891389, + "tokens_seen": 220659712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540381143430291, + "loss": 1.6971, + "theoretical_loss": 4.330778160056855, + "tokens_seen": 220725248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540220661985959, + "loss": 1.7276, + "theoretical_loss": 4.330620780045571, + "tokens_seen": 220790784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540060180541625, + "loss": 1.7717, + "theoretical_loss": 4.330463459817057, + "tokens_seen": 220856320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539899699097293, + "loss": 1.6917, + "theoretical_loss": 4.3303061993308765, + "tokens_seen": 220921856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539739217652959, + "loss": 1.6614, + "theoretical_loss": 4.330148998546628, + "tokens_seen": 220987392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539578736208627, + "loss": 1.6307, + "theoretical_loss": 4.329991857423953, + "tokens_seen": 221052928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539418254764293, + "loss": 1.5909, + "theoretical_loss": 4.329834775922529, + "tokens_seen": 221118464 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539257773319961, + "loss": 1.7151, + "theoretical_loss": 4.329677754002074, + "tokens_seen": 221184000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539097291875627, + "loss": 1.6996, + "theoretical_loss": 4.329520791622348, + "tokens_seen": 221249536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538936810431295, + "loss": 1.6523, + "theoretical_loss": 4.329363888743145, + "tokens_seen": 221315072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538776328986961, + "loss": 1.6329, + "theoretical_loss": 4.329207045324301, + "tokens_seen": 221380608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538615847542629, + "loss": 1.6521, + "theoretical_loss": 4.3290502613256905, + "tokens_seen": 221446144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538455366098295, + "loss": 1.8236, + "theoretical_loss": 4.328893536707228, + "tokens_seen": 221511680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538294884653963, + "loss": 1.7811, + "theoretical_loss": 4.328736871428864, + "tokens_seen": 221577216 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753813440320963, + "loss": 1.6121, + "theoretical_loss": 4.328580265450592, + "tokens_seen": 221642752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537973921765297, + "loss": 1.6411, + "theoretical_loss": 4.32842371873244, + "tokens_seen": 221708288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537813440320963, + "loss": 1.7128, + "theoretical_loss": 4.328267231234476, + "tokens_seen": 221773824 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753765295887663, + "loss": 1.7062, + "theoretical_loss": 4.328110802916811, + "tokens_seen": 221839360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537492477432297, + "loss": 1.8148, + "theoretical_loss": 4.327954433739588, + "tokens_seen": 221904896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537331995987964, + "loss": 1.6588, + "theoretical_loss": 4.327798123662992, + "tokens_seen": 221970432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537171514543632, + "loss": 1.5768, + "theoretical_loss": 4.327641872647248, + "tokens_seen": 222035968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537011033099298, + "loss": 1.7152, + "theoretical_loss": 4.327485680652615, + "tokens_seen": 222101504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007536850551654966, + "loss": 1.6536, + "theoretical_loss": 4.327329547639396, + "tokens_seen": 222167040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007536690070210632, + "loss": 1.6875, + "theoretical_loss": 4.327173473567927, + "tokens_seen": 222232576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075365295887663, + "loss": 1.7102, + "theoretical_loss": 4.327017458398587, + "tokens_seen": 222298112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007536369107321966, + "loss": 1.6412, + "theoretical_loss": 4.32686150209179, + "tokens_seen": 222363648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007536208625877632, + "loss": 1.6459, + "theoretical_loss": 4.32670560460799, + "tokens_seen": 222429184 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075360481444333, + "loss": 1.5749, + "theoretical_loss": 4.326549765907679, + "tokens_seen": 222494720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535887662988966, + "loss": 1.7364, + "theoretical_loss": 4.326393985951386, + "tokens_seen": 222560256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535727181544634, + "loss": 1.5982, + "theoretical_loss": 4.326238264699679, + "tokens_seen": 222625792 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075355667001003, + "loss": 1.7705, + "theoretical_loss": 4.326082602113164, + "tokens_seen": 222691328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535406218655968, + "loss": 1.6681, + "theoretical_loss": 4.325926998152487, + "tokens_seen": 222756864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 80592, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5793734788894653, + "objective/train/theoretical_loss": 4.325771452778328, + "objective/train/tokens_used": 243282400, + "theoretical_loss": 4.325771452778328, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535245737211634, + "loss": 1.6079, + "theoretical_loss": 4.325771452778328, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535085255767302, + "loss": 1.731, + "theoretical_loss": 4.325615965951408, + "tokens_seen": 222887936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534924774322969, + "loss": 1.6734, + "theoretical_loss": 4.325460537632483, + "tokens_seen": 222953472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534764292878636, + "loss": 1.6681, + "theoretical_loss": 4.325305167782352, + "tokens_seen": 223019008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534603811434303, + "loss": 1.6635, + "theoretical_loss": 4.325149856361845, + "tokens_seen": 223084544 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753444332998997, + "loss": 1.6284, + "theoretical_loss": 4.3249946033318345, + "tokens_seen": 223150080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534282848545637, + "loss": 1.7733, + "theoretical_loss": 4.32483940865323, + "tokens_seen": 223215616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534122367101304, + "loss": 1.6484, + "theoretical_loss": 4.324684272286978, + "tokens_seen": 223281152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533961885656971, + "loss": 1.6395, + "theoretical_loss": 4.324529194194062, + "tokens_seen": 223346688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533801404212638, + "loss": 1.6507, + "theoretical_loss": 4.324374174335503, + "tokens_seen": 223412224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533640922768305, + "loss": 1.7281, + "theoretical_loss": 4.324219212672363, + "tokens_seen": 223477760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533480441323972, + "loss": 1.7564, + "theoretical_loss": 4.324064309165734, + "tokens_seen": 223543296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533319959879639, + "loss": 1.5957, + "theoretical_loss": 4.3239094637767534, + "tokens_seen": 223608832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533159478435306, + "loss": 1.7318, + "theoretical_loss": 4.323754676466592, + "tokens_seen": 223674368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532998996990973, + "loss": 1.7811, + "theoretical_loss": 4.323599947196458, + "tokens_seen": 223739904 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753283851554664, + "loss": 1.7292, + "theoretical_loss": 4.323445275927597, + "tokens_seen": 223805440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532678034102307, + "loss": 1.712, + "theoretical_loss": 4.323290662621292, + "tokens_seen": 223870976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532517552657974, + "loss": 1.7663, + "theoretical_loss": 4.323136107238865, + "tokens_seen": 223936512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532357071213641, + "loss": 1.7012, + "theoretical_loss": 4.322981609741671, + "tokens_seen": 224002048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532196589769308, + "loss": 1.6102, + "theoretical_loss": 4.322827170091106, + "tokens_seen": 224067584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532036108324975, + "loss": 1.6167, + "theoretical_loss": 4.322672788248601, + "tokens_seen": 224133120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531875626880642, + "loss": 1.5478, + "theoretical_loss": 4.3225184641756265, + "tokens_seen": 224198656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531715145436309, + "loss": 1.6447, + "theoretical_loss": 4.322364197833684, + "tokens_seen": 224264192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531554663991976, + "loss": 1.618, + "theoretical_loss": 4.322209989184319, + "tokens_seen": 224329728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531394182547644, + "loss": 1.6174, + "theoretical_loss": 4.32205583818911, + "tokens_seen": 224395264 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753123370110331, + "loss": 1.6718, + "theoretical_loss": 4.321901744809672, + "tokens_seen": 224460800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531073219658978, + "loss": 1.6626, + "theoretical_loss": 4.321747709007658, + "tokens_seen": 224526336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530912738214644, + "loss": 1.6943, + "theoretical_loss": 4.321593730744759, + "tokens_seen": 224591872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530752256770312, + "loss": 1.7384, + "theoretical_loss": 4.3214398099826985, + "tokens_seen": 224657408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530591775325978, + "loss": 1.6295, + "theoretical_loss": 4.321285946683242, + "tokens_seen": 224722944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530431293881646, + "loss": 1.6245, + "theoretical_loss": 4.321132140808186, + "tokens_seen": 224788480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530270812437312, + "loss": 1.6358, + "theoretical_loss": 4.320978392319368, + "tokens_seen": 224854016 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753011033099298, + "loss": 1.6467, + "theoretical_loss": 4.320824701178661, + "tokens_seen": 224919552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529949849548646, + "loss": 1.6154, + "theoretical_loss": 4.320671067347972, + "tokens_seen": 224985088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529789368104314, + "loss": 1.7091, + "theoretical_loss": 4.320517490789246, + "tokens_seen": 225050624 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752962888665998, + "loss": 1.5441, + "theoretical_loss": 4.320363971464466, + "tokens_seen": 225116160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529468405215648, + "loss": 1.913, + "theoretical_loss": 4.320210509335649, + "tokens_seen": 225181696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529307923771314, + "loss": 1.7173, + "theoretical_loss": 4.320057104364848, + "tokens_seen": 225247232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529147442326982, + "loss": 1.5435, + "theoretical_loss": 4.3199037565141545, + "tokens_seen": 225312768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528986960882648, + "loss": 1.5532, + "theoretical_loss": 4.319750465745694, + "tokens_seen": 225378304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528826479438316, + "loss": 1.5809, + "theoretical_loss": 4.31959723202163, + "tokens_seen": 225443840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528665997993982, + "loss": 1.6615, + "theoretical_loss": 4.319444055304161, + "tokens_seen": 225509376 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752850551654965, + "loss": 1.6242, + "theoretical_loss": 4.319290935555521, + "tokens_seen": 225574912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528345035105316, + "loss": 1.6218, + "theoretical_loss": 4.319137872737979, + "tokens_seen": 225640448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528184553660984, + "loss": 1.7047, + "theoretical_loss": 4.318984866813844, + "tokens_seen": 225705984 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752802407221665, + "loss": 1.678, + "theoretical_loss": 4.318831917745458, + "tokens_seen": 225771520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527863590772318, + "loss": 1.6995, + "theoretical_loss": 4.318679025495197, + "tokens_seen": 225837056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527703109327984, + "loss": 1.6975, + "theoretical_loss": 4.318526190025478, + "tokens_seen": 225902592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527542627883652, + "loss": 1.7384, + "theoretical_loss": 4.318373411298749, + "tokens_seen": 225968128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527382146439318, + "loss": 1.6878, + "theoretical_loss": 4.318220689277496, + "tokens_seen": 226033664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 81462, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.65253746509552, + "objective/train/theoretical_loss": 4.318068023924241, + "objective/train/tokens_used": 246559200, + "theoretical_loss": 4.318068023924241, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527221664994986, + "loss": 1.7266, + "theoretical_loss": 4.318068023924241, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527061183550652, + "loss": 1.536, + "theoretical_loss": 4.317915415201538, + "tokens_seen": 226164736 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752690070210632, + "loss": 1.512, + "theoretical_loss": 4.317762863071982, + "tokens_seen": 226230272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007526740220661986, + "loss": 1.5732, + "theoretical_loss": 4.317610367498201, + "tokens_seen": 226295808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007526579739217654, + "loss": 1.7787, + "theoretical_loss": 4.317457928442856, + "tokens_seen": 226361344 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752641925777332, + "loss": 1.6781, + "theoretical_loss": 4.317305545868648, + "tokens_seen": 226426880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007526258776328988, + "loss": 1.7043, + "theoretical_loss": 4.317153219738311, + "tokens_seen": 226492416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007526098294884654, + "loss": 1.6454, + "theoretical_loss": 4.317000950014615, + "tokens_seen": 226557952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525937813440322, + "loss": 1.5971, + "theoretical_loss": 4.316848736660363, + "tokens_seen": 226623488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525777331995988, + "loss": 1.6237, + "theoretical_loss": 4.316696579638396, + "tokens_seen": 226689024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525616850551656, + "loss": 1.6166, + "theoretical_loss": 4.316544478911592, + "tokens_seen": 226754560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525456369107322, + "loss": 1.7207, + "theoretical_loss": 4.316392434442858, + "tokens_seen": 226820096 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752529588766299, + "loss": 1.6948, + "theoretical_loss": 4.316240446195142, + "tokens_seen": 226885632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525135406218656, + "loss": 1.6371, + "theoretical_loss": 4.3160885141314225, + "tokens_seen": 226951168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524974924774324, + "loss": 1.6582, + "theoretical_loss": 4.315936638214718, + "tokens_seen": 227016704 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752481444332999, + "loss": 1.6293, + "theoretical_loss": 4.315784818408079, + "tokens_seen": 227082240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524653961885658, + "loss": 1.6876, + "theoretical_loss": 4.31563305467459, + "tokens_seen": 227147776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524493480441324, + "loss": 1.6117, + "theoretical_loss": 4.315481346977374, + "tokens_seen": 227213312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524332998996992, + "loss": 1.5356, + "theoretical_loss": 4.315329695279584, + "tokens_seen": 227278848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524172517552658, + "loss": 1.7156, + "theoretical_loss": 4.315178099544413, + "tokens_seen": 227344384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524012036108326, + "loss": 1.5758, + "theoretical_loss": 4.315026559735085, + "tokens_seen": 227409920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523851554663992, + "loss": 1.7364, + "theoretical_loss": 4.314875075814861, + "tokens_seen": 227475456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523691073219659, + "loss": 1.716, + "theoretical_loss": 4.314723647747035, + "tokens_seen": 227540992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523530591775327, + "loss": 1.5991, + "theoretical_loss": 4.314572275494937, + "tokens_seen": 227606528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523370110330993, + "loss": 1.4546, + "theoretical_loss": 4.3144209590219305, + "tokens_seen": 227672064 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752320962888666, + "loss": 1.6459, + "theoretical_loss": 4.314269698291415, + "tokens_seen": 227737600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523049147442327, + "loss": 1.7276, + "theoretical_loss": 4.3141184932668235, + "tokens_seen": 227803136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522888665997995, + "loss": 1.6812, + "theoretical_loss": 4.3139673439116235, + "tokens_seen": 227868672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522728184553661, + "loss": 1.7196, + "theoretical_loss": 4.313816250189317, + "tokens_seen": 227934208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522567703109329, + "loss": 1.6224, + "theoretical_loss": 4.313665212063441, + "tokens_seen": 227999744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522407221664995, + "loss": 1.6276, + "theoretical_loss": 4.313514229497566, + "tokens_seen": 228065280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522246740220661, + "loss": 1.5987, + "theoretical_loss": 4.313363302455299, + "tokens_seen": 228130816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522086258776329, + "loss": 1.7328, + "theoretical_loss": 4.313212430900277, + "tokens_seen": 228196352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521925777331995, + "loss": 1.5894, + "theoretical_loss": 4.313061614796176, + "tokens_seen": 228261888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521765295887663, + "loss": 1.766, + "theoretical_loss": 4.312910854106702, + "tokens_seen": 228327424 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752160481444333, + "loss": 1.6051, + "theoretical_loss": 4.3127601487956, + "tokens_seen": 228392960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521444332998997, + "loss": 1.7288, + "theoretical_loss": 4.312609498826644, + "tokens_seen": 228458496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521283851554664, + "loss": 1.5024, + "theoretical_loss": 4.312458904163645, + "tokens_seen": 228524032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521123370110331, + "loss": 1.7327, + "theoretical_loss": 4.312308364770448, + "tokens_seen": 228589568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520962888665998, + "loss": 1.5368, + "theoretical_loss": 4.312157880610931, + "tokens_seen": 228655104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520802407221665, + "loss": 1.5935, + "theoretical_loss": 4.312007451649006, + "tokens_seen": 228720640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520641925777332, + "loss": 1.6157, + "theoretical_loss": 4.31185707784862, + "tokens_seen": 228786176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520481444332999, + "loss": 1.5835, + "theoretical_loss": 4.311706759173753, + "tokens_seen": 228851712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520320962888666, + "loss": 1.5946, + "theoretical_loss": 4.311556495588419, + "tokens_seen": 228917248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520160481444333, + "loss": 1.6795, + "theoretical_loss": 4.3114062870566645, + "tokens_seen": 228982784 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752, + "loss": 1.5991, + "theoretical_loss": 4.311256133542573, + "tokens_seen": 229048320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519839518555667, + "loss": 1.6723, + "theoretical_loss": 4.311106035010259, + "tokens_seen": 229113856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519679037111334, + "loss": 1.5913, + "theoretical_loss": 4.310955991423871, + "tokens_seen": 229179392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519518555667001, + "loss": 1.6719, + "theoretical_loss": 4.310806002747592, + "tokens_seen": 229244928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519358074222669, + "loss": 1.5921, + "theoretical_loss": 4.310656068945637, + "tokens_seen": 229310464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 82681, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5853383541107178, + "objective/train/theoretical_loss": 4.310506189982257, + "objective/train/tokens_used": 249836000, + "theoretical_loss": 4.310506189982257, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519197592778335, + "loss": 1.601, + "theoretical_loss": 4.310506189982257, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519037111334003, + "loss": 1.7332, + "theoretical_loss": 4.310356365821734, + "tokens_seen": 229441536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518876629889669, + "loss": 1.7276, + "theoretical_loss": 4.310206596428387, + "tokens_seen": 229507072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518716148445337, + "loss": 1.6907, + "theoretical_loss": 4.310056881766561, + "tokens_seen": 229572608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518555667001003, + "loss": 1.6095, + "theoretical_loss": 4.309907221800645, + "tokens_seen": 229638144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518395185556671, + "loss": 1.7901, + "theoretical_loss": 4.309757616495053, + "tokens_seen": 229703680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518234704112337, + "loss": 1.6816, + "theoretical_loss": 4.309608065814234, + "tokens_seen": 229769216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518074222668005, + "loss": 1.5572, + "theoretical_loss": 4.309458569722673, + "tokens_seen": 229834752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517913741223671, + "loss": 1.7167, + "theoretical_loss": 4.309309128184886, + "tokens_seen": 229900288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517753259779339, + "loss": 1.6913, + "theoretical_loss": 4.309159741165422, + "tokens_seen": 229965824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517592778335005, + "loss": 1.6231, + "theoretical_loss": 4.309010408628865, + "tokens_seen": 230031360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517432296890673, + "loss": 1.5985, + "theoretical_loss": 4.308861130539829, + "tokens_seen": 230096896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517271815446339, + "loss": 1.6563, + "theoretical_loss": 4.308711906862965, + "tokens_seen": 230162432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517111334002007, + "loss": 1.6262, + "theoretical_loss": 4.308562737562953, + "tokens_seen": 230227968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516950852557673, + "loss": 1.7089, + "theoretical_loss": 4.30841362260451, + "tokens_seen": 230293504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516790371113341, + "loss": 1.612, + "theoretical_loss": 4.308264561952381, + "tokens_seen": 230359040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516629889669007, + "loss": 1.6092, + "theoretical_loss": 4.30811555557135, + "tokens_seen": 230424576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516469408224675, + "loss": 1.6265, + "theoretical_loss": 4.307966603426227, + "tokens_seen": 230490112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516308926780341, + "loss": 1.7801, + "theoretical_loss": 4.307817705481861, + "tokens_seen": 230555648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516148445336009, + "loss": 1.7435, + "theoretical_loss": 4.307668861703131, + "tokens_seen": 230621184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515987963891675, + "loss": 1.6081, + "theoretical_loss": 4.307520072054947, + "tokens_seen": 230686720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515827482447343, + "loss": 1.5812, + "theoretical_loss": 4.307371336502254, + "tokens_seen": 230752256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515667001003009, + "loss": 1.6561, + "theoretical_loss": 4.307222655010031, + "tokens_seen": 230817792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515506519558677, + "loss": 1.6483, + "theoretical_loss": 4.307074027543287, + "tokens_seen": 230883328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515346038114343, + "loss": 1.7229, + "theoretical_loss": 4.3069254540670645, + "tokens_seen": 230948864 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515185556670011, + "loss": 1.7828, + "theoretical_loss": 4.306776934546436, + "tokens_seen": 231014400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515025075225677, + "loss": 1.5098, + "theoretical_loss": 4.306628468946512, + "tokens_seen": 231079936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514864593781345, + "loss": 1.55, + "theoretical_loss": 4.306480057232431, + "tokens_seen": 231145472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514704112337011, + "loss": 1.6725, + "theoretical_loss": 4.306331699369366, + "tokens_seen": 231211008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514543630892679, + "loss": 1.6515, + "theoretical_loss": 4.30618339532252, + "tokens_seen": 231276544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514383149448345, + "loss": 1.7387, + "theoretical_loss": 4.306035145057132, + "tokens_seen": 231342080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514222668004013, + "loss": 1.6858, + "theoretical_loss": 4.30588694853847, + "tokens_seen": 231407616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514062186559679, + "loss": 1.6754, + "theoretical_loss": 4.305738805731836, + "tokens_seen": 231473152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513901705115347, + "loss": 1.6655, + "theoretical_loss": 4.305590716602563, + "tokens_seen": 231538688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513741223671013, + "loss": 1.6448, + "theoretical_loss": 4.305442681116018, + "tokens_seen": 231604224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513580742226681, + "loss": 1.6165, + "theoretical_loss": 4.305294699237598, + "tokens_seen": 231669760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513420260782347, + "loss": 1.5662, + "theoretical_loss": 4.305146770932734, + "tokens_seen": 231735296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513259779338015, + "loss": 1.6911, + "theoretical_loss": 4.304998896166888, + "tokens_seen": 231800832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513099297893681, + "loss": 1.709, + "theoretical_loss": 4.304851074905554, + "tokens_seen": 231866368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512938816449349, + "loss": 1.6598, + "theoretical_loss": 4.304703307114258, + "tokens_seen": 231931904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512778335005015, + "loss": 1.6776, + "theoretical_loss": 4.304555592758559, + "tokens_seen": 231997440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512617853560683, + "loss": 1.5082, + "theoretical_loss": 4.304407931804047, + "tokens_seen": 232062976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512457372116349, + "loss": 1.6469, + "theoretical_loss": 4.304260324216344, + "tokens_seen": 232128512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512296890672017, + "loss": 1.6334, + "theoretical_loss": 4.304112769961102, + "tokens_seen": 232194048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512136409227683, + "loss": 1.532, + "theoretical_loss": 4.303965269004008, + "tokens_seen": 232259584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511975927783351, + "loss": 1.7242, + "theoretical_loss": 4.30381782131078, + "tokens_seen": 232325120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511815446339017, + "loss": 1.6597, + "theoretical_loss": 4.303670426847166, + "tokens_seen": 232390656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511654964894685, + "loss": 1.5863, + "theoretical_loss": 4.303523085578946, + "tokens_seen": 232456192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511494483450351, + "loss": 1.6315, + "theoretical_loss": 4.303375797471935, + "tokens_seen": 232521728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511334002006019, + "loss": 1.6744, + "theoretical_loss": 4.303228562491974, + "tokens_seen": 232587264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 83903, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6346615552902222, + "objective/train/theoretical_loss": 4.303081380604939, + "objective/train/tokens_used": 253112800, + "theoretical_loss": 4.303081380604939, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511173520561685, + "loss": 1.6698, + "theoretical_loss": 4.303081380604939, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511013039117353, + "loss": 1.5696, + "theoretical_loss": 4.302934251776739, + "tokens_seen": 232718336 + }, + { + "epoch": 0.07, + "learning_rate": 0.000751085255767302, + "loss": 1.7224, + "theoretical_loss": 4.302787175973311, + "tokens_seen": 232783872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510692076228687, + "loss": 1.7526, + "theoretical_loss": 4.302640153160625, + "tokens_seen": 232849408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510531594784353, + "loss": 1.6315, + "theoretical_loss": 4.302493183304683, + "tokens_seen": 232914944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510371113340021, + "loss": 1.6435, + "theoretical_loss": 4.302346266371517, + "tokens_seen": 232980480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510210631895687, + "loss": 1.6499, + "theoretical_loss": 4.302199402327191, + "tokens_seen": 233046016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510050150451355, + "loss": 1.6708, + "theoretical_loss": 4.302052591137802, + "tokens_seen": 233111552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509889669007021, + "loss": 1.6101, + "theoretical_loss": 4.301905832769474, + "tokens_seen": 233177088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509729187562688, + "loss": 1.6334, + "theoretical_loss": 4.301759127188367, + "tokens_seen": 233242624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509568706118356, + "loss": 1.7197, + "theoretical_loss": 4.301612474360669, + "tokens_seen": 233308160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509408224674022, + "loss": 1.7176, + "theoretical_loss": 4.301465874252601, + "tokens_seen": 233373696 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750924774322969, + "loss": 1.7187, + "theoretical_loss": 4.301319326830413, + "tokens_seen": 233439232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509087261785356, + "loss": 1.7268, + "theoretical_loss": 4.301172832060389, + "tokens_seen": 233504768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007508926780341024, + "loss": 1.6295, + "theoretical_loss": 4.3010263899088415, + "tokens_seen": 233570304 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750876629889669, + "loss": 1.6408, + "theoretical_loss": 4.300880000342114, + "tokens_seen": 233635840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007508605817452358, + "loss": 1.6239, + "theoretical_loss": 4.300733663326583, + "tokens_seen": 233701376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007508445336008024, + "loss": 1.5342, + "theoretical_loss": 4.300587378828655, + "tokens_seen": 233766912 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750828485456369, + "loss": 1.7235, + "theoretical_loss": 4.300441146814766, + "tokens_seen": 233832448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007508124373119358, + "loss": 1.619, + "theoretical_loss": 4.300294967251386, + "tokens_seen": 233897984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507963891675024, + "loss": 1.6433, + "theoretical_loss": 4.300148840105011, + "tokens_seen": 233963520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507803410230692, + "loss": 1.5639, + "theoretical_loss": 4.300002765342173, + "tokens_seen": 234029056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507642928786359, + "loss": 1.672, + "theoretical_loss": 4.299856742929431, + "tokens_seen": 234094592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507482447342026, + "loss": 1.6359, + "theoretical_loss": 4.299710772833377, + "tokens_seen": 234160128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507321965897694, + "loss": 1.7174, + "theoretical_loss": 4.299564855020631, + "tokens_seen": 234225664 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750716148445336, + "loss": 1.5812, + "theoretical_loss": 4.299418989457848, + "tokens_seen": 234291200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507001003009028, + "loss": 1.6371, + "theoretical_loss": 4.299273176111709, + "tokens_seen": 234356736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506840521564694, + "loss": 1.6428, + "theoretical_loss": 4.299127414948927, + "tokens_seen": 234422272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506680040120362, + "loss": 1.654, + "theoretical_loss": 4.298981705936248, + "tokens_seen": 234487808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506519558676028, + "loss": 1.7671, + "theoretical_loss": 4.298836049040444, + "tokens_seen": 234553344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506359077231696, + "loss": 1.7702, + "theoretical_loss": 4.298690444228322, + "tokens_seen": 234618880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506198595787362, + "loss": 1.5735, + "theoretical_loss": 4.298544891466717, + "tokens_seen": 234684416 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750603811434303, + "loss": 1.6077, + "theoretical_loss": 4.298399390722493, + "tokens_seen": 234749952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505877632898696, + "loss": 1.725, + "theoretical_loss": 4.298253941962548, + "tokens_seen": 234815488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505717151454364, + "loss": 1.5374, + "theoretical_loss": 4.2981085451538075, + "tokens_seen": 234881024 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750555667001003, + "loss": 1.6508, + "theoretical_loss": 4.297963200263228, + "tokens_seen": 234946560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505396188565698, + "loss": 1.7077, + "theoretical_loss": 4.297817907257797, + "tokens_seen": 235012096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505235707121364, + "loss": 1.6419, + "theoretical_loss": 4.29767266610453, + "tokens_seen": 235077632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505075225677032, + "loss": 1.6664, + "theoretical_loss": 4.297527476770476, + "tokens_seen": 235143168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504914744232698, + "loss": 1.6296, + "theoretical_loss": 4.297382339222711, + "tokens_seen": 235208704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504754262788366, + "loss": 1.693, + "theoretical_loss": 4.297237253428342, + "tokens_seen": 235274240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504593781344032, + "loss": 1.8014, + "theoretical_loss": 4.297092219354509, + "tokens_seen": 235339776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075044332998997, + "loss": 1.7015, + "theoretical_loss": 4.296947236968376, + "tokens_seen": 235405312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504272818455366, + "loss": 1.6181, + "theoretical_loss": 4.296802306237143, + "tokens_seen": 235470848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504112337011034, + "loss": 1.7072, + "theoretical_loss": 4.296657427128036, + "tokens_seen": 235536384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075039518555667, + "loss": 1.629, + "theoretical_loss": 4.296512599608311, + "tokens_seen": 235601920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503791374122368, + "loss": 1.6502, + "theoretical_loss": 4.296367823645259, + "tokens_seen": 235667456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503630892678034, + "loss": 1.671, + "theoretical_loss": 4.296223099206193, + "tokens_seen": 235732992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503470411233702, + "loss": 1.7477, + "theoretical_loss": 4.296078426258462, + "tokens_seen": 235798528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503309929789368, + "loss": 1.521, + "theoretical_loss": 4.295933804769442, + "tokens_seen": 235864064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 84743, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6421600580215454, + "objective/train/theoretical_loss": 4.295789234706538, + "objective/train/tokens_used": 256389600, + "theoretical_loss": 4.295789234706538, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503149448345036, + "loss": 1.7047, + "theoretical_loss": 4.295789234706538, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502988966900702, + "loss": 1.7024, + "theoretical_loss": 4.295644716037188, + "tokens_seen": 235995136 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750282848545637, + "loss": 1.7471, + "theoretical_loss": 4.295500248728856, + "tokens_seen": 236060672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502668004012036, + "loss": 1.7045, + "theoretical_loss": 4.295355832749038, + "tokens_seen": 236126208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502507522567704, + "loss": 1.7213, + "theoretical_loss": 4.295211468065258, + "tokens_seen": 236191744 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750234704112337, + "loss": 1.715, + "theoretical_loss": 4.295067154645071, + "tokens_seen": 236257280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502186559679038, + "loss": 1.6261, + "theoretical_loss": 4.294922892456061, + "tokens_seen": 236322816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502026078234704, + "loss": 1.601, + "theoretical_loss": 4.294778681465841, + "tokens_seen": 236388352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501865596790372, + "loss": 1.4909, + "theoretical_loss": 4.294634521642053, + "tokens_seen": 236453888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501705115346038, + "loss": 1.7144, + "theoretical_loss": 4.294490412952371, + "tokens_seen": 236519424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501544633901706, + "loss": 1.7742, + "theoretical_loss": 4.294346355364494, + "tokens_seen": 236584960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501384152457372, + "loss": 1.678, + "theoretical_loss": 4.294202348846156, + "tokens_seen": 236650496 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750122367101304, + "loss": 1.6133, + "theoretical_loss": 4.294058393365114, + "tokens_seen": 236716032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501063189568706, + "loss": 1.685, + "theoretical_loss": 4.293914488889161, + "tokens_seen": 236781568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500902708124374, + "loss": 1.6133, + "theoretical_loss": 4.293770635386112, + "tokens_seen": 236847104 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750074222668004, + "loss": 1.716, + "theoretical_loss": 4.293626832823818, + "tokens_seen": 236912640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500581745235708, + "loss": 1.6319, + "theoretical_loss": 4.293483081170153, + "tokens_seen": 236978176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500421263791374, + "loss": 1.5912, + "theoretical_loss": 4.293339380393027, + "tokens_seen": 237043712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500260782347042, + "loss": 1.6264, + "theoretical_loss": 4.293195730460372, + "tokens_seen": 237109248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500100300902708, + "loss": 1.5781, + "theoretical_loss": 4.293052131340154, + "tokens_seen": 237174784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007499939819458376, + "loss": 1.7007, + "theoretical_loss": 4.292908583000365, + "tokens_seen": 237240320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007499779338014042, + "loss": 1.663, + "theoretical_loss": 4.29276508540903, + "tokens_seen": 237305856 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749961885656971, + "loss": 1.5671, + "theoretical_loss": 4.292621638534198, + "tokens_seen": 237371392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007499458375125376, + "loss": 1.7502, + "theoretical_loss": 4.29247824234395, + "tokens_seen": 237436928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007499297893681044, + "loss": 1.715, + "theoretical_loss": 4.292334896806397, + "tokens_seen": 237502464 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749913741223671, + "loss": 1.6047, + "theoretical_loss": 4.292191601889673, + "tokens_seen": 237568000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498976930792378, + "loss": 1.6171, + "theoretical_loss": 4.2920483575619475, + "tokens_seen": 237633536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498816449348044, + "loss": 1.7442, + "theoretical_loss": 4.291905163791417, + "tokens_seen": 237699072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498655967903712, + "loss": 1.6901, + "theoretical_loss": 4.291762020546304, + "tokens_seen": 237764608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498495486459378, + "loss": 1.6801, + "theoretical_loss": 4.291618927794862, + "tokens_seen": 237830144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498335005015046, + "loss": 1.669, + "theoretical_loss": 4.291475885505374, + "tokens_seen": 237895680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498174523570712, + "loss": 1.6468, + "theoretical_loss": 4.291332893646149, + "tokens_seen": 237961216 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749801404212638, + "loss": 1.5978, + "theoretical_loss": 4.291189952185527, + "tokens_seen": 238026752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497853560682046, + "loss": 1.5644, + "theoretical_loss": 4.291047061091875, + "tokens_seen": 238092288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497693079237714, + "loss": 1.5605, + "theoretical_loss": 4.290904220333591, + "tokens_seen": 238157824 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749753259779338, + "loss": 1.6864, + "theoretical_loss": 4.2907614298790975, + "tokens_seen": 238223360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497372116349048, + "loss": 1.685, + "theoretical_loss": 4.290618689696848, + "tokens_seen": 238288896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497211634904714, + "loss": 1.6605, + "theoretical_loss": 4.290475999755326, + "tokens_seen": 238354432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497051153460382, + "loss": 1.7451, + "theoretical_loss": 4.29033336002304, + "tokens_seen": 238419968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496890672016048, + "loss": 1.5776, + "theoretical_loss": 4.290190770468529, + "tokens_seen": 238485504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496730190571716, + "loss": 1.6556, + "theoretical_loss": 4.290048231060361, + "tokens_seen": 238551040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496569709127382, + "loss": 1.5922, + "theoretical_loss": 4.289905741767129, + "tokens_seen": 238616576 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749640922768305, + "loss": 1.6411, + "theoretical_loss": 4.289763302557457, + "tokens_seen": 238682112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496248746238716, + "loss": 1.7339, + "theoretical_loss": 4.289620913399998, + "tokens_seen": 238747648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496088264794384, + "loss": 1.61, + "theoretical_loss": 4.2894785742634305, + "tokens_seen": 238813184 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749592778335005, + "loss": 1.6489, + "theoretical_loss": 4.289336285116463, + "tokens_seen": 238878720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495767301905718, + "loss": 1.6366, + "theoretical_loss": 4.289194045927832, + "tokens_seen": 238944256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495606820461385, + "loss": 1.57, + "theoretical_loss": 4.289051856666303, + "tokens_seen": 239009792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495446339017051, + "loss": 1.659, + "theoretical_loss": 4.288909717300667, + "tokens_seen": 239075328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495285857572719, + "loss": 1.6939, + "theoretical_loss": 4.288767627799745, + "tokens_seen": 239140864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 85669, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6066745519638062, + "objective/train/theoretical_loss": 4.288625588132385, + "objective/train/tokens_used": 259666400, + "theoretical_loss": 4.288625588132385, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495125376128385, + "loss": 1.6276, + "theoretical_loss": 4.288625588132385, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494964894684053, + "loss": 1.715, + "theoretical_loss": 4.288483598267465, + "tokens_seen": 239271936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494804413239719, + "loss": 1.5889, + "theoretical_loss": 4.288341658173888, + "tokens_seen": 239337472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494643931795387, + "loss": 1.7141, + "theoretical_loss": 4.288199767820587, + "tokens_seen": 239403008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494483450351053, + "loss": 1.682, + "theoretical_loss": 4.2880579271765225, + "tokens_seen": 239468544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494322968906721, + "loss": 1.5611, + "theoretical_loss": 4.287916136210682, + "tokens_seen": 239534080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494162487462387, + "loss": 1.6503, + "theoretical_loss": 4.287774394892082, + "tokens_seen": 239599616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494002006018055, + "loss": 1.6604, + "theoretical_loss": 4.287632703189766, + "tokens_seen": 239665152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493841524573721, + "loss": 1.5737, + "theoretical_loss": 4.2874910610728065, + "tokens_seen": 239730688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493681043129389, + "loss": 1.6437, + "theoretical_loss": 4.287349468510302, + "tokens_seen": 239796224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493520561685055, + "loss": 1.4903, + "theoretical_loss": 4.28720792547138, + "tokens_seen": 239861760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493360080240723, + "loss": 1.6124, + "theoretical_loss": 4.287066431925194, + "tokens_seen": 239927296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493199598796389, + "loss": 1.5895, + "theoretical_loss": 4.286924987840926, + "tokens_seen": 239992832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493039117352057, + "loss": 1.7164, + "theoretical_loss": 4.2867835931877885, + "tokens_seen": 240058368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492878635907723, + "loss": 1.6419, + "theoretical_loss": 4.286642247935017, + "tokens_seen": 240123904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492718154463391, + "loss": 1.6765, + "theoretical_loss": 4.286500952051876, + "tokens_seen": 240189440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492557673019057, + "loss": 1.5405, + "theoretical_loss": 4.286359705507659, + "tokens_seen": 240254976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492397191574725, + "loss": 1.558, + "theoretical_loss": 4.286218508271686, + "tokens_seen": 240320512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492236710130391, + "loss": 1.5783, + "theoretical_loss": 4.286077360313303, + "tokens_seen": 240386048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492076228686059, + "loss": 1.5213, + "theoretical_loss": 4.285936261601887, + "tokens_seen": 240451584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491915747241725, + "loss": 1.5629, + "theoretical_loss": 4.285795212106839, + "tokens_seen": 240517120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491755265797393, + "loss": 1.7373, + "theoretical_loss": 4.2856542117975875, + "tokens_seen": 240582656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491594784353059, + "loss": 1.7176, + "theoretical_loss": 4.285513260643591, + "tokens_seen": 240648192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491434302908727, + "loss": 1.6888, + "theoretical_loss": 4.285372358614332, + "tokens_seen": 240713728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491273821464393, + "loss": 1.6917, + "theoretical_loss": 4.285231505679324, + "tokens_seen": 240779264 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491113340020061, + "loss": 1.6539, + "theoretical_loss": 4.285090701808103, + "tokens_seen": 240844800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490952858575727, + "loss": 1.612, + "theoretical_loss": 4.284949946970236, + "tokens_seen": 240910336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490792377131395, + "loss": 1.7414, + "theoretical_loss": 4.284809241135315, + "tokens_seen": 240975872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490631895687061, + "loss": 1.6637, + "theoretical_loss": 4.284668584272962, + "tokens_seen": 241041408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490471414242729, + "loss": 1.6183, + "theoretical_loss": 4.284527976352823, + "tokens_seen": 241106944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490310932798395, + "loss": 1.6879, + "theoretical_loss": 4.284387417344572, + "tokens_seen": 241172480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490150451354063, + "loss": 1.6791, + "theoretical_loss": 4.28424690721791, + "tokens_seen": 241238016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489989969909729, + "loss": 1.7164, + "theoretical_loss": 4.284106445942565, + "tokens_seen": 241303552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489829488465397, + "loss": 1.6794, + "theoretical_loss": 4.2839660334882925, + "tokens_seen": 241369088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489669007021063, + "loss": 1.4591, + "theoretical_loss": 4.283825669824875, + "tokens_seen": 241434624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489508525576731, + "loss": 1.6535, + "theoretical_loss": 4.28368535492212, + "tokens_seen": 241500160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489348044132397, + "loss": 1.6211, + "theoretical_loss": 4.283545088749865, + "tokens_seen": 241565696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489187562688065, + "loss": 1.6274, + "theoretical_loss": 4.2834048712779715, + "tokens_seen": 241631232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489027081243731, + "loss": 1.5905, + "theoretical_loss": 4.28326470247633, + "tokens_seen": 241696768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488866599799399, + "loss": 1.6188, + "theoretical_loss": 4.283124582314856, + "tokens_seen": 241762304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488706118355065, + "loss": 1.5419, + "theoretical_loss": 4.282984510763493, + "tokens_seen": 241827840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488545636910733, + "loss": 1.6195, + "theoretical_loss": 4.282844487792209, + "tokens_seen": 241893376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488385155466399, + "loss": 1.6174, + "theoretical_loss": 4.282704513371002, + "tokens_seen": 241958912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488224674022067, + "loss": 1.5593, + "theoretical_loss": 4.282564587469896, + "tokens_seen": 242024448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488064192577733, + "loss": 1.5889, + "theoretical_loss": 4.282424710058938, + "tokens_seen": 242089984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487903711133401, + "loss": 1.5381, + "theoretical_loss": 4.282284881108206, + "tokens_seen": 242155520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487743229689067, + "loss": 1.6208, + "theoretical_loss": 4.282145100587802, + "tokens_seen": 242221056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487582748244735, + "loss": 1.6344, + "theoretical_loss": 4.282005368467856, + "tokens_seen": 242286592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487422266800401, + "loss": 1.6803, + "theoretical_loss": 4.2818656847185235, + "tokens_seen": 242352128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487261785356069, + "loss": 1.523, + "theoretical_loss": 4.281726049309987, + "tokens_seen": 242417664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 86947, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.533555507659912, + "objective/train/theoretical_loss": 4.281586462212455, + "objective/train/tokens_used": 262943200, + "theoretical_loss": 4.281586462212455, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487101303911735, + "loss": 1.5863, + "theoretical_loss": 4.281586462212455, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486940822467403, + "loss": 1.6372, + "theoretical_loss": 4.281446923396162, + "tokens_seen": 242548736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486780341023069, + "loss": 1.6006, + "theoretical_loss": 4.281307432831371, + "tokens_seen": 242614272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486619859578737, + "loss": 1.5637, + "theoretical_loss": 4.281167990488369, + "tokens_seen": 242679808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486459378134403, + "loss": 1.6963, + "theoretical_loss": 4.281028596337469, + "tokens_seen": 242745344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486298896690071, + "loss": 1.5761, + "theoretical_loss": 4.280889250349014, + "tokens_seen": 242810880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486138415245737, + "loss": 1.5397, + "theoretical_loss": 4.280749952493368, + "tokens_seen": 242876416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485977933801405, + "loss": 1.6975, + "theoretical_loss": 4.280610702740926, + "tokens_seen": 242941952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485817452357071, + "loss": 1.5826, + "theoretical_loss": 4.280471501062106, + "tokens_seen": 243007488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485656970912739, + "loss": 1.7064, + "theoretical_loss": 4.280332347427353, + "tokens_seen": 243073024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485496489468405, + "loss": 1.6329, + "theoretical_loss": 4.280193241807139, + "tokens_seen": 243138560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485336008024073, + "loss": 1.5267, + "theoretical_loss": 4.280054184171963, + "tokens_seen": 243204096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485175526579739, + "loss": 1.6792, + "theoretical_loss": 4.279915174492345, + "tokens_seen": 243269632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485015045135407, + "loss": 1.638, + "theoretical_loss": 4.279776212738838, + "tokens_seen": 243335168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484854563691073, + "loss": 1.6731, + "theoretical_loss": 4.279637298882016, + "tokens_seen": 243400704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484694082246741, + "loss": 1.5625, + "theoretical_loss": 4.27949843289248, + "tokens_seen": 243466240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484533600802407, + "loss": 1.6964, + "theoretical_loss": 4.2793596147408595, + "tokens_seen": 243531776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484373119358075, + "loss": 1.6091, + "theoretical_loss": 4.279220844397806, + "tokens_seen": 243597312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484212637913741, + "loss": 1.6271, + "theoretical_loss": 4.279082121834001, + "tokens_seen": 243662848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484052156469409, + "loss": 1.6291, + "theoretical_loss": 4.278943447020148, + "tokens_seen": 243728384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483891675025075, + "loss": 1.7249, + "theoretical_loss": 4.278804819926978, + "tokens_seen": 243793920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483731193580743, + "loss": 1.656, + "theoretical_loss": 4.278666240525249, + "tokens_seen": 243859456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483570712136409, + "loss": 1.7174, + "theoretical_loss": 4.278527708785743, + "tokens_seen": 243924992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483410230692077, + "loss": 1.6751, + "theoretical_loss": 4.278389224679268, + "tokens_seen": 243990528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483249749247745, + "loss": 1.4917, + "theoretical_loss": 4.27825078817666, + "tokens_seen": 244056064 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483089267803411, + "loss": 1.5895, + "theoretical_loss": 4.2781123992487755, + "tokens_seen": 244121600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482928786359077, + "loss": 1.6703, + "theoretical_loss": 4.277974057866503, + "tokens_seen": 244187136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482768304914745, + "loss": 1.5676, + "theoretical_loss": 4.277835764000752, + "tokens_seen": 244252672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482607823470411, + "loss": 1.6488, + "theoretical_loss": 4.277697517622459, + "tokens_seen": 244318208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482447342026079, + "loss": 1.6331, + "theoretical_loss": 4.277559318702588, + "tokens_seen": 244383744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482286860581746, + "loss": 1.5206, + "theoretical_loss": 4.277421167212125, + "tokens_seen": 244449280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482126379137413, + "loss": 1.6105, + "theoretical_loss": 4.277283063122083, + "tokens_seen": 244514816 + }, + { + "epoch": 0.07, + "learning_rate": 0.000748196589769308, + "loss": 1.6427, + "theoretical_loss": 4.277145006403503, + "tokens_seen": 244580352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481805416248747, + "loss": 1.7534, + "theoretical_loss": 4.2770069970274465, + "tokens_seen": 244645888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481644934804414, + "loss": 1.6926, + "theoretical_loss": 4.276869034965005, + "tokens_seen": 244711424 + }, + { + "epoch": 0.07, + "learning_rate": 0.000748148445336008, + "loss": 1.7342, + "theoretical_loss": 4.276731120187293, + "tokens_seen": 244776960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481323971915748, + "loss": 1.6409, + "theoretical_loss": 4.2765932526654495, + "tokens_seen": 244842496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481163490471414, + "loss": 1.6483, + "theoretical_loss": 4.276455432370643, + "tokens_seen": 244908032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481003009027082, + "loss": 1.5419, + "theoretical_loss": 4.276317659274062, + "tokens_seen": 244973568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480842527582748, + "loss": 1.5604, + "theoretical_loss": 4.276179933346924, + "tokens_seen": 245039104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480682046138416, + "loss": 1.4418, + "theoretical_loss": 4.27604225456047, + "tokens_seen": 245104640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480521564694082, + "loss": 1.6223, + "theoretical_loss": 4.275904622885967, + "tokens_seen": 245170176 + }, + { + "epoch": 0.07, + "learning_rate": 0.000748036108324975, + "loss": 1.6681, + "theoretical_loss": 4.275767038294708, + "tokens_seen": 245235712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480200601805416, + "loss": 1.4692, + "theoretical_loss": 4.275629500758008, + "tokens_seen": 245301248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480040120361084, + "loss": 1.5969, + "theoretical_loss": 4.27549201024721, + "tokens_seen": 245366784 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747987963891675, + "loss": 1.6522, + "theoretical_loss": 4.275354566733682, + "tokens_seen": 245432320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479719157472418, + "loss": 1.6379, + "theoretical_loss": 4.2752171701888155, + "tokens_seen": 245497856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479558676028084, + "loss": 1.5753, + "theoretical_loss": 4.275079820584029, + "tokens_seen": 245563392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479398194583752, + "loss": 1.6753, + "theoretical_loss": 4.274942517890764, + "tokens_seen": 245628928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479237713139418, + "loss": 1.6301, + "theoretical_loss": 4.2748052620804895, + "tokens_seen": 245694464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 87859, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.655349850654602, + "objective/train/theoretical_loss": 4.274668053124696, + "objective/train/tokens_used": 266220000, + "theoretical_loss": 4.274668053124696, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479077231695086, + "loss": 1.6396, + "theoretical_loss": 4.274668053124696, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007478916750250752, + "loss": 1.5806, + "theoretical_loss": 4.2745308909949005, + "tokens_seen": 245825536 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747875626880642, + "loss": 1.7082, + "theoretical_loss": 4.274393775662648, + "tokens_seen": 245891072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007478595787362086, + "loss": 1.6241, + "theoretical_loss": 4.274256707099502, + "tokens_seen": 245956608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007478435305917754, + "loss": 1.5844, + "theoretical_loss": 4.274119685277058, + "tokens_seen": 246022144 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747827482447342, + "loss": 1.5921, + "theoretical_loss": 4.273982710166931, + "tokens_seen": 246087680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007478114343029088, + "loss": 1.6682, + "theoretical_loss": 4.273845781740762, + "tokens_seen": 246153216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477953861584754, + "loss": 1.5675, + "theoretical_loss": 4.273708899970218, + "tokens_seen": 246218752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477793380140422, + "loss": 1.7248, + "theoretical_loss": 4.273572064826991, + "tokens_seen": 246284288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477632898696088, + "loss": 1.6493, + "theoretical_loss": 4.273435276282795, + "tokens_seen": 246349824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477472417251756, + "loss": 1.6431, + "theoretical_loss": 4.2732985343093715, + "tokens_seen": 246415360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477311935807422, + "loss": 1.5834, + "theoretical_loss": 4.273161838878485, + "tokens_seen": 246480896 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747715145436309, + "loss": 1.649, + "theoretical_loss": 4.273025189961926, + "tokens_seen": 246546432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476990972918756, + "loss": 1.5257, + "theoretical_loss": 4.272888587531508, + "tokens_seen": 246611968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476830491474424, + "loss": 1.7432, + "theoretical_loss": 4.272752031559071, + "tokens_seen": 246677504 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747667001003009, + "loss": 1.5807, + "theoretical_loss": 4.272615522016476, + "tokens_seen": 246743040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476509528585758, + "loss": 1.5903, + "theoretical_loss": 4.272479058875614, + "tokens_seen": 246808576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476349047141424, + "loss": 1.5591, + "theoretical_loss": 4.272342642108396, + "tokens_seen": 246874112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476188565697092, + "loss": 1.5049, + "theoretical_loss": 4.272206271686758, + "tokens_seen": 246939648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476028084252758, + "loss": 1.6674, + "theoretical_loss": 4.272069947582662, + "tokens_seen": 247005184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007475867602808426, + "loss": 1.5916, + "theoretical_loss": 4.271933669768094, + "tokens_seen": 247070720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007475707121364092, + "loss": 1.6944, + "theoretical_loss": 4.271797438215064, + "tokens_seen": 247136256 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747554663991976, + "loss": 1.5855, + "theoretical_loss": 4.271661252895605, + "tokens_seen": 247201792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007475386158475426, + "loss": 1.7893, + "theoretical_loss": 4.271525113781777, + "tokens_seen": 247267328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007475225677031094, + "loss": 1.6591, + "theoretical_loss": 4.271389020845662, + "tokens_seen": 247332864 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747506519558676, + "loss": 1.6706, + "theoretical_loss": 4.27125297405937, + "tokens_seen": 247398400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007474904714142428, + "loss": 1.6419, + "theoretical_loss": 4.271116973395028, + "tokens_seen": 247463936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474744232698094, + "loss": 1.7433, + "theoretical_loss": 4.270981018824795, + "tokens_seen": 247529472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474583751253762, + "loss": 1.6313, + "theoretical_loss": 4.270845110320851, + "tokens_seen": 247595008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474423269809428, + "loss": 1.6649, + "theoretical_loss": 4.270709247855397, + "tokens_seen": 247660544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474262788365096, + "loss": 1.5666, + "theoretical_loss": 4.270573431400663, + "tokens_seen": 247726080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474102306920762, + "loss": 1.729, + "theoretical_loss": 4.270437660928902, + "tokens_seen": 247791616 + }, + { + "epoch": 0.08, + "learning_rate": 0.000747394182547643, + "loss": 1.6638, + "theoretical_loss": 4.270301936412389, + "tokens_seen": 247857152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007473781344032096, + "loss": 1.6416, + "theoretical_loss": 4.2701662578234245, + "tokens_seen": 247922688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007473620862587764, + "loss": 1.5788, + "theoretical_loss": 4.270030625134334, + "tokens_seen": 247988224 + }, + { + "epoch": 0.08, + "learning_rate": 0.000747346038114343, + "loss": 1.5882, + "theoretical_loss": 4.269895038317464, + "tokens_seen": 248053760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007473299899699098, + "loss": 1.5104, + "theoretical_loss": 4.269759497345188, + "tokens_seen": 248119296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007473139418254764, + "loss": 1.6717, + "theoretical_loss": 4.269624002189901, + "tokens_seen": 248184832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472978936810432, + "loss": 1.7941, + "theoretical_loss": 4.269488552824024, + "tokens_seen": 248250368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472818455366098, + "loss": 1.7017, + "theoretical_loss": 4.269353149220001, + "tokens_seen": 248315904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472657973921766, + "loss": 1.5691, + "theoretical_loss": 4.2692177913503, + "tokens_seen": 248381440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472497492477432, + "loss": 1.5977, + "theoretical_loss": 4.2690824791874125, + "tokens_seen": 248446976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074723370110331, + "loss": 1.6137, + "theoretical_loss": 4.2689472127038535, + "tokens_seen": 248512512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472176529588766, + "loss": 1.5904, + "theoretical_loss": 4.2688119918721625, + "tokens_seen": 248578048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472016048144434, + "loss": 1.6154, + "theoretical_loss": 4.268676816664902, + "tokens_seen": 248643584 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074718555667001, + "loss": 1.6636, + "theoretical_loss": 4.26854168705466, + "tokens_seen": 248709120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007471695085255768, + "loss": 1.6526, + "theoretical_loss": 4.2684066030140455, + "tokens_seen": 248774656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007471534603811434, + "loss": 1.5885, + "theoretical_loss": 4.268271564515693, + "tokens_seen": 248840192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007471374122367102, + "loss": 1.6694, + "theoretical_loss": 4.26813657153226, + "tokens_seen": 248905728 + }, + { + "epoch": 0.08, + "learning_rate": 0.000747121364092277, + "loss": 1.6456, + "theoretical_loss": 4.268001624036429, + "tokens_seen": 248971264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 89113, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5617129802703857, + "objective/train/theoretical_loss": 4.267866722000902, + "objective/train/tokens_used": 269496800, + "theoretical_loss": 4.267866722000902, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007471053159478436, + "loss": 1.6179, + "theoretical_loss": 4.267866722000902, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470892678034103, + "loss": 1.8281, + "theoretical_loss": 4.26773186539841, + "tokens_seen": 249102336 + }, + { + "epoch": 0.08, + "learning_rate": 0.000747073219658977, + "loss": 1.6578, + "theoretical_loss": 4.267597054201705, + "tokens_seen": 249167872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470571715145438, + "loss": 1.7296, + "theoretical_loss": 4.267462288383561, + "tokens_seen": 249233408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470411233701104, + "loss": 1.5899, + "theoretical_loss": 4.267327567916777, + "tokens_seen": 249298944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470250752256772, + "loss": 1.6601, + "theoretical_loss": 4.267192892774176, + "tokens_seen": 249364480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470090270812438, + "loss": 1.5475, + "theoretical_loss": 4.267058262928604, + "tokens_seen": 249430016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007469929789368106, + "loss": 1.6116, + "theoretical_loss": 4.26692367835293, + "tokens_seen": 249495552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007469769307923772, + "loss": 1.4896, + "theoretical_loss": 4.266789139020045, + "tokens_seen": 249561088 + }, + { + "epoch": 0.08, + "learning_rate": 0.000746960882647944, + "loss": 1.734, + "theoretical_loss": 4.266654644902868, + "tokens_seen": 249626624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007469448345035106, + "loss": 1.6688, + "theoretical_loss": 4.266520195974336, + "tokens_seen": 249692160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007469287863590774, + "loss": 1.6633, + "theoretical_loss": 4.266385792207412, + "tokens_seen": 249757696 + }, + { + "epoch": 0.08, + "learning_rate": 0.000746912738214644, + "loss": 1.7313, + "theoretical_loss": 4.266251433575082, + "tokens_seen": 249823232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468966900702106, + "loss": 1.6623, + "theoretical_loss": 4.266117120050355, + "tokens_seen": 249888768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468806419257774, + "loss": 1.659, + "theoretical_loss": 4.265982851606263, + "tokens_seen": 249954304 + }, + { + "epoch": 0.08, + "learning_rate": 0.000746864593781344, + "loss": 1.7441, + "theoretical_loss": 4.265848628215862, + "tokens_seen": 250019840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468485456369108, + "loss": 1.7166, + "theoretical_loss": 4.265714449852229, + "tokens_seen": 250085376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468324974924775, + "loss": 1.7004, + "theoretical_loss": 4.265580316488467, + "tokens_seen": 250150912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468164493480442, + "loss": 1.6103, + "theoretical_loss": 4.2654462280977015, + "tokens_seen": 250216448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468004012036109, + "loss": 1.5756, + "theoretical_loss": 4.265312184653078, + "tokens_seen": 250281984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467843530591776, + "loss": 1.6858, + "theoretical_loss": 4.2651781861277716, + "tokens_seen": 250347520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467683049147443, + "loss": 1.7365, + "theoretical_loss": 4.265044232494972, + "tokens_seen": 250413056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467522567703109, + "loss": 1.7981, + "theoretical_loss": 4.264910323727898, + "tokens_seen": 250478592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467362086258777, + "loss": 1.6383, + "theoretical_loss": 4.264776459799791, + "tokens_seen": 250544128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467201604814443, + "loss": 1.6194, + "theoretical_loss": 4.264642640683911, + "tokens_seen": 250609664 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467041123370111, + "loss": 1.6848, + "theoretical_loss": 4.264508866353546, + "tokens_seen": 250675200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466880641925777, + "loss": 1.7286, + "theoretical_loss": 4.264375136782004, + "tokens_seen": 250740736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466720160481445, + "loss": 1.7089, + "theoretical_loss": 4.264241451942617, + "tokens_seen": 250806272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466559679037111, + "loss": 1.5735, + "theoretical_loss": 4.264107811808739, + "tokens_seen": 250871808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466399197592779, + "loss": 1.6118, + "theoretical_loss": 4.263974216353749, + "tokens_seen": 250937344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466238716148445, + "loss": 1.5047, + "theoretical_loss": 4.2638406655510455, + "tokens_seen": 251002880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466078234704113, + "loss": 1.6449, + "theoretical_loss": 4.263707159374051, + "tokens_seen": 251068416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465917753259779, + "loss": 1.5712, + "theoretical_loss": 4.263573697796213, + "tokens_seen": 251133952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465757271815447, + "loss": 1.6243, + "theoretical_loss": 4.263440280790999, + "tokens_seen": 251199488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465596790371113, + "loss": 1.6167, + "theoretical_loss": 4.263306908331899, + "tokens_seen": 251265024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465436308926781, + "loss": 1.6755, + "theoretical_loss": 4.2631735803924276, + "tokens_seen": 251330560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465275827482447, + "loss": 1.7118, + "theoretical_loss": 4.263040296946122, + "tokens_seen": 251396096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465115346038115, + "loss": 1.7401, + "theoretical_loss": 4.26290705796654, + "tokens_seen": 251461632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464954864593781, + "loss": 1.6834, + "theoretical_loss": 4.262773863427265, + "tokens_seen": 251527168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464794383149449, + "loss": 1.7491, + "theoretical_loss": 4.262640713301899, + "tokens_seen": 251592704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464633901705115, + "loss": 1.5683, + "theoretical_loss": 4.26250760756407, + "tokens_seen": 251658240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464473420260783, + "loss": 1.6373, + "theoretical_loss": 4.262374546187428, + "tokens_seen": 251723776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464312938816449, + "loss": 1.4961, + "theoretical_loss": 4.262241529145643, + "tokens_seen": 251789312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464152457372117, + "loss": 1.6324, + "theoretical_loss": 4.262108556412411, + "tokens_seen": 251854848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463991975927783, + "loss": 1.8033, + "theoretical_loss": 4.261975627961448, + "tokens_seen": 251920384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463831494483451, + "loss": 1.7477, + "theoretical_loss": 4.2618427437664925, + "tokens_seen": 251985920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463671013039117, + "loss": 1.6968, + "theoretical_loss": 4.261709903801307, + "tokens_seen": 252051456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463510531594785, + "loss": 1.6687, + "theoretical_loss": 4.261577108039674, + "tokens_seen": 252116992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463350050150451, + "loss": 1.7375, + "theoretical_loss": 4.261444356455402, + "tokens_seen": 252182528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463189568706119, + "loss": 1.6986, + "theoretical_loss": 4.261311649022318, + "tokens_seen": 252248064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 90341, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.549142599105835, + "objective/train/theoretical_loss": 4.261178985714273, + "objective/train/tokens_used": 272773600, + "theoretical_loss": 4.261178985714273, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463029087261785, + "loss": 1.6768, + "theoretical_loss": 4.261178985714273, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462868605817453, + "loss": 1.5756, + "theoretical_loss": 4.26104636650514, + "tokens_seen": 252379136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462708124373119, + "loss": 1.4787, + "theoretical_loss": 4.260913791368815, + "tokens_seen": 252444672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462547642928787, + "loss": 1.5706, + "theoretical_loss": 4.260781260279216, + "tokens_seen": 252510208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462387161484453, + "loss": 1.5784, + "theoretical_loss": 4.260648773210281, + "tokens_seen": 252575744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462226680040121, + "loss": 1.6752, + "theoretical_loss": 4.2605163301359745, + "tokens_seen": 252641280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462066198595787, + "loss": 1.5707, + "theoretical_loss": 4.260383931030279, + "tokens_seen": 252706816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461905717151455, + "loss": 1.6033, + "theoretical_loss": 4.2602515758672, + "tokens_seen": 252772352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461745235707121, + "loss": 1.7105, + "theoretical_loss": 4.260119264620768, + "tokens_seen": 252837888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461584754262789, + "loss": 1.7694, + "theoretical_loss": 4.259986997265033, + "tokens_seen": 252903424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461424272818455, + "loss": 1.5492, + "theoretical_loss": 4.259854773774066, + "tokens_seen": 252968960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461263791374123, + "loss": 1.5911, + "theoretical_loss": 4.259722594121963, + "tokens_seen": 253034496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461103309929789, + "loss": 1.6801, + "theoretical_loss": 4.259590458282839, + "tokens_seen": 253100032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460942828485457, + "loss": 1.6821, + "theoretical_loss": 4.259458366230835, + "tokens_seen": 253165568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460782347041123, + "loss": 1.6812, + "theoretical_loss": 4.2593263179401095, + "tokens_seen": 253231104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460621865596791, + "loss": 1.6885, + "theoretical_loss": 4.259194313384846, + "tokens_seen": 253296640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460461384152457, + "loss": 1.6347, + "theoretical_loss": 4.259062352539247, + "tokens_seen": 253362176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460300902708125, + "loss": 1.6389, + "theoretical_loss": 4.258930435377541, + "tokens_seen": 253427712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460140421263791, + "loss": 1.6788, + "theoretical_loss": 4.258798561873974, + "tokens_seen": 253493248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459979939819459, + "loss": 1.6139, + "theoretical_loss": 4.258666732002817, + "tokens_seen": 253558784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459819458375125, + "loss": 1.6973, + "theoretical_loss": 4.258534945738362, + "tokens_seen": 253624320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459658976930793, + "loss": 1.6564, + "theoretical_loss": 4.258403203054923, + "tokens_seen": 253689856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459498495486459, + "loss": 1.6421, + "theoretical_loss": 4.258271503926833, + "tokens_seen": 253755392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459338014042127, + "loss": 1.6334, + "theoretical_loss": 4.258139848328451, + "tokens_seen": 253820928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459177532597794, + "loss": 1.6213, + "theoretical_loss": 4.258008236234154, + "tokens_seen": 253886464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459017051153461, + "loss": 1.6186, + "theoretical_loss": 4.257876667618344, + "tokens_seen": 253952000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458856569709128, + "loss": 1.6503, + "theoretical_loss": 4.257745142455442, + "tokens_seen": 254017536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458696088264795, + "loss": 1.7115, + "theoretical_loss": 4.257613660719892, + "tokens_seen": 254083072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458535606820462, + "loss": 1.6601, + "theoretical_loss": 4.25748222238616, + "tokens_seen": 254148608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458375125376129, + "loss": 1.5442, + "theoretical_loss": 4.2573508274287315, + "tokens_seen": 254214144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458214643931796, + "loss": 1.4927, + "theoretical_loss": 4.257219475822116, + "tokens_seen": 254279680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458054162487463, + "loss": 1.5835, + "theoretical_loss": 4.257088167540843, + "tokens_seen": 254345216 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745789368104313, + "loss": 1.5372, + "theoretical_loss": 4.256956902559465, + "tokens_seen": 254410752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457733199598797, + "loss": 1.6738, + "theoretical_loss": 4.256825680852554, + "tokens_seen": 254476288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457572718154464, + "loss": 1.646, + "theoretical_loss": 4.256694502394704, + "tokens_seen": 254541824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457412236710131, + "loss": 1.6983, + "theoretical_loss": 4.256563367160533, + "tokens_seen": 254607360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457251755265798, + "loss": 1.6854, + "theoretical_loss": 4.256432275124676, + "tokens_seen": 254672896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457091273821465, + "loss": 1.6195, + "theoretical_loss": 4.256301226261794, + "tokens_seen": 254738432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456930792377133, + "loss": 1.5589, + "theoretical_loss": 4.256170220546565, + "tokens_seen": 254803968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456770310932799, + "loss": 1.4944, + "theoretical_loss": 4.2560392579536925, + "tokens_seen": 254869504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456609829488467, + "loss": 1.5076, + "theoretical_loss": 4.255908338457898, + "tokens_seen": 254935040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456449348044133, + "loss": 1.5108, + "theoretical_loss": 4.255777462033926, + "tokens_seen": 255000576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074562888665998, + "loss": 1.5947, + "theoretical_loss": 4.255646628656542, + "tokens_seen": 255066112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456128385155467, + "loss": 1.5601, + "theoretical_loss": 4.255515838300534, + "tokens_seen": 255131648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455967903711135, + "loss": 1.6591, + "theoretical_loss": 4.255385090940708, + "tokens_seen": 255197184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455807422266801, + "loss": 1.6293, + "theoretical_loss": 4.255254386551894, + "tokens_seen": 255262720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455646940822469, + "loss": 1.5927, + "theoretical_loss": 4.255123725108943, + "tokens_seen": 255328256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455486459378135, + "loss": 1.584, + "theoretical_loss": 4.254993106586725, + "tokens_seen": 255393792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455325977933803, + "loss": 1.6033, + "theoretical_loss": 4.254862530960134, + "tokens_seen": 255459328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455165496489469, + "loss": 1.628, + "theoretical_loss": 4.254731998204083, + "tokens_seen": 255524864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 91319, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5541565418243408, + "objective/train/theoretical_loss": 4.254601508293508, + "objective/train/tokens_used": 276050400, + "theoretical_loss": 4.254601508293508, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455005015045136, + "loss": 1.5715, + "theoretical_loss": 4.254601508293508, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454844533600803, + "loss": 1.6553, + "theoretical_loss": 4.254471061203364, + "tokens_seen": 255655936 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745468405215647, + "loss": 1.7168, + "theoretical_loss": 4.254340656908628, + "tokens_seen": 255721472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454523570712137, + "loss": 1.636, + "theoretical_loss": 4.254210295384298, + "tokens_seen": 255787008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454363089267804, + "loss": 1.6813, + "theoretical_loss": 4.254079976605394, + "tokens_seen": 255852544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454202607823471, + "loss": 1.5497, + "theoretical_loss": 4.253949700546956, + "tokens_seen": 255918080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454042126379138, + "loss": 1.5719, + "theoretical_loss": 4.2538194671840435, + "tokens_seen": 255983616 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453881644934805, + "loss": 1.7041, + "theoretical_loss": 4.25368927649174, + "tokens_seen": 256049152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453721163490472, + "loss": 1.5223, + "theoretical_loss": 4.253559128445148, + "tokens_seen": 256114688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453560682046138, + "loss": 1.6966, + "theoretical_loss": 4.253429023019391, + "tokens_seen": 256180224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453400200601806, + "loss": 1.6931, + "theoretical_loss": 4.253298960189614, + "tokens_seen": 256245760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453239719157472, + "loss": 1.5582, + "theoretical_loss": 4.253168939930982, + "tokens_seen": 256311296 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745307923771314, + "loss": 1.5669, + "theoretical_loss": 4.253038962218682, + "tokens_seen": 256376832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452918756268806, + "loss": 1.7125, + "theoretical_loss": 4.25290902702792, + "tokens_seen": 256442368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452758274824474, + "loss": 1.6, + "theoretical_loss": 4.252779134333926, + "tokens_seen": 256507904 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745259779338014, + "loss": 1.7343, + "theoretical_loss": 4.252649284111946, + "tokens_seen": 256573440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452437311935808, + "loss": 1.6937, + "theoretical_loss": 4.252519476337251, + "tokens_seen": 256638976 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452276830491474, + "loss": 1.5897, + "theoretical_loss": 4.252389710985131, + "tokens_seen": 256704512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452116349047142, + "loss": 1.7181, + "theoretical_loss": 4.252259988030896, + "tokens_seen": 256770048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451955867602808, + "loss": 1.6677, + "theoretical_loss": 4.252130307449878, + "tokens_seen": 256835584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451795386158476, + "loss": 1.6119, + "theoretical_loss": 4.252000669217429, + "tokens_seen": 256901120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451634904714142, + "loss": 1.722, + "theoretical_loss": 4.251871073308923, + "tokens_seen": 256966656 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745147442326981, + "loss": 1.6881, + "theoretical_loss": 4.251741519699751, + "tokens_seen": 257032192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451313941825476, + "loss": 1.6454, + "theoretical_loss": 4.251612008365328, + "tokens_seen": 257097728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451153460381144, + "loss": 1.527, + "theoretical_loss": 4.2514825392810875, + "tokens_seen": 257163264 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745099297893681, + "loss": 1.6586, + "theoretical_loss": 4.251353112422486, + "tokens_seen": 257228800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450832497492478, + "loss": 1.6275, + "theoretical_loss": 4.251223727764999, + "tokens_seen": 257294336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450672016048144, + "loss": 1.716, + "theoretical_loss": 4.251094385284121, + "tokens_seen": 257359872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450511534603812, + "loss": 1.6393, + "theoretical_loss": 4.25096508495537, + "tokens_seen": 257425408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450351053159478, + "loss": 1.6811, + "theoretical_loss": 4.250835826754281, + "tokens_seen": 257490944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450190571715146, + "loss": 1.6071, + "theoretical_loss": 4.2507066106564135, + "tokens_seen": 257556480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450030090270812, + "loss": 1.7105, + "theoretical_loss": 4.250577436637344, + "tokens_seen": 257622016 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744986960882648, + "loss": 1.6548, + "theoretical_loss": 4.25044830467267, + "tokens_seen": 257687552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007449709127382146, + "loss": 1.7264, + "theoretical_loss": 4.250319214738011, + "tokens_seen": 257753088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007449548645937814, + "loss": 1.6103, + "theoretical_loss": 4.250190166809005, + "tokens_seen": 257818624 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744938816449348, + "loss": 1.6217, + "theoretical_loss": 4.250061160861311, + "tokens_seen": 257884160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007449227683049148, + "loss": 1.5565, + "theoretical_loss": 4.249932196870608, + "tokens_seen": 257949696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007449067201604814, + "loss": 1.6584, + "theoretical_loss": 4.249803274812598, + "tokens_seen": 258015232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448906720160482, + "loss": 1.7406, + "theoretical_loss": 4.249674394662998, + "tokens_seen": 258080768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448746238716148, + "loss": 1.5355, + "theoretical_loss": 4.249545556397549, + "tokens_seen": 258146304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448585757271816, + "loss": 1.6406, + "theoretical_loss": 4.249416759992011, + "tokens_seen": 258211840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448425275827482, + "loss": 1.5055, + "theoretical_loss": 4.249288005422166, + "tokens_seen": 258277376 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744826479438315, + "loss": 1.5312, + "theoretical_loss": 4.249159292663812, + "tokens_seen": 258342912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448104312938816, + "loss": 1.6441, + "theoretical_loss": 4.249030621692772, + "tokens_seen": 258408448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447943831494484, + "loss": 1.6091, + "theoretical_loss": 4.248901992484885, + "tokens_seen": 258473984 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744778335005015, + "loss": 1.6485, + "theoretical_loss": 4.248773405016014, + "tokens_seen": 258539520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447622868605818, + "loss": 1.6405, + "theoretical_loss": 4.248644859262039, + "tokens_seen": 258605056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447462387161484, + "loss": 1.5547, + "theoretical_loss": 4.248516355198861, + "tokens_seen": 258670592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447301905717152, + "loss": 1.6182, + "theoretical_loss": 4.2483878928024, + "tokens_seen": 258736128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447141424272819, + "loss": 1.592, + "theoretical_loss": 4.248259472048598, + "tokens_seen": 258801664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 92500, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5000085830688477, + "objective/train/theoretical_loss": 4.248131092913416, + "objective/train/tokens_used": 279327200, + "theoretical_loss": 4.248131092913416, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446980942828486, + "loss": 1.5452, + "theoretical_loss": 4.248131092913416, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446820461384153, + "loss": 1.542, + "theoretical_loss": 4.248002755372836, + "tokens_seen": 258932736 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744665997993982, + "loss": 1.6977, + "theoretical_loss": 4.247874459402857, + "tokens_seen": 258998272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446499498495487, + "loss": 1.5523, + "theoretical_loss": 4.2477462049795, + "tokens_seen": 259063808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446339017051154, + "loss": 1.6554, + "theoretical_loss": 4.2476179920788075, + "tokens_seen": 259129344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446178535606821, + "loss": 1.557, + "theoretical_loss": 4.247489820676839, + "tokens_seen": 259194880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446018054162488, + "loss": 1.6801, + "theoretical_loss": 4.247361690749676, + "tokens_seen": 259260416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445857572718155, + "loss": 1.604, + "theoretical_loss": 4.2472336022734165, + "tokens_seen": 259325952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445697091273822, + "loss": 1.6637, + "theoretical_loss": 4.2471055552241825, + "tokens_seen": 259391488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445536609829489, + "loss": 1.6168, + "theoretical_loss": 4.246977549578113, + "tokens_seen": 259457024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445376128385156, + "loss": 1.6913, + "theoretical_loss": 4.246849585311368, + "tokens_seen": 259522560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445215646940823, + "loss": 1.6747, + "theoretical_loss": 4.246721662400128, + "tokens_seen": 259588096 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744505516549649, + "loss": 1.5241, + "theoretical_loss": 4.246593780820589, + "tokens_seen": 259653632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444894684052157, + "loss": 1.5544, + "theoretical_loss": 4.2464659405489735, + "tokens_seen": 259719168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444734202607824, + "loss": 1.4963, + "theoretical_loss": 4.246338141561518, + "tokens_seen": 259784704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444573721163491, + "loss": 1.5923, + "theoretical_loss": 4.24621038383448, + "tokens_seen": 259850240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444413239719158, + "loss": 1.643, + "theoretical_loss": 4.246082667344139, + "tokens_seen": 259915776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444252758274825, + "loss": 1.5979, + "theoretical_loss": 4.245954992066792, + "tokens_seen": 259981312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444092276830492, + "loss": 1.7024, + "theoretical_loss": 4.245827357978754, + "tokens_seen": 260046848 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744393179538616, + "loss": 1.5572, + "theoretical_loss": 4.245699765056365, + "tokens_seen": 260112384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007443771313941826, + "loss": 1.5896, + "theoretical_loss": 4.245572213275978, + "tokens_seen": 260177920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007443610832497493, + "loss": 1.8514, + "theoretical_loss": 4.245444702613971, + "tokens_seen": 260243456 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744345035105316, + "loss": 1.586, + "theoretical_loss": 4.245317233046738, + "tokens_seen": 260308992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007443289869608828, + "loss": 1.7146, + "theoretical_loss": 4.2451898045506935, + "tokens_seen": 260374528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007443129388164494, + "loss": 1.6225, + "theoretical_loss": 4.245062417102272, + "tokens_seen": 260440064 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442968906720162, + "loss": 1.592, + "theoretical_loss": 4.244935070677927, + "tokens_seen": 260505600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442808425275828, + "loss": 1.6766, + "theoretical_loss": 4.244807765254132, + "tokens_seen": 260571136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442647943831496, + "loss": 1.6643, + "theoretical_loss": 4.244680500807378, + "tokens_seen": 260636672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442487462387162, + "loss": 1.5448, + "theoretical_loss": 4.244553277314178, + "tokens_seen": 260702208 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744232698094283, + "loss": 1.6036, + "theoretical_loss": 4.244426094751063, + "tokens_seen": 260767744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442166499498496, + "loss": 1.6473, + "theoretical_loss": 4.244298953094583, + "tokens_seen": 260833280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442006018054164, + "loss": 1.6384, + "theoretical_loss": 4.244171852321308, + "tokens_seen": 260898816 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744184553660983, + "loss": 1.6358, + "theoretical_loss": 4.244044792407828, + "tokens_seen": 260964352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441685055165498, + "loss": 1.6318, + "theoretical_loss": 4.24391777333075, + "tokens_seen": 261029888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441524573721164, + "loss": 1.6431, + "theoretical_loss": 4.243790795066703, + "tokens_seen": 261095424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441364092276832, + "loss": 1.5781, + "theoretical_loss": 4.243663857592332, + "tokens_seen": 261160960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441203610832498, + "loss": 1.5393, + "theoretical_loss": 4.243536960884305, + "tokens_seen": 261226496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441043129388166, + "loss": 1.6555, + "theoretical_loss": 4.243410104919307, + "tokens_seen": 261292032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440882647943832, + "loss": 1.6852, + "theoretical_loss": 4.243283289674042, + "tokens_seen": 261357568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440722166499499, + "loss": 1.4922, + "theoretical_loss": 4.243156515125234, + "tokens_seen": 261423104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440561685055166, + "loss": 1.6475, + "theoretical_loss": 4.243029781249625, + "tokens_seen": 261488640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440401203610833, + "loss": 1.5878, + "theoretical_loss": 4.2429030880239775, + "tokens_seen": 261554176 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074402407221665, + "loss": 1.6059, + "theoretical_loss": 4.242776435425072, + "tokens_seen": 261619712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440080240722167, + "loss": 1.5873, + "theoretical_loss": 4.24264982342971, + "tokens_seen": 261685248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439919759277834, + "loss": 1.5429, + "theoretical_loss": 4.242523252014709, + "tokens_seen": 261750784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439759277833501, + "loss": 1.6651, + "theoretical_loss": 4.242396721156908, + "tokens_seen": 261816320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439598796389168, + "loss": 1.7161, + "theoretical_loss": 4.242270230833164, + "tokens_seen": 261881856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439438314944835, + "loss": 1.5095, + "theoretical_loss": 4.242143781020353, + "tokens_seen": 261947392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439277833500501, + "loss": 1.6434, + "theoretical_loss": 4.24201737169537, + "tokens_seen": 262012928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439117352056169, + "loss": 1.6189, + "theoretical_loss": 4.24189100283513, + "tokens_seen": 262078464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 93423, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6364566087722778, + "objective/train/theoretical_loss": 4.241764674416565, + "objective/train/tokens_used": 282604000, + "theoretical_loss": 4.241764674416565, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438956870611835, + "loss": 1.6476, + "theoretical_loss": 4.241764674416565, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438796389167503, + "loss": 1.5735, + "theoretical_loss": 4.241638386416628, + "tokens_seen": 262209536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438635907723169, + "loss": 1.6634, + "theoretical_loss": 4.241512138812288, + "tokens_seen": 262275072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438475426278837, + "loss": 1.5158, + "theoretical_loss": 4.2413859315805365, + "tokens_seen": 262340608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438314944834503, + "loss": 1.5613, + "theoretical_loss": 4.241259764698381, + "tokens_seen": 262406144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438154463390171, + "loss": 1.5698, + "theoretical_loss": 4.24113363814285, + "tokens_seen": 262471680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437993981945837, + "loss": 1.7, + "theoretical_loss": 4.241007551890988, + "tokens_seen": 262537216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437833500501505, + "loss": 1.5795, + "theoretical_loss": 4.2408815059198615, + "tokens_seen": 262602752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437673019057171, + "loss": 1.6302, + "theoretical_loss": 4.240755500206554, + "tokens_seen": 262668288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437512537612839, + "loss": 1.6311, + "theoretical_loss": 4.240629534728167, + "tokens_seen": 262733824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437352056168505, + "loss": 1.6255, + "theoretical_loss": 4.240503609461822, + "tokens_seen": 262799360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437191574724173, + "loss": 1.8019, + "theoretical_loss": 4.24037772438466, + "tokens_seen": 262864896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437031093279839, + "loss": 1.5956, + "theoretical_loss": 4.240251879473839, + "tokens_seen": 262930432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436870611835507, + "loss": 1.6039, + "theoretical_loss": 4.240126074706536, + "tokens_seen": 262995968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436710130391173, + "loss": 1.6342, + "theoretical_loss": 4.240000310059948, + "tokens_seen": 263061504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436549648946841, + "loss": 1.5717, + "theoretical_loss": 4.239874585511288, + "tokens_seen": 263127040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436389167502507, + "loss": 1.7075, + "theoretical_loss": 4.239748901037791, + "tokens_seen": 263192576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436228686058175, + "loss": 1.5863, + "theoretical_loss": 4.2396232566167065, + "tokens_seen": 263258112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436068204613841, + "loss": 1.7253, + "theoretical_loss": 4.239497652225307, + "tokens_seen": 263323648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435907723169509, + "loss": 1.5759, + "theoretical_loss": 4.23937208784088, + "tokens_seen": 263389184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435747241725175, + "loss": 1.5407, + "theoretical_loss": 4.239246563440734, + "tokens_seen": 263454720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435586760280843, + "loss": 1.512, + "theoretical_loss": 4.239121079002194, + "tokens_seen": 263520256 + }, + { + "epoch": 0.08, + "learning_rate": 0.000743542627883651, + "loss": 1.5218, + "theoretical_loss": 4.238995634502606, + "tokens_seen": 263585792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435265797392177, + "loss": 1.6162, + "theoretical_loss": 4.238870229919331, + "tokens_seen": 263651328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435105315947844, + "loss": 1.5465, + "theoretical_loss": 4.2387448652297515, + "tokens_seen": 263716864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434944834503511, + "loss": 1.6953, + "theoretical_loss": 4.238619540411268, + "tokens_seen": 263782400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434784353059178, + "loss": 1.6432, + "theoretical_loss": 4.238494255441296, + "tokens_seen": 263847936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434623871614845, + "loss": 1.6252, + "theoretical_loss": 4.238369010297275, + "tokens_seen": 263913472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434463390170512, + "loss": 1.6152, + "theoretical_loss": 4.238243804956658, + "tokens_seen": 263979008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434302908726179, + "loss": 1.6624, + "theoretical_loss": 4.23811863939692, + "tokens_seen": 264044544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434142427281846, + "loss": 1.605, + "theoretical_loss": 4.237993513595552, + "tokens_seen": 264110080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433981945837513, + "loss": 1.5313, + "theoretical_loss": 4.2378684275300635, + "tokens_seen": 264175616 + }, + { + "epoch": 0.08, + "learning_rate": 0.000743382146439318, + "loss": 1.5915, + "theoretical_loss": 4.237743381177983, + "tokens_seen": 264241152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433660982948847, + "loss": 1.6108, + "theoretical_loss": 4.237618374516858, + "tokens_seen": 264306688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433500501504514, + "loss": 1.6528, + "theoretical_loss": 4.237493407524253, + "tokens_seen": 264372224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433340020060181, + "loss": 1.5117, + "theoretical_loss": 4.23736848017775, + "tokens_seen": 264437760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433179538615848, + "loss": 1.6246, + "theoretical_loss": 4.237243592454951, + "tokens_seen": 264503296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433019057171515, + "loss": 1.6768, + "theoretical_loss": 4.2371187443334755, + "tokens_seen": 264568832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432858575727182, + "loss": 1.7698, + "theoretical_loss": 4.236993935790962, + "tokens_seen": 264634368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432698094282849, + "loss": 1.5859, + "theoretical_loss": 4.236869166805064, + "tokens_seen": 264699904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432537612838516, + "loss": 1.4231, + "theoretical_loss": 4.236744437353458, + "tokens_seen": 264765440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432377131394183, + "loss": 1.7275, + "theoretical_loss": 4.236619747413836, + "tokens_seen": 264830976 + }, + { + "epoch": 0.08, + "learning_rate": 0.000743221664994985, + "loss": 1.5382, + "theoretical_loss": 4.236495096963906, + "tokens_seen": 264896512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432056168505517, + "loss": 1.6893, + "theoretical_loss": 4.236370485981398, + "tokens_seen": 264962048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431895687061184, + "loss": 1.676, + "theoretical_loss": 4.236245914444058, + "tokens_seen": 265027584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431735205616851, + "loss": 1.5495, + "theoretical_loss": 4.2361213823296495, + "tokens_seen": 265093120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431574724172518, + "loss": 1.6253, + "theoretical_loss": 4.235996889615957, + "tokens_seen": 265158656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431414242728185, + "loss": 1.646, + "theoretical_loss": 4.235872436280779, + "tokens_seen": 265224192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431253761283852, + "loss": 1.7312, + "theoretical_loss": 4.235748022301935, + "tokens_seen": 265289728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431093279839519, + "loss": 1.5934, + "theoretical_loss": 4.235623647657261, + "tokens_seen": 265355264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 94744, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6758512258529663, + "objective/train/theoretical_loss": 4.235499312324611, + "objective/train/tokens_used": 285880800, + "theoretical_loss": 4.235499312324611, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430932798395186, + "loss": 1.6044, + "theoretical_loss": 4.235499312324611, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430772316950853, + "loss": 1.6172, + "theoretical_loss": 4.235375016281858, + "tokens_seen": 265486336 + }, + { + "epoch": 0.08, + "learning_rate": 0.000743061183550652, + "loss": 1.5535, + "theoretical_loss": 4.235250759506892, + "tokens_seen": 265551872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430451354062187, + "loss": 1.4944, + "theoretical_loss": 4.235126541977621, + "tokens_seen": 265617408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430290872617854, + "loss": 1.6137, + "theoretical_loss": 4.2350023636719705, + "tokens_seen": 265682944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430130391173521, + "loss": 1.4967, + "theoretical_loss": 4.234878224567885, + "tokens_seen": 265748480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429969909729188, + "loss": 1.5846, + "theoretical_loss": 4.2347541246433265, + "tokens_seen": 265814016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429809428284855, + "loss": 1.6215, + "theoretical_loss": 4.234630063876273, + "tokens_seen": 265879552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429648946840523, + "loss": 1.5888, + "theoretical_loss": 4.234506042244723, + "tokens_seen": 265945088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429488465396189, + "loss": 1.6198, + "theoretical_loss": 4.234382059726691, + "tokens_seen": 266010624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429327983951857, + "loss": 1.635, + "theoretical_loss": 4.234258116300211, + "tokens_seen": 266076160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429167502507523, + "loss": 1.5904, + "theoretical_loss": 4.234134211943332, + "tokens_seen": 266141696 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742900702106319, + "loss": 1.6317, + "theoretical_loss": 4.234010346634123, + "tokens_seen": 266207232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428846539618857, + "loss": 1.6059, + "theoretical_loss": 4.23388652035067, + "tokens_seen": 266272768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428686058174525, + "loss": 1.5262, + "theoretical_loss": 4.233762733071076, + "tokens_seen": 266338304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428525576730191, + "loss": 1.6322, + "theoretical_loss": 4.233638984773464, + "tokens_seen": 266403840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428365095285859, + "loss": 1.7282, + "theoretical_loss": 4.2335152754359715, + "tokens_seen": 266469376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428204613841525, + "loss": 1.5823, + "theoretical_loss": 4.233391605036756, + "tokens_seen": 266534912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428044132397193, + "loss": 1.555, + "theoretical_loss": 4.233267973553991, + "tokens_seen": 266600448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427883650952859, + "loss": 1.6832, + "theoretical_loss": 4.233144380965869, + "tokens_seen": 266665984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427723169508527, + "loss": 1.5752, + "theoretical_loss": 4.2330208272505985, + "tokens_seen": 266731520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427562688064193, + "loss": 1.6642, + "theoretical_loss": 4.232897312386408, + "tokens_seen": 266797056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427402206619861, + "loss": 1.6123, + "theoretical_loss": 4.232773836351541, + "tokens_seen": 266862592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427241725175527, + "loss": 1.6768, + "theoretical_loss": 4.2326503991242586, + "tokens_seen": 266928128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427081243731195, + "loss": 1.5605, + "theoretical_loss": 4.232527000682843, + "tokens_seen": 266993664 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426920762286861, + "loss": 1.6535, + "theoretical_loss": 4.232403641005589, + "tokens_seen": 267059200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426760280842528, + "loss": 1.6531, + "theoretical_loss": 4.232280320070812, + "tokens_seen": 267124736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426599799398195, + "loss": 1.6343, + "theoretical_loss": 4.232157037856844, + "tokens_seen": 267190272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426439317953862, + "loss": 1.6088, + "theoretical_loss": 4.232033794342033, + "tokens_seen": 267255808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426278836509529, + "loss": 1.5571, + "theoretical_loss": 4.231910589504748, + "tokens_seen": 267321344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426118355065196, + "loss": 1.626, + "theoretical_loss": 4.2317874233233725, + "tokens_seen": 267386880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007425957873620863, + "loss": 1.561, + "theoretical_loss": 4.231664295776307, + "tokens_seen": 267452416 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742579739217653, + "loss": 1.5373, + "theoretical_loss": 4.231541206841971, + "tokens_seen": 267517952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007425636910732197, + "loss": 1.5562, + "theoretical_loss": 4.2314181564988, + "tokens_seen": 267583488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007425476429287864, + "loss": 1.614, + "theoretical_loss": 4.23129514472525, + "tokens_seen": 267649024 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742531594784353, + "loss": 1.5684, + "theoretical_loss": 4.231172171499789, + "tokens_seen": 267714560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007425155466399198, + "loss": 1.6038, + "theoretical_loss": 4.231049236800907, + "tokens_seen": 267780096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424994984954864, + "loss": 1.5656, + "theoretical_loss": 4.230926340607109, + "tokens_seen": 267845632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424834503510532, + "loss": 1.6124, + "theoretical_loss": 4.230803482896918, + "tokens_seen": 267911168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424674022066198, + "loss": 1.6741, + "theoretical_loss": 4.230680663648872, + "tokens_seen": 267976704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424513540621866, + "loss": 1.6197, + "theoretical_loss": 4.230557882841531, + "tokens_seen": 268042240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424353059177532, + "loss": 1.7328, + "theoretical_loss": 4.2304351404534675, + "tokens_seen": 268107776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074241925777332, + "loss": 1.6919, + "theoretical_loss": 4.230312436463272, + "tokens_seen": 268173312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424032096288866, + "loss": 1.6116, + "theoretical_loss": 4.230189770849556, + "tokens_seen": 268238848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423871614844534, + "loss": 1.6098, + "theoretical_loss": 4.230067143590943, + "tokens_seen": 268304384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074237111334002, + "loss": 1.5206, + "theoretical_loss": 4.229944554666076, + "tokens_seen": 268369920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423550651955868, + "loss": 1.6026, + "theoretical_loss": 4.229822004053617, + "tokens_seen": 268435456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423390170511535, + "loss": 1.6134, + "theoretical_loss": 4.229699491732242, + "tokens_seen": 268500992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423229689067202, + "loss": 1.5862, + "theoretical_loss": 4.2295770176806435, + "tokens_seen": 268566528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423069207622869, + "loss": 1.6335, + "theoretical_loss": 4.229454581877535, + "tokens_seen": 268632064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 95678, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.629549503326416, + "objective/train/theoretical_loss": 4.229332184301644, + "objective/train/tokens_used": 289157600, + "theoretical_loss": 4.229332184301644, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422908726178536, + "loss": 1.6398, + "theoretical_loss": 4.229332184301644, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422748244734203, + "loss": 1.4887, + "theoretical_loss": 4.229209824931714, + "tokens_seen": 268763136 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742258776328987, + "loss": 1.6316, + "theoretical_loss": 4.22908750374651, + "tokens_seen": 268828672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422427281845537, + "loss": 1.5414, + "theoretical_loss": 4.22896522072481, + "tokens_seen": 268894208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422266800401204, + "loss": 1.6666, + "theoretical_loss": 4.22884297584541, + "tokens_seen": 268959744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422106318956871, + "loss": 1.6226, + "theoretical_loss": 4.228720769087124, + "tokens_seen": 269025280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421945837512538, + "loss": 1.4824, + "theoretical_loss": 4.22859860042878, + "tokens_seen": 269090816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421785356068205, + "loss": 1.574, + "theoretical_loss": 4.2284764698492285, + "tokens_seen": 269156352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421624874623872, + "loss": 1.6074, + "theoretical_loss": 4.22835437732733, + "tokens_seen": 269221888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421464393179539, + "loss": 1.6837, + "theoretical_loss": 4.228232322841967, + "tokens_seen": 269287424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421303911735206, + "loss": 1.5557, + "theoretical_loss": 4.228110306372036, + "tokens_seen": 269352960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421143430290873, + "loss": 1.5514, + "theoretical_loss": 4.227988327896453, + "tokens_seen": 269418496 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742098294884654, + "loss": 1.6616, + "theoretical_loss": 4.2278663873941476, + "tokens_seen": 269484032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420822467402207, + "loss": 1.5448, + "theoretical_loss": 4.227744484844068, + "tokens_seen": 269549568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420661985957874, + "loss": 1.5115, + "theoretical_loss": 4.227622620225182, + "tokens_seen": 269615104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420501504513541, + "loss": 1.6058, + "theoretical_loss": 4.227500793516466, + "tokens_seen": 269680640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420341023069208, + "loss": 1.6111, + "theoretical_loss": 4.227379004696923, + "tokens_seen": 269746176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420180541624875, + "loss": 1.5459, + "theoretical_loss": 4.227257253745566, + "tokens_seen": 269811712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420020060180542, + "loss": 1.6436, + "theoretical_loss": 4.227135540641426, + "tokens_seen": 269877248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419859578736209, + "loss": 1.6299, + "theoretical_loss": 4.227013865363553, + "tokens_seen": 269942784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419699097291876, + "loss": 1.5194, + "theoretical_loss": 4.226892227891012, + "tokens_seen": 270008320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419538615847543, + "loss": 1.5954, + "theoretical_loss": 4.2267706282028845, + "tokens_seen": 270073856 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741937813440321, + "loss": 1.596, + "theoretical_loss": 4.226649066278269, + "tokens_seen": 270139392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419217652958877, + "loss": 1.7046, + "theoretical_loss": 4.226527542096282, + "tokens_seen": 270204928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419057171514544, + "loss": 1.6551, + "theoretical_loss": 4.226406055636053, + "tokens_seen": 270270464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418896690070211, + "loss": 1.6659, + "theoretical_loss": 4.226284606876731, + "tokens_seen": 270336000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418736208625878, + "loss": 1.5405, + "theoretical_loss": 4.226163195797481, + "tokens_seen": 270401536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418575727181545, + "loss": 1.6297, + "theoretical_loss": 4.226041822377486, + "tokens_seen": 270467072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418415245737212, + "loss": 1.5975, + "theoretical_loss": 4.225920486595941, + "tokens_seen": 270532608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418254764292879, + "loss": 1.6092, + "theoretical_loss": 4.225799188432063, + "tokens_seen": 270598144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418094282848546, + "loss": 1.665, + "theoretical_loss": 4.225677927865083, + "tokens_seen": 270663680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417933801404213, + "loss": 1.5838, + "theoretical_loss": 4.225556704874247, + "tokens_seen": 270729216 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741777331995988, + "loss": 1.6257, + "theoretical_loss": 4.22543551943882, + "tokens_seen": 270794752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417612838515547, + "loss": 1.5903, + "theoretical_loss": 4.225314371538083, + "tokens_seen": 270860288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417452357071214, + "loss": 1.5667, + "theoretical_loss": 4.225193261151332, + "tokens_seen": 270925824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417291875626881, + "loss": 1.5673, + "theoretical_loss": 4.22507218825788, + "tokens_seen": 270991360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417131394182548, + "loss": 1.5828, + "theoretical_loss": 4.224951152837058, + "tokens_seen": 271056896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416970912738215, + "loss": 1.6085, + "theoretical_loss": 4.2248301548682115, + "tokens_seen": 271122432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416810431293882, + "loss": 1.5875, + "theoretical_loss": 4.2247091943307025, + "tokens_seen": 271187968 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741664994984955, + "loss": 1.5357, + "theoretical_loss": 4.224588271203911, + "tokens_seen": 271253504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416489468405216, + "loss": 1.7111, + "theoretical_loss": 4.22446738546723, + "tokens_seen": 271319040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416328986960883, + "loss": 1.6451, + "theoretical_loss": 4.224346537100074, + "tokens_seen": 271384576 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741616850551655, + "loss": 1.4726, + "theoretical_loss": 4.22422572608187, + "tokens_seen": 271450112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416008024072218, + "loss": 1.5835, + "theoretical_loss": 4.224104952392061, + "tokens_seen": 271515648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415847542627884, + "loss": 1.6244, + "theoretical_loss": 4.223984216010107, + "tokens_seen": 271581184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415687061183552, + "loss": 1.6205, + "theoretical_loss": 4.223863516915486, + "tokens_seen": 271646720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415526579739218, + "loss": 1.5239, + "theoretical_loss": 4.223742855087691, + "tokens_seen": 271712256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415366098294886, + "loss": 1.5946, + "theoretical_loss": 4.223622230506231, + "tokens_seen": 271777792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415205616850552, + "loss": 1.7028, + "theoretical_loss": 4.22350164315063, + "tokens_seen": 271843328 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741504513540622, + "loss": 1.5819, + "theoretical_loss": 4.22338109300043, + "tokens_seen": 271908864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 96867, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5667804479599, + "objective/train/theoretical_loss": 4.22326058003519, + "objective/train/tokens_used": 292434400, + "theoretical_loss": 4.22326058003519, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414884653961886, + "loss": 1.5744, + "theoretical_loss": 4.22326058003519, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414724172517554, + "loss": 1.6947, + "theoretical_loss": 4.223140104234482, + "tokens_seen": 272039936 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741456369107322, + "loss": 1.5442, + "theoretical_loss": 4.2230196655778975, + "tokens_seen": 272105472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414403209628888, + "loss": 1.707, + "theoretical_loss": 4.222899264045042, + "tokens_seen": 272171008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414242728184554, + "loss": 1.635, + "theoretical_loss": 4.2227788996155375, + "tokens_seen": 272236544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414082246740222, + "loss": 1.6033, + "theoretical_loss": 4.222658572269022, + "tokens_seen": 272302080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413921765295888, + "loss": 1.6143, + "theoretical_loss": 4.222538281985151, + "tokens_seen": 272367616 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413761283851556, + "loss": 1.6715, + "theoretical_loss": 4.2224180287435935, + "tokens_seen": 272433152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413600802407222, + "loss": 1.5306, + "theoretical_loss": 4.222297812524037, + "tokens_seen": 272498688 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741344032096289, + "loss": 1.6365, + "theoretical_loss": 4.222177633306183, + "tokens_seen": 272564224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413279839518556, + "loss": 1.657, + "theoretical_loss": 4.222057491069751, + "tokens_seen": 272629760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413119358074224, + "loss": 1.5467, + "theoretical_loss": 4.221937385794474, + "tokens_seen": 272695296 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741295887662989, + "loss": 1.5622, + "theoretical_loss": 4.221817317460104, + "tokens_seen": 272760832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412798395185557, + "loss": 1.53, + "theoretical_loss": 4.221697286046407, + "tokens_seen": 272826368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412637913741224, + "loss": 1.4924, + "theoretical_loss": 4.221577291533165, + "tokens_seen": 272891904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412477432296891, + "loss": 1.583, + "theoretical_loss": 4.221457333900176, + "tokens_seen": 272957440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412316950852558, + "loss": 1.5246, + "theoretical_loss": 4.221337413127254, + "tokens_seen": 273022976 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412156469408225, + "loss": 1.42, + "theoretical_loss": 4.221217529194231, + "tokens_seen": 273088512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411995987963892, + "loss": 1.6641, + "theoretical_loss": 4.22109768208095, + "tokens_seen": 273154048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411835506519559, + "loss": 1.5788, + "theoretical_loss": 4.220977871767275, + "tokens_seen": 273219584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411675025075226, + "loss": 1.6302, + "theoretical_loss": 4.220858098233082, + "tokens_seen": 273285120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411514543630893, + "loss": 1.5654, + "theoretical_loss": 4.220738361458266, + "tokens_seen": 273350656 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741135406218656, + "loss": 1.5599, + "theoretical_loss": 4.220618661422735, + "tokens_seen": 273416192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411193580742227, + "loss": 1.6897, + "theoretical_loss": 4.2204989981064145, + "tokens_seen": 273481728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411033099297894, + "loss": 1.5683, + "theoretical_loss": 4.220379371489246, + "tokens_seen": 273547264 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410872617853561, + "loss": 1.5457, + "theoretical_loss": 4.220259781551185, + "tokens_seen": 273612800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410712136409228, + "loss": 1.6194, + "theoretical_loss": 4.2201402282722045, + "tokens_seen": 273678336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410551654964895, + "loss": 1.6366, + "theoretical_loss": 4.2200207116322925, + "tokens_seen": 273743872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410391173520562, + "loss": 1.6136, + "theoretical_loss": 4.219901231611452, + "tokens_seen": 273809408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410230692076229, + "loss": 1.6047, + "theoretical_loss": 4.219781788189703, + "tokens_seen": 273874944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410070210631896, + "loss": 1.4999, + "theoretical_loss": 4.2196623813470815, + "tokens_seen": 273940480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409909729187563, + "loss": 1.6752, + "theoretical_loss": 4.219543011063637, + "tokens_seen": 274006016 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740974924774323, + "loss": 1.7212, + "theoretical_loss": 4.219423677319437, + "tokens_seen": 274071552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409588766298897, + "loss": 1.6442, + "theoretical_loss": 4.219304380094562, + "tokens_seen": 274137088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409428284854564, + "loss": 1.6143, + "theoretical_loss": 4.2191851193691114, + "tokens_seen": 274202624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409267803410231, + "loss": 1.6529, + "theoretical_loss": 4.219065895123197, + "tokens_seen": 274268160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409107321965898, + "loss": 1.6902, + "theoretical_loss": 4.218946707336949, + "tokens_seen": 274333696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408946840521565, + "loss": 1.5987, + "theoretical_loss": 4.2188275559905115, + "tokens_seen": 274399232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408786359077232, + "loss": 1.527, + "theoretical_loss": 4.218708441064044, + "tokens_seen": 274464768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408625877632899, + "loss": 1.5728, + "theoretical_loss": 4.218589362537723, + "tokens_seen": 274530304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408465396188566, + "loss": 1.5688, + "theoretical_loss": 4.218470320391739, + "tokens_seen": 274595840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408304914744233, + "loss": 1.5903, + "theoretical_loss": 4.218351314606298, + "tokens_seen": 274661376 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074081444332999, + "loss": 1.6266, + "theoretical_loss": 4.218232345161622, + "tokens_seen": 274726912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407983951855567, + "loss": 1.5244, + "theoretical_loss": 4.21811341203795, + "tokens_seen": 274792448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407823470411234, + "loss": 1.6264, + "theoretical_loss": 4.217994515215534, + "tokens_seen": 274857984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407662988966901, + "loss": 1.6983, + "theoretical_loss": 4.217875654674642, + "tokens_seen": 274923520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407502507522568, + "loss": 1.5818, + "theoretical_loss": 4.217756830395559, + "tokens_seen": 274989056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407342026078235, + "loss": 1.6596, + "theoretical_loss": 4.217638042358583, + "tokens_seen": 275054592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407181544633902, + "loss": 1.6276, + "theoretical_loss": 4.21751929054403, + "tokens_seen": 275120128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407021063189569, + "loss": 1.6848, + "theoretical_loss": 4.2174005749322285, + "tokens_seen": 275185664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 97725, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.526691198348999, + "objective/train/theoretical_loss": 4.2172818955035245, + "objective/train/tokens_used": 295711200, + "theoretical_loss": 4.2172818955035245, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406860581745236, + "loss": 1.5652, + "theoretical_loss": 4.2172818955035245, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406700100300903, + "loss": 1.5369, + "theoretical_loss": 4.217163252238279, + "tokens_seen": 275316736 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740653961885657, + "loss": 1.4076, + "theoretical_loss": 4.217044645116869, + "tokens_seen": 275382272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406379137412237, + "loss": 1.6474, + "theoretical_loss": 4.216926074119684, + "tokens_seen": 275447808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406218655967904, + "loss": 1.5036, + "theoretical_loss": 4.216807539227132, + "tokens_seen": 275513344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406058174523571, + "loss": 1.6137, + "theoretical_loss": 4.2166890404196335, + "tokens_seen": 275578880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405897693079238, + "loss": 1.6188, + "theoretical_loss": 4.216570577677627, + "tokens_seen": 275644416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405737211634905, + "loss": 1.6735, + "theoretical_loss": 4.216452150981566, + "tokens_seen": 275709952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405576730190572, + "loss": 1.4854, + "theoretical_loss": 4.216333760311916, + "tokens_seen": 275775488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405416248746239, + "loss": 1.6522, + "theoretical_loss": 4.216215405649161, + "tokens_seen": 275841024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405255767301906, + "loss": 1.5212, + "theoretical_loss": 4.2160970869738, + "tokens_seen": 275906560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405095285857573, + "loss": 1.7428, + "theoretical_loss": 4.215978804266345, + "tokens_seen": 275972096 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740493480441324, + "loss": 1.6548, + "theoretical_loss": 4.215860557507324, + "tokens_seen": 276037632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404774322968907, + "loss": 1.4961, + "theoretical_loss": 4.215742346677283, + "tokens_seen": 276103168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404613841524574, + "loss": 1.5168, + "theoretical_loss": 4.215624171756779, + "tokens_seen": 276168704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404453360080241, + "loss": 1.5582, + "theoretical_loss": 4.215506032726386, + "tokens_seen": 276234240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404292878635908, + "loss": 1.6843, + "theoretical_loss": 4.215387929566694, + "tokens_seen": 276299776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404132397191575, + "loss": 1.6185, + "theoretical_loss": 4.215269862258307, + "tokens_seen": 276365312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403971915747242, + "loss": 1.4809, + "theoretical_loss": 4.215151830781843, + "tokens_seen": 276430848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403811434302909, + "loss": 1.4883, + "theoretical_loss": 4.2150338351179375, + "tokens_seen": 276496384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403650952858576, + "loss": 1.6069, + "theoretical_loss": 4.21491587524724, + "tokens_seen": 276561920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403490471414243, + "loss": 1.7038, + "theoretical_loss": 4.214797951150415, + "tokens_seen": 276627456 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740332998996991, + "loss": 1.581, + "theoretical_loss": 4.214680062808142, + "tokens_seen": 276692992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403169508525577, + "loss": 1.542, + "theoretical_loss": 4.214562210201114, + "tokens_seen": 276758528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403009027081244, + "loss": 1.4822, + "theoretical_loss": 4.214444393310042, + "tokens_seen": 276824064 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402848545636911, + "loss": 1.5861, + "theoretical_loss": 4.2143266121156495, + "tokens_seen": 276889600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402688064192578, + "loss": 1.6427, + "theoretical_loss": 4.2142088665986766, + "tokens_seen": 276955136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402527582748245, + "loss": 1.535, + "theoretical_loss": 4.214091156739878, + "tokens_seen": 277020672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402367101303912, + "loss": 1.6281, + "theoretical_loss": 4.213973482520021, + "tokens_seen": 277086208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402206619859579, + "loss": 1.6517, + "theoretical_loss": 4.213855843919891, + "tokens_seen": 277151744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402046138415247, + "loss": 1.6089, + "theoretical_loss": 4.2137382409202875, + "tokens_seen": 277217280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401885656970913, + "loss": 1.6273, + "theoretical_loss": 4.2136206735020245, + "tokens_seen": 277282816 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740172517552658, + "loss": 1.5309, + "theoretical_loss": 4.21350314164593, + "tokens_seen": 277348352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401564694082247, + "loss": 1.612, + "theoretical_loss": 4.213385645332848, + "tokens_seen": 277413888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401404212637915, + "loss": 1.7019, + "theoretical_loss": 4.213268184543638, + "tokens_seen": 277479424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401243731193581, + "loss": 1.5525, + "theoretical_loss": 4.213150759259172, + "tokens_seen": 277544960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401083249749249, + "loss": 1.601, + "theoretical_loss": 4.213033369460339, + "tokens_seen": 277610496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400922768304915, + "loss": 1.55, + "theoretical_loss": 4.212916015128041, + "tokens_seen": 277676032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400762286860583, + "loss": 1.6188, + "theoretical_loss": 4.212798696243197, + "tokens_seen": 277741568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400601805416249, + "loss": 1.6865, + "theoretical_loss": 4.212681412786738, + "tokens_seen": 277807104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400441323971917, + "loss": 1.6119, + "theoretical_loss": 4.212564164739613, + "tokens_seen": 277872640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400280842527583, + "loss": 1.653, + "theoretical_loss": 4.212446952082783, + "tokens_seen": 277938176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400120361083251, + "loss": 1.5911, + "theoretical_loss": 4.212329774797225, + "tokens_seen": 278003712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399959879638917, + "loss": 1.5933, + "theoretical_loss": 4.212212632863931, + "tokens_seen": 278069248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399799398194585, + "loss": 1.5824, + "theoretical_loss": 4.212095526263905, + "tokens_seen": 278134784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399638916750251, + "loss": 1.593, + "theoretical_loss": 4.211978454978171, + "tokens_seen": 278200320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399478435305919, + "loss": 1.4985, + "theoretical_loss": 4.211861418987762, + "tokens_seen": 278265856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399317953861585, + "loss": 1.7031, + "theoretical_loss": 4.211744418273728, + "tokens_seen": 278331392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399157472417253, + "loss": 1.4938, + "theoretical_loss": 4.211627452817136, + "tokens_seen": 278396928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398996990972919, + "loss": 1.5638, + "theoretical_loss": 4.211510522599063, + "tokens_seen": 278462464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 98910, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5239521265029907, + "objective/train/theoretical_loss": 4.2113936276006045, + "objective/train/tokens_used": 298988000, + "theoretical_loss": 4.2113936276006045, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398836509528586, + "loss": 1.5001, + "theoretical_loss": 4.2113936276006045, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398676028084253, + "loss": 1.4508, + "theoretical_loss": 4.211276767802868, + "tokens_seen": 278593536 + }, + { + "epoch": 0.08, + "learning_rate": 0.000739851554663992, + "loss": 1.5051, + "theoretical_loss": 4.211159943186978, + "tokens_seen": 278659072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398355065195587, + "loss": 1.5914, + "theoretical_loss": 4.211043153734071, + "tokens_seen": 278724608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398194583751254, + "loss": 1.5629, + "theoretical_loss": 4.2109263994253, + "tokens_seen": 278790144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398034102306921, + "loss": 1.5097, + "theoretical_loss": 4.21080968024183, + "tokens_seen": 278855680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397873620862588, + "loss": 1.6827, + "theoretical_loss": 4.210692996164845, + "tokens_seen": 278921216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397713139418255, + "loss": 1.539, + "theoretical_loss": 4.210576347175539, + "tokens_seen": 278986752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397552657973922, + "loss": 1.6543, + "theoretical_loss": 4.210459733255123, + "tokens_seen": 279052288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397392176529589, + "loss": 1.5957, + "theoretical_loss": 4.2103431543848195, + "tokens_seen": 279117824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397231695085256, + "loss": 1.5742, + "theoretical_loss": 4.210226610545872, + "tokens_seen": 279183360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397071213640923, + "loss": 1.6426, + "theoretical_loss": 4.21011010171953, + "tokens_seen": 279248896 + }, + { + "epoch": 0.08, + "learning_rate": 0.000739691073219659, + "loss": 1.5525, + "theoretical_loss": 4.209993627887064, + "tokens_seen": 279314432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396750250752257, + "loss": 1.5869, + "theoretical_loss": 4.209877189029756, + "tokens_seen": 279379968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396589769307924, + "loss": 1.6383, + "theoretical_loss": 4.209760785128903, + "tokens_seen": 279445504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396429287863591, + "loss": 1.638, + "theoretical_loss": 4.2096444161658155, + "tokens_seen": 279511040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396268806419258, + "loss": 1.5023, + "theoretical_loss": 4.209528082121819, + "tokens_seen": 279576576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396108324974925, + "loss": 1.5516, + "theoretical_loss": 4.209411782978256, + "tokens_seen": 279642112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395947843530592, + "loss": 1.5677, + "theoretical_loss": 4.209295518716477, + "tokens_seen": 279707648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395787362086259, + "loss": 1.5189, + "theoretical_loss": 4.209179289317854, + "tokens_seen": 279773184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395626880641926, + "loss": 1.5775, + "theoretical_loss": 4.209063094763767, + "tokens_seen": 279838720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395466399197593, + "loss": 1.5707, + "theoretical_loss": 4.208946935035616, + "tokens_seen": 279904256 + }, + { + "epoch": 0.08, + "learning_rate": 0.000739530591775326, + "loss": 1.5592, + "theoretical_loss": 4.2088308101148115, + "tokens_seen": 279969792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395145436308927, + "loss": 1.6149, + "theoretical_loss": 4.208714719982778, + "tokens_seen": 280035328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394984954864594, + "loss": 1.5742, + "theoretical_loss": 4.208598664620958, + "tokens_seen": 280100864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394824473420261, + "loss": 1.6079, + "theoretical_loss": 4.208482644010804, + "tokens_seen": 280166400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394663991975928, + "loss": 1.5717, + "theoretical_loss": 4.2083666581337855, + "tokens_seen": 280231936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394503510531595, + "loss": 1.5169, + "theoretical_loss": 4.208250706971385, + "tokens_seen": 280297472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394343029087262, + "loss": 1.6327, + "theoretical_loss": 4.208134790505099, + "tokens_seen": 280363008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394182547642929, + "loss": 1.6121, + "theoretical_loss": 4.20801890871644, + "tokens_seen": 280428544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394022066198596, + "loss": 1.5979, + "theoretical_loss": 4.207903061586931, + "tokens_seen": 280494080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393861584754263, + "loss": 1.6794, + "theoretical_loss": 4.207787249098113, + "tokens_seen": 280559616 + }, + { + "epoch": 0.09, + "learning_rate": 0.000739370110330993, + "loss": 1.5258, + "theoretical_loss": 4.20767147123154, + "tokens_seen": 280625152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393540621865597, + "loss": 1.6724, + "theoretical_loss": 4.20755572796878, + "tokens_seen": 280690688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393380140421264, + "loss": 1.5395, + "theoretical_loss": 4.207440019291413, + "tokens_seen": 280756224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393219658976931, + "loss": 1.6244, + "theoretical_loss": 4.2073243451810365, + "tokens_seen": 280821760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393059177532598, + "loss": 1.6882, + "theoretical_loss": 4.20720870561926, + "tokens_seen": 280887296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392898696088265, + "loss": 1.4866, + "theoretical_loss": 4.2070931005877075, + "tokens_seen": 280952832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392738214643932, + "loss": 1.5088, + "theoretical_loss": 4.206977530068017, + "tokens_seen": 281018368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392577733199599, + "loss": 1.583, + "theoretical_loss": 4.206861994041842, + "tokens_seen": 281083904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392417251755266, + "loss": 1.5661, + "theoretical_loss": 4.206746492490847, + "tokens_seen": 281149440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392256770310933, + "loss": 1.6379, + "theoretical_loss": 4.206631025396714, + "tokens_seen": 281214976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00073920962888666, + "loss": 1.587, + "theoretical_loss": 4.206515592741135, + "tokens_seen": 281280512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391935807422267, + "loss": 1.6539, + "theoretical_loss": 4.20640019450582, + "tokens_seen": 281346048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391775325977934, + "loss": 1.6555, + "theoretical_loss": 4.206284830672491, + "tokens_seen": 281411584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391614844533601, + "loss": 1.6364, + "theoretical_loss": 4.206169501222885, + "tokens_seen": 281477120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391454363089268, + "loss": 1.5291, + "theoretical_loss": 4.20605420613875, + "tokens_seen": 281542656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391293881644935, + "loss": 1.5918, + "theoretical_loss": 4.205938945401851, + "tokens_seen": 281608192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391133400200602, + "loss": 1.5979, + "theoretical_loss": 4.205823718993966, + "tokens_seen": 281673728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390972918756269, + "loss": 1.6177, + "theoretical_loss": 4.2057085268968875, + "tokens_seen": 281739264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 99932, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.517197608947754, + "objective/train/theoretical_loss": 4.205593369092421, + "objective/train/tokens_used": 302264800, + "theoretical_loss": 4.205593369092421, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390812437311936, + "loss": 1.5729, + "theoretical_loss": 4.205593369092421, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390651955867603, + "loss": 1.6341, + "theoretical_loss": 4.205478245562386, + "tokens_seen": 281870336 + }, + { + "epoch": 0.09, + "learning_rate": 0.000739049147442327, + "loss": 1.6788, + "theoretical_loss": 4.205363156288615, + "tokens_seen": 281935872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390330992978937, + "loss": 1.621, + "theoretical_loss": 4.205248101252957, + "tokens_seen": 282001408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390170511534604, + "loss": 1.544, + "theoretical_loss": 4.205133080437273, + "tokens_seen": 282066944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390010030090271, + "loss": 1.6125, + "theoretical_loss": 4.205018093823437, + "tokens_seen": 282132480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389849548645938, + "loss": 1.5787, + "theoretical_loss": 4.204903141393338, + "tokens_seen": 282198016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389689067201605, + "loss": 1.6345, + "theoretical_loss": 4.204788223128879, + "tokens_seen": 282263552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389528585757272, + "loss": 1.5378, + "theoretical_loss": 4.204673339011976, + "tokens_seen": 282329088 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738936810431294, + "loss": 1.6958, + "theoretical_loss": 4.204558489024561, + "tokens_seen": 282394624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389207622868606, + "loss": 1.5529, + "theoretical_loss": 4.204443673148575, + "tokens_seen": 282460160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389047141424273, + "loss": 1.4651, + "theoretical_loss": 4.204328891365977, + "tokens_seen": 282525696 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738888665997994, + "loss": 1.551, + "theoretical_loss": 4.2042141436587395, + "tokens_seen": 282591232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388726178535607, + "loss": 1.5604, + "theoretical_loss": 4.204099430008846, + "tokens_seen": 282656768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388565697091274, + "loss": 1.5569, + "theoretical_loss": 4.203984750398296, + "tokens_seen": 282722304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388405215646942, + "loss": 1.6475, + "theoretical_loss": 4.203870104809101, + "tokens_seen": 282787840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388244734202608, + "loss": 1.5684, + "theoretical_loss": 4.203755493223289, + "tokens_seen": 282853376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388084252758276, + "loss": 1.7048, + "theoretical_loss": 4.203640915622898, + "tokens_seen": 282918912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387923771313942, + "loss": 1.6332, + "theoretical_loss": 4.203526371989983, + "tokens_seen": 282984448 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738776328986961, + "loss": 1.6873, + "theoretical_loss": 4.203411862306609, + "tokens_seen": 283049984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387602808425276, + "loss": 1.6564, + "theoretical_loss": 4.203297386554859, + "tokens_seen": 283115520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387442326980944, + "loss": 1.6088, + "theoretical_loss": 4.203182944716826, + "tokens_seen": 283181056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387281845536611, + "loss": 1.6262, + "theoretical_loss": 4.203068536774619, + "tokens_seen": 283246592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387121364092278, + "loss": 1.5406, + "theoretical_loss": 4.202954162710357, + "tokens_seen": 283312128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386960882647945, + "loss": 1.731, + "theoretical_loss": 4.202839822506178, + "tokens_seen": 283377664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386800401203612, + "loss": 1.6566, + "theoretical_loss": 4.202725516144228, + "tokens_seen": 283443200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386639919759279, + "loss": 1.6085, + "theoretical_loss": 4.202611243606671, + "tokens_seen": 283508736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386479438314946, + "loss": 1.7107, + "theoretical_loss": 4.202497004875681, + "tokens_seen": 283574272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386318956870613, + "loss": 1.6096, + "theoretical_loss": 4.202382799933449, + "tokens_seen": 283639808 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738615847542628, + "loss": 1.4388, + "theoretical_loss": 4.2022686287621775, + "tokens_seen": 283705344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385997993981946, + "loss": 1.588, + "theoretical_loss": 4.20215449134408, + "tokens_seen": 283770880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385837512537614, + "loss": 1.634, + "theoretical_loss": 4.202040387661389, + "tokens_seen": 283836416 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738567703109328, + "loss": 1.6546, + "theoretical_loss": 4.201926317696347, + "tokens_seen": 283901952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385516549648948, + "loss": 1.656, + "theoretical_loss": 4.201812281431209, + "tokens_seen": 283967488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385356068204614, + "loss": 1.6949, + "theoretical_loss": 4.201698278848246, + "tokens_seen": 284033024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385195586760282, + "loss": 1.5907, + "theoretical_loss": 4.201584309929743, + "tokens_seen": 284098560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385035105315948, + "loss": 1.6153, + "theoretical_loss": 4.201470374657993, + "tokens_seen": 284164096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384874623871615, + "loss": 1.6676, + "theoretical_loss": 4.201356473015308, + "tokens_seen": 284229632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384714142427282, + "loss": 1.6801, + "theoretical_loss": 4.201242604984013, + "tokens_seen": 284295168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384553660982949, + "loss": 1.7536, + "theoretical_loss": 4.201128770546442, + "tokens_seen": 284360704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384393179538616, + "loss": 1.6957, + "theoretical_loss": 4.201014969684948, + "tokens_seen": 284426240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384232698094283, + "loss": 1.6393, + "theoretical_loss": 4.200901202381893, + "tokens_seen": 284491776 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738407221664995, + "loss": 1.6103, + "theoretical_loss": 4.200787468619653, + "tokens_seen": 284557312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383911735205617, + "loss": 1.6311, + "theoretical_loss": 4.200673768380621, + "tokens_seen": 284622848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383751253761284, + "loss": 1.6207, + "theoretical_loss": 4.200560101647198, + "tokens_seen": 284688384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383590772316951, + "loss": 1.5322, + "theoretical_loss": 4.2004464684018, + "tokens_seen": 284753920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383430290872618, + "loss": 1.5648, + "theoretical_loss": 4.200332868626861, + "tokens_seen": 284819456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383269809428285, + "loss": 1.4811, + "theoretical_loss": 4.200219302304821, + "tokens_seen": 284884992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383109327983952, + "loss": 1.5884, + "theoretical_loss": 4.200105769418138, + "tokens_seen": 284950528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382948846539619, + "loss": 1.6323, + "theoretical_loss": 4.199992269949281, + "tokens_seen": 285016064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 101193, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5234239101409912, + "objective/train/theoretical_loss": 4.199878803880733, + "objective/train/tokens_used": 305541600, + "theoretical_loss": 4.199878803880733, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382788365095286, + "loss": 1.4773, + "theoretical_loss": 4.199878803880733, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382627883650953, + "loss": 1.4435, + "theoretical_loss": 4.1997653711949905, + "tokens_seen": 285147136 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738246740220662, + "loss": 1.5471, + "theoretical_loss": 4.199651971874562, + "tokens_seen": 285212672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382306920762287, + "loss": 1.5007, + "theoretical_loss": 4.199538605901973, + "tokens_seen": 285278208 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382146439317954, + "loss": 1.6037, + "theoretical_loss": 4.199425273259756, + "tokens_seen": 285343744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381985957873621, + "loss": 1.5669, + "theoretical_loss": 4.199311973930461, + "tokens_seen": 285409280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381825476429288, + "loss": 1.5726, + "theoretical_loss": 4.199198707896651, + "tokens_seen": 285474816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381664994984955, + "loss": 1.5804, + "theoretical_loss": 4.199085475140899, + "tokens_seen": 285540352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381504513540622, + "loss": 1.6743, + "theoretical_loss": 4.1989722756457954, + "tokens_seen": 285605888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381344032096289, + "loss": 1.5639, + "theoretical_loss": 4.198859109393941, + "tokens_seen": 285671424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381183550651956, + "loss": 1.5105, + "theoretical_loss": 4.198745976367949, + "tokens_seen": 285736960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381023069207623, + "loss": 1.5629, + "theoretical_loss": 4.198632876550449, + "tokens_seen": 285802496 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738086258776329, + "loss": 1.5286, + "theoretical_loss": 4.19851980992408, + "tokens_seen": 285868032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380702106318957, + "loss": 1.6526, + "theoretical_loss": 4.198406776471497, + "tokens_seen": 285933568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380541624874624, + "loss": 1.7248, + "theoretical_loss": 4.198293776175365, + "tokens_seen": 285999104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380381143430291, + "loss": 1.4311, + "theoretical_loss": 4.198180809018366, + "tokens_seen": 286064640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380220661985958, + "loss": 1.5445, + "theoretical_loss": 4.198067874983192, + "tokens_seen": 286130176 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380060180541625, + "loss": 1.58, + "theoretical_loss": 4.197954974052546, + "tokens_seen": 286195712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379899699097292, + "loss": 1.5693, + "theoretical_loss": 4.19784210620915, + "tokens_seen": 286261248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379739217652959, + "loss": 1.6604, + "theoretical_loss": 4.197729271435736, + "tokens_seen": 286326784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379578736208626, + "loss": 1.6563, + "theoretical_loss": 4.197616469715046, + "tokens_seen": 286392320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379418254764293, + "loss": 1.75, + "theoretical_loss": 4.19750370102984, + "tokens_seen": 286457856 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737925777331996, + "loss": 1.7021, + "theoretical_loss": 4.197390965362887, + "tokens_seen": 286523392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379097291875627, + "loss": 1.567, + "theoretical_loss": 4.197278262696971, + "tokens_seen": 286588928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378936810431294, + "loss": 1.5716, + "theoretical_loss": 4.197165593014889, + "tokens_seen": 286654464 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378776328986961, + "loss": 1.6708, + "theoretical_loss": 4.197052956299449, + "tokens_seen": 286720000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378615847542628, + "loss": 1.5052, + "theoretical_loss": 4.196940352533474, + "tokens_seen": 286785536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378455366098295, + "loss": 1.5812, + "theoretical_loss": 4.196827781699799, + "tokens_seen": 286851072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378294884653962, + "loss": 1.6132, + "theoretical_loss": 4.196715243781273, + "tokens_seen": 286916608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378134403209629, + "loss": 1.6742, + "theoretical_loss": 4.196602738760754, + "tokens_seen": 286982144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377973921765296, + "loss": 1.6104, + "theoretical_loss": 4.196490266621119, + "tokens_seen": 287047680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377813440320963, + "loss": 1.6694, + "theoretical_loss": 4.196377827345252, + "tokens_seen": 287113216 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737765295887663, + "loss": 1.5387, + "theoretical_loss": 4.196265420916053, + "tokens_seen": 287178752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377492477432297, + "loss": 1.6054, + "theoretical_loss": 4.196153047316433, + "tokens_seen": 287244288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377331995987964, + "loss": 1.5317, + "theoretical_loss": 4.196040706529319, + "tokens_seen": 287309824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377171514543631, + "loss": 1.6415, + "theoretical_loss": 4.195928398537647, + "tokens_seen": 287375360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377011033099298, + "loss": 1.6341, + "theoretical_loss": 4.195816123324367, + "tokens_seen": 287440896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376850551654965, + "loss": 1.6622, + "theoretical_loss": 4.195703880872443, + "tokens_seen": 287506432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376690070210632, + "loss": 1.5524, + "theoretical_loss": 4.195591671164852, + "tokens_seen": 287571968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376529588766299, + "loss": 1.5814, + "theoretical_loss": 4.195479494184581, + "tokens_seen": 287637504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376369107321966, + "loss": 1.5762, + "theoretical_loss": 4.195367349914631, + "tokens_seen": 287703040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376208625877633, + "loss": 1.5798, + "theoretical_loss": 4.195255238338016, + "tokens_seen": 287768576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00073760481444333, + "loss": 1.5229, + "theoretical_loss": 4.195143159437764, + "tokens_seen": 287834112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375887662988967, + "loss": 1.5433, + "theoretical_loss": 4.195031113196913, + "tokens_seen": 287899648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375727181544634, + "loss": 1.6659, + "theoretical_loss": 4.194919099598516, + "tokens_seen": 287965184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375566700100301, + "loss": 1.5775, + "theoretical_loss": 4.194807118625636, + "tokens_seen": 288030720 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375406218655968, + "loss": 1.6525, + "theoretical_loss": 4.194695170261353, + "tokens_seen": 288096256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375245737211636, + "loss": 1.5722, + "theoretical_loss": 4.194583254488754, + "tokens_seen": 288161792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375085255767302, + "loss": 1.5539, + "theoretical_loss": 4.194471371290944, + "tokens_seen": 288227328 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737492477432297, + "loss": 1.6119, + "theoretical_loss": 4.194359520651036, + "tokens_seen": 288292864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 102045, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5714504718780518, + "objective/train/theoretical_loss": 4.1942477025521585, + "objective/train/tokens_used": 308818400, + "theoretical_loss": 4.1942477025521585, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007374764292878637, + "loss": 1.5811, + "theoretical_loss": 4.1942477025521585, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007374603811434304, + "loss": 1.6291, + "theoretical_loss": 4.194135916977452, + "tokens_seen": 288423936 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737444332998997, + "loss": 1.6362, + "theoretical_loss": 4.19402416391007, + "tokens_seen": 288489472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007374282848545638, + "loss": 1.5195, + "theoretical_loss": 4.193912443333177, + "tokens_seen": 288555008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007374122367101305, + "loss": 1.474, + "theoretical_loss": 4.193800755229951, + "tokens_seen": 288620544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373961885656972, + "loss": 1.6996, + "theoretical_loss": 4.193689099583582, + "tokens_seen": 288686080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373801404212639, + "loss": 1.5586, + "theoretical_loss": 4.193577476377275, + "tokens_seen": 288751616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373640922768306, + "loss": 1.5385, + "theoretical_loss": 4.193465885594242, + "tokens_seen": 288817152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373480441323973, + "loss": 1.5508, + "theoretical_loss": 4.1933543272177145, + "tokens_seen": 288882688 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737331995987964, + "loss": 1.6673, + "theoretical_loss": 4.193242801230931, + "tokens_seen": 288948224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373159478435307, + "loss": 1.6181, + "theoretical_loss": 4.193131307617145, + "tokens_seen": 289013760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372998996990974, + "loss": 1.6803, + "theoretical_loss": 4.193019846359622, + "tokens_seen": 289079296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372838515546641, + "loss": 1.6104, + "theoretical_loss": 4.192908417441639, + "tokens_seen": 289144832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372678034102308, + "loss": 1.6341, + "theoretical_loss": 4.192797020846487, + "tokens_seen": 289210368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372517552657975, + "loss": 1.5669, + "theoretical_loss": 4.192685656557468, + "tokens_seen": 289275904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372357071213642, + "loss": 1.6607, + "theoretical_loss": 4.1925743245578975, + "tokens_seen": 289341440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372196589769309, + "loss": 1.5765, + "theoretical_loss": 4.1924630248311034, + "tokens_seen": 289406976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372036108324975, + "loss": 1.48, + "theoretical_loss": 4.192351757360425, + "tokens_seen": 289472512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371875626880643, + "loss": 1.5329, + "theoretical_loss": 4.1922405221292145, + "tokens_seen": 289538048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371715145436309, + "loss": 1.4819, + "theoretical_loss": 4.192129319120838, + "tokens_seen": 289603584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371554663991977, + "loss": 1.6302, + "theoretical_loss": 4.1920181483186685, + "tokens_seen": 289669120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371394182547643, + "loss": 1.5838, + "theoretical_loss": 4.1919070097060995, + "tokens_seen": 289734656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371233701103311, + "loss": 1.5787, + "theoretical_loss": 4.19179590326653, + "tokens_seen": 289800192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371073219658977, + "loss": 1.5751, + "theoretical_loss": 4.191684828983375, + "tokens_seen": 289865728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370912738214645, + "loss": 1.664, + "theoretical_loss": 4.191573786840061, + "tokens_seen": 289931264 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370752256770311, + "loss": 1.5749, + "theoretical_loss": 4.191462776820025, + "tokens_seen": 289996800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370591775325978, + "loss": 1.604, + "theoretical_loss": 4.191351798906719, + "tokens_seen": 290062336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370431293881645, + "loss": 1.7533, + "theoretical_loss": 4.191240853083604, + "tokens_seen": 290127872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370270812437312, + "loss": 1.6221, + "theoretical_loss": 4.191129939334159, + "tokens_seen": 290193408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370110330992979, + "loss": 1.5681, + "theoretical_loss": 4.191019057641868, + "tokens_seen": 290258944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369949849548646, + "loss": 1.6289, + "theoretical_loss": 4.1909082079902324, + "tokens_seen": 290324480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369789368104313, + "loss": 1.7017, + "theoretical_loss": 4.190797390362764, + "tokens_seen": 290390016 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736962888665998, + "loss": 1.5707, + "theoretical_loss": 4.190686604742986, + "tokens_seen": 290455552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369468405215647, + "loss": 1.6355, + "theoretical_loss": 4.190575851114437, + "tokens_seen": 290521088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369307923771314, + "loss": 1.7072, + "theoretical_loss": 4.190465129460662, + "tokens_seen": 290586624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369147442326981, + "loss": 1.6748, + "theoretical_loss": 4.190354439765224, + "tokens_seen": 290652160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368986960882648, + "loss": 1.5803, + "theoretical_loss": 4.190243782011696, + "tokens_seen": 290717696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368826479438315, + "loss": 1.6931, + "theoretical_loss": 4.190133156183663, + "tokens_seen": 290783232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368665997993982, + "loss": 1.5436, + "theoretical_loss": 4.190022562264721, + "tokens_seen": 290848768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368505516549649, + "loss": 1.6706, + "theoretical_loss": 4.189912000238479, + "tokens_seen": 290914304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368345035105316, + "loss": 1.5295, + "theoretical_loss": 4.189801470088559, + "tokens_seen": 290979840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368184553660983, + "loss": 1.6353, + "theoretical_loss": 4.189690971798596, + "tokens_seen": 291045376 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736802407221665, + "loss": 1.6281, + "theoretical_loss": 4.189580505352232, + "tokens_seen": 291110912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367863590772317, + "loss": 1.6095, + "theoretical_loss": 4.189470070733129, + "tokens_seen": 291176448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367703109327984, + "loss": 1.6304, + "theoretical_loss": 4.189359667924952, + "tokens_seen": 291241984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367542627883651, + "loss": 1.6004, + "theoretical_loss": 4.189249296911386, + "tokens_seen": 291307520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367382146439318, + "loss": 1.6946, + "theoretical_loss": 4.189138957676125, + "tokens_seen": 291373056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367221664994985, + "loss": 1.7297, + "theoretical_loss": 4.1890286502028715, + "tokens_seen": 291438592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367061183550652, + "loss": 1.6308, + "theoretical_loss": 4.188918374475347, + "tokens_seen": 291504128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366900702106319, + "loss": 1.6745, + "theoretical_loss": 4.188808130477279, + "tokens_seen": 291569664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 103221, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6348395347595215, + "objective/train/theoretical_loss": 4.1886979181924096, + "objective/train/tokens_used": 312095200, + "theoretical_loss": 4.1886979181924096, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366740220661986, + "loss": 1.647, + "theoretical_loss": 4.1886979181924096, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366579739217653, + "loss": 1.7113, + "theoretical_loss": 4.188587737604493, + "tokens_seen": 291700736 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736641925777332, + "loss": 1.6427, + "theoretical_loss": 4.188477588697295, + "tokens_seen": 291766272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366258776328987, + "loss": 1.5969, + "theoretical_loss": 4.188367471454593, + "tokens_seen": 291831808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366098294884654, + "loss": 1.4803, + "theoretical_loss": 4.188257385860177, + "tokens_seen": 291897344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365937813440321, + "loss": 1.6042, + "theoretical_loss": 4.188147331897849, + "tokens_seen": 291962880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365777331995988, + "loss": 1.6471, + "theoretical_loss": 4.1880373095514205, + "tokens_seen": 292028416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365616850551655, + "loss": 1.6034, + "theoretical_loss": 4.18792731880472, + "tokens_seen": 292093952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365456369107322, + "loss": 1.5614, + "theoretical_loss": 4.187817359641581, + "tokens_seen": 292159488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365295887662989, + "loss": 1.5428, + "theoretical_loss": 4.1877074320458565, + "tokens_seen": 292225024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365135406218656, + "loss": 1.5486, + "theoretical_loss": 4.187597536001406, + "tokens_seen": 292290560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364974924774323, + "loss": 1.5498, + "theoretical_loss": 4.187487671492102, + "tokens_seen": 292356096 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736481444332999, + "loss": 1.6196, + "theoretical_loss": 4.18737783850183, + "tokens_seen": 292421632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364653961885657, + "loss": 1.6716, + "theoretical_loss": 4.187268037014486, + "tokens_seen": 292487168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364493480441324, + "loss": 1.6067, + "theoretical_loss": 4.187158267013979, + "tokens_seen": 292552704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364332998996991, + "loss": 1.517, + "theoretical_loss": 4.18704852848423, + "tokens_seen": 292618240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364172517552658, + "loss": 1.6312, + "theoretical_loss": 4.18693882140917, + "tokens_seen": 292683776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364012036108325, + "loss": 1.4959, + "theoretical_loss": 4.186829145772743, + "tokens_seen": 292749312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363851554663992, + "loss": 1.5707, + "theoretical_loss": 4.186719501558905, + "tokens_seen": 292814848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363691073219659, + "loss": 1.5797, + "theoretical_loss": 4.186609888751623, + "tokens_seen": 292880384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363530591775326, + "loss": 1.6421, + "theoretical_loss": 4.186500307334878, + "tokens_seen": 292945920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363370110330993, + "loss": 1.6258, + "theoretical_loss": 4.186390757292659, + "tokens_seen": 293011456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363209628886661, + "loss": 1.5723, + "theoretical_loss": 4.18628123860897, + "tokens_seen": 293076992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363049147442327, + "loss": 1.638, + "theoretical_loss": 4.186171751267825, + "tokens_seen": 293142528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362888665997995, + "loss": 1.6085, + "theoretical_loss": 4.186062295253249, + "tokens_seen": 293208064 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362728184553661, + "loss": 1.6619, + "theoretical_loss": 4.185952870549283, + "tokens_seen": 293273600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362567703109329, + "loss": 1.5903, + "theoretical_loss": 4.185843477139974, + "tokens_seen": 293339136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362407221664995, + "loss": 1.555, + "theoretical_loss": 4.185734115009383, + "tokens_seen": 293404672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362246740220663, + "loss": 1.5025, + "theoretical_loss": 4.185624784141585, + "tokens_seen": 293470208 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736208625877633, + "loss": 1.5115, + "theoretical_loss": 4.185515484520664, + "tokens_seen": 293535744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361925777331997, + "loss": 1.6593, + "theoretical_loss": 4.1854062161307155, + "tokens_seen": 293601280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361765295887663, + "loss": 1.7188, + "theoretical_loss": 4.185296978955848, + "tokens_seen": 293666816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361604814443331, + "loss": 1.5893, + "theoretical_loss": 4.185187772980181, + "tokens_seen": 293732352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361444332998997, + "loss": 1.4278, + "theoretical_loss": 4.185078598187846, + "tokens_seen": 293797888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361283851554665, + "loss": 1.4631, + "theoretical_loss": 4.184969454562985, + "tokens_seen": 293863424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361123370110332, + "loss": 1.5732, + "theoretical_loss": 4.1848603420897525, + "tokens_seen": 293928960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360962888665999, + "loss": 1.5025, + "theoretical_loss": 4.184751260752316, + "tokens_seen": 293994496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360802407221666, + "loss": 1.5542, + "theoretical_loss": 4.18464221053485, + "tokens_seen": 294060032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360641925777333, + "loss": 1.6657, + "theoretical_loss": 4.184533191421547, + "tokens_seen": 294125568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360481444333, + "loss": 1.7267, + "theoretical_loss": 4.184424203396606, + "tokens_seen": 294191104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360320962888667, + "loss": 1.4338, + "theoretical_loss": 4.184315246444239, + "tokens_seen": 294256640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360160481444334, + "loss": 1.5626, + "theoretical_loss": 4.18420632054867, + "tokens_seen": 294322176 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360000000000001, + "loss": 1.6043, + "theoretical_loss": 4.1840974256941355, + "tokens_seen": 294387712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359839518555668, + "loss": 1.5946, + "theoretical_loss": 4.183988561864879, + "tokens_seen": 294453248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359679037111335, + "loss": 1.5876, + "theoretical_loss": 4.183879729045163, + "tokens_seen": 294518784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359518555667002, + "loss": 1.6075, + "theoretical_loss": 4.183770927219255, + "tokens_seen": 294584320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359358074222669, + "loss": 1.5569, + "theoretical_loss": 4.183662156371436, + "tokens_seen": 294649856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359197592778336, + "loss": 1.6115, + "theoretical_loss": 4.183553416485999, + "tokens_seen": 294715392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359037111334003, + "loss": 1.5446, + "theoretical_loss": 4.183444707547249, + "tokens_seen": 294780928 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735887662988967, + "loss": 1.6602, + "theoretical_loss": 4.1833360295395, + "tokens_seen": 294846464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 104159, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.72190523147583, + "objective/train/theoretical_loss": 4.183227382447081, + "objective/train/tokens_used": 315372000, + "theoretical_loss": 4.183227382447081, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358716148445337, + "loss": 1.6478, + "theoretical_loss": 4.183227382447081, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358555667001004, + "loss": 1.6181, + "theoretical_loss": 4.183118766254328, + "tokens_seen": 294977536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358395185556671, + "loss": 1.6082, + "theoretical_loss": 4.183010180945593, + "tokens_seen": 295043072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358234704112338, + "loss": 1.5291, + "theoretical_loss": 4.182901626505237, + "tokens_seen": 295108608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358074222668004, + "loss": 1.6135, + "theoretical_loss": 4.182793102917632, + "tokens_seen": 295174144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357913741223672, + "loss": 1.5915, + "theoretical_loss": 4.182684610167162, + "tokens_seen": 295239680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357753259779338, + "loss": 1.6561, + "theoretical_loss": 4.1825761482382235, + "tokens_seen": 295305216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357592778335006, + "loss": 1.7016, + "theoretical_loss": 4.182467717115221, + "tokens_seen": 295370752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357432296890672, + "loss": 1.6305, + "theoretical_loss": 4.182359316782576, + "tokens_seen": 295436288 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735727181544634, + "loss": 1.5485, + "theoretical_loss": 4.1822509472247145, + "tokens_seen": 295501824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357111334002006, + "loss": 1.6077, + "theoretical_loss": 4.18214260842608, + "tokens_seen": 295567360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356950852557674, + "loss": 1.5885, + "theoretical_loss": 4.182034300371122, + "tokens_seen": 295632896 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735679037111334, + "loss": 1.564, + "theoretical_loss": 4.181926023044306, + "tokens_seen": 295698432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356629889669007, + "loss": 1.5082, + "theoretical_loss": 4.181817776430106, + "tokens_seen": 295763968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356469408224674, + "loss": 1.711, + "theoretical_loss": 4.181709560513008, + "tokens_seen": 295829504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356308926780341, + "loss": 1.616, + "theoretical_loss": 4.181601375277509, + "tokens_seen": 295895040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356148445336008, + "loss": 1.6577, + "theoretical_loss": 4.181493220708117, + "tokens_seen": 295960576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355987963891675, + "loss": 1.6458, + "theoretical_loss": 4.181385096789353, + "tokens_seen": 296026112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355827482447342, + "loss": 1.6822, + "theoretical_loss": 4.181277003505747, + "tokens_seen": 296091648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355667001003009, + "loss": 1.5702, + "theoretical_loss": 4.181168940841843, + "tokens_seen": 296157184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355506519558676, + "loss": 1.6557, + "theoretical_loss": 4.181060908782191, + "tokens_seen": 296222720 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355346038114343, + "loss": 1.5546, + "theoretical_loss": 4.18095290731136, + "tokens_seen": 296288256 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735518555667001, + "loss": 1.53, + "theoretical_loss": 4.180844936413922, + "tokens_seen": 296353792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355025075225677, + "loss": 1.562, + "theoretical_loss": 4.180736996074465, + "tokens_seen": 296419328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354864593781344, + "loss": 1.7042, + "theoretical_loss": 4.180629086277589, + "tokens_seen": 296484864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354704112337011, + "loss": 1.5895, + "theoretical_loss": 4.180521207007902, + "tokens_seen": 296550400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354543630892678, + "loss": 1.6513, + "theoretical_loss": 4.180413358250024, + "tokens_seen": 296615936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354383149448345, + "loss": 1.6866, + "theoretical_loss": 4.180305539988588, + "tokens_seen": 296681472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354222668004012, + "loss": 1.5783, + "theoretical_loss": 4.180197752208235, + "tokens_seen": 296747008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354062186559679, + "loss": 1.6043, + "theoretical_loss": 4.1800899948936205, + "tokens_seen": 296812544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353901705115346, + "loss": 1.565, + "theoretical_loss": 4.179982268029409, + "tokens_seen": 296878080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353741223671013, + "loss": 1.5138, + "theoretical_loss": 4.179874571600277, + "tokens_seen": 296943616 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735358074222668, + "loss": 1.6457, + "theoretical_loss": 4.1797669055909115, + "tokens_seen": 297009152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353420260782347, + "loss": 1.6078, + "theoretical_loss": 4.1796592699860105, + "tokens_seen": 297074688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353259779338014, + "loss": 1.691, + "theoretical_loss": 4.179551664770283, + "tokens_seen": 297140224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353099297893681, + "loss": 1.5473, + "theoretical_loss": 4.1794440899284515, + "tokens_seen": 297205760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352938816449348, + "loss": 1.5952, + "theoretical_loss": 4.179336545445245, + "tokens_seen": 297271296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352778335005015, + "loss": 1.5846, + "theoretical_loss": 4.1792290313054075, + "tokens_seen": 297336832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352617853560682, + "loss": 1.5803, + "theoretical_loss": 4.179121547493692, + "tokens_seen": 297402368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352457372116349, + "loss": 1.6163, + "theoretical_loss": 4.179014093994862, + "tokens_seen": 297467904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352296890672016, + "loss": 1.59, + "theoretical_loss": 4.178906670793695, + "tokens_seen": 297533440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352136409227683, + "loss": 1.6357, + "theoretical_loss": 4.178799277874977, + "tokens_seen": 297598976 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735197592778335, + "loss": 1.6383, + "theoretical_loss": 4.178691915223505, + "tokens_seen": 297664512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351815446339017, + "loss": 1.5541, + "theoretical_loss": 4.178584582824088, + "tokens_seen": 297730048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351654964894684, + "loss": 1.7037, + "theoretical_loss": 4.1784772806615464, + "tokens_seen": 297795584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351494483450351, + "loss": 1.6525, + "theoretical_loss": 4.178370008720709, + "tokens_seen": 297861120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351334002006018, + "loss": 1.5659, + "theoretical_loss": 4.178262766986418, + "tokens_seen": 297926656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351173520561686, + "loss": 1.5114, + "theoretical_loss": 4.178155555443525, + "tokens_seen": 297992192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351013039117352, + "loss": 1.7272, + "theoretical_loss": 4.178048374076894, + "tokens_seen": 298057728 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735085255767302, + "loss": 1.6131, + "theoretical_loss": 4.1779412228714, + "tokens_seen": 298123264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 105423, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5215253829956055, + "objective/train/theoretical_loss": 4.177834101811927, + "objective/train/tokens_used": 318648800, + "theoretical_loss": 4.177834101811927, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007350692076228686, + "loss": 1.5549, + "theoretical_loss": 4.177834101811927, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007350531594784354, + "loss": 1.4696, + "theoretical_loss": 4.1777270108833715, + "tokens_seen": 298254336 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735037111334002, + "loss": 1.6553, + "theoretical_loss": 4.177619950070639, + "tokens_seen": 298319872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007350210631895688, + "loss": 1.6148, + "theoretical_loss": 4.177512919358649, + "tokens_seen": 298385408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007350050150451354, + "loss": 1.6731, + "theoretical_loss": 4.17740591873233, + "tokens_seen": 298450944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349889669007022, + "loss": 1.6382, + "theoretical_loss": 4.177298948176619, + "tokens_seen": 298516480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349729187562688, + "loss": 1.4472, + "theoretical_loss": 4.17719200767647, + "tokens_seen": 298582016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349568706118356, + "loss": 1.6397, + "theoretical_loss": 4.177085097216842, + "tokens_seen": 298647552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349408224674022, + "loss": 1.6669, + "theoretical_loss": 4.1769782167827065, + "tokens_seen": 298713088 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734924774322969, + "loss": 1.6121, + "theoretical_loss": 4.176871366359047, + "tokens_seen": 298778624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349087261785356, + "loss": 1.4917, + "theoretical_loss": 4.176764545930858, + "tokens_seen": 298844160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348926780341024, + "loss": 1.5772, + "theoretical_loss": 4.176657755483143, + "tokens_seen": 298909696 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734876629889669, + "loss": 1.6754, + "theoretical_loss": 4.176550995000917, + "tokens_seen": 298975232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348605817452358, + "loss": 1.6027, + "theoretical_loss": 4.176444264469206, + "tokens_seen": 299040768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348445336008024, + "loss": 1.7495, + "theoretical_loss": 4.176337563873046, + "tokens_seen": 299106304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348284854563692, + "loss": 1.6497, + "theoretical_loss": 4.176230893197486, + "tokens_seen": 299171840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348124373119358, + "loss": 1.51, + "theoretical_loss": 4.1761242524275834, + "tokens_seen": 299237376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347963891675026, + "loss": 1.5526, + "theoretical_loss": 4.176017641548407, + "tokens_seen": 299302912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347803410230692, + "loss": 1.7034, + "theoretical_loss": 4.175911060545037, + "tokens_seen": 299368448 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734764292878636, + "loss": 1.6063, + "theoretical_loss": 4.175804509402562, + "tokens_seen": 299433984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347482447342027, + "loss": 1.4916, + "theoretical_loss": 4.175697988106085, + "tokens_seen": 299499520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347321965897694, + "loss": 1.7321, + "theoretical_loss": 4.175591496640718, + "tokens_seen": 299565056 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734716148445336, + "loss": 1.5392, + "theoretical_loss": 4.175485034991581, + "tokens_seen": 299630592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347001003009028, + "loss": 1.5464, + "theoretical_loss": 4.17537860314381, + "tokens_seen": 299696128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346840521564695, + "loss": 1.5023, + "theoretical_loss": 4.175272201082547, + "tokens_seen": 299761664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346680040120362, + "loss": 1.5762, + "theoretical_loss": 4.175165828792946, + "tokens_seen": 299827200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346519558676029, + "loss": 1.6361, + "theoretical_loss": 4.1750594862601735, + "tokens_seen": 299892736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346359077231696, + "loss": 1.6891, + "theoretical_loss": 4.1749531734694045, + "tokens_seen": 299958272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346198595787363, + "loss": 1.5498, + "theoretical_loss": 4.174846890405825, + "tokens_seen": 300023808 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734603811434303, + "loss": 1.5768, + "theoretical_loss": 4.174740637054632, + "tokens_seen": 300089344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345877632898697, + "loss": 1.5761, + "theoretical_loss": 4.174634413401034, + "tokens_seen": 300154880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345717151454364, + "loss": 1.5583, + "theoretical_loss": 4.174528219430247, + "tokens_seen": 300220416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345556670010031, + "loss": 1.5313, + "theoretical_loss": 4.174422055127502, + "tokens_seen": 300285952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345396188565698, + "loss": 1.6252, + "theoretical_loss": 4.174315920478037, + "tokens_seen": 300351488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345235707121365, + "loss": 1.5832, + "theoretical_loss": 4.174209815467102, + "tokens_seen": 300417024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345075225677032, + "loss": 1.5806, + "theoretical_loss": 4.174103740079958, + "tokens_seen": 300482560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344914744232699, + "loss": 1.6372, + "theoretical_loss": 4.1739976943018755, + "tokens_seen": 300548096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344754262788366, + "loss": 1.7141, + "theoretical_loss": 4.173891678118135, + "tokens_seen": 300613632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344593781344033, + "loss": 1.592, + "theoretical_loss": 4.173785691514031, + "tokens_seen": 300679168 + }, + { + "epoch": 0.09, + "learning_rate": 0.00073444332998997, + "loss": 1.6567, + "theoretical_loss": 4.173679734474862, + "tokens_seen": 300744704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344272818455367, + "loss": 1.5908, + "theoretical_loss": 4.173573806985945, + "tokens_seen": 300810240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344112337011033, + "loss": 1.6119, + "theoretical_loss": 4.173467909032602, + "tokens_seen": 300875776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343951855566701, + "loss": 1.6013, + "theoretical_loss": 4.173362040600166, + "tokens_seen": 300941312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343791374122367, + "loss": 1.6848, + "theoretical_loss": 4.173256201673983, + "tokens_seen": 301006848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343630892678035, + "loss": 1.5417, + "theoretical_loss": 4.173150392239406, + "tokens_seen": 301072384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343470411233701, + "loss": 1.5092, + "theoretical_loss": 4.173044612281802, + "tokens_seen": 301137920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343309929789369, + "loss": 1.5109, + "theoretical_loss": 4.172938861786546, + "tokens_seen": 301203456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343149448345035, + "loss": 1.5317, + "theoretical_loss": 4.1728331407390264, + "tokens_seen": 301268992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342988966900703, + "loss": 1.58, + "theoretical_loss": 4.172727449124636, + "tokens_seen": 301334528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342828485456369, + "loss": 1.625, + "theoretical_loss": 4.172621786928786, + "tokens_seen": 301400064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 106715, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5944786071777344, + "objective/train/theoretical_loss": 4.17251615413689, + "objective/train/tokens_used": 321925600, + "theoretical_loss": 4.17251615413689, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342668004012036, + "loss": 1.6601, + "theoretical_loss": 4.17251615413689, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342507522567703, + "loss": 1.5005, + "theoretical_loss": 4.172410550734378, + "tokens_seen": 301531136 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734234704112337, + "loss": 1.5192, + "theoretical_loss": 4.172304976706689, + "tokens_seen": 301596672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342186559679037, + "loss": 1.5185, + "theoretical_loss": 4.17219943203927, + "tokens_seen": 301662208 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342026078234704, + "loss": 1.7129, + "theoretical_loss": 4.172093916717581, + "tokens_seen": 301727744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341865596790371, + "loss": 1.5986, + "theoretical_loss": 4.171988430727091, + "tokens_seen": 301793280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341705115346038, + "loss": 1.5291, + "theoretical_loss": 4.17188297405328, + "tokens_seen": 301858816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341544633901705, + "loss": 1.6157, + "theoretical_loss": 4.171777546681638, + "tokens_seen": 301924352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341384152457372, + "loss": 1.6012, + "theoretical_loss": 4.171672148597665, + "tokens_seen": 301989888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341223671013039, + "loss": 1.7156, + "theoretical_loss": 4.171566779786872, + "tokens_seen": 302055424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341063189568706, + "loss": 1.5974, + "theoretical_loss": 4.17146144023478, + "tokens_seen": 302120960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340902708124373, + "loss": 1.5717, + "theoretical_loss": 4.171356129926921, + "tokens_seen": 302186496 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734074222668004, + "loss": 1.647, + "theoretical_loss": 4.171250848848835, + "tokens_seen": 302252032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340581745235707, + "loss": 1.7013, + "theoretical_loss": 4.171145596986076, + "tokens_seen": 302317568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340421263791374, + "loss": 1.5124, + "theoretical_loss": 4.171040374324204, + "tokens_seen": 302383104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340260782347041, + "loss": 1.5927, + "theoretical_loss": 4.170935180848793, + "tokens_seen": 302448640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340100300902708, + "loss": 1.5633, + "theoretical_loss": 4.170830016545425, + "tokens_seen": 302514176 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339939819458375, + "loss": 1.7269, + "theoretical_loss": 4.1707248813996936, + "tokens_seen": 302579712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339779338014042, + "loss": 1.6634, + "theoretical_loss": 4.170619775397201, + "tokens_seen": 302645248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339618856569709, + "loss": 1.7733, + "theoretical_loss": 4.1705146985235615, + "tokens_seen": 302710784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339458375125376, + "loss": 1.6171, + "theoretical_loss": 4.170409650764397, + "tokens_seen": 302776320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339297893681043, + "loss": 1.49, + "theoretical_loss": 4.170304632105344, + "tokens_seen": 302841856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339137412236711, + "loss": 1.6787, + "theoretical_loss": 4.170199642532045, + "tokens_seen": 302907392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338976930792377, + "loss": 1.5294, + "theoretical_loss": 4.170094682030154, + "tokens_seen": 302972928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338816449348045, + "loss": 1.5229, + "theoretical_loss": 4.169989750585335, + "tokens_seen": 303038464 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338655967903711, + "loss": 1.6258, + "theoretical_loss": 4.169884848183264, + "tokens_seen": 303104000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338495486459379, + "loss": 1.5401, + "theoretical_loss": 4.169779974809624, + "tokens_seen": 303169536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338335005015045, + "loss": 1.6634, + "theoretical_loss": 4.169675130450111, + "tokens_seen": 303235072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338174523570713, + "loss": 1.5992, + "theoretical_loss": 4.16957031509043, + "tokens_seen": 303300608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338014042126379, + "loss": 1.601, + "theoretical_loss": 4.169465528716295, + "tokens_seen": 303366144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337853560682047, + "loss": 1.4711, + "theoretical_loss": 4.169360771313434, + "tokens_seen": 303431680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337693079237713, + "loss": 1.7924, + "theoretical_loss": 4.169256042867579, + "tokens_seen": 303497216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337532597793381, + "loss": 1.6994, + "theoretical_loss": 4.169151343364476, + "tokens_seen": 303562752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337372116349047, + "loss": 1.5778, + "theoretical_loss": 4.169046672789882, + "tokens_seen": 303628288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337211634904715, + "loss": 1.6038, + "theoretical_loss": 4.168942031129562, + "tokens_seen": 303693824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337051153460381, + "loss": 1.6144, + "theoretical_loss": 4.168837418369292, + "tokens_seen": 303759360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336890672016049, + "loss": 1.6138, + "theoretical_loss": 4.168732834494857, + "tokens_seen": 303824896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336730190571715, + "loss": 1.5502, + "theoretical_loss": 4.1686282794920535, + "tokens_seen": 303890432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336569709127383, + "loss": 1.7256, + "theoretical_loss": 4.168523753346687, + "tokens_seen": 303955968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336409227683049, + "loss": 1.5377, + "theoretical_loss": 4.168419256044573, + "tokens_seen": 304021504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336248746238717, + "loss": 1.632, + "theoretical_loss": 4.168314787571538, + "tokens_seen": 304087040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336088264794383, + "loss": 1.626, + "theoretical_loss": 4.168210347913418, + "tokens_seen": 304152576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335927783350051, + "loss": 1.6017, + "theoretical_loss": 4.168105937056059, + "tokens_seen": 304218112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335767301905717, + "loss": 1.7149, + "theoretical_loss": 4.168001554985317, + "tokens_seen": 304283648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335606820461385, + "loss": 1.6081, + "theoretical_loss": 4.167897201687056, + "tokens_seen": 304349184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335446339017051, + "loss": 1.6175, + "theoretical_loss": 4.167792877147155, + "tokens_seen": 304414720 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335285857572719, + "loss": 1.5451, + "theoretical_loss": 4.1676885813514986, + "tokens_seen": 304480256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335125376128385, + "loss": 1.6038, + "theoretical_loss": 4.167584314285982, + "tokens_seen": 304545792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334964894684053, + "loss": 1.5534, + "theoretical_loss": 4.167480075936512, + "tokens_seen": 304611328 + }, + { + "epoch": 0.09, + "learning_rate": 0.000733480441323972, + "loss": 1.5889, + "theoretical_loss": 4.167375866289004, + "tokens_seen": 304676864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 107694, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6512261629104614, + "objective/train/theoretical_loss": 4.167271685329383, + "objective/train/tokens_used": 325202400, + "theoretical_loss": 4.167271685329383, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334643931795387, + "loss": 1.5713, + "theoretical_loss": 4.167271685329383, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334483450351053, + "loss": 1.6476, + "theoretical_loss": 4.167167533043585, + "tokens_seen": 304807936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334322968906721, + "loss": 1.63, + "theoretical_loss": 4.1670634094175565, + "tokens_seen": 304873472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334162487462387, + "loss": 1.541, + "theoretical_loss": 4.166959314437252, + "tokens_seen": 304939008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334002006018055, + "loss": 1.6109, + "theoretical_loss": 4.166855248088637, + "tokens_seen": 305004544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333841524573721, + "loss": 1.6243, + "theoretical_loss": 4.1667512103576865, + "tokens_seen": 305070080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333681043129389, + "loss": 1.6751, + "theoretical_loss": 4.166647201230386, + "tokens_seen": 305135616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333520561685056, + "loss": 1.6589, + "theoretical_loss": 4.166543220692731, + "tokens_seen": 305201152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333360080240723, + "loss": 1.5901, + "theoretical_loss": 4.166439268730724, + "tokens_seen": 305266688 + }, + { + "epoch": 0.09, + "learning_rate": 0.000733319959879639, + "loss": 1.653, + "theoretical_loss": 4.166335345330383, + "tokens_seen": 305332224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333039117352057, + "loss": 1.6598, + "theoretical_loss": 4.16623145047773, + "tokens_seen": 305397760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332878635907724, + "loss": 1.5896, + "theoretical_loss": 4.166127584158801, + "tokens_seen": 305463296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332718154463391, + "loss": 1.6481, + "theoretical_loss": 4.166023746359639, + "tokens_seen": 305528832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332557673019058, + "loss": 1.6581, + "theoretical_loss": 4.165919937066299, + "tokens_seen": 305594368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332397191574725, + "loss": 1.6001, + "theoretical_loss": 4.165816156264844, + "tokens_seen": 305659904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332236710130392, + "loss": 1.7288, + "theoretical_loss": 4.165712403941348, + "tokens_seen": 305725440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332076228686059, + "loss": 1.5835, + "theoretical_loss": 4.1656086800818946, + "tokens_seen": 305790976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331915747241726, + "loss": 1.6093, + "theoretical_loss": 4.165504984672577, + "tokens_seen": 305856512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331755265797393, + "loss": 1.4599, + "theoretical_loss": 4.165401317699498, + "tokens_seen": 305922048 + }, + { + "epoch": 0.09, + "learning_rate": 0.000733159478435306, + "loss": 1.6132, + "theoretical_loss": 4.16529767914877, + "tokens_seen": 305987584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331434302908727, + "loss": 1.5185, + "theoretical_loss": 4.165194069006516, + "tokens_seen": 306053120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331273821464394, + "loss": 1.6518, + "theoretical_loss": 4.165090487258867, + "tokens_seen": 306118656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331113340020061, + "loss": 1.5807, + "theoretical_loss": 4.164986933891968, + "tokens_seen": 306184192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330952858575728, + "loss": 1.6028, + "theoretical_loss": 4.164883408891968, + "tokens_seen": 306249728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330792377131395, + "loss": 1.6098, + "theoretical_loss": 4.164779912245029, + "tokens_seen": 306315264 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330631895687062, + "loss": 1.5847, + "theoretical_loss": 4.1646764439373225, + "tokens_seen": 306380800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330471414242729, + "loss": 1.5105, + "theoretical_loss": 4.164573003955029, + "tokens_seen": 306446336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330310932798396, + "loss": 1.5924, + "theoretical_loss": 4.164469592284338, + "tokens_seen": 306511872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330150451354062, + "loss": 1.7551, + "theoretical_loss": 4.164366208911453, + "tokens_seen": 306577408 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732998996990973, + "loss": 1.7007, + "theoretical_loss": 4.1642628538225805, + "tokens_seen": 306642944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329829488465396, + "loss": 1.6358, + "theoretical_loss": 4.1641595270039415, + "tokens_seen": 306708480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329669007021064, + "loss": 1.6819, + "theoretical_loss": 4.1640562284417655, + "tokens_seen": 306774016 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732950852557673, + "loss": 1.5444, + "theoretical_loss": 4.16395295812229, + "tokens_seen": 306839552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329348044132398, + "loss": 1.6675, + "theoretical_loss": 4.1638497160317645, + "tokens_seen": 306905088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329187562688064, + "loss": 1.6466, + "theoretical_loss": 4.163746502156448, + "tokens_seen": 306970624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329027081243732, + "loss": 1.4845, + "theoretical_loss": 4.163643316482606, + "tokens_seen": 307036160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328866599799398, + "loss": 1.6061, + "theoretical_loss": 4.1635401589965175, + "tokens_seen": 307101696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328706118355065, + "loss": 1.6429, + "theoretical_loss": 4.163437029684469, + "tokens_seen": 307167232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328545636910732, + "loss": 1.6264, + "theoretical_loss": 4.163333928532758, + "tokens_seen": 307232768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328385155466399, + "loss": 1.6924, + "theoretical_loss": 4.163230855527688, + "tokens_seen": 307298304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328224674022066, + "loss": 1.6224, + "theoretical_loss": 4.163127810655578, + "tokens_seen": 307363840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328064192577733, + "loss": 1.6408, + "theoretical_loss": 4.163024793902752, + "tokens_seen": 307429376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00073279037111334, + "loss": 1.7319, + "theoretical_loss": 4.162921805255543, + "tokens_seen": 307494912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327743229689067, + "loss": 1.501, + "theoretical_loss": 4.162818844700298, + "tokens_seen": 307560448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327582748244734, + "loss": 1.6282, + "theoretical_loss": 4.162715912223369, + "tokens_seen": 307625984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327422266800401, + "loss": 1.5315, + "theoretical_loss": 4.162613007811122, + "tokens_seen": 307691520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327261785356068, + "loss": 1.5266, + "theoretical_loss": 4.1625101314499275, + "tokens_seen": 307757056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327101303911736, + "loss": 1.5706, + "theoretical_loss": 4.162407283126169, + "tokens_seen": 307822592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007326940822467402, + "loss": 1.5297, + "theoretical_loss": 4.1623044628262384, + "tokens_seen": 307888128 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732678034102307, + "loss": 1.6028, + "theoretical_loss": 4.162201670536537, + "tokens_seen": 307953664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 108952, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5811710357666016, + "objective/train/theoretical_loss": 4.162098906243477, + "objective/train/tokens_used": 328479200, + "theoretical_loss": 4.162098906243477, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007326619859578736, + "loss": 1.6008, + "theoretical_loss": 4.162098906243477, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007326459378134404, + "loss": 1.5509, + "theoretical_loss": 4.161996169933477, + "tokens_seen": 308084736 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732629889669007, + "loss": 1.6285, + "theoretical_loss": 4.161893461592968, + "tokens_seen": 308150272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007326138415245738, + "loss": 1.6352, + "theoretical_loss": 4.16179078120839, + "tokens_seen": 308215808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325977933801404, + "loss": 1.5552, + "theoretical_loss": 4.161688128766191, + "tokens_seen": 308281344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325817452357072, + "loss": 1.5523, + "theoretical_loss": 4.16158550425283, + "tokens_seen": 308346880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325656970912738, + "loss": 1.6311, + "theoretical_loss": 4.161482907654775, + "tokens_seen": 308412416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325496489468406, + "loss": 1.6177, + "theoretical_loss": 4.161380338958502, + "tokens_seen": 308477952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325336008024072, + "loss": 1.6076, + "theoretical_loss": 4.161277798150498, + "tokens_seen": 308543488 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732517552657974, + "loss": 1.5029, + "theoretical_loss": 4.16117528521726, + "tokens_seen": 308609024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325015045135406, + "loss": 1.6536, + "theoretical_loss": 4.161072800145292, + "tokens_seen": 308674560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324854563691074, + "loss": 1.5278, + "theoretical_loss": 4.16097034292111, + "tokens_seen": 308740096 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732469408224674, + "loss": 1.4431, + "theoretical_loss": 4.160867913531238, + "tokens_seen": 308805632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324533600802408, + "loss": 1.6229, + "theoretical_loss": 4.160765511962209, + "tokens_seen": 308871168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324373119358074, + "loss": 1.5469, + "theoretical_loss": 4.160663138200567, + "tokens_seen": 308936704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324212637913742, + "loss": 1.6081, + "theoretical_loss": 4.1605607922328645, + "tokens_seen": 309002240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324052156469408, + "loss": 1.6369, + "theoretical_loss": 4.160458474045662, + "tokens_seen": 309067776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007323891675025076, + "loss": 1.736, + "theoretical_loss": 4.16035618362553, + "tokens_seen": 309133312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007323731193580742, + "loss": 1.4865, + "theoretical_loss": 4.16025392095905, + "tokens_seen": 309198848 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732357071213641, + "loss": 1.6449, + "theoretical_loss": 4.160151686032814, + "tokens_seen": 309264384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007323410230692076, + "loss": 1.622, + "theoretical_loss": 4.160049478833416, + "tokens_seen": 309329920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007323249749247744, + "loss": 1.681, + "theoretical_loss": 4.159947299347468, + "tokens_seen": 309395456 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732308926780341, + "loss": 1.5762, + "theoretical_loss": 4.159845147561587, + "tokens_seen": 309460992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322928786359078, + "loss": 1.5544, + "theoretical_loss": 4.1597430234624, + "tokens_seen": 309526528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322768304914744, + "loss": 1.4294, + "theoretical_loss": 4.159640927036541, + "tokens_seen": 309592064 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322607823470412, + "loss": 1.5333, + "theoretical_loss": 4.159538858270659, + "tokens_seen": 309657600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322447342026078, + "loss": 1.5659, + "theoretical_loss": 4.159436817151407, + "tokens_seen": 309723136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322286860581746, + "loss": 1.6312, + "theoretical_loss": 4.159334803665448, + "tokens_seen": 309788672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322126379137412, + "loss": 1.5842, + "theoretical_loss": 4.159232817799458, + "tokens_seen": 309854208 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732196589769308, + "loss": 1.5808, + "theoretical_loss": 4.1591308595401175, + "tokens_seen": 309919744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321805416248746, + "loss": 1.5817, + "theoretical_loss": 4.159028928874118, + "tokens_seen": 309985280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321644934804414, + "loss": 1.6191, + "theoretical_loss": 4.158927025788162, + "tokens_seen": 310050816 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732148445336008, + "loss": 1.624, + "theoretical_loss": 4.158825150268958, + "tokens_seen": 310116352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321323971915748, + "loss": 1.5717, + "theoretical_loss": 4.158723302303227, + "tokens_seen": 310181888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321163490471414, + "loss": 1.6036, + "theoretical_loss": 4.158621481877697, + "tokens_seen": 310247424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321003009027082, + "loss": 1.6511, + "theoretical_loss": 4.158519688979106, + "tokens_seen": 310312960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320842527582748, + "loss": 1.5586, + "theoretical_loss": 4.1584179235942, + "tokens_seen": 310378496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320682046138416, + "loss": 1.6026, + "theoretical_loss": 4.158316185709737, + "tokens_seen": 310444032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320521564694082, + "loss": 1.7038, + "theoretical_loss": 4.158214475312481, + "tokens_seen": 310509568 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732036108324975, + "loss": 1.7325, + "theoretical_loss": 4.158112792389206, + "tokens_seen": 310575104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320200601805416, + "loss": 1.6455, + "theoretical_loss": 4.158011136926698, + "tokens_seen": 310640640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320040120361084, + "loss": 1.5333, + "theoretical_loss": 4.157909508911747, + "tokens_seen": 310706176 + }, + { + "epoch": 0.09, + "learning_rate": 0.000731987963891675, + "loss": 1.663, + "theoretical_loss": 4.157807908331157, + "tokens_seen": 310771712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319719157472418, + "loss": 1.6373, + "theoretical_loss": 4.157706335171738, + "tokens_seen": 310837248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319558676028085, + "loss": 1.6038, + "theoretical_loss": 4.15760478942031, + "tokens_seen": 310902784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319398194583752, + "loss": 1.6834, + "theoretical_loss": 4.157503271063703, + "tokens_seen": 310968320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319237713139419, + "loss": 1.529, + "theoretical_loss": 4.157401780088756, + "tokens_seen": 311033856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319077231695086, + "loss": 1.671, + "theoretical_loss": 4.157300316482315, + "tokens_seen": 311099392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318916750250753, + "loss": 1.6511, + "theoretical_loss": 4.157198880231238, + "tokens_seen": 311164928 + }, + { + "epoch": 0.09, + "learning_rate": 0.000731875626880642, + "loss": 1.5657, + "theoretical_loss": 4.157097471322389, + "tokens_seen": 311230464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 110010, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5765893459320068, + "objective/train/theoretical_loss": 4.156996089742645, + "objective/train/tokens_used": 331756000, + "theoretical_loss": 4.156996089742645, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318595787362087, + "loss": 1.56, + "theoretical_loss": 4.156996089742645, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318435305917754, + "loss": 1.7442, + "theoretical_loss": 4.156894735478888, + "tokens_seen": 311361536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318274824473421, + "loss": 1.5361, + "theoretical_loss": 4.156793408518011, + "tokens_seen": 311427072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318114343029088, + "loss": 1.6328, + "theoretical_loss": 4.156692108846916, + "tokens_seen": 311492608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317953861584755, + "loss": 1.6673, + "theoretical_loss": 4.156590836452514, + "tokens_seen": 311558144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317793380140422, + "loss": 1.7033, + "theoretical_loss": 4.156489591321726, + "tokens_seen": 311623680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317632898696089, + "loss": 1.4632, + "theoretical_loss": 4.15638837344148, + "tokens_seen": 311689216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317472417251756, + "loss": 1.6694, + "theoretical_loss": 4.156287182798714, + "tokens_seen": 311754752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317311935807423, + "loss": 1.552, + "theoretical_loss": 4.156186019380375, + "tokens_seen": 311820288 + }, + { + "epoch": 0.09, + "learning_rate": 0.000731715145436309, + "loss": 1.6538, + "theoretical_loss": 4.156084883173419, + "tokens_seen": 311885824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316990972918757, + "loss": 1.5271, + "theoretical_loss": 4.155983774164811, + "tokens_seen": 311951360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316830491474424, + "loss": 1.4608, + "theoretical_loss": 4.1558826923415255, + "tokens_seen": 312016896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316670010030091, + "loss": 1.6252, + "theoretical_loss": 4.155781637690545, + "tokens_seen": 312082432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316509528585758, + "loss": 1.5328, + "theoretical_loss": 4.1556806101988615, + "tokens_seen": 312147968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316349047141425, + "loss": 1.5291, + "theoretical_loss": 4.155579609853476, + "tokens_seen": 312213504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316188565697092, + "loss": 1.5796, + "theoretical_loss": 4.155478636641398, + "tokens_seen": 312279040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316028084252759, + "loss": 1.7283, + "theoretical_loss": 4.155377690549646, + "tokens_seen": 312344576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315867602808425, + "loss": 1.5384, + "theoretical_loss": 4.155276771565248, + "tokens_seen": 312410112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315707121364093, + "loss": 1.6876, + "theoretical_loss": 4.155175879675241, + "tokens_seen": 312475648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315546639919759, + "loss": 1.5711, + "theoretical_loss": 4.155075014866671, + "tokens_seen": 312541184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315386158475427, + "loss": 1.6038, + "theoretical_loss": 4.15497417712659, + "tokens_seen": 312606720 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315225677031093, + "loss": 1.6181, + "theoretical_loss": 4.1548733664420645, + "tokens_seen": 312672256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315065195586761, + "loss": 1.6023, + "theoretical_loss": 4.154772582800165, + "tokens_seen": 312737792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314904714142427, + "loss": 1.6421, + "theoretical_loss": 4.154671826187972, + "tokens_seen": 312803328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314744232698095, + "loss": 1.6457, + "theoretical_loss": 4.154571096592576, + "tokens_seen": 312868864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314583751253761, + "loss": 1.6198, + "theoretical_loss": 4.154470394001077, + "tokens_seen": 312934400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314423269809429, + "loss": 1.5943, + "theoretical_loss": 4.154369718400583, + "tokens_seen": 312999936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314262788365095, + "loss": 1.5763, + "theoretical_loss": 4.154269069778207, + "tokens_seen": 313065472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314102306920763, + "loss": 1.6197, + "theoretical_loss": 4.154168448121078, + "tokens_seen": 313131008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313941825476429, + "loss": 1.6308, + "theoretical_loss": 4.15406785341633, + "tokens_seen": 313196544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313781344032097, + "loss": 1.488, + "theoretical_loss": 4.153967285651105, + "tokens_seen": 313262080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313620862587763, + "loss": 1.7241, + "theoretical_loss": 4.153866744812555, + "tokens_seen": 313327616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313460381143431, + "loss": 1.488, + "theoretical_loss": 4.1537662308878405, + "tokens_seen": 313393152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313299899699097, + "loss": 1.685, + "theoretical_loss": 4.153665743864131, + "tokens_seen": 313458688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007313139418254765, + "loss": 1.5907, + "theoretical_loss": 4.153565283728606, + "tokens_seen": 313524224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312978936810431, + "loss": 1.6424, + "theoretical_loss": 4.153464850468452, + "tokens_seen": 313589760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312818455366099, + "loss": 1.6169, + "theoretical_loss": 4.1533644440708635, + "tokens_seen": 313655296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312657973921765, + "loss": 1.6373, + "theoretical_loss": 4.153264064523047, + "tokens_seen": 313720832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312497492477433, + "loss": 1.4888, + "theoretical_loss": 4.1531637118122156, + "tokens_seen": 313786368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312337011033099, + "loss": 1.6178, + "theoretical_loss": 4.153063385925591, + "tokens_seen": 313851904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312176529588767, + "loss": 1.6062, + "theoretical_loss": 4.152963086850405, + "tokens_seen": 313917440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312016048144433, + "loss": 1.7214, + "theoretical_loss": 4.152862814573895, + "tokens_seen": 313982976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311855566700101, + "loss": 1.5936, + "theoretical_loss": 4.152762569083313, + "tokens_seen": 314048512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311695085255767, + "loss": 1.6886, + "theoretical_loss": 4.1526623503659135, + "tokens_seen": 314114048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311534603811435, + "loss": 1.543, + "theoretical_loss": 4.152562158408962, + "tokens_seen": 314179584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311374122367101, + "loss": 1.5865, + "theoretical_loss": 4.1524619931997355, + "tokens_seen": 314245120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311213640922769, + "loss": 1.5747, + "theoretical_loss": 4.152361854725515, + "tokens_seen": 314310656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311053159478435, + "loss": 1.6538, + "theoretical_loss": 4.152261742973594, + "tokens_seen": 314376192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310892678034103, + "loss": 1.6593, + "theoretical_loss": 4.152161657931273, + "tokens_seen": 314441728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310732196589769, + "loss": 1.6103, + "theoretical_loss": 4.15206159958586, + "tokens_seen": 314507264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 111213, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5043652057647705, + "objective/train/theoretical_loss": 4.1519615679246735, + "objective/train/tokens_used": 335032800, + "theoretical_loss": 4.1519615679246735, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310571715145437, + "loss": 1.5245, + "theoretical_loss": 4.1519615679246735, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310411233701103, + "loss": 1.5779, + "theoretical_loss": 4.151861562935041, + "tokens_seen": 314638336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310250752256771, + "loss": 1.5589, + "theoretical_loss": 4.151761584604298, + "tokens_seen": 314703872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310090270812437, + "loss": 1.5793, + "theoretical_loss": 4.1516616329197875, + "tokens_seen": 314769408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309929789368105, + "loss": 1.6619, + "theoretical_loss": 4.151561707868863, + "tokens_seen": 314834944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309769307923771, + "loss": 1.6171, + "theoretical_loss": 4.151461809438885, + "tokens_seen": 314900480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309608826479439, + "loss": 1.5683, + "theoretical_loss": 4.1513619376172235, + "tokens_seen": 314966016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309448345035105, + "loss": 1.6547, + "theoretical_loss": 4.151262092391257, + "tokens_seen": 315031552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309287863590773, + "loss": 1.6485, + "theoretical_loss": 4.1511622737483735, + "tokens_seen": 315097088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309127382146439, + "loss": 1.5209, + "theoretical_loss": 4.151062481675968, + "tokens_seen": 315162624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308966900702107, + "loss": 1.5688, + "theoretical_loss": 4.150962716161445, + "tokens_seen": 315228160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308806419257773, + "loss": 1.5452, + "theoretical_loss": 4.150862977192217, + "tokens_seen": 315293696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308645937813441, + "loss": 1.555, + "theoretical_loss": 4.150763264755706, + "tokens_seen": 315359232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308485456369107, + "loss": 1.7569, + "theoretical_loss": 4.150663578839342, + "tokens_seen": 315424768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308324974924775, + "loss": 1.598, + "theoretical_loss": 4.150563919430562, + "tokens_seen": 315490304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308164493480441, + "loss": 1.4641, + "theoretical_loss": 4.150464286516816, + "tokens_seen": 315555840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308004012036109, + "loss": 1.5865, + "theoretical_loss": 4.150364680085558, + "tokens_seen": 315621376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307843530591775, + "loss": 1.5584, + "theoretical_loss": 4.150265100124253, + "tokens_seen": 315686912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307683049147443, + "loss": 1.6787, + "theoretical_loss": 4.150165546620373, + "tokens_seen": 315752448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307522567703109, + "loss": 1.7157, + "theoretical_loss": 4.1500660195614, + "tokens_seen": 315817984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307362086258777, + "loss": 1.5774, + "theoretical_loss": 4.149966518934824, + "tokens_seen": 315883520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307201604814443, + "loss": 1.5534, + "theoretical_loss": 4.149867044728143, + "tokens_seen": 315949056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307041123370111, + "loss": 1.597, + "theoretical_loss": 4.149767596928863, + "tokens_seen": 316014592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306880641925777, + "loss": 1.8193, + "theoretical_loss": 4.149668175524502, + "tokens_seen": 316080128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306720160481445, + "loss": 1.5885, + "theoretical_loss": 4.14956878050258, + "tokens_seen": 316145664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306559679037111, + "loss": 1.6594, + "theoretical_loss": 4.1494694118506334, + "tokens_seen": 316211200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306399197592779, + "loss": 1.6892, + "theoretical_loss": 4.1493700695562, + "tokens_seen": 316276736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306238716148446, + "loss": 1.5594, + "theoretical_loss": 4.14927075360683, + "tokens_seen": 316342272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306078234704113, + "loss": 1.6457, + "theoretical_loss": 4.149171463990082, + "tokens_seen": 316407808 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730591775325978, + "loss": 1.5986, + "theoretical_loss": 4.149072200693521, + "tokens_seen": 316473344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305757271815447, + "loss": 1.6219, + "theoretical_loss": 4.148972963704722, + "tokens_seen": 316538880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305596790371114, + "loss": 1.6861, + "theoretical_loss": 4.148873753011269, + "tokens_seen": 316604416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305436308926781, + "loss": 1.6092, + "theoretical_loss": 4.148774568600752, + "tokens_seen": 316669952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305275827482448, + "loss": 1.5524, + "theoretical_loss": 4.148675410460772, + "tokens_seen": 316735488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305115346038115, + "loss": 1.5586, + "theoretical_loss": 4.148576278578937, + "tokens_seen": 316801024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304954864593782, + "loss": 1.5904, + "theoretical_loss": 4.148477172942863, + "tokens_seen": 316866560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304794383149449, + "loss": 1.745, + "theoretical_loss": 4.1483780935401775, + "tokens_seen": 316932096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304633901705116, + "loss": 1.5839, + "theoretical_loss": 4.148279040358511, + "tokens_seen": 316997632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304473420260783, + "loss": 1.5903, + "theoretical_loss": 4.148180013385507, + "tokens_seen": 317063168 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730431293881645, + "loss": 1.7813, + "theoretical_loss": 4.148081012608817, + "tokens_seen": 317128704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304152457372117, + "loss": 1.8107, + "theoretical_loss": 4.147982038016096, + "tokens_seen": 317194240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303991975927784, + "loss": 1.7651, + "theoretical_loss": 4.147883089595015, + "tokens_seen": 317259776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303831494483451, + "loss": 1.8327, + "theoretical_loss": 4.147784167333247, + "tokens_seen": 317325312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303671013039118, + "loss": 1.8399, + "theoretical_loss": 4.147685271218477, + "tokens_seen": 317390848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303510531594785, + "loss": 1.8931, + "theoretical_loss": 4.147586401238396, + "tokens_seen": 317456384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303350050150452, + "loss": 1.8639, + "theoretical_loss": 4.147487557380705, + "tokens_seen": 317521920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303189568706119, + "loss": 1.8167, + "theoretical_loss": 4.147388739633113, + "tokens_seen": 317587456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303029087261786, + "loss": 1.86, + "theoretical_loss": 4.147289947983337, + "tokens_seen": 317652992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302868605817453, + "loss": 1.9685, + "theoretical_loss": 4.1471911824191015, + "tokens_seen": 317718528 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730270812437312, + "loss": 1.7142, + "theoretical_loss": 4.147092442928141, + "tokens_seen": 317784064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 112065, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 2.024470567703247, + "objective/train/theoretical_loss": 4.146993729498197, + "objective/train/tokens_used": 338309600, + "theoretical_loss": 4.146993729498197, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302547642928787, + "loss": 1.9022, + "theoretical_loss": 4.146993729498197, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302387161484454, + "loss": 1.9061, + "theoretical_loss": 4.146895042117021, + "tokens_seen": 317915136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302226680040121, + "loss": 1.9165, + "theoretical_loss": 4.146796380772369, + "tokens_seen": 317980672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302066198595788, + "loss": 1.9143, + "theoretical_loss": 4.146697745452011, + "tokens_seen": 318046208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301905717151454, + "loss": 1.8095, + "theoretical_loss": 4.146599136143719, + "tokens_seen": 318111744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301745235707122, + "loss": 1.7185, + "theoretical_loss": 4.146500552835278, + "tokens_seen": 318177280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301584754262788, + "loss": 1.8265, + "theoretical_loss": 4.146401995514479, + "tokens_seen": 318242816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301424272818456, + "loss": 1.8853, + "theoretical_loss": 4.146303464169123, + "tokens_seen": 318308352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301263791374122, + "loss": 1.8499, + "theoretical_loss": 4.146204958787017, + "tokens_seen": 318373888 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730110330992979, + "loss": 1.6803, + "theoretical_loss": 4.146106479355978, + "tokens_seen": 318439424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300942828485456, + "loss": 1.8647, + "theoretical_loss": 4.1460080258638285, + "tokens_seen": 318504960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300782347041124, + "loss": 1.8665, + "theoretical_loss": 4.145909598298404, + "tokens_seen": 318570496 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730062186559679, + "loss": 1.8231, + "theoretical_loss": 4.145811196647544, + "tokens_seen": 318636032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300461384152458, + "loss": 1.8109, + "theoretical_loss": 4.145712820899098, + "tokens_seen": 318701568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300300902708124, + "loss": 1.7429, + "theoretical_loss": 4.145614471040923, + "tokens_seen": 318767104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300140421263792, + "loss": 1.8226, + "theoretical_loss": 4.145516147060884, + "tokens_seen": 318832640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299979939819458, + "loss": 1.701, + "theoretical_loss": 4.145417848946857, + "tokens_seen": 318898176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299819458375126, + "loss": 1.8668, + "theoretical_loss": 4.145319576686721, + "tokens_seen": 318963712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299658976930792, + "loss": 1.6904, + "theoretical_loss": 4.1452213302683685, + "tokens_seen": 319029248 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729949849548646, + "loss": 1.9191, + "theoretical_loss": 4.145123109679696, + "tokens_seen": 319094784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299338014042126, + "loss": 1.9909, + "theoretical_loss": 4.1450249149086105, + "tokens_seen": 319160320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299177532597794, + "loss": 1.9859, + "theoretical_loss": 4.144926745943026, + "tokens_seen": 319225856 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729901705115346, + "loss": 1.8796, + "theoretical_loss": 4.1448286027708665, + "tokens_seen": 319291392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298856569709128, + "loss": 1.7589, + "theoretical_loss": 4.144730485380062, + "tokens_seen": 319356928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298696088264794, + "loss": 1.881, + "theoretical_loss": 4.144632393758551, + "tokens_seen": 319422464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298535606820462, + "loss": 1.9404, + "theoretical_loss": 4.144534327894281, + "tokens_seen": 319488000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298375125376128, + "loss": 1.8255, + "theoretical_loss": 4.144436287775206, + "tokens_seen": 319553536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298214643931796, + "loss": 1.8244, + "theoretical_loss": 4.144338273389291, + "tokens_seen": 319619072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298054162487462, + "loss": 1.7481, + "theoretical_loss": 4.144240284724505, + "tokens_seen": 319684608 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729789368104313, + "loss": 1.8876, + "theoretical_loss": 4.144142321768831, + "tokens_seen": 319750144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007297733199598796, + "loss": 1.7792, + "theoretical_loss": 4.144044384510253, + "tokens_seen": 319815680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007297572718154464, + "loss": 1.7007, + "theoretical_loss": 4.1439464729367685, + "tokens_seen": 319881216 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729741223671013, + "loss": 1.5911, + "theoretical_loss": 4.14384858703638, + "tokens_seen": 319946752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007297251755265798, + "loss": 1.7342, + "theoretical_loss": 4.1437507267971, + "tokens_seen": 320012288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007297091273821464, + "loss": 1.7468, + "theoretical_loss": 4.143652892206948, + "tokens_seen": 320077824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296930792377132, + "loss": 1.7422, + "theoretical_loss": 4.1435550832539505, + "tokens_seen": 320143360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296770310932798, + "loss": 1.6767, + "theoretical_loss": 4.143457299926146, + "tokens_seen": 320208896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296609829488466, + "loss": 1.7012, + "theoretical_loss": 4.143359542211576, + "tokens_seen": 320274432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296449348044132, + "loss": 1.6575, + "theoretical_loss": 4.143261810098293, + "tokens_seen": 320339968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00072962888665998, + "loss": 1.8258, + "theoretical_loss": 4.1431641035743585, + "tokens_seen": 320405504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296128385155466, + "loss": 1.7058, + "theoretical_loss": 4.143066422627838, + "tokens_seen": 320471040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295967903711134, + "loss": 1.7022, + "theoretical_loss": 4.142968767246808, + "tokens_seen": 320536576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00072958074222668, + "loss": 1.658, + "theoretical_loss": 4.142871137419354, + "tokens_seen": 320602112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295646940822468, + "loss": 1.7651, + "theoretical_loss": 4.142773533133565, + "tokens_seen": 320667648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295486459378134, + "loss": 1.7998, + "theoretical_loss": 4.142675954377543, + "tokens_seen": 320733184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295325977933802, + "loss": 1.8644, + "theoretical_loss": 4.142578401139396, + "tokens_seen": 320798720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295165496489468, + "loss": 1.778, + "theoretical_loss": 4.142480873407239, + "tokens_seen": 320864256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295005015045136, + "loss": 1.7293, + "theoretical_loss": 4.1423833711691955, + "tokens_seen": 320929792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007294844533600802, + "loss": 1.6623, + "theoretical_loss": 4.142285894413398, + "tokens_seen": 320995328 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729468405215647, + "loss": 1.768, + "theoretical_loss": 4.142188443127985, + "tokens_seen": 321060864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 113306, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5380922555923462, + "objective/train/theoretical_loss": 4.142091017301105, + "objective/train/tokens_used": 341586400, + "theoretical_loss": 4.142091017301105, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007294523570712136, + "loss": 1.6093, + "theoretical_loss": 4.142091017301105, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007294363089267804, + "loss": 1.6369, + "theoretical_loss": 4.141993616920914, + "tokens_seen": 321191936 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729420260782347, + "loss": 1.71, + "theoretical_loss": 4.141896241975575, + "tokens_seen": 321257472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007294042126379138, + "loss": 1.4959, + "theoretical_loss": 4.141798892453259, + "tokens_seen": 321323008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293881644934804, + "loss": 1.7129, + "theoretical_loss": 4.141701568342145, + "tokens_seen": 321388544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293721163490472, + "loss": 1.6549, + "theoretical_loss": 4.141604269630422, + "tokens_seen": 321454080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293560682046138, + "loss": 1.5904, + "theoretical_loss": 4.1415069963062825, + "tokens_seen": 321519616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293400200601806, + "loss": 1.6822, + "theoretical_loss": 4.1414097483579315, + "tokens_seen": 321585152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293239719157472, + "loss": 1.6719, + "theoretical_loss": 4.1413125257735794, + "tokens_seen": 321650688 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729307923771314, + "loss": 1.5757, + "theoretical_loss": 4.141215328541445, + "tokens_seen": 321716224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292918756268806, + "loss": 1.5815, + "theoretical_loss": 4.141118156649753, + "tokens_seen": 321781760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292758274824474, + "loss": 1.6802, + "theoretical_loss": 4.1410210100867415, + "tokens_seen": 321847296 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729259779338014, + "loss": 1.6105, + "theoretical_loss": 4.14092388884065, + "tokens_seen": 321912832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292437311935808, + "loss": 1.5999, + "theoretical_loss": 4.1408267928997295, + "tokens_seen": 321978368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292276830491475, + "loss": 1.6539, + "theoretical_loss": 4.140729722252239, + "tokens_seen": 322043904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292116349047142, + "loss": 1.6241, + "theoretical_loss": 4.140632676886442, + "tokens_seen": 322109440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291955867602809, + "loss": 1.6331, + "theoretical_loss": 4.140535656790615, + "tokens_seen": 322174976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291795386158476, + "loss": 1.6988, + "theoretical_loss": 4.140438661953038, + "tokens_seen": 322240512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291634904714143, + "loss": 1.4855, + "theoretical_loss": 4.140341692362, + "tokens_seen": 322306048 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729147442326981, + "loss": 1.682, + "theoretical_loss": 4.140244748005799, + "tokens_seen": 322371584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291313941825477, + "loss": 1.7468, + "theoretical_loss": 4.14014782887274, + "tokens_seen": 322437120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291153460381144, + "loss": 1.6284, + "theoretical_loss": 4.140050934951135, + "tokens_seen": 322502656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290992978936812, + "loss": 1.6126, + "theoretical_loss": 4.139954066229304, + "tokens_seen": 322568192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290832497492478, + "loss": 1.6776, + "theoretical_loss": 4.139857222695578, + "tokens_seen": 322633728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290672016048146, + "loss": 1.6517, + "theoretical_loss": 4.139760404338291, + "tokens_seen": 322699264 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290511534603812, + "loss": 1.6009, + "theoretical_loss": 4.139663611145786, + "tokens_seen": 322764800 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729035105315948, + "loss": 1.6975, + "theoretical_loss": 4.139566843106417, + "tokens_seen": 322830336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290190571715146, + "loss": 1.6493, + "theoretical_loss": 4.139470100208542, + "tokens_seen": 322895872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290030090270814, + "loss": 1.7209, + "theoretical_loss": 4.139373382440528, + "tokens_seen": 322961408 + }, + { + "epoch": 0.1, + "learning_rate": 0.000728986960882648, + "loss": 1.6022, + "theoretical_loss": 4.1392766897907505, + "tokens_seen": 323026944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289709127382148, + "loss": 1.6735, + "theoretical_loss": 4.139180022247592, + "tokens_seen": 323092480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289548645937814, + "loss": 1.5358, + "theoretical_loss": 4.139083379799443, + "tokens_seen": 323158016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289388164493481, + "loss": 1.5424, + "theoretical_loss": 4.1389867624347, + "tokens_seen": 323223552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289227683049148, + "loss": 1.6485, + "theoretical_loss": 4.138890170141771, + "tokens_seen": 323289088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289067201604815, + "loss": 1.5949, + "theoretical_loss": 4.138793602909068, + "tokens_seen": 323354624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288906720160482, + "loss": 1.7056, + "theoretical_loss": 4.138697060725013, + "tokens_seen": 323420160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288746238716149, + "loss": 1.6545, + "theoretical_loss": 4.138600543578034, + "tokens_seen": 323485696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288585757271816, + "loss": 1.7005, + "theoretical_loss": 4.138504051456568, + "tokens_seen": 323551232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288425275827483, + "loss": 1.6943, + "theoretical_loss": 4.13840758434906, + "tokens_seen": 323616768 + }, + { + "epoch": 0.1, + "learning_rate": 0.000728826479438315, + "loss": 1.5391, + "theoretical_loss": 4.1383111422439605, + "tokens_seen": 323682304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288104312938817, + "loss": 1.6891, + "theoretical_loss": 4.13821472512973, + "tokens_seen": 323747840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287943831494483, + "loss": 1.6866, + "theoretical_loss": 4.138118332994837, + "tokens_seen": 323813376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287783350050151, + "loss": 1.69, + "theoretical_loss": 4.138021965827753, + "tokens_seen": 323878912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287622868605817, + "loss": 1.5937, + "theoretical_loss": 4.1379256236169635, + "tokens_seen": 323944448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287462387161485, + "loss": 1.6, + "theoretical_loss": 4.137829306350957, + "tokens_seen": 324009984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287301905717151, + "loss": 1.6746, + "theoretical_loss": 4.137733014018233, + "tokens_seen": 324075520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287141424272819, + "loss": 1.6548, + "theoretical_loss": 4.137636746607295, + "tokens_seen": 324141056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286980942828485, + "loss": 1.5518, + "theoretical_loss": 4.137540504106657, + "tokens_seen": 324206592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286820461384153, + "loss": 1.7232, + "theoretical_loss": 4.137444286504841, + "tokens_seen": 324272128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286659979939819, + "loss": 1.6377, + "theoretical_loss": 4.137348093790372, + "tokens_seen": 324337664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 114269, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6439099311828613, + "objective/train/theoretical_loss": 4.13725192595179, + "objective/train/tokens_used": 344863200, + "theoretical_loss": 4.13725192595179, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286499498495487, + "loss": 1.5966, + "theoretical_loss": 4.13725192595179, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286339017051153, + "loss": 1.7042, + "theoretical_loss": 4.1371557829776355, + "tokens_seen": 324468736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286178535606821, + "loss": 1.6878, + "theoretical_loss": 4.13705966485646, + "tokens_seen": 324534272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286018054162487, + "loss": 1.641, + "theoretical_loss": 4.136963571576825, + "tokens_seen": 324599808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285857572718155, + "loss": 1.6288, + "theoretical_loss": 4.136867503127292, + "tokens_seen": 324665344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285697091273821, + "loss": 1.5733, + "theoretical_loss": 4.136771459496439, + "tokens_seen": 324730880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285536609829489, + "loss": 1.5923, + "theoretical_loss": 4.136675440672844, + "tokens_seen": 324796416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285376128385155, + "loss": 1.7097, + "theoretical_loss": 4.136579446645098, + "tokens_seen": 324861952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285215646940823, + "loss": 1.6223, + "theoretical_loss": 4.136483477401798, + "tokens_seen": 324927488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285055165496489, + "loss": 1.6171, + "theoretical_loss": 4.136387532931546, + "tokens_seen": 324993024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284894684052157, + "loss": 1.555, + "theoretical_loss": 4.136291613222955, + "tokens_seen": 325058560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284734202607823, + "loss": 1.5637, + "theoretical_loss": 4.136195718264644, + "tokens_seen": 325124096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284573721163491, + "loss": 1.5865, + "theoretical_loss": 4.136099848045239, + "tokens_seen": 325189632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284413239719157, + "loss": 1.695, + "theoretical_loss": 4.136004002553375, + "tokens_seen": 325255168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284252758274825, + "loss": 1.5485, + "theoretical_loss": 4.135908181777693, + "tokens_seen": 325320704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284092276830491, + "loss": 1.6128, + "theoretical_loss": 4.135812385706842, + "tokens_seen": 325386240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283931795386159, + "loss": 1.7098, + "theoretical_loss": 4.135716614329479, + "tokens_seen": 325451776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283771313941825, + "loss": 1.7113, + "theoretical_loss": 4.135620867634269, + "tokens_seen": 325517312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283610832497493, + "loss": 1.6371, + "theoretical_loss": 4.135525145609881, + "tokens_seen": 325582848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283450351053159, + "loss": 1.7067, + "theoretical_loss": 4.135429448244997, + "tokens_seen": 325648384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283289869608827, + "loss": 1.609, + "theoretical_loss": 4.135333775528302, + "tokens_seen": 325713920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283129388164493, + "loss": 1.4926, + "theoretical_loss": 4.13523812744849, + "tokens_seen": 325779456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282968906720161, + "loss": 1.7475, + "theoretical_loss": 4.135142503994263, + "tokens_seen": 325844992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282808425275827, + "loss": 1.539, + "theoretical_loss": 4.13504690515433, + "tokens_seen": 325910528 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282647943831495, + "loss": 1.6961, + "theoretical_loss": 4.134951330917408, + "tokens_seen": 325976064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282487462387161, + "loss": 1.5638, + "theoretical_loss": 4.134855781272218, + "tokens_seen": 326041600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282326980942829, + "loss": 1.6864, + "theoretical_loss": 4.134760256207495, + "tokens_seen": 326107136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282166499498495, + "loss": 1.4862, + "theoretical_loss": 4.1346647557119764, + "tokens_seen": 326172672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282006018054163, + "loss": 1.6338, + "theoretical_loss": 4.134569279774407, + "tokens_seen": 326238208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281845536609829, + "loss": 1.6086, + "theoretical_loss": 4.134473828383541, + "tokens_seen": 326303744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281685055165497, + "loss": 1.4595, + "theoretical_loss": 4.13437840152814, + "tokens_seen": 326369280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281524573721163, + "loss": 1.6468, + "theoretical_loss": 4.134282999196973, + "tokens_seen": 326434816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281364092276831, + "loss": 1.6356, + "theoretical_loss": 4.134187621378813, + "tokens_seen": 326500352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281203610832497, + "loss": 1.546, + "theoretical_loss": 4.134092268062446, + "tokens_seen": 326565888 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281043129388165, + "loss": 1.4938, + "theoretical_loss": 4.133996939236661, + "tokens_seen": 326631424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280882647943831, + "loss": 1.6343, + "theoretical_loss": 4.133901634890256, + "tokens_seen": 326696960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280722166499499, + "loss": 1.5978, + "theoretical_loss": 4.1338063550120365, + "tokens_seen": 326762496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280561685055165, + "loss": 1.7451, + "theoretical_loss": 4.133711099590815, + "tokens_seen": 326828032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280401203610833, + "loss": 1.6231, + "theoretical_loss": 4.133615868615411, + "tokens_seen": 326893568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280240722166499, + "loss": 1.6108, + "theoretical_loss": 4.133520662074654, + "tokens_seen": 326959104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280080240722167, + "loss": 1.6044, + "theoretical_loss": 4.133425479957375, + "tokens_seen": 327024640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279919759277833, + "loss": 1.6349, + "theoretical_loss": 4.133330322252419, + "tokens_seen": 327090176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279759277833501, + "loss": 1.6286, + "theoretical_loss": 4.1332351889486345, + "tokens_seen": 327155712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279598796389167, + "loss": 1.4971, + "theoretical_loss": 4.133140080034878, + "tokens_seen": 327221248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279438314944835, + "loss": 1.5259, + "theoretical_loss": 4.133044995500013, + "tokens_seen": 327286784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279277833500503, + "loss": 1.6473, + "theoretical_loss": 4.1329499353329116, + "tokens_seen": 327352320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279117352056169, + "loss": 1.5083, + "theoretical_loss": 4.132854899522453, + "tokens_seen": 327417856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278956870611837, + "loss": 1.7212, + "theoretical_loss": 4.132759888057521, + "tokens_seen": 327483392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278796389167503, + "loss": 1.6026, + "theoretical_loss": 4.13266490092701, + "tokens_seen": 327548928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278635907723171, + "loss": 1.5403, + "theoretical_loss": 4.132569938119821, + "tokens_seen": 327614464 + }, + { + "debugging/Compilability": 0.0, + "debugging/distinct-1-grams": 0.7371483758339369, + "debugging/entropy-1-grams": 5.361149495192262, + "debugging/length": 581.1666666666666, + "debugging/num_segments": 12, + "debugging/score": 0.06659683474910605, + "debugging/score_std": 0.015715120837281396, + "epoch": 0.1, + "objective/train/docs_used": 115553, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6277862787246704, + "objective/train/theoretical_loss": 4.132474999624861, + "objective/train/tokens_used": 348140000, + "theoretical_loss": 4.132474999624861, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278475426278837, + "loss": 1.6737, + "theoretical_loss": 4.132474999624861, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278314944834505, + "loss": 1.5806, + "theoretical_loss": 4.132380085431046, + "tokens_seen": 327745536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278154463390171, + "loss": 1.641, + "theoretical_loss": 4.132285195527297, + "tokens_seen": 327811072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277993981945839, + "loss": 1.6191, + "theoretical_loss": 4.132190329902544, + "tokens_seen": 327876608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277833500501505, + "loss": 1.5115, + "theoretical_loss": 4.132095488545724, + "tokens_seen": 327942144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277673019057173, + "loss": 1.6416, + "theoretical_loss": 4.132000671445781, + "tokens_seen": 328007680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277512537612839, + "loss": 1.5818, + "theoretical_loss": 4.131905878591667, + "tokens_seen": 328073216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277352056168507, + "loss": 1.5892, + "theoretical_loss": 4.13181110997234, + "tokens_seen": 328138752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277191574724173, + "loss": 1.6217, + "theoretical_loss": 4.131716365576765, + "tokens_seen": 328204288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277031093279841, + "loss": 1.4981, + "theoretical_loss": 4.131621645393917, + "tokens_seen": 328269824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276870611835507, + "loss": 1.5857, + "theoretical_loss": 4.131526949412774, + "tokens_seen": 328335360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276710130391175, + "loss": 1.53, + "theoretical_loss": 4.131432277622325, + "tokens_seen": 328400896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276549648946841, + "loss": 1.5972, + "theoretical_loss": 4.131337630011564, + "tokens_seen": 328466432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276389167502509, + "loss": 1.5262, + "theoretical_loss": 4.131243006569493, + "tokens_seen": 328531968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276228686058175, + "loss": 1.5769, + "theoretical_loss": 4.131148407285121, + "tokens_seen": 328597504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276068204613843, + "loss": 1.6371, + "theoretical_loss": 4.131053832147465, + "tokens_seen": 328663040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275907723169509, + "loss": 1.6341, + "theoretical_loss": 4.130959281145547, + "tokens_seen": 328728576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275747241725177, + "loss": 1.6886, + "theoretical_loss": 4.130864754268399, + "tokens_seen": 328794112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275586760280843, + "loss": 1.5881, + "theoretical_loss": 4.130770251505058, + "tokens_seen": 328859648 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727542627883651, + "loss": 1.5361, + "theoretical_loss": 4.130675772844569, + "tokens_seen": 328925184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275265797392177, + "loss": 1.6476, + "theoretical_loss": 4.130581318275984, + "tokens_seen": 328990720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275105315947844, + "loss": 1.6211, + "theoretical_loss": 4.130486887788362, + "tokens_seen": 329056256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274944834503511, + "loss": 1.5303, + "theoretical_loss": 4.130392481370771, + "tokens_seen": 329121792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274784353059178, + "loss": 1.5864, + "theoretical_loss": 4.130298099012283, + "tokens_seen": 329187328 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274623871614845, + "loss": 1.5658, + "theoretical_loss": 4.130203740701979, + "tokens_seen": 329252864 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274463390170512, + "loss": 1.6121, + "theoretical_loss": 4.1301094064289465, + "tokens_seen": 329318400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274302908726179, + "loss": 1.7239, + "theoretical_loss": 4.130015096182282, + "tokens_seen": 329383936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274142427281846, + "loss": 1.5618, + "theoretical_loss": 4.129920809951085, + "tokens_seen": 329449472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273981945837512, + "loss": 1.594, + "theoretical_loss": 4.129826547724467, + "tokens_seen": 329515008 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727382146439318, + "loss": 1.6085, + "theoretical_loss": 4.129732309491542, + "tokens_seen": 329580544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273660982948846, + "loss": 1.7381, + "theoretical_loss": 4.129638095241436, + "tokens_seen": 329646080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273500501504514, + "loss": 1.608, + "theoretical_loss": 4.129543904963278, + "tokens_seen": 329711616 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727334002006018, + "loss": 1.6218, + "theoretical_loss": 4.129449738646205, + "tokens_seen": 329777152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273179538615848, + "loss": 1.5262, + "theoretical_loss": 4.129355596279362, + "tokens_seen": 329842688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273019057171514, + "loss": 1.585, + "theoretical_loss": 4.129261477851902, + "tokens_seen": 329908224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272858575727182, + "loss": 1.517, + "theoretical_loss": 4.129167383352982, + "tokens_seen": 329973760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272698094282848, + "loss": 1.5909, + "theoretical_loss": 4.129073312771768, + "tokens_seen": 330039296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272537612838516, + "loss": 1.6328, + "theoretical_loss": 4.128979266097433, + "tokens_seen": 330104832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272377131394182, + "loss": 1.5654, + "theoretical_loss": 4.128885243319158, + "tokens_seen": 330170368 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727221664994985, + "loss": 1.6423, + "theoretical_loss": 4.12879124442613, + "tokens_seen": 330235904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272056168505516, + "loss": 1.5612, + "theoretical_loss": 4.128697269407541, + "tokens_seen": 330301440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271895687061184, + "loss": 1.6453, + "theoretical_loss": 4.128603318252593, + "tokens_seen": 330366976 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727173520561685, + "loss": 1.6937, + "theoretical_loss": 4.128509390950494, + "tokens_seen": 330432512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271574724172518, + "loss": 1.6754, + "theoretical_loss": 4.128415487490461, + "tokens_seen": 330498048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271414242728184, + "loss": 1.5799, + "theoretical_loss": 4.128321607861713, + "tokens_seen": 330563584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271253761283852, + "loss": 1.6589, + "theoretical_loss": 4.128227752053482, + "tokens_seen": 330629120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271093279839518, + "loss": 1.5626, + "theoretical_loss": 4.128133920055001, + "tokens_seen": 330694656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007270932798395186, + "loss": 1.5973, + "theoretical_loss": 4.1280401118555154, + "tokens_seen": 330760192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007270772316950852, + "loss": 1.5759, + "theoretical_loss": 4.127946327444274, + "tokens_seen": 330825728 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727061183550652, + "loss": 1.4935, + "theoretical_loss": 4.127852566810535, + "tokens_seen": 330891264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 116426, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7715214490890503, + "objective/train/theoretical_loss": 4.127758829943562, + "objective/train/tokens_used": 351416800, + "theoretical_loss": 4.127758829943562, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007270451354062186, + "loss": 1.6378, + "theoretical_loss": 4.127758829943562, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007270290872617854, + "loss": 1.6115, + "theoretical_loss": 4.127665116832626, + "tokens_seen": 331022336 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727013039117352, + "loss": 1.4952, + "theoretical_loss": 4.127571427467005, + "tokens_seen": 331087872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269969909729188, + "loss": 1.5364, + "theoretical_loss": 4.1274777618359835, + "tokens_seen": 331153408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269809428284854, + "loss": 1.5265, + "theoretical_loss": 4.127384119928854, + "tokens_seen": 331218944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269648946840522, + "loss": 1.5606, + "theoretical_loss": 4.127290501734914, + "tokens_seen": 331284480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269488465396188, + "loss": 1.6189, + "theoretical_loss": 4.127196907243472, + "tokens_seen": 331350016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269327983951856, + "loss": 1.6621, + "theoretical_loss": 4.127103336443839, + "tokens_seen": 331415552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269167502507522, + "loss": 1.6471, + "theoretical_loss": 4.127009789325334, + "tokens_seen": 331481088 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726900702106319, + "loss": 1.5859, + "theoretical_loss": 4.126916265877284, + "tokens_seen": 331546624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268846539618856, + "loss": 1.5848, + "theoretical_loss": 4.126822766089024, + "tokens_seen": 331612160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268686058174524, + "loss": 1.7124, + "theoretical_loss": 4.126729289949893, + "tokens_seen": 331677696 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726852557673019, + "loss": 1.582, + "theoretical_loss": 4.126635837449238, + "tokens_seen": 331743232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268365095285858, + "loss": 1.5718, + "theoretical_loss": 4.126542408576414, + "tokens_seen": 331808768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268204613841524, + "loss": 1.5834, + "theoretical_loss": 4.126449003320783, + "tokens_seen": 331874304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268044132397192, + "loss": 1.5014, + "theoretical_loss": 4.126355621671711, + "tokens_seen": 331939840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267883650952858, + "loss": 1.4939, + "theoretical_loss": 4.126262263618574, + "tokens_seen": 332005376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267723169508526, + "loss": 1.5887, + "theoretical_loss": 4.1261689291507535, + "tokens_seen": 332070912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267562688064192, + "loss": 1.6777, + "theoretical_loss": 4.126075618257639, + "tokens_seen": 332136448 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726740220661986, + "loss": 1.6688, + "theoretical_loss": 4.125982330928624, + "tokens_seen": 332201984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267241725175528, + "loss": 1.6503, + "theoretical_loss": 4.125889067153113, + "tokens_seen": 332267520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267081243731194, + "loss": 1.5699, + "theoretical_loss": 4.125795826920514, + "tokens_seen": 332333056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266920762286862, + "loss": 1.5449, + "theoretical_loss": 4.125702610220244, + "tokens_seen": 332398592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266760280842528, + "loss": 1.6834, + "theoretical_loss": 4.1256094170417255, + "tokens_seen": 332464128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266599799398196, + "loss": 1.6401, + "theoretical_loss": 4.125516247374387, + "tokens_seen": 332529664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266439317953862, + "loss": 1.7152, + "theoretical_loss": 4.125423101207668, + "tokens_seen": 332595200 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726627883650953, + "loss": 1.4565, + "theoretical_loss": 4.125329978531009, + "tokens_seen": 332660736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266118355065196, + "loss": 1.6387, + "theoretical_loss": 4.125236879333861, + "tokens_seen": 332726272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265957873620864, + "loss": 1.5454, + "theoretical_loss": 4.125143803605683, + "tokens_seen": 332791808 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726579739217653, + "loss": 1.6302, + "theoretical_loss": 4.125050751335936, + "tokens_seen": 332857344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265636910732198, + "loss": 1.6075, + "theoretical_loss": 4.124957722514091, + "tokens_seen": 332922880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265476429287864, + "loss": 1.6286, + "theoretical_loss": 4.124864717129629, + "tokens_seen": 332988416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265315947843532, + "loss": 1.4963, + "theoretical_loss": 4.124771735172031, + "tokens_seen": 333053952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265155466399198, + "loss": 1.559, + "theoretical_loss": 4.1246787766307875, + "tokens_seen": 333119488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264994984954866, + "loss": 1.6646, + "theoretical_loss": 4.124585841495398, + "tokens_seen": 333185024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264834503510532, + "loss": 1.4918, + "theoretical_loss": 4.124492929755367, + "tokens_seen": 333250560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00072646740220662, + "loss": 1.5617, + "theoretical_loss": 4.1244000414002056, + "tokens_seen": 333316096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264513540621866, + "loss": 1.6774, + "theoretical_loss": 4.124307176419432, + "tokens_seen": 333381632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264353059177534, + "loss": 1.5307, + "theoretical_loss": 4.12421433480257, + "tokens_seen": 333447168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00072641925777332, + "loss": 1.5039, + "theoretical_loss": 4.124121516539153, + "tokens_seen": 333512704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264032096288868, + "loss": 1.6459, + "theoretical_loss": 4.124028721618719, + "tokens_seen": 333578240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263871614844534, + "loss": 1.5771, + "theoretical_loss": 4.123935950030812, + "tokens_seen": 333643776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263711133400202, + "loss": 1.5749, + "theoretical_loss": 4.123843201764984, + "tokens_seen": 333709312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263550651955868, + "loss": 1.5776, + "theoretical_loss": 4.1237504768107955, + "tokens_seen": 333774848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263390170511536, + "loss": 1.6285, + "theoretical_loss": 4.12365777515781, + "tokens_seen": 333840384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263229689067202, + "loss": 1.739, + "theoretical_loss": 4.1235650967956, + "tokens_seen": 333905920 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726306920762287, + "loss": 1.3803, + "theoretical_loss": 4.123472441713743, + "tokens_seen": 333971456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262908726178536, + "loss": 1.6053, + "theoretical_loss": 4.123379809901827, + "tokens_seen": 334036992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262748244734204, + "loss": 1.5184, + "theoretical_loss": 4.123287201349443, + "tokens_seen": 334102528 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726258776328987, + "loss": 1.4989, + "theoretical_loss": 4.123194616046189, + "tokens_seen": 334168064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 117616, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7517592906951904, + "objective/train/theoretical_loss": 4.123102053981672, + "objective/train/tokens_used": 354693600, + "theoretical_loss": 4.123102053981672, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262427281845538, + "loss": 1.6572, + "theoretical_loss": 4.123102053981672, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262266800401204, + "loss": 1.5332, + "theoretical_loss": 4.123009515145503, + "tokens_seen": 334299136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262106318956872, + "loss": 1.5541, + "theoretical_loss": 4.122916999527301, + "tokens_seen": 334364672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261945837512538, + "loss": 1.6208, + "theoretical_loss": 4.122824507116691, + "tokens_seen": 334430208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261785356068206, + "loss": 1.5833, + "theoretical_loss": 4.122732037903308, + "tokens_seen": 334495744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261624874623872, + "loss": 1.6827, + "theoretical_loss": 4.122639591876789, + "tokens_seen": 334561280 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726146439317954, + "loss": 1.5785, + "theoretical_loss": 4.122547169026778, + "tokens_seen": 334626816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261303911735206, + "loss": 1.6094, + "theoretical_loss": 4.12245476934293, + "tokens_seen": 334692352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261143430290873, + "loss": 1.5737, + "theoretical_loss": 4.122362392814903, + "tokens_seen": 334757888 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726098294884654, + "loss": 1.5096, + "theoretical_loss": 4.122270039432362, + "tokens_seen": 334823424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260822467402207, + "loss": 1.5614, + "theoretical_loss": 4.122177709184978, + "tokens_seen": 334888960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260661985957874, + "loss": 1.6424, + "theoretical_loss": 4.1220854020624325, + "tokens_seen": 334954496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260501504513541, + "loss": 1.6124, + "theoretical_loss": 4.121993118054409, + "tokens_seen": 335020032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260341023069208, + "loss": 1.5596, + "theoretical_loss": 4.1219008571505995, + "tokens_seen": 335085568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260180541624875, + "loss": 1.6681, + "theoretical_loss": 4.121808619340704, + "tokens_seen": 335151104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260020060180541, + "loss": 1.5883, + "theoretical_loss": 4.121716404614426, + "tokens_seen": 335216640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259859578736209, + "loss": 1.4965, + "theoretical_loss": 4.121624212961478, + "tokens_seen": 335282176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259699097291875, + "loss": 1.6037, + "theoretical_loss": 4.121532044371579, + "tokens_seen": 335347712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259538615847543, + "loss": 1.7225, + "theoretical_loss": 4.121439898834454, + "tokens_seen": 335413248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259378134403209, + "loss": 1.69, + "theoretical_loss": 4.1213477763398325, + "tokens_seen": 335478784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259217652958877, + "loss": 1.5917, + "theoretical_loss": 4.121255676877455, + "tokens_seen": 335544320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259057171514543, + "loss": 1.7592, + "theoretical_loss": 4.1211636004370655, + "tokens_seen": 335609856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258896690070211, + "loss": 1.722, + "theoretical_loss": 4.121071547008415, + "tokens_seen": 335675392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258736208625877, + "loss": 1.5877, + "theoretical_loss": 4.120979516581262, + "tokens_seen": 335740928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258575727181545, + "loss": 1.5069, + "theoretical_loss": 4.120887509145371, + "tokens_seen": 335806464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258415245737211, + "loss": 1.5241, + "theoretical_loss": 4.120795524690512, + "tokens_seen": 335872000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258254764292879, + "loss": 1.5952, + "theoretical_loss": 4.1207035632064635, + "tokens_seen": 335937536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258094282848545, + "loss": 1.4847, + "theoretical_loss": 4.120611624683008, + "tokens_seen": 336003072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257933801404213, + "loss": 1.7648, + "theoretical_loss": 4.120519709109938, + "tokens_seen": 336068608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257773319959879, + "loss": 1.5907, + "theoretical_loss": 4.12042781647705, + "tokens_seen": 336134144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257612838515547, + "loss": 1.4965, + "theoretical_loss": 4.120335946774147, + "tokens_seen": 336199680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257452357071213, + "loss": 1.6617, + "theoretical_loss": 4.12024409999104, + "tokens_seen": 336265216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257291875626881, + "loss": 1.5381, + "theoretical_loss": 4.120152276117545, + "tokens_seen": 336330752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257131394182547, + "loss": 1.5592, + "theoretical_loss": 4.120060475143486, + "tokens_seen": 336396288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256970912738215, + "loss": 1.5193, + "theoretical_loss": 4.119968697058692, + "tokens_seen": 336461824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256810431293881, + "loss": 1.5876, + "theoretical_loss": 4.119876941852999, + "tokens_seen": 336527360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256649949849549, + "loss": 1.652, + "theoretical_loss": 4.11978520951625, + "tokens_seen": 336592896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256489468405215, + "loss": 1.5175, + "theoretical_loss": 4.119693500038294, + "tokens_seen": 336658432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256328986960883, + "loss": 1.5785, + "theoretical_loss": 4.119601813408988, + "tokens_seen": 336723968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256168505516549, + "loss": 1.6262, + "theoretical_loss": 4.1195101496181925, + "tokens_seen": 336789504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256008024072217, + "loss": 1.5974, + "theoretical_loss": 4.119418508655777, + "tokens_seen": 336855040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255847542627883, + "loss": 1.5781, + "theoretical_loss": 4.119326890511615, + "tokens_seen": 336920576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255687061183551, + "loss": 1.4977, + "theoretical_loss": 4.1192352951755895, + "tokens_seen": 336986112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255526579739217, + "loss": 1.5431, + "theoretical_loss": 4.119143722637588, + "tokens_seen": 337051648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255366098294885, + "loss": 1.5899, + "theoretical_loss": 4.119052172887506, + "tokens_seen": 337117184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255205616850552, + "loss": 1.6135, + "theoretical_loss": 4.118960645915243, + "tokens_seen": 337182720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255045135406219, + "loss": 1.6049, + "theoretical_loss": 4.118869141710706, + "tokens_seen": 337248256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254884653961886, + "loss": 1.6426, + "theoretical_loss": 4.11877766026381, + "tokens_seen": 337313792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254724172517553, + "loss": 1.5858, + "theoretical_loss": 4.118686201564475, + "tokens_seen": 337379328 + }, + { + "epoch": 0.1, + "learning_rate": 0.000725456369107322, + "loss": 1.7204, + "theoretical_loss": 4.1185947656026265, + "tokens_seen": 337444864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 118525, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.7721240520477295, + "objective/train/theoretical_loss": 4.118503352368197, + "objective/train/tokens_used": 357970400, + "theoretical_loss": 4.118503352368197, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254403209628887, + "loss": 1.6626, + "theoretical_loss": 4.118503352368197, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254242728184554, + "loss": 1.506, + "theoretical_loss": 4.118411961851129, + "tokens_seen": 337575936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254082246740221, + "loss": 1.5272, + "theoretical_loss": 4.118320594041366, + "tokens_seen": 337641472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253921765295888, + "loss": 1.5772, + "theoretical_loss": 4.118229248928859, + "tokens_seen": 337707008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253761283851555, + "loss": 1.5166, + "theoretical_loss": 4.11813792650357, + "tokens_seen": 337772544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253600802407223, + "loss": 1.6504, + "theoretical_loss": 4.118046626755461, + "tokens_seen": 337838080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253440320962889, + "loss": 1.543, + "theoretical_loss": 4.117955349674505, + "tokens_seen": 337903616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253279839518557, + "loss": 1.6382, + "theoretical_loss": 4.117864095250679, + "tokens_seen": 337969152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253119358074223, + "loss": 1.5598, + "theoretical_loss": 4.117772863473968, + "tokens_seen": 338034688 + }, + { + "epoch": 0.1, + "learning_rate": 0.000725295887662989, + "loss": 1.626, + "theoretical_loss": 4.117681654334361, + "tokens_seen": 338100224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252798395185557, + "loss": 1.5688, + "theoretical_loss": 4.117590467821857, + "tokens_seen": 338165760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252637913741225, + "loss": 1.5726, + "theoretical_loss": 4.117499303926457, + "tokens_seen": 338231296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252477432296891, + "loss": 1.5728, + "theoretical_loss": 4.1174081626381716, + "tokens_seen": 338296832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252316950852559, + "loss": 1.5736, + "theoretical_loss": 4.117317043947017, + "tokens_seen": 338362368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252156469408225, + "loss": 1.6962, + "theoretical_loss": 4.117225947843014, + "tokens_seen": 338427904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251995987963893, + "loss": 1.5178, + "theoretical_loss": 4.117134874316194, + "tokens_seen": 338493440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251835506519559, + "loss": 1.5646, + "theoretical_loss": 4.117043823356589, + "tokens_seen": 338558976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251675025075227, + "loss": 1.7531, + "theoretical_loss": 4.116952794954242, + "tokens_seen": 338624512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251514543630893, + "loss": 1.4881, + "theoretical_loss": 4.1168617890992, + "tokens_seen": 338690048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251354062186561, + "loss": 1.5268, + "theoretical_loss": 4.116770805781517, + "tokens_seen": 338755584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251193580742227, + "loss": 1.404, + "theoretical_loss": 4.116679844991253, + "tokens_seen": 338821120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251033099297895, + "loss": 1.6254, + "theoretical_loss": 4.116588906718475, + "tokens_seen": 338886656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250872617853561, + "loss": 1.5061, + "theoretical_loss": 4.116497990953255, + "tokens_seen": 338952192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250712136409229, + "loss": 1.4944, + "theoretical_loss": 4.1164070976856735, + "tokens_seen": 339017728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250551654964895, + "loss": 1.6102, + "theoretical_loss": 4.1163162269058144, + "tokens_seen": 339083264 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250391173520563, + "loss": 1.5782, + "theoretical_loss": 4.11622537860377, + "tokens_seen": 339148800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250230692076229, + "loss": 1.7109, + "theoretical_loss": 4.116134552769638, + "tokens_seen": 339214336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250070210631897, + "loss": 1.6032, + "theoretical_loss": 4.116043749393523, + "tokens_seen": 339279872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249909729187563, + "loss": 1.533, + "theoretical_loss": 4.115952968465535, + "tokens_seen": 339345408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249749247743231, + "loss": 1.4966, + "theoretical_loss": 4.115862209975791, + "tokens_seen": 339410944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249588766298897, + "loss": 1.5367, + "theoretical_loss": 4.115771473914415, + "tokens_seen": 339476480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249428284854565, + "loss": 1.6097, + "theoretical_loss": 4.115680760271535, + "tokens_seen": 339542016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249267803410231, + "loss": 1.67, + "theoretical_loss": 4.115590069037287, + "tokens_seen": 339607552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249107321965899, + "loss": 1.7401, + "theoretical_loss": 4.115499400201813, + "tokens_seen": 339673088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248946840521565, + "loss": 1.6455, + "theoretical_loss": 4.1154087537552595, + "tokens_seen": 339738624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248786359077233, + "loss": 1.6652, + "theoretical_loss": 4.1153181296877825, + "tokens_seen": 339804160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248625877632899, + "loss": 1.4987, + "theoretical_loss": 4.115227527989543, + "tokens_seen": 339869696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248465396188567, + "loss": 1.6819, + "theoretical_loss": 4.115136948650705, + "tokens_seen": 339935232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248304914744233, + "loss": 1.512, + "theoretical_loss": 4.115046391661443, + "tokens_seen": 340000768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248144433299901, + "loss": 1.5822, + "theoretical_loss": 4.114955857011937, + "tokens_seen": 340066304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247983951855567, + "loss": 1.6232, + "theoretical_loss": 4.114865344692371, + "tokens_seen": 340131840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247823470411235, + "loss": 1.5821, + "theoretical_loss": 4.1147748546929375, + "tokens_seen": 340197376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247662988966901, + "loss": 1.484, + "theoretical_loss": 4.114684387003833, + "tokens_seen": 340262912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247502507522569, + "loss": 1.6424, + "theoretical_loss": 4.114593941615262, + "tokens_seen": 340328448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247342026078235, + "loss": 1.4232, + "theoretical_loss": 4.114503518517435, + "tokens_seen": 340393984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247181544633902, + "loss": 1.6698, + "theoretical_loss": 4.114413117700568, + "tokens_seen": 340459520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247021063189569, + "loss": 1.6268, + "theoretical_loss": 4.114322739154883, + "tokens_seen": 340525056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246860581745236, + "loss": 1.7974, + "theoretical_loss": 4.114232382870609, + "tokens_seen": 340590592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246700100300903, + "loss": 1.6961, + "theoretical_loss": 4.114142048837982, + "tokens_seen": 340656128 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724653961885657, + "loss": 1.6857, + "theoretical_loss": 4.114051737047241, + "tokens_seen": 340721664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 119921, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6314363479614258, + "objective/train/theoretical_loss": 4.1139614474886335, + "objective/train/tokens_used": 361247200, + "theoretical_loss": 4.1139614474886335, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246379137412237, + "loss": 1.5892, + "theoretical_loss": 4.1139614474886335, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246218655967904, + "loss": 1.6219, + "theoretical_loss": 4.1138711801524135, + "tokens_seen": 340852736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246058174523571, + "loss": 1.5791, + "theoretical_loss": 4.11378093502884, + "tokens_seen": 340918272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245897693079238, + "loss": 1.498, + "theoretical_loss": 4.113690712108179, + "tokens_seen": 340983808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245737211634904, + "loss": 1.5305, + "theoretical_loss": 4.113600511380701, + "tokens_seen": 341049344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245576730190572, + "loss": 1.4932, + "theoretical_loss": 4.113510332836685, + "tokens_seen": 341114880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245416248746238, + "loss": 1.5771, + "theoretical_loss": 4.1134201764664144, + "tokens_seen": 341180416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245255767301906, + "loss": 1.6052, + "theoretical_loss": 4.113330042260181, + "tokens_seen": 341245952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245095285857572, + "loss": 1.4641, + "theoretical_loss": 4.113239930208278, + "tokens_seen": 341311488 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724493480441324, + "loss": 1.6998, + "theoretical_loss": 4.113149840301009, + "tokens_seen": 341377024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007244774322968906, + "loss": 1.6401, + "theoretical_loss": 4.113059772528683, + "tokens_seen": 341442560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007244613841524574, + "loss": 1.5859, + "theoretical_loss": 4.112969726881613, + "tokens_seen": 341508096 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724445336008024, + "loss": 1.6477, + "theoretical_loss": 4.112879703350122, + "tokens_seen": 341573632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007244292878635908, + "loss": 1.701, + "theoretical_loss": 4.112789701924535, + "tokens_seen": 341639168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007244132397191574, + "loss": 1.5066, + "theoretical_loss": 4.112699722595185, + "tokens_seen": 341704704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243971915747242, + "loss": 1.5769, + "theoretical_loss": 4.11260976535241, + "tokens_seen": 341770240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243811434302908, + "loss": 1.613, + "theoretical_loss": 4.112519830186557, + "tokens_seen": 341835776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243650952858576, + "loss": 1.5651, + "theoretical_loss": 4.112429917087974, + "tokens_seen": 341901312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243490471414242, + "loss": 1.6294, + "theoretical_loss": 4.112340026047022, + "tokens_seen": 341966848 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724332998996991, + "loss": 1.5895, + "theoretical_loss": 4.112250157054061, + "tokens_seen": 342032384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243169508525577, + "loss": 1.8001, + "theoretical_loss": 4.112160310099461, + "tokens_seen": 342097920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243009027081244, + "loss": 1.5778, + "theoretical_loss": 4.112070485173598, + "tokens_seen": 342163456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242848545636911, + "loss": 1.6557, + "theoretical_loss": 4.111980682266853, + "tokens_seen": 342228992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242688064192578, + "loss": 1.5965, + "theoretical_loss": 4.111890901369612, + "tokens_seen": 342294528 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242527582748245, + "loss": 1.6546, + "theoretical_loss": 4.11180114247227, + "tokens_seen": 342360064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242367101303912, + "loss": 1.6265, + "theoretical_loss": 4.1117114055652255, + "tokens_seen": 342425600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242206619859579, + "loss": 1.5941, + "theoretical_loss": 4.111621690638884, + "tokens_seen": 342491136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242046138415246, + "loss": 1.4372, + "theoretical_loss": 4.111531997683658, + "tokens_seen": 342556672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241885656970913, + "loss": 1.5377, + "theoretical_loss": 4.1114423266899625, + "tokens_seen": 342622208 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724172517552658, + "loss": 1.6013, + "theoretical_loss": 4.111352677648224, + "tokens_seen": 342687744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241564694082247, + "loss": 1.5588, + "theoretical_loss": 4.111263050548869, + "tokens_seen": 342753280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241404212637914, + "loss": 1.7092, + "theoretical_loss": 4.111173445382335, + "tokens_seen": 342818816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241243731193581, + "loss": 1.5877, + "theoretical_loss": 4.111083862139063, + "tokens_seen": 342884352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241083249749248, + "loss": 1.569, + "theoretical_loss": 4.110994300809501, + "tokens_seen": 342949888 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007240922768304915, + "loss": 1.5495, + "theoretical_loss": 4.1109047613841, + "tokens_seen": 343015424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007240762286860582, + "loss": 1.5503, + "theoretical_loss": 4.110815243853323, + "tokens_seen": 343080960 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724060180541625, + "loss": 1.6105, + "theoretical_loss": 4.110725748207633, + "tokens_seen": 343146496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007240441323971916, + "loss": 1.4616, + "theoretical_loss": 4.110636274437502, + "tokens_seen": 343212032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007240280842527583, + "loss": 1.6878, + "theoretical_loss": 4.110546822533407, + "tokens_seen": 343277568 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724012036108325, + "loss": 1.6503, + "theoretical_loss": 4.110457392485832, + "tokens_seen": 343343104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239959879638917, + "loss": 1.7304, + "theoretical_loss": 4.110367984285265, + "tokens_seen": 343408640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239799398194584, + "loss": 1.6316, + "theoretical_loss": 4.1102785979222025, + "tokens_seen": 343474176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239638916750252, + "loss": 1.6789, + "theoretical_loss": 4.110189233387146, + "tokens_seen": 343539712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239478435305918, + "loss": 1.4762, + "theoretical_loss": 4.1100998906706, + "tokens_seen": 343605248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239317953861586, + "loss": 1.6151, + "theoretical_loss": 4.110010569763081, + "tokens_seen": 343670784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239157472417252, + "loss": 1.5582, + "theoretical_loss": 4.109921270655106, + "tokens_seen": 343736320 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723899699097292, + "loss": 1.5535, + "theoretical_loss": 4.109831993337201, + "tokens_seen": 343801856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238836509528586, + "loss": 1.6652, + "theoretical_loss": 4.109742737799895, + "tokens_seen": 343867392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238676028084254, + "loss": 1.5522, + "theoretical_loss": 4.109653504033726, + "tokens_seen": 343932928 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723851554663992, + "loss": 1.4251, + "theoretical_loss": 4.109564292029235, + "tokens_seen": 343998464 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 121082, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.569899082183838, + "objective/train/theoretical_loss": 4.109475101776974, + "objective/train/tokens_used": 364524000, + "theoretical_loss": 4.109475101776974, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238355065195588, + "loss": 1.6178, + "theoretical_loss": 4.109475101776974, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238194583751254, + "loss": 1.6877, + "theoretical_loss": 4.109385933267495, + "tokens_seen": 344129536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238034102306922, + "loss": 1.4787, + "theoretical_loss": 4.109296786491359, + "tokens_seen": 344195072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237873620862588, + "loss": 1.5375, + "theoretical_loss": 4.1092076614391315, + "tokens_seen": 344260608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237713139418256, + "loss": 1.7169, + "theoretical_loss": 4.109118558101386, + "tokens_seen": 344326144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237552657973922, + "loss": 1.5425, + "theoretical_loss": 4.1090294764687, + "tokens_seen": 344391680 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723739217652959, + "loss": 1.6181, + "theoretical_loss": 4.108940416531656, + "tokens_seen": 344457216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237231695085256, + "loss": 1.5927, + "theoretical_loss": 4.108851378280846, + "tokens_seen": 344522752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237071213640924, + "loss": 1.7046, + "theoretical_loss": 4.108762361706865, + "tokens_seen": 344588288 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723691073219659, + "loss": 1.6399, + "theoretical_loss": 4.108673366800313, + "tokens_seen": 344653824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236750250752258, + "loss": 1.5645, + "theoretical_loss": 4.1085843935518, + "tokens_seen": 344719360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236589769307924, + "loss": 1.4864, + "theoretical_loss": 4.108495441951937, + "tokens_seen": 344784896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236429287863592, + "loss": 1.6584, + "theoretical_loss": 4.108406511991344, + "tokens_seen": 344850432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236268806419258, + "loss": 1.5342, + "theoretical_loss": 4.108317603660646, + "tokens_seen": 344915968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236108324974926, + "loss": 1.6394, + "theoretical_loss": 4.108228716950474, + "tokens_seen": 344981504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007235947843530592, + "loss": 1.5845, + "theoretical_loss": 4.108139851851464, + "tokens_seen": 345047040 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723578736208626, + "loss": 1.4446, + "theoretical_loss": 4.108051008354258, + "tokens_seen": 345112576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007235626880641926, + "loss": 1.5017, + "theoretical_loss": 4.107962186449506, + "tokens_seen": 345178112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007235466399197594, + "loss": 1.5043, + "theoretical_loss": 4.107873386127861, + "tokens_seen": 345243648 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723530591775326, + "loss": 1.4736, + "theoretical_loss": 4.107784607379983, + "tokens_seen": 345309184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007235145436308928, + "loss": 1.4901, + "theoretical_loss": 4.107695850196539, + "tokens_seen": 345374720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234984954864594, + "loss": 1.5762, + "theoretical_loss": 4.107607114568197, + "tokens_seen": 345440256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234824473420262, + "loss": 1.5355, + "theoretical_loss": 4.107518400485638, + "tokens_seen": 345505792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234663991975928, + "loss": 1.6054, + "theoretical_loss": 4.107429707939545, + "tokens_seen": 345571328 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234503510531596, + "loss": 1.6409, + "theoretical_loss": 4.107341036920604, + "tokens_seen": 345636864 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234343029087262, + "loss": 1.5958, + "theoretical_loss": 4.107252387419513, + "tokens_seen": 345702400 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723418254764293, + "loss": 1.6453, + "theoretical_loss": 4.107163759426971, + "tokens_seen": 345767936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234022066198596, + "loss": 1.6439, + "theoretical_loss": 4.1070751529336835, + "tokens_seen": 345833472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233861584754264, + "loss": 1.5924, + "theoretical_loss": 4.106986567930364, + "tokens_seen": 345899008 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723370110330993, + "loss": 1.6914, + "theoretical_loss": 4.1068980044077295, + "tokens_seen": 345964544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233540621865598, + "loss": 1.5754, + "theoretical_loss": 4.1068094623565035, + "tokens_seen": 346030080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233380140421264, + "loss": 1.5374, + "theoretical_loss": 4.106720941767417, + "tokens_seen": 346095616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233219658976931, + "loss": 1.6939, + "theoretical_loss": 4.106632442631203, + "tokens_seen": 346161152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233059177532598, + "loss": 1.6309, + "theoretical_loss": 4.106543964938604, + "tokens_seen": 346226688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007232898696088265, + "loss": 1.5436, + "theoretical_loss": 4.106455508680366, + "tokens_seen": 346292224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007232738214643932, + "loss": 1.5128, + "theoretical_loss": 4.106367073847241, + "tokens_seen": 346357760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007232577733199599, + "loss": 1.4556, + "theoretical_loss": 4.106278660429988, + "tokens_seen": 346423296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007232417251755266, + "loss": 1.5254, + "theoretical_loss": 4.10619026841937, + "tokens_seen": 346488832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007232256770310933, + "loss": 1.7047, + "theoretical_loss": 4.106101897806156, + "tokens_seen": 346554368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00072320962888666, + "loss": 1.5841, + "theoretical_loss": 4.1060135485811236, + "tokens_seen": 346619904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231935807422267, + "loss": 1.6565, + "theoretical_loss": 4.105925220735052, + "tokens_seen": 346685440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231775325977933, + "loss": 1.5367, + "theoretical_loss": 4.105836914258728, + "tokens_seen": 346750976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231614844533601, + "loss": 1.5487, + "theoretical_loss": 4.105748629142945, + "tokens_seen": 346816512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231454363089267, + "loss": 1.5987, + "theoretical_loss": 4.1056603653785, + "tokens_seen": 346882048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231293881644935, + "loss": 1.6058, + "theoretical_loss": 4.1055721229561986, + "tokens_seen": 346947584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231133400200602, + "loss": 1.6939, + "theoretical_loss": 4.105483901866848, + "tokens_seen": 347013120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230972918756269, + "loss": 1.628, + "theoretical_loss": 4.105395702101266, + "tokens_seen": 347078656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230812437311936, + "loss": 1.6857, + "theoretical_loss": 4.1053075236502705, + "tokens_seen": 347144192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230651955867603, + "loss": 1.535, + "theoretical_loss": 4.1052193665046905, + "tokens_seen": 347209728 + }, + { + "epoch": 0.11, + "learning_rate": 0.000723049147442327, + "loss": 1.5932, + "theoretical_loss": 4.105131230655358, + "tokens_seen": 347275264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 122020, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6148931980133057, + "objective/train/theoretical_loss": 4.1050431160931105, + "objective/train/tokens_used": 367800800, + "theoretical_loss": 4.1050431160931105, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230330992978937, + "loss": 1.5267, + "theoretical_loss": 4.1050431160931105, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230170511534604, + "loss": 1.6963, + "theoretical_loss": 4.104955022808792, + "tokens_seen": 347406336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230010030090271, + "loss": 1.597, + "theoretical_loss": 4.104866950793251, + "tokens_seen": 347471872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229849548645938, + "loss": 1.6458, + "theoretical_loss": 4.104778900037343, + "tokens_seen": 347537408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229689067201605, + "loss": 1.5932, + "theoretical_loss": 4.104690870531929, + "tokens_seen": 347602944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229528585757272, + "loss": 1.6048, + "theoretical_loss": 4.104602862267875, + "tokens_seen": 347668480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229368104312939, + "loss": 1.5327, + "theoretical_loss": 4.1045148752360525, + "tokens_seen": 347734016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229207622868606, + "loss": 1.6515, + "theoretical_loss": 4.104426909427339, + "tokens_seen": 347799552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229047141424273, + "loss": 1.7427, + "theoretical_loss": 4.104338964832619, + "tokens_seen": 347865088 + }, + { + "epoch": 0.11, + "learning_rate": 0.000722888665997994, + "loss": 1.4355, + "theoretical_loss": 4.10425104144278, + "tokens_seen": 347930624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228726178535607, + "loss": 1.5693, + "theoretical_loss": 4.104163139248717, + "tokens_seen": 347996160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228565697091274, + "loss": 1.7103, + "theoretical_loss": 4.1040752582413305, + "tokens_seen": 348061696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228405215646941, + "loss": 1.5576, + "theoretical_loss": 4.103987398411524, + "tokens_seen": 348127232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228244734202608, + "loss": 1.5204, + "theoretical_loss": 4.103899559750213, + "tokens_seen": 348192768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228084252758275, + "loss": 1.5165, + "theoretical_loss": 4.10381174224831, + "tokens_seen": 348258304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227923771313942, + "loss": 1.4986, + "theoretical_loss": 4.10372394589674, + "tokens_seen": 348323840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227763289869609, + "loss": 1.6011, + "theoretical_loss": 4.10363617068643, + "tokens_seen": 348389376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227602808425276, + "loss": 1.5675, + "theoretical_loss": 4.103548416608316, + "tokens_seen": 348454912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227442326980943, + "loss": 1.6869, + "theoretical_loss": 4.103460683653335, + "tokens_seen": 348520448 + }, + { + "epoch": 0.11, + "learning_rate": 0.000722728184553661, + "loss": 1.5663, + "theoretical_loss": 4.103372971812432, + "tokens_seen": 348585984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227121364092277, + "loss": 1.605, + "theoretical_loss": 4.103285281076558, + "tokens_seen": 348651520 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226960882647944, + "loss": 1.5792, + "theoretical_loss": 4.10319761143667, + "tokens_seen": 348717056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226800401203611, + "loss": 1.5252, + "theoretical_loss": 4.103109962883728, + "tokens_seen": 348782592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226639919759278, + "loss": 1.57, + "theoretical_loss": 4.1030223354087, + "tokens_seen": 348848128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226479438314945, + "loss": 1.5449, + "theoretical_loss": 4.102934729002559, + "tokens_seen": 348913664 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226318956870612, + "loss": 1.4946, + "theoretical_loss": 4.102847143656284, + "tokens_seen": 348979200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226158475426279, + "loss": 1.548, + "theoretical_loss": 4.102759579360857, + "tokens_seen": 349044736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225997993981947, + "loss": 1.6362, + "theoretical_loss": 4.102672036107269, + "tokens_seen": 349110272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225837512537613, + "loss": 1.5507, + "theoretical_loss": 4.102584513886514, + "tokens_seen": 349175808 + }, + { + "epoch": 0.11, + "learning_rate": 0.000722567703109328, + "loss": 1.5163, + "theoretical_loss": 4.102497012689594, + "tokens_seen": 349241344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225516549648947, + "loss": 1.6004, + "theoretical_loss": 4.102409532507513, + "tokens_seen": 349306880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225356068204615, + "loss": 1.4954, + "theoretical_loss": 4.102322073331285, + "tokens_seen": 349372416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225195586760281, + "loss": 1.4697, + "theoretical_loss": 4.102234635151925, + "tokens_seen": 349437952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225035105315949, + "loss": 1.5609, + "theoretical_loss": 4.102147217960457, + "tokens_seen": 349503488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224874623871615, + "loss": 1.5016, + "theoretical_loss": 4.1020598217479085, + "tokens_seen": 349569024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224714142427283, + "loss": 1.6255, + "theoretical_loss": 4.101972446505314, + "tokens_seen": 349634560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224553660982949, + "loss": 1.5068, + "theoretical_loss": 4.101885092223712, + "tokens_seen": 349700096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224393179538617, + "loss": 1.6375, + "theoretical_loss": 4.101797758894147, + "tokens_seen": 349765632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224232698094283, + "loss": 1.5521, + "theoretical_loss": 4.101710446507669, + "tokens_seen": 349831168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224072216649951, + "loss": 1.5221, + "theoretical_loss": 4.101623155055336, + "tokens_seen": 349896704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223911735205617, + "loss": 1.5779, + "theoretical_loss": 4.101535884528206, + "tokens_seen": 349962240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223751253761285, + "loss": 1.5925, + "theoretical_loss": 4.1014486349173485, + "tokens_seen": 350027776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223590772316951, + "loss": 1.6624, + "theoretical_loss": 4.101361406213834, + "tokens_seen": 350093312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223430290872619, + "loss": 1.5511, + "theoretical_loss": 4.101274198408741, + "tokens_seen": 350158848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223269809428285, + "loss": 1.6351, + "theoretical_loss": 4.101187011493153, + "tokens_seen": 350224384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223109327983953, + "loss": 1.6788, + "theoretical_loss": 4.101099845458156, + "tokens_seen": 350289920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222948846539619, + "loss": 1.6737, + "theoretical_loss": 4.101012700294848, + "tokens_seen": 350355456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222788365095287, + "loss": 1.6357, + "theoretical_loss": 4.100925575994325, + "tokens_seen": 350420992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222627883650953, + "loss": 1.5646, + "theoretical_loss": 4.100838472547695, + "tokens_seen": 350486528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222467402206621, + "loss": 1.5503, + "theoretical_loss": 4.100751389946067, + "tokens_seen": 350552064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 123333, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5236725807189941, + "objective/train/theoretical_loss": 4.100664328180557, + "objective/train/tokens_used": 371077600, + "theoretical_loss": 4.100664328180557, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222306920762287, + "loss": 1.5919, + "theoretical_loss": 4.100664328180557, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222146439317955, + "loss": 1.6033, + "theoretical_loss": 4.100577287242286, + "tokens_seen": 350683136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221985957873621, + "loss": 1.4229, + "theoretical_loss": 4.100490267122381, + "tokens_seen": 350748672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221825476429289, + "loss": 1.4672, + "theoretical_loss": 4.100403267811975, + "tokens_seen": 350814208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221664994984955, + "loss": 1.6758, + "theoretical_loss": 4.100316289302206, + "tokens_seen": 350879744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221504513540623, + "loss": 1.5765, + "theoretical_loss": 4.100229331584215, + "tokens_seen": 350945280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221344032096289, + "loss": 1.6201, + "theoretical_loss": 4.100142394649152, + "tokens_seen": 351010816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221183550651957, + "loss": 1.5798, + "theoretical_loss": 4.100055478488171, + "tokens_seen": 351076352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221023069207623, + "loss": 1.5945, + "theoretical_loss": 4.099968583092431, + "tokens_seen": 351141888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220862587763291, + "loss": 1.7934, + "theoretical_loss": 4.099881708453097, + "tokens_seen": 351207424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220702106318957, + "loss": 1.5929, + "theoretical_loss": 4.099794854561339, + "tokens_seen": 351272960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220541624874625, + "loss": 1.5479, + "theoretical_loss": 4.0997080214083335, + "tokens_seen": 351338496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220381143430291, + "loss": 1.5969, + "theoretical_loss": 4.09962120898526, + "tokens_seen": 351404032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220220661985959, + "loss": 1.8095, + "theoretical_loss": 4.099534417283305, + "tokens_seen": 351469568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220060180541625, + "loss": 1.5545, + "theoretical_loss": 4.099447646293662, + "tokens_seen": 351535104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219899699097293, + "loss": 1.4616, + "theoretical_loss": 4.099360896007527, + "tokens_seen": 351600640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219739217652959, + "loss": 1.5338, + "theoretical_loss": 4.099274166416102, + "tokens_seen": 351666176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219578736208627, + "loss": 1.5553, + "theoretical_loss": 4.099187457510595, + "tokens_seen": 351731712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219418254764293, + "loss": 1.6277, + "theoretical_loss": 4.099100769282221, + "tokens_seen": 351797248 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721925777331996, + "loss": 1.5872, + "theoretical_loss": 4.099014101722196, + "tokens_seen": 351862784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219097291875627, + "loss": 1.549, + "theoretical_loss": 4.098927454821747, + "tokens_seen": 351928320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218936810431294, + "loss": 1.7034, + "theoretical_loss": 4.098840828572101, + "tokens_seen": 351993856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218776328986961, + "loss": 1.5264, + "theoretical_loss": 4.0987542229644935, + "tokens_seen": 352059392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218615847542628, + "loss": 1.5467, + "theoretical_loss": 4.0986676379901645, + "tokens_seen": 352124928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218455366098295, + "loss": 1.5454, + "theoretical_loss": 4.098581073640361, + "tokens_seen": 352190464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218294884653962, + "loss": 1.6108, + "theoretical_loss": 4.098494529906331, + "tokens_seen": 352256000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218134403209629, + "loss": 1.625, + "theoretical_loss": 4.098408006779332, + "tokens_seen": 352321536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217973921765296, + "loss": 1.6397, + "theoretical_loss": 4.098321504250626, + "tokens_seen": 352387072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217813440320963, + "loss": 1.651, + "theoretical_loss": 4.098235022311479, + "tokens_seen": 352452608 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721765295887663, + "loss": 1.5686, + "theoretical_loss": 4.098148560953163, + "tokens_seen": 352518144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217492477432297, + "loss": 1.5725, + "theoretical_loss": 4.098062120166956, + "tokens_seen": 352583680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217331995987964, + "loss": 1.5605, + "theoretical_loss": 4.097975699944141, + "tokens_seen": 352649216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217171514543631, + "loss": 1.4939, + "theoretical_loss": 4.097889300276005, + "tokens_seen": 352714752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217011033099298, + "loss": 1.425, + "theoretical_loss": 4.097802921153842, + "tokens_seen": 352780288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216850551654965, + "loss": 1.5532, + "theoretical_loss": 4.097716562568951, + "tokens_seen": 352845824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216690070210632, + "loss": 1.6839, + "theoretical_loss": 4.097630224512635, + "tokens_seen": 352911360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216529588766299, + "loss": 1.6294, + "theoretical_loss": 4.097543906976204, + "tokens_seen": 352976896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216369107321966, + "loss": 1.5198, + "theoretical_loss": 4.097457609950973, + "tokens_seen": 353042432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216208625877633, + "loss": 1.5761, + "theoretical_loss": 4.09737133342826, + "tokens_seen": 353107968 + }, + { + "epoch": 0.11, + "learning_rate": 0.00072160481444333, + "loss": 1.6117, + "theoretical_loss": 4.097285077399392, + "tokens_seen": 353173504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215887662988967, + "loss": 1.6125, + "theoretical_loss": 4.097198841855699, + "tokens_seen": 353239040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215727181544634, + "loss": 1.6285, + "theoretical_loss": 4.097112626788516, + "tokens_seen": 353304576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215566700100301, + "loss": 1.5738, + "theoretical_loss": 4.097026432189185, + "tokens_seen": 353370112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215406218655968, + "loss": 1.6719, + "theoretical_loss": 4.096940258049052, + "tokens_seen": 353435648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215245737211635, + "loss": 1.5116, + "theoretical_loss": 4.096854104359467, + "tokens_seen": 353501184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215085255767302, + "loss": 1.4971, + "theoretical_loss": 4.096767971111789, + "tokens_seen": 353566720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214924774322969, + "loss": 1.6234, + "theoretical_loss": 4.096681858297378, + "tokens_seen": 353632256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214764292878636, + "loss": 1.5845, + "theoretical_loss": 4.096595765907603, + "tokens_seen": 353697792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214603811434303, + "loss": 1.4026, + "theoretical_loss": 4.096509693933835, + "tokens_seen": 353763328 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721444332998997, + "loss": 1.745, + "theoretical_loss": 4.096423642367453, + "tokens_seen": 353828864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 124136, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5353221893310547, + "objective/train/theoretical_loss": 4.096337611199838, + "objective/train/tokens_used": 374354400, + "theoretical_loss": 4.096337611199838, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214282848545637, + "loss": 1.4744, + "theoretical_loss": 4.096337611199838, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214122367101304, + "loss": 1.5854, + "theoretical_loss": 4.096251600422381, + "tokens_seen": 353959936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213961885656971, + "loss": 1.5289, + "theoretical_loss": 4.096165610026474, + "tokens_seen": 354025472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213801404212638, + "loss": 1.5319, + "theoretical_loss": 4.096079640003515, + "tokens_seen": 354091008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213640922768305, + "loss": 1.469, + "theoretical_loss": 4.095993690344908, + "tokens_seen": 354156544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213480441323972, + "loss": 1.5532, + "theoretical_loss": 4.0959077610420636, + "tokens_seen": 354222080 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721331995987964, + "loss": 1.503, + "theoretical_loss": 4.095821852086394, + "tokens_seen": 354287616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213159478435306, + "loss": 1.4657, + "theoretical_loss": 4.095735963469322, + "tokens_seen": 354353152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212998996990973, + "loss": 1.4184, + "theoretical_loss": 4.095650095182268, + "tokens_seen": 354418688 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721283851554664, + "loss": 1.525, + "theoretical_loss": 4.0955642472166645, + "tokens_seen": 354484224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212678034102307, + "loss": 1.524, + "theoretical_loss": 4.095478419563947, + "tokens_seen": 354549760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212517552657974, + "loss": 1.4833, + "theoretical_loss": 4.095392612215555, + "tokens_seen": 354615296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212357071213642, + "loss": 1.486, + "theoretical_loss": 4.095306825162933, + "tokens_seen": 354680832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212196589769308, + "loss": 1.6026, + "theoretical_loss": 4.095221058397534, + "tokens_seen": 354746368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212036108324976, + "loss": 1.493, + "theoretical_loss": 4.095135311910812, + "tokens_seen": 354811904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007211875626880642, + "loss": 1.4913, + "theoretical_loss": 4.095049585694229, + "tokens_seen": 354877440 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721171514543631, + "loss": 1.5869, + "theoretical_loss": 4.09496387973925, + "tokens_seen": 354942976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007211554663991976, + "loss": 1.4811, + "theoretical_loss": 4.094878194037348, + "tokens_seen": 355008512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007211394182547644, + "loss": 1.5131, + "theoretical_loss": 4.094792528579999, + "tokens_seen": 355074048 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721123370110331, + "loss": 1.5777, + "theoretical_loss": 4.094706883358684, + "tokens_seen": 355139584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007211073219658978, + "loss": 1.5027, + "theoretical_loss": 4.09462125836489, + "tokens_seen": 355205120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210912738214644, + "loss": 1.4404, + "theoretical_loss": 4.0945356535901105, + "tokens_seen": 355270656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210752256770312, + "loss": 1.5339, + "theoretical_loss": 4.094450069025841, + "tokens_seen": 355336192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210591775325978, + "loss": 1.454, + "theoretical_loss": 4.094364504663584, + "tokens_seen": 355401728 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210431293881646, + "loss": 1.6805, + "theoretical_loss": 4.094278960494847, + "tokens_seen": 355467264 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210270812437312, + "loss": 1.4953, + "theoretical_loss": 4.094193436511143, + "tokens_seen": 355532800 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721011033099298, + "loss": 1.4764, + "theoretical_loss": 4.094107932703991, + "tokens_seen": 355598336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209949849548646, + "loss": 1.6513, + "theoretical_loss": 4.09402244906491, + "tokens_seen": 355663872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209789368104314, + "loss": 1.6311, + "theoretical_loss": 4.093936985585432, + "tokens_seen": 355729408 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720962888665998, + "loss": 1.6119, + "theoretical_loss": 4.093851542257088, + "tokens_seen": 355794944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209468405215648, + "loss": 1.6171, + "theoretical_loss": 4.093766119071416, + "tokens_seen": 355860480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209307923771314, + "loss": 1.6012, + "theoretical_loss": 4.093680716019961, + "tokens_seen": 355926016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209147442326982, + "loss": 1.5785, + "theoretical_loss": 4.093595333094269, + "tokens_seen": 355991552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208986960882648, + "loss": 1.5399, + "theoretical_loss": 4.093509970285895, + "tokens_seen": 356057088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208826479438316, + "loss": 1.5283, + "theoretical_loss": 4.093424627586399, + "tokens_seen": 356122624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208665997993982, + "loss": 1.4739, + "theoretical_loss": 4.093339304987341, + "tokens_seen": 356188160 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720850551654965, + "loss": 1.4281, + "theoretical_loss": 4.093254002480293, + "tokens_seen": 356253696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208345035105316, + "loss": 1.7098, + "theoretical_loss": 4.093168720056828, + "tokens_seen": 356319232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208184553660984, + "loss": 1.4742, + "theoretical_loss": 4.093083457708524, + "tokens_seen": 356384768 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720802407221665, + "loss": 1.6874, + "theoretical_loss": 4.092998215426967, + "tokens_seen": 356450304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207863590772318, + "loss": 1.5701, + "theoretical_loss": 4.092912993203744, + "tokens_seen": 356515840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207703109327984, + "loss": 1.5932, + "theoretical_loss": 4.09282779103045, + "tokens_seen": 356581376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207542627883652, + "loss": 1.5365, + "theoretical_loss": 4.092742608898685, + "tokens_seen": 356646912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207382146439318, + "loss": 1.6133, + "theoretical_loss": 4.092657446800052, + "tokens_seen": 356712448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207221664994986, + "loss": 1.5243, + "theoretical_loss": 4.092572304726161, + "tokens_seen": 356777984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207061183550653, + "loss": 1.5389, + "theoretical_loss": 4.092487182668627, + "tokens_seen": 356843520 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720690070210632, + "loss": 1.6216, + "theoretical_loss": 4.092402080619069, + "tokens_seen": 356909056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007206740220661987, + "loss": 1.7329, + "theoretical_loss": 4.09231699856911, + "tokens_seen": 356974592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007206579739217654, + "loss": 1.5615, + "theoretical_loss": 4.092231936510383, + "tokens_seen": 357040128 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720641925777332, + "loss": 1.5868, + "theoretical_loss": 4.09214689443452, + "tokens_seen": 357105664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 125479, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5331692695617676, + "objective/train/theoretical_loss": 4.0920618723331605, + "objective/train/tokens_used": 377631200, + "theoretical_loss": 4.0920618723331605, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007206258776328988, + "loss": 1.4486, + "theoretical_loss": 4.0920618723331605, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007206098294884654, + "loss": 1.621, + "theoretical_loss": 4.091976870197951, + "tokens_seen": 357236736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205937813440322, + "loss": 1.5364, + "theoretical_loss": 4.091891888020539, + "tokens_seen": 357302272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205777331995988, + "loss": 1.4859, + "theoretical_loss": 4.091806925792582, + "tokens_seen": 357367808 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205616850551656, + "loss": 1.6564, + "theoretical_loss": 4.091721983505737, + "tokens_seen": 357433344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205456369107322, + "loss": 1.4599, + "theoretical_loss": 4.091637061151671, + "tokens_seen": 357498880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205295887662989, + "loss": 1.4924, + "theoretical_loss": 4.091552158722052, + "tokens_seen": 357564416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205135406218656, + "loss": 1.4918, + "theoretical_loss": 4.091467276208554, + "tokens_seen": 357629952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204974924774323, + "loss": 1.5315, + "theoretical_loss": 4.091382413602861, + "tokens_seen": 357695488 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720481444332999, + "loss": 1.6625, + "theoretical_loss": 4.0912975708966535, + "tokens_seen": 357761024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204653961885657, + "loss": 1.5066, + "theoretical_loss": 4.091212748081622, + "tokens_seen": 357826560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204493480441324, + "loss": 1.5501, + "theoretical_loss": 4.091127945149463, + "tokens_seen": 357892096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204332998996991, + "loss": 1.5951, + "theoretical_loss": 4.091043162091876, + "tokens_seen": 357957632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204172517552658, + "loss": 1.6305, + "theoretical_loss": 4.090958398900564, + "tokens_seen": 358023168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204012036108325, + "loss": 1.4798, + "theoretical_loss": 4.090873655567238, + "tokens_seen": 358088704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203851554663992, + "loss": 1.5176, + "theoretical_loss": 4.090788932083613, + "tokens_seen": 358154240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203691073219659, + "loss": 1.5244, + "theoretical_loss": 4.090704228441408, + "tokens_seen": 358219776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203530591775326, + "loss": 1.4846, + "theoretical_loss": 4.090619544632347, + "tokens_seen": 358285312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203370110330993, + "loss": 1.5317, + "theoretical_loss": 4.0905348806481605, + "tokens_seen": 358350848 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720320962888666, + "loss": 1.5263, + "theoretical_loss": 4.090450236480583, + "tokens_seen": 358416384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203049147442327, + "loss": 1.4949, + "theoretical_loss": 4.090365612121355, + "tokens_seen": 358481920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202888665997994, + "loss": 1.5811, + "theoretical_loss": 4.0902810075622185, + "tokens_seen": 358547456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202728184553661, + "loss": 1.5137, + "theoretical_loss": 4.090196422794925, + "tokens_seen": 358612992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202567703109328, + "loss": 1.6153, + "theoretical_loss": 4.0901118578112285, + "tokens_seen": 358678528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202407221664995, + "loss": 1.5427, + "theoretical_loss": 4.090027312602888, + "tokens_seen": 358744064 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202246740220662, + "loss": 1.5705, + "theoretical_loss": 4.089942787161667, + "tokens_seen": 358809600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202086258776329, + "loss": 1.6536, + "theoretical_loss": 4.089858281479335, + "tokens_seen": 358875136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201925777331996, + "loss": 1.5034, + "theoretical_loss": 4.089773795547668, + "tokens_seen": 358940672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201765295887663, + "loss": 1.6133, + "theoretical_loss": 4.089689329358443, + "tokens_seen": 359006208 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720160481444333, + "loss": 1.6104, + "theoretical_loss": 4.089604882903444, + "tokens_seen": 359071744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201444332998997, + "loss": 1.5082, + "theoretical_loss": 4.089520456174461, + "tokens_seen": 359137280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201283851554664, + "loss": 1.6146, + "theoretical_loss": 4.089436049163287, + "tokens_seen": 359202816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201123370110331, + "loss": 1.6244, + "theoretical_loss": 4.089351661861722, + "tokens_seen": 359268352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200962888665998, + "loss": 1.5419, + "theoretical_loss": 4.089267294261566, + "tokens_seen": 359333888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200802407221665, + "loss": 1.6289, + "theoretical_loss": 4.089182946354631, + "tokens_seen": 359399424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200641925777332, + "loss": 1.5434, + "theoretical_loss": 4.08909861813273, + "tokens_seen": 359464960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200481444332999, + "loss": 1.4964, + "theoretical_loss": 4.0890143095876805, + "tokens_seen": 359530496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200320962888666, + "loss": 1.6228, + "theoretical_loss": 4.088930020711305, + "tokens_seen": 359596032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200160481444333, + "loss": 1.5421, + "theoretical_loss": 4.088845751495433, + "tokens_seen": 359661568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00072, + "loss": 1.4741, + "theoretical_loss": 4.088761501931897, + "tokens_seen": 359727104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199839518555667, + "loss": 1.6506, + "theoretical_loss": 4.088677272012535, + "tokens_seen": 359792640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199679037111334, + "loss": 1.6194, + "theoretical_loss": 4.088593061729188, + "tokens_seen": 359858176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199518555667001, + "loss": 1.5317, + "theoretical_loss": 4.0885088710737065, + "tokens_seen": 359923712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199358074222668, + "loss": 1.6348, + "theoretical_loss": 4.088424700037941, + "tokens_seen": 359989248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199197592778335, + "loss": 1.5538, + "theoretical_loss": 4.08834054861375, + "tokens_seen": 360054784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199037111334002, + "loss": 1.5956, + "theoretical_loss": 4.088256416792994, + "tokens_seen": 360120320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198876629889669, + "loss": 1.4913, + "theoretical_loss": 4.088172304567541, + "tokens_seen": 360185856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198716148445337, + "loss": 1.6122, + "theoretical_loss": 4.088088211929263, + "tokens_seen": 360251392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198555667001003, + "loss": 1.4953, + "theoretical_loss": 4.088004138870037, + "tokens_seen": 360316928 + }, + { + "epoch": 0.11, + "learning_rate": 0.000719839518555667, + "loss": 1.5233, + "theoretical_loss": 4.087920085381745, + "tokens_seen": 360382464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 126440, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6889604330062866, + "objective/train/theoretical_loss": 4.0878360514562715, + "objective/train/tokens_used": 380908000, + "theoretical_loss": 4.0878360514562715, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198234704112337, + "loss": 1.6539, + "theoretical_loss": 4.0878360514562715, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198074222668005, + "loss": 1.4704, + "theoretical_loss": 4.087752037085509, + "tokens_seen": 360513536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197913741223671, + "loss": 1.5366, + "theoretical_loss": 4.087668042261353, + "tokens_seen": 360579072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197753259779339, + "loss": 1.4443, + "theoretical_loss": 4.087584066975706, + "tokens_seen": 360644608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197592778335005, + "loss": 1.5863, + "theoretical_loss": 4.087500111220472, + "tokens_seen": 360710144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197432296890673, + "loss": 1.5266, + "theoretical_loss": 4.087416174987563, + "tokens_seen": 360775680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197271815446339, + "loss": 1.6049, + "theoretical_loss": 4.087332258268893, + "tokens_seen": 360841216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197111334002007, + "loss": 1.5389, + "theoretical_loss": 4.087248361056384, + "tokens_seen": 360906752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196950852557673, + "loss": 1.7153, + "theoretical_loss": 4.087164483341959, + "tokens_seen": 360972288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196790371113341, + "loss": 1.4725, + "theoretical_loss": 4.087080625117549, + "tokens_seen": 361037824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196629889669007, + "loss": 1.4997, + "theoretical_loss": 4.086996786375089, + "tokens_seen": 361103360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196469408224675, + "loss": 1.5327, + "theoretical_loss": 4.0869129671065165, + "tokens_seen": 361168896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196308926780341, + "loss": 1.5279, + "theoretical_loss": 4.086829167303778, + "tokens_seen": 361234432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196148445336009, + "loss": 1.6238, + "theoretical_loss": 4.086745386958821, + "tokens_seen": 361299968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195987963891675, + "loss": 1.6077, + "theoretical_loss": 4.0866616260636, + "tokens_seen": 361365504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195827482447343, + "loss": 1.5136, + "theoretical_loss": 4.086577884610074, + "tokens_seen": 361431040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195667001003009, + "loss": 1.5304, + "theoretical_loss": 4.086494162590206, + "tokens_seen": 361496576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195506519558677, + "loss": 1.6102, + "theoretical_loss": 4.086410459995964, + "tokens_seen": 361562112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195346038114343, + "loss": 1.6007, + "theoretical_loss": 4.0863267768193205, + "tokens_seen": 361627648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195185556670011, + "loss": 1.5662, + "theoretical_loss": 4.086243113052254, + "tokens_seen": 361693184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195025075225678, + "loss": 1.4989, + "theoretical_loss": 4.0861594686867475, + "tokens_seen": 361758720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194864593781345, + "loss": 1.5979, + "theoretical_loss": 4.086075843714786, + "tokens_seen": 361824256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194704112337012, + "loss": 1.6645, + "theoretical_loss": 4.085992238128364, + "tokens_seen": 361889792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194543630892679, + "loss": 1.7224, + "theoretical_loss": 4.085908651919477, + "tokens_seen": 361955328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194383149448346, + "loss": 1.5489, + "theoretical_loss": 4.085825085080128, + "tokens_seen": 362020864 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194222668004013, + "loss": 1.5459, + "theoretical_loss": 4.08574153760232, + "tokens_seen": 362086400 + }, + { + "epoch": 0.11, + "learning_rate": 0.000719406218655968, + "loss": 1.7234, + "theoretical_loss": 4.085658009478067, + "tokens_seen": 362151936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193901705115347, + "loss": 1.4414, + "theoretical_loss": 4.0855745006993835, + "tokens_seen": 362217472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193741223671014, + "loss": 1.5663, + "theoretical_loss": 4.08549101125829, + "tokens_seen": 362283008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193580742226681, + "loss": 1.6014, + "theoretical_loss": 4.085407541146812, + "tokens_seen": 362348544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193420260782348, + "loss": 1.6279, + "theoretical_loss": 4.085324090356979, + "tokens_seen": 362414080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193259779338015, + "loss": 1.531, + "theoretical_loss": 4.085240658880827, + "tokens_seen": 362479616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193099297893682, + "loss": 1.5496, + "theoretical_loss": 4.085157246710394, + "tokens_seen": 362545152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192938816449349, + "loss": 1.5263, + "theoretical_loss": 4.0850738538377245, + "tokens_seen": 362610688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192778335005016, + "loss": 1.6056, + "theoretical_loss": 4.084990480254867, + "tokens_seen": 362676224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192617853560683, + "loss": 1.6295, + "theoretical_loss": 4.084907125953876, + "tokens_seen": 362741760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192457372116349, + "loss": 1.5451, + "theoretical_loss": 4.0848237909268095, + "tokens_seen": 362807296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192296890672017, + "loss": 1.6121, + "theoretical_loss": 4.084740475165729, + "tokens_seen": 362872832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192136409227683, + "loss": 1.5199, + "theoretical_loss": 4.084657178662704, + "tokens_seen": 362938368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191975927783351, + "loss": 1.5887, + "theoretical_loss": 4.084573901409807, + "tokens_seen": 363003904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191815446339017, + "loss": 1.6182, + "theoretical_loss": 4.084490643399113, + "tokens_seen": 363069440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191654964894685, + "loss": 1.5459, + "theoretical_loss": 4.084407404622706, + "tokens_seen": 363134976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191494483450351, + "loss": 1.6282, + "theoretical_loss": 4.084324185072671, + "tokens_seen": 363200512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191334002006019, + "loss": 1.6513, + "theoretical_loss": 4.084240984741099, + "tokens_seen": 363266048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191173520561685, + "loss": 1.5754, + "theoretical_loss": 4.084157803620086, + "tokens_seen": 363331584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191013039117352, + "loss": 1.6044, + "theoretical_loss": 4.084074641701734, + "tokens_seen": 363397120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190852557673019, + "loss": 1.6218, + "theoretical_loss": 4.083991498978146, + "tokens_seen": 363462656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190692076228686, + "loss": 1.549, + "theoretical_loss": 4.083908375441433, + "tokens_seen": 363528192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190531594784353, + "loss": 1.5066, + "theoretical_loss": 4.083825271083709, + "tokens_seen": 363593728 + }, + { + "epoch": 0.11, + "learning_rate": 0.000719037111334002, + "loss": 1.7009, + "theoretical_loss": 4.083742185897093, + "tokens_seen": 363659264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 127545, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4759583473205566, + "objective/train/theoretical_loss": 4.08365911987371, + "objective/train/tokens_used": 384184800, + "theoretical_loss": 4.08365911987371, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190210631895687, + "loss": 1.4025, + "theoretical_loss": 4.08365911987371, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190050150451354, + "loss": 1.6018, + "theoretical_loss": 4.083576073005687, + "tokens_seen": 363790336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189889669007021, + "loss": 1.381, + "theoretical_loss": 4.083493045285158, + "tokens_seen": 363855872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189729187562688, + "loss": 1.4797, + "theoretical_loss": 4.08341003670426, + "tokens_seen": 363921408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189568706118355, + "loss": 1.477, + "theoretical_loss": 4.0833270472551355, + "tokens_seen": 363986944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189408224674022, + "loss": 1.6203, + "theoretical_loss": 4.083244076929932, + "tokens_seen": 364052480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189247743229689, + "loss": 1.669, + "theoretical_loss": 4.083161125720801, + "tokens_seen": 364118016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189087261785356, + "loss": 1.5458, + "theoretical_loss": 4.083078193619899, + "tokens_seen": 364183552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188926780341023, + "loss": 1.6065, + "theoretical_loss": 4.082995280619386, + "tokens_seen": 364249088 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718876629889669, + "loss": 1.4797, + "theoretical_loss": 4.0829123867114285, + "tokens_seen": 364314624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188605817452357, + "loss": 1.752, + "theoretical_loss": 4.082829511888197, + "tokens_seen": 364380160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188445336008024, + "loss": 1.5957, + "theoretical_loss": 4.082746656141865, + "tokens_seen": 364445696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188284854563691, + "loss": 1.6762, + "theoretical_loss": 4.082663819464613, + "tokens_seen": 364511232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188124373119358, + "loss": 1.6709, + "theoretical_loss": 4.082581001848624, + "tokens_seen": 364576768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187963891675025, + "loss": 1.4828, + "theoretical_loss": 4.082498203286088, + "tokens_seen": 364642304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187803410230692, + "loss": 1.5422, + "theoretical_loss": 4.082415423769197, + "tokens_seen": 364707840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187642928786359, + "loss": 1.4115, + "theoretical_loss": 4.082332663290149, + "tokens_seen": 364773376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187482447342026, + "loss": 1.5521, + "theoretical_loss": 4.082249921841147, + "tokens_seen": 364838912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187321965897693, + "loss": 1.5435, + "theoretical_loss": 4.082167199414398, + "tokens_seen": 364904448 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718716148445336, + "loss": 1.5592, + "theoretical_loss": 4.082084496002113, + "tokens_seen": 364969984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187001003009027, + "loss": 1.521, + "theoretical_loss": 4.082001811596509, + "tokens_seen": 365035520 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186840521564694, + "loss": 1.5971, + "theoretical_loss": 4.081919146189806, + "tokens_seen": 365101056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186680040120361, + "loss": 1.3846, + "theoretical_loss": 4.08183649977423, + "tokens_seen": 365166592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186519558676028, + "loss": 1.6659, + "theoretical_loss": 4.08175387234201, + "tokens_seen": 365232128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186359077231695, + "loss": 1.5476, + "theoretical_loss": 4.081671263885381, + "tokens_seen": 365297664 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186198595787362, + "loss": 1.501, + "theoretical_loss": 4.081588674396583, + "tokens_seen": 365363200 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718603811434303, + "loss": 1.5564, + "theoretical_loss": 4.081506103867858, + "tokens_seen": 365428736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185877632898696, + "loss": 1.6682, + "theoretical_loss": 4.081423552291456, + "tokens_seen": 365494272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185717151454363, + "loss": 1.5907, + "theoretical_loss": 4.081341019659628, + "tokens_seen": 365559808 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718555667001003, + "loss": 1.5072, + "theoretical_loss": 4.081258505964633, + "tokens_seen": 365625344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185396188565697, + "loss": 1.629, + "theoretical_loss": 4.081176011198731, + "tokens_seen": 365690880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185235707121364, + "loss": 1.6701, + "theoretical_loss": 4.08109353535419, + "tokens_seen": 365756416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185075225677032, + "loss": 1.6575, + "theoretical_loss": 4.0810110784232805, + "tokens_seen": 365821952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184914744232698, + "loss": 1.6648, + "theoretical_loss": 4.080928640398278, + "tokens_seen": 365887488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184754262788366, + "loss": 1.5334, + "theoretical_loss": 4.080846221271461, + "tokens_seen": 365953024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184593781344032, + "loss": 1.6324, + "theoretical_loss": 4.0807638210351165, + "tokens_seen": 366018560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00071844332998997, + "loss": 1.7469, + "theoretical_loss": 4.080681439681533, + "tokens_seen": 366084096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184272818455366, + "loss": 1.6183, + "theoretical_loss": 4.080599077203003, + "tokens_seen": 366149632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184112337011034, + "loss": 1.4391, + "theoretical_loss": 4.080516733591827, + "tokens_seen": 366215168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00071839518555667, + "loss": 1.578, + "theoretical_loss": 4.080434408840304, + "tokens_seen": 366280704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183791374122368, + "loss": 1.6214, + "theoretical_loss": 4.080352102940744, + "tokens_seen": 366346240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183630892678034, + "loss": 1.583, + "theoretical_loss": 4.080269815885458, + "tokens_seen": 366411776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183470411233702, + "loss": 1.6004, + "theoretical_loss": 4.080187547666763, + "tokens_seen": 366477312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183309929789368, + "loss": 1.5257, + "theoretical_loss": 4.080105298276978, + "tokens_seen": 366542848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183149448345036, + "loss": 1.4851, + "theoretical_loss": 4.08002306770843, + "tokens_seen": 366608384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182988966900703, + "loss": 1.554, + "theoretical_loss": 4.079940855953447, + "tokens_seen": 366673920 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718282848545637, + "loss": 1.5353, + "theoretical_loss": 4.079858663004364, + "tokens_seen": 366739456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182668004012037, + "loss": 1.5238, + "theoretical_loss": 4.079776488853521, + "tokens_seen": 366804992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182507522567704, + "loss": 1.5154, + "theoretical_loss": 4.079694333493259, + "tokens_seen": 366870528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182347041123371, + "loss": 1.6315, + "theoretical_loss": 4.079612196915927, + "tokens_seen": 366936064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 128594, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5575971603393555, + "objective/train/theoretical_loss": 4.079530079113876, + "objective/train/tokens_used": 387461600, + "theoretical_loss": 4.079530079113876, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182186559679038, + "loss": 1.559, + "theoretical_loss": 4.079530079113876, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182026078234705, + "loss": 1.6243, + "theoretical_loss": 4.079447980079465, + "tokens_seen": 367067136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181865596790372, + "loss": 1.5679, + "theoretical_loss": 4.079365899805053, + "tokens_seen": 367132672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181705115346039, + "loss": 1.5415, + "theoretical_loss": 4.079283838283007, + "tokens_seen": 367198208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181544633901706, + "loss": 1.63, + "theoretical_loss": 4.079201795505695, + "tokens_seen": 367263744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181384152457373, + "loss": 1.5294, + "theoretical_loss": 4.079119771465494, + "tokens_seen": 367329280 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718122367101304, + "loss": 1.5443, + "theoretical_loss": 4.079037766154781, + "tokens_seen": 367394816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181063189568707, + "loss": 1.6435, + "theoretical_loss": 4.078955779565941, + "tokens_seen": 367460352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180902708124374, + "loss": 1.4562, + "theoretical_loss": 4.078873811691361, + "tokens_seen": 367525888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180742226680041, + "loss": 1.6974, + "theoretical_loss": 4.078791862523433, + "tokens_seen": 367591424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180581745235708, + "loss": 1.5949, + "theoretical_loss": 4.078709932054554, + "tokens_seen": 367656960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180421263791375, + "loss": 1.5187, + "theoretical_loss": 4.078628020277128, + "tokens_seen": 367722496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180260782347042, + "loss": 1.547, + "theoretical_loss": 4.078546127183556, + "tokens_seen": 367788032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180100300902709, + "loss": 1.5075, + "theoretical_loss": 4.078464252766252, + "tokens_seen": 367853568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179939819458376, + "loss": 1.4956, + "theoretical_loss": 4.078382397017628, + "tokens_seen": 367919104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179779338014043, + "loss": 1.6077, + "theoretical_loss": 4.0783005599301045, + "tokens_seen": 367984640 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717961885656971, + "loss": 1.6948, + "theoretical_loss": 4.078218741496104, + "tokens_seen": 368050176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179458375125377, + "loss": 1.5008, + "theoretical_loss": 4.078136941708055, + "tokens_seen": 368115712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179297893681044, + "loss": 1.4755, + "theoretical_loss": 4.078055160558391, + "tokens_seen": 368181248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179137412236711, + "loss": 1.6208, + "theoretical_loss": 4.0779733980395445, + "tokens_seen": 368246784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178976930792378, + "loss": 1.5518, + "theoretical_loss": 4.077891654143961, + "tokens_seen": 368312320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178816449348045, + "loss": 1.746, + "theoretical_loss": 4.0778099288640846, + "tokens_seen": 368377856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178655967903712, + "loss": 1.4664, + "theoretical_loss": 4.0777282221923645, + "tokens_seen": 368443392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178495486459378, + "loss": 1.6716, + "theoretical_loss": 4.077646534121256, + "tokens_seen": 368508928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178335005015046, + "loss": 1.6311, + "theoretical_loss": 4.077564864643217, + "tokens_seen": 368574464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178174523570712, + "loss": 1.5638, + "theoretical_loss": 4.077483213750711, + "tokens_seen": 368640000 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717801404212638, + "loss": 1.6408, + "theoretical_loss": 4.077401581436206, + "tokens_seen": 368705536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177853560682046, + "loss": 1.5993, + "theoretical_loss": 4.077319967692173, + "tokens_seen": 368771072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177693079237714, + "loss": 1.5674, + "theoretical_loss": 4.077238372511088, + "tokens_seen": 368836608 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717753259779338, + "loss": 1.5769, + "theoretical_loss": 4.077156795885434, + "tokens_seen": 368902144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177372116349048, + "loss": 1.4508, + "theoretical_loss": 4.077075237807693, + "tokens_seen": 368967680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177211634904714, + "loss": 1.5182, + "theoretical_loss": 4.076993698270357, + "tokens_seen": 369033216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177051153460381, + "loss": 1.5911, + "theoretical_loss": 4.0769121772659185, + "tokens_seen": 369098752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176890672016048, + "loss": 1.5392, + "theoretical_loss": 4.076830674786876, + "tokens_seen": 369164288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176730190571715, + "loss": 1.5293, + "theoretical_loss": 4.076749190825732, + "tokens_seen": 369229824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176569709127382, + "loss": 1.5865, + "theoretical_loss": 4.076667725374993, + "tokens_seen": 369295360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176409227683049, + "loss": 1.5412, + "theoretical_loss": 4.076586278427171, + "tokens_seen": 369360896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176248746238716, + "loss": 1.4512, + "theoretical_loss": 4.076504849974781, + "tokens_seen": 369426432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176088264794383, + "loss": 1.6467, + "theoretical_loss": 4.076423440010343, + "tokens_seen": 369491968 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717592778335005, + "loss": 1.5657, + "theoretical_loss": 4.076342048526382, + "tokens_seen": 369557504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175767301905717, + "loss": 1.5009, + "theoretical_loss": 4.076260675515427, + "tokens_seen": 369623040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175606820461384, + "loss": 1.4412, + "theoretical_loss": 4.07617932097001, + "tokens_seen": 369688576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175446339017051, + "loss": 1.5958, + "theoretical_loss": 4.076097984882669, + "tokens_seen": 369754112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175285857572718, + "loss": 1.5223, + "theoretical_loss": 4.076016667245945, + "tokens_seen": 369819648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175125376128385, + "loss": 1.5297, + "theoretical_loss": 4.075935368052384, + "tokens_seen": 369885184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174964894684052, + "loss": 1.5543, + "theoretical_loss": 4.075854087294538, + "tokens_seen": 369950720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174804413239719, + "loss": 1.4428, + "theoretical_loss": 4.075772824964961, + "tokens_seen": 370016256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174643931795386, + "loss": 1.483, + "theoretical_loss": 4.07569158105621, + "tokens_seen": 370081792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174483450351053, + "loss": 1.5444, + "theoretical_loss": 4.075610355560851, + "tokens_seen": 370147328 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717432296890672, + "loss": 1.5556, + "theoretical_loss": 4.075529148471451, + "tokens_seen": 370212864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 129872, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5275757312774658, + "objective/train/theoretical_loss": 4.075447959780581, + "objective/train/tokens_used": 390738400, + "theoretical_loss": 4.075447959780581, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174162487462387, + "loss": 1.517, + "theoretical_loss": 4.075447959780581, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174002006018054, + "loss": 1.5282, + "theoretical_loss": 4.075366789480818, + "tokens_seen": 370343936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173841524573721, + "loss": 1.5652, + "theoretical_loss": 4.075285637564743, + "tokens_seen": 370409472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173681043129388, + "loss": 1.5969, + "theoretical_loss": 4.07520450402494, + "tokens_seen": 370475008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173520561685055, + "loss": 1.4542, + "theoretical_loss": 4.075123388854, + "tokens_seen": 370540544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173360080240722, + "loss": 1.619, + "theoretical_loss": 4.075042292044515, + "tokens_seen": 370606080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173199598796389, + "loss": 1.5211, + "theoretical_loss": 4.074961213589082, + "tokens_seen": 370671616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173039117352056, + "loss": 1.6464, + "theoretical_loss": 4.074880153480304, + "tokens_seen": 370737152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172878635907723, + "loss": 1.528, + "theoretical_loss": 4.074799111710789, + "tokens_seen": 370802688 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717271815446339, + "loss": 1.5398, + "theoretical_loss": 4.074718088273144, + "tokens_seen": 370868224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172557673019057, + "loss": 1.496, + "theoretical_loss": 4.074637083159988, + "tokens_seen": 370933760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172397191574724, + "loss": 1.4893, + "theoretical_loss": 4.0745560963639385, + "tokens_seen": 370999296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172236710130391, + "loss": 1.5709, + "theoretical_loss": 4.0744751278776175, + "tokens_seen": 371064832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172076228686058, + "loss": 1.6092, + "theoretical_loss": 4.074394177693655, + "tokens_seen": 371130368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171915747241725, + "loss": 1.464, + "theoretical_loss": 4.074313245804683, + "tokens_seen": 371195904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171755265797392, + "loss": 1.5461, + "theoretical_loss": 4.074232332203336, + "tokens_seen": 371261440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171594784353059, + "loss": 1.4682, + "theoretical_loss": 4.074151436882256, + "tokens_seen": 371326976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171434302908726, + "loss": 1.5951, + "theoretical_loss": 4.074070559834088, + "tokens_seen": 371392512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171273821464393, + "loss": 1.5379, + "theoretical_loss": 4.073989701051481, + "tokens_seen": 371458048 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717111334002006, + "loss": 1.5889, + "theoretical_loss": 4.073908860527087, + "tokens_seen": 371523584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170952858575728, + "loss": 1.5812, + "theoretical_loss": 4.0738280382535645, + "tokens_seen": 371589120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170792377131395, + "loss": 1.403, + "theoretical_loss": 4.0737472342235765, + "tokens_seen": 371654656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170631895687062, + "loss": 1.5452, + "theoretical_loss": 4.073666448429787, + "tokens_seen": 371720192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170471414242729, + "loss": 1.5192, + "theoretical_loss": 4.073585680864868, + "tokens_seen": 371785728 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170310932798396, + "loss": 1.4643, + "theoretical_loss": 4.073504931521493, + "tokens_seen": 371851264 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170150451354063, + "loss": 1.5878, + "theoretical_loss": 4.073424200392342, + "tokens_seen": 371916800 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716998996990973, + "loss": 1.6744, + "theoretical_loss": 4.073343487470097, + "tokens_seen": 371982336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169829488465397, + "loss": 1.5849, + "theoretical_loss": 4.073262792747445, + "tokens_seen": 372047872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169669007021064, + "loss": 1.5167, + "theoretical_loss": 4.073182116217079, + "tokens_seen": 372113408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169508525576731, + "loss": 1.5442, + "theoretical_loss": 4.073101457871693, + "tokens_seen": 372178944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169348044132398, + "loss": 1.5109, + "theoretical_loss": 4.073020817703988, + "tokens_seen": 372244480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169187562688065, + "loss": 1.5497, + "theoretical_loss": 4.072940195706668, + "tokens_seen": 372310016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169027081243732, + "loss": 1.5475, + "theoretical_loss": 4.072859591872441, + "tokens_seen": 372375552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168866599799399, + "loss": 1.5128, + "theoretical_loss": 4.07277900619402, + "tokens_seen": 372441088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168706118355066, + "loss": 1.5664, + "theoretical_loss": 4.072698438664121, + "tokens_seen": 372506624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168545636910733, + "loss": 1.5994, + "theoretical_loss": 4.072617889275467, + "tokens_seen": 372572160 + }, + { + "epoch": 0.11, + "learning_rate": 0.00071683851554664, + "loss": 1.5367, + "theoretical_loss": 4.07253735802078, + "tokens_seen": 372637696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168224674022067, + "loss": 1.595, + "theoretical_loss": 4.072456844892792, + "tokens_seen": 372703232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168064192577734, + "loss": 1.5945, + "theoretical_loss": 4.072376349884236, + "tokens_seen": 372768768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167903711133401, + "loss": 1.4935, + "theoretical_loss": 4.072295872987848, + "tokens_seen": 372834304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167743229689068, + "loss": 1.5228, + "theoretical_loss": 4.072215414196373, + "tokens_seen": 372899840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167582748244735, + "loss": 1.5967, + "theoretical_loss": 4.072134973502554, + "tokens_seen": 372965376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167422266800402, + "loss": 1.5616, + "theoretical_loss": 4.072054550899144, + "tokens_seen": 373030912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167261785356069, + "loss": 1.4757, + "theoretical_loss": 4.071974146378896, + "tokens_seen": 373096448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167101303911736, + "loss": 1.5998, + "theoretical_loss": 4.071893759934568, + "tokens_seen": 373161984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166940822467403, + "loss": 1.4561, + "theoretical_loss": 4.0718133915589245, + "tokens_seen": 373227520 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716678034102307, + "loss": 1.5062, + "theoretical_loss": 4.071733041244732, + "tokens_seen": 373293056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166619859578737, + "loss": 1.5552, + "theoretical_loss": 4.071652708984761, + "tokens_seen": 373358592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166459378134404, + "loss": 1.4439, + "theoretical_loss": 4.0715723947717875, + "tokens_seen": 373424128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166298896690071, + "loss": 1.5569, + "theoretical_loss": 4.07149209859859, + "tokens_seen": 373489664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 131043, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4238359928131104, + "objective/train/theoretical_loss": 4.0714118204579535, + "objective/train/tokens_used": 394015200, + "theoretical_loss": 4.0714118204579535, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166138415245738, + "loss": 1.432, + "theoretical_loss": 4.0714118204579535, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165977933801405, + "loss": 1.4711, + "theoretical_loss": 4.0713315603426645, + "tokens_seen": 373620736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165817452357072, + "loss": 1.7222, + "theoretical_loss": 4.071251318245516, + "tokens_seen": 373686272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165656970912739, + "loss": 1.4663, + "theoretical_loss": 4.071171094159304, + "tokens_seen": 373751808 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165496489468406, + "loss": 1.5914, + "theoretical_loss": 4.071090888076828, + "tokens_seen": 373817344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165336008024073, + "loss": 1.4531, + "theoretical_loss": 4.071010699990892, + "tokens_seen": 373882880 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716517552657974, + "loss": 1.5228, + "theoretical_loss": 4.070930529894306, + "tokens_seen": 373948416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165015045135407, + "loss": 1.6131, + "theoretical_loss": 4.070850377779882, + "tokens_seen": 374013952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164854563691074, + "loss": 1.578, + "theoretical_loss": 4.070770243640436, + "tokens_seen": 374079488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164694082246741, + "loss": 1.6162, + "theoretical_loss": 4.07069012746879, + "tokens_seen": 374145024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164533600802407, + "loss": 1.4642, + "theoretical_loss": 4.070610029257768, + "tokens_seen": 374210560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164373119358075, + "loss": 1.596, + "theoretical_loss": 4.0705299490002, + "tokens_seen": 374276096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164212637913741, + "loss": 1.5887, + "theoretical_loss": 4.070449886688919, + "tokens_seen": 374341632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164052156469409, + "loss": 1.466, + "theoretical_loss": 4.0703698423167625, + "tokens_seen": 374407168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163891675025075, + "loss": 1.55, + "theoretical_loss": 4.070289815876571, + "tokens_seen": 374472704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163731193580743, + "loss": 1.6113, + "theoretical_loss": 4.070209807361191, + "tokens_seen": 374538240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163570712136409, + "loss": 1.5242, + "theoretical_loss": 4.0701298167634725, + "tokens_seen": 374603776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163410230692077, + "loss": 1.5492, + "theoretical_loss": 4.0700498440762685, + "tokens_seen": 374669312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163249749247743, + "loss": 1.5498, + "theoretical_loss": 4.069969889292437, + "tokens_seen": 374734848 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716308926780341, + "loss": 1.6181, + "theoretical_loss": 4.069889952404841, + "tokens_seen": 374800384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162928786359077, + "loss": 1.4905, + "theoretical_loss": 4.069810033406345, + "tokens_seen": 374865920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162768304914744, + "loss": 1.6594, + "theoretical_loss": 4.06973013228982, + "tokens_seen": 374931456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162607823470411, + "loss": 1.59, + "theoretical_loss": 4.0696502490481405, + "tokens_seen": 374996992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162447342026078, + "loss": 1.5982, + "theoretical_loss": 4.069570383674185, + "tokens_seen": 375062528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162286860581745, + "loss": 1.5867, + "theoretical_loss": 4.069490536160835, + "tokens_seen": 375128064 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162126379137412, + "loss": 1.5181, + "theoretical_loss": 4.069410706500978, + "tokens_seen": 375193600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161965897693079, + "loss": 1.5475, + "theoretical_loss": 4.0693308946875035, + "tokens_seen": 375259136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161805416248746, + "loss": 1.5137, + "theoretical_loss": 4.069251100713307, + "tokens_seen": 375324672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161644934804413, + "loss": 1.558, + "theoretical_loss": 4.069171324571288, + "tokens_seen": 375390208 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716148445336008, + "loss": 1.5597, + "theoretical_loss": 4.069091566254347, + "tokens_seen": 375455744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161323971915747, + "loss": 1.5059, + "theoretical_loss": 4.069011825755393, + "tokens_seen": 375521280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161163490471414, + "loss": 1.4777, + "theoretical_loss": 4.068932103067335, + "tokens_seen": 375586816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161003009027081, + "loss": 1.5031, + "theoretical_loss": 4.06885239818309, + "tokens_seen": 375652352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160842527582748, + "loss": 1.5192, + "theoretical_loss": 4.068772711095575, + "tokens_seen": 375717888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160682046138415, + "loss": 1.6938, + "theoretical_loss": 4.068693041797715, + "tokens_seen": 375783424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160521564694082, + "loss": 1.6846, + "theoretical_loss": 4.068613390282437, + "tokens_seen": 375848960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160361083249749, + "loss": 1.6436, + "theoretical_loss": 4.06853375654267, + "tokens_seen": 375914496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160200601805416, + "loss": 1.4518, + "theoretical_loss": 4.068454140571351, + "tokens_seen": 375980032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160040120361083, + "loss": 1.6677, + "theoretical_loss": 4.068374542361418, + "tokens_seen": 376045568 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715987963891675, + "loss": 1.6108, + "theoretical_loss": 4.068294961905817, + "tokens_seen": 376111104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159719157472417, + "loss": 1.5951, + "theoretical_loss": 4.068215399197492, + "tokens_seen": 376176640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159558676028084, + "loss": 1.6626, + "theoretical_loss": 4.068135854229396, + "tokens_seen": 376242176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159398194583751, + "loss": 1.5851, + "theoretical_loss": 4.068056326994485, + "tokens_seen": 376307712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159237713139418, + "loss": 1.549, + "theoretical_loss": 4.067976817485717, + "tokens_seen": 376373248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159077231695085, + "loss": 1.534, + "theoretical_loss": 4.067897325696056, + "tokens_seen": 376438784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158916750250753, + "loss": 1.6247, + "theoretical_loss": 4.06781785161847, + "tokens_seen": 376504320 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715875626880642, + "loss": 1.574, + "theoretical_loss": 4.067738395245929, + "tokens_seen": 376569856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158595787362087, + "loss": 1.5068, + "theoretical_loss": 4.067658956571409, + "tokens_seen": 376635392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158435305917753, + "loss": 1.56, + "theoretical_loss": 4.067579535587891, + "tokens_seen": 376700928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158274824473421, + "loss": 1.5091, + "theoretical_loss": 4.067500132288357, + "tokens_seen": 376766464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 131935, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4308035373687744, + "objective/train/theoretical_loss": 4.067420746665794, + "objective/train/tokens_used": 397292000, + "theoretical_loss": 4.067420746665794, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158114343029087, + "loss": 1.5559, + "theoretical_loss": 4.067420746665794, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157953861584755, + "loss": 1.5028, + "theoretical_loss": 4.067341378713195, + "tokens_seen": 376897536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157793380140421, + "loss": 1.5996, + "theoretical_loss": 4.0672620284235546, + "tokens_seen": 376963072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157632898696089, + "loss": 1.5491, + "theoretical_loss": 4.067182695789872, + "tokens_seen": 377028608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157472417251756, + "loss": 1.5053, + "theoretical_loss": 4.06710338080515, + "tokens_seen": 377094144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157311935807423, + "loss": 1.5987, + "theoretical_loss": 4.067024083462399, + "tokens_seen": 377159680 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715715145436309, + "loss": 1.4151, + "theoretical_loss": 4.066944803754628, + "tokens_seen": 377225216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156990972918757, + "loss": 1.4533, + "theoretical_loss": 4.0668655416748525, + "tokens_seen": 377290752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156830491474424, + "loss": 1.5566, + "theoretical_loss": 4.066786297216092, + "tokens_seen": 377356288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156670010030091, + "loss": 1.4728, + "theoretical_loss": 4.066707070371371, + "tokens_seen": 377421824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156509528585758, + "loss": 1.5469, + "theoretical_loss": 4.066627861133716, + "tokens_seen": 377487360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156349047141425, + "loss": 1.4933, + "theoretical_loss": 4.066548669496157, + "tokens_seen": 377552896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156188565697092, + "loss": 1.5501, + "theoretical_loss": 4.066469495451732, + "tokens_seen": 377618432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156028084252759, + "loss": 1.6296, + "theoretical_loss": 4.066390338993479, + "tokens_seen": 377683968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155867602808426, + "loss": 1.6332, + "theoretical_loss": 4.0663112001144395, + "tokens_seen": 377749504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155707121364093, + "loss": 1.6292, + "theoretical_loss": 4.066232078807664, + "tokens_seen": 377815040 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715554663991976, + "loss": 1.5593, + "theoretical_loss": 4.0661529750662, + "tokens_seen": 377880576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155386158475427, + "loss": 1.5156, + "theoretical_loss": 4.066073888883105, + "tokens_seen": 377946112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155225677031094, + "loss": 1.5514, + "theoretical_loss": 4.065994820251438, + "tokens_seen": 378011648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155065195586761, + "loss": 1.4833, + "theoretical_loss": 4.065915769164262, + "tokens_seen": 378077184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154904714142428, + "loss": 1.6372, + "theoretical_loss": 4.065836735614642, + "tokens_seen": 378142720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154744232698095, + "loss": 1.6646, + "theoretical_loss": 4.06575771959565, + "tokens_seen": 378208256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154583751253762, + "loss": 1.6973, + "theoretical_loss": 4.065678721100362, + "tokens_seen": 378273792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154423269809429, + "loss": 1.6011, + "theoretical_loss": 4.065599740121855, + "tokens_seen": 378339328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154262788365096, + "loss": 1.5382, + "theoretical_loss": 4.065520776653212, + "tokens_seen": 378404864 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154102306920763, + "loss": 1.5479, + "theoretical_loss": 4.06544183068752, + "tokens_seen": 378470400 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715394182547643, + "loss": 1.5046, + "theoretical_loss": 4.0653629022178706, + "tokens_seen": 378535936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153781344032097, + "loss": 1.502, + "theoretical_loss": 4.065283991237355, + "tokens_seen": 378601472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153620862587764, + "loss": 1.553, + "theoretical_loss": 4.065205097739075, + "tokens_seen": 378667008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153460381143431, + "loss": 1.5149, + "theoretical_loss": 4.065126221716131, + "tokens_seen": 378732544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153299899699098, + "loss": 1.5736, + "theoretical_loss": 4.0650473631616295, + "tokens_seen": 378798080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153139418254765, + "loss": 1.5831, + "theoretical_loss": 4.064968522068681, + "tokens_seen": 378863616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152978936810432, + "loss": 1.6488, + "theoretical_loss": 4.0648896984304, + "tokens_seen": 378929152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152818455366099, + "loss": 1.5114, + "theoretical_loss": 4.064810892239903, + "tokens_seen": 378994688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152657973921766, + "loss": 1.4931, + "theoretical_loss": 4.064732103490312, + "tokens_seen": 379060224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152497492477433, + "loss": 1.5598, + "theoretical_loss": 4.064653332174753, + "tokens_seen": 379125760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00071523370110331, + "loss": 1.6029, + "theoretical_loss": 4.064574578286356, + "tokens_seen": 379191296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152176529588767, + "loss": 1.6075, + "theoretical_loss": 4.064495841818255, + "tokens_seen": 379256832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152016048144434, + "loss": 1.5738, + "theoretical_loss": 4.064417122763587, + "tokens_seen": 379322368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007151855566700101, + "loss": 1.5304, + "theoretical_loss": 4.064338421115491, + "tokens_seen": 379387904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007151695085255768, + "loss": 1.5089, + "theoretical_loss": 4.064259736867116, + "tokens_seen": 379453440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007151534603811435, + "loss": 1.5295, + "theoretical_loss": 4.0641810700116086, + "tokens_seen": 379518976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007151374122367102, + "loss": 1.553, + "theoretical_loss": 4.064102420542122, + "tokens_seen": 379584512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007151213640922769, + "loss": 1.5785, + "theoretical_loss": 4.064023788451814, + "tokens_seen": 379650048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007151053159478436, + "loss": 1.5725, + "theoretical_loss": 4.063945173733844, + "tokens_seen": 379715584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007150892678034103, + "loss": 1.3953, + "theoretical_loss": 4.063866576381377, + "tokens_seen": 379781120 + }, + { + "epoch": 0.12, + "learning_rate": 0.000715073219658977, + "loss": 1.4806, + "theoretical_loss": 4.063787996387583, + "tokens_seen": 379846656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007150571715145436, + "loss": 1.5389, + "theoretical_loss": 4.063709433745632, + "tokens_seen": 379912192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007150411233701104, + "loss": 1.5711, + "theoretical_loss": 4.063630888448701, + "tokens_seen": 379977728 + }, + { + "epoch": 0.12, + "learning_rate": 0.000715025075225677, + "loss": 1.6166, + "theoretical_loss": 4.06355236048997, + "tokens_seen": 380043264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 133175, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.637108325958252, + "objective/train/theoretical_loss": 4.063473849862623, + "objective/train/tokens_used": 400568800, + "theoretical_loss": 4.063473849862623, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007150090270812438, + "loss": 1.6212, + "theoretical_loss": 4.063473849862623, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149929789368104, + "loss": 1.4958, + "theoretical_loss": 4.063395356559848, + "tokens_seen": 380174336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149769307923772, + "loss": 1.5352, + "theoretical_loss": 4.063316880574836, + "tokens_seen": 380239872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149608826479438, + "loss": 1.5525, + "theoretical_loss": 4.063238421900783, + "tokens_seen": 380305408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149448345035106, + "loss": 1.6395, + "theoretical_loss": 4.063159980530887, + "tokens_seen": 380370944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149287863590772, + "loss": 1.5751, + "theoretical_loss": 4.063081556458353, + "tokens_seen": 380436480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149127382146439, + "loss": 1.5307, + "theoretical_loss": 4.063003149676387, + "tokens_seen": 380502016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148966900702106, + "loss": 1.6268, + "theoretical_loss": 4.062924760178198, + "tokens_seen": 380567552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148806419257773, + "loss": 1.4769, + "theoretical_loss": 4.062846387957005, + "tokens_seen": 380633088 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714864593781344, + "loss": 1.4427, + "theoretical_loss": 4.062768033006023, + "tokens_seen": 380698624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148485456369107, + "loss": 1.5785, + "theoretical_loss": 4.062689695318475, + "tokens_seen": 380764160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148324974924774, + "loss": 1.7042, + "theoretical_loss": 4.062611374887587, + "tokens_seen": 380829696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148164493480441, + "loss": 1.4121, + "theoretical_loss": 4.062533071706589, + "tokens_seen": 380895232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148004012036108, + "loss": 1.529, + "theoretical_loss": 4.062454785768716, + "tokens_seen": 380960768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147843530591775, + "loss": 1.4973, + "theoretical_loss": 4.062376517067204, + "tokens_seen": 381026304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147683049147442, + "loss": 1.5547, + "theoretical_loss": 4.062298265595294, + "tokens_seen": 381091840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147522567703109, + "loss": 1.4552, + "theoretical_loss": 4.062220031346234, + "tokens_seen": 381157376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147362086258776, + "loss": 1.5088, + "theoretical_loss": 4.06214181431327, + "tokens_seen": 381222912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147201604814443, + "loss": 1.5998, + "theoretical_loss": 4.062063614489656, + "tokens_seen": 381288448 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714704112337011, + "loss": 1.3875, + "theoretical_loss": 4.061985431868648, + "tokens_seen": 381353984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146880641925778, + "loss": 1.5849, + "theoretical_loss": 4.061907266443508, + "tokens_seen": 381419520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146720160481444, + "loss": 1.5982, + "theoretical_loss": 4.061829118207498, + "tokens_seen": 381485056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146559679037112, + "loss": 1.5771, + "theoretical_loss": 4.061750987153888, + "tokens_seen": 381550592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146399197592778, + "loss": 1.6583, + "theoretical_loss": 4.061672873275949, + "tokens_seen": 381616128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146238716148446, + "loss": 1.5146, + "theoretical_loss": 4.061594776566958, + "tokens_seen": 381681664 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146078234704112, + "loss": 1.6203, + "theoretical_loss": 4.061516697020191, + "tokens_seen": 381747200 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714591775325978, + "loss": 1.4899, + "theoretical_loss": 4.061438634628933, + "tokens_seen": 381812736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007145757271815446, + "loss": 1.4785, + "theoretical_loss": 4.061360589386473, + "tokens_seen": 381878272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007145596790371114, + "loss": 1.5746, + "theoretical_loss": 4.061282561286098, + "tokens_seen": 381943808 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714543630892678, + "loss": 1.4623, + "theoretical_loss": 4.061204550321104, + "tokens_seen": 382009344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007145275827482448, + "loss": 1.4579, + "theoretical_loss": 4.061126556484791, + "tokens_seen": 382074880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007145115346038114, + "loss": 1.7142, + "theoretical_loss": 4.06104857977046, + "tokens_seen": 382140416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144954864593782, + "loss": 1.4523, + "theoretical_loss": 4.060970620171415, + "tokens_seen": 382205952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144794383149448, + "loss": 1.569, + "theoretical_loss": 4.060892677680967, + "tokens_seen": 382271488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144633901705116, + "loss": 1.56, + "theoretical_loss": 4.06081475229243, + "tokens_seen": 382337024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144473420260782, + "loss": 1.595, + "theoretical_loss": 4.060736843999121, + "tokens_seen": 382402560 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714431293881645, + "loss": 1.4555, + "theoretical_loss": 4.060658952794359, + "tokens_seen": 382468096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144152457372116, + "loss": 1.5569, + "theoretical_loss": 4.06058107867147, + "tokens_seen": 382533632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143991975927784, + "loss": 1.5, + "theoretical_loss": 4.060503221623783, + "tokens_seen": 382599168 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714383149448345, + "loss": 1.5665, + "theoretical_loss": 4.060425381644629, + "tokens_seen": 382664704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143671013039118, + "loss": 1.5303, + "theoretical_loss": 4.060347558727344, + "tokens_seen": 382730240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143510531594785, + "loss": 1.5049, + "theoretical_loss": 4.060269752865268, + "tokens_seen": 382795776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143350050150452, + "loss": 1.5552, + "theoretical_loss": 4.060191964051744, + "tokens_seen": 382861312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143189568706119, + "loss": 1.3659, + "theoretical_loss": 4.060114192280119, + "tokens_seen": 382926848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143029087261786, + "loss": 1.5707, + "theoretical_loss": 4.0600364375437445, + "tokens_seen": 382992384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142868605817453, + "loss": 1.644, + "theoretical_loss": 4.059958699835974, + "tokens_seen": 383057920 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714270812437312, + "loss": 1.5578, + "theoretical_loss": 4.059880979150167, + "tokens_seen": 383123456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142547642928787, + "loss": 1.4528, + "theoretical_loss": 4.059803275479684, + "tokens_seen": 383188992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142387161484454, + "loss": 1.5817, + "theoretical_loss": 4.059725588817891, + "tokens_seen": 383254528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142226680040121, + "loss": 1.4266, + "theoretical_loss": 4.0596479191581585, + "tokens_seen": 383320064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 134130, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6504217386245728, + "objective/train/theoretical_loss": 4.059570266493859, + "objective/train/tokens_used": 403845600, + "theoretical_loss": 4.059570266493859, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142066198595788, + "loss": 1.6386, + "theoretical_loss": 4.059570266493859, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141905717151455, + "loss": 1.5463, + "theoretical_loss": 4.059492630818369, + "tokens_seen": 383451136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141745235707122, + "loss": 1.6065, + "theoretical_loss": 4.05941501212507, + "tokens_seen": 383516672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141584754262789, + "loss": 1.4486, + "theoretical_loss": 4.059337410407346, + "tokens_seen": 383582208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141424272818456, + "loss": 1.5162, + "theoretical_loss": 4.059259825658584, + "tokens_seen": 383647744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141263791374123, + "loss": 1.5191, + "theoretical_loss": 4.059182257872177, + "tokens_seen": 383713280 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714110330992979, + "loss": 1.6177, + "theoretical_loss": 4.059104707041519, + "tokens_seen": 383778816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140942828485457, + "loss": 1.4974, + "theoretical_loss": 4.059027173160011, + "tokens_seen": 383844352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140782347041124, + "loss": 1.5206, + "theoretical_loss": 4.0589496562210545, + "tokens_seen": 383909888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140621865596791, + "loss": 1.5041, + "theoretical_loss": 4.058872156218055, + "tokens_seen": 383975424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140461384152458, + "loss": 1.4481, + "theoretical_loss": 4.058794673144425, + "tokens_seen": 384040960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140300902708125, + "loss": 1.4762, + "theoretical_loss": 4.0587172069935775, + "tokens_seen": 384106496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140140421263792, + "loss": 1.5379, + "theoretical_loss": 4.05863975775893, + "tokens_seen": 384172032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139979939819459, + "loss": 1.6316, + "theoretical_loss": 4.058562325433902, + "tokens_seen": 384237568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139819458375126, + "loss": 1.4715, + "theoretical_loss": 4.058484910011922, + "tokens_seen": 384303104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139658976930793, + "loss": 1.535, + "theoretical_loss": 4.058407511486415, + "tokens_seen": 384368640 + }, + { + "epoch": 0.12, + "learning_rate": 0.000713949849548646, + "loss": 1.591, + "theoretical_loss": 4.058330129850816, + "tokens_seen": 384434176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139338014042127, + "loss": 1.4796, + "theoretical_loss": 4.058252765098558, + "tokens_seen": 384499712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139177532597794, + "loss": 1.3905, + "theoretical_loss": 4.058175417223084, + "tokens_seen": 384565248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139017051153461, + "loss": 1.6414, + "theoretical_loss": 4.058098086217836, + "tokens_seen": 384630784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138856569709128, + "loss": 1.4973, + "theoretical_loss": 4.058020772076259, + "tokens_seen": 384696320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138696088264795, + "loss": 1.6776, + "theoretical_loss": 4.057943474791806, + "tokens_seen": 384761856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138535606820462, + "loss": 1.4589, + "theoretical_loss": 4.0578661943579295, + "tokens_seen": 384827392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138375125376129, + "loss": 1.5205, + "theoretical_loss": 4.05778893076809, + "tokens_seen": 384892928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138214643931796, + "loss": 1.6148, + "theoretical_loss": 4.057711684015747, + "tokens_seen": 384958464 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138054162487463, + "loss": 1.4956, + "theoretical_loss": 4.057634454094365, + "tokens_seen": 385024000 + }, + { + "epoch": 0.12, + "learning_rate": 0.000713789368104313, + "loss": 1.5085, + "theoretical_loss": 4.057557240997415, + "tokens_seen": 385089536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137733199598797, + "loss": 1.4795, + "theoretical_loss": 4.057480044718368, + "tokens_seen": 385155072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137572718154464, + "loss": 1.533, + "theoretical_loss": 4.057402865250701, + "tokens_seen": 385220608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137412236710131, + "loss": 1.5496, + "theoretical_loss": 4.057325702587893, + "tokens_seen": 385286144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137251755265798, + "loss": 1.6184, + "theoretical_loss": 4.057248556723429, + "tokens_seen": 385351680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137091273821465, + "loss": 1.5929, + "theoretical_loss": 4.057171427650793, + "tokens_seen": 385417216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136930792377132, + "loss": 1.5673, + "theoretical_loss": 4.0570943153634795, + "tokens_seen": 385482752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136770310932799, + "loss": 1.4974, + "theoretical_loss": 4.0570172198549805, + "tokens_seen": 385548288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136609829488466, + "loss": 1.5252, + "theoretical_loss": 4.056940141118794, + "tokens_seen": 385613824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136449348044133, + "loss": 1.5163, + "theoretical_loss": 4.056863079148422, + "tokens_seen": 385679360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136288866599799, + "loss": 1.5341, + "theoretical_loss": 4.05678603393737, + "tokens_seen": 385744896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136128385155467, + "loss": 1.5027, + "theoretical_loss": 4.056709005479147, + "tokens_seen": 385810432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135967903711133, + "loss": 1.4236, + "theoretical_loss": 4.056631993767264, + "tokens_seen": 385875968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135807422266801, + "loss": 1.6033, + "theoretical_loss": 4.056554998795239, + "tokens_seen": 385941504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135646940822467, + "loss": 1.5528, + "theoretical_loss": 4.05647802055659, + "tokens_seen": 386007040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135486459378135, + "loss": 1.6238, + "theoretical_loss": 4.056401059044841, + "tokens_seen": 386072576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135325977933801, + "loss": 1.5508, + "theoretical_loss": 4.056324114253519, + "tokens_seen": 386138112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135165496489468, + "loss": 1.4138, + "theoretical_loss": 4.056247186176154, + "tokens_seen": 386203648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135005015045135, + "loss": 1.5629, + "theoretical_loss": 4.056170274806281, + "tokens_seen": 386269184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134844533600803, + "loss": 1.5687, + "theoretical_loss": 4.056093380137436, + "tokens_seen": 386334720 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134684052156469, + "loss": 1.5048, + "theoretical_loss": 4.056016502163162, + "tokens_seen": 386400256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134523570712137, + "loss": 1.6054, + "theoretical_loss": 4.055939640877003, + "tokens_seen": 386465792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134363089267803, + "loss": 1.5434, + "theoretical_loss": 4.055862796272508, + "tokens_seen": 386531328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134202607823471, + "loss": 1.4143, + "theoretical_loss": 4.055785968343228, + "tokens_seen": 386596864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 135378, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6471995115280151, + "objective/train/theoretical_loss": 4.05570915708272, + "objective/train/tokens_used": 407122400, + "theoretical_loss": 4.05570915708272, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134042126379137, + "loss": 1.5954, + "theoretical_loss": 4.05570915708272, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133881644934805, + "loss": 1.367, + "theoretical_loss": 4.055632362484542, + "tokens_seen": 386727936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133721163490471, + "loss": 1.5976, + "theoretical_loss": 4.055555584542257, + "tokens_seen": 386793472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133560682046139, + "loss": 1.4761, + "theoretical_loss": 4.055478823249432, + "tokens_seen": 386859008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133400200601805, + "loss": 1.4676, + "theoretical_loss": 4.055402078599638, + "tokens_seen": 386924544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133239719157473, + "loss": 1.4819, + "theoretical_loss": 4.055325350586445, + "tokens_seen": 386990080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133079237713139, + "loss": 1.3562, + "theoretical_loss": 4.055248639203433, + "tokens_seen": 387055616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132918756268807, + "loss": 1.561, + "theoretical_loss": 4.055171944444182, + "tokens_seen": 387121152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132758274824473, + "loss": 1.5905, + "theoretical_loss": 4.055095266302276, + "tokens_seen": 387186688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132597793380141, + "loss": 1.5024, + "theoretical_loss": 4.055018604771302, + "tokens_seen": 387252224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132437311935807, + "loss": 1.5481, + "theoretical_loss": 4.054941959844853, + "tokens_seen": 387317760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132276830491475, + "loss": 1.5958, + "theoretical_loss": 4.054865331516523, + "tokens_seen": 387383296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132116349047141, + "loss": 1.4962, + "theoretical_loss": 4.054788719779911, + "tokens_seen": 387448832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131955867602809, + "loss": 1.6014, + "theoretical_loss": 4.054712124628617, + "tokens_seen": 387514368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131795386158475, + "loss": 1.569, + "theoretical_loss": 4.054635546056248, + "tokens_seen": 387579904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131634904714143, + "loss": 1.4515, + "theoretical_loss": 4.054558984056414, + "tokens_seen": 387645440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131474423269809, + "loss": 1.509, + "theoretical_loss": 4.054482438622727, + "tokens_seen": 387710976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131313941825477, + "loss": 1.3481, + "theoretical_loss": 4.0544059097488026, + "tokens_seen": 387776512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131153460381143, + "loss": 1.5887, + "theoretical_loss": 4.05432939742826, + "tokens_seen": 387842048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130992978936811, + "loss": 1.5527, + "theoretical_loss": 4.054252901654724, + "tokens_seen": 387907584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130832497492477, + "loss": 1.5288, + "theoretical_loss": 4.054176422421821, + "tokens_seen": 387973120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130672016048145, + "loss": 1.575, + "theoretical_loss": 4.05409995972318, + "tokens_seen": 388038656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130511534603811, + "loss": 1.54, + "theoretical_loss": 4.054023513552437, + "tokens_seen": 388104192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130351053159479, + "loss": 1.476, + "theoretical_loss": 4.0539470839032266, + "tokens_seen": 388169728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130190571715146, + "loss": 1.5659, + "theoretical_loss": 4.053870670769192, + "tokens_seen": 388235264 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130030090270813, + "loss": 1.5637, + "theoretical_loss": 4.053794274143977, + "tokens_seen": 388300800 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712986960882648, + "loss": 1.5868, + "theoretical_loss": 4.053717894021229, + "tokens_seen": 388366336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129709127382147, + "loss": 1.4002, + "theoretical_loss": 4.0536415303946, + "tokens_seen": 388431872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129548645937814, + "loss": 1.591, + "theoretical_loss": 4.053565183257745, + "tokens_seen": 388497408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129388164493481, + "loss": 1.5506, + "theoretical_loss": 4.053488852604322, + "tokens_seen": 388562944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129227683049148, + "loss": 1.483, + "theoretical_loss": 4.053412538427993, + "tokens_seen": 388628480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129067201604815, + "loss": 1.6479, + "theoretical_loss": 4.053336240722423, + "tokens_seen": 388694016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128906720160482, + "loss": 1.4564, + "theoretical_loss": 4.053259959481283, + "tokens_seen": 388759552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128746238716149, + "loss": 1.532, + "theoretical_loss": 4.053183694698244, + "tokens_seen": 388825088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128585757271816, + "loss": 1.4543, + "theoretical_loss": 4.053107446366981, + "tokens_seen": 388890624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128425275827483, + "loss": 1.7257, + "theoretical_loss": 4.053031214481175, + "tokens_seen": 388956160 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712826479438315, + "loss": 1.5432, + "theoretical_loss": 4.052954999034509, + "tokens_seen": 389021696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128104312938817, + "loss": 1.3693, + "theoretical_loss": 4.052878800020668, + "tokens_seen": 389087232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127943831494484, + "loss": 1.5255, + "theoretical_loss": 4.052802617433344, + "tokens_seen": 389152768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127783350050151, + "loss": 1.5014, + "theoretical_loss": 4.0527264512662295, + "tokens_seen": 389218304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127622868605818, + "loss": 1.5613, + "theoretical_loss": 4.052650301513021, + "tokens_seen": 389283840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127462387161485, + "loss": 1.4304, + "theoretical_loss": 4.0525741681674186, + "tokens_seen": 389349376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127301905717152, + "loss": 1.4755, + "theoretical_loss": 4.052498051223127, + "tokens_seen": 389414912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127141424272819, + "loss": 1.7319, + "theoretical_loss": 4.052421950673853, + "tokens_seen": 389480448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126980942828486, + "loss": 1.6418, + "theoretical_loss": 4.052345866513308, + "tokens_seen": 389545984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126820461384153, + "loss": 1.4938, + "theoretical_loss": 4.052269798735207, + "tokens_seen": 389611520 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712665997993982, + "loss": 1.4466, + "theoretical_loss": 4.052193747333265, + "tokens_seen": 389677056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126499498495487, + "loss": 1.6005, + "theoretical_loss": 4.052117712301206, + "tokens_seen": 389742592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126339017051154, + "loss": 1.4522, + "theoretical_loss": 4.052041693632754, + "tokens_seen": 389808128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126178535606821, + "loss": 1.5045, + "theoretical_loss": 4.0519656913216355, + "tokens_seen": 389873664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 136223, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4804840087890625, + "objective/train/theoretical_loss": 4.051889705361584, + "objective/train/tokens_used": 410399200, + "theoretical_loss": 4.051889705361584, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126018054162488, + "loss": 1.5287, + "theoretical_loss": 4.051889705361584, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125857572718155, + "loss": 1.4526, + "theoretical_loss": 4.051813735746334, + "tokens_seen": 390004736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125697091273822, + "loss": 1.6437, + "theoretical_loss": 4.051737782469624, + "tokens_seen": 390070272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125536609829489, + "loss": 1.6544, + "theoretical_loss": 4.051661845525196, + "tokens_seen": 390135808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125376128385156, + "loss": 1.4723, + "theoretical_loss": 4.051585924906796, + "tokens_seen": 390201344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125215646940823, + "loss": 1.7279, + "theoretical_loss": 4.051510020608171, + "tokens_seen": 390266880 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712505516549649, + "loss": 1.5472, + "theoretical_loss": 4.051434132623075, + "tokens_seen": 390332416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124894684052157, + "loss": 1.699, + "theoretical_loss": 4.051358260945262, + "tokens_seen": 390397952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124734202607824, + "loss": 1.6237, + "theoretical_loss": 4.051282405568494, + "tokens_seen": 390463488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124573721163491, + "loss": 1.5859, + "theoretical_loss": 4.051206566486531, + "tokens_seen": 390529024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124413239719158, + "loss": 1.5356, + "theoretical_loss": 4.0511307436931405, + "tokens_seen": 390594560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124252758274825, + "loss": 1.558, + "theoretical_loss": 4.051054937182092, + "tokens_seen": 390660096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124092276830492, + "loss": 1.5667, + "theoretical_loss": 4.0509791469471566, + "tokens_seen": 390725632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123931795386159, + "loss": 1.5082, + "theoretical_loss": 4.050903372982113, + "tokens_seen": 390791168 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123771313941826, + "loss": 1.6031, + "theoretical_loss": 4.050827615280739, + "tokens_seen": 390856704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123610832497493, + "loss": 1.5781, + "theoretical_loss": 4.050751873836819, + "tokens_seen": 390922240 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712345035105316, + "loss": 1.4028, + "theoretical_loss": 4.050676148644139, + "tokens_seen": 390987776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123289869608827, + "loss": 1.6489, + "theoretical_loss": 4.050600439696489, + "tokens_seen": 391053312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123129388164494, + "loss": 1.473, + "theoretical_loss": 4.050524746987664, + "tokens_seen": 391118848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122968906720161, + "loss": 1.5548, + "theoretical_loss": 4.050449070511458, + "tokens_seen": 391184384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122808425275828, + "loss": 1.5028, + "theoretical_loss": 4.050373410261673, + "tokens_seen": 391249920 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122647943831495, + "loss": 1.5715, + "theoretical_loss": 4.050297766232113, + "tokens_seen": 391315456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122487462387162, + "loss": 1.7176, + "theoretical_loss": 4.050222138416583, + "tokens_seen": 391380992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122326980942828, + "loss": 1.5573, + "theoretical_loss": 4.050146526808896, + "tokens_seen": 391446528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122166499498496, + "loss": 1.6053, + "theoretical_loss": 4.050070931402864, + "tokens_seen": 391512064 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122006018054162, + "loss": 1.4945, + "theoretical_loss": 4.049995352192305, + "tokens_seen": 391577600 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712184553660983, + "loss": 1.6051, + "theoretical_loss": 4.04991978917104, + "tokens_seen": 391643136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007121685055165496, + "loss": 1.6117, + "theoretical_loss": 4.049844242332892, + "tokens_seen": 391708672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007121524573721164, + "loss": 1.5436, + "theoretical_loss": 4.049768711671689, + "tokens_seen": 391774208 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712136409227683, + "loss": 1.5156, + "theoretical_loss": 4.049693197181262, + "tokens_seen": 391839744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007121203610832498, + "loss": 1.5343, + "theoretical_loss": 4.049617698855444, + "tokens_seen": 391905280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007121043129388164, + "loss": 1.5006, + "theoretical_loss": 4.049542216688074, + "tokens_seen": 391970816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120882647943832, + "loss": 1.5838, + "theoretical_loss": 4.049466750672994, + "tokens_seen": 392036352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120722166499498, + "loss": 1.6256, + "theoretical_loss": 4.049391300804045, + "tokens_seen": 392101888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120561685055166, + "loss": 1.65, + "theoretical_loss": 4.049315867075077, + "tokens_seen": 392167424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120401203610832, + "loss": 1.4785, + "theoretical_loss": 4.0492404494799406, + "tokens_seen": 392232960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00071202407221665, + "loss": 1.5928, + "theoretical_loss": 4.049165048012489, + "tokens_seen": 392298496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120080240722166, + "loss": 1.4771, + "theoretical_loss": 4.049089662666582, + "tokens_seen": 392364032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119919759277834, + "loss": 1.4962, + "theoretical_loss": 4.04901429343608, + "tokens_seen": 392429568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00071197592778335, + "loss": 1.5464, + "theoretical_loss": 4.048938940314848, + "tokens_seen": 392495104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119598796389168, + "loss": 1.439, + "theoretical_loss": 4.048863603296753, + "tokens_seen": 392560640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119438314944834, + "loss": 1.4998, + "theoretical_loss": 4.048788282375666, + "tokens_seen": 392626176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119277833500502, + "loss": 1.5768, + "theoretical_loss": 4.048712977545463, + "tokens_seen": 392691712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119117352056168, + "loss": 1.4402, + "theoretical_loss": 4.048637688800021, + "tokens_seen": 392757248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007118956870611836, + "loss": 1.5783, + "theoretical_loss": 4.048562416133221, + "tokens_seen": 392822784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007118796389167502, + "loss": 1.5998, + "theoretical_loss": 4.048487159538949, + "tokens_seen": 392888320 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711863590772317, + "loss": 1.5081, + "theoretical_loss": 4.048411919011091, + "tokens_seen": 392953856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007118475426278836, + "loss": 1.5313, + "theoretical_loss": 4.04833669454354, + "tokens_seen": 393019392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007118314944834504, + "loss": 1.5179, + "theoretical_loss": 4.0482614861301895, + "tokens_seen": 393084928 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711815446339017, + "loss": 1.5538, + "theoretical_loss": 4.048186293764939, + "tokens_seen": 393150464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 137415, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5672664642333984, + "objective/train/theoretical_loss": 4.048111117441689, + "objective/train/tokens_used": 413676000, + "theoretical_loss": 4.048111117441689, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117993981945838, + "loss": 1.6009, + "theoretical_loss": 4.048111117441689, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117833500501504, + "loss": 1.6339, + "theoretical_loss": 4.048035957154343, + "tokens_seen": 393281536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117673019057172, + "loss": 1.6105, + "theoretical_loss": 4.047960812896811, + "tokens_seen": 393347072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117512537612838, + "loss": 1.6307, + "theoretical_loss": 4.047885684663004, + "tokens_seen": 393412608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117352056168506, + "loss": 1.5179, + "theoretical_loss": 4.047810572446836, + "tokens_seen": 393478144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117191574724172, + "loss": 1.617, + "theoretical_loss": 4.047735476242225, + "tokens_seen": 393543680 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711703109327984, + "loss": 1.6172, + "theoretical_loss": 4.047660396043092, + "tokens_seen": 393609216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116870611835506, + "loss": 1.4355, + "theoretical_loss": 4.047585331843363, + "tokens_seen": 393674752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116710130391174, + "loss": 1.5347, + "theoretical_loss": 4.047510283636965, + "tokens_seen": 393740288 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711654964894684, + "loss": 1.4901, + "theoretical_loss": 4.04743525141783, + "tokens_seen": 393805824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116389167502508, + "loss": 1.5138, + "theoretical_loss": 4.047360235179891, + "tokens_seen": 393871360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116228686058175, + "loss": 1.5696, + "theoretical_loss": 4.047285234917087, + "tokens_seen": 393936896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116068204613842, + "loss": 1.4289, + "theoretical_loss": 4.04721025062336, + "tokens_seen": 394002432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115907723169509, + "loss": 1.5431, + "theoretical_loss": 4.047135282292653, + "tokens_seen": 394067968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115747241725176, + "loss": 1.5128, + "theoretical_loss": 4.0470603299189145, + "tokens_seen": 394133504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115586760280843, + "loss": 1.5557, + "theoretical_loss": 4.0469853934960955, + "tokens_seen": 394199040 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711542627883651, + "loss": 1.5148, + "theoretical_loss": 4.046910473018151, + "tokens_seen": 394264576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115265797392177, + "loss": 1.6591, + "theoretical_loss": 4.046835568479038, + "tokens_seen": 394330112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115105315947844, + "loss": 1.4581, + "theoretical_loss": 4.046760679872717, + "tokens_seen": 394395648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114944834503511, + "loss": 1.4585, + "theoretical_loss": 4.046685807193153, + "tokens_seen": 394461184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114784353059178, + "loss": 1.5061, + "theoretical_loss": 4.046610950434314, + "tokens_seen": 394526720 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114623871614845, + "loss": 1.5028, + "theoretical_loss": 4.04653610959017, + "tokens_seen": 394592256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114463390170512, + "loss": 1.5819, + "theoretical_loss": 4.046461284654696, + "tokens_seen": 394657792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114302908726179, + "loss": 1.5578, + "theoretical_loss": 4.046386475621869, + "tokens_seen": 394723328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114142427281846, + "loss": 1.5475, + "theoretical_loss": 4.046311682485669, + "tokens_seen": 394788864 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113981945837513, + "loss": 1.5736, + "theoretical_loss": 4.0462369052400815, + "tokens_seen": 394854400 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711382146439318, + "loss": 1.5481, + "theoretical_loss": 4.0461621438790925, + "tokens_seen": 394919936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113660982948847, + "loss": 1.6394, + "theoretical_loss": 4.046087398396693, + "tokens_seen": 394985472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113500501504514, + "loss": 1.5032, + "theoretical_loss": 4.046012668786877, + "tokens_seen": 395051008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113340020060181, + "loss": 1.5094, + "theoretical_loss": 4.045937955043641, + "tokens_seen": 395116544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113179538615848, + "loss": 1.6007, + "theoretical_loss": 4.045863257160986, + "tokens_seen": 395182080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113019057171515, + "loss": 1.4421, + "theoretical_loss": 4.045788575132915, + "tokens_seen": 395247616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112858575727182, + "loss": 1.5609, + "theoretical_loss": 4.045713908953435, + "tokens_seen": 395313152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112698094282849, + "loss": 1.4736, + "theoretical_loss": 4.045639258616557, + "tokens_seen": 395378688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112537612838516, + "loss": 1.5692, + "theoretical_loss": 4.045564624116293, + "tokens_seen": 395444224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112377131394183, + "loss": 1.5839, + "theoretical_loss": 4.04549000544666, + "tokens_seen": 395509760 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711221664994985, + "loss": 1.3912, + "theoretical_loss": 4.045415402601678, + "tokens_seen": 395575296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112056168505517, + "loss": 1.4101, + "theoretical_loss": 4.045340815575372, + "tokens_seen": 395640832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111895687061184, + "loss": 1.582, + "theoretical_loss": 4.045266244361765, + "tokens_seen": 395706368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111735205616851, + "loss": 1.5496, + "theoretical_loss": 4.045191688954889, + "tokens_seen": 395771904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111574724172518, + "loss": 1.5711, + "theoretical_loss": 4.045117149348776, + "tokens_seen": 395837440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111414242728185, + "loss": 1.4717, + "theoretical_loss": 4.045042625537462, + "tokens_seen": 395902976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111253761283852, + "loss": 1.5493, + "theoretical_loss": 4.044968117514987, + "tokens_seen": 395968512 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711109327983952, + "loss": 1.5777, + "theoretical_loss": 4.044893625275392, + "tokens_seen": 396034048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110932798395186, + "loss": 1.4861, + "theoretical_loss": 4.044819148812726, + "tokens_seen": 396099584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110772316950854, + "loss": 1.5666, + "theoretical_loss": 4.044744688121035, + "tokens_seen": 396165120 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711061183550652, + "loss": 1.451, + "theoretical_loss": 4.044670243194372, + "tokens_seen": 396230656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110451354062188, + "loss": 1.5889, + "theoretical_loss": 4.044595814026794, + "tokens_seen": 396296192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110290872617854, + "loss": 1.41, + "theoretical_loss": 4.044521400612357, + "tokens_seen": 396361728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110130391173522, + "loss": 1.5389, + "theoretical_loss": 4.044447002945125, + "tokens_seen": 396427264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 138316, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4225183725357056, + "objective/train/theoretical_loss": 4.044372621019162, + "objective/train/tokens_used": 416952800, + "theoretical_loss": 4.044372621019162, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109969909729188, + "loss": 1.3996, + "theoretical_loss": 4.044372621019162, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109809428284855, + "loss": 1.6224, + "theoretical_loss": 4.044298254828538, + "tokens_seen": 396558336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109648946840522, + "loss": 1.3741, + "theoretical_loss": 4.044223904367324, + "tokens_seen": 396623872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109488465396189, + "loss": 1.5343, + "theoretical_loss": 4.044149569629594, + "tokens_seen": 396689408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109327983951856, + "loss": 1.5944, + "theoretical_loss": 4.044075250609426, + "tokens_seen": 396754944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109167502507523, + "loss": 1.4888, + "theoretical_loss": 4.044000947300901, + "tokens_seen": 396820480 + }, + { + "epoch": 0.12, + "learning_rate": 0.000710900702106319, + "loss": 1.4342, + "theoretical_loss": 4.043926659698106, + "tokens_seen": 396886016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108846539618857, + "loss": 1.6039, + "theoretical_loss": 4.043852387795125, + "tokens_seen": 396951552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108686058174524, + "loss": 1.5694, + "theoretical_loss": 4.043778131586052, + "tokens_seen": 397017088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108525576730191, + "loss": 1.6645, + "theoretical_loss": 4.043703891064979, + "tokens_seen": 397082624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108365095285857, + "loss": 1.5529, + "theoretical_loss": 4.043629666226003, + "tokens_seen": 397148160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108204613841525, + "loss": 1.4129, + "theoretical_loss": 4.043555457063226, + "tokens_seen": 397213696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108044132397191, + "loss": 1.5982, + "theoretical_loss": 4.043481263570751, + "tokens_seen": 397279232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107883650952859, + "loss": 1.5183, + "theoretical_loss": 4.043407085742684, + "tokens_seen": 397344768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107723169508525, + "loss": 1.5224, + "theoretical_loss": 4.0433329235731374, + "tokens_seen": 397410304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107562688064193, + "loss": 1.576, + "theoretical_loss": 4.043258777056221, + "tokens_seen": 397475840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107402206619859, + "loss": 1.5558, + "theoretical_loss": 4.043184646186053, + "tokens_seen": 397541376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107241725175527, + "loss": 1.5146, + "theoretical_loss": 4.043110530956754, + "tokens_seen": 397606912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107081243731193, + "loss": 1.5506, + "theoretical_loss": 4.043036431362444, + "tokens_seen": 397672448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106920762286861, + "loss": 1.5227, + "theoretical_loss": 4.042962347397252, + "tokens_seen": 397737984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106760280842527, + "loss": 1.467, + "theoretical_loss": 4.042888279055304, + "tokens_seen": 397803520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106599799398195, + "loss": 1.4999, + "theoretical_loss": 4.0428142263307345, + "tokens_seen": 397869056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106439317953861, + "loss": 1.5015, + "theoretical_loss": 4.042740189217677, + "tokens_seen": 397934592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106278836509529, + "loss": 1.4486, + "theoretical_loss": 4.042666167710273, + "tokens_seen": 398000128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106118355065195, + "loss": 1.5478, + "theoretical_loss": 4.042592161802662, + "tokens_seen": 398065664 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105957873620863, + "loss": 1.4975, + "theoretical_loss": 4.042518171488989, + "tokens_seen": 398131200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105797392176529, + "loss": 1.5275, + "theoretical_loss": 4.042444196763403, + "tokens_seen": 398196736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105636910732197, + "loss": 1.5028, + "theoretical_loss": 4.042370237620053, + "tokens_seen": 398262272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105476429287863, + "loss": 1.4962, + "theoretical_loss": 4.042296294053097, + "tokens_seen": 398327808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105315947843531, + "loss": 1.6266, + "theoretical_loss": 4.0422223660566905, + "tokens_seen": 398393344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105155466399197, + "loss": 1.5788, + "theoretical_loss": 4.0421484536249945, + "tokens_seen": 398458880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104994984954865, + "loss": 1.7934, + "theoretical_loss": 4.042074556752172, + "tokens_seen": 398524416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104834503510531, + "loss": 1.629, + "theoretical_loss": 4.042000675432391, + "tokens_seen": 398589952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104674022066199, + "loss": 1.4217, + "theoretical_loss": 4.041926809659821, + "tokens_seen": 398655488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104513540621865, + "loss": 1.5271, + "theoretical_loss": 4.0418529594286365, + "tokens_seen": 398721024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104353059177533, + "loss": 1.5935, + "theoretical_loss": 4.041779124733012, + "tokens_seen": 398786560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104192577733199, + "loss": 1.4487, + "theoretical_loss": 4.041705305567129, + "tokens_seen": 398852096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104032096288867, + "loss": 1.7103, + "theoretical_loss": 4.041631501925169, + "tokens_seen": 398917632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103871614844533, + "loss": 1.5326, + "theoretical_loss": 4.041557713801319, + "tokens_seen": 398983168 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103711133400201, + "loss": 1.5504, + "theoretical_loss": 4.041483941189766, + "tokens_seen": 399048704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103550651955867, + "loss": 1.58, + "theoretical_loss": 4.041410184084703, + "tokens_seen": 399114240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103390170511535, + "loss": 1.5606, + "theoretical_loss": 4.041336442480326, + "tokens_seen": 399179776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103229689067201, + "loss": 1.6753, + "theoretical_loss": 4.041262716370833, + "tokens_seen": 399245312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103069207622869, + "loss": 1.6089, + "theoretical_loss": 4.041189005750424, + "tokens_seen": 399310848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102908726178535, + "loss": 1.4364, + "theoretical_loss": 4.041115310613307, + "tokens_seen": 399376384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102748244734203, + "loss": 1.6079, + "theoretical_loss": 4.0410416309536865, + "tokens_seen": 399441920 + }, + { + "epoch": 0.12, + "learning_rate": 0.000710258776328987, + "loss": 1.5491, + "theoretical_loss": 4.040967966765774, + "tokens_seen": 399507456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102427281845537, + "loss": 1.5805, + "theoretical_loss": 4.040894318043785, + "tokens_seen": 399572992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102266800401204, + "loss": 1.6833, + "theoretical_loss": 4.040820684781934, + "tokens_seen": 399638528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102106318956871, + "loss": 1.5103, + "theoretical_loss": 4.040747066974443, + "tokens_seen": 399704064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 139589, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5951530933380127, + "objective/train/theoretical_loss": 4.040673464615535, + "objective/train/tokens_used": 420229600, + "theoretical_loss": 4.040673464615535, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101945837512538, + "loss": 1.5524, + "theoretical_loss": 4.040673464615535, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101785356068205, + "loss": 1.5063, + "theoretical_loss": 4.040599877699437, + "tokens_seen": 399835136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101624874623872, + "loss": 1.5295, + "theoretical_loss": 4.040526306220377, + "tokens_seen": 399900672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101464393179539, + "loss": 1.6562, + "theoretical_loss": 4.040452750172588, + "tokens_seen": 399966208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101303911735206, + "loss": 1.5602, + "theoretical_loss": 4.040379209550307, + "tokens_seen": 400031744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101143430290873, + "loss": 1.5235, + "theoretical_loss": 4.0403056843477705, + "tokens_seen": 400097280 + }, + { + "epoch": 0.12, + "learning_rate": 0.000710098294884654, + "loss": 1.4961, + "theoretical_loss": 4.040232174559223, + "tokens_seen": 400162816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100822467402207, + "loss": 1.5648, + "theoretical_loss": 4.040158680178907, + "tokens_seen": 400228352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100661985957874, + "loss": 1.4735, + "theoretical_loss": 4.040085201201072, + "tokens_seen": 400293888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100501504513541, + "loss": 1.5627, + "theoretical_loss": 4.0400117376199685, + "tokens_seen": 400359424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100341023069208, + "loss": 1.6345, + "theoretical_loss": 4.039938289429852, + "tokens_seen": 400424960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100180541624875, + "loss": 1.6676, + "theoretical_loss": 4.039864856624979, + "tokens_seen": 400490496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100020060180542, + "loss": 1.5924, + "theoretical_loss": 4.039791439199609, + "tokens_seen": 400556032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099859578736209, + "loss": 1.5713, + "theoretical_loss": 4.039718037148006, + "tokens_seen": 400621568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099699097291876, + "loss": 1.5793, + "theoretical_loss": 4.039644650464438, + "tokens_seen": 400687104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099538615847543, + "loss": 1.5998, + "theoretical_loss": 4.039571279143172, + "tokens_seen": 400752640 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709937813440321, + "loss": 1.541, + "theoretical_loss": 4.039497923178484, + "tokens_seen": 400818176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099217652958877, + "loss": 1.4307, + "theoretical_loss": 4.039424582564646, + "tokens_seen": 400883712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099057171514545, + "loss": 1.4884, + "theoretical_loss": 4.03935125729594, + "tokens_seen": 400949248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098896690070211, + "loss": 1.6443, + "theoretical_loss": 4.039277947366647, + "tokens_seen": 401014784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098736208625879, + "loss": 1.5543, + "theoretical_loss": 4.039204652771051, + "tokens_seen": 401080320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098575727181545, + "loss": 1.4671, + "theoretical_loss": 4.039131373503441, + "tokens_seen": 401145856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098415245737213, + "loss": 1.5812, + "theoretical_loss": 4.039058109558107, + "tokens_seen": 401211392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098254764292879, + "loss": 1.6335, + "theoretical_loss": 4.038984860929346, + "tokens_seen": 401276928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098094282848547, + "loss": 1.5945, + "theoretical_loss": 4.038911627611451, + "tokens_seen": 401342464 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097933801404213, + "loss": 1.5385, + "theoretical_loss": 4.038838409598727, + "tokens_seen": 401408000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097773319959881, + "loss": 1.491, + "theoretical_loss": 4.038765206885472, + "tokens_seen": 401473536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097612838515547, + "loss": 1.5328, + "theoretical_loss": 4.038692019465997, + "tokens_seen": 401539072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097452357071215, + "loss": 1.573, + "theoretical_loss": 4.038618847334609, + "tokens_seen": 401604608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097291875626881, + "loss": 1.7025, + "theoretical_loss": 4.038545690485622, + "tokens_seen": 401670144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097131394182549, + "loss": 1.5662, + "theoretical_loss": 4.038472548913349, + "tokens_seen": 401735680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096970912738215, + "loss": 1.59, + "theoretical_loss": 4.038399422612112, + "tokens_seen": 401801216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096810431293883, + "loss": 1.5199, + "theoretical_loss": 4.038326311576229, + "tokens_seen": 401866752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096649949849549, + "loss": 1.542, + "theoretical_loss": 4.038253215800028, + "tokens_seen": 401932288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096489468405217, + "loss": 1.5482, + "theoretical_loss": 4.038180135277834, + "tokens_seen": 401997824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096328986960883, + "loss": 1.6522, + "theoretical_loss": 4.038107070003979, + "tokens_seen": 402063360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096168505516551, + "loss": 1.5936, + "theoretical_loss": 4.038034019972796, + "tokens_seen": 402128896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096008024072217, + "loss": 1.6195, + "theoretical_loss": 4.037960985178623, + "tokens_seen": 402194432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095847542627884, + "loss": 1.5161, + "theoretical_loss": 4.037887965615798, + "tokens_seen": 402259968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095687061183551, + "loss": 1.5262, + "theoretical_loss": 4.0378149612786665, + "tokens_seen": 402325504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095526579739218, + "loss": 1.6219, + "theoretical_loss": 4.037741972161571, + "tokens_seen": 402391040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095366098294885, + "loss": 1.584, + "theoretical_loss": 4.037668998258863, + "tokens_seen": 402456576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095205616850552, + "loss": 1.532, + "theoretical_loss": 4.037596039564892, + "tokens_seen": 402522112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095045135406219, + "loss": 1.6007, + "theoretical_loss": 4.0375230960740165, + "tokens_seen": 402587648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007094884653961886, + "loss": 1.5508, + "theoretical_loss": 4.03745016778059, + "tokens_seen": 402653184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007094724172517553, + "loss": 1.5275, + "theoretical_loss": 4.037377254678977, + "tokens_seen": 402718720 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709456369107322, + "loss": 1.587, + "theoretical_loss": 4.037304356763539, + "tokens_seen": 402784256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007094403209628886, + "loss": 1.5588, + "theoretical_loss": 4.037231474028644, + "tokens_seen": 402849792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007094242728184554, + "loss": 1.4966, + "theoretical_loss": 4.037158606468663, + "tokens_seen": 402915328 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709408224674022, + "loss": 1.5709, + "theoretical_loss": 4.037085754077966, + "tokens_seen": 402980864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 140534, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5789742469787598, + "objective/train/theoretical_loss": 4.037012916850932, + "objective/train/tokens_used": 423506400, + "theoretical_loss": 4.037012916850932, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093921765295888, + "loss": 1.5523, + "theoretical_loss": 4.037012916850932, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093761283851554, + "loss": 1.5749, + "theoretical_loss": 4.036940094781937, + "tokens_seen": 403111936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093600802407222, + "loss": 1.596, + "theoretical_loss": 4.036867287865366, + "tokens_seen": 403177472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093440320962888, + "loss": 1.4619, + "theoretical_loss": 4.036794496095602, + "tokens_seen": 403243008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093279839518556, + "loss": 1.587, + "theoretical_loss": 4.0367217194670335, + "tokens_seen": 403308544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093119358074222, + "loss": 1.5353, + "theoretical_loss": 4.036648957974052, + "tokens_seen": 403374080 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709295887662989, + "loss": 1.5422, + "theoretical_loss": 4.03657621161105, + "tokens_seen": 403439616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007092798395185556, + "loss": 1.6412, + "theoretical_loss": 4.036503480372424, + "tokens_seen": 403505152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007092637913741224, + "loss": 1.5548, + "theoretical_loss": 4.036430764252577, + "tokens_seen": 403570688 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709247743229689, + "loss": 1.5112, + "theoretical_loss": 4.036358063245908, + "tokens_seen": 403636224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007092316950852558, + "loss": 1.515, + "theoretical_loss": 4.036285377346826, + "tokens_seen": 403701760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007092156469408224, + "loss": 1.422, + "theoretical_loss": 4.036212706549738, + "tokens_seen": 403767296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091995987963892, + "loss": 1.6223, + "theoretical_loss": 4.036140050849056, + "tokens_seen": 403832832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091835506519558, + "loss": 1.4715, + "theoretical_loss": 4.036067410239196, + "tokens_seen": 403898368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091675025075226, + "loss": 1.5133, + "theoretical_loss": 4.035994784714574, + "tokens_seen": 403963904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091514543630892, + "loss": 1.4933, + "theoretical_loss": 4.035922174269611, + "tokens_seen": 404029440 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709135406218656, + "loss": 1.3653, + "theoretical_loss": 4.035849578898731, + "tokens_seen": 404094976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091193580742226, + "loss": 1.5697, + "theoretical_loss": 4.035776998596361, + "tokens_seen": 404160512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091033099297894, + "loss": 1.5925, + "theoretical_loss": 4.035704433356932, + "tokens_seen": 404226048 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709087261785356, + "loss": 1.5928, + "theoretical_loss": 4.035631883174873, + "tokens_seen": 404291584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090712136409228, + "loss": 1.51, + "theoretical_loss": 4.035559348044622, + "tokens_seen": 404357120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090551654964894, + "loss": 1.5758, + "theoretical_loss": 4.035486827960617, + "tokens_seen": 404422656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090391173520562, + "loss": 1.4794, + "theoretical_loss": 4.035414322917299, + "tokens_seen": 404488192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090230692076228, + "loss": 1.643, + "theoretical_loss": 4.035341832909114, + "tokens_seen": 404553728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090070210631896, + "loss": 1.5227, + "theoretical_loss": 4.035269357930507, + "tokens_seen": 404619264 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007089909729187562, + "loss": 1.5348, + "theoretical_loss": 4.03519689797593, + "tokens_seen": 404684800 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708974924774323, + "loss": 1.5649, + "theoretical_loss": 4.035124453039836, + "tokens_seen": 404750336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007089588766298896, + "loss": 1.6123, + "theoretical_loss": 4.035052023116679, + "tokens_seen": 404815872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007089428284854564, + "loss": 1.5052, + "theoretical_loss": 4.0349796082009215, + "tokens_seen": 404881408 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708926780341023, + "loss": 1.493, + "theoretical_loss": 4.034907208287024, + "tokens_seen": 404946944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007089107321965898, + "loss": 1.6608, + "theoretical_loss": 4.034834823369451, + "tokens_seen": 405012480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088946840521565, + "loss": 1.5893, + "theoretical_loss": 4.034762453442672, + "tokens_seen": 405078016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088786359077232, + "loss": 1.4565, + "theoretical_loss": 4.034690098501155, + "tokens_seen": 405143552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088625877632899, + "loss": 1.5087, + "theoretical_loss": 4.034617758539377, + "tokens_seen": 405209088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088465396188566, + "loss": 1.5578, + "theoretical_loss": 4.034545433551814, + "tokens_seen": 405274624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088304914744233, + "loss": 1.4697, + "theoretical_loss": 4.034473123532944, + "tokens_seen": 405340160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00070881444332999, + "loss": 1.5152, + "theoretical_loss": 4.034400828477251, + "tokens_seen": 405405696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087983951855567, + "loss": 1.5794, + "theoretical_loss": 4.03432854837922, + "tokens_seen": 405471232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087823470411234, + "loss": 1.4242, + "theoretical_loss": 4.03425628323334, + "tokens_seen": 405536768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087662988966901, + "loss": 1.5615, + "theoretical_loss": 4.034184033034102, + "tokens_seen": 405602304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087502507522568, + "loss": 1.5607, + "theoretical_loss": 4.034111797776001, + "tokens_seen": 405667840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087342026078235, + "loss": 1.5839, + "theoretical_loss": 4.034039577453533, + "tokens_seen": 405733376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087181544633902, + "loss": 1.3925, + "theoretical_loss": 4.033967372061199, + "tokens_seen": 405798912 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708702106318957, + "loss": 1.5731, + "theoretical_loss": 4.033895181593501, + "tokens_seen": 405864448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086860581745236, + "loss": 1.4833, + "theoretical_loss": 4.033823006044947, + "tokens_seen": 405929984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086700100300904, + "loss": 1.5742, + "theoretical_loss": 4.033750845410044, + "tokens_seen": 405995520 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708653961885657, + "loss": 1.5395, + "theoretical_loss": 4.033678699683303, + "tokens_seen": 406061056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086379137412238, + "loss": 1.5878, + "theoretical_loss": 4.033606568859241, + "tokens_seen": 406126592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086218655967904, + "loss": 1.4696, + "theoretical_loss": 4.033534452932376, + "tokens_seen": 406192128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086058174523572, + "loss": 1.5745, + "theoretical_loss": 4.033462351897225, + "tokens_seen": 406257664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 141740, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.756189227104187, + "objective/train/theoretical_loss": 4.033390265748315, + "objective/train/tokens_used": 426783200, + "theoretical_loss": 4.033390265748315, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085897693079238, + "loss": 1.5844, + "theoretical_loss": 4.033390265748315, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085737211634906, + "loss": 1.4169, + "theoretical_loss": 4.033318194480169, + "tokens_seen": 406388736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085576730190572, + "loss": 1.6124, + "theoretical_loss": 4.0332461380873195, + "tokens_seen": 406454272 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708541624874624, + "loss": 1.6184, + "theoretical_loss": 4.033174096564295, + "tokens_seen": 406519808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085255767301906, + "loss": 1.4786, + "theoretical_loss": 4.033102069905635, + "tokens_seen": 406585344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085095285857574, + "loss": 1.4864, + "theoretical_loss": 4.033030058105872, + "tokens_seen": 406650880 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708493480441324, + "loss": 1.5946, + "theoretical_loss": 4.032958061159551, + "tokens_seen": 406716416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084774322968908, + "loss": 1.5697, + "theoretical_loss": 4.032886079061214, + "tokens_seen": 406781952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084613841524574, + "loss": 1.6082, + "theoretical_loss": 4.032814111805407, + "tokens_seen": 406847488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084453360080242, + "loss": 1.6108, + "theoretical_loss": 4.03274215938668, + "tokens_seen": 406913024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084292878635908, + "loss": 1.5825, + "theoretical_loss": 4.0326702217995845, + "tokens_seen": 406978560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084132397191576, + "loss": 1.6002, + "theoretical_loss": 4.0325982990386775, + "tokens_seen": 407044096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083971915747242, + "loss": 1.5136, + "theoretical_loss": 4.032526391098515, + "tokens_seen": 407109632 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708381143430291, + "loss": 1.5522, + "theoretical_loss": 4.032454497973659, + "tokens_seen": 407175168 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083650952858576, + "loss": 1.5036, + "theoretical_loss": 4.032382619658673, + "tokens_seen": 407240704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083490471414244, + "loss": 1.5827, + "theoretical_loss": 4.0323107561481235, + "tokens_seen": 407306240 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708332998996991, + "loss": 1.4544, + "theoretical_loss": 4.0322389074365805, + "tokens_seen": 407371776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083169508525578, + "loss": 1.6027, + "theoretical_loss": 4.032167073518615, + "tokens_seen": 407437312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083009027081244, + "loss": 1.5711, + "theoretical_loss": 4.032095254388803, + "tokens_seen": 407502848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082848545636912, + "loss": 1.5284, + "theoretical_loss": 4.032023450041724, + "tokens_seen": 407568384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082688064192578, + "loss": 1.5628, + "theoretical_loss": 4.031951660471956, + "tokens_seen": 407633920 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082527582748246, + "loss": 1.5589, + "theoretical_loss": 4.031879885674085, + "tokens_seen": 407699456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082367101303912, + "loss": 1.6216, + "theoretical_loss": 4.031808125642697, + "tokens_seen": 407764992 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708220661985958, + "loss": 1.552, + "theoretical_loss": 4.031736380372382, + "tokens_seen": 407830528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082046138415246, + "loss": 1.4943, + "theoretical_loss": 4.031664649857731, + "tokens_seen": 407896064 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081885656970914, + "loss": 1.5147, + "theoretical_loss": 4.031592934093341, + "tokens_seen": 407961600 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708172517552658, + "loss": 1.4208, + "theoretical_loss": 4.031521233073809, + "tokens_seen": 408027136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081564694082247, + "loss": 1.5565, + "theoretical_loss": 4.031449546793736, + "tokens_seen": 408092672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081404212637914, + "loss": 1.5643, + "theoretical_loss": 4.031377875247726, + "tokens_seen": 408158208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081243731193581, + "loss": 1.6523, + "theoretical_loss": 4.031306218430385, + "tokens_seen": 408223744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081083249749248, + "loss": 1.5344, + "theoretical_loss": 4.031234576336322, + "tokens_seen": 408289280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080922768304915, + "loss": 1.5772, + "theoretical_loss": 4.0311629489601515, + "tokens_seen": 408354816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080762286860582, + "loss": 1.6145, + "theoretical_loss": 4.031091336296487, + "tokens_seen": 408420352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080601805416249, + "loss": 1.5051, + "theoretical_loss": 4.031019738339946, + "tokens_seen": 408485888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080441323971915, + "loss": 1.5164, + "theoretical_loss": 4.03094815508515, + "tokens_seen": 408551424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080280842527583, + "loss": 1.5636, + "theoretical_loss": 4.030876586526722, + "tokens_seen": 408616960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080120361083249, + "loss": 1.3939, + "theoretical_loss": 4.0308050326592895, + "tokens_seen": 408682496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079959879638917, + "loss": 1.4653, + "theoretical_loss": 4.030733493477481, + "tokens_seen": 408748032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079799398194583, + "loss": 1.5727, + "theoretical_loss": 4.030661968975929, + "tokens_seen": 408813568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079638916750251, + "loss": 1.5448, + "theoretical_loss": 4.030590459149267, + "tokens_seen": 408879104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079478435305917, + "loss": 1.453, + "theoretical_loss": 4.030518963992135, + "tokens_seen": 408944640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079317953861585, + "loss": 1.5661, + "theoretical_loss": 4.030447483499171, + "tokens_seen": 409010176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079157472417251, + "loss": 1.504, + "theoretical_loss": 4.03037601766502, + "tokens_seen": 409075712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078996990972919, + "loss": 1.4467, + "theoretical_loss": 4.030304566484329, + "tokens_seen": 409141248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078836509528585, + "loss": 1.5275, + "theoretical_loss": 4.030233129951744, + "tokens_seen": 409206784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078676028084253, + "loss": 1.5733, + "theoretical_loss": 4.03016170806192, + "tokens_seen": 409272320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078515546639919, + "loss": 1.4817, + "theoretical_loss": 4.030090300809509, + "tokens_seen": 409337856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078355065195587, + "loss": 1.6054, + "theoretical_loss": 4.03001890818917, + "tokens_seen": 409403392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078194583751253, + "loss": 1.4895, + "theoretical_loss": 4.029947530195563, + "tokens_seen": 409468928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078034102306921, + "loss": 1.547, + "theoretical_loss": 4.029876166823351, + "tokens_seen": 409534464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 142735, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6000092029571533, + "objective/train/theoretical_loss": 4.029804818067199, + "objective/train/tokens_used": 430060000, + "theoretical_loss": 4.029804818067199, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077873620862587, + "loss": 1.5053, + "theoretical_loss": 4.029804818067199, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077713139418255, + "loss": 1.4704, + "theoretical_loss": 4.029733483921777, + "tokens_seen": 409665536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077552657973921, + "loss": 1.5721, + "theoretical_loss": 4.0296621643817545, + "tokens_seen": 409731072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077392176529589, + "loss": 1.6005, + "theoretical_loss": 4.029590859441807, + "tokens_seen": 409796608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077231695085255, + "loss": 1.5829, + "theoretical_loss": 4.029519569096612, + "tokens_seen": 409862144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077071213640923, + "loss": 1.5834, + "theoretical_loss": 4.029448293340847, + "tokens_seen": 409927680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076910732196589, + "loss": 1.4792, + "theoretical_loss": 4.029377032169197, + "tokens_seen": 409993216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076750250752257, + "loss": 1.4813, + "theoretical_loss": 4.029305785576347, + "tokens_seen": 410058752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076589769307923, + "loss": 1.597, + "theoretical_loss": 4.029234553556985, + "tokens_seen": 410124288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076429287863591, + "loss": 1.5541, + "theoretical_loss": 4.029163336105801, + "tokens_seen": 410189824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076268806419257, + "loss": 1.5452, + "theoretical_loss": 4.029092133217489, + "tokens_seen": 410255360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076108324974925, + "loss": 1.4583, + "theoretical_loss": 4.029020944886746, + "tokens_seen": 410320896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075947843530591, + "loss": 1.5313, + "theoretical_loss": 4.0289497711082705, + "tokens_seen": 410386432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075787362086259, + "loss": 1.5252, + "theoretical_loss": 4.028878611876765, + "tokens_seen": 410451968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075626880641925, + "loss": 1.5533, + "theoretical_loss": 4.028807467186935, + "tokens_seen": 410517504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075466399197593, + "loss": 1.5356, + "theoretical_loss": 4.028736337033488, + "tokens_seen": 410583040 + }, + { + "epoch": 0.12, + "learning_rate": 0.000707530591775326, + "loss": 1.4687, + "theoretical_loss": 4.028665221411133, + "tokens_seen": 410648576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075145436308927, + "loss": 1.5098, + "theoretical_loss": 4.028594120314585, + "tokens_seen": 410714112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074984954864595, + "loss": 1.5612, + "theoretical_loss": 4.028523033738558, + "tokens_seen": 410779648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074824473420261, + "loss": 1.4409, + "theoretical_loss": 4.028451961677772, + "tokens_seen": 410845184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074663991975929, + "loss": 1.3957, + "theoretical_loss": 4.028380904126948, + "tokens_seen": 410910720 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074503510531595, + "loss": 1.616, + "theoretical_loss": 4.0283098610808095, + "tokens_seen": 410976256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074343029087263, + "loss": 1.5218, + "theoretical_loss": 4.028238832534084, + "tokens_seen": 411041792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074182547642929, + "loss": 1.5817, + "theoretical_loss": 4.028167818481502, + "tokens_seen": 411107328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074022066198597, + "loss": 1.5122, + "theoretical_loss": 4.0280968189177955, + "tokens_seen": 411172864 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073861584754263, + "loss": 1.5176, + "theoretical_loss": 4.028025833837699, + "tokens_seen": 411238400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073701103309931, + "loss": 1.5894, + "theoretical_loss": 4.027954863235951, + "tokens_seen": 411303936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073540621865597, + "loss": 1.3881, + "theoretical_loss": 4.0278839071072925, + "tokens_seen": 411369472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073380140421265, + "loss": 1.492, + "theoretical_loss": 4.027812965446467, + "tokens_seen": 411435008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073219658976931, + "loss": 1.6257, + "theoretical_loss": 4.02774203824822, + "tokens_seen": 411500544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073059177532599, + "loss": 1.4058, + "theoretical_loss": 4.0276711255073, + "tokens_seen": 411566080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072898696088265, + "loss": 1.4869, + "theoretical_loss": 4.027600227218461, + "tokens_seen": 411631616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072738214643933, + "loss": 1.5341, + "theoretical_loss": 4.027529343376456, + "tokens_seen": 411697152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072577733199599, + "loss": 1.541, + "theoretical_loss": 4.027458473976041, + "tokens_seen": 411762688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072417251755267, + "loss": 1.5091, + "theoretical_loss": 4.027387619011977, + "tokens_seen": 411828224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072256770310933, + "loss": 1.4396, + "theoretical_loss": 4.027316778479027, + "tokens_seen": 411893760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072096288866601, + "loss": 1.5064, + "theoretical_loss": 4.027245952371956, + "tokens_seen": 411959296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071935807422267, + "loss": 1.4344, + "theoretical_loss": 4.027175140685532, + "tokens_seen": 412024832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071775325977935, + "loss": 1.4299, + "theoretical_loss": 4.027104343414527, + "tokens_seen": 412090368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071614844533601, + "loss": 1.509, + "theoretical_loss": 4.027033560553713, + "tokens_seen": 412155904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071454363089269, + "loss": 1.4293, + "theoretical_loss": 4.026962792097867, + "tokens_seen": 412221440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071293881644935, + "loss": 1.423, + "theoretical_loss": 4.026892038041767, + "tokens_seen": 412286976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071133400200603, + "loss": 1.6205, + "theoretical_loss": 4.026821298380197, + "tokens_seen": 412352512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007070972918756269, + "loss": 1.5077, + "theoretical_loss": 4.026750573107939, + "tokens_seen": 412418048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007070812437311937, + "loss": 1.5667, + "theoretical_loss": 4.0266798622197815, + "tokens_seen": 412483584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070651955867603, + "loss": 1.5787, + "theoretical_loss": 4.026609165710514, + "tokens_seen": 412549120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070491474423271, + "loss": 1.5379, + "theoretical_loss": 4.02653848357493, + "tokens_seen": 412614656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070330992978937, + "loss": 1.4793, + "theoretical_loss": 4.026467815807823, + "tokens_seen": 412680192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070170511534605, + "loss": 1.5342, + "theoretical_loss": 4.026397162403994, + "tokens_seen": 412745728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070010030090271, + "loss": 1.5203, + "theoretical_loss": 4.0263265233582395, + "tokens_seen": 412811264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 143929, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.583418369293213, + "objective/train/theoretical_loss": 4.026255898665367, + "objective/train/tokens_used": 433336800, + "theoretical_loss": 4.026255898665367, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069849548645939, + "loss": 1.5316, + "theoretical_loss": 4.026255898665367, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069689067201605, + "loss": 1.5474, + "theoretical_loss": 4.02618528832018, + "tokens_seen": 412942336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069528585757273, + "loss": 1.5566, + "theoretical_loss": 4.026114692317489, + "tokens_seen": 413007872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069368104312939, + "loss": 1.2872, + "theoretical_loss": 4.026044110652105, + "tokens_seen": 413073408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069207622868607, + "loss": 1.5705, + "theoretical_loss": 4.025973543318841, + "tokens_seen": 413138944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069047141424273, + "loss": 1.5252, + "theoretical_loss": 4.025902990312517, + "tokens_seen": 413204480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068886659979941, + "loss": 1.5701, + "theoretical_loss": 4.025832451627949, + "tokens_seen": 413270016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068726178535607, + "loss": 1.5828, + "theoretical_loss": 4.025761927259961, + "tokens_seen": 413335552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068565697091275, + "loss": 1.4946, + "theoretical_loss": 4.0256914172033795, + "tokens_seen": 413401088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068405215646941, + "loss": 1.5515, + "theoretical_loss": 4.025620921453029, + "tokens_seen": 413466624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068244734202609, + "loss": 1.639, + "theoretical_loss": 4.025550440003744, + "tokens_seen": 413532160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068084252758275, + "loss": 1.549, + "theoretical_loss": 4.025479972850354, + "tokens_seen": 413597696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067923771313943, + "loss": 1.4433, + "theoretical_loss": 4.025409519987696, + "tokens_seen": 413663232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067763289869609, + "loss": 1.4926, + "theoretical_loss": 4.025339081410609, + "tokens_seen": 413728768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067602808425276, + "loss": 1.5137, + "theoretical_loss": 4.025268657113934, + "tokens_seen": 413794304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067442326980943, + "loss": 1.6172, + "theoretical_loss": 4.025198247092514, + "tokens_seen": 413859840 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706728184553661, + "loss": 1.5152, + "theoretical_loss": 4.025127851341196, + "tokens_seen": 413925376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067121364092277, + "loss": 1.4704, + "theoretical_loss": 4.025057469854831, + "tokens_seen": 413990912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066960882647944, + "loss": 1.454, + "theoretical_loss": 4.024987102628267, + "tokens_seen": 414056448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066800401203611, + "loss": 1.482, + "theoretical_loss": 4.024916749656362, + "tokens_seen": 414121984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066639919759278, + "loss": 1.4719, + "theoretical_loss": 4.024846410933971, + "tokens_seen": 414187520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066479438314945, + "loss": 1.523, + "theoretical_loss": 4.024776086455955, + "tokens_seen": 414253056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066318956870612, + "loss": 1.4781, + "theoretical_loss": 4.024705776217177, + "tokens_seen": 414318592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066158475426278, + "loss": 1.5754, + "theoretical_loss": 4.024635480212501, + "tokens_seen": 414384128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065997993981946, + "loss": 1.5037, + "theoretical_loss": 4.024565198436796, + "tokens_seen": 414449664 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065837512537612, + "loss": 1.5883, + "theoretical_loss": 4.024494930884931, + "tokens_seen": 414515200 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706567703109328, + "loss": 1.5003, + "theoretical_loss": 4.024424677551781, + "tokens_seen": 414580736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065516549648946, + "loss": 1.4816, + "theoretical_loss": 4.02435443843222, + "tokens_seen": 414646272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065356068204614, + "loss": 1.6653, + "theoretical_loss": 4.024284213521128, + "tokens_seen": 414711808 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706519558676028, + "loss": 1.5012, + "theoretical_loss": 4.024214002813386, + "tokens_seen": 414777344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065035105315948, + "loss": 1.4605, + "theoretical_loss": 4.024143806303877, + "tokens_seen": 414842880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064874623871614, + "loss": 1.4218, + "theoretical_loss": 4.024073623987487, + "tokens_seen": 414908416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064714142427282, + "loss": 1.5412, + "theoretical_loss": 4.024003455859107, + "tokens_seen": 414973952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064553660982948, + "loss": 1.4357, + "theoretical_loss": 4.023933301913628, + "tokens_seen": 415039488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064393179538616, + "loss": 1.5242, + "theoretical_loss": 4.023863162145943, + "tokens_seen": 415105024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064232698094282, + "loss": 1.6922, + "theoretical_loss": 4.023793036550952, + "tokens_seen": 415170560 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706407221664995, + "loss": 1.4891, + "theoretical_loss": 4.023722925123551, + "tokens_seen": 415236096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063911735205616, + "loss": 1.5164, + "theoretical_loss": 4.023652827858646, + "tokens_seen": 415301632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063751253761284, + "loss": 1.537, + "theoretical_loss": 4.023582744751138, + "tokens_seen": 415367168 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706359077231695, + "loss": 1.437, + "theoretical_loss": 4.0235126757959385, + "tokens_seen": 415432704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063430290872618, + "loss": 1.467, + "theoretical_loss": 4.0234426209879555, + "tokens_seen": 415498240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063269809428284, + "loss": 1.5664, + "theoretical_loss": 4.0233725803221025, + "tokens_seen": 415563776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063109327983952, + "loss": 1.5127, + "theoretical_loss": 4.023302553793295, + "tokens_seen": 415629312 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706294884653962, + "loss": 1.4854, + "theoretical_loss": 4.023232541396451, + "tokens_seen": 415694848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007062788365095286, + "loss": 1.5801, + "theoretical_loss": 4.023162543126491, + "tokens_seen": 415760384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007062627883650954, + "loss": 1.5972, + "theoretical_loss": 4.023092558978339, + "tokens_seen": 415825920 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706246740220662, + "loss": 1.5104, + "theoretical_loss": 4.023022588946921, + "tokens_seen": 415891456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007062306920762288, + "loss": 1.3768, + "theoretical_loss": 4.022952633027164, + "tokens_seen": 415956992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007062146439317954, + "loss": 1.5053, + "theoretical_loss": 4.022882691214002, + "tokens_seen": 416022528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061985957873622, + "loss": 1.6381, + "theoretical_loss": 4.022812763502367, + "tokens_seen": 416088064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 144846, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4289838075637817, + "objective/train/theoretical_loss": 4.022742849887195, + "objective/train/tokens_used": 436613600, + "theoretical_loss": 4.022742849887195, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061825476429288, + "loss": 1.5006, + "theoretical_loss": 4.022742849887195, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061664994984956, + "loss": 1.5972, + "theoretical_loss": 4.022672950363427, + "tokens_seen": 416219136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061504513540622, + "loss": 1.5633, + "theoretical_loss": 4.022603064926004, + "tokens_seen": 416284672 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706134403209629, + "loss": 1.578, + "theoretical_loss": 4.02253319356987, + "tokens_seen": 416350208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061183550651956, + "loss": 1.475, + "theoretical_loss": 4.022463336289971, + "tokens_seen": 416415744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061023069207624, + "loss": 1.4692, + "theoretical_loss": 4.022393493081259, + "tokens_seen": 416481280 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706086258776329, + "loss": 1.4357, + "theoretical_loss": 4.022323663938684, + "tokens_seen": 416546816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060702106318958, + "loss": 1.4955, + "theoretical_loss": 4.022253848857201, + "tokens_seen": 416612352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060541624874624, + "loss": 1.544, + "theoretical_loss": 4.022184047831768, + "tokens_seen": 416677888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060381143430292, + "loss": 1.5028, + "theoretical_loss": 4.022114260857345, + "tokens_seen": 416743424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060220661985958, + "loss": 1.523, + "theoretical_loss": 4.022044487928895, + "tokens_seen": 416808960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060060180541626, + "loss": 1.5272, + "theoretical_loss": 4.021974729041381, + "tokens_seen": 416874496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007059899699097292, + "loss": 1.5397, + "theoretical_loss": 4.021904984189773, + "tokens_seen": 416940032 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705973921765296, + "loss": 1.5114, + "theoretical_loss": 4.021835253369041, + "tokens_seen": 417005568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007059578736208626, + "loss": 1.6011, + "theoretical_loss": 4.021765536574157, + "tokens_seen": 417071104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007059418254764294, + "loss": 1.5084, + "theoretical_loss": 4.021695833800098, + "tokens_seen": 417136640 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705925777331996, + "loss": 1.5491, + "theoretical_loss": 4.02162614504184, + "tokens_seen": 417202176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007059097291875628, + "loss": 1.4945, + "theoretical_loss": 4.021556470294366, + "tokens_seen": 417267712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058936810431294, + "loss": 1.5515, + "theoretical_loss": 4.021486809552657, + "tokens_seen": 417333248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058776328986962, + "loss": 1.4881, + "theoretical_loss": 4.021417162811701, + "tokens_seen": 417398784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058615847542628, + "loss": 1.6362, + "theoretical_loss": 4.021347530066485, + "tokens_seen": 417464320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058455366098296, + "loss": 1.5221, + "theoretical_loss": 4.0212779113120005, + "tokens_seen": 417529856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058294884653962, + "loss": 1.5595, + "theoretical_loss": 4.021208306543242, + "tokens_seen": 417595392 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705813440320963, + "loss": 1.5748, + "theoretical_loss": 4.021138715755204, + "tokens_seen": 417660928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057973921765296, + "loss": 1.5799, + "theoretical_loss": 4.021069138942886, + "tokens_seen": 417726464 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057813440320964, + "loss": 1.5296, + "theoretical_loss": 4.020999576101291, + "tokens_seen": 417792000 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705765295887663, + "loss": 1.4946, + "theoretical_loss": 4.02093002722542, + "tokens_seen": 417857536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057492477432298, + "loss": 1.583, + "theoretical_loss": 4.020860492310281, + "tokens_seen": 417923072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057331995987964, + "loss": 1.5426, + "theoretical_loss": 4.020790971350884, + "tokens_seen": 417988608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057171514543632, + "loss": 1.5689, + "theoretical_loss": 4.020721464342238, + "tokens_seen": 418054144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057011033099298, + "loss": 1.4474, + "theoretical_loss": 4.02065197127936, + "tokens_seen": 418119680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007056850551654966, + "loss": 1.5951, + "theoretical_loss": 4.020582492157265, + "tokens_seen": 418185216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007056690070210632, + "loss": 1.5839, + "theoretical_loss": 4.020513026970973, + "tokens_seen": 418250752 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070565295887663, + "loss": 1.5332, + "theoretical_loss": 4.020443575715506, + "tokens_seen": 418316288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007056369107321966, + "loss": 1.5696, + "theoretical_loss": 4.020374138385887, + "tokens_seen": 418381824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007056208625877634, + "loss": 1.6332, + "theoretical_loss": 4.020304714977145, + "tokens_seen": 418447360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070560481444333, + "loss": 1.5723, + "theoretical_loss": 4.020235305484308, + "tokens_seen": 418512896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055887662988968, + "loss": 1.5005, + "theoretical_loss": 4.020165909902409, + "tokens_seen": 418578432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055727181544634, + "loss": 1.4726, + "theoretical_loss": 4.020096528226483, + "tokens_seen": 418643968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055566700100302, + "loss": 1.59, + "theoretical_loss": 4.020027160451566, + "tokens_seen": 418709504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055406218655968, + "loss": 1.4871, + "theoretical_loss": 4.019957806572697, + "tokens_seen": 418775040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055245737211636, + "loss": 1.54, + "theoretical_loss": 4.019888466584921, + "tokens_seen": 418840576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055085255767302, + "loss": 1.4329, + "theoretical_loss": 4.019819140483281, + "tokens_seen": 418906112 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705492477432297, + "loss": 1.613, + "theoretical_loss": 4.019749828262825, + "tokens_seen": 418971648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007054764292878636, + "loss": 1.501, + "theoretical_loss": 4.0196805299186025, + "tokens_seen": 419037184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007054603811434304, + "loss": 1.3925, + "theoretical_loss": 4.019611245445666, + "tokens_seen": 419102720 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705444332998997, + "loss": 1.4554, + "theoretical_loss": 4.01954197483907, + "tokens_seen": 419168256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007054282848545638, + "loss": 1.4464, + "theoretical_loss": 4.019472718093874, + "tokens_seen": 419233792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007054122367101304, + "loss": 1.5303, + "theoretical_loss": 4.019403475205135, + "tokens_seen": 419299328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053961885656972, + "loss": 1.5512, + "theoretical_loss": 4.019334246167919, + "tokens_seen": 419364864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 145965, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.599278211593628, + "objective/train/theoretical_loss": 4.019265030977288, + "objective/train/tokens_used": 439890400, + "theoretical_loss": 4.019265030977288, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053801404212638, + "loss": 1.615, + "theoretical_loss": 4.019265030977288, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053640922768305, + "loss": 1.4472, + "theoretical_loss": 4.019195829628311, + "tokens_seen": 419495936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053480441323972, + "loss": 1.5333, + "theoretical_loss": 4.019126642116059, + "tokens_seen": 419561472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053319959879639, + "loss": 1.4841, + "theoretical_loss": 4.019057468435603, + "tokens_seen": 419627008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053159478435306, + "loss": 1.4736, + "theoretical_loss": 4.01898830858202, + "tokens_seen": 419692544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052998996990973, + "loss": 1.5241, + "theoretical_loss": 4.018919162550386, + "tokens_seen": 419758080 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705283851554664, + "loss": 1.5164, + "theoretical_loss": 4.018850030335783, + "tokens_seen": 419823616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052678034102307, + "loss": 1.4566, + "theoretical_loss": 4.018780911933293, + "tokens_seen": 419889152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052517552657974, + "loss": 1.629, + "theoretical_loss": 4.018711807338001, + "tokens_seen": 419954688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052357071213641, + "loss": 1.5485, + "theoretical_loss": 4.018642716544994, + "tokens_seen": 420020224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052196589769307, + "loss": 1.5464, + "theoretical_loss": 4.018573639549364, + "tokens_seen": 420085760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052036108324975, + "loss": 1.5059, + "theoretical_loss": 4.018504576346203, + "tokens_seen": 420151296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051875626880641, + "loss": 1.5752, + "theoretical_loss": 4.018435526930607, + "tokens_seen": 420216832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051715145436309, + "loss": 1.6088, + "theoretical_loss": 4.018366491297673, + "tokens_seen": 420282368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051554663991975, + "loss": 1.4821, + "theoretical_loss": 4.018297469442502, + "tokens_seen": 420347904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051394182547643, + "loss": 1.4243, + "theoretical_loss": 4.018228461360197, + "tokens_seen": 420413440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051233701103309, + "loss": 1.538, + "theoretical_loss": 4.018159467045862, + "tokens_seen": 420478976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051073219658977, + "loss": 1.4714, + "theoretical_loss": 4.018090486494607, + "tokens_seen": 420544512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050912738214644, + "loss": 1.4927, + "theoretical_loss": 4.0180215197015405, + "tokens_seen": 420610048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050752256770311, + "loss": 1.4405, + "theoretical_loss": 4.017952566661776, + "tokens_seen": 420675584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050591775325978, + "loss": 1.6563, + "theoretical_loss": 4.017883627370431, + "tokens_seen": 420741120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050431293881645, + "loss": 1.5569, + "theoretical_loss": 4.017814701822619, + "tokens_seen": 420806656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050270812437312, + "loss": 1.5939, + "theoretical_loss": 4.017745790013464, + "tokens_seen": 420872192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050110330992979, + "loss": 1.6454, + "theoretical_loss": 4.017676891938088, + "tokens_seen": 420937728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049949849548647, + "loss": 1.5453, + "theoretical_loss": 4.0176080075916145, + "tokens_seen": 421003264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049789368104313, + "loss": 1.5243, + "theoretical_loss": 4.017539136969174, + "tokens_seen": 421068800 + }, + { + "epoch": 0.13, + "learning_rate": 0.000704962888665998, + "loss": 1.6324, + "theoretical_loss": 4.017470280065895, + "tokens_seen": 421134336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049468405215647, + "loss": 1.4929, + "theoretical_loss": 4.017401436876911, + "tokens_seen": 421199872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049307923771315, + "loss": 1.4403, + "theoretical_loss": 4.017332607397358, + "tokens_seen": 421265408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049147442326981, + "loss": 1.4566, + "theoretical_loss": 4.017263791622373, + "tokens_seen": 421330944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048986960882649, + "loss": 1.5641, + "theoretical_loss": 4.017194989547095, + "tokens_seen": 421396480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048826479438315, + "loss": 1.4466, + "theoretical_loss": 4.017126201166668, + "tokens_seen": 421462016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048665997993983, + "loss": 1.6451, + "theoretical_loss": 4.017057426476239, + "tokens_seen": 421527552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048505516549649, + "loss": 1.5085, + "theoretical_loss": 4.016988665470953, + "tokens_seen": 421593088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048345035105317, + "loss": 1.528, + "theoretical_loss": 4.016919918145961, + "tokens_seen": 421658624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048184553660983, + "loss": 1.4832, + "theoretical_loss": 4.016851184496415, + "tokens_seen": 421724160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048024072216651, + "loss": 1.338, + "theoretical_loss": 4.016782464517472, + "tokens_seen": 421789696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047863590772317, + "loss": 1.4554, + "theoretical_loss": 4.016713758204288, + "tokens_seen": 421855232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047703109327985, + "loss": 1.5311, + "theoretical_loss": 4.016645065552023, + "tokens_seen": 421920768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047542627883651, + "loss": 1.4613, + "theoretical_loss": 4.0165763865558395, + "tokens_seen": 421986304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047382146439319, + "loss": 1.3784, + "theoretical_loss": 4.016507721210904, + "tokens_seen": 422051840 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047221664994985, + "loss": 1.5047, + "theoretical_loss": 4.016439069512382, + "tokens_seen": 422117376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047061183550653, + "loss": 1.5729, + "theoretical_loss": 4.016370431455444, + "tokens_seen": 422182912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046900702106319, + "loss": 1.5015, + "theoretical_loss": 4.016301807035264, + "tokens_seen": 422248448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046740220661987, + "loss": 1.4857, + "theoretical_loss": 4.016233196247014, + "tokens_seen": 422313984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046579739217653, + "loss": 1.4861, + "theoretical_loss": 4.016164599085872, + "tokens_seen": 422379520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046419257773321, + "loss": 1.3585, + "theoretical_loss": 4.016096015547021, + "tokens_seen": 422445056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046258776328987, + "loss": 1.7261, + "theoretical_loss": 4.016027445625638, + "tokens_seen": 422510592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046098294884655, + "loss": 1.4888, + "theoretical_loss": 4.015958889316911, + "tokens_seen": 422576128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045937813440321, + "loss": 1.505, + "theoretical_loss": 4.015890346616026, + "tokens_seen": 422641664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 147195, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.464314579963684, + "objective/train/theoretical_loss": 4.015821817518173, + "objective/train/tokens_used": 443167200, + "theoretical_loss": 4.015821817518173, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045777331995989, + "loss": 1.4434, + "theoretical_loss": 4.015821817518173, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045616850551655, + "loss": 1.5134, + "theoretical_loss": 4.015753302018544, + "tokens_seen": 422772736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045456369107323, + "loss": 1.3918, + "theoretical_loss": 4.015684800112332, + "tokens_seen": 422838272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045295887662989, + "loss": 1.4096, + "theoretical_loss": 4.015616311794735, + "tokens_seen": 422903808 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045135406218657, + "loss": 1.5184, + "theoretical_loss": 4.015547837060953, + "tokens_seen": 422969344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044974924774323, + "loss": 1.4947, + "theoretical_loss": 4.015479375906187, + "tokens_seen": 423034880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044814443329991, + "loss": 1.5867, + "theoretical_loss": 4.0154109283256405, + "tokens_seen": 423100416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044653961885657, + "loss": 1.5139, + "theoretical_loss": 4.015342494314521, + "tokens_seen": 423165952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044493480441325, + "loss": 1.5439, + "theoretical_loss": 4.015274073868037, + "tokens_seen": 423231488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044332998996991, + "loss": 1.5311, + "theoretical_loss": 4.015205666981401, + "tokens_seen": 423297024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044172517552659, + "loss": 1.5146, + "theoretical_loss": 4.015137273649826, + "tokens_seen": 423362560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044012036108325, + "loss": 1.6298, + "theoretical_loss": 4.015068893868528, + "tokens_seen": 423428096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043851554663993, + "loss": 1.3893, + "theoretical_loss": 4.015000527632726, + "tokens_seen": 423493632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043691073219659, + "loss": 1.452, + "theoretical_loss": 4.014932174937642, + "tokens_seen": 423559168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043530591775327, + "loss": 1.4128, + "theoretical_loss": 4.014863835778499, + "tokens_seen": 423624704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043370110330993, + "loss": 1.5542, + "theoretical_loss": 4.014795510150522, + "tokens_seen": 423690240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043209628886661, + "loss": 1.6039, + "theoretical_loss": 4.014727198048942, + "tokens_seen": 423755776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043049147442327, + "loss": 1.5696, + "theoretical_loss": 4.014658899468988, + "tokens_seen": 423821312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042888665997995, + "loss": 1.566, + "theoretical_loss": 4.014590614405893, + "tokens_seen": 423886848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042728184553661, + "loss": 1.5941, + "theoretical_loss": 4.014522342854895, + "tokens_seen": 423952384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042567703109329, + "loss": 1.4703, + "theoretical_loss": 4.01445408481123, + "tokens_seen": 424017920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042407221664995, + "loss": 1.5825, + "theoretical_loss": 4.014385840270139, + "tokens_seen": 424083456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042246740220663, + "loss": 1.5623, + "theoretical_loss": 4.014317609226865, + "tokens_seen": 424148992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042086258776329, + "loss": 1.466, + "theoretical_loss": 4.014249391676652, + "tokens_seen": 424214528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041925777331997, + "loss": 1.4422, + "theoretical_loss": 4.014181187614751, + "tokens_seen": 424280064 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041765295887663, + "loss": 1.4518, + "theoretical_loss": 4.01411299703641, + "tokens_seen": 424345600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041604814443331, + "loss": 1.5206, + "theoretical_loss": 4.014044819936882, + "tokens_seen": 424411136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041444332998997, + "loss": 1.4531, + "theoretical_loss": 4.013976656311422, + "tokens_seen": 424476672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041283851554665, + "loss": 1.5651, + "theoretical_loss": 4.013908506155287, + "tokens_seen": 424542208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041123370110331, + "loss": 1.5244, + "theoretical_loss": 4.013840369463737, + "tokens_seen": 424607744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040962888665999, + "loss": 1.5536, + "theoretical_loss": 4.013772246232035, + "tokens_seen": 424673280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040802407221665, + "loss": 1.4733, + "theoretical_loss": 4.013704136455445, + "tokens_seen": 424738816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040641925777333, + "loss": 1.4762, + "theoretical_loss": 4.013636040129233, + "tokens_seen": 424804352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040481444332999, + "loss": 1.4481, + "theoretical_loss": 4.0135679572486715, + "tokens_seen": 424869888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040320962888667, + "loss": 1.5199, + "theoretical_loss": 4.013499887809028, + "tokens_seen": 424935424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040160481444333, + "loss": 1.5524, + "theoretical_loss": 4.01343183180558, + "tokens_seen": 425000960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040000000000001, + "loss": 1.5354, + "theoretical_loss": 4.013363789233602, + "tokens_seen": 425066496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039839518555667, + "loss": 1.4442, + "theoretical_loss": 4.013295760088374, + "tokens_seen": 425132032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039679037111334, + "loss": 1.4176, + "theoretical_loss": 4.013227744365178, + "tokens_seen": 425197568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039518555667001, + "loss": 1.4971, + "theoretical_loss": 4.013159742059296, + "tokens_seen": 425263104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039358074222668, + "loss": 1.4715, + "theoretical_loss": 4.013091753166015, + "tokens_seen": 425328640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039197592778335, + "loss": 1.5929, + "theoretical_loss": 4.013023777680624, + "tokens_seen": 425394176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039037111334002, + "loss": 1.5164, + "theoretical_loss": 4.0129558155984135, + "tokens_seen": 425459712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038876629889669, + "loss": 1.395, + "theoretical_loss": 4.012887866914675, + "tokens_seen": 425525248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038716148445336, + "loss": 1.4983, + "theoretical_loss": 4.0128199316247075, + "tokens_seen": 425590784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038555667001003, + "loss": 1.5055, + "theoretical_loss": 4.0127520097238065, + "tokens_seen": 425656320 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703839518555667, + "loss": 1.5522, + "theoretical_loss": 4.012684101207273, + "tokens_seen": 425721856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038234704112337, + "loss": 1.3876, + "theoretical_loss": 4.01261620607041, + "tokens_seen": 425787392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038074222668004, + "loss": 1.5193, + "theoretical_loss": 4.012548324308522, + "tokens_seen": 425852928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037913741223671, + "loss": 1.5764, + "theoretical_loss": 4.012480455916918, + "tokens_seen": 425918464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 148172, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.435992956161499, + "objective/train/theoretical_loss": 4.012412600890906, + "objective/train/tokens_used": 446444000, + "theoretical_loss": 4.012412600890906, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037753259779338, + "loss": 1.5679, + "theoretical_loss": 4.012412600890906, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037592778335005, + "loss": 1.5018, + "theoretical_loss": 4.012344759225799, + "tokens_seen": 426049536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037432296890672, + "loss": 1.5631, + "theoretical_loss": 4.0122769309169115, + "tokens_seen": 426115072 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703727181544634, + "loss": 1.5465, + "theoretical_loss": 4.0122091159595605, + "tokens_seen": 426180608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037111334002006, + "loss": 1.4451, + "theoretical_loss": 4.0121413143490665, + "tokens_seen": 426246144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036950852557673, + "loss": 1.479, + "theoretical_loss": 4.012073526080749, + "tokens_seen": 426311680 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703679037111334, + "loss": 1.5421, + "theoretical_loss": 4.012005751149934, + "tokens_seen": 426377216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036629889669007, + "loss": 1.416, + "theoretical_loss": 4.011937989551946, + "tokens_seen": 426442752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036469408224674, + "loss": 1.5782, + "theoretical_loss": 4.011870241282114, + "tokens_seen": 426508288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036308926780342, + "loss": 1.4749, + "theoretical_loss": 4.011802506335771, + "tokens_seen": 426573824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036148445336008, + "loss": 1.515, + "theoretical_loss": 4.011734784708249, + "tokens_seen": 426639360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035987963891676, + "loss": 1.5876, + "theoretical_loss": 4.011667076394884, + "tokens_seen": 426704896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035827482447342, + "loss": 1.4823, + "theoretical_loss": 4.011599381391014, + "tokens_seen": 426770432 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703566700100301, + "loss": 1.5256, + "theoretical_loss": 4.0115316996919805, + "tokens_seen": 426835968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035506519558676, + "loss": 1.4832, + "theoretical_loss": 4.011464031293125, + "tokens_seen": 426901504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035346038114344, + "loss": 1.4891, + "theoretical_loss": 4.0113963761897935, + "tokens_seen": 426967040 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703518555667001, + "loss": 1.4984, + "theoretical_loss": 4.011328734377333, + "tokens_seen": 427032576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035025075225678, + "loss": 1.4975, + "theoretical_loss": 4.011261105851093, + "tokens_seen": 427098112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034864593781344, + "loss": 1.51, + "theoretical_loss": 4.011193490606427, + "tokens_seen": 427163648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034704112337012, + "loss": 1.5087, + "theoretical_loss": 4.01112588863869, + "tokens_seen": 427229184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034543630892678, + "loss": 1.4949, + "theoretical_loss": 4.011058299943237, + "tokens_seen": 427294720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034383149448346, + "loss": 1.501, + "theoretical_loss": 4.010990724515427, + "tokens_seen": 427360256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034222668004012, + "loss": 1.4712, + "theoretical_loss": 4.010923162350624, + "tokens_seen": 427425792 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703406218655968, + "loss": 1.5337, + "theoretical_loss": 4.01085561344419, + "tokens_seen": 427491328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033901705115346, + "loss": 1.5073, + "theoretical_loss": 4.010788077791491, + "tokens_seen": 427556864 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033741223671014, + "loss": 1.4828, + "theoretical_loss": 4.0107205553878975, + "tokens_seen": 427622400 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703358074222668, + "loss": 1.5587, + "theoretical_loss": 4.010653046228779, + "tokens_seen": 427687936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033420260782348, + "loss": 1.4948, + "theoretical_loss": 4.010585550309508, + "tokens_seen": 427753472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033259779338014, + "loss": 1.4606, + "theoretical_loss": 4.010518067625461, + "tokens_seen": 427819008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033099297893682, + "loss": 1.5048, + "theoretical_loss": 4.010450598172016, + "tokens_seen": 427884544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032938816449348, + "loss": 1.6048, + "theoretical_loss": 4.010383141944553, + "tokens_seen": 427950080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032778335005016, + "loss": 1.567, + "theoretical_loss": 4.010315698938455, + "tokens_seen": 428015616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032617853560682, + "loss": 1.4507, + "theoretical_loss": 4.0102482691491055, + "tokens_seen": 428081152 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703245737211635, + "loss": 1.5643, + "theoretical_loss": 4.010180852571893, + "tokens_seen": 428146688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032296890672016, + "loss": 1.5148, + "theoretical_loss": 4.010113449202206, + "tokens_seen": 428212224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032136409227684, + "loss": 1.4841, + "theoretical_loss": 4.010046059035437, + "tokens_seen": 428277760 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703197592778335, + "loss": 1.438, + "theoretical_loss": 4.009978682066979, + "tokens_seen": 428343296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031815446339018, + "loss": 1.5299, + "theoretical_loss": 4.00991131829223, + "tokens_seen": 428408832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031654964894684, + "loss": 1.5179, + "theoretical_loss": 4.009843967706586, + "tokens_seen": 428474368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031494483450352, + "loss": 1.4384, + "theoretical_loss": 4.009776630305451, + "tokens_seen": 428539904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031334002006018, + "loss": 1.4572, + "theoretical_loss": 4.009709306084227, + "tokens_seen": 428605440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031173520561686, + "loss": 1.5282, + "theoretical_loss": 4.00964199503832, + "tokens_seen": 428670976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031013039117352, + "loss": 1.476, + "theoretical_loss": 4.009574697163137, + "tokens_seen": 428736512 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703085255767302, + "loss": 1.6019, + "theoretical_loss": 4.00950741245409, + "tokens_seen": 428802048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007030692076228686, + "loss": 1.4104, + "theoretical_loss": 4.009440140906589, + "tokens_seen": 428867584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007030531594784354, + "loss": 1.6, + "theoretical_loss": 4.00937288251605, + "tokens_seen": 428933120 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703037111334002, + "loss": 1.5577, + "theoretical_loss": 4.009305637277892, + "tokens_seen": 428998656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007030210631895688, + "loss": 1.5977, + "theoretical_loss": 4.009238405187531, + "tokens_seen": 429064192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007030050150451354, + "loss": 1.57, + "theoretical_loss": 4.009171186240391, + "tokens_seen": 429129728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029889669007022, + "loss": 1.478, + "theoretical_loss": 4.009103980431896, + "tokens_seen": 429195264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 149293, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.394667625427246, + "objective/train/theoretical_loss": 4.009036787757472, + "objective/train/tokens_used": 449720800, + "theoretical_loss": 4.009036787757472, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029729187562688, + "loss": 1.4451, + "theoretical_loss": 4.009036787757472, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029568706118356, + "loss": 1.6264, + "theoretical_loss": 4.0089696082125466, + "tokens_seen": 429326336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029408224674022, + "loss": 1.6049, + "theoretical_loss": 4.008902441792552, + "tokens_seen": 429391872 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702924774322969, + "loss": 1.6117, + "theoretical_loss": 4.00883528849292, + "tokens_seen": 429457408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029087261785356, + "loss": 1.6004, + "theoretical_loss": 4.008768148309087, + "tokens_seen": 429522944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028926780341024, + "loss": 1.5711, + "theoretical_loss": 4.00870102123649, + "tokens_seen": 429588480 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702876629889669, + "loss": 1.4821, + "theoretical_loss": 4.008633907270571, + "tokens_seen": 429654016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028605817452358, + "loss": 1.5774, + "theoretical_loss": 4.008566806406771, + "tokens_seen": 429719552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028445336008024, + "loss": 1.3801, + "theoretical_loss": 4.0084997186405324, + "tokens_seen": 429785088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028284854563692, + "loss": 1.5135, + "theoretical_loss": 4.008432643967305, + "tokens_seen": 429850624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028124373119358, + "loss": 1.4067, + "theoretical_loss": 4.008365582382538, + "tokens_seen": 429916160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027963891675026, + "loss": 1.6318, + "theoretical_loss": 4.008298533881681, + "tokens_seen": 429981696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027803410230692, + "loss": 1.5392, + "theoretical_loss": 4.008231498460189, + "tokens_seen": 430047232 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702764292878636, + "loss": 1.4467, + "theoretical_loss": 4.008164476113517, + "tokens_seen": 430112768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027482447342026, + "loss": 1.5168, + "theoretical_loss": 4.008097466837124, + "tokens_seen": 430178304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027321965897694, + "loss": 1.5886, + "theoretical_loss": 4.00803047062647, + "tokens_seen": 430243840 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702716148445336, + "loss": 1.5773, + "theoretical_loss": 4.007963487477018, + "tokens_seen": 430309376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027001003009028, + "loss": 1.5376, + "theoretical_loss": 4.007896517384233, + "tokens_seen": 430374912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026840521564694, + "loss": 1.5965, + "theoretical_loss": 4.007829560343582, + "tokens_seen": 430440448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026680040120362, + "loss": 1.5795, + "theoretical_loss": 4.007762616350535, + "tokens_seen": 430505984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026519558676028, + "loss": 1.4647, + "theoretical_loss": 4.007695685400563, + "tokens_seen": 430571520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026359077231696, + "loss": 1.552, + "theoretical_loss": 4.007628767489142, + "tokens_seen": 430637056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026198595787362, + "loss": 1.4869, + "theoretical_loss": 4.007561862611746, + "tokens_seen": 430702592 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702603811434303, + "loss": 1.4319, + "theoretical_loss": 4.007494970763855, + "tokens_seen": 430768128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025877632898696, + "loss": 1.5502, + "theoretical_loss": 4.00742809194095, + "tokens_seen": 430833664 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025717151454363, + "loss": 1.5765, + "theoretical_loss": 4.007361226138513, + "tokens_seen": 430899200 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702555667001003, + "loss": 1.4739, + "theoretical_loss": 4.00729437335203, + "tokens_seen": 430964736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025396188565697, + "loss": 1.3935, + "theoretical_loss": 4.007227533576988, + "tokens_seen": 431030272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025235707121364, + "loss": 1.5863, + "theoretical_loss": 4.007160706808879, + "tokens_seen": 431095808 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025075225677031, + "loss": 1.4851, + "theoretical_loss": 4.007093893043192, + "tokens_seen": 431161344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024914744232698, + "loss": 1.5307, + "theoretical_loss": 4.007027092275424, + "tokens_seen": 431226880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024754262788365, + "loss": 1.5869, + "theoretical_loss": 4.006960304501069, + "tokens_seen": 431292416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024593781344032, + "loss": 1.5409, + "theoretical_loss": 4.0068935297156285, + "tokens_seen": 431357952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024433299899699, + "loss": 1.572, + "theoretical_loss": 4.0068267679146015, + "tokens_seen": 431423488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024272818455366, + "loss": 1.4711, + "theoretical_loss": 4.006760019093493, + "tokens_seen": 431489024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024112337011033, + "loss": 1.4946, + "theoretical_loss": 4.006693283247808, + "tokens_seen": 431554560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070239518555667, + "loss": 1.4717, + "theoretical_loss": 4.0066265603730535, + "tokens_seen": 431620096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023791374122367, + "loss": 1.5803, + "theoretical_loss": 4.006559850464741, + "tokens_seen": 431685632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023630892678034, + "loss": 1.652, + "theoretical_loss": 4.006493153518381, + "tokens_seen": 431751168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023470411233701, + "loss": 1.5243, + "theoretical_loss": 4.006426469529488, + "tokens_seen": 431816704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023309929789368, + "loss": 1.4888, + "theoretical_loss": 4.006359798493582, + "tokens_seen": 431882240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023149448345035, + "loss": 1.5452, + "theoretical_loss": 4.006293140406179, + "tokens_seen": 431947776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022988966900702, + "loss": 1.5458, + "theoretical_loss": 4.0062264952627995, + "tokens_seen": 432013312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022828485456369, + "loss": 1.6067, + "theoretical_loss": 4.00615986305897, + "tokens_seen": 432078848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022668004012037, + "loss": 1.5937, + "theoretical_loss": 4.006093243790214, + "tokens_seen": 432144384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022507522567703, + "loss": 1.6446, + "theoretical_loss": 4.0060266374520594, + "tokens_seen": 432209920 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702234704112337, + "loss": 1.5289, + "theoretical_loss": 4.005960044040037, + "tokens_seen": 432275456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022186559679037, + "loss": 1.5373, + "theoretical_loss": 4.005893463549679, + "tokens_seen": 432340992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022026078234705, + "loss": 1.502, + "theoretical_loss": 4.0058268959765195, + "tokens_seen": 432406528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021865596790371, + "loss": 1.4267, + "theoretical_loss": 4.005760341316096, + "tokens_seen": 432472064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 150155, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2991299629211426, + "objective/train/theoretical_loss": 4.0056937995639466, + "objective/train/tokens_used": 452997600, + "theoretical_loss": 4.0056937995639466, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021705115346039, + "loss": 1.4196, + "theoretical_loss": 4.0056937995639466, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021544633901705, + "loss": 1.5659, + "theoretical_loss": 4.005627270715614, + "tokens_seen": 432603136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021384152457373, + "loss": 1.5304, + "theoretical_loss": 4.00556075476664, + "tokens_seen": 432668672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021223671013039, + "loss": 1.5103, + "theoretical_loss": 4.005494251712571, + "tokens_seen": 432734208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021063189568707, + "loss": 1.4877, + "theoretical_loss": 4.005427761548955, + "tokens_seen": 432799744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020902708124373, + "loss": 1.5152, + "theoretical_loss": 4.005361284271341, + "tokens_seen": 432865280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020742226680041, + "loss": 1.3662, + "theoretical_loss": 4.005294819875283, + "tokens_seen": 432930816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020581745235707, + "loss": 1.5065, + "theoretical_loss": 4.005228368356335, + "tokens_seen": 432996352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020421263791375, + "loss": 1.5918, + "theoretical_loss": 4.005161929710052, + "tokens_seen": 433061888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020260782347041, + "loss": 1.5184, + "theoretical_loss": 4.0050955039319955, + "tokens_seen": 433127424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020100300902709, + "loss": 1.4619, + "theoretical_loss": 4.0050290910177235, + "tokens_seen": 433192960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019939819458375, + "loss": 1.5408, + "theoretical_loss": 4.004962690962802, + "tokens_seen": 433258496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019779338014043, + "loss": 1.4712, + "theoretical_loss": 4.004896303762797, + "tokens_seen": 433324032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019618856569709, + "loss": 1.4964, + "theoretical_loss": 4.0048299294132725, + "tokens_seen": 433389568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019458375125377, + "loss": 1.5561, + "theoretical_loss": 4.0047635679098015, + "tokens_seen": 433455104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019297893681043, + "loss": 1.4142, + "theoretical_loss": 4.004697219247955, + "tokens_seen": 433520640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019137412236711, + "loss": 1.5248, + "theoretical_loss": 4.004630883423308, + "tokens_seen": 433586176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018976930792377, + "loss": 1.5179, + "theoretical_loss": 4.004564560431437, + "tokens_seen": 433651712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018816449348045, + "loss": 1.6083, + "theoretical_loss": 4.004498250267919, + "tokens_seen": 433717248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018655967903711, + "loss": 1.4272, + "theoretical_loss": 4.004431952928337, + "tokens_seen": 433782784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018495486459379, + "loss": 1.5259, + "theoretical_loss": 4.004365668408273, + "tokens_seen": 433848320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018335005015045, + "loss": 1.4605, + "theoretical_loss": 4.004299396703312, + "tokens_seen": 433913856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018174523570713, + "loss": 1.5143, + "theoretical_loss": 4.004233137809042, + "tokens_seen": 433979392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018014042126379, + "loss": 1.5833, + "theoretical_loss": 4.004166891721052, + "tokens_seen": 434044928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017853560682047, + "loss": 1.4719, + "theoretical_loss": 4.004100658434934, + "tokens_seen": 434110464 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017693079237713, + "loss": 1.5535, + "theoretical_loss": 4.004034437946283, + "tokens_seen": 434176000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017532597793381, + "loss": 1.4578, + "theoretical_loss": 4.003968230250694, + "tokens_seen": 434241536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017372116349047, + "loss": 1.5173, + "theoretical_loss": 4.003902035343766, + "tokens_seen": 434307072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017211634904715, + "loss": 1.5396, + "theoretical_loss": 4.003835853221099, + "tokens_seen": 434372608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017051153460381, + "loss": 1.3807, + "theoretical_loss": 4.003769683878296, + "tokens_seen": 434438144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016890672016049, + "loss": 1.4859, + "theoretical_loss": 4.003703527310962, + "tokens_seen": 434503680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016730190571715, + "loss": 1.4641, + "theoretical_loss": 4.003637383514704, + "tokens_seen": 434569216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016569709127383, + "loss": 1.5701, + "theoretical_loss": 4.00357125248513, + "tokens_seen": 434634752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016409227683049, + "loss": 1.5586, + "theoretical_loss": 4.003505134217853, + "tokens_seen": 434700288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016248746238717, + "loss": 1.5719, + "theoretical_loss": 4.0034390287084864, + "tokens_seen": 434765824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016088264794383, + "loss": 1.6704, + "theoretical_loss": 4.003372935952646, + "tokens_seen": 434831360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015927783350051, + "loss": 1.5484, + "theoretical_loss": 4.003306855945948, + "tokens_seen": 434896896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015767301905717, + "loss": 1.4919, + "theoretical_loss": 4.003240788684015, + "tokens_seen": 434962432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015606820461385, + "loss": 1.5402, + "theoretical_loss": 4.003174734162467, + "tokens_seen": 435027968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015446339017051, + "loss": 1.515, + "theoretical_loss": 4.00310869237693, + "tokens_seen": 435093504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015285857572719, + "loss": 1.5989, + "theoretical_loss": 4.003042663323029, + "tokens_seen": 435159040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015125376128385, + "loss": 1.4633, + "theoretical_loss": 4.0029766469963945, + "tokens_seen": 435224576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014964894684053, + "loss": 1.4019, + "theoretical_loss": 4.0029106433926565, + "tokens_seen": 435290112 + }, + { + "epoch": 0.13, + "learning_rate": 0.000701480441323972, + "loss": 1.5274, + "theoretical_loss": 4.002844652507449, + "tokens_seen": 435355648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014643931795387, + "loss": 1.4794, + "theoretical_loss": 4.002778674336405, + "tokens_seen": 435421184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014483450351054, + "loss": 1.5274, + "theoretical_loss": 4.002712708875162, + "tokens_seen": 435486720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014322968906721, + "loss": 1.5422, + "theoretical_loss": 4.002646756119362, + "tokens_seen": 435552256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014162487462388, + "loss": 1.5547, + "theoretical_loss": 4.002580816064645, + "tokens_seen": 435617792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014002006018055, + "loss": 1.5411, + "theoretical_loss": 4.002514888706655, + "tokens_seen": 435683328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013841524573722, + "loss": 1.374, + "theoretical_loss": 4.002448974041037, + "tokens_seen": 435748864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 151381, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.41636323928833, + "objective/train/theoretical_loss": 4.0023830720634415, + "objective/train/tokens_used": 456274400, + "theoretical_loss": 4.0023830720634415, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013681043129389, + "loss": 1.3912, + "theoretical_loss": 4.0023830720634415, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013520561685056, + "loss": 1.4124, + "theoretical_loss": 4.0023171827695165, + "tokens_seen": 435879936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013360080240723, + "loss": 1.4398, + "theoretical_loss": 4.002251306154916, + "tokens_seen": 435945472 + }, + { + "epoch": 0.13, + "learning_rate": 0.000701319959879639, + "loss": 1.5039, + "theoretical_loss": 4.002185442215293, + "tokens_seen": 436011008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013039117352057, + "loss": 1.4421, + "theoretical_loss": 4.002119590946305, + "tokens_seen": 436076544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012878635907723, + "loss": 1.5099, + "theoretical_loss": 4.002053752343611, + "tokens_seen": 436142080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012718154463391, + "loss": 1.523, + "theoretical_loss": 4.001987926402871, + "tokens_seen": 436207616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012557673019057, + "loss": 1.46, + "theoretical_loss": 4.001922113119749, + "tokens_seen": 436273152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012397191574725, + "loss": 1.542, + "theoretical_loss": 4.00185631248991, + "tokens_seen": 436338688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012236710130391, + "loss": 1.4672, + "theoretical_loss": 4.001790524509021, + "tokens_seen": 436404224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012076228686059, + "loss": 1.5367, + "theoretical_loss": 4.001724749172752, + "tokens_seen": 436469760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011915747241725, + "loss": 1.5853, + "theoretical_loss": 4.001658986476775, + "tokens_seen": 436535296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011755265797393, + "loss": 1.4906, + "theoretical_loss": 4.0015932364167615, + "tokens_seen": 436600832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011594784353059, + "loss": 1.5542, + "theoretical_loss": 4.00152749898839, + "tokens_seen": 436666368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011434302908726, + "loss": 1.3937, + "theoretical_loss": 4.001461774187338, + "tokens_seen": 436731904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011273821464393, + "loss": 1.6308, + "theoretical_loss": 4.001396062009285, + "tokens_seen": 436797440 + }, + { + "epoch": 0.13, + "learning_rate": 0.000701111334002006, + "loss": 1.5693, + "theoretical_loss": 4.001330362449914, + "tokens_seen": 436862976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010952858575727, + "loss": 1.4551, + "theoretical_loss": 4.001264675504908, + "tokens_seen": 436928512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010792377131394, + "loss": 1.5887, + "theoretical_loss": 4.001199001169955, + "tokens_seen": 436994048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010631895687061, + "loss": 1.5951, + "theoretical_loss": 4.001133339440743, + "tokens_seen": 437059584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010471414242728, + "loss": 1.448, + "theoretical_loss": 4.001067690312963, + "tokens_seen": 437125120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010310932798395, + "loss": 1.4664, + "theoretical_loss": 4.001002053782307, + "tokens_seen": 437190656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010150451354062, + "loss": 1.4245, + "theoretical_loss": 4.0009364298444705, + "tokens_seen": 437256192 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700998996990973, + "loss": 1.464, + "theoretical_loss": 4.000870818495152, + "tokens_seen": 437321728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009829488465396, + "loss": 1.5072, + "theoretical_loss": 4.000805219730049, + "tokens_seen": 437387264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009669007021063, + "loss": 1.3711, + "theoretical_loss": 4.000739633544863, + "tokens_seen": 437452800 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700950852557673, + "loss": 1.5206, + "theoretical_loss": 4.000674059935298, + "tokens_seen": 437518336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009348044132397, + "loss": 1.5586, + "theoretical_loss": 4.000608498897059, + "tokens_seen": 437583872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009187562688064, + "loss": 1.6071, + "theoretical_loss": 4.000542950425855, + "tokens_seen": 437649408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009027081243731, + "loss": 1.4701, + "theoretical_loss": 4.000477414517394, + "tokens_seen": 437714944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008866599799398, + "loss": 1.4597, + "theoretical_loss": 4.000411891167389, + "tokens_seen": 437780480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008706118355066, + "loss": 1.5, + "theoretical_loss": 4.000346380371554, + "tokens_seen": 437846016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008545636910732, + "loss": 1.4292, + "theoretical_loss": 4.000280882125605, + "tokens_seen": 437911552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070083851554664, + "loss": 1.45, + "theoretical_loss": 4.000215396425259, + "tokens_seen": 437977088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008224674022066, + "loss": 1.5011, + "theoretical_loss": 4.000149923266237, + "tokens_seen": 438042624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008064192577734, + "loss": 1.4882, + "theoretical_loss": 4.0000844626442635, + "tokens_seen": 438108160 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070079037111334, + "loss": 1.5182, + "theoretical_loss": 4.000019014555059, + "tokens_seen": 438173696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007743229689068, + "loss": 1.402, + "theoretical_loss": 3.9999535789943543, + "tokens_seen": 438239232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007582748244734, + "loss": 1.591, + "theoretical_loss": 3.9998881559578754, + "tokens_seen": 438304768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007422266800402, + "loss": 1.4926, + "theoretical_loss": 3.999822745441353, + "tokens_seen": 438370304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007261785356068, + "loss": 1.504, + "theoretical_loss": 3.9997573474405215, + "tokens_seen": 438435840 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007101303911736, + "loss": 1.4095, + "theoretical_loss": 3.9996919619511146, + "tokens_seen": 438501376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007006940822467402, + "loss": 1.5043, + "theoretical_loss": 3.9996265889688702, + "tokens_seen": 438566912 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700678034102307, + "loss": 1.4652, + "theoretical_loss": 3.999561228489527, + "tokens_seen": 438632448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007006619859578736, + "loss": 1.444, + "theoretical_loss": 3.9994958805088268, + "tokens_seen": 438697984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007006459378134404, + "loss": 1.5337, + "theoretical_loss": 3.999430545022512, + "tokens_seen": 438763520 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700629889669007, + "loss": 1.5445, + "theoretical_loss": 3.999365222026329, + "tokens_seen": 438829056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007006138415245738, + "loss": 1.689, + "theoretical_loss": 3.999299911516024, + "tokens_seen": 438894592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005977933801404, + "loss": 1.5859, + "theoretical_loss": 3.9992346134873484, + "tokens_seen": 438960128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005817452357072, + "loss": 1.4005, + "theoretical_loss": 3.999169327936052, + "tokens_seen": 439025664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 152335, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6528295278549194, + "objective/train/theoretical_loss": 3.99910405485789, + "objective/train/tokens_used": 459551200, + "theoretical_loss": 3.99910405485789, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005656970912738, + "loss": 1.5898, + "theoretical_loss": 3.99910405485789, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005496489468406, + "loss": 1.601, + "theoretical_loss": 3.9990387942486167, + "tokens_seen": 439156736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005336008024072, + "loss": 1.5703, + "theoretical_loss": 3.9989735461039917, + "tokens_seen": 439222272 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700517552657974, + "loss": 1.5155, + "theoretical_loss": 3.9989083104197736, + "tokens_seen": 439287808 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005015045135406, + "loss": 1.5189, + "theoretical_loss": 3.9988430871917253, + "tokens_seen": 439353344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004854563691074, + "loss": 1.4124, + "theoretical_loss": 3.9987778764156108, + "tokens_seen": 439418880 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700469408224674, + "loss": 1.4934, + "theoretical_loss": 3.9987126780871955, + "tokens_seen": 439484416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004533600802408, + "loss": 1.4822, + "theoretical_loss": 3.998647492202249, + "tokens_seen": 439549952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004373119358074, + "loss": 1.4562, + "theoretical_loss": 3.99858231875654, + "tokens_seen": 439615488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004212637913742, + "loss": 1.6713, + "theoretical_loss": 3.9985171577458427, + "tokens_seen": 439681024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004052156469408, + "loss": 1.4871, + "theoretical_loss": 3.99845200916593, + "tokens_seen": 439746560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007003891675025076, + "loss": 1.5559, + "theoretical_loss": 3.9983868730125796, + "tokens_seen": 439812096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007003731193580742, + "loss": 1.4817, + "theoretical_loss": 3.998321749281569, + "tokens_seen": 439877632 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700357071213641, + "loss": 1.6303, + "theoretical_loss": 3.99825663796868, + "tokens_seen": 439943168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007003410230692076, + "loss": 1.5342, + "theoretical_loss": 3.998191539069694, + "tokens_seen": 440008704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007003249749247744, + "loss": 1.4689, + "theoretical_loss": 3.9981264525803972, + "tokens_seen": 440074240 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700308926780341, + "loss": 1.636, + "theoretical_loss": 3.9980613784965753, + "tokens_seen": 440139776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002928786359078, + "loss": 1.4691, + "theoretical_loss": 3.9979963168140182, + "tokens_seen": 440205312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002768304914745, + "loss": 1.5015, + "theoretical_loss": 3.997931267528516, + "tokens_seen": 440270848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002607823470412, + "loss": 1.5555, + "theoretical_loss": 3.997866230635862, + "tokens_seen": 440336384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002447342026079, + "loss": 1.4922, + "theoretical_loss": 3.9978012061318515, + "tokens_seen": 440401920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002286860581746, + "loss": 1.5052, + "theoretical_loss": 3.997736194012281, + "tokens_seen": 440467456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002126379137413, + "loss": 1.4865, + "theoretical_loss": 3.99767119427295, + "tokens_seen": 440532992 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700196589769308, + "loss": 1.5147, + "theoretical_loss": 3.9976062069096603, + "tokens_seen": 440598528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001805416248747, + "loss": 1.5291, + "theoretical_loss": 3.9975412319182144, + "tokens_seen": 440664064 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001644934804414, + "loss": 1.4955, + "theoretical_loss": 3.9974762692944172, + "tokens_seen": 440729600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001484453360081, + "loss": 1.5683, + "theoretical_loss": 3.997411319034077, + "tokens_seen": 440795136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001323971915748, + "loss": 1.4643, + "theoretical_loss": 3.9973463811330037, + "tokens_seen": 440860672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001163490471415, + "loss": 1.436, + "theoretical_loss": 3.997281455587007, + "tokens_seen": 440926208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001003009027082, + "loss": 1.6007, + "theoretical_loss": 3.9972165423919015, + "tokens_seen": 440991744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000842527582749, + "loss": 1.4685, + "theoretical_loss": 3.9971516415435024, + "tokens_seen": 441057280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000682046138416, + "loss": 1.5206, + "theoretical_loss": 3.9970867530376273, + "tokens_seen": 441122816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000521564694083, + "loss": 1.4993, + "theoretical_loss": 3.9970218768700967, + "tokens_seen": 441188352 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700036108324975, + "loss": 1.48, + "theoretical_loss": 3.99695701303673, + "tokens_seen": 441253888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000200601805417, + "loss": 1.4757, + "theoretical_loss": 3.9968921615333532, + "tokens_seen": 441319424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000040120361084, + "loss": 1.5006, + "theoretical_loss": 3.996827322355791, + "tokens_seen": 441384960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999879638916751, + "loss": 1.432, + "theoretical_loss": 3.9967624954998713, + "tokens_seen": 441450496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999719157472418, + "loss": 1.7472, + "theoretical_loss": 3.9966976809614234, + "tokens_seen": 441516032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999558676028085, + "loss": 1.4991, + "theoretical_loss": 3.99663287873628, + "tokens_seen": 441581568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999398194583752, + "loss": 1.5962, + "theoretical_loss": 3.9965680888202737, + "tokens_seen": 441647104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999237713139419, + "loss": 1.4266, + "theoretical_loss": 3.996503311209242, + "tokens_seen": 441712640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999077231695086, + "loss": 1.587, + "theoretical_loss": 3.9964385458990206, + "tokens_seen": 441778176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006998916750250752, + "loss": 1.5794, + "theoretical_loss": 3.9963737928854517, + "tokens_seen": 441843712 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699875626880642, + "loss": 1.4391, + "theoretical_loss": 3.996309052164376, + "tokens_seen": 441909248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006998595787362086, + "loss": 1.5746, + "theoretical_loss": 3.996244323731638, + "tokens_seen": 441974784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006998435305917754, + "loss": 1.5798, + "theoretical_loss": 3.996179607583082, + "tokens_seen": 442040320 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699827482447342, + "loss": 1.3749, + "theoretical_loss": 3.9961149037145587, + "tokens_seen": 442105856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006998114343029088, + "loss": 1.554, + "theoretical_loss": 3.996050212121916, + "tokens_seen": 442171392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997953861584754, + "loss": 1.4281, + "theoretical_loss": 3.995985532801007, + "tokens_seen": 442236928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997793380140422, + "loss": 1.5208, + "theoretical_loss": 3.995920865747685, + "tokens_seen": 442302464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 153589, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5176739692687988, + "objective/train/theoretical_loss": 3.9958562109578066, + "objective/train/tokens_used": 462828000, + "theoretical_loss": 3.9958562109578066, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997632898696088, + "loss": 1.4968, + "theoretical_loss": 3.9958562109578066, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997472417251755, + "loss": 1.5038, + "theoretical_loss": 3.9957915684272294, + "tokens_seen": 442433536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997311935807422, + "loss": 1.4439, + "theoretical_loss": 3.995726938151815, + "tokens_seen": 442499072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997151454363089, + "loss": 1.644, + "theoretical_loss": 3.995662320127423, + "tokens_seen": 442564608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996990972918756, + "loss": 1.4451, + "theoretical_loss": 3.9955977143499197, + "tokens_seen": 442630144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996830491474423, + "loss": 1.612, + "theoretical_loss": 3.99553312081517, + "tokens_seen": 442695680 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699667001003009, + "loss": 1.6093, + "theoretical_loss": 3.995468539519042, + "tokens_seen": 442761216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996509528585757, + "loss": 1.4835, + "theoretical_loss": 3.9954039704574065, + "tokens_seen": 442826752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996349047141424, + "loss": 1.6179, + "theoretical_loss": 3.9953394136261355, + "tokens_seen": 442892288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996188565697091, + "loss": 1.5285, + "theoretical_loss": 3.995274869021103, + "tokens_seen": 442957824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996028084252758, + "loss": 1.5937, + "theoretical_loss": 3.995210336638185, + "tokens_seen": 443023360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995867602808425, + "loss": 1.6425, + "theoretical_loss": 3.9951458164732596, + "tokens_seen": 443088896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995707121364092, + "loss": 1.468, + "theoretical_loss": 3.9950813085222068, + "tokens_seen": 443154432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995546639919759, + "loss": 1.4978, + "theoretical_loss": 3.99501681278091, + "tokens_seen": 443219968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995386158475426, + "loss": 1.4505, + "theoretical_loss": 3.994952329245251, + "tokens_seen": 443285504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995225677031093, + "loss": 1.5477, + "theoretical_loss": 3.994887857911118, + "tokens_seen": 443351040 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699506519558676, + "loss": 1.5128, + "theoretical_loss": 3.9948233987743986, + "tokens_seen": 443416576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994904714142427, + "loss": 1.529, + "theoretical_loss": 3.994758951830982, + "tokens_seen": 443482112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994744232698095, + "loss": 1.5579, + "theoretical_loss": 3.9946945170767614, + "tokens_seen": 443547648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994583751253761, + "loss": 1.4624, + "theoretical_loss": 3.994630094507631, + "tokens_seen": 443613184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994423269809429, + "loss": 1.5406, + "theoretical_loss": 3.994565684119486, + "tokens_seen": 443678720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994262788365095, + "loss": 1.5907, + "theoretical_loss": 3.9945012859082247, + "tokens_seen": 443744256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994102306920763, + "loss": 1.4722, + "theoretical_loss": 3.9944368998697475, + "tokens_seen": 443809792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993941825476429, + "loss": 1.4417, + "theoretical_loss": 3.9943725259999563, + "tokens_seen": 443875328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993781344032097, + "loss": 1.5681, + "theoretical_loss": 3.994308164294755, + "tokens_seen": 443940864 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993620862587763, + "loss": 1.3864, + "theoretical_loss": 3.9942438147500496, + "tokens_seen": 444006400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993460381143431, + "loss": 1.5363, + "theoretical_loss": 3.9941794773617487, + "tokens_seen": 444071936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993299899699097, + "loss": 1.572, + "theoretical_loss": 3.9941151521257616, + "tokens_seen": 444137472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993139418254765, + "loss": 1.4912, + "theoretical_loss": 3.9940508390380005, + "tokens_seen": 444203008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992978936810431, + "loss": 1.6135, + "theoretical_loss": 3.9939865380943793, + "tokens_seen": 444268544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992818455366099, + "loss": 1.5042, + "theoretical_loss": 3.993922249290814, + "tokens_seen": 444334080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992657973921765, + "loss": 1.4777, + "theoretical_loss": 3.993857972623222, + "tokens_seen": 444399616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992497492477433, + "loss": 1.5358, + "theoretical_loss": 3.993793708087524, + "tokens_seen": 444465152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992337011033099, + "loss": 1.4613, + "theoretical_loss": 3.9937294556796417, + "tokens_seen": 444530688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992176529588767, + "loss": 1.6225, + "theoretical_loss": 3.993665215395498, + "tokens_seen": 444596224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992016048144433, + "loss": 1.5545, + "theoretical_loss": 3.9936009872310194, + "tokens_seen": 444661760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991855566700101, + "loss": 1.5691, + "theoretical_loss": 3.993536771182134, + "tokens_seen": 444727296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991695085255767, + "loss": 1.4441, + "theoretical_loss": 3.993472567244771, + "tokens_seen": 444792832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991534603811435, + "loss": 1.6013, + "theoretical_loss": 3.9934083754148615, + "tokens_seen": 444858368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991374122367101, + "loss": 1.4713, + "theoretical_loss": 3.9933441956883398, + "tokens_seen": 444923904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991213640922769, + "loss": 1.6447, + "theoretical_loss": 3.9932800280611422, + "tokens_seen": 444989440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991053159478435, + "loss": 1.4422, + "theoretical_loss": 3.9932158725292055, + "tokens_seen": 445054976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990892678034103, + "loss": 1.4835, + "theoretical_loss": 3.993151729088469, + "tokens_seen": 445120512 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699073219658977, + "loss": 1.4059, + "theoretical_loss": 3.993087597734875, + "tokens_seen": 445186048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990571715145437, + "loss": 1.4937, + "theoretical_loss": 3.993023478464366, + "tokens_seen": 445251584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990411233701104, + "loss": 1.4638, + "theoretical_loss": 3.992959371272888, + "tokens_seen": 445317120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990250752256771, + "loss": 1.57, + "theoretical_loss": 3.992895276156388, + "tokens_seen": 445382656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990090270812438, + "loss": 1.4905, + "theoretical_loss": 3.992831193110816, + "tokens_seen": 445448192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989929789368105, + "loss": 1.4754, + "theoretical_loss": 3.992767122132123, + "tokens_seen": 445513728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989769307923772, + "loss": 1.5195, + "theoretical_loss": 3.9927030632162612, + "tokens_seen": 445579264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 154520, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.673126220703125, + "objective/train/theoretical_loss": 3.9926390163591874, + "objective/train/tokens_used": 466104800, + "theoretical_loss": 3.9926390163591874, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989608826479439, + "loss": 1.5465, + "theoretical_loss": 3.9926390163591874, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989448345035106, + "loss": 1.4801, + "theoretical_loss": 3.9925749815568583, + "tokens_seen": 445710336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989287863590773, + "loss": 1.4372, + "theoretical_loss": 3.9925109588052328, + "tokens_seen": 445775872 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698912738214644, + "loss": 1.5426, + "theoretical_loss": 3.9924469481002713, + "tokens_seen": 445841408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988966900702107, + "loss": 1.5802, + "theoretical_loss": 3.9923829494379377, + "tokens_seen": 445906944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988806419257774, + "loss": 1.5393, + "theoretical_loss": 3.9923189628141964, + "tokens_seen": 445972480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988645937813441, + "loss": 1.4943, + "theoretical_loss": 3.9922549882250147, + "tokens_seen": 446038016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988485456369108, + "loss": 1.4797, + "theoretical_loss": 3.9921910256663606, + "tokens_seen": 446103552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988324974924775, + "loss": 1.4582, + "theoretical_loss": 3.9921270751342064, + "tokens_seen": 446169088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988164493480442, + "loss": 1.3838, + "theoretical_loss": 3.992063136624523, + "tokens_seen": 446234624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988004012036109, + "loss": 1.4467, + "theoretical_loss": 3.9919992101332866, + "tokens_seen": 446300160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987843530591776, + "loss": 1.5887, + "theoretical_loss": 3.991935295656473, + "tokens_seen": 446365696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987683049147443, + "loss": 1.4229, + "theoretical_loss": 3.9918713931900602, + "tokens_seen": 446431232 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698752256770311, + "loss": 1.5085, + "theoretical_loss": 3.9918075027300297, + "tokens_seen": 446496768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987362086258777, + "loss": 1.5026, + "theoretical_loss": 3.991743624272363, + "tokens_seen": 446562304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987201604814444, + "loss": 1.5529, + "theoretical_loss": 3.9916797578130456, + "tokens_seen": 446627840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987041123370111, + "loss": 1.4314, + "theoretical_loss": 3.9916159033480625, + "tokens_seen": 446693376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986880641925778, + "loss": 1.5042, + "theoretical_loss": 3.9915520608734028, + "tokens_seen": 446758912 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986720160481445, + "loss": 1.5427, + "theoretical_loss": 3.991488230385056, + "tokens_seen": 446824448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986559679037112, + "loss": 1.5715, + "theoretical_loss": 3.991424411879014, + "tokens_seen": 446889984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986399197592779, + "loss": 1.532, + "theoretical_loss": 3.9913606053512716, + "tokens_seen": 446955520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986238716148446, + "loss": 1.4223, + "theoretical_loss": 3.9912968107978237, + "tokens_seen": 447021056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986078234704113, + "loss": 1.5344, + "theoretical_loss": 3.9912330282146695, + "tokens_seen": 447086592 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698591775325978, + "loss": 1.606, + "theoretical_loss": 3.9911692575978077, + "tokens_seen": 447152128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985757271815447, + "loss": 1.4802, + "theoretical_loss": 3.9911054989432397, + "tokens_seen": 447217664 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985596790371114, + "loss": 1.4954, + "theoretical_loss": 3.9910417522469706, + "tokens_seen": 447283200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985436308926781, + "loss": 1.4643, + "theoretical_loss": 3.990978017505004, + "tokens_seen": 447348736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985275827482448, + "loss": 1.4879, + "theoretical_loss": 3.990914294713348, + "tokens_seen": 447414272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985115346038115, + "loss": 1.3527, + "theoretical_loss": 3.9908505838680126, + "tokens_seen": 447479808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984954864593781, + "loss": 1.5504, + "theoretical_loss": 3.9907868849650088, + "tokens_seen": 447545344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984794383149449, + "loss": 1.4939, + "theoretical_loss": 3.9907231980003495, + "tokens_seen": 447610880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984633901705115, + "loss": 1.4516, + "theoretical_loss": 3.99065952297005, + "tokens_seen": 447676416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984473420260783, + "loss": 1.4114, + "theoretical_loss": 3.990595859870127, + "tokens_seen": 447741952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984312938816449, + "loss": 1.6211, + "theoretical_loss": 3.9905322086966004, + "tokens_seen": 447807488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984152457372117, + "loss": 1.5145, + "theoretical_loss": 3.990468569445489, + "tokens_seen": 447873024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983991975927783, + "loss": 1.5833, + "theoretical_loss": 3.990404942112818, + "tokens_seen": 447938560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983831494483451, + "loss": 1.4963, + "theoretical_loss": 3.9903413266946104, + "tokens_seen": 448004096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983671013039117, + "loss": 1.4331, + "theoretical_loss": 3.9902777231868933, + "tokens_seen": 448069632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983510531594784, + "loss": 1.4788, + "theoretical_loss": 3.990214131585695, + "tokens_seen": 448135168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983350050150451, + "loss": 1.4973, + "theoretical_loss": 3.990150551887047, + "tokens_seen": 448200704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983189568706118, + "loss": 1.4562, + "theoretical_loss": 3.9900869840869793, + "tokens_seen": 448266240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983029087261785, + "loss": 1.6024, + "theoretical_loss": 3.9900234281815283, + "tokens_seen": 448331776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982868605817452, + "loss": 1.4377, + "theoretical_loss": 3.989959884166728, + "tokens_seen": 448397312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982708124373119, + "loss": 1.5387, + "theoretical_loss": 3.9898963520386186, + "tokens_seen": 448462848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982547642928786, + "loss": 1.5395, + "theoretical_loss": 3.9898328317932386, + "tokens_seen": 448528384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982387161484453, + "loss": 1.5434, + "theoretical_loss": 3.98976932342663, + "tokens_seen": 448593920 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698222668004012, + "loss": 1.6005, + "theoretical_loss": 3.9897058269348364, + "tokens_seen": 448659456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982066198595787, + "loss": 1.5173, + "theoretical_loss": 3.9896423423139034, + "tokens_seen": 448724992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981905717151454, + "loss": 1.4321, + "theoretical_loss": 3.989578869559879, + "tokens_seen": 448790528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981745235707121, + "loss": 1.3811, + "theoretical_loss": 3.989515408668812, + "tokens_seen": 448856064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 155690, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5394407510757446, + "objective/train/theoretical_loss": 3.989451959636754, + "objective/train/tokens_used": 469381600, + "theoretical_loss": 3.989451959636754, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981584754262788, + "loss": 1.4618, + "theoretical_loss": 3.989451959636754, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981424272818456, + "loss": 1.5767, + "theoretical_loss": 3.989388522459757, + "tokens_seen": 448987136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981263791374122, + "loss": 1.5553, + "theoretical_loss": 3.989325097133878, + "tokens_seen": 449052672 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698110330992979, + "loss": 1.59, + "theoretical_loss": 3.989261683655172, + "tokens_seen": 449118208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980942828485456, + "loss": 1.5345, + "theoretical_loss": 3.989198282019699, + "tokens_seen": 449183744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980782347041124, + "loss": 1.4384, + "theoretical_loss": 3.989134892223519, + "tokens_seen": 449249280 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698062186559679, + "loss": 1.5084, + "theoretical_loss": 3.9890715142626956, + "tokens_seen": 449314816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980461384152458, + "loss": 1.4859, + "theoretical_loss": 3.9890081481332924, + "tokens_seen": 449380352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980300902708124, + "loss": 1.5314, + "theoretical_loss": 3.988944793831376, + "tokens_seen": 449445888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980140421263792, + "loss": 1.4309, + "theoretical_loss": 3.9888814513530138, + "tokens_seen": 449511424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979979939819458, + "loss": 1.5783, + "theoretical_loss": 3.988818120694277, + "tokens_seen": 449576960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979819458375126, + "loss": 1.423, + "theoretical_loss": 3.988754801851237, + "tokens_seen": 449642496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979658976930792, + "loss": 1.517, + "theoretical_loss": 3.9886914948199683, + "tokens_seen": 449708032 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697949849548646, + "loss": 1.5909, + "theoretical_loss": 3.988628199596546, + "tokens_seen": 449773568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979338014042126, + "loss": 1.4381, + "theoretical_loss": 3.9885649161770473, + "tokens_seen": 449839104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979177532597794, + "loss": 1.4802, + "theoretical_loss": 3.988501644557553, + "tokens_seen": 449904640 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697901705115346, + "loss": 1.4035, + "theoretical_loss": 3.9884383847341436, + "tokens_seen": 449970176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978856569709128, + "loss": 1.5362, + "theoretical_loss": 3.988375136702902, + "tokens_seen": 450035712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978696088264795, + "loss": 1.4519, + "theoretical_loss": 3.988311900459914, + "tokens_seen": 450101248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978535606820462, + "loss": 1.5602, + "theoretical_loss": 3.988248676001267, + "tokens_seen": 450166784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978375125376129, + "loss": 1.4926, + "theoretical_loss": 3.9881854633230485, + "tokens_seen": 450232320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978214643931796, + "loss": 1.5247, + "theoretical_loss": 3.9881222624213497, + "tokens_seen": 450297856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978054162487463, + "loss": 1.5003, + "theoretical_loss": 3.9880590732922636, + "tokens_seen": 450363392 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697789368104313, + "loss": 1.7043, + "theoretical_loss": 3.9879958959318844, + "tokens_seen": 450428928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977733199598797, + "loss": 1.583, + "theoretical_loss": 3.9879327303363086, + "tokens_seen": 450494464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977572718154464, + "loss": 1.5782, + "theoretical_loss": 3.9878695765016334, + "tokens_seen": 450560000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977412236710131, + "loss": 1.4689, + "theoretical_loss": 3.98780643442396, + "tokens_seen": 450625536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977251755265798, + "loss": 1.4843, + "theoretical_loss": 3.9877433040993897, + "tokens_seen": 450691072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977091273821465, + "loss": 1.54, + "theoretical_loss": 3.9876801855240265, + "tokens_seen": 450756608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976930792377132, + "loss": 1.4755, + "theoretical_loss": 3.987617078693975, + "tokens_seen": 450822144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976770310932799, + "loss": 1.4973, + "theoretical_loss": 3.987553983605345, + "tokens_seen": 450887680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976609829488466, + "loss": 1.6195, + "theoretical_loss": 3.9874909002542434, + "tokens_seen": 450953216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976449348044133, + "loss": 1.4774, + "theoretical_loss": 3.987427828636782, + "tokens_seen": 451018752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00069762888665998, + "loss": 1.5001, + "theoretical_loss": 3.9873647687490745, + "tokens_seen": 451084288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976128385155467, + "loss": 1.6324, + "theoretical_loss": 3.9873017205872356, + "tokens_seen": 451149824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975967903711134, + "loss": 1.432, + "theoretical_loss": 3.9872386841473815, + "tokens_seen": 451215360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975807422266801, + "loss": 1.5045, + "theoretical_loss": 3.987175659425631, + "tokens_seen": 451280896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975646940822468, + "loss": 1.5344, + "theoretical_loss": 3.9871126464181046, + "tokens_seen": 451346432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975486459378135, + "loss": 1.5424, + "theoretical_loss": 3.9870496451209245, + "tokens_seen": 451411968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975325977933802, + "loss": 1.605, + "theoretical_loss": 3.986986655530215, + "tokens_seen": 451477504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975165496489469, + "loss": 1.5406, + "theoretical_loss": 3.9869236776421015, + "tokens_seen": 451543040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975005015045136, + "loss": 1.4135, + "theoretical_loss": 3.986860711452713, + "tokens_seen": 451608576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974844533600803, + "loss": 1.5881, + "theoretical_loss": 3.986797756958178, + "tokens_seen": 451674112 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697468405215647, + "loss": 1.5084, + "theoretical_loss": 3.9867348141546284, + "tokens_seen": 451739648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974523570712137, + "loss": 1.5528, + "theoretical_loss": 3.9866718830381975, + "tokens_seen": 451805184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974363089267804, + "loss": 1.4579, + "theoretical_loss": 3.98660896360502, + "tokens_seen": 451870720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974202607823471, + "loss": 1.5445, + "theoretical_loss": 3.9865460558512336, + "tokens_seen": 451936256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974042126379138, + "loss": 1.3889, + "theoretical_loss": 3.986483159772977, + "tokens_seen": 452001792 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973881644934805, + "loss": 1.4828, + "theoretical_loss": 3.986420275366391, + "tokens_seen": 452067328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973721163490472, + "loss": 1.5615, + "theoretical_loss": 3.9863574026276174, + "tokens_seen": 452132864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 156834, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4055612087249756, + "objective/train/theoretical_loss": 3.986294541552801, + "objective/train/tokens_used": 472658400, + "theoretical_loss": 3.986294541552801, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973560682046139, + "loss": 1.4389, + "theoretical_loss": 3.986294541552801, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973400200601806, + "loss": 1.5344, + "theoretical_loss": 3.9862316921380883, + "tokens_seen": 452263936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973239719157473, + "loss": 1.5517, + "theoretical_loss": 3.9861688543796268, + "tokens_seen": 452329472 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697307923771314, + "loss": 1.4465, + "theoretical_loss": 3.986106028273567, + "tokens_seen": 452395008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972918756268807, + "loss": 1.4825, + "theoretical_loss": 3.9860432138160595, + "tokens_seen": 452460544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972758274824474, + "loss": 1.3684, + "theoretical_loss": 3.985980411003259, + "tokens_seen": 452526080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972597793380141, + "loss": 1.4558, + "theoretical_loss": 3.98591761983132, + "tokens_seen": 452591616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972437311935808, + "loss": 1.5356, + "theoretical_loss": 3.9858548402964, + "tokens_seen": 452657152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972276830491475, + "loss": 1.4826, + "theoretical_loss": 3.985792072394658, + "tokens_seen": 452722688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972116349047142, + "loss": 1.4988, + "theoretical_loss": 3.9857293161222542, + "tokens_seen": 452788224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971955867602809, + "loss": 1.5111, + "theoretical_loss": 3.985666571475352, + "tokens_seen": 452853760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971795386158476, + "loss": 1.5831, + "theoretical_loss": 3.985603838450116, + "tokens_seen": 452919296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971634904714143, + "loss": 1.5667, + "theoretical_loss": 3.9855411170427115, + "tokens_seen": 452984832 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697147442326981, + "loss": 1.4883, + "theoretical_loss": 3.985478407249307, + "tokens_seen": 453050368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971313941825477, + "loss": 1.5357, + "theoretical_loss": 3.985415709066073, + "tokens_seen": 453115904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971153460381144, + "loss": 1.4585, + "theoretical_loss": 3.98535302248918, + "tokens_seen": 453181440 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697099297893681, + "loss": 1.4994, + "theoretical_loss": 3.9852903475148027, + "tokens_seen": 453246976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970832497492478, + "loss": 1.5574, + "theoretical_loss": 3.985227684139116, + "tokens_seen": 453312512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970672016048144, + "loss": 1.5326, + "theoretical_loss": 3.9851650323582977, + "tokens_seen": 453378048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970511534603812, + "loss": 1.4839, + "theoretical_loss": 3.9851023921685256, + "tokens_seen": 453443584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970351053159478, + "loss": 1.471, + "theoretical_loss": 3.9850397635659807, + "tokens_seen": 453509120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970190571715146, + "loss": 1.4931, + "theoretical_loss": 3.9849771465468464, + "tokens_seen": 453574656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970030090270812, + "loss": 1.5389, + "theoretical_loss": 3.9849145411073064, + "tokens_seen": 453640192 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696986960882648, + "loss": 1.6396, + "theoretical_loss": 3.984851947243547, + "tokens_seen": 453705728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006969709127382146, + "loss": 1.4131, + "theoretical_loss": 3.984789364951757, + "tokens_seen": 453771264 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006969548645937813, + "loss": 1.534, + "theoretical_loss": 3.9847267942281253, + "tokens_seen": 453836800 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696938816449348, + "loss": 1.4727, + "theoretical_loss": 3.984664235068843, + "tokens_seen": 453902336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006969227683049147, + "loss": 1.4447, + "theoretical_loss": 3.9846016874701053, + "tokens_seen": 453967872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006969067201604814, + "loss": 1.5291, + "theoretical_loss": 3.9845391514281063, + "tokens_seen": 454033408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968906720160481, + "loss": 1.465, + "theoretical_loss": 3.9844766269390437, + "tokens_seen": 454098944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968746238716148, + "loss": 1.5212, + "theoretical_loss": 3.984414113999115, + "tokens_seen": 454164480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968585757271815, + "loss": 1.4242, + "theoretical_loss": 3.984351612604522, + "tokens_seen": 454230016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968425275827482, + "loss": 1.4277, + "theoretical_loss": 3.984289122751467, + "tokens_seen": 454295552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968264794383149, + "loss": 1.5631, + "theoretical_loss": 3.984226644436154, + "tokens_seen": 454361088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968104312938816, + "loss": 1.4569, + "theoretical_loss": 3.9841641776547885, + "tokens_seen": 454426624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967943831494483, + "loss": 1.4869, + "theoretical_loss": 3.98410172240358, + "tokens_seen": 454492160 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696778335005015, + "loss": 1.4457, + "theoretical_loss": 3.984039278678736, + "tokens_seen": 454557696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967622868605817, + "loss": 1.5251, + "theoretical_loss": 3.9839768464764695, + "tokens_seen": 454623232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967462387161485, + "loss": 1.6093, + "theoretical_loss": 3.9839144257929933, + "tokens_seen": 454688768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967301905717151, + "loss": 1.409, + "theoretical_loss": 3.983852016624522, + "tokens_seen": 454754304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967141424272819, + "loss": 1.5454, + "theoretical_loss": 3.9837896189672724, + "tokens_seen": 454819840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966980942828485, + "loss": 1.5249, + "theoretical_loss": 3.9837272328174635, + "tokens_seen": 454885376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966820461384153, + "loss": 1.4779, + "theoretical_loss": 3.9836648581713154, + "tokens_seen": 454950912 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696665997993982, + "loss": 1.5053, + "theoretical_loss": 3.9836024950250497, + "tokens_seen": 455016448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966499498495487, + "loss": 1.5391, + "theoretical_loss": 3.983540143374891, + "tokens_seen": 455081984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966339017051154, + "loss": 1.5406, + "theoretical_loss": 3.983477803217066, + "tokens_seen": 455147520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966178535606821, + "loss": 1.6182, + "theoretical_loss": 3.9834154745478, + "tokens_seen": 455213056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966018054162488, + "loss": 1.3834, + "theoretical_loss": 3.9833531573633234, + "tokens_seen": 455278592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965857572718155, + "loss": 1.5062, + "theoretical_loss": 3.9832908516598673, + "tokens_seen": 455344128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965697091273822, + "loss": 1.509, + "theoretical_loss": 3.983228557433664, + "tokens_seen": 455409664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 157689, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5005377531051636, + "objective/train/theoretical_loss": 3.9831662746809493, + "objective/train/tokens_used": 475935200, + "theoretical_loss": 3.9831662746809493, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965536609829489, + "loss": 1.5773, + "theoretical_loss": 3.9831662746809493, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965376128385156, + "loss": 1.516, + "theoretical_loss": 3.9831040033979583, + "tokens_seen": 455540736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965215646940823, + "loss": 1.3727, + "theoretical_loss": 3.9830417435809298, + "tokens_seen": 455606272 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696505516549649, + "loss": 1.4682, + "theoretical_loss": 3.9829794952261035, + "tokens_seen": 455671808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964894684052157, + "loss": 1.5151, + "theoretical_loss": 3.9829172583297208, + "tokens_seen": 455737344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964734202607824, + "loss": 1.4777, + "theoretical_loss": 3.9828550328880263, + "tokens_seen": 455802880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964573721163491, + "loss": 1.6122, + "theoretical_loss": 3.982792818897264, + "tokens_seen": 455868416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964413239719158, + "loss": 1.5951, + "theoretical_loss": 3.9827306163536815, + "tokens_seen": 455933952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964252758274825, + "loss": 1.5475, + "theoretical_loss": 3.982668425253528, + "tokens_seen": 455999488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964092276830492, + "loss": 1.4892, + "theoretical_loss": 3.982606245593053, + "tokens_seen": 456065024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963931795386159, + "loss": 1.5035, + "theoretical_loss": 3.9825440773685092, + "tokens_seen": 456130560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963771313941826, + "loss": 1.493, + "theoretical_loss": 3.9824819205761512, + "tokens_seen": 456196096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963610832497493, + "loss": 1.5549, + "theoretical_loss": 3.982419775212235, + "tokens_seen": 456261632 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696345035105316, + "loss": 1.6392, + "theoretical_loss": 3.9823576412730173, + "tokens_seen": 456327168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963289869608827, + "loss": 1.5156, + "theoretical_loss": 3.982295518754758, + "tokens_seen": 456392704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963129388164494, + "loss": 1.5393, + "theoretical_loss": 3.982233407653718, + "tokens_seen": 456458240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962968906720161, + "loss": 1.5114, + "theoretical_loss": 3.982171307966161, + "tokens_seen": 456523776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962808425275828, + "loss": 1.5031, + "theoretical_loss": 3.9821092196883505, + "tokens_seen": 456589312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962647943831495, + "loss": 1.5498, + "theoretical_loss": 3.982047142816554, + "tokens_seen": 456654848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962487462387162, + "loss": 1.5781, + "theoretical_loss": 3.981985077347039, + "tokens_seen": 456720384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962326980942829, + "loss": 1.5119, + "theoretical_loss": 3.981923023276075, + "tokens_seen": 456785920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962166499498496, + "loss": 1.5188, + "theoretical_loss": 3.9818609805999348, + "tokens_seen": 456851456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962006018054163, + "loss": 1.4882, + "theoretical_loss": 3.981798949314891, + "tokens_seen": 456916992 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696184553660983, + "loss": 1.6035, + "theoretical_loss": 3.98173692941722, + "tokens_seen": 456982528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961685055165497, + "loss": 1.5139, + "theoretical_loss": 3.981674920903197, + "tokens_seen": 457048064 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961524573721164, + "loss": 1.5006, + "theoretical_loss": 3.9816129237691023, + "tokens_seen": 457113600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961364092276831, + "loss": 1.5368, + "theoretical_loss": 3.981550938011215, + "tokens_seen": 457179136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961203610832498, + "loss": 1.4274, + "theoretical_loss": 3.981488963625818, + "tokens_seen": 457244672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961043129388165, + "loss": 1.3543, + "theoretical_loss": 3.9814270006091954, + "tokens_seen": 457310208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960882647943832, + "loss": 1.5751, + "theoretical_loss": 3.981365048957633, + "tokens_seen": 457375744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960722166499499, + "loss": 1.5665, + "theoretical_loss": 3.9813031086674173, + "tokens_seen": 457441280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960561685055166, + "loss": 1.4666, + "theoretical_loss": 3.9812411797348384, + "tokens_seen": 457506816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960401203610833, + "loss": 1.4307, + "theoretical_loss": 3.9811792621561874, + "tokens_seen": 457572352 + }, + { + "epoch": 0.14, + "learning_rate": 0.00069602407221665, + "loss": 1.4939, + "theoretical_loss": 3.981117355927756, + "tokens_seen": 457637888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960080240722167, + "loss": 1.541, + "theoretical_loss": 3.981055461045839, + "tokens_seen": 457703424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959919759277834, + "loss": 1.5948, + "theoretical_loss": 3.980993577506733, + "tokens_seen": 457768960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959759277833501, + "loss": 1.5795, + "theoretical_loss": 3.980931705306735, + "tokens_seen": 457834496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959598796389168, + "loss": 1.519, + "theoretical_loss": 3.9808698444421458, + "tokens_seen": 457900032 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959438314944835, + "loss": 1.5537, + "theoretical_loss": 3.9808079949092656, + "tokens_seen": 457965568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959277833500502, + "loss": 1.5433, + "theoretical_loss": 3.9807461567043987, + "tokens_seen": 458031104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959117352056169, + "loss": 1.4325, + "theoretical_loss": 3.980684329823849, + "tokens_seen": 458096640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958956870611836, + "loss": 1.5263, + "theoretical_loss": 3.980622514263923, + "tokens_seen": 458162176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958796389167503, + "loss": 1.5642, + "theoretical_loss": 3.9805607100209297, + "tokens_seen": 458227712 + }, + { + "epoch": 0.14, + "learning_rate": 0.000695863590772317, + "loss": 1.4736, + "theoretical_loss": 3.980498917091179, + "tokens_seen": 458293248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958475426278837, + "loss": 1.5423, + "theoretical_loss": 3.980437135470982, + "tokens_seen": 458358784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958314944834504, + "loss": 1.5497, + "theoretical_loss": 3.980375365156653, + "tokens_seen": 458424320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958154463390171, + "loss": 1.4201, + "theoretical_loss": 3.9803136061445064, + "tokens_seen": 458489856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957993981945838, + "loss": 1.5075, + "theoretical_loss": 3.9802518584308597, + "tokens_seen": 458555392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957833500501505, + "loss": 1.407, + "theoretical_loss": 3.980190122012032, + "tokens_seen": 458620928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957673019057172, + "loss": 1.4852, + "theoretical_loss": 3.980128396884343, + "tokens_seen": 458686464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 158992, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4671831130981445, + "objective/train/theoretical_loss": 3.9800666830441145, + "objective/train/tokens_used": 479212000, + "theoretical_loss": 3.9800666830441145, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957512537612839, + "loss": 1.5057, + "theoretical_loss": 3.9800666830441145, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957352056168506, + "loss": 1.3742, + "theoretical_loss": 3.9800049804876716, + "tokens_seen": 458817536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957191574724173, + "loss": 1.6351, + "theoretical_loss": 3.9799432892113384, + "tokens_seen": 458883072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957031093279839, + "loss": 1.4647, + "theoretical_loss": 3.9798816092114437, + "tokens_seen": 458948608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956870611835507, + "loss": 1.4541, + "theoretical_loss": 3.9798199404843153, + "tokens_seen": 459014144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956710130391173, + "loss": 1.5378, + "theoretical_loss": 3.9797582830262845, + "tokens_seen": 459079680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956549648946841, + "loss": 1.4761, + "theoretical_loss": 3.9796966368336832, + "tokens_seen": 459145216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956389167502507, + "loss": 1.5889, + "theoretical_loss": 3.979635001902847, + "tokens_seen": 459210752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956228686058175, + "loss": 1.6021, + "theoretical_loss": 3.9795733782301097, + "tokens_seen": 459276288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956068204613841, + "loss": 1.5399, + "theoretical_loss": 3.979511765811811, + "tokens_seen": 459341824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955907723169509, + "loss": 1.5921, + "theoretical_loss": 3.9794501646442884, + "tokens_seen": 459407360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955747241725175, + "loss": 1.5187, + "theoretical_loss": 3.9793885747238837, + "tokens_seen": 459472896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955586760280842, + "loss": 1.5166, + "theoretical_loss": 3.97932699604694, + "tokens_seen": 459538432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955426278836509, + "loss": 1.4833, + "theoretical_loss": 3.979265428609801, + "tokens_seen": 459603968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955265797392176, + "loss": 1.5999, + "theoretical_loss": 3.9792038724088137, + "tokens_seen": 459669504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955105315947843, + "loss": 1.5948, + "theoretical_loss": 3.9791423274403255, + "tokens_seen": 459735040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954944834503511, + "loss": 1.4881, + "theoretical_loss": 3.9790807937006853, + "tokens_seen": 459800576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954784353059177, + "loss": 1.488, + "theoretical_loss": 3.979019271186246, + "tokens_seen": 459866112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954623871614845, + "loss": 1.506, + "theoretical_loss": 3.9789577598933588, + "tokens_seen": 459931648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954463390170511, + "loss": 1.4603, + "theoretical_loss": 3.9788962598183795, + "tokens_seen": 459997184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954302908726179, + "loss": 1.5106, + "theoretical_loss": 3.978834770957665, + "tokens_seen": 460062720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954142427281846, + "loss": 1.5574, + "theoretical_loss": 3.9787732933075715, + "tokens_seen": 460128256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953981945837513, + "loss": 1.6217, + "theoretical_loss": 3.97871182686446, + "tokens_seen": 460193792 + }, + { + "epoch": 0.14, + "learning_rate": 0.000695382146439318, + "loss": 1.4491, + "theoretical_loss": 3.978650371624692, + "tokens_seen": 460259328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953660982948847, + "loss": 1.5272, + "theoretical_loss": 3.9785889275846307, + "tokens_seen": 460324864 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953500501504514, + "loss": 1.5238, + "theoretical_loss": 3.978527494740641, + "tokens_seen": 460390400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953340020060181, + "loss": 1.5595, + "theoretical_loss": 3.9784660730890895, + "tokens_seen": 460455936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953179538615848, + "loss": 1.5279, + "theoretical_loss": 3.978404662626344, + "tokens_seen": 460521472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953019057171515, + "loss": 1.5194, + "theoretical_loss": 3.9783432633487745, + "tokens_seen": 460587008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952858575727182, + "loss": 1.5553, + "theoretical_loss": 3.9782818752527533, + "tokens_seen": 460652544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952698094282849, + "loss": 1.481, + "theoretical_loss": 3.978220498334654, + "tokens_seen": 460718080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952537612838516, + "loss": 1.5989, + "theoretical_loss": 3.97815913259085, + "tokens_seen": 460783616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952377131394183, + "loss": 1.4864, + "theoretical_loss": 3.9780977780177205, + "tokens_seen": 460849152 + }, + { + "epoch": 0.14, + "learning_rate": 0.000695221664994985, + "loss": 1.535, + "theoretical_loss": 3.978036434611642, + "tokens_seen": 460914688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952056168505517, + "loss": 1.4559, + "theoretical_loss": 3.977975102368995, + "tokens_seen": 460980224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951895687061184, + "loss": 1.5792, + "theoretical_loss": 3.977913781286162, + "tokens_seen": 461045760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951735205616851, + "loss": 1.5258, + "theoretical_loss": 3.9778524713595256, + "tokens_seen": 461111296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951574724172518, + "loss": 1.4682, + "theoretical_loss": 3.977791172585472, + "tokens_seen": 461176832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951414242728185, + "loss": 1.4718, + "theoretical_loss": 3.977729884960387, + "tokens_seen": 461242368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951253761283852, + "loss": 1.5455, + "theoretical_loss": 3.9776686084806596, + "tokens_seen": 461307904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951093279839519, + "loss": 1.4778, + "theoretical_loss": 3.9776073431426804, + "tokens_seen": 461373440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950932798395186, + "loss": 1.5595, + "theoretical_loss": 3.977546088942841, + "tokens_seen": 461438976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950772316950853, + "loss": 1.5265, + "theoretical_loss": 3.9774848458775347, + "tokens_seen": 461504512 + }, + { + "epoch": 0.14, + "learning_rate": 0.000695061183550652, + "loss": 1.5641, + "theoretical_loss": 3.9774236139431576, + "tokens_seen": 461570048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950451354062187, + "loss": 1.4548, + "theoretical_loss": 3.977362393136106, + "tokens_seen": 461635584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950290872617854, + "loss": 1.4131, + "theoretical_loss": 3.977301183452779, + "tokens_seen": 461701120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950130391173521, + "loss": 1.4352, + "theoretical_loss": 3.977239984889576, + "tokens_seen": 461766656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949969909729188, + "loss": 1.4112, + "theoretical_loss": 3.9771787974429, + "tokens_seen": 461832192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949809428284855, + "loss": 1.443, + "theoretical_loss": 3.977117621109154, + "tokens_seen": 461897728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949648946840522, + "loss": 1.4848, + "theoretical_loss": 3.977056455884744, + "tokens_seen": 461963264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 159827, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.441860318183899, + "objective/train/theoretical_loss": 3.9769953017660766, + "objective/train/tokens_used": 482488800, + "theoretical_loss": 3.9769953017660766, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949488465396189, + "loss": 1.4389, + "theoretical_loss": 3.9769953017660766, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949327983951856, + "loss": 1.4872, + "theoretical_loss": 3.97693415874956, + "tokens_seen": 462094336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949167502507523, + "loss": 1.4321, + "theoretical_loss": 3.9768730268316057, + "tokens_seen": 462159872 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694900702106319, + "loss": 1.4834, + "theoretical_loss": 3.976811906008625, + "tokens_seen": 462225408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948846539618857, + "loss": 1.4452, + "theoretical_loss": 3.976750796277031, + "tokens_seen": 462290944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948686058174524, + "loss": 1.6206, + "theoretical_loss": 3.9766896976332404, + "tokens_seen": 462356480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948525576730191, + "loss": 1.493, + "theoretical_loss": 3.9766286100736696, + "tokens_seen": 462422016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948365095285858, + "loss": 1.479, + "theoretical_loss": 3.976567533594737, + "tokens_seen": 462487552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948204613841525, + "loss": 1.5357, + "theoretical_loss": 3.9765064681928637, + "tokens_seen": 462553088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948044132397192, + "loss": 1.464, + "theoretical_loss": 3.9764454138644703, + "tokens_seen": 462618624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947883650952859, + "loss": 1.469, + "theoretical_loss": 3.976384370605982, + "tokens_seen": 462684160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947723169508526, + "loss": 1.4872, + "theoretical_loss": 3.9763233384138235, + "tokens_seen": 462749696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947562688064193, + "loss": 1.5855, + "theoretical_loss": 3.976262317284421, + "tokens_seen": 462815232 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694740220661986, + "loss": 1.5168, + "theoretical_loss": 3.976201307214205, + "tokens_seen": 462880768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947241725175527, + "loss": 1.5409, + "theoretical_loss": 3.9761403081996045, + "tokens_seen": 462946304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947081243731194, + "loss": 1.5067, + "theoretical_loss": 3.9760793202370524, + "tokens_seen": 463011840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946920762286861, + "loss": 1.6068, + "theoretical_loss": 3.976018343322981, + "tokens_seen": 463077376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946760280842528, + "loss": 1.4944, + "theoretical_loss": 3.975957377453826, + "tokens_seen": 463142912 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946599799398195, + "loss": 1.4191, + "theoretical_loss": 3.9758964226260254, + "tokens_seen": 463208448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946439317953862, + "loss": 1.5305, + "theoretical_loss": 3.975835478836016, + "tokens_seen": 463273984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946278836509529, + "loss": 1.581, + "theoretical_loss": 3.97577454608024, + "tokens_seen": 463339520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946118355065196, + "loss": 1.5766, + "theoretical_loss": 3.975713624355139, + "tokens_seen": 463405056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945957873620863, + "loss": 1.4439, + "theoretical_loss": 3.975652713657155, + "tokens_seen": 463470592 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694579739217653, + "loss": 1.5426, + "theoretical_loss": 3.975591813982734, + "tokens_seen": 463536128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945636910732197, + "loss": 1.4779, + "theoretical_loss": 3.9755309253283233, + "tokens_seen": 463601664 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945476429287864, + "loss": 1.5188, + "theoretical_loss": 3.9754700476903713, + "tokens_seen": 463667200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945315947843531, + "loss": 1.4893, + "theoretical_loss": 3.975409181065328, + "tokens_seen": 463732736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945155466399198, + "loss": 1.5916, + "theoretical_loss": 3.9753483254496444, + "tokens_seen": 463798272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944994984954865, + "loss": 1.5098, + "theoretical_loss": 3.9752874808397753, + "tokens_seen": 463863808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944834503510532, + "loss": 1.5938, + "theoretical_loss": 3.975226647232175, + "tokens_seen": 463929344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944674022066199, + "loss": 1.5668, + "theoretical_loss": 3.9751658246233, + "tokens_seen": 463994880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944513540621866, + "loss": 1.5402, + "theoretical_loss": 3.9751050130096095, + "tokens_seen": 464060416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944353059177533, + "loss": 1.521, + "theoretical_loss": 3.9750442123875622, + "tokens_seen": 464125952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00069441925777332, + "loss": 1.5605, + "theoretical_loss": 3.9749834227536214, + "tokens_seen": 464191488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944032096288867, + "loss": 1.441, + "theoretical_loss": 3.9749226441042493, + "tokens_seen": 464257024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943871614844534, + "loss": 1.5675, + "theoretical_loss": 3.9748618764359107, + "tokens_seen": 464322560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943711133400201, + "loss": 1.5599, + "theoretical_loss": 3.9748011197450728, + "tokens_seen": 464388096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943550651955868, + "loss": 1.6316, + "theoretical_loss": 3.974740374028203, + "tokens_seen": 464453632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943390170511535, + "loss": 1.4587, + "theoretical_loss": 3.9746796392817716, + "tokens_seen": 464519168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943229689067202, + "loss": 1.4971, + "theoretical_loss": 3.9746189155022504, + "tokens_seen": 464584704 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694306920762287, + "loss": 1.5041, + "theoretical_loss": 3.9745582026861124, + "tokens_seen": 464650240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006942908726178536, + "loss": 1.6098, + "theoretical_loss": 3.9744975008298313, + "tokens_seen": 464715776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006942748244734202, + "loss": 1.5333, + "theoretical_loss": 3.974436809929885, + "tokens_seen": 464781312 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694258776328987, + "loss": 1.5918, + "theoretical_loss": 3.9743761299827494, + "tokens_seen": 464846848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006942427281845536, + "loss": 1.4116, + "theoretical_loss": 3.974315460984906, + "tokens_seen": 464912384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006942266800401204, + "loss": 1.5304, + "theoretical_loss": 3.974254802932836, + "tokens_seen": 464977920 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694210631895687, + "loss": 1.5563, + "theoretical_loss": 3.974194155823021, + "tokens_seen": 465043456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941945837512538, + "loss": 1.5092, + "theoretical_loss": 3.9741335196519465, + "tokens_seen": 465108992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941785356068204, + "loss": 1.5468, + "theoretical_loss": 3.9740728944160977, + "tokens_seen": 465174528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941624874623872, + "loss": 1.4083, + "theoretical_loss": 3.974012280111963, + "tokens_seen": 465240064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 161129, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5128990411758423, + "objective/train/theoretical_loss": 3.973951676736032, + "objective/train/tokens_used": 485765600, + "theoretical_loss": 3.973951676736032, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941464393179538, + "loss": 1.4757, + "theoretical_loss": 3.973951676736032, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941303911735206, + "loss": 1.6992, + "theoretical_loss": 3.973891084284795, + "tokens_seen": 465371136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941143430290872, + "loss": 1.4481, + "theoretical_loss": 3.973830502754745, + "tokens_seen": 465436672 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694098294884654, + "loss": 1.3695, + "theoretical_loss": 3.973769932142376, + "tokens_seen": 465502208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940822467402206, + "loss": 1.4866, + "theoretical_loss": 3.9737093724441843, + "tokens_seen": 465567744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940661985957874, + "loss": 1.6144, + "theoretical_loss": 3.9736488236566667, + "tokens_seen": 465633280 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694050150451354, + "loss": 1.494, + "theoretical_loss": 3.973588285776323, + "tokens_seen": 465698816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940341023069208, + "loss": 1.3829, + "theoretical_loss": 3.9735277587996527, + "tokens_seen": 465764352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940180541624875, + "loss": 1.511, + "theoretical_loss": 3.973467242723159, + "tokens_seen": 465829888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940020060180542, + "loss": 1.5361, + "theoretical_loss": 3.9734067375433466, + "tokens_seen": 465895424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939859578736209, + "loss": 1.4227, + "theoretical_loss": 3.9733462432567195, + "tokens_seen": 465960960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939699097291876, + "loss": 1.5223, + "theoretical_loss": 3.9732857598597855, + "tokens_seen": 466026496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939538615847543, + "loss": 1.587, + "theoretical_loss": 3.9732252873490532, + "tokens_seen": 466092032 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693937813440321, + "loss": 1.4361, + "theoretical_loss": 3.9731648257210335, + "tokens_seen": 466157568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939217652958877, + "loss": 1.5038, + "theoretical_loss": 3.9731043749722375, + "tokens_seen": 466223104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939057171514544, + "loss": 1.5343, + "theoretical_loss": 3.9730439350991795, + "tokens_seen": 466288640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938896690070211, + "loss": 1.5162, + "theoretical_loss": 3.9729835060983745, + "tokens_seen": 466354176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938736208625878, + "loss": 1.4384, + "theoretical_loss": 3.972923087966339, + "tokens_seen": 466419712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938575727181545, + "loss": 1.5003, + "theoretical_loss": 3.9728626806995915, + "tokens_seen": 466485248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938415245737212, + "loss": 1.5118, + "theoretical_loss": 3.9728022842946524, + "tokens_seen": 466550784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938254764292879, + "loss": 1.518, + "theoretical_loss": 3.972741898748043, + "tokens_seen": 466616320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938094282848546, + "loss": 1.5349, + "theoretical_loss": 3.972681524056286, + "tokens_seen": 466681856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937933801404213, + "loss": 1.6516, + "theoretical_loss": 3.9726211602159074, + "tokens_seen": 466747392 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693777331995988, + "loss": 1.5143, + "theoretical_loss": 3.9725608072234326, + "tokens_seen": 466812928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937612838515547, + "loss": 1.5186, + "theoretical_loss": 3.97250046507539, + "tokens_seen": 466878464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937452357071214, + "loss": 1.4036, + "theoretical_loss": 3.972440133768309, + "tokens_seen": 466944000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937291875626881, + "loss": 1.4624, + "theoretical_loss": 3.972379813298721, + "tokens_seen": 467009536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937131394182548, + "loss": 1.4537, + "theoretical_loss": 3.972319503663159, + "tokens_seen": 467075072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936970912738215, + "loss": 1.4609, + "theoretical_loss": 3.972259204858157, + "tokens_seen": 467140608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936810431293882, + "loss": 1.5882, + "theoretical_loss": 3.9721989168802505, + "tokens_seen": 467206144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936649949849549, + "loss": 1.5135, + "theoretical_loss": 3.9721386397259786, + "tokens_seen": 467271680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936489468405216, + "loss": 1.6007, + "theoretical_loss": 3.972078373391879, + "tokens_seen": 467337216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936328986960883, + "loss": 1.3934, + "theoretical_loss": 3.9720181178744935, + "tokens_seen": 467402752 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693616850551655, + "loss": 1.4687, + "theoretical_loss": 3.971957873170364, + "tokens_seen": 467468288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936008024072217, + "loss": 1.4979, + "theoretical_loss": 3.971897639276035, + "tokens_seen": 467533824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935847542627884, + "loss": 1.4511, + "theoretical_loss": 3.9718374161880505, + "tokens_seen": 467599360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935687061183551, + "loss": 1.396, + "theoretical_loss": 3.971777203902959, + "tokens_seen": 467664896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935526579739218, + "loss": 1.5844, + "theoretical_loss": 3.971717002417309, + "tokens_seen": 467730432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935366098294885, + "loss": 1.6607, + "theoretical_loss": 3.9716568117276503, + "tokens_seen": 467795968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935205616850552, + "loss": 1.5169, + "theoretical_loss": 3.9715966318305354, + "tokens_seen": 467861504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935045135406219, + "loss": 1.4717, + "theoretical_loss": 3.9715364627225176, + "tokens_seen": 467927040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934884653961886, + "loss": 1.3652, + "theoretical_loss": 3.9714763044001513, + "tokens_seen": 467992576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934724172517553, + "loss": 1.5736, + "theoretical_loss": 3.971416156859995, + "tokens_seen": 468058112 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693456369107322, + "loss": 1.4701, + "theoretical_loss": 3.9713560200986042, + "tokens_seen": 468123648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934403209628887, + "loss": 1.4428, + "theoretical_loss": 3.9712958941125405, + "tokens_seen": 468189184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934242728184554, + "loss": 1.6038, + "theoretical_loss": 3.971235778898365, + "tokens_seen": 468254720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934082246740221, + "loss": 1.5205, + "theoretical_loss": 3.971175674452641, + "tokens_seen": 468320256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933921765295888, + "loss": 1.5535, + "theoretical_loss": 3.9711155807719316, + "tokens_seen": 468385792 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933761283851555, + "loss": 1.5537, + "theoretical_loss": 3.971055497852804, + "tokens_seen": 468451328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933600802407222, + "loss": 1.4948, + "theoretical_loss": 3.9709954256918265, + "tokens_seen": 468516864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 161996, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3592880964279175, + "objective/train/theoretical_loss": 3.970935364285567, + "objective/train/tokens_used": 489042400, + "theoretical_loss": 3.970935364285567, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933440320962889, + "loss": 1.4138, + "theoretical_loss": 3.970935364285567, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933279839518556, + "loss": 1.438, + "theoretical_loss": 3.9708753136305974, + "tokens_seen": 468647936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933119358074223, + "loss": 1.4692, + "theoretical_loss": 3.9708152737234887, + "tokens_seen": 468713472 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693295887662989, + "loss": 1.4952, + "theoretical_loss": 3.9707552445608165, + "tokens_seen": 468779008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932798395185557, + "loss": 1.482, + "theoretical_loss": 3.9706952261391555, + "tokens_seen": 468844544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932637913741224, + "loss": 1.4899, + "theoretical_loss": 3.970635218455083, + "tokens_seen": 468910080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932477432296891, + "loss": 1.4871, + "theoretical_loss": 3.970575221505178, + "tokens_seen": 468975616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932316950852558, + "loss": 1.5286, + "theoretical_loss": 3.97051523528602, + "tokens_seen": 469041152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932156469408225, + "loss": 1.5471, + "theoretical_loss": 3.970455259794191, + "tokens_seen": 469106688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931995987963892, + "loss": 1.5182, + "theoretical_loss": 3.970395295026275, + "tokens_seen": 469172224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931835506519559, + "loss": 1.6327, + "theoretical_loss": 3.970335340978856, + "tokens_seen": 469237760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931675025075226, + "loss": 1.4887, + "theoretical_loss": 3.970275397648522, + "tokens_seen": 469303296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931514543630893, + "loss": 1.5198, + "theoretical_loss": 3.9702154650318597, + "tokens_seen": 469368832 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693135406218656, + "loss": 1.5937, + "theoretical_loss": 3.9701555431254594, + "tokens_seen": 469434368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931193580742227, + "loss": 1.4071, + "theoretical_loss": 3.9700956319259113, + "tokens_seen": 469499904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931033099297894, + "loss": 1.4972, + "theoretical_loss": 3.97003573142981, + "tokens_seen": 469565440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930872617853562, + "loss": 1.5, + "theoretical_loss": 3.9699758416337483, + "tokens_seen": 469630976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930712136409228, + "loss": 1.561, + "theoretical_loss": 3.9699159625343223, + "tokens_seen": 469696512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930551654964896, + "loss": 1.3793, + "theoretical_loss": 3.96985609412813, + "tokens_seen": 469762048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930391173520562, + "loss": 1.534, + "theoretical_loss": 3.96979623641177, + "tokens_seen": 469827584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930230692076229, + "loss": 1.5069, + "theoretical_loss": 3.9697363893818434, + "tokens_seen": 469893120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930070210631896, + "loss": 1.503, + "theoretical_loss": 3.969676553034951, + "tokens_seen": 469958656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929909729187563, + "loss": 1.5206, + "theoretical_loss": 3.969616727367698, + "tokens_seen": 470024192 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692974924774323, + "loss": 1.5285, + "theoretical_loss": 3.9695569123766887, + "tokens_seen": 470089728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929588766298897, + "loss": 1.4659, + "theoretical_loss": 3.9694971080585297, + "tokens_seen": 470155264 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929428284854564, + "loss": 1.4967, + "theoretical_loss": 3.9694373144098307, + "tokens_seen": 470220800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929267803410231, + "loss": 1.514, + "theoretical_loss": 3.9693775314271997, + "tokens_seen": 470286336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929107321965898, + "loss": 1.4917, + "theoretical_loss": 3.9693177591072493, + "tokens_seen": 470351872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928946840521565, + "loss": 1.4453, + "theoretical_loss": 3.9692579974465922, + "tokens_seen": 470417408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928786359077231, + "loss": 1.4894, + "theoretical_loss": 3.9691982464418434, + "tokens_seen": 470482944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928625877632899, + "loss": 1.5014, + "theoretical_loss": 3.9691385060896174, + "tokens_seen": 470548480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928465396188565, + "loss": 1.5743, + "theoretical_loss": 3.9690787763865334, + "tokens_seen": 470614016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928304914744233, + "loss": 1.5716, + "theoretical_loss": 3.96901905732921, + "tokens_seen": 470679552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928144433299899, + "loss": 1.5425, + "theoretical_loss": 3.968959348914268, + "tokens_seen": 470745088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927983951855567, + "loss": 1.482, + "theoretical_loss": 3.9688996511383294, + "tokens_seen": 470810624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927823470411233, + "loss": 1.4986, + "theoretical_loss": 3.968839963998018, + "tokens_seen": 470876160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927662988966901, + "loss": 1.4271, + "theoretical_loss": 3.9687802874899596, + "tokens_seen": 470941696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927502507522567, + "loss": 1.4772, + "theoretical_loss": 3.9687206216107804, + "tokens_seen": 471007232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927342026078235, + "loss": 1.5334, + "theoretical_loss": 3.968660966357109, + "tokens_seen": 471072768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927181544633901, + "loss": 1.5458, + "theoretical_loss": 3.9686013217255756, + "tokens_seen": 471138304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927021063189569, + "loss": 1.4409, + "theoretical_loss": 3.9685416877128112, + "tokens_seen": 471203840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926860581745235, + "loss": 1.5034, + "theoretical_loss": 3.9684820643154497, + "tokens_seen": 471269376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926700100300903, + "loss": 1.4848, + "theoretical_loss": 3.9684224515301247, + "tokens_seen": 471334912 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692653961885657, + "loss": 1.4413, + "theoretical_loss": 3.968362849353473, + "tokens_seen": 471400448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926379137412237, + "loss": 1.543, + "theoretical_loss": 3.9683032577821313, + "tokens_seen": 471465984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926218655967904, + "loss": 1.4091, + "theoretical_loss": 3.9682436768127394, + "tokens_seen": 471531520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926058174523571, + "loss": 1.5432, + "theoretical_loss": 3.968184106441938, + "tokens_seen": 471597056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925897693079238, + "loss": 1.5376, + "theoretical_loss": 3.9681245466663695, + "tokens_seen": 471662592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925737211634905, + "loss": 1.4423, + "theoretical_loss": 3.9680649974826774, + "tokens_seen": 471728128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925576730190572, + "loss": 1.4998, + "theoretical_loss": 3.9680054588875064, + "tokens_seen": 471793664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 163162, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4037538766860962, + "objective/train/theoretical_loss": 3.9679459308775042, + "objective/train/tokens_used": 492319200, + "theoretical_loss": 3.9679459308775042, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925416248746239, + "loss": 1.4952, + "theoretical_loss": 3.9679459308775042, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925255767301906, + "loss": 1.5488, + "theoretical_loss": 3.9678864134493184, + "tokens_seen": 471924736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925095285857573, + "loss": 1.5842, + "theoretical_loss": 3.9678269065996, + "tokens_seen": 471990272 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692493480441324, + "loss": 1.5198, + "theoretical_loss": 3.967767410324999, + "tokens_seen": 472055808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924774322968907, + "loss": 1.4667, + "theoretical_loss": 3.967707924622169, + "tokens_seen": 472121344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924613841524574, + "loss": 1.5051, + "theoretical_loss": 3.967648449487764, + "tokens_seen": 472186880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924453360080241, + "loss": 1.5073, + "theoretical_loss": 3.9675889849184407, + "tokens_seen": 472252416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924292878635908, + "loss": 1.5508, + "theoretical_loss": 3.9675295309108556, + "tokens_seen": 472317952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924132397191575, + "loss": 1.4599, + "theoretical_loss": 3.9674700874616686, + "tokens_seen": 472383488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923971915747242, + "loss": 1.6051, + "theoretical_loss": 3.9674106545675403, + "tokens_seen": 472449024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923811434302909, + "loss": 1.4511, + "theoretical_loss": 3.967351232225132, + "tokens_seen": 472514560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923650952858576, + "loss": 1.4974, + "theoretical_loss": 3.9672918204311074, + "tokens_seen": 472580096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923490471414243, + "loss": 1.5511, + "theoretical_loss": 3.9672324191821318, + "tokens_seen": 472645632 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692332998996991, + "loss": 1.6074, + "theoretical_loss": 3.967173028474871, + "tokens_seen": 472711168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923169508525577, + "loss": 1.6054, + "theoretical_loss": 3.967113648305995, + "tokens_seen": 472776704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923009027081244, + "loss": 1.5523, + "theoretical_loss": 3.9670542786721716, + "tokens_seen": 472842240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922848545636911, + "loss": 1.4466, + "theoretical_loss": 3.9669949195700727, + "tokens_seen": 472907776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922688064192578, + "loss": 1.5414, + "theoretical_loss": 3.9669355709963705, + "tokens_seen": 472973312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922527582748245, + "loss": 1.4645, + "theoretical_loss": 3.9668762329477403, + "tokens_seen": 473038848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922367101303912, + "loss": 1.4129, + "theoretical_loss": 3.966816905420856, + "tokens_seen": 473104384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922206619859579, + "loss": 1.5528, + "theoretical_loss": 3.966757588412396, + "tokens_seen": 473169920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922046138415246, + "loss": 1.6118, + "theoretical_loss": 3.9666982819190384, + "tokens_seen": 473235456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921885656970913, + "loss": 1.4257, + "theoretical_loss": 3.9666389859374642, + "tokens_seen": 473300992 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692172517552658, + "loss": 1.5467, + "theoretical_loss": 3.9665797004643544, + "tokens_seen": 473366528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921564694082247, + "loss": 1.5235, + "theoretical_loss": 3.966520425496392, + "tokens_seen": 473432064 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921404212637914, + "loss": 1.4722, + "theoretical_loss": 3.9664611610302627, + "tokens_seen": 473497600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921243731193581, + "loss": 1.3878, + "theoretical_loss": 3.966401907062652, + "tokens_seen": 473563136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921083249749248, + "loss": 1.5362, + "theoretical_loss": 3.9663426635902477, + "tokens_seen": 473628672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920922768304915, + "loss": 1.6144, + "theoretical_loss": 3.9662834306097388, + "tokens_seen": 473694208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920762286860582, + "loss": 1.4815, + "theoretical_loss": 3.9662242081178167, + "tokens_seen": 473759744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920601805416249, + "loss": 1.508, + "theoretical_loss": 3.9661649961111727, + "tokens_seen": 473825280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920441323971916, + "loss": 1.4545, + "theoretical_loss": 3.9661057945865017, + "tokens_seen": 473890816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920280842527583, + "loss": 1.565, + "theoretical_loss": 3.966046603540498, + "tokens_seen": 473956352 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692012036108325, + "loss": 1.4851, + "theoretical_loss": 3.9659874229698584, + "tokens_seen": 474021888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919959879638917, + "loss": 1.4032, + "theoretical_loss": 3.9659282528712816, + "tokens_seen": 474087424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919799398194584, + "loss": 1.4323, + "theoretical_loss": 3.965869093241467, + "tokens_seen": 474152960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919638916750251, + "loss": 1.519, + "theoretical_loss": 3.965809944077116, + "tokens_seen": 474218496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919478435305918, + "loss": 1.4708, + "theoretical_loss": 3.965750805374931, + "tokens_seen": 474284032 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919317953861585, + "loss": 1.4264, + "theoretical_loss": 3.965691677131617, + "tokens_seen": 474349568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919157472417252, + "loss": 1.5769, + "theoretical_loss": 3.9656325593438786, + "tokens_seen": 474415104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918996990972919, + "loss": 1.4914, + "theoretical_loss": 3.9655734520084236, + "tokens_seen": 474480640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918836509528587, + "loss": 1.5013, + "theoretical_loss": 3.965514355121961, + "tokens_seen": 474546176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918676028084253, + "loss": 1.5206, + "theoretical_loss": 3.9654552686812004, + "tokens_seen": 474611712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918515546639921, + "loss": 1.452, + "theoretical_loss": 3.9653961926828534, + "tokens_seen": 474677248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918355065195587, + "loss": 1.5568, + "theoretical_loss": 3.9653371271236337, + "tokens_seen": 474742784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918194583751255, + "loss": 1.576, + "theoretical_loss": 3.9652780720002556, + "tokens_seen": 474808320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918034102306921, + "loss": 1.544, + "theoretical_loss": 3.965219027309435, + "tokens_seen": 474873856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917873620862589, + "loss": 1.5643, + "theoretical_loss": 3.9651599930478905, + "tokens_seen": 474939392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917713139418255, + "loss": 1.4463, + "theoretical_loss": 3.9651009692123402, + "tokens_seen": 475004928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917552657973923, + "loss": 1.5332, + "theoretical_loss": 3.9650419557995047, + "tokens_seen": 475070464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 164144, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6853489875793457, + "objective/train/theoretical_loss": 3.964982952806107, + "objective/train/tokens_used": 495596000, + "theoretical_loss": 3.964982952806107, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917392176529589, + "loss": 1.5972, + "theoretical_loss": 3.964982952806107, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917231695085257, + "loss": 1.4829, + "theoretical_loss": 3.9649239602288695, + "tokens_seen": 475201536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917071213640923, + "loss": 1.3827, + "theoretical_loss": 3.9648649780645178, + "tokens_seen": 475267072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916910732196591, + "loss": 1.4239, + "theoretical_loss": 3.9648060063097788, + "tokens_seen": 475332608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916750250752257, + "loss": 1.4822, + "theoretical_loss": 3.9647470449613795, + "tokens_seen": 475398144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916589769307925, + "loss": 1.4893, + "theoretical_loss": 3.9646880940160507, + "tokens_seen": 475463680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916429287863591, + "loss": 1.4965, + "theoretical_loss": 3.9646291534705225, + "tokens_seen": 475529216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916268806419258, + "loss": 1.5566, + "theoretical_loss": 3.9645702233215268, + "tokens_seen": 475594752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916108324974925, + "loss": 1.4053, + "theoretical_loss": 3.964511303565799, + "tokens_seen": 475660288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006915947843530592, + "loss": 1.5627, + "theoretical_loss": 3.964452394200073, + "tokens_seen": 475725824 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691578736208626, + "loss": 1.4794, + "theoretical_loss": 3.9643934952210866, + "tokens_seen": 475791360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006915626880641926, + "loss": 1.484, + "theoretical_loss": 3.9643346066255782, + "tokens_seen": 475856896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006915466399197593, + "loss": 1.5084, + "theoretical_loss": 3.964275728410287, + "tokens_seen": 475922432 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691530591775326, + "loss": 1.4504, + "theoretical_loss": 3.9642168605719545, + "tokens_seen": 475987968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006915145436308928, + "loss": 1.5379, + "theoretical_loss": 3.9641580031073236, + "tokens_seen": 476053504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914984954864594, + "loss": 1.5919, + "theoretical_loss": 3.9640991560131384, + "tokens_seen": 476119040 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691482447342026, + "loss": 1.4496, + "theoretical_loss": 3.964040319286145, + "tokens_seen": 476184576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914663991975928, + "loss": 1.4655, + "theoretical_loss": 3.9639814929230894, + "tokens_seen": 476250112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914503510531594, + "loss": 1.5148, + "theoretical_loss": 3.9639226769207214, + "tokens_seen": 476315648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914343029087262, + "loss": 1.4979, + "theoretical_loss": 3.963863871275791, + "tokens_seen": 476381184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914182547642928, + "loss": 1.5029, + "theoretical_loss": 3.963805075985049, + "tokens_seen": 476446720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914022066198596, + "loss": 1.5035, + "theoretical_loss": 3.9637462910452497, + "tokens_seen": 476512256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006913861584754262, + "loss": 1.6175, + "theoretical_loss": 3.963687516453146, + "tokens_seen": 476577792 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691370110330993, + "loss": 1.4935, + "theoretical_loss": 3.9636287522054947, + "tokens_seen": 476643328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006913540621865596, + "loss": 1.5319, + "theoretical_loss": 3.9635699982990538, + "tokens_seen": 476708864 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006913380140421264, + "loss": 1.3645, + "theoretical_loss": 3.963511254730581, + "tokens_seen": 476774400 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691321965897693, + "loss": 1.4714, + "theoretical_loss": 3.9634525214968375, + "tokens_seen": 476839936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006913059177532598, + "loss": 1.4609, + "theoretical_loss": 3.963393798594585, + "tokens_seen": 476905472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912898696088265, + "loss": 1.5017, + "theoretical_loss": 3.963335086020586, + "tokens_seen": 476971008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912738214643932, + "loss": 1.5675, + "theoretical_loss": 3.9632763837716065, + "tokens_seen": 477036544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912577733199599, + "loss": 1.5274, + "theoretical_loss": 3.9632176918444113, + "tokens_seen": 477102080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912417251755266, + "loss": 1.4453, + "theoretical_loss": 3.9631590102357697, + "tokens_seen": 477167616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912256770310933, + "loss": 1.491, + "theoretical_loss": 3.963100338942449, + "tokens_seen": 477233152 + }, + { + "epoch": 0.14, + "learning_rate": 0.00069120962888666, + "loss": 1.4311, + "theoretical_loss": 3.963041677961221, + "tokens_seen": 477298688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911935807422267, + "loss": 1.5531, + "theoretical_loss": 3.962983027288857, + "tokens_seen": 477364224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911775325977934, + "loss": 1.4859, + "theoretical_loss": 3.9629243869221304, + "tokens_seen": 477429760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911614844533601, + "loss": 1.5019, + "theoretical_loss": 3.962865756857817, + "tokens_seen": 477495296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911454363089268, + "loss": 1.5433, + "theoretical_loss": 3.9628071370926925, + "tokens_seen": 477560832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911293881644935, + "loss": 1.5089, + "theoretical_loss": 3.9627485276235346, + "tokens_seen": 477626368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911133400200602, + "loss": 1.5039, + "theoretical_loss": 3.962689928447123, + "tokens_seen": 477691904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910972918756269, + "loss": 1.6092, + "theoretical_loss": 3.962631339560238, + "tokens_seen": 477757440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910812437311936, + "loss": 1.5068, + "theoretical_loss": 3.962572760959662, + "tokens_seen": 477822976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910651955867603, + "loss": 1.4194, + "theoretical_loss": 3.9625141926421783, + "tokens_seen": 477888512 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691049147442327, + "loss": 1.5435, + "theoretical_loss": 3.9624556346045727, + "tokens_seen": 477954048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910330992978937, + "loss": 1.4924, + "theoretical_loss": 3.962397086843631, + "tokens_seen": 478019584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910170511534604, + "loss": 1.5266, + "theoretical_loss": 3.962338549356141, + "tokens_seen": 478085120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910010030090271, + "loss": 1.5045, + "theoretical_loss": 3.962280022138893, + "tokens_seen": 478150656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909849548645938, + "loss": 1.5396, + "theoretical_loss": 3.962221505188677, + "tokens_seen": 478216192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909689067201605, + "loss": 1.4155, + "theoretical_loss": 3.9621629985022855, + "tokens_seen": 478281728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909528585757272, + "loss": 1.482, + "theoretical_loss": 3.9621045020765124, + "tokens_seen": 478347264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 165388, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.461626648902893, + "objective/train/theoretical_loss": 3.9620460159081525, + "objective/train/tokens_used": 498872800, + "theoretical_loss": 3.9620460159081525, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909368104312939, + "loss": 1.4965, + "theoretical_loss": 3.9620460159081525, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909207622868606, + "loss": 1.5332, + "theoretical_loss": 3.9619875399940025, + "tokens_seen": 478478336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006909047141424273, + "loss": 1.4555, + "theoretical_loss": 3.9619290743308606, + "tokens_seen": 478543872 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690888665997994, + "loss": 1.5502, + "theoretical_loss": 3.961870618915526, + "tokens_seen": 478609408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908726178535607, + "loss": 1.4432, + "theoretical_loss": 3.961812173744801, + "tokens_seen": 478674944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908565697091274, + "loss": 1.4627, + "theoretical_loss": 3.961753738815486, + "tokens_seen": 478740480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908405215646941, + "loss": 1.4945, + "theoretical_loss": 3.961695314124385, + "tokens_seen": 478806016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908244734202608, + "loss": 1.5489, + "theoretical_loss": 3.961636899668304, + "tokens_seen": 478871552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908084252758275, + "loss": 1.5541, + "theoretical_loss": 3.96157849544405, + "tokens_seen": 478937088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907923771313942, + "loss": 1.5218, + "theoretical_loss": 3.96152010144843, + "tokens_seen": 479002624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907763289869609, + "loss": 1.476, + "theoretical_loss": 3.961461717678254, + "tokens_seen": 479068160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907602808425276, + "loss": 1.3915, + "theoretical_loss": 3.961403344130333, + "tokens_seen": 479133696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907442326980943, + "loss": 1.5185, + "theoretical_loss": 3.9613449808014796, + "tokens_seen": 479199232 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690728184553661, + "loss": 1.6288, + "theoretical_loss": 3.9612866276885073, + "tokens_seen": 479264768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907121364092277, + "loss": 1.5618, + "theoretical_loss": 3.961228284788231, + "tokens_seen": 479330304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906960882647944, + "loss": 1.403, + "theoretical_loss": 3.9611699520974684, + "tokens_seen": 479395840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906800401203612, + "loss": 1.5271, + "theoretical_loss": 3.961111629613036, + "tokens_seen": 479461376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906639919759278, + "loss": 1.627, + "theoretical_loss": 3.9610533173317553, + "tokens_seen": 479526912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906479438314946, + "loss": 1.4974, + "theoretical_loss": 3.960995015250446, + "tokens_seen": 479592448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906318956870612, + "loss": 1.4971, + "theoretical_loss": 3.96093672336593, + "tokens_seen": 479657984 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690615847542628, + "loss": 1.6243, + "theoretical_loss": 3.9608784416750322, + "tokens_seen": 479723520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905997993981946, + "loss": 1.4766, + "theoretical_loss": 3.9608201701745775, + "tokens_seen": 479789056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905837512537614, + "loss": 1.5663, + "theoretical_loss": 3.960761908861392, + "tokens_seen": 479854592 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690567703109328, + "loss": 1.4331, + "theoretical_loss": 3.9607036577323047, + "tokens_seen": 479920128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905516549648948, + "loss": 1.3873, + "theoretical_loss": 3.9606454167841445, + "tokens_seen": 479985664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905356068204614, + "loss": 1.5245, + "theoretical_loss": 3.960587186013742, + "tokens_seen": 480051200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905195586760282, + "loss": 1.5018, + "theoretical_loss": 3.96052896541793, + "tokens_seen": 480116736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905035105315948, + "loss": 1.4737, + "theoretical_loss": 3.9604707549935423, + "tokens_seen": 480182272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006904874623871616, + "loss": 1.5611, + "theoretical_loss": 3.9604125547374136, + "tokens_seen": 480247808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006904714142427282, + "loss": 1.4181, + "theoretical_loss": 3.9603543646463812, + "tokens_seen": 480313344 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690455366098295, + "loss": 1.5261, + "theoretical_loss": 3.9602961847172824, + "tokens_seen": 480378880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006904393179538616, + "loss": 1.5351, + "theoretical_loss": 3.960238014946957, + "tokens_seen": 480444416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006904232698094284, + "loss": 1.4876, + "theoretical_loss": 3.960179855332245, + "tokens_seen": 480509952 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690407221664995, + "loss": 1.4839, + "theoretical_loss": 3.96012170586999, + "tokens_seen": 480575488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903911735205618, + "loss": 1.3866, + "theoretical_loss": 3.9600635665570345, + "tokens_seen": 480641024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903751253761284, + "loss": 1.5976, + "theoretical_loss": 3.9600054373902243, + "tokens_seen": 480706560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903590772316952, + "loss": 1.4092, + "theoretical_loss": 3.959947318366405, + "tokens_seen": 480772096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903430290872618, + "loss": 1.4633, + "theoretical_loss": 3.9598892094824256, + "tokens_seen": 480837632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903269809428286, + "loss": 1.5516, + "theoretical_loss": 3.9598311107351347, + "tokens_seen": 480903168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903109327983952, + "loss": 1.5714, + "theoretical_loss": 3.959773022121383, + "tokens_seen": 480968704 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690294884653962, + "loss": 1.4482, + "theoretical_loss": 3.959714943638023, + "tokens_seen": 481034240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006902788365095286, + "loss": 1.4323, + "theoretical_loss": 3.9596568752819072, + "tokens_seen": 481099776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006902627883650954, + "loss": 1.5344, + "theoretical_loss": 3.9595988170498915, + "tokens_seen": 481165312 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690246740220662, + "loss": 1.5042, + "theoretical_loss": 3.959540768938832, + "tokens_seen": 481230848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006902306920762287, + "loss": 1.6065, + "theoretical_loss": 3.9594827309455862, + "tokens_seen": 481296384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006902146439317954, + "loss": 1.5888, + "theoretical_loss": 3.9594247030670133, + "tokens_seen": 481361920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901985957873621, + "loss": 1.4552, + "theoretical_loss": 3.959366685299974, + "tokens_seen": 481427456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901825476429288, + "loss": 1.564, + "theoretical_loss": 3.9593086776413298, + "tokens_seen": 481492992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901664994984955, + "loss": 1.4304, + "theoretical_loss": 3.959250680087945, + "tokens_seen": 481558528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901504513540622, + "loss": 1.4006, + "theoretical_loss": 3.959192692636683, + "tokens_seen": 481624064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 166207, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4316575527191162, + "objective/train/theoretical_loss": 3.959134715284411, + "objective/train/tokens_used": 502149600, + "theoretical_loss": 3.959134715284411, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901344032096289, + "loss": 1.5149, + "theoretical_loss": 3.959134715284411, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901183550651957, + "loss": 1.5353, + "theoretical_loss": 3.959076748027996, + "tokens_seen": 481755136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901023069207623, + "loss": 1.5337, + "theoretical_loss": 3.959018790864307, + "tokens_seen": 481820672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900862587763289, + "loss": 1.535, + "theoretical_loss": 3.9589608437902144, + "tokens_seen": 481886208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900702106318957, + "loss": 1.5224, + "theoretical_loss": 3.9589029068025896, + "tokens_seen": 481951744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900541624874623, + "loss": 1.4818, + "theoretical_loss": 3.9588449798983065, + "tokens_seen": 482017280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900381143430291, + "loss": 1.4084, + "theoretical_loss": 3.9587870630742383, + "tokens_seen": 482082816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900220661985957, + "loss": 1.6073, + "theoretical_loss": 3.958729156327262, + "tokens_seen": 482148352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900060180541625, + "loss": 1.5083, + "theoretical_loss": 3.9586712596542553, + "tokens_seen": 482213888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899899699097291, + "loss": 1.4613, + "theoretical_loss": 3.958613373052095, + "tokens_seen": 482279424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899739217652959, + "loss": 1.4953, + "theoretical_loss": 3.958555496517663, + "tokens_seen": 482344960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899578736208625, + "loss": 1.4788, + "theoretical_loss": 3.95849763004784, + "tokens_seen": 482410496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899418254764293, + "loss": 1.4487, + "theoretical_loss": 3.958439773639509, + "tokens_seen": 482476032 + }, + { + "epoch": 0.15, + "learning_rate": 0.000689925777331996, + "loss": 1.4259, + "theoretical_loss": 3.958381927289554, + "tokens_seen": 482541568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899097291875627, + "loss": 1.4236, + "theoretical_loss": 3.958324090994861, + "tokens_seen": 482607104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898936810431294, + "loss": 1.4207, + "theoretical_loss": 3.958266264752317, + "tokens_seen": 482672640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898776328986961, + "loss": 1.5267, + "theoretical_loss": 3.9582084485588096, + "tokens_seen": 482738176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898615847542628, + "loss": 1.5434, + "theoretical_loss": 3.95815064241123, + "tokens_seen": 482803712 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898455366098295, + "loss": 1.4206, + "theoretical_loss": 3.958092846306468, + "tokens_seen": 482869248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898294884653962, + "loss": 1.5472, + "theoretical_loss": 3.958035060241417, + "tokens_seen": 482934784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898134403209629, + "loss": 1.5298, + "theoretical_loss": 3.957977284212971, + "tokens_seen": 483000320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897973921765296, + "loss": 1.3356, + "theoretical_loss": 3.9579195182180245, + "tokens_seen": 483065856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897813440320963, + "loss": 1.4347, + "theoretical_loss": 3.957861762253475, + "tokens_seen": 483131392 + }, + { + "epoch": 0.15, + "learning_rate": 0.000689765295887663, + "loss": 1.4678, + "theoretical_loss": 3.9578040163162203, + "tokens_seen": 483196928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897492477432297, + "loss": 1.5097, + "theoretical_loss": 3.95774628040316, + "tokens_seen": 483262464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897331995987964, + "loss": 1.4883, + "theoretical_loss": 3.9576885545111944, + "tokens_seen": 483328000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897171514543631, + "loss": 1.4399, + "theoretical_loss": 3.9576308386372263, + "tokens_seen": 483393536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897011033099298, + "loss": 1.6046, + "theoretical_loss": 3.957573132778159, + "tokens_seen": 483459072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896850551654965, + "loss": 1.499, + "theoretical_loss": 3.9575154369308985, + "tokens_seen": 483524608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896690070210632, + "loss": 1.482, + "theoretical_loss": 3.957457751092349, + "tokens_seen": 483590144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896529588766299, + "loss": 1.3449, + "theoretical_loss": 3.9574000752594203, + "tokens_seen": 483655680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896369107321966, + "loss": 1.466, + "theoretical_loss": 3.9573424094290206, + "tokens_seen": 483721216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896208625877633, + "loss": 1.3929, + "theoretical_loss": 3.9572847535980604, + "tokens_seen": 483786752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068960481444333, + "loss": 1.446, + "theoretical_loss": 3.9572271077634515, + "tokens_seen": 483852288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895887662988967, + "loss": 1.5478, + "theoretical_loss": 3.957169471922107, + "tokens_seen": 483917824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895727181544634, + "loss": 1.4874, + "theoretical_loss": 3.9571118460709425, + "tokens_seen": 483983360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895566700100301, + "loss": 1.4437, + "theoretical_loss": 3.9570542302068725, + "tokens_seen": 484048896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895406218655968, + "loss": 1.5434, + "theoretical_loss": 3.9569966243268153, + "tokens_seen": 484114432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895245737211635, + "loss": 1.4222, + "theoretical_loss": 3.9569390284276893, + "tokens_seen": 484179968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895085255767302, + "loss": 1.4055, + "theoretical_loss": 3.9568814425064147, + "tokens_seen": 484245504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894924774322969, + "loss": 1.5564, + "theoretical_loss": 3.9568238665599127, + "tokens_seen": 484311040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894764292878637, + "loss": 1.5019, + "theoretical_loss": 3.9567663005851057, + "tokens_seen": 484376576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894603811434303, + "loss": 1.4021, + "theoretical_loss": 3.956708744578919, + "tokens_seen": 484442112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894443329989971, + "loss": 1.4476, + "theoretical_loss": 3.9566511985382773, + "tokens_seen": 484507648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894282848545637, + "loss": 1.4908, + "theoretical_loss": 3.956593662460108, + "tokens_seen": 484573184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894122367101305, + "loss": 1.5107, + "theoretical_loss": 3.9565361363413385, + "tokens_seen": 484638720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893961885656971, + "loss": 1.4567, + "theoretical_loss": 3.9564786201788995, + "tokens_seen": 484704256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893801404212639, + "loss": 1.5791, + "theoretical_loss": 3.956421113969721, + "tokens_seen": 484769792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893640922768305, + "loss": 1.4383, + "theoretical_loss": 3.9563636177107364, + "tokens_seen": 484835328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893480441323973, + "loss": 1.4124, + "theoretical_loss": 3.956306131398878, + "tokens_seen": 484900864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 167502, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4446344375610352, + "objective/train/theoretical_loss": 3.9562486550310823, + "objective/train/tokens_used": 505426400, + "theoretical_loss": 3.9562486550310823, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893319959879639, + "loss": 1.3573, + "theoretical_loss": 3.9562486550310823, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893159478435307, + "loss": 1.4464, + "theoretical_loss": 3.9561911886042846, + "tokens_seen": 485031936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892998996990973, + "loss": 1.4097, + "theoretical_loss": 3.9561337321154237, + "tokens_seen": 485097472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892838515546641, + "loss": 1.4955, + "theoretical_loss": 3.956076285561438, + "tokens_seen": 485163008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892678034102307, + "loss": 1.4266, + "theoretical_loss": 3.9560188489392676, + "tokens_seen": 485228544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892517552657975, + "loss": 1.4735, + "theoretical_loss": 3.9559614222458555, + "tokens_seen": 485294080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892357071213641, + "loss": 1.4313, + "theoretical_loss": 3.955904005478144, + "tokens_seen": 485359616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892196589769309, + "loss": 1.4503, + "theoretical_loss": 3.9558465986330784, + "tokens_seen": 485425152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892036108324975, + "loss": 1.5692, + "theoretical_loss": 3.9557892017076037, + "tokens_seen": 485490688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891875626880643, + "loss": 1.4592, + "theoretical_loss": 3.9557318146986677, + "tokens_seen": 485556224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891715145436309, + "loss": 1.4865, + "theoretical_loss": 3.9556744376032196, + "tokens_seen": 485621760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891554663991977, + "loss": 1.554, + "theoretical_loss": 3.9556170704182083, + "tokens_seen": 485687296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891394182547643, + "loss": 1.4508, + "theoretical_loss": 3.955559713140585, + "tokens_seen": 485752832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891233701103311, + "loss": 1.4282, + "theoretical_loss": 3.9555023657673036, + "tokens_seen": 485818368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891073219658977, + "loss": 1.4978, + "theoretical_loss": 3.955445028295317, + "tokens_seen": 485883904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890912738214645, + "loss": 1.418, + "theoretical_loss": 3.955387700721581, + "tokens_seen": 485949440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890752256770311, + "loss": 1.5058, + "theoretical_loss": 3.9553303830430524, + "tokens_seen": 486014976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890591775325979, + "loss": 1.5994, + "theoretical_loss": 3.955273075256689, + "tokens_seen": 486080512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890431293881645, + "loss": 1.4623, + "theoretical_loss": 3.9552157773594496, + "tokens_seen": 486146048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890270812437313, + "loss": 1.4523, + "theoretical_loss": 3.9551584893482965, + "tokens_seen": 486211584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890110330992979, + "loss": 1.5572, + "theoretical_loss": 3.9551012112201906, + "tokens_seen": 486277120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889949849548647, + "loss": 1.4486, + "theoretical_loss": 3.9550439429720954, + "tokens_seen": 486342656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889789368104313, + "loss": 1.4769, + "theoretical_loss": 3.9549866846009762, + "tokens_seen": 486408192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889628886659981, + "loss": 1.6685, + "theoretical_loss": 3.9549294361037988, + "tokens_seen": 486473728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889468405215647, + "loss": 1.5421, + "theoretical_loss": 3.9548721974775303, + "tokens_seen": 486539264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889307923771315, + "loss": 1.477, + "theoretical_loss": 3.95481496871914, + "tokens_seen": 486604800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889147442326981, + "loss": 1.5481, + "theoretical_loss": 3.954757749825598, + "tokens_seen": 486670336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888986960882649, + "loss": 1.4039, + "theoretical_loss": 3.9547005407938753, + "tokens_seen": 486735872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888826479438315, + "loss": 1.3672, + "theoretical_loss": 3.9546433416209448, + "tokens_seen": 486801408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888665997993983, + "loss": 1.5134, + "theoretical_loss": 3.9545861523037815, + "tokens_seen": 486866944 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688850551654965, + "loss": 1.5121, + "theoretical_loss": 3.95452897283936, + "tokens_seen": 486932480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888345035105317, + "loss": 1.5094, + "theoretical_loss": 3.9544718032246573, + "tokens_seen": 486998016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888184553660983, + "loss": 1.5549, + "theoretical_loss": 3.9544146434566514, + "tokens_seen": 487063552 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688802407221665, + "loss": 1.4202, + "theoretical_loss": 3.9543574935323216, + "tokens_seen": 487129088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887863590772317, + "loss": 1.4835, + "theoretical_loss": 3.95430035344865, + "tokens_seen": 487194624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887703109327984, + "loss": 1.4217, + "theoretical_loss": 3.954243223202617, + "tokens_seen": 487260160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887542627883652, + "loss": 1.4977, + "theoretical_loss": 3.9541861027912066, + "tokens_seen": 487325696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887382146439318, + "loss": 1.5066, + "theoretical_loss": 3.9541289922114045, + "tokens_seen": 487391232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887221664994986, + "loss": 1.5499, + "theoretical_loss": 3.954071891460196, + "tokens_seen": 487456768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887061183550652, + "loss": 1.4346, + "theoretical_loss": 3.954014800534569, + "tokens_seen": 487522304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886900702106318, + "loss": 1.4414, + "theoretical_loss": 3.9539577194315116, + "tokens_seen": 487587840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886740220661986, + "loss": 1.5233, + "theoretical_loss": 3.953900648148015, + "tokens_seen": 487653376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886579739217652, + "loss": 1.5257, + "theoretical_loss": 3.953843586681069, + "tokens_seen": 487718912 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688641925777332, + "loss": 1.4625, + "theoretical_loss": 3.9537865350276684, + "tokens_seen": 487784448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886258776328986, + "loss": 1.6209, + "theoretical_loss": 3.953729493184806, + "tokens_seen": 487849984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886098294884654, + "loss": 1.4365, + "theoretical_loss": 3.953672461149477, + "tokens_seen": 487915520 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688593781344032, + "loss": 1.5948, + "theoretical_loss": 3.953615438918679, + "tokens_seen": 487981056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885777331995988, + "loss": 1.4913, + "theoretical_loss": 3.9535584264894092, + "tokens_seen": 488046592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885616850551655, + "loss": 1.4834, + "theoretical_loss": 3.9535014238586683, + "tokens_seen": 488112128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885456369107322, + "loss": 1.5498, + "theoretical_loss": 3.953444431023456, + "tokens_seen": 488177664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 168681, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5218801498413086, + "objective/train/theoretical_loss": 3.9533874479807745, + "objective/train/tokens_used": 508703200, + "theoretical_loss": 3.9533874479807745, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885295887662989, + "loss": 1.4728, + "theoretical_loss": 3.9533874479807745, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885135406218656, + "loss": 1.4928, + "theoretical_loss": 3.9533304747276263, + "tokens_seen": 488308736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884974924774323, + "loss": 1.4964, + "theoretical_loss": 3.953273511261018, + "tokens_seen": 488374272 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688481444332999, + "loss": 1.4585, + "theoretical_loss": 3.953216557577954, + "tokens_seen": 488439808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884653961885657, + "loss": 1.5129, + "theoretical_loss": 3.9531596136754423, + "tokens_seen": 488505344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884493480441324, + "loss": 1.3073, + "theoretical_loss": 3.953102679550491, + "tokens_seen": 488570880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884332998996991, + "loss": 1.6385, + "theoretical_loss": 3.953045755200111, + "tokens_seen": 488636416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884172517552658, + "loss": 1.4704, + "theoretical_loss": 3.9529888406213125, + "tokens_seen": 488701952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884012036108325, + "loss": 1.4706, + "theoretical_loss": 3.952931935811108, + "tokens_seen": 488767488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883851554663992, + "loss": 1.4269, + "theoretical_loss": 3.9528750407665125, + "tokens_seen": 488833024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883691073219659, + "loss": 1.6005, + "theoretical_loss": 3.9528181554845405, + "tokens_seen": 488898560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883530591775326, + "loss": 1.3151, + "theoretical_loss": 3.9527612799622087, + "tokens_seen": 488964096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883370110330993, + "loss": 1.3791, + "theoretical_loss": 3.9527044141965346, + "tokens_seen": 489029632 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688320962888666, + "loss": 1.5089, + "theoretical_loss": 3.9526475581845375, + "tokens_seen": 489095168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883049147442327, + "loss": 1.4356, + "theoretical_loss": 3.9525907119232384, + "tokens_seen": 489160704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882888665997994, + "loss": 1.4122, + "theoretical_loss": 3.952533875409658, + "tokens_seen": 489226240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882728184553662, + "loss": 1.6054, + "theoretical_loss": 3.95247704864082, + "tokens_seen": 489291776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882567703109328, + "loss": 1.5323, + "theoretical_loss": 3.952420231613748, + "tokens_seen": 489357312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882407221664996, + "loss": 1.4921, + "theoretical_loss": 3.952363424325469, + "tokens_seen": 489422848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882246740220662, + "loss": 1.5348, + "theoretical_loss": 3.952306626773009, + "tokens_seen": 489488384 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688208625877633, + "loss": 1.2855, + "theoretical_loss": 3.9522498389533967, + "tokens_seen": 489553920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881925777331996, + "loss": 1.4844, + "theoretical_loss": 3.952193060863661, + "tokens_seen": 489619456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881765295887664, + "loss": 1.4188, + "theoretical_loss": 3.9521362925008336, + "tokens_seen": 489684992 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688160481444333, + "loss": 1.3951, + "theoretical_loss": 3.952079533861947, + "tokens_seen": 489750528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881444332998998, + "loss": 1.5724, + "theoretical_loss": 3.9520227849440337, + "tokens_seen": 489816064 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881283851554664, + "loss": 1.3901, + "theoretical_loss": 3.951966045744129, + "tokens_seen": 489881600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881123370110332, + "loss": 1.5279, + "theoretical_loss": 3.9519093162592687, + "tokens_seen": 489947136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880962888665998, + "loss": 1.4731, + "theoretical_loss": 3.95185259648649, + "tokens_seen": 490012672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880802407221666, + "loss": 1.4999, + "theoretical_loss": 3.951795886422833, + "tokens_seen": 490078208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880641925777332, + "loss": 1.5234, + "theoretical_loss": 3.9517391860653364, + "tokens_seen": 490143744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880481444333, + "loss": 1.5716, + "theoretical_loss": 3.951682495411041, + "tokens_seen": 490209280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880320962888666, + "loss": 1.4127, + "theoretical_loss": 3.9516258144569907, + "tokens_seen": 490274816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880160481444334, + "loss": 1.4481, + "theoretical_loss": 3.9515691432002296, + "tokens_seen": 490340352 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688, + "loss": 1.5564, + "theoretical_loss": 3.951512481637801, + "tokens_seen": 490405888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879839518555668, + "loss": 1.5187, + "theoretical_loss": 3.9514558297667532, + "tokens_seen": 490471424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879679037111334, + "loss": 1.5075, + "theoretical_loss": 3.9513991875841334, + "tokens_seen": 490536960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879518555667002, + "loss": 1.4442, + "theoretical_loss": 3.95134255508699, + "tokens_seen": 490602496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879358074222668, + "loss": 1.5834, + "theoretical_loss": 3.9512859322723743, + "tokens_seen": 490668032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879197592778336, + "loss": 1.5289, + "theoretical_loss": 3.9512293191373375, + "tokens_seen": 490733568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879037111334002, + "loss": 1.6062, + "theoretical_loss": 3.9511727156789327, + "tokens_seen": 490799104 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687887662988967, + "loss": 1.5124, + "theoretical_loss": 3.951116121894214, + "tokens_seen": 490864640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006878716148445336, + "loss": 1.4595, + "theoretical_loss": 3.951059537780237, + "tokens_seen": 490930176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006878555667001004, + "loss": 1.5176, + "theoretical_loss": 3.951002963334059, + "tokens_seen": 490995712 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687839518555667, + "loss": 1.4955, + "theoretical_loss": 3.9509463985527375, + "tokens_seen": 491061248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006878234704112338, + "loss": 1.4248, + "theoretical_loss": 3.9508898434333313, + "tokens_seen": 491126784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006878074222668004, + "loss": 1.5181, + "theoretical_loss": 3.9508332979729026, + "tokens_seen": 491192320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877913741223672, + "loss": 1.4302, + "theoretical_loss": 3.950776762168512, + "tokens_seen": 491257856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877753259779338, + "loss": 1.4632, + "theoretical_loss": 3.9507202360172236, + "tokens_seen": 491323392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877592778335006, + "loss": 1.3747, + "theoretical_loss": 3.9506637195161014, + "tokens_seen": 491388928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877432296890672, + "loss": 1.3571, + "theoretical_loss": 3.950607212662211, + "tokens_seen": 491454464 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 169679, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.443345069885254, + "objective/train/theoretical_loss": 3.950550715452621, + "objective/train/tokens_used": 511980000, + "theoretical_loss": 3.950550715452621, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687727181544634, + "loss": 1.4414, + "theoretical_loss": 3.950550715452621, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877111334002006, + "loss": 1.5504, + "theoretical_loss": 3.9504942278843984, + "tokens_seen": 491585536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876950852557674, + "loss": 1.4341, + "theoretical_loss": 3.950437749954613, + "tokens_seen": 491651072 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687679037111334, + "loss": 1.5328, + "theoretical_loss": 3.950381281660336, + "tokens_seen": 491716608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876629889669008, + "loss": 1.5646, + "theoretical_loss": 3.950324822998639, + "tokens_seen": 491782144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876469408224674, + "loss": 1.5351, + "theoretical_loss": 3.950268373966597, + "tokens_seen": 491847680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876308926780342, + "loss": 1.5962, + "theoretical_loss": 3.950211934561284, + "tokens_seen": 491913216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876148445336008, + "loss": 1.5011, + "theoretical_loss": 3.950155504779775, + "tokens_seen": 491978752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875987963891676, + "loss": 1.5318, + "theoretical_loss": 3.9500990846191484, + "tokens_seen": 492044288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875827482447342, + "loss": 1.4562, + "theoretical_loss": 3.950042674076483, + "tokens_seen": 492109824 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687566700100301, + "loss": 1.5587, + "theoretical_loss": 3.949986273148858, + "tokens_seen": 492175360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875506519558676, + "loss": 1.6206, + "theoretical_loss": 3.9499298818333557, + "tokens_seen": 492240896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875346038114344, + "loss": 1.5126, + "theoretical_loss": 3.9498735001270573, + "tokens_seen": 492306432 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687518555667001, + "loss": 1.6095, + "theoretical_loss": 3.949817128027047, + "tokens_seen": 492371968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875025075225678, + "loss": 1.4721, + "theoretical_loss": 3.9497607655304097, + "tokens_seen": 492437504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874864593781344, + "loss": 1.4875, + "theoretical_loss": 3.949704412634232, + "tokens_seen": 492503040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874704112337012, + "loss": 1.5354, + "theoretical_loss": 3.949648069335601, + "tokens_seen": 492568576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874543630892678, + "loss": 1.5265, + "theoretical_loss": 3.9495917356316057, + "tokens_seen": 492634112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874383149448346, + "loss": 1.4206, + "theoretical_loss": 3.9495354115193364, + "tokens_seen": 492699648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874222668004012, + "loss": 1.5544, + "theoretical_loss": 3.9494790969958835, + "tokens_seen": 492765184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874062186559679, + "loss": 1.5174, + "theoretical_loss": 3.949422792058341, + "tokens_seen": 492830720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873901705115347, + "loss": 1.4034, + "theoretical_loss": 3.949366496703802, + "tokens_seen": 492896256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873741223671013, + "loss": 1.5041, + "theoretical_loss": 3.9493102109293616, + "tokens_seen": 492961792 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687358074222668, + "loss": 1.5309, + "theoretical_loss": 3.9492539347321167, + "tokens_seen": 493027328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873420260782347, + "loss": 1.3729, + "theoretical_loss": 3.9491976681091643, + "tokens_seen": 493092864 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873259779338015, + "loss": 1.428, + "theoretical_loss": 3.949141411057604, + "tokens_seen": 493158400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873099297893681, + "loss": 1.5152, + "theoretical_loss": 3.9490851635745354, + "tokens_seen": 493223936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872938816449349, + "loss": 1.4597, + "theoretical_loss": 3.9490289256570605, + "tokens_seen": 493289472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872778335005015, + "loss": 1.5602, + "theoretical_loss": 3.9489726973022816, + "tokens_seen": 493355008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872617853560681, + "loss": 1.4065, + "theoretical_loss": 3.948916478507303, + "tokens_seen": 493420544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872457372116349, + "loss": 1.4266, + "theoretical_loss": 3.94886026926923, + "tokens_seen": 493486080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872296890672015, + "loss": 1.4491, + "theoretical_loss": 3.9488040695851687, + "tokens_seen": 493551616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872136409227683, + "loss": 1.4786, + "theoretical_loss": 3.9487478794522275, + "tokens_seen": 493617152 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687197592778335, + "loss": 1.5208, + "theoretical_loss": 3.948691698867515, + "tokens_seen": 493682688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871815446339017, + "loss": 1.4405, + "theoretical_loss": 3.9486355278281415, + "tokens_seen": 493748224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871654964894684, + "loss": 1.4269, + "theoretical_loss": 3.948579366331219, + "tokens_seen": 493813760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871494483450351, + "loss": 1.524, + "theoretical_loss": 3.94852321437386, + "tokens_seen": 493879296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871334002006018, + "loss": 1.4391, + "theoretical_loss": 3.9484670719531785, + "tokens_seen": 493944832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871173520561685, + "loss": 1.5082, + "theoretical_loss": 3.9484109390662896, + "tokens_seen": 494010368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871013039117352, + "loss": 1.5184, + "theoretical_loss": 3.94835481571031, + "tokens_seen": 494075904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870852557673019, + "loss": 1.4863, + "theoretical_loss": 3.948298701882358, + "tokens_seen": 494141440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870692076228687, + "loss": 1.5287, + "theoretical_loss": 3.9482425975795525, + "tokens_seen": 494206976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870531594784353, + "loss": 1.5385, + "theoretical_loss": 3.9481865027990137, + "tokens_seen": 494272512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870371113340021, + "loss": 1.5284, + "theoretical_loss": 3.9481304175378633, + "tokens_seen": 494338048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870210631895687, + "loss": 1.4945, + "theoretical_loss": 3.948074341793224, + "tokens_seen": 494403584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870050150451355, + "loss": 1.5887, + "theoretical_loss": 3.94801827556222, + "tokens_seen": 494469120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869889669007021, + "loss": 1.4482, + "theoretical_loss": 3.9479622188419765, + "tokens_seen": 494534656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869729187562689, + "loss": 1.4412, + "theoretical_loss": 3.9479061716296204, + "tokens_seen": 494600192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869568706118355, + "loss": 1.5186, + "theoretical_loss": 3.9478501339222793, + "tokens_seen": 494665728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869408224674023, + "loss": 1.4655, + "theoretical_loss": 3.9477941057170822, + "tokens_seen": 494731264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 170930, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5508875846862793, + "objective/train/theoretical_loss": 3.9477380870111602, + "objective/train/tokens_used": 515256800, + "theoretical_loss": 3.9477380870111602, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869247743229689, + "loss": 1.4898, + "theoretical_loss": 3.9477380870111602, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869087261785357, + "loss": 1.5979, + "theoretical_loss": 3.9476820778016437, + "tokens_seen": 494862336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868926780341023, + "loss": 1.5028, + "theoretical_loss": 3.9476260780856665, + "tokens_seen": 494927872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868766298896691, + "loss": 1.5157, + "theoretical_loss": 3.947570087860362, + "tokens_seen": 494993408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868605817452357, + "loss": 1.4887, + "theoretical_loss": 3.947514107122866, + "tokens_seen": 495058944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868445336008025, + "loss": 1.5196, + "theoretical_loss": 3.9474581358703147, + "tokens_seen": 495124480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868284854563691, + "loss": 1.4206, + "theoretical_loss": 3.947402174099847, + "tokens_seen": 495190016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868124373119359, + "loss": 1.4628, + "theoretical_loss": 3.9473462218085995, + "tokens_seen": 495255552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867963891675025, + "loss": 1.468, + "theoretical_loss": 3.947290278993715, + "tokens_seen": 495321088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867803410230693, + "loss": 1.4707, + "theoretical_loss": 3.947234345652334, + "tokens_seen": 495386624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867642928786359, + "loss": 1.4693, + "theoretical_loss": 3.9471784217815995, + "tokens_seen": 495452160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867482447342027, + "loss": 1.524, + "theoretical_loss": 3.9471225073786553, + "tokens_seen": 495517696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867321965897693, + "loss": 1.5436, + "theoretical_loss": 3.9470666024406467, + "tokens_seen": 495583232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867161484453361, + "loss": 1.5024, + "theoretical_loss": 3.9470107069647202, + "tokens_seen": 495648768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867001003009027, + "loss": 1.3839, + "theoretical_loss": 3.946954820948024, + "tokens_seen": 495714304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866840521564695, + "loss": 1.5284, + "theoretical_loss": 3.946898944387706, + "tokens_seen": 495779840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866680040120361, + "loss": 1.5712, + "theoretical_loss": 3.946843077280917, + "tokens_seen": 495845376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866519558676029, + "loss": 1.6285, + "theoretical_loss": 3.946787219624809, + "tokens_seen": 495910912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866359077231695, + "loss": 1.6073, + "theoretical_loss": 3.9467313714165337, + "tokens_seen": 495976448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866198595787363, + "loss": 1.4352, + "theoretical_loss": 3.946675532653246, + "tokens_seen": 496041984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866038114343029, + "loss": 1.514, + "theoretical_loss": 3.9466197033321, + "tokens_seen": 496107520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865877632898697, + "loss": 1.5126, + "theoretical_loss": 3.9465638834502528, + "tokens_seen": 496173056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865717151454363, + "loss": 1.4771, + "theoretical_loss": 3.9465080730048614, + "tokens_seen": 496238592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865556670010031, + "loss": 1.4483, + "theoretical_loss": 3.9464522719930857, + "tokens_seen": 496304128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865396188565697, + "loss": 1.5381, + "theoretical_loss": 3.9463964804120844, + "tokens_seen": 496369664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865235707121365, + "loss": 1.5251, + "theoretical_loss": 3.9463406982590197, + "tokens_seen": 496435200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865075225677031, + "loss": 1.4486, + "theoretical_loss": 3.946284925531054, + "tokens_seen": 496500736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864914744232699, + "loss": 1.5538, + "theoretical_loss": 3.9462291622253503, + "tokens_seen": 496566272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864754262788365, + "loss": 1.3553, + "theoretical_loss": 3.946173408339075, + "tokens_seen": 496631808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864593781344033, + "loss": 1.4294, + "theoretical_loss": 3.9461176638693933, + "tokens_seen": 496697344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864433299899699, + "loss": 1.5686, + "theoretical_loss": 3.9460619288134726, + "tokens_seen": 496762880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864272818455367, + "loss": 1.4742, + "theoretical_loss": 3.9460062031684817, + "tokens_seen": 496828416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864112337011033, + "loss": 1.3954, + "theoretical_loss": 3.945950486931591, + "tokens_seen": 496893952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863951855566701, + "loss": 1.5157, + "theoretical_loss": 3.9458947800999713, + "tokens_seen": 496959488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863791374122367, + "loss": 1.4627, + "theoretical_loss": 3.9458390826707945, + "tokens_seen": 497025024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863630892678035, + "loss": 1.4251, + "theoretical_loss": 3.945783394641235, + "tokens_seen": 497090560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863470411233701, + "loss": 1.6089, + "theoretical_loss": 3.9457277160084665, + "tokens_seen": 497156096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863309929789369, + "loss": 1.5565, + "theoretical_loss": 3.9456720467696655, + "tokens_seen": 497221632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863149448345035, + "loss": 1.5502, + "theoretical_loss": 3.9456163869220093, + "tokens_seen": 497287168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862988966900703, + "loss": 1.4875, + "theoretical_loss": 3.945560736462677, + "tokens_seen": 497352704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862828485456369, + "loss": 1.4912, + "theoretical_loss": 3.945505095388847, + "tokens_seen": 497418240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862668004012037, + "loss": 1.4862, + "theoretical_loss": 3.9454494636977007, + "tokens_seen": 497483776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862507522567703, + "loss": 1.5397, + "theoretical_loss": 3.9453938413864207, + "tokens_seen": 497549312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862347041123371, + "loss": 1.5082, + "theoretical_loss": 3.94533822845219, + "tokens_seen": 497614848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862186559679037, + "loss": 1.5358, + "theoretical_loss": 3.9452826248921924, + "tokens_seen": 497680384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862026078234705, + "loss": 1.4845, + "theoretical_loss": 3.9452270307036144, + "tokens_seen": 497745920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861865596790371, + "loss": 1.4408, + "theoretical_loss": 3.945171445883643, + "tokens_seen": 497811456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861705115346039, + "loss": 1.475, + "theoretical_loss": 3.9451158704294658, + "tokens_seen": 497876992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861544633901705, + "loss": 1.4578, + "theoretical_loss": 3.945060304338273, + "tokens_seen": 497942528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861384152457373, + "loss": 1.4505, + "theoretical_loss": 3.9450047476072547, + "tokens_seen": 498008064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 171822, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6084593534469604, + "objective/train/theoretical_loss": 3.9449492002336024, + "objective/train/tokens_used": 518533600, + "theoretical_loss": 3.9449492002336024, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.000686122367101304, + "loss": 1.5196, + "theoretical_loss": 3.9449492002336024, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861063189568707, + "loss": 1.5035, + "theoretical_loss": 3.9448936622145103, + "tokens_seen": 498139136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860902708124373, + "loss": 1.49, + "theoretical_loss": 3.9448381335471714, + "tokens_seen": 498204672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860742226680041, + "loss": 1.5774, + "theoretical_loss": 3.9447826142287816, + "tokens_seen": 498270208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860581745235707, + "loss": 1.4464, + "theoretical_loss": 3.944727104256537, + "tokens_seen": 498335744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860421263791375, + "loss": 1.5822, + "theoretical_loss": 3.9446716036276364, + "tokens_seen": 498401280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860260782347042, + "loss": 1.4544, + "theoretical_loss": 3.944616112339279, + "tokens_seen": 498466816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860100300902708, + "loss": 1.4174, + "theoretical_loss": 3.944560630388664, + "tokens_seen": 498532352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006859939819458376, + "loss": 1.4449, + "theoretical_loss": 3.9445051577729937, + "tokens_seen": 498597888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006859779338014042, + "loss": 1.4734, + "theoretical_loss": 3.944449694489471, + "tokens_seen": 498663424 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685961885656971, + "loss": 1.5683, + "theoretical_loss": 3.9443942405352987, + "tokens_seen": 498728960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006859458375125376, + "loss": 1.4521, + "theoretical_loss": 3.9443387959076825, + "tokens_seen": 498794496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006859297893681044, + "loss": 1.4705, + "theoretical_loss": 3.944283360603829, + "tokens_seen": 498860032 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685913741223671, + "loss": 1.5388, + "theoretical_loss": 3.9442279346209457, + "tokens_seen": 498925568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858976930792378, + "loss": 1.5262, + "theoretical_loss": 3.944172517956241, + "tokens_seen": 498991104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858816449348044, + "loss": 1.5162, + "theoretical_loss": 3.944117110606925, + "tokens_seen": 499056640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858655967903712, + "loss": 1.4281, + "theoretical_loss": 3.9440617125702087, + "tokens_seen": 499122176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858495486459378, + "loss": 1.4013, + "theoretical_loss": 3.9440063238433045, + "tokens_seen": 499187712 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858335005015046, + "loss": 1.4952, + "theoretical_loss": 3.9439509444234258, + "tokens_seen": 499253248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858174523570712, + "loss": 1.4168, + "theoretical_loss": 3.943895574307788, + "tokens_seen": 499318784 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685801404212638, + "loss": 1.4904, + "theoretical_loss": 3.9438402134936057, + "tokens_seen": 499384320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857853560682046, + "loss": 1.4545, + "theoretical_loss": 3.9437848619780973, + "tokens_seen": 499449856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857693079237714, + "loss": 1.5395, + "theoretical_loss": 3.94372951975848, + "tokens_seen": 499515392 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685753259779338, + "loss": 1.5102, + "theoretical_loss": 3.9436741868319745, + "tokens_seen": 499580928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857372116349048, + "loss": 1.5157, + "theoretical_loss": 3.943618863195801, + "tokens_seen": 499646464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857211634904714, + "loss": 1.4785, + "theoretical_loss": 3.943563548847181, + "tokens_seen": 499712000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857051153460382, + "loss": 1.6383, + "theoretical_loss": 3.943508243783338, + "tokens_seen": 499777536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856890672016048, + "loss": 1.5405, + "theoretical_loss": 3.9434529480014966, + "tokens_seen": 499843072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856730190571716, + "loss": 1.4985, + "theoretical_loss": 3.9433976614988815, + "tokens_seen": 499908608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856569709127382, + "loss": 1.4688, + "theoretical_loss": 3.9433423842727198, + "tokens_seen": 499974144 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685640922768305, + "loss": 1.5811, + "theoretical_loss": 3.9432871163202394, + "tokens_seen": 500039680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856248746238716, + "loss": 1.4328, + "theoretical_loss": 3.9432318576386693, + "tokens_seen": 500105216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856088264794384, + "loss": 1.3487, + "theoretical_loss": 3.94317660822524, + "tokens_seen": 500170752 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685592778335005, + "loss": 1.5404, + "theoretical_loss": 3.9431213680771826, + "tokens_seen": 500236288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855767301905718, + "loss": 1.6125, + "theoretical_loss": 3.94306613719173, + "tokens_seen": 500301824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855606820461384, + "loss": 1.5333, + "theoretical_loss": 3.9430109155661164, + "tokens_seen": 500367360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855446339017052, + "loss": 1.5013, + "theoretical_loss": 3.9429557031975753, + "tokens_seen": 500432896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855285857572718, + "loss": 1.3822, + "theoretical_loss": 3.9429005000833444, + "tokens_seen": 500498432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855125376128386, + "loss": 1.3441, + "theoretical_loss": 3.9428453062206605, + "tokens_seen": 500563968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854964894684052, + "loss": 1.5172, + "theoretical_loss": 3.942790121606763, + "tokens_seen": 500629504 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685480441323972, + "loss": 1.522, + "theoretical_loss": 3.9427349462388905, + "tokens_seen": 500695040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854643931795386, + "loss": 1.5221, + "theoretical_loss": 3.942679780114284, + "tokens_seen": 500760576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854483450351054, + "loss": 1.5553, + "theoretical_loss": 3.9426246232301865, + "tokens_seen": 500826112 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685432296890672, + "loss": 1.4746, + "theoretical_loss": 3.9425694755838405, + "tokens_seen": 500891648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854162487462388, + "loss": 1.5228, + "theoretical_loss": 3.9425143371724913, + "tokens_seen": 500957184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854002006018054, + "loss": 1.5312, + "theoretical_loss": 3.942459207993384, + "tokens_seen": 501022720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853841524573722, + "loss": 1.4847, + "theoretical_loss": 3.9424040880437654, + "tokens_seen": 501088256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853681043129388, + "loss": 1.5158, + "theoretical_loss": 3.9423489773208837, + "tokens_seen": 501153792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853520561685056, + "loss": 1.5185, + "theoretical_loss": 3.9422938758219885, + "tokens_seen": 501219328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853360080240722, + "loss": 1.5378, + "theoretical_loss": 3.9422387835443296, + "tokens_seen": 501284864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 173143, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4856337308883667, + "objective/train/theoretical_loss": 3.9421837004851588, + "objective/train/tokens_used": 521810400, + "theoretical_loss": 3.9421837004851588, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685319959879639, + "loss": 1.4838, + "theoretical_loss": 3.9421837004851588, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853039117352056, + "loss": 1.4919, + "theoretical_loss": 3.942128626641729, + "tokens_seen": 501415936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852878635907724, + "loss": 1.5944, + "theoretical_loss": 3.942073562011294, + "tokens_seen": 501481472 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685271815446339, + "loss": 1.4147, + "theoretical_loss": 3.9420185065911086, + "tokens_seen": 501547008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852557673019058, + "loss": 1.5705, + "theoretical_loss": 3.94196346037843, + "tokens_seen": 501612544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852397191574724, + "loss": 1.4792, + "theoretical_loss": 3.941908423370515, + "tokens_seen": 501678080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852236710130392, + "loss": 1.4602, + "theoretical_loss": 3.941853395564622, + "tokens_seen": 501743616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852076228686058, + "loss": 1.4439, + "theoretical_loss": 3.9417983769580114, + "tokens_seen": 501809152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006851915747241726, + "loss": 1.5512, + "theoretical_loss": 3.941743367547944, + "tokens_seen": 501874688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006851755265797392, + "loss": 1.416, + "theoretical_loss": 3.9416883673316816, + "tokens_seen": 501940224 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685159478435306, + "loss": 1.5061, + "theoretical_loss": 3.9416333763064886, + "tokens_seen": 502005760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006851434302908726, + "loss": 1.4738, + "theoretical_loss": 3.941578394469628, + "tokens_seen": 502071296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006851273821464394, + "loss": 1.4258, + "theoretical_loss": 3.941523421818366, + "tokens_seen": 502136832 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685111334002006, + "loss": 1.5415, + "theoretical_loss": 3.94146845834997, + "tokens_seen": 502202368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850952858575728, + "loss": 1.4216, + "theoretical_loss": 3.9414135040617078, + "tokens_seen": 502267904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850792377131394, + "loss": 1.3867, + "theoretical_loss": 3.9413585589508484, + "tokens_seen": 502333440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850631895687062, + "loss": 1.4369, + "theoretical_loss": 3.9413036230146625, + "tokens_seen": 502398976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850471414242728, + "loss": 1.5165, + "theoretical_loss": 3.941248696250421, + "tokens_seen": 502464512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850310932798396, + "loss": 1.5472, + "theoretical_loss": 3.941193778655397, + "tokens_seen": 502530048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850150451354062, + "loss": 1.5286, + "theoretical_loss": 3.9411388702268644, + "tokens_seen": 502595584 + }, + { + "epoch": 0.15, + "learning_rate": 0.000684998996990973, + "loss": 1.6273, + "theoretical_loss": 3.941083970962098, + "tokens_seen": 502661120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849829488465396, + "loss": 1.4459, + "theoretical_loss": 3.9410290808583746, + "tokens_seen": 502726656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849669007021064, + "loss": 1.4898, + "theoretical_loss": 3.9409741999129704, + "tokens_seen": 502792192 + }, + { + "epoch": 0.15, + "learning_rate": 0.000684950852557673, + "loss": 1.4672, + "theoretical_loss": 3.9409193281231647, + "tokens_seen": 502857728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849348044132398, + "loss": 1.4665, + "theoretical_loss": 3.9408644654862375, + "tokens_seen": 502923264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849187562688064, + "loss": 1.6371, + "theoretical_loss": 3.9408096119994687, + "tokens_seen": 502988800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849027081243732, + "loss": 1.5104, + "theoretical_loss": 3.940754767660141, + "tokens_seen": 503054336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848866599799398, + "loss": 1.4865, + "theoretical_loss": 3.940699932465537, + "tokens_seen": 503119872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848706118355066, + "loss": 1.4007, + "theoretical_loss": 3.9406451064129415, + "tokens_seen": 503185408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848545636910732, + "loss": 1.5524, + "theoretical_loss": 3.94059028949964, + "tokens_seen": 503250944 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068483851554664, + "loss": 1.541, + "theoretical_loss": 3.940535481722919, + "tokens_seen": 503316480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848224674022066, + "loss": 1.4049, + "theoretical_loss": 3.9404806830800667, + "tokens_seen": 503382016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848064192577734, + "loss": 1.4502, + "theoretical_loss": 3.940425893568371, + "tokens_seen": 503447552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068479037111334, + "loss": 1.4578, + "theoretical_loss": 3.9403711131851233, + "tokens_seen": 503513088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847743229689068, + "loss": 1.4604, + "theoretical_loss": 3.940316341927614, + "tokens_seen": 503578624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847582748244734, + "loss": 1.507, + "theoretical_loss": 3.9402615797931357, + "tokens_seen": 503644160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847422266800402, + "loss": 1.5525, + "theoretical_loss": 3.940206826778982, + "tokens_seen": 503709696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847261785356068, + "loss": 1.4689, + "theoretical_loss": 3.9401520828824483, + "tokens_seen": 503775232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847101303911736, + "loss": 1.6258, + "theoretical_loss": 3.9400973481008297, + "tokens_seen": 503840768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006846940822467402, + "loss": 1.5935, + "theoretical_loss": 3.9400426224314233, + "tokens_seen": 503906304 + }, + { + "epoch": 0.15, + "learning_rate": 0.000684678034102307, + "loss": 1.4517, + "theoretical_loss": 3.9399879058715275, + "tokens_seen": 503971840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006846619859578737, + "loss": 1.4911, + "theoretical_loss": 3.9399331984184416, + "tokens_seen": 504037376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006846459378134404, + "loss": 1.5333, + "theoretical_loss": 3.939878500069466, + "tokens_seen": 504102912 + }, + { + "epoch": 0.15, + "learning_rate": 0.000684629889669007, + "loss": 1.5188, + "theoretical_loss": 3.939823810821903, + "tokens_seen": 504168448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006846138415245737, + "loss": 1.5142, + "theoretical_loss": 3.9397691306730547, + "tokens_seen": 504233984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845977933801405, + "loss": 1.5587, + "theoretical_loss": 3.9397144596202254, + "tokens_seen": 504299520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845817452357071, + "loss": 1.4182, + "theoretical_loss": 3.93965979766072, + "tokens_seen": 504365056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845656970912739, + "loss": 1.393, + "theoretical_loss": 3.9396051447918445, + "tokens_seen": 504430592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845496489468405, + "loss": 1.4089, + "theoretical_loss": 3.939550501010907, + "tokens_seen": 504496128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845336008024073, + "loss": 1.4498, + "theoretical_loss": 3.939495866315216, + "tokens_seen": 504561664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 174029, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3834893703460693, + "objective/train/theoretical_loss": 3.93944124070208, + "objective/train/tokens_used": 525087200, + "theoretical_loss": 3.93944124070208, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845175526579739, + "loss": 1.4513, + "theoretical_loss": 3.93944124070208, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845015045135407, + "loss": 1.5259, + "theoretical_loss": 3.9393866241688116, + "tokens_seen": 504692736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844854563691073, + "loss": 1.4621, + "theoretical_loss": 3.9393320167127213, + "tokens_seen": 504758272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844694082246741, + "loss": 1.4741, + "theoretical_loss": 3.939277418331123, + "tokens_seen": 504823808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844533600802407, + "loss": 1.5203, + "theoretical_loss": 3.9392228290213316, + "tokens_seen": 504889344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844373119358075, + "loss": 1.5894, + "theoretical_loss": 3.939168248780661, + "tokens_seen": 504954880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844212637913741, + "loss": 1.4675, + "theoretical_loss": 3.9391136776064286, + "tokens_seen": 505020416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844052156469409, + "loss": 1.5257, + "theoretical_loss": 3.939059115495952, + "tokens_seen": 505085952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843891675025075, + "loss": 1.4885, + "theoretical_loss": 3.93900456244655, + "tokens_seen": 505151488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843731193580743, + "loss": 1.4047, + "theoretical_loss": 3.938950018455543, + "tokens_seen": 505217024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843570712136409, + "loss": 1.4484, + "theoretical_loss": 3.938895483520252, + "tokens_seen": 505282560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843410230692077, + "loss": 1.4234, + "theoretical_loss": 3.938840957637998, + "tokens_seen": 505348096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843249749247743, + "loss": 1.5042, + "theoretical_loss": 3.9387864408061066, + "tokens_seen": 505413632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843089267803411, + "loss": 1.3835, + "theoretical_loss": 3.9387319330219004, + "tokens_seen": 505479168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842928786359077, + "loss": 1.4732, + "theoretical_loss": 3.938677434282706, + "tokens_seen": 505544704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842768304914745, + "loss": 1.5227, + "theoretical_loss": 3.93862294458585, + "tokens_seen": 505610240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842607823470411, + "loss": 1.5028, + "theoretical_loss": 3.938568463928661, + "tokens_seen": 505675776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842447342026079, + "loss": 1.5603, + "theoretical_loss": 3.9385139923084678, + "tokens_seen": 505741312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842286860581745, + "loss": 1.5817, + "theoretical_loss": 3.9384595297225995, + "tokens_seen": 505806848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842126379137413, + "loss": 1.4385, + "theoretical_loss": 3.9384050761683884, + "tokens_seen": 505872384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841965897693079, + "loss": 1.4871, + "theoretical_loss": 3.938350631643168, + "tokens_seen": 505937920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841805416248747, + "loss": 1.5354, + "theoretical_loss": 3.9382961961442695, + "tokens_seen": 506003456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841644934804413, + "loss": 1.593, + "theoretical_loss": 3.93824176966903, + "tokens_seen": 506068992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841484453360081, + "loss": 1.6305, + "theoretical_loss": 3.9381873522147837, + "tokens_seen": 506134528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841323971915747, + "loss": 1.5211, + "theoretical_loss": 3.938132943778869, + "tokens_seen": 506200064 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841163490471415, + "loss": 1.5019, + "theoretical_loss": 3.938078544358623, + "tokens_seen": 506265600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841003009027081, + "loss": 1.4484, + "theoretical_loss": 3.9380241539513854, + "tokens_seen": 506331136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840842527582749, + "loss": 1.5122, + "theoretical_loss": 3.9379697725544967, + "tokens_seen": 506396672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840682046138415, + "loss": 1.5404, + "theoretical_loss": 3.9379154001652985, + "tokens_seen": 506462208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840521564694083, + "loss": 1.4067, + "theoretical_loss": 3.9378610367811326, + "tokens_seen": 506527744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840361083249749, + "loss": 1.5415, + "theoretical_loss": 3.9378066823993443, + "tokens_seen": 506593280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840200601805417, + "loss": 1.5637, + "theoretical_loss": 3.937752337017278, + "tokens_seen": 506658816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840040120361083, + "loss": 1.4408, + "theoretical_loss": 3.9376980006322784, + "tokens_seen": 506724352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839879638916751, + "loss": 1.4735, + "theoretical_loss": 3.937643673241695, + "tokens_seen": 506789888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839719157472417, + "loss": 1.3714, + "theoretical_loss": 3.9375893548428738, + "tokens_seen": 506855424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839558676028085, + "loss": 1.4486, + "theoretical_loss": 3.9375350454331657, + "tokens_seen": 506920960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839398194583751, + "loss": 1.4675, + "theoretical_loss": 3.937480745009921, + "tokens_seen": 506986496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839237713139419, + "loss": 1.5516, + "theoretical_loss": 3.937426453570491, + "tokens_seen": 507052032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839077231695085, + "loss": 1.3864, + "theoretical_loss": 3.9373721711122287, + "tokens_seen": 507117568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838916750250753, + "loss": 1.4253, + "theoretical_loss": 3.937317897632488, + "tokens_seen": 507183104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838756268806419, + "loss": 1.4409, + "theoretical_loss": 3.9372636331286244, + "tokens_seen": 507248640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838595787362087, + "loss": 1.5317, + "theoretical_loss": 3.937209377597993, + "tokens_seen": 507314176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838435305917753, + "loss": 1.4448, + "theoretical_loss": 3.9371551310379527, + "tokens_seen": 507379712 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838274824473421, + "loss": 1.5911, + "theoretical_loss": 3.93710089344586, + "tokens_seen": 507445248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838114343029087, + "loss": 1.4736, + "theoretical_loss": 3.937046664819076, + "tokens_seen": 507510784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837953861584755, + "loss": 1.4326, + "theoretical_loss": 3.93699244515496, + "tokens_seen": 507576320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837793380140421, + "loss": 1.3809, + "theoretical_loss": 3.936938234450875, + "tokens_seen": 507641856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837632898696089, + "loss": 1.4961, + "theoretical_loss": 3.936884032704183, + "tokens_seen": 507707392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837472417251755, + "loss": 1.4297, + "theoretical_loss": 3.9368298399122486, + "tokens_seen": 507772928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837311935807423, + "loss": 1.5158, + "theoretical_loss": 3.9367756560724363, + "tokens_seen": 507838464 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 175381, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2951419353485107, + "objective/train/theoretical_loss": 3.936721481182113, + "objective/train/tokens_used": 528364000, + "theoretical_loss": 3.936721481182113, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837151454363089, + "loss": 1.4057, + "theoretical_loss": 3.936721481182113, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836990972918757, + "loss": 1.4186, + "theoretical_loss": 3.9366673152386458, + "tokens_seen": 507969536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836830491474423, + "loss": 1.4403, + "theoretical_loss": 3.9366131582394024, + "tokens_seen": 508035072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836670010030091, + "loss": 1.5401, + "theoretical_loss": 3.936559010181753, + "tokens_seen": 508100608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836509528585757, + "loss": 1.4831, + "theoretical_loss": 3.936504871063069, + "tokens_seen": 508166144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836349047141425, + "loss": 1.4929, + "theoretical_loss": 3.9364507408807206, + "tokens_seen": 508231680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836188565697091, + "loss": 1.4945, + "theoretical_loss": 3.936396619632082, + "tokens_seen": 508297216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836028084252759, + "loss": 1.4826, + "theoretical_loss": 3.936342507314527, + "tokens_seen": 508362752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835867602808425, + "loss": 1.459, + "theoretical_loss": 3.9362884039254302, + "tokens_seen": 508428288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835707121364093, + "loss": 1.3537, + "theoretical_loss": 3.9362343094621677, + "tokens_seen": 508493824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835546639919759, + "loss": 1.4776, + "theoretical_loss": 3.936180223922118, + "tokens_seen": 508559360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835386158475427, + "loss": 1.5716, + "theoretical_loss": 3.936126147302658, + "tokens_seen": 508624896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835225677031093, + "loss": 1.5438, + "theoretical_loss": 3.9360720796011686, + "tokens_seen": 508690432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835065195586761, + "loss": 1.5372, + "theoretical_loss": 3.93601802081503, + "tokens_seen": 508755968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834904714142427, + "loss": 1.4193, + "theoretical_loss": 3.9359639709416236, + "tokens_seen": 508821504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834744232698095, + "loss": 1.5186, + "theoretical_loss": 3.9359099299783327, + "tokens_seen": 508887040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834583751253763, + "loss": 1.4852, + "theoretical_loss": 3.935855897922541, + "tokens_seen": 508952576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834423269809429, + "loss": 1.4447, + "theoretical_loss": 3.935801874771634, + "tokens_seen": 509018112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834262788365097, + "loss": 1.4547, + "theoretical_loss": 3.935747860522997, + "tokens_seen": 509083648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834102306920763, + "loss": 1.4285, + "theoretical_loss": 3.9356938551740184, + "tokens_seen": 509149184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833941825476431, + "loss": 1.5165, + "theoretical_loss": 3.9356398587220864, + "tokens_seen": 509214720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833781344032097, + "loss": 1.5487, + "theoretical_loss": 3.9355858711645895, + "tokens_seen": 509280256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833620862587765, + "loss": 1.5374, + "theoretical_loss": 3.9355318924989193, + "tokens_seen": 509345792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833460381143431, + "loss": 1.3921, + "theoretical_loss": 3.9354779227224674, + "tokens_seen": 509411328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833299899699097, + "loss": 1.4561, + "theoretical_loss": 3.935423961832626, + "tokens_seen": 509476864 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833139418254765, + "loss": 1.5147, + "theoretical_loss": 3.93537000982679, + "tokens_seen": 509542400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832978936810431, + "loss": 1.4813, + "theoretical_loss": 3.9353160667023532, + "tokens_seen": 509607936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832818455366099, + "loss": 1.4487, + "theoretical_loss": 3.935262132456713, + "tokens_seen": 509673472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832657973921766, + "loss": 1.4581, + "theoretical_loss": 3.9352082070872654, + "tokens_seen": 509739008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832497492477433, + "loss": 1.5631, + "theoretical_loss": 3.9351542905914094, + "tokens_seen": 509804544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068323370110331, + "loss": 1.4613, + "theoretical_loss": 3.935100382966544, + "tokens_seen": 509870080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832176529588766, + "loss": 1.4773, + "theoretical_loss": 3.93504648421007, + "tokens_seen": 509935616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832016048144434, + "loss": 1.49, + "theoretical_loss": 3.9349925943193886, + "tokens_seen": 510001152 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068318555667001, + "loss": 1.457, + "theoretical_loss": 3.934938713291903, + "tokens_seen": 510066688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831695085255768, + "loss": 1.4519, + "theoretical_loss": 3.934884841125017, + "tokens_seen": 510132224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831534603811434, + "loss": 1.3485, + "theoretical_loss": 3.9348309778161346, + "tokens_seen": 510197760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831374122367102, + "loss": 1.4214, + "theoretical_loss": 3.9347771233626627, + "tokens_seen": 510263296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831213640922768, + "loss": 1.365, + "theoretical_loss": 3.934723277762008, + "tokens_seen": 510328832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831053159478436, + "loss": 1.4693, + "theoretical_loss": 3.934669441011579, + "tokens_seen": 510394368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006830892678034102, + "loss": 1.4993, + "theoretical_loss": 3.934615613108784, + "tokens_seen": 510459904 + }, + { + "epoch": 0.15, + "learning_rate": 0.000683073219658977, + "loss": 1.4738, + "theoretical_loss": 3.9345617940510342, + "tokens_seen": 510525440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006830571715145436, + "loss": 1.3746, + "theoretical_loss": 3.934507983835741, + "tokens_seen": 510590976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006830411233701104, + "loss": 1.4135, + "theoretical_loss": 3.9344541824603163, + "tokens_seen": 510656512 + }, + { + "epoch": 0.15, + "learning_rate": 0.000683025075225677, + "loss": 1.5538, + "theoretical_loss": 3.934400389922174, + "tokens_seen": 510722048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006830090270812438, + "loss": 1.5079, + "theoretical_loss": 3.934346606218729, + "tokens_seen": 510787584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829929789368104, + "loss": 1.5737, + "theoretical_loss": 3.9342928313473964, + "tokens_seen": 510853120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829769307923772, + "loss": 1.4594, + "theoretical_loss": 3.9342390653055945, + "tokens_seen": 510918656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829608826479438, + "loss": 1.4723, + "theoretical_loss": 3.93418530809074, + "tokens_seen": 510984192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829448345035106, + "loss": 1.4413, + "theoretical_loss": 3.9341315597002517, + "tokens_seen": 511049728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829287863590772, + "loss": 1.4883, + "theoretical_loss": 3.934077820131551, + "tokens_seen": 511115264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 176389, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4067481756210327, + "objective/train/theoretical_loss": 3.934024089382058, + "objective/train/tokens_used": 531640800, + "theoretical_loss": 3.934024089382058, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.000682912738214644, + "loss": 1.4437, + "theoretical_loss": 3.934024089382058, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006828966900702106, + "loss": 1.4399, + "theoretical_loss": 3.933970367449195, + "tokens_seen": 511246336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006828806419257774, + "loss": 1.5355, + "theoretical_loss": 3.9339166543303863, + "tokens_seen": 511311872 + }, + { + "epoch": 0.15, + "learning_rate": 0.000682864593781344, + "loss": 1.4865, + "theoretical_loss": 3.9338629500230553, + "tokens_seen": 511377408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006828485456369108, + "loss": 1.4984, + "theoretical_loss": 3.9338092545246286, + "tokens_seen": 511442944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006828324974924774, + "loss": 1.4916, + "theoretical_loss": 3.933755567832532, + "tokens_seen": 511508480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006828164493480442, + "loss": 1.5456, + "theoretical_loss": 3.933701889944194, + "tokens_seen": 511574016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006828004012036108, + "loss": 1.5706, + "theoretical_loss": 3.933648220857042, + "tokens_seen": 511639552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006827843530591776, + "loss": 1.4834, + "theoretical_loss": 3.9335945605685074, + "tokens_seen": 511705088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006827683049147442, + "loss": 1.4373, + "theoretical_loss": 3.9335409090760196, + "tokens_seen": 511770624 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682752256770311, + "loss": 1.443, + "theoretical_loss": 3.933487266377012, + "tokens_seen": 511836160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006827362086258776, + "loss": 1.4371, + "theoretical_loss": 3.933433632468917, + "tokens_seen": 511901696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006827201604814444, + "loss": 1.5022, + "theoretical_loss": 3.9333800073491694, + "tokens_seen": 511967232 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682704112337011, + "loss": 1.4562, + "theoretical_loss": 3.9333263910152034, + "tokens_seen": 512032768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826880641925778, + "loss": 1.441, + "theoretical_loss": 3.9332727834644565, + "tokens_seen": 512098304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826720160481444, + "loss": 1.4132, + "theoretical_loss": 3.933219184694365, + "tokens_seen": 512163840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826559679037112, + "loss": 1.5127, + "theoretical_loss": 3.933165594702368, + "tokens_seen": 512229376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826399197592778, + "loss": 1.4125, + "theoretical_loss": 3.9331120134859052, + "tokens_seen": 512294912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826238716148446, + "loss": 1.4094, + "theoretical_loss": 3.9330584410424168, + "tokens_seen": 512360448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826078234704112, + "loss": 1.5539, + "theoretical_loss": 3.9330048773693447, + "tokens_seen": 512425984 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682591775325978, + "loss": 1.5014, + "theoretical_loss": 3.932951322464132, + "tokens_seen": 512491520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006825757271815446, + "loss": 1.4989, + "theoretical_loss": 3.9328977763242223, + "tokens_seen": 512557056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006825596790371114, + "loss": 1.5426, + "theoretical_loss": 3.93284423894706, + "tokens_seen": 512622592 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682543630892678, + "loss": 1.4491, + "theoretical_loss": 3.932790710330092, + "tokens_seen": 512688128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006825275827482448, + "loss": 1.4457, + "theoretical_loss": 3.932737190470765, + "tokens_seen": 512753664 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006825115346038114, + "loss": 1.4772, + "theoretical_loss": 3.9326836793665265, + "tokens_seen": 512819200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824954864593782, + "loss": 1.6298, + "theoretical_loss": 3.9326301770148264, + "tokens_seen": 512884736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824794383149448, + "loss": 1.5865, + "theoretical_loss": 3.9325766834131155, + "tokens_seen": 512950272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824633901705116, + "loss": 1.4053, + "theoretical_loss": 3.932523198558844, + "tokens_seen": 513015808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824473420260782, + "loss": 1.3849, + "theoretical_loss": 3.932469722449464, + "tokens_seen": 513081344 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682431293881645, + "loss": 1.3922, + "theoretical_loss": 3.932416255082431, + "tokens_seen": 513146880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824152457372116, + "loss": 1.4653, + "theoretical_loss": 3.932362796455198, + "tokens_seen": 513212416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823991975927784, + "loss": 1.3947, + "theoretical_loss": 3.932309346565221, + "tokens_seen": 513277952 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682383149448345, + "loss": 1.4863, + "theoretical_loss": 3.9322559054099564, + "tokens_seen": 513343488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823671013039118, + "loss": 1.5967, + "theoretical_loss": 3.932202472986862, + "tokens_seen": 513409024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823510531594784, + "loss": 1.4689, + "theoretical_loss": 3.932149049293397, + "tokens_seen": 513474560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823350050150452, + "loss": 1.3834, + "theoretical_loss": 3.9320956343270206, + "tokens_seen": 513540096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823189568706118, + "loss": 1.4431, + "theoretical_loss": 3.9320422280851943, + "tokens_seen": 513605632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823029087261786, + "loss": 1.4312, + "theoretical_loss": 3.9319888305653805, + "tokens_seen": 513671168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822868605817452, + "loss": 1.4436, + "theoretical_loss": 3.9319354417650416, + "tokens_seen": 513736704 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682270812437312, + "loss": 1.4754, + "theoretical_loss": 3.931882061681641, + "tokens_seen": 513802240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822547642928787, + "loss": 1.4687, + "theoretical_loss": 3.9318286903126456, + "tokens_seen": 513867776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822387161484454, + "loss": 1.536, + "theoretical_loss": 3.93177532765552, + "tokens_seen": 513933312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822226680040121, + "loss": 1.4208, + "theoretical_loss": 3.9317219737077327, + "tokens_seen": 513998848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822066198595788, + "loss": 1.5384, + "theoretical_loss": 3.9316686284667517, + "tokens_seen": 514064384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006821905717151455, + "loss": 1.5302, + "theoretical_loss": 3.9316152919300458, + "tokens_seen": 514129920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006821745235707122, + "loss": 1.5152, + "theoretical_loss": 3.9315619640950863, + "tokens_seen": 514195456 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682158475426279, + "loss": 1.4262, + "theoretical_loss": 3.9315086449593446, + "tokens_seen": 514260992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006821424272818456, + "loss": 1.5165, + "theoretical_loss": 3.931455334520293, + "tokens_seen": 514326528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006821263791374124, + "loss": 1.5204, + "theoretical_loss": 3.931402032775405, + "tokens_seen": 514392064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 177587, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4545185565948486, + "objective/train/theoretical_loss": 3.9313487397221554, + "objective/train/tokens_used": 534917600, + "theoretical_loss": 3.9313487397221554, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682110330992979, + "loss": 1.442, + "theoretical_loss": 3.9313487397221554, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820942828485458, + "loss": 1.4798, + "theoretical_loss": 3.9312954553580206, + "tokens_seen": 514523136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820782347041124, + "loss": 1.5157, + "theoretical_loss": 3.931242179680477, + "tokens_seen": 514588672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820621865596792, + "loss": 1.4057, + "theoretical_loss": 3.931188912687002, + "tokens_seen": 514654208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820461384152458, + "loss": 1.4713, + "theoretical_loss": 3.931135654375075, + "tokens_seen": 514719744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820300902708126, + "loss": 1.4986, + "theoretical_loss": 3.9310824047421757, + "tokens_seen": 514785280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820140421263792, + "loss": 1.5335, + "theoretical_loss": 3.9310291637857855, + "tokens_seen": 514850816 + }, + { + "epoch": 0.16, + "learning_rate": 0.000681997993981946, + "loss": 1.4282, + "theoretical_loss": 3.930975931503386, + "tokens_seen": 514916352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006819819458375126, + "loss": 1.5608, + "theoretical_loss": 3.930922707892461, + "tokens_seen": 514981888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006819658976930794, + "loss": 1.3664, + "theoretical_loss": 3.930869492950494, + "tokens_seen": 515047424 + }, + { + "epoch": 0.16, + "learning_rate": 0.000681949849548646, + "loss": 1.4341, + "theoretical_loss": 3.9308162866749705, + "tokens_seen": 515112960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006819338014042126, + "loss": 1.3829, + "theoretical_loss": 3.930763089063377, + "tokens_seen": 515178496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006819177532597794, + "loss": 1.517, + "theoretical_loss": 3.9307099001132, + "tokens_seen": 515244032 + }, + { + "epoch": 0.16, + "learning_rate": 0.000681901705115346, + "loss": 1.5803, + "theoretical_loss": 3.930656719821929, + "tokens_seen": 515309568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818856569709128, + "loss": 1.4556, + "theoretical_loss": 3.9306035481870527, + "tokens_seen": 515375104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818696088264795, + "loss": 1.5231, + "theoretical_loss": 3.9305503852060615, + "tokens_seen": 515440640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818535606820462, + "loss": 1.5268, + "theoretical_loss": 3.9304972308764476, + "tokens_seen": 515506176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818375125376129, + "loss": 1.4497, + "theoretical_loss": 3.9304440851957025, + "tokens_seen": 515571712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818214643931796, + "loss": 1.5811, + "theoretical_loss": 3.930390948161321, + "tokens_seen": 515637248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818054162487463, + "loss": 1.5048, + "theoretical_loss": 3.9303378197707968, + "tokens_seen": 515702784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817893681043129, + "loss": 1.4945, + "theoretical_loss": 3.9302847000216263, + "tokens_seen": 515768320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817733199598797, + "loss": 1.5569, + "theoretical_loss": 3.9302315889113055, + "tokens_seen": 515833856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817572718154463, + "loss": 1.4553, + "theoretical_loss": 3.930178486437333, + "tokens_seen": 515899392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817412236710131, + "loss": 1.3959, + "theoretical_loss": 3.930125392597207, + "tokens_seen": 515964928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817251755265797, + "loss": 1.3709, + "theoretical_loss": 3.9300723073884276, + "tokens_seen": 516030464 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817091273821465, + "loss": 1.4881, + "theoretical_loss": 3.9300192308084956, + "tokens_seen": 516096000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816930792377131, + "loss": 1.447, + "theoretical_loss": 3.929966162854913, + "tokens_seen": 516161536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816770310932799, + "loss": 1.5391, + "theoretical_loss": 3.929913103525183, + "tokens_seen": 516227072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816609829488465, + "loss": 1.4131, + "theoretical_loss": 3.929860052816809, + "tokens_seen": 516292608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816449348044133, + "loss": 1.5676, + "theoretical_loss": 3.9298070107272967, + "tokens_seen": 516358144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816288866599799, + "loss": 1.5717, + "theoretical_loss": 3.929753977254152, + "tokens_seen": 516423680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816128385155467, + "loss": 1.5477, + "theoretical_loss": 3.929700952394882, + "tokens_seen": 516489216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815967903711133, + "loss": 1.616, + "theoretical_loss": 3.9296479361469947, + "tokens_seen": 516554752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815807422266801, + "loss": 1.5208, + "theoretical_loss": 3.929594928508, + "tokens_seen": 516620288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815646940822467, + "loss": 1.5779, + "theoretical_loss": 3.9295419294754077, + "tokens_seen": 516685824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815486459378135, + "loss": 1.5485, + "theoretical_loss": 3.9294889390467285, + "tokens_seen": 516751360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815325977933801, + "loss": 1.4398, + "theoretical_loss": 3.929435957219476, + "tokens_seen": 516816896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815165496489469, + "loss": 1.4488, + "theoretical_loss": 3.9293829839911623, + "tokens_seen": 516882432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815005015045135, + "loss": 1.4929, + "theoretical_loss": 3.9293300193593024, + "tokens_seen": 516947968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814844533600803, + "loss": 1.5023, + "theoretical_loss": 3.929277063321412, + "tokens_seen": 517013504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814684052156469, + "loss": 1.5235, + "theoretical_loss": 3.929224115875007, + "tokens_seen": 517079040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814523570712137, + "loss": 1.4841, + "theoretical_loss": 3.929171177017605, + "tokens_seen": 517144576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814363089267803, + "loss": 1.5463, + "theoretical_loss": 3.9291182467467247, + "tokens_seen": 517210112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814202607823471, + "loss": 1.4908, + "theoretical_loss": 3.9290653250598857, + "tokens_seen": 517275648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814042126379137, + "loss": 1.4972, + "theoretical_loss": 3.9290124119546084, + "tokens_seen": 517341184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813881644934805, + "loss": 1.4464, + "theoretical_loss": 3.9289595074284143, + "tokens_seen": 517406720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813721163490471, + "loss": 1.5389, + "theoretical_loss": 3.9289066114788267, + "tokens_seen": 517472256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813560682046139, + "loss": 1.4443, + "theoretical_loss": 3.9288537241033685, + "tokens_seen": 517537792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813400200601805, + "loss": 1.404, + "theoretical_loss": 3.9288008452995644, + "tokens_seen": 517603328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813239719157473, + "loss": 1.5549, + "theoretical_loss": 3.9287479750649412, + "tokens_seen": 517668864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 178536, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4981567859649658, + "objective/train/theoretical_loss": 3.9286951133970245, + "objective/train/tokens_used": 538194400, + "theoretical_loss": 3.9286951133970245, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813079237713139, + "loss": 1.4896, + "theoretical_loss": 3.9286951133970245, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812918756268807, + "loss": 1.3853, + "theoretical_loss": 3.9286422602933424, + "tokens_seen": 517799936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812758274824473, + "loss": 1.4381, + "theoretical_loss": 3.928589415751424, + "tokens_seen": 517865472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812597793380141, + "loss": 1.4726, + "theoretical_loss": 3.9285365797687986, + "tokens_seen": 517931008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812437311935807, + "loss": 1.5329, + "theoretical_loss": 3.9284837523429976, + "tokens_seen": 517996544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812276830491475, + "loss": 1.3644, + "theoretical_loss": 3.9284309334715526, + "tokens_seen": 518062080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812116349047141, + "loss": 1.516, + "theoretical_loss": 3.9283781231519965, + "tokens_seen": 518127616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811955867602809, + "loss": 1.5407, + "theoretical_loss": 3.9283253213818634, + "tokens_seen": 518193152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811795386158475, + "loss": 1.5017, + "theoretical_loss": 3.9282725281586885, + "tokens_seen": 518258688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811634904714143, + "loss": 1.5143, + "theoretical_loss": 3.928219743480007, + "tokens_seen": 518324224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811474423269809, + "loss": 1.5884, + "theoretical_loss": 3.9281669673433566, + "tokens_seen": 518389760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811313941825477, + "loss": 1.552, + "theoretical_loss": 3.9281141997462754, + "tokens_seen": 518455296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811153460381143, + "loss": 1.5766, + "theoretical_loss": 3.9280614406863013, + "tokens_seen": 518520832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810992978936811, + "loss": 1.474, + "theoretical_loss": 3.9280086901609756, + "tokens_seen": 518586368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810832497492477, + "loss": 1.4442, + "theoretical_loss": 3.927955948167839, + "tokens_seen": 518651904 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810672016048145, + "loss": 1.4731, + "theoretical_loss": 3.9279032147044335, + "tokens_seen": 518717440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810511534603812, + "loss": 1.4481, + "theoretical_loss": 3.9278504897683018, + "tokens_seen": 518782976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810351053159479, + "loss": 1.5885, + "theoretical_loss": 3.927797773356989, + "tokens_seen": 518848512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810190571715146, + "loss": 1.3918, + "theoretical_loss": 3.9277450654680397, + "tokens_seen": 518914048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810030090270813, + "loss": 1.4471, + "theoretical_loss": 3.9276923660990004, + "tokens_seen": 518979584 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680986960882648, + "loss": 1.5669, + "theoretical_loss": 3.9276396752474176, + "tokens_seen": 519045120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809709127382147, + "loss": 1.5003, + "theoretical_loss": 3.9275869929108405, + "tokens_seen": 519110656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809548645937814, + "loss": 1.4697, + "theoretical_loss": 3.927534319086817, + "tokens_seen": 519176192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809388164493481, + "loss": 1.438, + "theoretical_loss": 3.9274816537728983, + "tokens_seen": 519241728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809227683049148, + "loss": 1.428, + "theoretical_loss": 3.9274289969666354, + "tokens_seen": 519307264 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809067201604815, + "loss": 1.5598, + "theoretical_loss": 3.9273763486655806, + "tokens_seen": 519372800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808906720160482, + "loss": 1.4526, + "theoretical_loss": 3.9273237088672874, + "tokens_seen": 519438336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808746238716149, + "loss": 1.4842, + "theoretical_loss": 3.92727107756931, + "tokens_seen": 519503872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808585757271816, + "loss": 1.4134, + "theoretical_loss": 3.927218454769203, + "tokens_seen": 519569408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808425275827483, + "loss": 1.4679, + "theoretical_loss": 3.9271658404645233, + "tokens_seen": 519634944 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680826479438315, + "loss": 1.5984, + "theoretical_loss": 3.9271132346528286, + "tokens_seen": 519700480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808104312938817, + "loss": 1.5113, + "theoretical_loss": 3.9270606373316763, + "tokens_seen": 519766016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807943831494484, + "loss": 1.5097, + "theoretical_loss": 3.9270080484986267, + "tokens_seen": 519831552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807783350050151, + "loss": 1.5268, + "theoretical_loss": 3.9269554681512395, + "tokens_seen": 519897088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807622868605819, + "loss": 1.4653, + "theoretical_loss": 3.926902896287076, + "tokens_seen": 519962624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807462387161485, + "loss": 1.4649, + "theoretical_loss": 3.9268503329036992, + "tokens_seen": 520028160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807301905717153, + "loss": 1.5718, + "theoretical_loss": 3.926797777998672, + "tokens_seen": 520093696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807141424272819, + "loss": 1.357, + "theoretical_loss": 3.9267452315695586, + "tokens_seen": 520159232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806980942828487, + "loss": 1.468, + "theoretical_loss": 3.9266926936139246, + "tokens_seen": 520224768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806820461384153, + "loss": 1.4507, + "theoretical_loss": 3.9266401641293367, + "tokens_seen": 520290304 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680665997993982, + "loss": 1.3782, + "theoretical_loss": 3.926587643113362, + "tokens_seen": 520355840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806499498495487, + "loss": 1.4731, + "theoretical_loss": 3.9265351305635687, + "tokens_seen": 520421376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806339017051155, + "loss": 1.4242, + "theoretical_loss": 3.9264826264775268, + "tokens_seen": 520486912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806178535606821, + "loss": 1.5955, + "theoretical_loss": 3.9264301308528067, + "tokens_seen": 520552448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806018054162489, + "loss": 1.4677, + "theoretical_loss": 3.926377643686979, + "tokens_seen": 520617984 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805857572718155, + "loss": 1.539, + "theoretical_loss": 3.926325164977616, + "tokens_seen": 520683520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805697091273823, + "loss": 1.4778, + "theoretical_loss": 3.9262726947222926, + "tokens_seen": 520749056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805536609829489, + "loss": 1.4355, + "theoretical_loss": 3.926220232918582, + "tokens_seen": 520814592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805376128385156, + "loss": 1.5164, + "theoretical_loss": 3.92616777956406, + "tokens_seen": 520880128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805215646940823, + "loss": 1.3718, + "theoretical_loss": 3.926115334656303, + "tokens_seen": 520945664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 179706, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5349757671356201, + "objective/train/theoretical_loss": 3.926062898192888, + "objective/train/tokens_used": 541471200, + "theoretical_loss": 3.926062898192888, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680505516549649, + "loss": 1.5216, + "theoretical_loss": 3.926062898192888, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804894684052157, + "loss": 1.4301, + "theoretical_loss": 3.926010470171394, + "tokens_seen": 521076736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804734202607824, + "loss": 1.5091, + "theoretical_loss": 3.9259580505894007, + "tokens_seen": 521142272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804573721163491, + "loss": 1.4445, + "theoretical_loss": 3.925905639444488, + "tokens_seen": 521207808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804413239719158, + "loss": 1.5386, + "theoretical_loss": 3.9258532367342367, + "tokens_seen": 521273344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804252758274825, + "loss": 1.4605, + "theoretical_loss": 3.9258008424562307, + "tokens_seen": 521338880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804092276830492, + "loss": 1.5114, + "theoretical_loss": 3.9257484566080523, + "tokens_seen": 521404416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803931795386158, + "loss": 1.5215, + "theoretical_loss": 3.9256960791872855, + "tokens_seen": 521469952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803771313941826, + "loss": 1.4821, + "theoretical_loss": 3.925643710191517, + "tokens_seen": 521535488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803610832497492, + "loss": 1.4289, + "theoretical_loss": 3.9255913496183332, + "tokens_seen": 521601024 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680345035105316, + "loss": 1.4774, + "theoretical_loss": 3.9255389974653196, + "tokens_seen": 521666560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803289869608826, + "loss": 1.496, + "theoretical_loss": 3.9254866537300668, + "tokens_seen": 521732096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803129388164494, + "loss": 1.5232, + "theoretical_loss": 3.925434318410163, + "tokens_seen": 521797632 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680296890672016, + "loss": 1.5735, + "theoretical_loss": 3.9253819915031993, + "tokens_seen": 521863168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802808425275828, + "loss": 1.4249, + "theoretical_loss": 3.9253296730067664, + "tokens_seen": 521928704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802647943831494, + "loss": 1.5218, + "theoretical_loss": 3.9252773629184565, + "tokens_seen": 521994240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802487462387162, + "loss": 1.394, + "theoretical_loss": 3.925225061235863, + "tokens_seen": 522059776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802326980942828, + "loss": 1.3936, + "theoretical_loss": 3.925172767956581, + "tokens_seen": 522125312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802166499498496, + "loss": 1.6192, + "theoretical_loss": 3.9251204830782056, + "tokens_seen": 522190848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802006018054162, + "loss": 1.3342, + "theoretical_loss": 3.925068206598332, + "tokens_seen": 522256384 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680184553660983, + "loss": 1.5168, + "theoretical_loss": 3.9250159385145595, + "tokens_seen": 522321920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006801685055165496, + "loss": 1.4312, + "theoretical_loss": 3.9249636788244846, + "tokens_seen": 522387456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006801524573721164, + "loss": 1.4457, + "theoretical_loss": 3.924911427525708, + "tokens_seen": 522452992 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680136409227683, + "loss": 1.4418, + "theoretical_loss": 3.9248591846158285, + "tokens_seen": 522518528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006801203610832498, + "loss": 1.4766, + "theoretical_loss": 3.9248069500924485, + "tokens_seen": 522584064 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006801043129388164, + "loss": 1.4706, + "theoretical_loss": 3.9247547239531695, + "tokens_seen": 522649600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800882647943832, + "loss": 1.3625, + "theoretical_loss": 3.924702506195595, + "tokens_seen": 522715136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800722166499498, + "loss": 1.6163, + "theoretical_loss": 3.9246502968173296, + "tokens_seen": 522780672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800561685055166, + "loss": 1.4221, + "theoretical_loss": 3.9245980958159787, + "tokens_seen": 522846208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800401203610832, + "loss": 1.5205, + "theoretical_loss": 3.924545903189147, + "tokens_seen": 522911744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00068002407221665, + "loss": 1.5592, + "theoretical_loss": 3.9244937189344435, + "tokens_seen": 522977280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800080240722166, + "loss": 1.4894, + "theoretical_loss": 3.9244415430494755, + "tokens_seen": 523042816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799919759277834, + "loss": 1.4424, + "theoretical_loss": 3.9243893755318515, + "tokens_seen": 523108352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00067997592778335, + "loss": 1.3891, + "theoretical_loss": 3.924337216379183, + "tokens_seen": 523173888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799598796389168, + "loss": 1.4588, + "theoretical_loss": 3.92428506558908, + "tokens_seen": 523239424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799438314944834, + "loss": 1.5871, + "theoretical_loss": 3.924232923159155, + "tokens_seen": 523304960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799277833500502, + "loss": 1.4269, + "theoretical_loss": 3.9241807890870213, + "tokens_seen": 523370496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799117352056168, + "loss": 1.5134, + "theoretical_loss": 3.9241286633702925, + "tokens_seen": 523436032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798956870611836, + "loss": 1.5179, + "theoretical_loss": 3.9240765460065834, + "tokens_seen": 523501568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798796389167503, + "loss": 1.515, + "theoretical_loss": 3.924024436993511, + "tokens_seen": 523567104 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679863590772317, + "loss": 1.481, + "theoretical_loss": 3.9239723363286916, + "tokens_seen": 523632640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798475426278837, + "loss": 1.4266, + "theoretical_loss": 3.923920244009743, + "tokens_seen": 523698176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798314944834504, + "loss": 1.4077, + "theoretical_loss": 3.9238681600342846, + "tokens_seen": 523763712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798154463390171, + "loss": 1.4432, + "theoretical_loss": 3.9238160843999363, + "tokens_seen": 523829248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797993981945838, + "loss": 1.4181, + "theoretical_loss": 3.923764017104318, + "tokens_seen": 523894784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797833500501505, + "loss": 1.4425, + "theoretical_loss": 3.9237119581450535, + "tokens_seen": 523960320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797673019057172, + "loss": 1.5611, + "theoretical_loss": 3.923659907519764, + "tokens_seen": 524025856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797512537612839, + "loss": 1.3971, + "theoretical_loss": 3.9236078652260735, + "tokens_seen": 524091392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797352056168506, + "loss": 1.5388, + "theoretical_loss": 3.9235558312616075, + "tokens_seen": 524156928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797191574724173, + "loss": 1.4835, + "theoretical_loss": 3.9235038056239917, + "tokens_seen": 524222464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 180943, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5657446384429932, + "objective/train/theoretical_loss": 3.9234517883108517, + "objective/train/tokens_used": 544748000, + "theoretical_loss": 3.9234517883108517, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679703109327984, + "loss": 1.5641, + "theoretical_loss": 3.9234517883108517, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796870611835507, + "loss": 1.4348, + "theoretical_loss": 3.9233997793198165, + "tokens_seen": 524353536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796710130391174, + "loss": 1.5001, + "theoretical_loss": 3.9233477786485143, + "tokens_seen": 524419072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796549648946841, + "loss": 1.5435, + "theoretical_loss": 3.923295786294575, + "tokens_seen": 524484608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796389167502508, + "loss": 1.4204, + "theoretical_loss": 3.9232438022556293, + "tokens_seen": 524550144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796228686058175, + "loss": 1.4743, + "theoretical_loss": 3.923191826529308, + "tokens_seen": 524615680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796068204613842, + "loss": 1.471, + "theoretical_loss": 3.9231398591132445, + "tokens_seen": 524681216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795907723169509, + "loss": 1.4699, + "theoretical_loss": 3.9230879000050716, + "tokens_seen": 524746752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795747241725176, + "loss": 1.526, + "theoretical_loss": 3.9230359492024247, + "tokens_seen": 524812288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795586760280843, + "loss": 1.5139, + "theoretical_loss": 3.9229840067029382, + "tokens_seen": 524877824 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679542627883651, + "loss": 1.4791, + "theoretical_loss": 3.92293207250425, + "tokens_seen": 524943360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795265797392177, + "loss": 1.4533, + "theoretical_loss": 3.9228801466039958, + "tokens_seen": 525008896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795105315947844, + "loss": 1.4722, + "theoretical_loss": 3.9228282289998155, + "tokens_seen": 525074432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794944834503511, + "loss": 1.358, + "theoretical_loss": 3.9227763196893477, + "tokens_seen": 525139968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794784353059178, + "loss": 1.5151, + "theoretical_loss": 3.9227244186702332, + "tokens_seen": 525205504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794623871614845, + "loss": 1.4097, + "theoretical_loss": 3.9226725259401123, + "tokens_seen": 525271040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794463390170512, + "loss": 1.5695, + "theoretical_loss": 3.9226206414966285, + "tokens_seen": 525336576 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679430290872618, + "loss": 1.3796, + "theoretical_loss": 3.9225687653374237, + "tokens_seen": 525402112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794142427281846, + "loss": 1.4475, + "theoretical_loss": 3.922516897460143, + "tokens_seen": 525467648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793981945837513, + "loss": 1.4249, + "theoretical_loss": 3.9224650378624313, + "tokens_seen": 525533184 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679382146439318, + "loss": 1.5164, + "theoretical_loss": 3.9224131865419345, + "tokens_seen": 525598720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793660982948848, + "loss": 1.4282, + "theoretical_loss": 3.9223613434963003, + "tokens_seen": 525664256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793500501504514, + "loss": 1.4594, + "theoretical_loss": 3.922309508723176, + "tokens_seen": 525729792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793340020060182, + "loss": 1.4641, + "theoretical_loss": 3.9222576822202107, + "tokens_seen": 525795328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793179538615848, + "loss": 1.5236, + "theoretical_loss": 3.922205863985055, + "tokens_seen": 525860864 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793019057171516, + "loss": 1.4442, + "theoretical_loss": 3.9221540540153588, + "tokens_seen": 525926400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006792858575727182, + "loss": 1.3884, + "theoretical_loss": 3.922102252308775, + "tokens_seen": 525991936 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679269809428285, + "loss": 1.5972, + "theoretical_loss": 3.9220504588629552, + "tokens_seen": 526057472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006792537612838516, + "loss": 1.4631, + "theoretical_loss": 3.9219986736755548, + "tokens_seen": 526123008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006792377131394184, + "loss": 1.5174, + "theoretical_loss": 3.921946896744227, + "tokens_seen": 526188544 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679221664994985, + "loss": 1.3494, + "theoretical_loss": 3.9218951280666285, + "tokens_seen": 526254080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006792056168505518, + "loss": 1.3861, + "theoretical_loss": 3.9218433676404154, + "tokens_seen": 526319616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791895687061184, + "loss": 1.3822, + "theoretical_loss": 3.9217916154632455, + "tokens_seen": 526385152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791735205616852, + "loss": 1.5359, + "theoretical_loss": 3.9217398715327776, + "tokens_seen": 526450688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791574724172518, + "loss": 1.5608, + "theoretical_loss": 3.9216881358466713, + "tokens_seen": 526516224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791414242728185, + "loss": 1.3272, + "theoretical_loss": 3.9216364084025868, + "tokens_seen": 526581760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791253761283852, + "loss": 1.5005, + "theoretical_loss": 3.921584689198185, + "tokens_seen": 526647296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791093279839519, + "loss": 1.3935, + "theoretical_loss": 3.921532978231129, + "tokens_seen": 526712832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790932798395186, + "loss": 1.4883, + "theoretical_loss": 3.921481275499082, + "tokens_seen": 526778368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790772316950853, + "loss": 1.5002, + "theoretical_loss": 3.921429580999709, + "tokens_seen": 526843904 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679061183550652, + "loss": 1.538, + "theoretical_loss": 3.921377894730674, + "tokens_seen": 526909440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790451354062187, + "loss": 1.5555, + "theoretical_loss": 3.9213262166896445, + "tokens_seen": 526974976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790290872617854, + "loss": 1.4427, + "theoretical_loss": 3.9212745468742867, + "tokens_seen": 527040512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790130391173521, + "loss": 1.3753, + "theoretical_loss": 3.921222885282268, + "tokens_seen": 527106048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789969909729187, + "loss": 1.3694, + "theoretical_loss": 3.9211712319112597, + "tokens_seen": 527171584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789809428284855, + "loss": 1.4033, + "theoretical_loss": 3.92111958675893, + "tokens_seen": 527237120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789648946840521, + "loss": 1.5529, + "theoretical_loss": 3.9210679498229504, + "tokens_seen": 527302656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789488465396189, + "loss": 1.5309, + "theoretical_loss": 3.9210163211009927, + "tokens_seen": 527368192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789327983951855, + "loss": 1.4576, + "theoretical_loss": 3.92096470059073, + "tokens_seen": 527433728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789167502507523, + "loss": 1.4784, + "theoretical_loss": 3.9209130882898364, + "tokens_seen": 527499264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 181844, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4468753337860107, + "objective/train/theoretical_loss": 3.920861484195986, + "objective/train/tokens_used": 548024800, + "theoretical_loss": 3.920861484195986, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789007021063189, + "loss": 1.4834, + "theoretical_loss": 3.920861484195986, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788846539618857, + "loss": 1.3464, + "theoretical_loss": 3.920809888306854, + "tokens_seen": 527630336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788686058174523, + "loss": 1.4429, + "theoretical_loss": 3.9207583006201183, + "tokens_seen": 527695872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788525576730191, + "loss": 1.5239, + "theoretical_loss": 3.920706721133456, + "tokens_seen": 527761408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788365095285857, + "loss": 1.4154, + "theoretical_loss": 3.9206551498445465, + "tokens_seen": 527826944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788204613841525, + "loss": 1.3948, + "theoretical_loss": 3.920603586751068, + "tokens_seen": 527892480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788044132397191, + "loss": 1.5827, + "theoretical_loss": 3.9205520318507006, + "tokens_seen": 527958016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787883650952859, + "loss": 1.4211, + "theoretical_loss": 3.9205004851411274, + "tokens_seen": 528023552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787723169508525, + "loss": 1.378, + "theoretical_loss": 3.920448946620029, + "tokens_seen": 528089088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787562688064193, + "loss": 1.4033, + "theoretical_loss": 3.9203974162850903, + "tokens_seen": 528154624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787402206619859, + "loss": 1.5083, + "theoretical_loss": 3.920345894133994, + "tokens_seen": 528220160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787241725175527, + "loss": 1.4078, + "theoretical_loss": 3.9202943801644263, + "tokens_seen": 528285696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787081243731193, + "loss": 1.5383, + "theoretical_loss": 3.9202428743740727, + "tokens_seen": 528351232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786920762286861, + "loss": 1.3915, + "theoretical_loss": 3.92019137676062, + "tokens_seen": 528416768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786760280842528, + "loss": 1.4579, + "theoretical_loss": 3.920139887321757, + "tokens_seen": 528482304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786599799398195, + "loss": 1.4671, + "theoretical_loss": 3.9200884060551724, + "tokens_seen": 528547840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786439317953862, + "loss": 1.3536, + "theoretical_loss": 3.9200369329585563, + "tokens_seen": 528613376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786278836509529, + "loss": 1.4441, + "theoretical_loss": 3.9199854680295982, + "tokens_seen": 528678912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786118355065196, + "loss": 1.3964, + "theoretical_loss": 3.9199340112659913, + "tokens_seen": 528744448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785957873620863, + "loss": 1.4516, + "theoretical_loss": 3.9198825626654275, + "tokens_seen": 528809984 + }, + { + "epoch": 0.16, + "learning_rate": 0.000678579739217653, + "loss": 1.4887, + "theoretical_loss": 3.9198311222256, + "tokens_seen": 528875520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785636910732197, + "loss": 1.5368, + "theoretical_loss": 3.919779689944205, + "tokens_seen": 528941056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785476429287864, + "loss": 1.4402, + "theoretical_loss": 3.9197282658189363, + "tokens_seen": 529006592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785315947843531, + "loss": 1.5339, + "theoretical_loss": 3.919676849847491, + "tokens_seen": 529072128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785155466399198, + "loss": 1.4247, + "theoretical_loss": 3.9196254420275665, + "tokens_seen": 529137664 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784994984954865, + "loss": 1.4668, + "theoretical_loss": 3.919574042356861, + "tokens_seen": 529203200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784834503510532, + "loss": 1.4803, + "theoretical_loss": 3.9195226508330734, + "tokens_seen": 529268736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784674022066199, + "loss": 1.3684, + "theoretical_loss": 3.919471267453905, + "tokens_seen": 529334272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784513540621866, + "loss": 1.524, + "theoretical_loss": 3.919419892217055, + "tokens_seen": 529399808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784353059177533, + "loss": 1.4664, + "theoretical_loss": 3.919368525120227, + "tokens_seen": 529465344 + }, + { + "epoch": 0.16, + "learning_rate": 0.00067841925777332, + "loss": 1.4618, + "theoretical_loss": 3.9193171661611235, + "tokens_seen": 529530880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784032096288867, + "loss": 1.4584, + "theoretical_loss": 3.9192658153374484, + "tokens_seen": 529596416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783871614844534, + "loss": 1.4449, + "theoretical_loss": 3.9192144726469063, + "tokens_seen": 529661952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783711133400201, + "loss": 1.5014, + "theoretical_loss": 3.919163138087203, + "tokens_seen": 529727488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783550651955868, + "loss": 1.5635, + "theoretical_loss": 3.919111811656046, + "tokens_seen": 529793024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783390170511535, + "loss": 1.4419, + "theoretical_loss": 3.9190604933511417, + "tokens_seen": 529858560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783229689067202, + "loss": 1.448, + "theoretical_loss": 3.9190091831701994, + "tokens_seen": 529924096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783069207622869, + "loss": 1.3953, + "theoretical_loss": 3.918957881110928, + "tokens_seen": 529989632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782908726178536, + "loss": 1.5078, + "theoretical_loss": 3.9189065871710387, + "tokens_seen": 530055168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782748244734203, + "loss": 1.3386, + "theoretical_loss": 3.918855301348242, + "tokens_seen": 530120704 + }, + { + "epoch": 0.16, + "learning_rate": 0.000678258776328987, + "loss": 1.5389, + "theoretical_loss": 3.9188040236402513, + "tokens_seen": 530186240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782427281845537, + "loss": 1.4875, + "theoretical_loss": 3.918752754044778, + "tokens_seen": 530251776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782266800401204, + "loss": 1.3773, + "theoretical_loss": 3.9187014925595385, + "tokens_seen": 530317312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782106318956871, + "loss": 1.564, + "theoretical_loss": 3.918650239182246, + "tokens_seen": 530382848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781945837512538, + "loss": 1.5289, + "theoretical_loss": 3.918598993910617, + "tokens_seen": 530448384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781785356068205, + "loss": 1.4389, + "theoretical_loss": 3.9185477567423685, + "tokens_seen": 530513920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781624874623872, + "loss": 1.5537, + "theoretical_loss": 3.9184965276752184, + "tokens_seen": 530579456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781464393179539, + "loss": 1.385, + "theoretical_loss": 3.9184453067068854, + "tokens_seen": 530644992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781303911735206, + "loss": 1.3913, + "theoretical_loss": 3.918394093835089, + "tokens_seen": 530710528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781143430290873, + "loss": 1.4841, + "theoretical_loss": 3.91834288905755, + "tokens_seen": 530776064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 183006, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5242764949798584, + "objective/train/theoretical_loss": 3.91829169237199, + "objective/train/tokens_used": 551301600, + "theoretical_loss": 3.91829169237199, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.000678098294884654, + "loss": 1.5453, + "theoretical_loss": 3.91829169237199, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780822467402207, + "loss": 1.5267, + "theoretical_loss": 3.918240503776131, + "tokens_seen": 530907136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780661985957874, + "loss": 1.5107, + "theoretical_loss": 3.9181893232676965, + "tokens_seen": 530972672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780501504513541, + "loss": 1.4461, + "theoretical_loss": 3.9181381508444106, + "tokens_seen": 531038208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780341023069208, + "loss": 1.4501, + "theoretical_loss": 3.918086986504, + "tokens_seen": 531103744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780180541624875, + "loss": 1.46, + "theoretical_loss": 3.9180358302441887, + "tokens_seen": 531169280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780020060180543, + "loss": 1.388, + "theoretical_loss": 3.9179846820627047, + "tokens_seen": 531234816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779859578736209, + "loss": 1.3978, + "theoretical_loss": 3.917933541957276, + "tokens_seen": 531300352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779699097291877, + "loss": 1.5003, + "theoretical_loss": 3.9178824099256317, + "tokens_seen": 531365888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779538615847543, + "loss": 1.5086, + "theoretical_loss": 3.9178312859655016, + "tokens_seen": 531431424 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677937813440321, + "loss": 1.5874, + "theoretical_loss": 3.9177801700746153, + "tokens_seen": 531496960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779217652958877, + "loss": 1.4059, + "theoretical_loss": 3.9177290622507055, + "tokens_seen": 531562496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779057171514545, + "loss": 1.5381, + "theoretical_loss": 3.9176779624915055, + "tokens_seen": 531628032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778896690070211, + "loss": 1.4416, + "theoretical_loss": 3.917626870794747, + "tokens_seen": 531693568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778736208625879, + "loss": 1.4973, + "theoretical_loss": 3.9175757871581647, + "tokens_seen": 531759104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778575727181545, + "loss": 1.4864, + "theoretical_loss": 3.917524711579495, + "tokens_seen": 531824640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778415245737213, + "loss": 1.5042, + "theoretical_loss": 3.9174736440564732, + "tokens_seen": 531890176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778254764292879, + "loss": 1.6653, + "theoretical_loss": 3.917422584586837, + "tokens_seen": 531955712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778094282848547, + "loss": 1.408, + "theoretical_loss": 3.9173715331683243, + "tokens_seen": 532021248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777933801404213, + "loss": 1.3939, + "theoretical_loss": 3.9173204897986738, + "tokens_seen": 532086784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777773319959881, + "loss": 1.4191, + "theoretical_loss": 3.9172694544756252, + "tokens_seen": 532152320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777612838515547, + "loss": 1.4525, + "theoretical_loss": 3.91721842719692, + "tokens_seen": 532217856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777452357071214, + "loss": 1.4859, + "theoretical_loss": 3.9171674079603, + "tokens_seen": 532283392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777291875626881, + "loss": 1.4312, + "theoretical_loss": 3.9171163967635065, + "tokens_seen": 532348928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777131394182548, + "loss": 1.5149, + "theoretical_loss": 3.9170653936042843, + "tokens_seen": 532414464 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776970912738215, + "loss": 1.4395, + "theoretical_loss": 3.917014398480377, + "tokens_seen": 532480000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776810431293882, + "loss": 1.5004, + "theoretical_loss": 3.9169634113895304, + "tokens_seen": 532545536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776649949849549, + "loss": 1.4336, + "theoretical_loss": 3.9169124323294913, + "tokens_seen": 532611072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776489468405216, + "loss": 1.5267, + "theoretical_loss": 3.916861461298005, + "tokens_seen": 532676608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776328986960883, + "loss": 1.5136, + "theoretical_loss": 3.916810498292822, + "tokens_seen": 532742144 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677616850551655, + "loss": 1.4758, + "theoretical_loss": 3.9167595433116897, + "tokens_seen": 532807680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776008024072216, + "loss": 1.4975, + "theoretical_loss": 3.9167085963523585, + "tokens_seen": 532873216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775847542627884, + "loss": 1.6303, + "theoretical_loss": 3.916657657412579, + "tokens_seen": 532938752 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677568706118355, + "loss": 1.5686, + "theoretical_loss": 3.9166067264901026, + "tokens_seen": 533004288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775526579739218, + "loss": 1.5364, + "theoretical_loss": 3.916555803582683, + "tokens_seen": 533069824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775366098294884, + "loss": 1.4351, + "theoretical_loss": 3.9165048886880722, + "tokens_seen": 533135360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775205616850552, + "loss": 1.5138, + "theoretical_loss": 3.916453981804026, + "tokens_seen": 533200896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775045135406218, + "loss": 1.3943, + "theoretical_loss": 3.916403082928299, + "tokens_seen": 533266432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774884653961886, + "loss": 1.4264, + "theoretical_loss": 3.916352192058648, + "tokens_seen": 533331968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774724172517553, + "loss": 1.4411, + "theoretical_loss": 3.9163013091928294, + "tokens_seen": 533397504 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677456369107322, + "loss": 1.4073, + "theoretical_loss": 3.9162504343286013, + "tokens_seen": 533463040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774403209628887, + "loss": 1.4156, + "theoretical_loss": 3.9161995674637233, + "tokens_seen": 533528576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774242728184554, + "loss": 1.4649, + "theoretical_loss": 3.916148708595955, + "tokens_seen": 533594112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774082246740221, + "loss": 1.4853, + "theoretical_loss": 3.9160978577230567, + "tokens_seen": 533659648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773921765295888, + "loss": 1.5875, + "theoretical_loss": 3.91604701484279, + "tokens_seen": 533725184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773761283851555, + "loss": 1.4837, + "theoretical_loss": 3.915996179952918, + "tokens_seen": 533790720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773600802407222, + "loss": 1.44, + "theoretical_loss": 3.915945353051205, + "tokens_seen": 533856256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773440320962889, + "loss": 1.5419, + "theoretical_loss": 3.9158945341354134, + "tokens_seen": 533921792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773279839518556, + "loss": 1.5078, + "theoretical_loss": 3.9158437232033094, + "tokens_seen": 533987328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773119358074223, + "loss": 1.4417, + "theoretical_loss": 3.9157929202526596, + "tokens_seen": 534052864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 184018, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4757671356201172, + "objective/train/theoretical_loss": 3.9157421252812306, + "objective/train/tokens_used": 554578400, + "theoretical_loss": 3.9157421252812306, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677295887662989, + "loss": 1.4674, + "theoretical_loss": 3.9157421252812306, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772798395185557, + "loss": 1.5966, + "theoretical_loss": 3.9156913382867895, + "tokens_seen": 534183936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772637913741224, + "loss": 1.4379, + "theoretical_loss": 3.915640559267107, + "tokens_seen": 534249472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772477432296891, + "loss": 1.5897, + "theoretical_loss": 3.9155897882199513, + "tokens_seen": 534315008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772316950852558, + "loss": 1.393, + "theoretical_loss": 3.9155390251430937, + "tokens_seen": 534380544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772156469408225, + "loss": 1.5403, + "theoretical_loss": 3.915488270034306, + "tokens_seen": 534446080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771995987963892, + "loss": 1.4583, + "theoretical_loss": 3.9154375228913594, + "tokens_seen": 534511616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771835506519559, + "loss": 1.3791, + "theoretical_loss": 3.9153867837120284, + "tokens_seen": 534577152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771675025075226, + "loss": 1.4518, + "theoretical_loss": 3.9153360524940872, + "tokens_seen": 534642688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771514543630893, + "loss": 1.4759, + "theoretical_loss": 3.915285329235311, + "tokens_seen": 534708224 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677135406218656, + "loss": 1.5145, + "theoretical_loss": 3.915234613933475, + "tokens_seen": 534773760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771193580742227, + "loss": 1.5178, + "theoretical_loss": 3.915183906586357, + "tokens_seen": 534839296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771033099297894, + "loss": 1.3855, + "theoretical_loss": 3.915133207191734, + "tokens_seen": 534904832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770872617853561, + "loss": 1.4519, + "theoretical_loss": 3.915082515747385, + "tokens_seen": 534970368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770712136409228, + "loss": 1.5169, + "theoretical_loss": 3.9150318322510893, + "tokens_seen": 535035904 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770551654964895, + "loss": 1.4437, + "theoretical_loss": 3.9149811567006285, + "tokens_seen": 535101440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770391173520562, + "loss": 1.4334, + "theoretical_loss": 3.9149304890937833, + "tokens_seen": 535166976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770230692076229, + "loss": 1.4117, + "theoretical_loss": 3.9148798294283353, + "tokens_seen": 535232512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770070210631896, + "loss": 1.5312, + "theoretical_loss": 3.9148291777020683, + "tokens_seen": 535298048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769909729187563, + "loss": 1.4469, + "theoretical_loss": 3.9147785339127665, + "tokens_seen": 535363584 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676974924774323, + "loss": 1.5636, + "theoretical_loss": 3.914727898058215, + "tokens_seen": 535429120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769588766298897, + "loss": 1.4036, + "theoretical_loss": 3.914677270136199, + "tokens_seen": 535494656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769428284854564, + "loss": 1.5086, + "theoretical_loss": 3.914626650144505, + "tokens_seen": 535560192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769267803410231, + "loss": 1.3943, + "theoretical_loss": 3.914576038080922, + "tokens_seen": 535625728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769107321965898, + "loss": 1.3243, + "theoretical_loss": 3.9145254339432367, + "tokens_seen": 535691264 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768946840521565, + "loss": 1.4929, + "theoretical_loss": 3.9144748377292395, + "tokens_seen": 535756800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768786359077232, + "loss": 1.4033, + "theoretical_loss": 3.9144242494367205, + "tokens_seen": 535822336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768625877632899, + "loss": 1.4823, + "theoretical_loss": 3.914373669063471, + "tokens_seen": 535887872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768465396188566, + "loss": 1.4225, + "theoretical_loss": 3.914323096607283, + "tokens_seen": 535953408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768304914744233, + "loss": 1.4116, + "theoretical_loss": 3.914272532065949, + "tokens_seen": 536018944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00067681444332999, + "loss": 1.4862, + "theoretical_loss": 3.914221975437264, + "tokens_seen": 536084480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767983951855567, + "loss": 1.4409, + "theoretical_loss": 3.914171426719021, + "tokens_seen": 536150016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767823470411234, + "loss": 1.5067, + "theoretical_loss": 3.9141208859090164, + "tokens_seen": 536215552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767662988966901, + "loss": 1.4402, + "theoretical_loss": 3.9140703530050467, + "tokens_seen": 536281088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767502507522568, + "loss": 1.6256, + "theoretical_loss": 3.914019828004909, + "tokens_seen": 536346624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767342026078235, + "loss": 1.4947, + "theoretical_loss": 3.9139693109064018, + "tokens_seen": 536412160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767181544633902, + "loss": 1.4468, + "theoretical_loss": 3.9139188017073243, + "tokens_seen": 536477696 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676702106318957, + "loss": 1.4703, + "theoretical_loss": 3.9138683004054764, + "tokens_seen": 536543232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766860581745236, + "loss": 1.3957, + "theoretical_loss": 3.913817806998659, + "tokens_seen": 536608768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766700100300903, + "loss": 1.4385, + "theoretical_loss": 3.9137673214846735, + "tokens_seen": 536674304 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676653961885657, + "loss": 1.3962, + "theoretical_loss": 3.913716843861323, + "tokens_seen": 536739840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766379137412238, + "loss": 1.5503, + "theoretical_loss": 3.91366637412641, + "tokens_seen": 536805376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766218655967904, + "loss": 1.571, + "theoretical_loss": 3.9136159122777405, + "tokens_seen": 536870912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766058174523572, + "loss": 1.546, + "theoretical_loss": 3.9135654583131183, + "tokens_seen": 536936448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765897693079238, + "loss": 1.5006, + "theoretical_loss": 3.9135150122303504, + "tokens_seen": 537001984 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765737211634906, + "loss": 1.5824, + "theoretical_loss": 3.913464574027244, + "tokens_seen": 537067520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765576730190572, + "loss": 1.5226, + "theoretical_loss": 3.913414143701606, + "tokens_seen": 537133056 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676541624874624, + "loss": 1.5175, + "theoretical_loss": 3.913363721251246, + "tokens_seen": 537198592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765255767301906, + "loss": 1.3869, + "theoretical_loss": 3.913313306673974, + "tokens_seen": 537264128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765095285857574, + "loss": 1.5287, + "theoretical_loss": 3.913262899967599, + "tokens_seen": 537329664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 185299, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3653303384780884, + "objective/train/theoretical_loss": 3.9132125011299337, + "objective/train/tokens_used": 557855200, + "theoretical_loss": 3.9132125011299337, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676493480441324, + "loss": 1.3709, + "theoretical_loss": 3.9132125011299337, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764774322968908, + "loss": 1.4701, + "theoretical_loss": 3.9131621101587903, + "tokens_seen": 537460736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764613841524574, + "loss": 1.4026, + "theoretical_loss": 3.913111727051982, + "tokens_seen": 537526272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764453360080242, + "loss": 1.6027, + "theoretical_loss": 3.9130613518073227, + "tokens_seen": 537591808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764292878635908, + "loss": 1.4285, + "theoretical_loss": 3.913010984422627, + "tokens_seen": 537657344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764132397191576, + "loss": 1.4628, + "theoretical_loss": 3.9129606248957103, + "tokens_seen": 537722880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763971915747242, + "loss": 1.5315, + "theoretical_loss": 3.91291027322439, + "tokens_seen": 537788416 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676381143430291, + "loss": 1.3342, + "theoretical_loss": 3.9128599294064834, + "tokens_seen": 537853952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763650952858576, + "loss": 1.6502, + "theoretical_loss": 3.9128095934398095, + "tokens_seen": 537919488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763490471414244, + "loss": 1.4484, + "theoretical_loss": 3.9127592653221868, + "tokens_seen": 537985024 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676332998996991, + "loss": 1.4775, + "theoretical_loss": 3.912708945051435, + "tokens_seen": 538050560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763169508525577, + "loss": 1.6435, + "theoretical_loss": 3.9126586326253765, + "tokens_seen": 538116096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763009027081244, + "loss": 1.5078, + "theoretical_loss": 3.9126083280418325, + "tokens_seen": 538181632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762848545636911, + "loss": 1.4297, + "theoretical_loss": 3.9125580312986257, + "tokens_seen": 538247168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762688064192578, + "loss": 1.4096, + "theoretical_loss": 3.9125077423935792, + "tokens_seen": 538312704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762527582748245, + "loss": 1.4451, + "theoretical_loss": 3.912457461324519, + "tokens_seen": 538378240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762367101303912, + "loss": 1.5723, + "theoretical_loss": 3.9124071880892686, + "tokens_seen": 538443776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762206619859579, + "loss": 1.4255, + "theoretical_loss": 3.912356922685655, + "tokens_seen": 538509312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762046138415246, + "loss": 1.6908, + "theoretical_loss": 3.912306665111506, + "tokens_seen": 538574848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761885656970913, + "loss": 1.4579, + "theoretical_loss": 3.9122564153646486, + "tokens_seen": 538640384 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676172517552658, + "loss": 1.4806, + "theoretical_loss": 3.9122061734429128, + "tokens_seen": 538705920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761564694082247, + "loss": 1.3943, + "theoretical_loss": 3.9121559393441268, + "tokens_seen": 538771456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761404212637914, + "loss": 1.4735, + "theoretical_loss": 3.912105713066122, + "tokens_seen": 538836992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761243731193581, + "loss": 1.5643, + "theoretical_loss": 3.91205549460673, + "tokens_seen": 538902528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761083249749248, + "loss": 1.4788, + "theoretical_loss": 3.9120052839637824, + "tokens_seen": 538968064 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760922768304915, + "loss": 1.5398, + "theoretical_loss": 3.9119550811351127, + "tokens_seen": 539033600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760762286860582, + "loss": 1.5037, + "theoretical_loss": 3.9119048861185552, + "tokens_seen": 539099136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760601805416249, + "loss": 1.4558, + "theoretical_loss": 3.9118546989119447, + "tokens_seen": 539164672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760441323971916, + "loss": 1.5235, + "theoretical_loss": 3.9118045195131166, + "tokens_seen": 539230208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760280842527583, + "loss": 1.4225, + "theoretical_loss": 3.911754347919908, + "tokens_seen": 539295744 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676012036108325, + "loss": 1.4986, + "theoretical_loss": 3.911704184130156, + "tokens_seen": 539361280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759959879638917, + "loss": 1.4661, + "theoretical_loss": 3.911654028141699, + "tokens_seen": 539426816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759799398194584, + "loss": 1.4318, + "theoretical_loss": 3.9116038799523762, + "tokens_seen": 539492352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759638916750251, + "loss": 1.478, + "theoretical_loss": 3.911553739560028, + "tokens_seen": 539557888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759478435305918, + "loss": 1.4314, + "theoretical_loss": 3.9115036069624947, + "tokens_seen": 539623424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759317953861585, + "loss": 1.471, + "theoretical_loss": 3.9114534821576186, + "tokens_seen": 539688960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759157472417252, + "loss": 1.3858, + "theoretical_loss": 3.9114033651432414, + "tokens_seen": 539754496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758996990972919, + "loss": 1.4939, + "theoretical_loss": 3.9113532559172084, + "tokens_seen": 539820032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758836509528586, + "loss": 1.5082, + "theoretical_loss": 3.9113031544773618, + "tokens_seen": 539885568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758676028084253, + "loss": 1.545, + "theoretical_loss": 3.9112530608215486, + "tokens_seen": 539951104 + }, + { + "epoch": 0.16, + "learning_rate": 0.000675851554663992, + "loss": 1.5214, + "theoretical_loss": 3.9112029749476145, + "tokens_seen": 540016640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758355065195587, + "loss": 1.5001, + "theoretical_loss": 3.9111528968534053, + "tokens_seen": 540082176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758194583751254, + "loss": 1.5725, + "theoretical_loss": 3.91110282653677, + "tokens_seen": 540147712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758034102306921, + "loss": 1.4774, + "theoretical_loss": 3.9110527639955563, + "tokens_seen": 540213248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757873620862588, + "loss": 1.5428, + "theoretical_loss": 3.911002709227615, + "tokens_seen": 540278784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757713139418255, + "loss": 1.5055, + "theoretical_loss": 3.9109526622307955, + "tokens_seen": 540344320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757552657973922, + "loss": 1.409, + "theoretical_loss": 3.910902623002949, + "tokens_seen": 540409856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757392176529589, + "loss": 1.4916, + "theoretical_loss": 3.9108525915419277, + "tokens_seen": 540475392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757231695085256, + "loss": 1.4927, + "theoretical_loss": 3.9108025678455847, + "tokens_seen": 540540928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757071213640923, + "loss": 1.4452, + "theoretical_loss": 3.9107525519117736, + "tokens_seen": 540606464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 186273, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.601438283920288, + "objective/train/theoretical_loss": 3.910702543738349, + "objective/train/tokens_used": 561132000, + "theoretical_loss": 3.910702543738349, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.000675691073219659, + "loss": 1.559, + "theoretical_loss": 3.910702543738349, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756750250752257, + "loss": 1.4619, + "theoretical_loss": 3.9106525433231663, + "tokens_seen": 540737536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756589769307924, + "loss": 1.4786, + "theoretical_loss": 3.9106025506640822, + "tokens_seen": 540803072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756429287863591, + "loss": 1.4277, + "theoretical_loss": 3.9105525657589535, + "tokens_seen": 540868608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756268806419258, + "loss": 1.5143, + "theoretical_loss": 3.910502588605638, + "tokens_seen": 540934144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756108324974925, + "loss": 1.5267, + "theoretical_loss": 3.9104526192019957, + "tokens_seen": 540999680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755947843530592, + "loss": 1.4738, + "theoretical_loss": 3.9104026575458857, + "tokens_seen": 541065216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755787362086259, + "loss": 1.4764, + "theoretical_loss": 3.9103527036351684, + "tokens_seen": 541130752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755626880641926, + "loss": 1.4191, + "theoretical_loss": 3.910302757467705, + "tokens_seen": 541196288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755466399197593, + "loss": 1.3898, + "theoretical_loss": 3.910252819041358, + "tokens_seen": 541261824 + }, + { + "epoch": 0.16, + "learning_rate": 0.000675530591775326, + "loss": 1.4358, + "theoretical_loss": 3.910202888353991, + "tokens_seen": 541327360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755145436308927, + "loss": 1.5174, + "theoretical_loss": 3.9101529654034675, + "tokens_seen": 541392896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754984954864594, + "loss": 1.5144, + "theoretical_loss": 3.9101030501876526, + "tokens_seen": 541458432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754824473420261, + "loss": 1.4981, + "theoretical_loss": 3.9100531427044114, + "tokens_seen": 541523968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754663991975928, + "loss": 1.4326, + "theoretical_loss": 3.9100032429516114, + "tokens_seen": 541589504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754503510531595, + "loss": 1.4579, + "theoretical_loss": 3.909953350927119, + "tokens_seen": 541655040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754343029087262, + "loss": 1.4463, + "theoretical_loss": 3.909903466628803, + "tokens_seen": 541720576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754182547642929, + "loss": 1.5862, + "theoretical_loss": 3.9098535900545324, + "tokens_seen": 541786112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754022066198596, + "loss": 1.5375, + "theoretical_loss": 3.909803721202177, + "tokens_seen": 541851648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753861584754263, + "loss": 1.5433, + "theoretical_loss": 3.909753860069608, + "tokens_seen": 541917184 + }, + { + "epoch": 0.16, + "learning_rate": 0.000675370110330993, + "loss": 1.5082, + "theoretical_loss": 3.909704006654695, + "tokens_seen": 541982720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753540621865597, + "loss": 1.4676, + "theoretical_loss": 3.9096541609553137, + "tokens_seen": 542048256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753380140421264, + "loss": 1.4795, + "theoretical_loss": 3.909604322969335, + "tokens_seen": 542113792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753219658976931, + "loss": 1.5508, + "theoretical_loss": 3.9095544926946344, + "tokens_seen": 542179328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753059177532598, + "loss": 1.4308, + "theoretical_loss": 3.909504670129085, + "tokens_seen": 542244864 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752898696088265, + "loss": 1.4758, + "theoretical_loss": 3.909454855270565, + "tokens_seen": 542310400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752738214643933, + "loss": 1.4732, + "theoretical_loss": 3.909405048116949, + "tokens_seen": 542375936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752577733199599, + "loss": 1.4179, + "theoretical_loss": 3.909355248666116, + "tokens_seen": 542441472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752417251755267, + "loss": 1.4859, + "theoretical_loss": 3.9093054569159436, + "tokens_seen": 542507008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752256770310933, + "loss": 1.4269, + "theoretical_loss": 3.909255672864311, + "tokens_seen": 542572544 + }, + { + "epoch": 0.16, + "learning_rate": 0.00067520962888666, + "loss": 1.4713, + "theoretical_loss": 3.909205896509098, + "tokens_seen": 542638080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751935807422267, + "loss": 1.4976, + "theoretical_loss": 3.9091561278481866, + "tokens_seen": 542703616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751775325977935, + "loss": 1.5367, + "theoretical_loss": 3.909106366879457, + "tokens_seen": 542769152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751614844533601, + "loss": 1.4783, + "theoretical_loss": 3.909056613600793, + "tokens_seen": 542834688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751454363089269, + "loss": 1.5162, + "theoretical_loss": 3.9090068680100765, + "tokens_seen": 542900224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751293881644935, + "loss": 1.4326, + "theoretical_loss": 3.9089571301051933, + "tokens_seen": 542965760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751133400200603, + "loss": 1.45, + "theoretical_loss": 3.9089073998840274, + "tokens_seen": 543031296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750972918756269, + "loss": 1.5227, + "theoretical_loss": 3.908857677344465, + "tokens_seen": 543096832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750812437311937, + "loss": 1.3709, + "theoretical_loss": 3.908807962484393, + "tokens_seen": 543162368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750651955867603, + "loss": 1.5308, + "theoretical_loss": 3.908758255301699, + "tokens_seen": 543227904 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750491474423271, + "loss": 1.471, + "theoretical_loss": 3.9087085557942713, + "tokens_seen": 543293440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750330992978937, + "loss": 1.5801, + "theoretical_loss": 3.908658863959999, + "tokens_seen": 543358976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750170511534605, + "loss": 1.4352, + "theoretical_loss": 3.9086091797967724, + "tokens_seen": 543424512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750010030090271, + "loss": 1.3739, + "theoretical_loss": 3.908559503302482, + "tokens_seen": 543490048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749849548645939, + "loss": 1.474, + "theoretical_loss": 3.9085098344750193, + "tokens_seen": 543555584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749689067201605, + "loss": 1.3797, + "theoretical_loss": 3.908460173312278, + "tokens_seen": 543621120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749528585757273, + "loss": 1.5149, + "theoretical_loss": 3.9084105198121506, + "tokens_seen": 543686656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749368104312939, + "loss": 1.5111, + "theoretical_loss": 3.9083608739725317, + "tokens_seen": 543752192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749207622868606, + "loss": 1.4539, + "theoretical_loss": 3.9083112357913166, + "tokens_seen": 543817728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749047141424273, + "loss": 1.4421, + "theoretical_loss": 3.9082616052664, + "tokens_seen": 543883264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 187447, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5180284976959229, + "objective/train/theoretical_loss": 3.90821198239568, + "objective/train/tokens_used": 564408800, + "theoretical_loss": 3.90821198239568, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.000674888665997994, + "loss": 1.4843, + "theoretical_loss": 3.90821198239568, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748726178535607, + "loss": 1.3792, + "theoretical_loss": 3.9081623671770536, + "tokens_seen": 544014336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748565697091274, + "loss": 1.6008, + "theoretical_loss": 3.9081127596084193, + "tokens_seen": 544079872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748405215646941, + "loss": 1.4053, + "theoretical_loss": 3.908063159687676, + "tokens_seen": 544145408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748244734202608, + "loss": 1.474, + "theoretical_loss": 3.9080135674127243, + "tokens_seen": 544210944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748084252758275, + "loss": 1.532, + "theoretical_loss": 3.9079639827814647, + "tokens_seen": 544276480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006747923771313942, + "loss": 1.4896, + "theoretical_loss": 3.907914405791799, + "tokens_seen": 544342016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006747763289869609, + "loss": 1.4168, + "theoretical_loss": 3.9078648364416297, + "tokens_seen": 544407552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006747602808425276, + "loss": 1.4467, + "theoretical_loss": 3.9078152747288604, + "tokens_seen": 544473088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006747442326980943, + "loss": 1.5824, + "theoretical_loss": 3.907765720651395, + "tokens_seen": 544538624 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674728184553661, + "loss": 1.5888, + "theoretical_loss": 3.907716174207139, + "tokens_seen": 544604160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006747121364092277, + "loss": 1.5042, + "theoretical_loss": 3.9076666353939977, + "tokens_seen": 544669696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746960882647944, + "loss": 1.5183, + "theoretical_loss": 3.9076171042098777, + "tokens_seen": 544735232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746800401203611, + "loss": 1.4646, + "theoretical_loss": 3.907567580652687, + "tokens_seen": 544800768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746639919759278, + "loss": 1.4199, + "theoretical_loss": 3.9075180647203336, + "tokens_seen": 544866304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746479438314945, + "loss": 1.4331, + "theoretical_loss": 3.907468556410727, + "tokens_seen": 544931840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746318956870612, + "loss": 1.4481, + "theoretical_loss": 3.907419055721777, + "tokens_seen": 544997376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746158475426279, + "loss": 1.4933, + "theoretical_loss": 3.907369562651394, + "tokens_seen": 545062912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745997993981946, + "loss": 1.4835, + "theoretical_loss": 3.9073200771974905, + "tokens_seen": 545128448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745837512537613, + "loss": 1.4513, + "theoretical_loss": 3.907270599357978, + "tokens_seen": 545193984 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674567703109328, + "loss": 1.5083, + "theoretical_loss": 3.9072211291307704, + "tokens_seen": 545259520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745516549648947, + "loss": 1.4462, + "theoretical_loss": 3.907171666513781, + "tokens_seen": 545325056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745356068204614, + "loss": 1.4709, + "theoretical_loss": 3.907122211504926, + "tokens_seen": 545390592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745195586760281, + "loss": 1.5978, + "theoretical_loss": 3.9070727641021206, + "tokens_seen": 545456128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745035105315948, + "loss": 1.4448, + "theoretical_loss": 3.907023324303281, + "tokens_seen": 545521664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744874623871615, + "loss": 1.4641, + "theoretical_loss": 3.9069738921063246, + "tokens_seen": 545587200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744714142427282, + "loss": 1.5723, + "theoretical_loss": 3.9069244675091697, + "tokens_seen": 545652736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744553660982949, + "loss": 1.4894, + "theoretical_loss": 3.9068750505097354, + "tokens_seen": 545718272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744393179538616, + "loss": 1.4264, + "theoretical_loss": 3.906825641105942, + "tokens_seen": 545783808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744232698094283, + "loss": 1.4719, + "theoretical_loss": 3.9067762392957093, + "tokens_seen": 545849344 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674407221664995, + "loss": 1.468, + "theoretical_loss": 3.90672684507696, + "tokens_seen": 545914880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743911735205617, + "loss": 1.3939, + "theoretical_loss": 3.9066774584476147, + "tokens_seen": 545980416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743751253761284, + "loss": 1.4936, + "theoretical_loss": 3.9066280794055976, + "tokens_seen": 546045952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743590772316951, + "loss": 1.4917, + "theoretical_loss": 3.9065787079488326, + "tokens_seen": 546111488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743430290872618, + "loss": 1.5062, + "theoretical_loss": 3.9065293440752447, + "tokens_seen": 546177024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743269809428285, + "loss": 1.4176, + "theoretical_loss": 3.9064799877827587, + "tokens_seen": 546242560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743109327983952, + "loss": 1.5045, + "theoretical_loss": 3.9064306390693018, + "tokens_seen": 546308096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742948846539619, + "loss": 1.5332, + "theoretical_loss": 3.9063812979328, + "tokens_seen": 546373632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742788365095286, + "loss": 1.4321, + "theoretical_loss": 3.9063319643711827, + "tokens_seen": 546439168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742627883650953, + "loss": 1.4583, + "theoretical_loss": 3.906282638382377, + "tokens_seen": 546504704 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674246740220662, + "loss": 1.4618, + "theoretical_loss": 3.906233319964315, + "tokens_seen": 546570240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742306920762287, + "loss": 1.4734, + "theoretical_loss": 3.9061840091149254, + "tokens_seen": 546635776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742146439317954, + "loss": 1.4433, + "theoretical_loss": 3.90613470583214, + "tokens_seen": 546701312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741985957873621, + "loss": 1.4171, + "theoretical_loss": 3.9060854101138904, + "tokens_seen": 546766848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741825476429288, + "loss": 1.552, + "theoretical_loss": 3.90603612195811, + "tokens_seen": 546832384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741664994984955, + "loss": 1.4249, + "theoretical_loss": 3.9059868413627328, + "tokens_seen": 546897920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741504513540622, + "loss": 1.4977, + "theoretical_loss": 3.905937568325692, + "tokens_seen": 546963456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741344032096289, + "loss": 1.4874, + "theoretical_loss": 3.9058883028449247, + "tokens_seen": 547028992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741183550651956, + "loss": 1.573, + "theoretical_loss": 3.905839044918366, + "tokens_seen": 547094528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741023069207623, + "loss": 1.4674, + "theoretical_loss": 3.905789794543953, + "tokens_seen": 547160064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 188319, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3917874097824097, + "objective/train/theoretical_loss": 3.9057405517196235, + "objective/train/tokens_used": 567685600, + "theoretical_loss": 3.9057405517196235, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674086258776329, + "loss": 1.3673, + "theoretical_loss": 3.9057405517196235, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740702106318957, + "loss": 1.4805, + "theoretical_loss": 3.905691316443316, + "tokens_seen": 547291136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740541624874624, + "loss": 1.4456, + "theoretical_loss": 3.90564208871297, + "tokens_seen": 547356672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740381143430291, + "loss": 1.4466, + "theoretical_loss": 3.9055928685265258, + "tokens_seen": 547422208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740220661985958, + "loss": 1.3941, + "theoretical_loss": 3.9055436558819245, + "tokens_seen": 547487744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740060180541625, + "loss": 1.3788, + "theoretical_loss": 3.905494450777107, + "tokens_seen": 547553280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006739899699097292, + "loss": 1.6164, + "theoretical_loss": 3.9054452532100177, + "tokens_seen": 547618816 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673973921765296, + "loss": 1.4857, + "theoretical_loss": 3.905396063178598, + "tokens_seen": 547684352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006739578736208626, + "loss": 1.4982, + "theoretical_loss": 3.905346880680794, + "tokens_seen": 547749888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006739418254764293, + "loss": 1.355, + "theoretical_loss": 3.905297705714549, + "tokens_seen": 547815424 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673925777331996, + "loss": 1.496, + "theoretical_loss": 3.9052485382778097, + "tokens_seen": 547880960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006739097291875627, + "loss": 1.4519, + "theoretical_loss": 3.905199378368523, + "tokens_seen": 547946496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738936810431294, + "loss": 1.5708, + "theoretical_loss": 3.9051502259846362, + "tokens_seen": 548012032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738776328986962, + "loss": 1.4104, + "theoretical_loss": 3.9051010811240974, + "tokens_seen": 548077568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738615847542629, + "loss": 1.6264, + "theoretical_loss": 3.905051943784856, + "tokens_seen": 548143104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738455366098296, + "loss": 1.4225, + "theoretical_loss": 3.905002813964861, + "tokens_seen": 548208640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738294884653963, + "loss": 1.4701, + "theoretical_loss": 3.904953691662064, + "tokens_seen": 548274176 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673813440320963, + "loss": 1.4369, + "theoretical_loss": 3.904904576874416, + "tokens_seen": 548339712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737973921765297, + "loss": 1.4516, + "theoretical_loss": 3.90485546959987, + "tokens_seen": 548405248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737813440320964, + "loss": 1.4014, + "theoretical_loss": 3.904806369836378, + "tokens_seen": 548470784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737652958876631, + "loss": 1.4709, + "theoretical_loss": 3.904757277581895, + "tokens_seen": 548536320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737492477432298, + "loss": 1.55, + "theoretical_loss": 3.9047081928343745, + "tokens_seen": 548601856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737331995987965, + "loss": 1.497, + "theoretical_loss": 3.904659115591773, + "tokens_seen": 548667392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737171514543632, + "loss": 1.4167, + "theoretical_loss": 3.904610045852046, + "tokens_seen": 548732928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737011033099299, + "loss": 1.5021, + "theoretical_loss": 3.904560983613152, + "tokens_seen": 548798464 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006736850551654966, + "loss": 1.4396, + "theoretical_loss": 3.9045119288730468, + "tokens_seen": 548864000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006736690070210632, + "loss": 1.5089, + "theoretical_loss": 3.9044628816296907, + "tokens_seen": 548929536 + }, + { + "epoch": 0.17, + "learning_rate": 0.00067365295887663, + "loss": 1.3947, + "theoretical_loss": 3.9044138418810426, + "tokens_seen": 548995072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006736369107321966, + "loss": 1.4812, + "theoretical_loss": 3.904364809625063, + "tokens_seen": 549060608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006736208625877634, + "loss": 1.3479, + "theoretical_loss": 3.9043157848597128, + "tokens_seen": 549126144 + }, + { + "epoch": 0.17, + "learning_rate": 0.00067360481444333, + "loss": 1.3436, + "theoretical_loss": 3.904266767582954, + "tokens_seen": 549191680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735887662988968, + "loss": 1.4552, + "theoretical_loss": 3.9042177577927495, + "tokens_seen": 549257216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735727181544634, + "loss": 1.4793, + "theoretical_loss": 3.904168755487062, + "tokens_seen": 549322752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735566700100302, + "loss": 1.4728, + "theoretical_loss": 3.9041197606638565, + "tokens_seen": 549388288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735406218655968, + "loss": 1.5343, + "theoretical_loss": 3.9040707733210978, + "tokens_seen": 549453824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735245737211635, + "loss": 1.5119, + "theoretical_loss": 3.9040217934567525, + "tokens_seen": 549519360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735085255767302, + "loss": 1.4605, + "theoretical_loss": 3.903972821068786, + "tokens_seen": 549584896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734924774322969, + "loss": 1.3218, + "theoretical_loss": 3.903923856155167, + "tokens_seen": 549650432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734764292878636, + "loss": 1.4982, + "theoretical_loss": 3.9038748987138625, + "tokens_seen": 549715968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734603811434303, + "loss": 1.4314, + "theoretical_loss": 3.9038259487428424, + "tokens_seen": 549781504 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673444332998997, + "loss": 1.4938, + "theoretical_loss": 3.9037770062400767, + "tokens_seen": 549847040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734282848545637, + "loss": 1.4628, + "theoretical_loss": 3.903728071203535, + "tokens_seen": 549912576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734122367101304, + "loss": 1.3642, + "theoretical_loss": 3.90367914363119, + "tokens_seen": 549978112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733961885656971, + "loss": 1.4482, + "theoretical_loss": 3.903630223521013, + "tokens_seen": 550043648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733801404212638, + "loss": 1.5198, + "theoretical_loss": 3.903581310870978, + "tokens_seen": 550109184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733640922768305, + "loss": 1.4351, + "theoretical_loss": 3.9035324056790577, + "tokens_seen": 550174720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733480441323972, + "loss": 1.4349, + "theoretical_loss": 3.9034835079432275, + "tokens_seen": 550240256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733319959879639, + "loss": 1.5413, + "theoretical_loss": 3.9034346176614623, + "tokens_seen": 550305792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733159478435306, + "loss": 1.5968, + "theoretical_loss": 3.9033857348317387, + "tokens_seen": 550371328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732998996990973, + "loss": 1.5534, + "theoretical_loss": 3.9033368594520335, + "tokens_seen": 550436864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 189606, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5754977464675903, + "objective/train/theoretical_loss": 3.903287991520324, + "objective/train/tokens_used": 570962400, + "theoretical_loss": 3.903287991520324, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673283851554664, + "loss": 1.502, + "theoretical_loss": 3.903287991520324, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732678034102307, + "loss": 1.4596, + "theoretical_loss": 3.903239131034589, + "tokens_seen": 550567936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732517552657974, + "loss": 1.4343, + "theoretical_loss": 3.9031902779928087, + "tokens_seen": 550633472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732357071213641, + "loss": 1.4176, + "theoretical_loss": 3.903141432392962, + "tokens_seen": 550699008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732196589769308, + "loss": 1.4912, + "theoretical_loss": 3.903092594233031, + "tokens_seen": 550764544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732036108324975, + "loss": 1.4471, + "theoretical_loss": 3.903043763510996, + "tokens_seen": 550830080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731875626880642, + "loss": 1.4121, + "theoretical_loss": 3.9029949402248407, + "tokens_seen": 550895616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731715145436309, + "loss": 1.4698, + "theoretical_loss": 3.902946124372548, + "tokens_seen": 550961152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731554663991976, + "loss": 1.4843, + "theoretical_loss": 3.902897315952102, + "tokens_seen": 551026688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731394182547643, + "loss": 1.4583, + "theoretical_loss": 3.9028485149614873, + "tokens_seen": 551092224 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673123370110331, + "loss": 1.4265, + "theoretical_loss": 3.90279972139869, + "tokens_seen": 551157760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731073219658977, + "loss": 1.4766, + "theoretical_loss": 3.902750935261696, + "tokens_seen": 551223296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730912738214644, + "loss": 1.5156, + "theoretical_loss": 3.902702156548493, + "tokens_seen": 551288832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730752256770311, + "loss": 1.5538, + "theoretical_loss": 3.9026533852570684, + "tokens_seen": 551354368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730591775325978, + "loss": 1.4477, + "theoretical_loss": 3.9026046213854118, + "tokens_seen": 551419904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730431293881645, + "loss": 1.4707, + "theoretical_loss": 3.9025558649315117, + "tokens_seen": 551485440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730270812437312, + "loss": 1.5043, + "theoretical_loss": 3.9025071158933597, + "tokens_seen": 551550976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730110330992979, + "loss": 1.4054, + "theoretical_loss": 3.9024583742689463, + "tokens_seen": 551616512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729949849548646, + "loss": 1.3994, + "theoretical_loss": 3.902409640056263, + "tokens_seen": 551682048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729789368104313, + "loss": 1.427, + "theoretical_loss": 3.902360913253303, + "tokens_seen": 551747584 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672962888665998, + "loss": 1.4008, + "theoretical_loss": 3.9023121938580605, + "tokens_seen": 551813120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729468405215647, + "loss": 1.4376, + "theoretical_loss": 3.9022634818685282, + "tokens_seen": 551878656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729307923771314, + "loss": 1.554, + "theoretical_loss": 3.902214777282702, + "tokens_seen": 551944192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729147442326981, + "loss": 1.3726, + "theoretical_loss": 3.902166080098578, + "tokens_seen": 552009728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728986960882648, + "loss": 1.394, + "theoretical_loss": 3.902117390314152, + "tokens_seen": 552075264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728826479438315, + "loss": 1.5451, + "theoretical_loss": 3.902068707927422, + "tokens_seen": 552140800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728665997993982, + "loss": 1.5443, + "theoretical_loss": 3.9020200329363863, + "tokens_seen": 552206336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728505516549649, + "loss": 1.4525, + "theoretical_loss": 3.901971365339043, + "tokens_seen": 552271872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728345035105316, + "loss": 1.345, + "theoretical_loss": 3.9019227051333925, + "tokens_seen": 552337408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728184553660983, + "loss": 1.4898, + "theoretical_loss": 3.901874052317435, + "tokens_seen": 552402944 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672802407221665, + "loss": 1.5711, + "theoretical_loss": 3.9018254068891727, + "tokens_seen": 552468480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727863590772317, + "loss": 1.6135, + "theoretical_loss": 3.901776768846606, + "tokens_seen": 552534016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727703109327984, + "loss": 1.6138, + "theoretical_loss": 3.901728138187739, + "tokens_seen": 552599552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727542627883651, + "loss": 1.3739, + "theoretical_loss": 3.9016795149105747, + "tokens_seen": 552665088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727382146439318, + "loss": 1.3964, + "theoretical_loss": 3.901630899013118, + "tokens_seen": 552730624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727221664994985, + "loss": 1.4253, + "theoretical_loss": 3.901582290493374, + "tokens_seen": 552796160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727061183550652, + "loss": 1.4719, + "theoretical_loss": 3.901533689349348, + "tokens_seen": 552861696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726900702106319, + "loss": 1.4899, + "theoretical_loss": 3.9014850955790474, + "tokens_seen": 552927232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726740220661986, + "loss": 1.3868, + "theoretical_loss": 3.9014365091804795, + "tokens_seen": 552992768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726579739217654, + "loss": 1.3792, + "theoretical_loss": 3.9013879301516523, + "tokens_seen": 553058304 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672641925777332, + "loss": 1.5142, + "theoretical_loss": 3.901339358490575, + "tokens_seen": 553123840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726258776328988, + "loss": 1.3746, + "theoretical_loss": 3.9012907941952575, + "tokens_seen": 553189376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726098294884654, + "loss": 1.4594, + "theoretical_loss": 3.9012422372637103, + "tokens_seen": 553254912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725937813440322, + "loss": 1.5213, + "theoretical_loss": 3.901193687693945, + "tokens_seen": 553320448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725777331995988, + "loss": 1.4227, + "theoretical_loss": 3.9011451454839734, + "tokens_seen": 553385984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725616850551656, + "loss": 1.5528, + "theoretical_loss": 3.901096610631808, + "tokens_seen": 553451520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725456369107322, + "loss": 1.4144, + "theoretical_loss": 3.901048083135464, + "tokens_seen": 553517056 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672529588766299, + "loss": 1.4536, + "theoretical_loss": 3.900999562992954, + "tokens_seen": 553582592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725135406218657, + "loss": 1.4655, + "theoretical_loss": 3.9009510502022944, + "tokens_seen": 553648128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724974924774324, + "loss": 1.5144, + "theoretical_loss": 3.9009025447615002, + "tokens_seen": 553713664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 190485, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3175182342529297, + "objective/train/theoretical_loss": 3.9008540466685897, + "objective/train/tokens_used": 574239200, + "theoretical_loss": 3.9008540466685897, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672481444332999, + "loss": 1.3439, + "theoretical_loss": 3.9008540466685897, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724653961885658, + "loss": 1.4714, + "theoretical_loss": 3.900805555921579, + "tokens_seen": 553844736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724493480441325, + "loss": 1.4252, + "theoretical_loss": 3.9007570725184872, + "tokens_seen": 553910272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724332998996992, + "loss": 1.4653, + "theoretical_loss": 3.900708596457333, + "tokens_seen": 553975808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724172517552659, + "loss": 1.4083, + "theoretical_loss": 3.900660127736136, + "tokens_seen": 554041344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724012036108326, + "loss": 1.4722, + "theoretical_loss": 3.9006116663529173, + "tokens_seen": 554106880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723851554663993, + "loss": 1.476, + "theoretical_loss": 3.9005632123056984, + "tokens_seen": 554172416 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672369107321966, + "loss": 1.5502, + "theoretical_loss": 3.9005147655925008, + "tokens_seen": 554237952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723530591775327, + "loss": 1.4592, + "theoretical_loss": 3.9004663262113475, + "tokens_seen": 554303488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723370110330994, + "loss": 1.5817, + "theoretical_loss": 3.900417894160263, + "tokens_seen": 554369024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723209628886661, + "loss": 1.4163, + "theoretical_loss": 3.900369469437271, + "tokens_seen": 554434560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723049147442328, + "loss": 1.4168, + "theoretical_loss": 3.900321052040397, + "tokens_seen": 554500096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722888665997995, + "loss": 1.3777, + "theoretical_loss": 3.900272641967666, + "tokens_seen": 554565632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722728184553661, + "loss": 1.4316, + "theoretical_loss": 3.900224239217106, + "tokens_seen": 554631168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722567703109329, + "loss": 1.4393, + "theoretical_loss": 3.9001758437867435, + "tokens_seen": 554696704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722407221664995, + "loss": 1.5377, + "theoretical_loss": 3.900127455674608, + "tokens_seen": 554762240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722246740220663, + "loss": 1.371, + "theoretical_loss": 3.9000790748787275, + "tokens_seen": 554827776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722086258776329, + "loss": 1.4876, + "theoretical_loss": 3.9000307013971325, + "tokens_seen": 554893312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721925777331997, + "loss": 1.5017, + "theoretical_loss": 3.899982335227852, + "tokens_seen": 554958848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721765295887663, + "loss": 1.4599, + "theoretical_loss": 3.89993397636892, + "tokens_seen": 555024384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721604814443331, + "loss": 1.5532, + "theoretical_loss": 3.899885624818366, + "tokens_seen": 555089920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721444332998997, + "loss": 1.4082, + "theoretical_loss": 3.8998372805742245, + "tokens_seen": 555155456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721283851554664, + "loss": 1.4126, + "theoretical_loss": 3.8997889436345283, + "tokens_seen": 555220992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721123370110331, + "loss": 1.5716, + "theoretical_loss": 3.899740613997312, + "tokens_seen": 555286528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720962888665998, + "loss": 1.5568, + "theoretical_loss": 3.8996922916606103, + "tokens_seen": 555352064 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720802407221665, + "loss": 1.4742, + "theoretical_loss": 3.89964397662246, + "tokens_seen": 555417600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720641925777332, + "loss": 1.4739, + "theoretical_loss": 3.899595668880897, + "tokens_seen": 555483136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720481444332999, + "loss": 1.418, + "theoretical_loss": 3.899547368433959, + "tokens_seen": 555548672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720320962888666, + "loss": 1.4476, + "theoretical_loss": 3.899499075279684, + "tokens_seen": 555614208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720160481444333, + "loss": 1.4526, + "theoretical_loss": 3.899450789416111, + "tokens_seen": 555679744 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672, + "loss": 1.522, + "theoretical_loss": 3.8994025108412793, + "tokens_seen": 555745280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719839518555667, + "loss": 1.4267, + "theoretical_loss": 3.8993542395532304, + "tokens_seen": 555810816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719679037111334, + "loss": 1.5176, + "theoretical_loss": 3.8993059755500044, + "tokens_seen": 555876352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719518555667001, + "loss": 1.4775, + "theoretical_loss": 3.8992577188296433, + "tokens_seen": 555941888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719358074222668, + "loss": 1.5034, + "theoretical_loss": 3.8992094693901906, + "tokens_seen": 556007424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719197592778335, + "loss": 1.428, + "theoretical_loss": 3.8991612272296896, + "tokens_seen": 556072960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719037111334002, + "loss": 1.4193, + "theoretical_loss": 3.899112992346184, + "tokens_seen": 556138496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718876629889669, + "loss": 1.4228, + "theoretical_loss": 3.899064764737719, + "tokens_seen": 556204032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718716148445336, + "loss": 1.4621, + "theoretical_loss": 3.8990165444023406, + "tokens_seen": 556269568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718555667001003, + "loss": 1.5343, + "theoretical_loss": 3.8989683313380947, + "tokens_seen": 556335104 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671839518555667, + "loss": 1.4186, + "theoretical_loss": 3.8989201255430292, + "tokens_seen": 556400640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718234704112337, + "loss": 1.5301, + "theoretical_loss": 3.8988719270151924, + "tokens_seen": 556466176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718074222668004, + "loss": 1.4505, + "theoretical_loss": 3.8988237357526314, + "tokens_seen": 556531712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717913741223671, + "loss": 1.4219, + "theoretical_loss": 3.8987755517533973, + "tokens_seen": 556597248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717753259779338, + "loss": 1.5398, + "theoretical_loss": 3.8987273750155396, + "tokens_seen": 556662784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717592778335005, + "loss": 1.539, + "theoretical_loss": 3.89867920553711, + "tokens_seen": 556728320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717432296890672, + "loss": 1.4986, + "theoretical_loss": 3.8986310433161595, + "tokens_seen": 556793856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717271815446339, + "loss": 1.4106, + "theoretical_loss": 3.898582888350741, + "tokens_seen": 556859392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717111334002006, + "loss": 1.4857, + "theoretical_loss": 3.8985347406389077, + "tokens_seen": 556924928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716950852557673, + "loss": 1.3531, + "theoretical_loss": 3.898486600178714, + "tokens_seen": 556990464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 191608, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5120327472686768, + "objective/train/theoretical_loss": 3.898438466968214, + "objective/train/tokens_used": 577516000, + "theoretical_loss": 3.898438466968214, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671679037111334, + "loss": 1.4921, + "theoretical_loss": 3.898438466968214, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716629889669007, + "loss": 1.4352, + "theoretical_loss": 3.8983903410054634, + "tokens_seen": 557121536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716469408224674, + "loss": 1.4466, + "theoretical_loss": 3.8983422222885187, + "tokens_seen": 557187072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716308926780341, + "loss": 1.4628, + "theoretical_loss": 3.8982941108154368, + "tokens_seen": 557252608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716148445336008, + "loss": 1.3886, + "theoretical_loss": 3.8982460065842757, + "tokens_seen": 557318144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715987963891675, + "loss": 1.4199, + "theoretical_loss": 3.8981979095930934, + "tokens_seen": 557383680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715827482447342, + "loss": 1.5233, + "theoretical_loss": 3.89814981983995, + "tokens_seen": 557449216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715667001003009, + "loss": 1.39, + "theoretical_loss": 3.8981017373229045, + "tokens_seen": 557514752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715506519558676, + "loss": 1.4756, + "theoretical_loss": 3.8980536620400184, + "tokens_seen": 557580288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715346038114343, + "loss": 1.3975, + "theoretical_loss": 3.8980055939893523, + "tokens_seen": 557645824 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671518555667001, + "loss": 1.4887, + "theoretical_loss": 3.89795753316897, + "tokens_seen": 557711360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715025075225677, + "loss": 1.4431, + "theoretical_loss": 3.8979094795769336, + "tokens_seen": 557776896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714864593781344, + "loss": 1.5638, + "theoretical_loss": 3.8978614332113066, + "tokens_seen": 557842432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714704112337011, + "loss": 1.5003, + "theoretical_loss": 3.8978133940701536, + "tokens_seen": 557907968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714543630892679, + "loss": 1.3973, + "theoretical_loss": 3.8977653621515405, + "tokens_seen": 557973504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714383149448345, + "loss": 1.4663, + "theoretical_loss": 3.8977173374535328, + "tokens_seen": 558039040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714222668004013, + "loss": 1.5457, + "theoretical_loss": 3.8976693199741974, + "tokens_seen": 558104576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714062186559679, + "loss": 1.4807, + "theoretical_loss": 3.8976213097116013, + "tokens_seen": 558170112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713901705115347, + "loss": 1.4396, + "theoretical_loss": 3.8975733066638134, + "tokens_seen": 558235648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713741223671013, + "loss": 1.5034, + "theoretical_loss": 3.897525310828902, + "tokens_seen": 558301184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713580742226681, + "loss": 1.3895, + "theoretical_loss": 3.8974773222049377, + "tokens_seen": 558366720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713420260782347, + "loss": 1.4724, + "theoretical_loss": 3.89742934078999, + "tokens_seen": 558432256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713259779338015, + "loss": 1.3561, + "theoretical_loss": 3.8973813665821306, + "tokens_seen": 558497792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713099297893681, + "loss": 1.5061, + "theoretical_loss": 3.8973333995794315, + "tokens_seen": 558563328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712938816449349, + "loss": 1.4293, + "theoretical_loss": 3.897285439779965, + "tokens_seen": 558628864 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712778335005015, + "loss": 1.3869, + "theoretical_loss": 3.8972374871818047, + "tokens_seen": 558694400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712617853560683, + "loss": 1.5328, + "theoretical_loss": 3.8971895417830247, + "tokens_seen": 558759936 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671245737211635, + "loss": 1.4658, + "theoretical_loss": 3.8971416035817006, + "tokens_seen": 558825472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712296890672017, + "loss": 1.4214, + "theoretical_loss": 3.8970936725759073, + "tokens_seen": 558891008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712136409227683, + "loss": 1.4339, + "theoretical_loss": 3.8970457487637216, + "tokens_seen": 558956544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711975927783351, + "loss": 1.4222, + "theoretical_loss": 3.89699783214322, + "tokens_seen": 559022080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711815446339017, + "loss": 1.5009, + "theoretical_loss": 3.8969499227124804, + "tokens_seen": 559087616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711654964894685, + "loss": 1.3898, + "theoretical_loss": 3.8969020204695823, + "tokens_seen": 559153152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711494483450352, + "loss": 1.5702, + "theoretical_loss": 3.8968541254126046, + "tokens_seen": 559218688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711334002006019, + "loss": 1.4097, + "theoretical_loss": 3.896806237539627, + "tokens_seen": 559284224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711173520561686, + "loss": 1.5051, + "theoretical_loss": 3.8967583568487307, + "tokens_seen": 559349760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711013039117353, + "loss": 1.4303, + "theoretical_loss": 3.8967104833379973, + "tokens_seen": 559415296 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671085255767302, + "loss": 1.5536, + "theoretical_loss": 3.8966626170055085, + "tokens_seen": 559480832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710692076228687, + "loss": 1.4769, + "theoretical_loss": 3.896614757849348, + "tokens_seen": 559546368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710531594784354, + "loss": 1.4104, + "theoretical_loss": 3.8965669058675996, + "tokens_seen": 559611904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710371113340021, + "loss": 1.5694, + "theoretical_loss": 3.8965190610583473, + "tokens_seen": 559677440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710210631895688, + "loss": 1.4871, + "theoretical_loss": 3.896471223419677, + "tokens_seen": 559742976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710050150451355, + "loss": 1.3676, + "theoretical_loss": 3.896423392949674, + "tokens_seen": 559808512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709889669007022, + "loss": 1.4637, + "theoretical_loss": 3.896375569646425, + "tokens_seen": 559874048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709729187562689, + "loss": 1.3481, + "theoretical_loss": 3.896327753508018, + "tokens_seen": 559939584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709568706118356, + "loss": 1.5005, + "theoretical_loss": 3.896279944532541, + "tokens_seen": 560005120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709408224674023, + "loss": 1.5627, + "theoretical_loss": 3.8962321427180826, + "tokens_seen": 560070656 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670924774322969, + "loss": 1.4232, + "theoretical_loss": 3.8961843480627323, + "tokens_seen": 560136192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709087261785357, + "loss": 1.4844, + "theoretical_loss": 3.8961365605645817, + "tokens_seen": 560201728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708926780341024, + "loss": 1.4625, + "theoretical_loss": 3.89608878022172, + "tokens_seen": 560267264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 192944, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4706947803497314, + "objective/train/theoretical_loss": 3.896041007032241, + "objective/train/tokens_used": 580792800, + "theoretical_loss": 3.896041007032241, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708766298896691, + "loss": 1.4728, + "theoretical_loss": 3.896041007032241, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708605817452358, + "loss": 1.5247, + "theoretical_loss": 3.895993240994236, + "tokens_seen": 560398336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708445336008024, + "loss": 1.4447, + "theoretical_loss": 3.895945482105798, + "tokens_seen": 560463872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708284854563692, + "loss": 1.4128, + "theoretical_loss": 3.8958977303650224, + "tokens_seen": 560529408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708124373119358, + "loss": 1.5002, + "theoretical_loss": 3.8958499857700035, + "tokens_seen": 560594944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707963891675026, + "loss": 1.424, + "theoretical_loss": 3.8958022483188364, + "tokens_seen": 560660480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707803410230692, + "loss": 1.5017, + "theoretical_loss": 3.895754518009617, + "tokens_seen": 560726016 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670764292878636, + "loss": 1.4005, + "theoretical_loss": 3.8957067948404434, + "tokens_seen": 560791552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707482447342026, + "loss": 1.5501, + "theoretical_loss": 3.895659078809412, + "tokens_seen": 560857088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707321965897693, + "loss": 1.5037, + "theoretical_loss": 3.895611369914622, + "tokens_seen": 560922624 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670716148445336, + "loss": 1.4869, + "theoretical_loss": 3.895563668154173, + "tokens_seen": 560988160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707001003009027, + "loss": 1.5724, + "theoretical_loss": 3.8955159735261637, + "tokens_seen": 561053696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706840521564694, + "loss": 1.4572, + "theoretical_loss": 3.895468286028695, + "tokens_seen": 561119232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706680040120361, + "loss": 1.3411, + "theoretical_loss": 3.8954206056598695, + "tokens_seen": 561184768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706519558676028, + "loss": 1.4937, + "theoretical_loss": 3.8953729324177875, + "tokens_seen": 561250304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706359077231695, + "loss": 1.436, + "theoretical_loss": 3.895325266300553, + "tokens_seen": 561315840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706198595787362, + "loss": 1.545, + "theoretical_loss": 3.8952776073062685, + "tokens_seen": 561381376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706038114343029, + "loss": 1.404, + "theoretical_loss": 3.895229955433039, + "tokens_seen": 561446912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705877632898696, + "loss": 1.3153, + "theoretical_loss": 3.8951823106789694, + "tokens_seen": 561512448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705717151454363, + "loss": 1.4576, + "theoretical_loss": 3.895134673042165, + "tokens_seen": 561577984 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670555667001003, + "loss": 1.4926, + "theoretical_loss": 3.895087042520733, + "tokens_seen": 561643520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705396188565697, + "loss": 1.4651, + "theoretical_loss": 3.895039419112779, + "tokens_seen": 561709056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705235707121364, + "loss": 1.4288, + "theoretical_loss": 3.8949918028164126, + "tokens_seen": 561774592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705075225677031, + "loss": 1.4602, + "theoretical_loss": 3.8949441936297413, + "tokens_seen": 561840128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704914744232698, + "loss": 1.4691, + "theoretical_loss": 3.894896591550875, + "tokens_seen": 561905664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704754262788365, + "loss": 1.4379, + "theoretical_loss": 3.894848996577924, + "tokens_seen": 561971200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704593781344032, + "loss": 1.5564, + "theoretical_loss": 3.8948014087089975, + "tokens_seen": 562036736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704433299899699, + "loss": 1.414, + "theoretical_loss": 3.8947538279422083, + "tokens_seen": 562102272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704272818455366, + "loss": 1.2931, + "theoretical_loss": 3.8947062542756683, + "tokens_seen": 562167808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704112337011033, + "loss": 1.5324, + "theoretical_loss": 3.8946586877074907, + "tokens_seen": 562233344 + }, + { + "epoch": 0.17, + "learning_rate": 0.00067039518555667, + "loss": 1.5057, + "theoretical_loss": 3.894611128235789, + "tokens_seen": 562298880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703791374122367, + "loss": 1.4521, + "theoretical_loss": 3.894563575858677, + "tokens_seen": 562364416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703630892678034, + "loss": 1.5724, + "theoretical_loss": 3.8945160305742705, + "tokens_seen": 562429952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703470411233701, + "loss": 1.5203, + "theoretical_loss": 3.8944684923806845, + "tokens_seen": 562495488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703309929789368, + "loss": 1.4232, + "theoretical_loss": 3.8944209612760368, + "tokens_seen": 562561024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703149448345035, + "loss": 1.4636, + "theoretical_loss": 3.894373437258443, + "tokens_seen": 562626560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702988966900702, + "loss": 1.454, + "theoretical_loss": 3.8943259203260228, + "tokens_seen": 562692096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702828485456369, + "loss": 1.4675, + "theoretical_loss": 3.8942784104768937, + "tokens_seen": 562757632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702668004012036, + "loss": 1.3461, + "theoretical_loss": 3.8942309077091757, + "tokens_seen": 562823168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702507522567704, + "loss": 1.4379, + "theoretical_loss": 3.894183412020988, + "tokens_seen": 562888704 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670234704112337, + "loss": 1.5108, + "theoretical_loss": 3.894135923410453, + "tokens_seen": 562954240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702186559679038, + "loss": 1.418, + "theoretical_loss": 3.8940884418756907, + "tokens_seen": 563019776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702026078234704, + "loss": 1.491, + "theoretical_loss": 3.894040967414824, + "tokens_seen": 563085312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701865596790372, + "loss": 1.3897, + "theoretical_loss": 3.893993500025976, + "tokens_seen": 563150848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701705115346038, + "loss": 1.53, + "theoretical_loss": 3.893946039707271, + "tokens_seen": 563216384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701544633901706, + "loss": 1.5299, + "theoretical_loss": 3.893898586456832, + "tokens_seen": 563281920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701384152457372, + "loss": 1.3898, + "theoretical_loss": 3.893851140272785, + "tokens_seen": 563347456 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670122367101304, + "loss": 1.4234, + "theoretical_loss": 3.8938037011532556, + "tokens_seen": 563412992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701063189568706, + "loss": 1.506, + "theoretical_loss": 3.893756269096371, + "tokens_seen": 563478528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700902708124374, + "loss": 1.5202, + "theoretical_loss": 3.8937088441002574, + "tokens_seen": 563544064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 193835, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5278165340423584, + "objective/train/theoretical_loss": 3.893661426163044, + "objective/train/tokens_used": 584069600, + "theoretical_loss": 3.893661426163044, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670074222668004, + "loss": 1.5765, + "theoretical_loss": 3.893661426163044, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700581745235708, + "loss": 1.502, + "theoretical_loss": 3.8936140152828584, + "tokens_seen": 563675136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700421263791374, + "loss": 1.4192, + "theoretical_loss": 3.8935666114578305, + "tokens_seen": 563740672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700260782347042, + "loss": 1.4857, + "theoretical_loss": 3.8935192146860906, + "tokens_seen": 563806208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700100300902708, + "loss": 1.5107, + "theoretical_loss": 3.8934718249657694, + "tokens_seen": 563871744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006699939819458376, + "loss": 1.4932, + "theoretical_loss": 3.8934244422949984, + "tokens_seen": 563937280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006699779338014042, + "loss": 1.4328, + "theoretical_loss": 3.89337706667191, + "tokens_seen": 564002816 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669961885656971, + "loss": 1.4154, + "theoretical_loss": 3.8933296980946372, + "tokens_seen": 564068352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006699458375125376, + "loss": 1.4524, + "theoretical_loss": 3.8932823365613136, + "tokens_seen": 564133888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006699297893681044, + "loss": 1.5674, + "theoretical_loss": 3.893234982070074, + "tokens_seen": 564199424 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669913741223671, + "loss": 1.478, + "theoretical_loss": 3.8931876346190517, + "tokens_seen": 564264960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698976930792378, + "loss": 1.48, + "theoretical_loss": 3.8931402942063853, + "tokens_seen": 564330496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698816449348044, + "loss": 1.5694, + "theoretical_loss": 3.89309296083021, + "tokens_seen": 564396032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698655967903712, + "loss": 1.4293, + "theoretical_loss": 3.8930456344886624, + "tokens_seen": 564461568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698495486459378, + "loss": 1.3529, + "theoretical_loss": 3.8929983151798817, + "tokens_seen": 564527104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698335005015046, + "loss": 1.4832, + "theoretical_loss": 3.892951002902006, + "tokens_seen": 564592640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698174523570712, + "loss": 1.5263, + "theoretical_loss": 3.892903697653174, + "tokens_seen": 564658176 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669801404212638, + "loss": 1.4369, + "theoretical_loss": 3.8928563994315266, + "tokens_seen": 564723712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697853560682047, + "loss": 1.3109, + "theoretical_loss": 3.8928091082352045, + "tokens_seen": 564789248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697693079237714, + "loss": 1.3746, + "theoretical_loss": 3.8927618240623496, + "tokens_seen": 564854784 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669753259779338, + "loss": 1.524, + "theoretical_loss": 3.8927145469111033, + "tokens_seen": 564920320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697372116349048, + "loss": 1.4817, + "theoretical_loss": 3.892667276779609, + "tokens_seen": 564985856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697211634904715, + "loss": 1.4604, + "theoretical_loss": 3.89262001366601, + "tokens_seen": 565051392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697051153460382, + "loss": 1.4828, + "theoretical_loss": 3.892572757568451, + "tokens_seen": 565116928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696890672016049, + "loss": 1.419, + "theoretical_loss": 3.892525508485077, + "tokens_seen": 565182464 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696730190571716, + "loss": 1.4302, + "theoretical_loss": 3.892478266414033, + "tokens_seen": 565248000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696569709127383, + "loss": 1.4737, + "theoretical_loss": 3.8924310313534667, + "tokens_seen": 565313536 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669640922768305, + "loss": 1.436, + "theoretical_loss": 3.892383803301524, + "tokens_seen": 565379072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696248746238717, + "loss": 1.385, + "theoretical_loss": 3.8923365822563536, + "tokens_seen": 565444608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696088264794384, + "loss": 1.3827, + "theoretical_loss": 3.8922893682161037, + "tokens_seen": 565510144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695927783350051, + "loss": 1.4165, + "theoretical_loss": 3.892242161178924, + "tokens_seen": 565575680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695767301905718, + "loss": 1.3998, + "theoretical_loss": 3.892194961142964, + "tokens_seen": 565641216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695606820461385, + "loss": 1.4822, + "theoretical_loss": 3.8921477681063736, + "tokens_seen": 565706752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695446339017052, + "loss": 1.5143, + "theoretical_loss": 3.892100582067306, + "tokens_seen": 565772288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695285857572719, + "loss": 1.5652, + "theoretical_loss": 3.8920534030239122, + "tokens_seen": 565837824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695125376128386, + "loss": 1.4798, + "theoretical_loss": 3.892006230974345, + "tokens_seen": 565903360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694964894684053, + "loss": 1.423, + "theoretical_loss": 3.8919590659167578, + "tokens_seen": 565968896 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669480441323972, + "loss": 1.4732, + "theoretical_loss": 3.8919119078493045, + "tokens_seen": 566034432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694643931795387, + "loss": 1.4357, + "theoretical_loss": 3.8918647567701417, + "tokens_seen": 566099968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694483450351053, + "loss": 1.481, + "theoretical_loss": 3.8918176126774227, + "tokens_seen": 566165504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694322968906721, + "loss": 1.513, + "theoretical_loss": 3.891770475569305, + "tokens_seen": 566231040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694162487462387, + "loss": 1.4065, + "theoretical_loss": 3.8917233454439453, + "tokens_seen": 566296576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694002006018055, + "loss": 1.4289, + "theoretical_loss": 3.8916762222995014, + "tokens_seen": 566362112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693841524573721, + "loss": 1.4075, + "theoretical_loss": 3.8916291061341317, + "tokens_seen": 566427648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693681043129389, + "loss": 1.3731, + "theoretical_loss": 3.8915819969459946, + "tokens_seen": 566493184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693520561685055, + "loss": 1.4602, + "theoretical_loss": 3.891534894733251, + "tokens_seen": 566558720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693360080240723, + "loss": 1.4231, + "theoretical_loss": 3.891487799494061, + "tokens_seen": 566624256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693199598796389, + "loss": 1.3516, + "theoretical_loss": 3.891440711226585, + "tokens_seen": 566689792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693039117352056, + "loss": 1.4852, + "theoretical_loss": 3.891393629928986, + "tokens_seen": 566755328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692878635907723, + "loss": 1.5724, + "theoretical_loss": 3.8913465555994255, + "tokens_seen": 566820864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 195104, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5167372226715088, + "objective/train/theoretical_loss": 3.8912994882360676, + "objective/train/tokens_used": 587346400, + "theoretical_loss": 3.8912994882360676, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669271815446339, + "loss": 1.5297, + "theoretical_loss": 3.8912994882360676, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692557673019057, + "loss": 1.4265, + "theoretical_loss": 3.891252427837076, + "tokens_seen": 566951936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692397191574724, + "loss": 1.4251, + "theoretical_loss": 3.8912053744006156, + "tokens_seen": 567017472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692236710130391, + "loss": 1.5477, + "theoretical_loss": 3.8911583279248507, + "tokens_seen": 567083008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692076228686058, + "loss": 1.4827, + "theoretical_loss": 3.8911112884079486, + "tokens_seen": 567148544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691915747241725, + "loss": 1.3542, + "theoretical_loss": 3.8910642558480752, + "tokens_seen": 567214080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691755265797392, + "loss": 1.5154, + "theoretical_loss": 3.8910172302433987, + "tokens_seen": 567279616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691594784353059, + "loss": 1.3885, + "theoretical_loss": 3.8909702115920863, + "tokens_seen": 567345152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691434302908726, + "loss": 1.4874, + "theoretical_loss": 3.890923199892308, + "tokens_seen": 567410688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691273821464393, + "loss": 1.4088, + "theoretical_loss": 3.890876195142232, + "tokens_seen": 567476224 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669111334002006, + "loss": 1.4489, + "theoretical_loss": 3.8908291973400297, + "tokens_seen": 567541760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690952858575727, + "loss": 1.3928, + "theoretical_loss": 3.890782206483871, + "tokens_seen": 567607296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690792377131394, + "loss": 1.4766, + "theoretical_loss": 3.8907352225719283, + "tokens_seen": 567672832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690631895687061, + "loss": 1.3147, + "theoretical_loss": 3.890688245602373, + "tokens_seen": 567738368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690471414242729, + "loss": 1.4016, + "theoretical_loss": 3.890641275573379, + "tokens_seen": 567803904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690310932798395, + "loss": 1.3566, + "theoretical_loss": 3.8905943124831195, + "tokens_seen": 567869440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690150451354063, + "loss": 1.3658, + "theoretical_loss": 3.890547356329769, + "tokens_seen": 567934976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689989969909729, + "loss": 1.4371, + "theoretical_loss": 3.890500407111502, + "tokens_seen": 568000512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689829488465397, + "loss": 1.4648, + "theoretical_loss": 3.8904534648264955, + "tokens_seen": 568066048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689669007021063, + "loss": 1.3354, + "theoretical_loss": 3.8904065294729246, + "tokens_seen": 568131584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689508525576731, + "loss": 1.5189, + "theoretical_loss": 3.890359601048967, + "tokens_seen": 568197120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689348044132397, + "loss": 1.5132, + "theoretical_loss": 3.8903126795528014, + "tokens_seen": 568262656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689187562688065, + "loss": 1.4662, + "theoretical_loss": 3.8902657649826047, + "tokens_seen": 568328192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689027081243731, + "loss": 1.4783, + "theoretical_loss": 3.890218857336557, + "tokens_seen": 568393728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688866599799399, + "loss": 1.3859, + "theoretical_loss": 3.890171956612838, + "tokens_seen": 568459264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688706118355065, + "loss": 1.4601, + "theoretical_loss": 3.890125062809628, + "tokens_seen": 568524800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688545636910733, + "loss": 1.4535, + "theoretical_loss": 3.8900781759251086, + "tokens_seen": 568590336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688385155466399, + "loss": 1.3373, + "theoretical_loss": 3.8900312959574617, + "tokens_seen": 568655872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688224674022067, + "loss": 1.4347, + "theoretical_loss": 3.88998442290487, + "tokens_seen": 568721408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688064192577733, + "loss": 1.4231, + "theoretical_loss": 3.889937556765517, + "tokens_seen": 568786944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687903711133401, + "loss": 1.3824, + "theoretical_loss": 3.8898906975375858, + "tokens_seen": 568852480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687743229689067, + "loss": 1.4538, + "theoretical_loss": 3.8898438452192616, + "tokens_seen": 568918016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687582748244735, + "loss": 1.4741, + "theoretical_loss": 3.8897969998087305, + "tokens_seen": 568983552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687422266800401, + "loss": 1.4618, + "theoretical_loss": 3.8897501613041783, + "tokens_seen": 569049088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687261785356069, + "loss": 1.4505, + "theoretical_loss": 3.8897033297037904, + "tokens_seen": 569114624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687101303911735, + "loss": 1.429, + "theoretical_loss": 3.889656505005756, + "tokens_seen": 569180160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686940822467403, + "loss": 1.5221, + "theoretical_loss": 3.889609687208262, + "tokens_seen": 569245696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686780341023069, + "loss": 1.4228, + "theoretical_loss": 3.8895628763094976, + "tokens_seen": 569311232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686619859578737, + "loss": 1.47, + "theoretical_loss": 3.8895160723076527, + "tokens_seen": 569376768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686459378134403, + "loss": 1.4073, + "theoretical_loss": 3.889469275200917, + "tokens_seen": 569442304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686298896690071, + "loss": 1.4508, + "theoretical_loss": 3.889422484987481, + "tokens_seen": 569507840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686138415245737, + "loss": 1.3578, + "theoretical_loss": 3.889375701665537, + "tokens_seen": 569573376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685977933801405, + "loss": 1.4829, + "theoretical_loss": 3.8893289252332766, + "tokens_seen": 569638912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685817452357071, + "loss": 1.4044, + "theoretical_loss": 3.8892821556888935, + "tokens_seen": 569704448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685656970912739, + "loss": 1.4279, + "theoretical_loss": 3.8892353930305807, + "tokens_seen": 569769984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685496489468405, + "loss": 1.6804, + "theoretical_loss": 3.8891886372565327, + "tokens_seen": 569835520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685336008024073, + "loss": 1.4291, + "theoretical_loss": 3.889141888364944, + "tokens_seen": 569901056 + }, + { + "epoch": 0.17, + "learning_rate": 0.000668517552657974, + "loss": 1.4354, + "theoretical_loss": 3.8890951463540104, + "tokens_seen": 569966592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685015045135407, + "loss": 1.4495, + "theoretical_loss": 3.8890484112219283, + "tokens_seen": 570032128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684854563691073, + "loss": 1.4318, + "theoretical_loss": 3.889001682966895, + "tokens_seen": 570097664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 195990, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5162267684936523, + "objective/train/theoretical_loss": 3.8889549615871077, + "objective/train/tokens_used": 590623200, + "theoretical_loss": 3.8889549615871077, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684694082246741, + "loss": 1.4552, + "theoretical_loss": 3.8889549615871077, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684533600802407, + "loss": 1.4672, + "theoretical_loss": 3.8889082470807645, + "tokens_seen": 570228736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684373119358075, + "loss": 1.4537, + "theoretical_loss": 3.888861539446066, + "tokens_seen": 570294272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684212637913742, + "loss": 1.4429, + "theoretical_loss": 3.88881483868121, + "tokens_seen": 570359808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684052156469409, + "loss": 1.4702, + "theoretical_loss": 3.8887681447843976, + "tokens_seen": 570425344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683891675025076, + "loss": 1.4375, + "theoretical_loss": 3.88872145775383, + "tokens_seen": 570490880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683731193580743, + "loss": 1.4581, + "theoretical_loss": 3.8886747775877084, + "tokens_seen": 570556416 + }, + { + "epoch": 0.17, + "learning_rate": 0.000668357071213641, + "loss": 1.3596, + "theoretical_loss": 3.888628104284236, + "tokens_seen": 570621952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683410230692077, + "loss": 1.5164, + "theoretical_loss": 3.888581437841615, + "tokens_seen": 570687488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683249749247744, + "loss": 1.4334, + "theoretical_loss": 3.8885347782580504, + "tokens_seen": 570753024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683089267803411, + "loss": 1.4766, + "theoretical_loss": 3.8884881255317456, + "tokens_seen": 570818560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682928786359078, + "loss": 1.4858, + "theoretical_loss": 3.888441479660906, + "tokens_seen": 570884096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682768304914745, + "loss": 1.532, + "theoretical_loss": 3.8883948406437376, + "tokens_seen": 570949632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682607823470412, + "loss": 1.4114, + "theoretical_loss": 3.8883482084784466, + "tokens_seen": 571015168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682447342026079, + "loss": 1.4769, + "theoretical_loss": 3.8883015831632406, + "tokens_seen": 571080704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682286860581746, + "loss": 1.5367, + "theoretical_loss": 3.8882549646963263, + "tokens_seen": 571146240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682126379137413, + "loss": 1.3905, + "theoretical_loss": 3.888208353075914, + "tokens_seen": 571211776 + }, + { + "epoch": 0.17, + "learning_rate": 0.000668196589769308, + "loss": 1.4432, + "theoretical_loss": 3.8881617483002113, + "tokens_seen": 571277312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681805416248747, + "loss": 1.4893, + "theoretical_loss": 3.8881151503674283, + "tokens_seen": 571342848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681644934804414, + "loss": 1.375, + "theoretical_loss": 3.888068559275776, + "tokens_seen": 571408384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681484453360081, + "loss": 1.4334, + "theoretical_loss": 3.8880219750234657, + "tokens_seen": 571473920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681323971915748, + "loss": 1.5079, + "theoretical_loss": 3.8879753976087086, + "tokens_seen": 571539456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681163490471415, + "loss": 1.4816, + "theoretical_loss": 3.8879288270297176, + "tokens_seen": 571604992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681003009027082, + "loss": 1.4003, + "theoretical_loss": 3.8878822632847063, + "tokens_seen": 571670528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680842527582749, + "loss": 1.5134, + "theoretical_loss": 3.887835706371888, + "tokens_seen": 571736064 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680682046138416, + "loss": 1.4662, + "theoretical_loss": 3.887789156289477, + "tokens_seen": 571801600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680521564694082, + "loss": 1.4893, + "theoretical_loss": 3.887742613035689, + "tokens_seen": 571867136 + }, + { + "epoch": 0.17, + "learning_rate": 0.000668036108324975, + "loss": 1.5036, + "theoretical_loss": 3.8876960766087407, + "tokens_seen": 571932672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680200601805416, + "loss": 1.4776, + "theoretical_loss": 3.8876495470068475, + "tokens_seen": 571998208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680040120361084, + "loss": 1.4827, + "theoretical_loss": 3.8876030242282265, + "tokens_seen": 572063744 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667987963891675, + "loss": 1.3667, + "theoretical_loss": 3.8875565082710963, + "tokens_seen": 572129280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679719157472418, + "loss": 1.436, + "theoretical_loss": 3.887509999133675, + "tokens_seen": 572194816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679558676028084, + "loss": 1.4718, + "theoretical_loss": 3.887463496814182, + "tokens_seen": 572260352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679398194583752, + "loss": 1.5012, + "theoretical_loss": 3.887417001310838, + "tokens_seen": 572325888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679237713139418, + "loss": 1.5284, + "theoretical_loss": 3.887370512621862, + "tokens_seen": 572391424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679077231695085, + "loss": 1.3893, + "theoretical_loss": 3.887324030745477, + "tokens_seen": 572456960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678916750250752, + "loss": 1.4374, + "theoretical_loss": 3.887277555679903, + "tokens_seen": 572522496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678756268806419, + "loss": 1.3893, + "theoretical_loss": 3.8872310874233644, + "tokens_seen": 572588032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678595787362086, + "loss": 1.4841, + "theoretical_loss": 3.887184625974083, + "tokens_seen": 572653568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678435305917754, + "loss": 1.4397, + "theoretical_loss": 3.8871381713302835, + "tokens_seen": 572719104 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667827482447342, + "loss": 1.4047, + "theoretical_loss": 3.8870917234901907, + "tokens_seen": 572784640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678114343029088, + "loss": 1.4294, + "theoretical_loss": 3.8870452824520294, + "tokens_seen": 572850176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677953861584754, + "loss": 1.6088, + "theoretical_loss": 3.8869988482140254, + "tokens_seen": 572915712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677793380140422, + "loss": 1.5765, + "theoretical_loss": 3.886952420774406, + "tokens_seen": 572981248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677632898696088, + "loss": 1.4694, + "theoretical_loss": 3.8869060001313973, + "tokens_seen": 573046784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677472417251756, + "loss": 1.4458, + "theoretical_loss": 3.8868595862832285, + "tokens_seen": 573112320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677311935807422, + "loss": 1.4668, + "theoretical_loss": 3.886813179228127, + "tokens_seen": 573177856 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667715145436309, + "loss": 1.3994, + "theoretical_loss": 3.886766778964323, + "tokens_seen": 573243392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676990972918756, + "loss": 1.4859, + "theoretical_loss": 3.8867203854900456, + "tokens_seen": 573308928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676830491474424, + "loss": 1.3269, + "theoretical_loss": 3.886673998803526, + "tokens_seen": 573374464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 197322, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.453871488571167, + "objective/train/theoretical_loss": 3.886627618902995, + "objective/train/tokens_used": 593900000, + "theoretical_loss": 3.886627618902995, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667667001003009, + "loss": 1.5145, + "theoretical_loss": 3.886627618902995, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676509528585758, + "loss": 1.4646, + "theoretical_loss": 3.886581245786685, + "tokens_seen": 573505536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676349047141424, + "loss": 1.252, + "theoretical_loss": 3.8865348794528276, + "tokens_seen": 573571072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676188565697092, + "loss": 1.4531, + "theoretical_loss": 3.8864885198996575, + "tokens_seen": 573636608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676028084252758, + "loss": 1.4677, + "theoretical_loss": 3.8864421671254066, + "tokens_seen": 573702144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006675867602808426, + "loss": 1.4749, + "theoretical_loss": 3.8863958211283114, + "tokens_seen": 573767680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006675707121364092, + "loss": 1.471, + "theoretical_loss": 3.8863494819066062, + "tokens_seen": 573833216 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667554663991976, + "loss": 1.5011, + "theoretical_loss": 3.8863031494585267, + "tokens_seen": 573898752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006675386158475426, + "loss": 1.5202, + "theoretical_loss": 3.8862568237823103, + "tokens_seen": 573964288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006675225677031094, + "loss": 1.4638, + "theoretical_loss": 3.8862105048761926, + "tokens_seen": 574029824 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667506519558676, + "loss": 1.5591, + "theoretical_loss": 3.8861641927384127, + "tokens_seen": 574095360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674904714142428, + "loss": 1.3434, + "theoretical_loss": 3.8861178873672095, + "tokens_seen": 574160896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674744232698094, + "loss": 1.4476, + "theoretical_loss": 3.886071588760821, + "tokens_seen": 574226432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674583751253762, + "loss": 1.4522, + "theoretical_loss": 3.886025296917487, + "tokens_seen": 574291968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674423269809428, + "loss": 1.4525, + "theoretical_loss": 3.8859790118354494, + "tokens_seen": 574357504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674262788365096, + "loss": 1.3981, + "theoretical_loss": 3.8859327335129477, + "tokens_seen": 574423040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674102306920762, + "loss": 1.4895, + "theoretical_loss": 3.8858864619482247, + "tokens_seen": 574488576 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667394182547643, + "loss": 1.4367, + "theoretical_loss": 3.8858401971395224, + "tokens_seen": 574554112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006673781344032096, + "loss": 1.4269, + "theoretical_loss": 3.885793939085085, + "tokens_seen": 574619648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006673620862587764, + "loss": 1.4373, + "theoretical_loss": 3.8857476877831543, + "tokens_seen": 574685184 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667346038114343, + "loss": 1.4741, + "theoretical_loss": 3.8857014432319765, + "tokens_seen": 574750720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006673299899699098, + "loss": 1.5079, + "theoretical_loss": 3.8856552054297957, + "tokens_seen": 574816256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006673139418254764, + "loss": 1.5025, + "theoretical_loss": 3.8856089743748576, + "tokens_seen": 574881792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672978936810432, + "loss": 1.3655, + "theoretical_loss": 3.885562750065409, + "tokens_seen": 574947328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672818455366098, + "loss": 1.6031, + "theoretical_loss": 3.8855165324996976, + "tokens_seen": 575012864 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672657973921766, + "loss": 1.432, + "theoretical_loss": 3.88547032167597, + "tokens_seen": 575078400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672497492477432, + "loss": 1.5351, + "theoretical_loss": 3.885424117592475, + "tokens_seen": 575143936 + }, + { + "epoch": 0.17, + "learning_rate": 0.00066723370110331, + "loss": 1.4087, + "theoretical_loss": 3.885377920247461, + "tokens_seen": 575209472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672176529588766, + "loss": 1.5188, + "theoretical_loss": 3.885331729639179, + "tokens_seen": 575275008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672016048144434, + "loss": 1.5046, + "theoretical_loss": 3.8852855457658784, + "tokens_seen": 575340544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00066718555667001, + "loss": 1.4645, + "theoretical_loss": 3.88523936862581, + "tokens_seen": 575406080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671695085255768, + "loss": 1.4773, + "theoretical_loss": 3.885193198217226, + "tokens_seen": 575471616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671534603811434, + "loss": 1.5131, + "theoretical_loss": 3.8851470345383783, + "tokens_seen": 575537152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671374122367102, + "loss": 1.4086, + "theoretical_loss": 3.8851008775875204, + "tokens_seen": 575602688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671213640922768, + "loss": 1.4833, + "theoretical_loss": 3.8850547273629052, + "tokens_seen": 575668224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671053159478436, + "loss": 1.3597, + "theoretical_loss": 3.8850085838627875, + "tokens_seen": 575733760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006670892678034102, + "loss": 1.4577, + "theoretical_loss": 3.8849624470854223, + "tokens_seen": 575799296 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667073219658977, + "loss": 1.3629, + "theoretical_loss": 3.8849163170290644, + "tokens_seen": 575864832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006670571715145436, + "loss": 1.3028, + "theoretical_loss": 3.8848701936919703, + "tokens_seen": 575930368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006670411233701104, + "loss": 1.54, + "theoretical_loss": 3.8848240770723974, + "tokens_seen": 575995904 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667025075225677, + "loss": 1.4641, + "theoretical_loss": 3.8847779671686022, + "tokens_seen": 576061440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006670090270812438, + "loss": 1.403, + "theoretical_loss": 3.8847318639788435, + "tokens_seen": 576126976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669929789368105, + "loss": 1.4338, + "theoretical_loss": 3.8846857675013804, + "tokens_seen": 576192512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669769307923772, + "loss": 1.5129, + "theoretical_loss": 3.884639677734471, + "tokens_seen": 576258048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669608826479439, + "loss": 1.3515, + "theoretical_loss": 3.884593594676377, + "tokens_seen": 576323584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669448345035106, + "loss": 1.3883, + "theoretical_loss": 3.884547518325359, + "tokens_seen": 576389120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669287863590773, + "loss": 1.4969, + "theoretical_loss": 3.8845014486796776, + "tokens_seen": 576454656 + }, + { + "epoch": 0.17, + "learning_rate": 0.000666912738214644, + "loss": 1.3977, + "theoretical_loss": 3.8844553857375947, + "tokens_seen": 576520192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668966900702107, + "loss": 1.3918, + "theoretical_loss": 3.884409329497374, + "tokens_seen": 576585728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668806419257774, + "loss": 1.3791, + "theoretical_loss": 3.884363279957278, + "tokens_seen": 576651264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 198260, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4905140399932861, + "objective/train/theoretical_loss": 3.884317237115571, + "objective/train/tokens_used": 597176800, + "theoretical_loss": 3.884317237115571, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668645937813441, + "loss": 1.4027, + "theoretical_loss": 3.884317237115571, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668485456369108, + "loss": 1.4067, + "theoretical_loss": 3.884271200970517, + "tokens_seen": 576782336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668324974924775, + "loss": 1.5032, + "theoretical_loss": 3.8842251715203826, + "tokens_seen": 576847872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668164493480442, + "loss": 1.4525, + "theoretical_loss": 3.8841791487634327, + "tokens_seen": 576913408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668004012036109, + "loss": 1.5495, + "theoretical_loss": 3.8841331326979347, + "tokens_seen": 576978944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667843530591776, + "loss": 1.4502, + "theoretical_loss": 3.8840871233221548, + "tokens_seen": 577044480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667683049147443, + "loss": 1.49, + "theoretical_loss": 3.8840411206343615, + "tokens_seen": 577110016 + }, + { + "epoch": 0.17, + "learning_rate": 0.000666752256770311, + "loss": 1.4969, + "theoretical_loss": 3.883995124632823, + "tokens_seen": 577175552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667362086258777, + "loss": 1.495, + "theoretical_loss": 3.8839491353158087, + "tokens_seen": 577241088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667201604814444, + "loss": 1.5002, + "theoretical_loss": 3.8839031526815884, + "tokens_seen": 577306624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667041123370111, + "loss": 1.4853, + "theoretical_loss": 3.8838571767284327, + "tokens_seen": 577372160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006666880641925778, + "loss": 1.4603, + "theoretical_loss": 3.883811207454612, + "tokens_seen": 577437696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666720160481445, + "loss": 1.3529, + "theoretical_loss": 3.883765244858399, + "tokens_seen": 577503232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666559679037111, + "loss": 1.4831, + "theoretical_loss": 3.8837192889380647, + "tokens_seen": 577568768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666399197592779, + "loss": 1.4656, + "theoretical_loss": 3.8836733396918834, + "tokens_seen": 577634304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666238716148445, + "loss": 1.4363, + "theoretical_loss": 3.8836273971181283, + "tokens_seen": 577699840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666078234704113, + "loss": 1.44, + "theoretical_loss": 3.883581461215074, + "tokens_seen": 577765376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665917753259779, + "loss": 1.4905, + "theoretical_loss": 3.8835355319809954, + "tokens_seen": 577830912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665757271815447, + "loss": 1.5494, + "theoretical_loss": 3.883489609414167, + "tokens_seen": 577896448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665596790371113, + "loss": 1.4153, + "theoretical_loss": 3.8834436935128664, + "tokens_seen": 577961984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665436308926781, + "loss": 1.3772, + "theoretical_loss": 3.8833977842753704, + "tokens_seen": 578027520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665275827482447, + "loss": 1.4353, + "theoretical_loss": 3.883351881699956, + "tokens_seen": 578093056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665115346038115, + "loss": 1.5437, + "theoretical_loss": 3.883305985784901, + "tokens_seen": 578158592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664954864593781, + "loss": 1.4388, + "theoretical_loss": 3.8832600965284847, + "tokens_seen": 578224128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664794383149449, + "loss": 1.4035, + "theoretical_loss": 3.8832142139289862, + "tokens_seen": 578289664 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664633901705115, + "loss": 1.614, + "theoretical_loss": 3.883168337984686, + "tokens_seen": 578355200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664473420260783, + "loss": 1.4377, + "theoretical_loss": 3.883122468693865, + "tokens_seen": 578420736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664312938816449, + "loss": 1.4435, + "theoretical_loss": 3.8830766060548036, + "tokens_seen": 578486272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664152457372117, + "loss": 1.4986, + "theoretical_loss": 3.8830307500657844, + "tokens_seen": 578551808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663991975927783, + "loss": 1.37, + "theoretical_loss": 3.8829849007250905, + "tokens_seen": 578617344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663831494483451, + "loss": 1.489, + "theoretical_loss": 3.8829390580310044, + "tokens_seen": 578682880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663671013039117, + "loss": 1.4244, + "theoretical_loss": 3.88289322198181, + "tokens_seen": 578748416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663510531594785, + "loss": 1.4809, + "theoretical_loss": 3.8828473925757923, + "tokens_seen": 578813952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663350050150451, + "loss": 1.5996, + "theoretical_loss": 3.882801569811236, + "tokens_seen": 578879488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663189568706119, + "loss": 1.4475, + "theoretical_loss": 3.8827557536864274, + "tokens_seen": 578945024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663029087261785, + "loss": 1.5018, + "theoretical_loss": 3.882709944199653, + "tokens_seen": 579010560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662868605817453, + "loss": 1.5429, + "theoretical_loss": 3.8826641413491987, + "tokens_seen": 579076096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662708124373119, + "loss": 1.4644, + "theoretical_loss": 3.882618345133354, + "tokens_seen": 579141632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662547642928787, + "loss": 1.4266, + "theoretical_loss": 3.882572555550406, + "tokens_seen": 579207168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662387161484453, + "loss": 1.5529, + "theoretical_loss": 3.882526772598644, + "tokens_seen": 579272704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662226680040121, + "loss": 1.4612, + "theoretical_loss": 3.8824809962763576, + "tokens_seen": 579338240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662066198595787, + "loss": 1.4295, + "theoretical_loss": 3.882435226581837, + "tokens_seen": 579403776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661905717151455, + "loss": 1.3675, + "theoretical_loss": 3.8823894635133738, + "tokens_seen": 579469312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661745235707121, + "loss": 1.4033, + "theoretical_loss": 3.8823437070692584, + "tokens_seen": 579534848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661584754262789, + "loss": 1.3765, + "theoretical_loss": 3.882297957247784, + "tokens_seen": 579600384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661424272818455, + "loss": 1.4589, + "theoretical_loss": 3.8822522140472424, + "tokens_seen": 579665920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661263791374123, + "loss": 1.4647, + "theoretical_loss": 3.882206477465928, + "tokens_seen": 579731456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661103309929789, + "loss": 1.3975, + "theoretical_loss": 3.882160747502134, + "tokens_seen": 579796992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660942828485457, + "loss": 1.5154, + "theoretical_loss": 3.8821150241541558, + "tokens_seen": 579862528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660782347041123, + "loss": 1.5099, + "theoretical_loss": 3.8820693074202888, + "tokens_seen": 579928064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 199439, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4916222095489502, + "objective/train/theoretical_loss": 3.8820235972988275, + "objective/train/tokens_used": 600453600, + "theoretical_loss": 3.8820235972988275, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660621865596791, + "loss": 1.4725, + "theoretical_loss": 3.8820235972988275, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660461384152457, + "loss": 1.5312, + "theoretical_loss": 3.8819778937880702, + "tokens_seen": 580059136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660300902708125, + "loss": 1.481, + "theoretical_loss": 3.8819321968863134, + "tokens_seen": 580124672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660140421263791, + "loss": 1.4392, + "theoretical_loss": 3.8818865065918553, + "tokens_seen": 580190208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659979939819459, + "loss": 1.282, + "theoretical_loss": 3.8818408229029937, + "tokens_seen": 580255744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659819458375125, + "loss": 1.43, + "theoretical_loss": 3.881795145818028, + "tokens_seen": 580321280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659658976930793, + "loss": 1.5045, + "theoretical_loss": 3.8817494753352584, + "tokens_seen": 580386816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659498495486459, + "loss": 1.3841, + "theoretical_loss": 3.881703811452985, + "tokens_seen": 580452352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659338014042127, + "loss": 1.4778, + "theoretical_loss": 3.8816581541695085, + "tokens_seen": 580517888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659177532597793, + "loss": 1.4307, + "theoretical_loss": 3.8816125034831312, + "tokens_seen": 580583424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659017051153461, + "loss": 1.3677, + "theoretical_loss": 3.881566859392154, + "tokens_seen": 580648960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658856569709127, + "loss": 1.4426, + "theoretical_loss": 3.8815212218948814, + "tokens_seen": 580714496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658696088264795, + "loss": 1.3904, + "theoretical_loss": 3.8814755909896164, + "tokens_seen": 580780032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658535606820461, + "loss": 1.4756, + "theoretical_loss": 3.881429966674663, + "tokens_seen": 580845568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658375125376129, + "loss": 1.4131, + "theoretical_loss": 3.881384348948325, + "tokens_seen": 580911104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658214643931795, + "loss": 1.3903, + "theoretical_loss": 3.8813387378089095, + "tokens_seen": 580976640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658054162487463, + "loss": 1.3678, + "theoretical_loss": 3.8812931332547214, + "tokens_seen": 581042176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657893681043129, + "loss": 1.4463, + "theoretical_loss": 3.881247535284068, + "tokens_seen": 581107712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657733199598797, + "loss": 1.3848, + "theoretical_loss": 3.881201943895256, + "tokens_seen": 581173248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657572718154463, + "loss": 1.4144, + "theoretical_loss": 3.8811563590865936, + "tokens_seen": 581238784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657412236710131, + "loss": 1.304, + "theoretical_loss": 3.8811107808563894, + "tokens_seen": 581304320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657251755265797, + "loss": 1.3599, + "theoretical_loss": 3.8810652092029523, + "tokens_seen": 581369856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657091273821465, + "loss": 1.4867, + "theoretical_loss": 3.881019644124593, + "tokens_seen": 581435392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656930792377131, + "loss": 1.3985, + "theoretical_loss": 3.8809740856196204, + "tokens_seen": 581500928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656770310932799, + "loss": 1.4878, + "theoretical_loss": 3.880928533686346, + "tokens_seen": 581566464 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656609829488466, + "loss": 1.5318, + "theoretical_loss": 3.8808829883230826, + "tokens_seen": 581632000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656449348044133, + "loss": 1.4466, + "theoretical_loss": 3.880837449528141, + "tokens_seen": 581697536 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066562888665998, + "loss": 1.4171, + "theoretical_loss": 3.8807919172998346, + "tokens_seen": 581763072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656128385155467, + "loss": 1.3872, + "theoretical_loss": 3.8807463916364773, + "tokens_seen": 581828608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655967903711134, + "loss": 1.508, + "theoretical_loss": 3.8807008725363836, + "tokens_seen": 581894144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655807422266801, + "loss": 1.506, + "theoretical_loss": 3.880655359997867, + "tokens_seen": 581959680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655646940822468, + "loss": 1.2942, + "theoretical_loss": 3.8806098540192435, + "tokens_seen": 582025216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655486459378135, + "loss": 1.5053, + "theoretical_loss": 3.880564354598829, + "tokens_seen": 582090752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655325977933802, + "loss": 1.4198, + "theoretical_loss": 3.8805188617349406, + "tokens_seen": 582156288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655165496489469, + "loss": 1.5112, + "theoretical_loss": 3.880473375425895, + "tokens_seen": 582221824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655005015045136, + "loss": 1.468, + "theoretical_loss": 3.880427895670011, + "tokens_seen": 582287360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654844533600803, + "loss": 1.4688, + "theoretical_loss": 3.8803824224656056, + "tokens_seen": 582352896 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665468405215647, + "loss": 1.3934, + "theoretical_loss": 3.880336955810999, + "tokens_seen": 582418432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654523570712137, + "loss": 1.5792, + "theoretical_loss": 3.880291495704511, + "tokens_seen": 582483968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654363089267805, + "loss": 1.3663, + "theoretical_loss": 3.880246042144461, + "tokens_seen": 582549504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654202607823471, + "loss": 1.4836, + "theoretical_loss": 3.880200595129171, + "tokens_seen": 582615040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654042126379139, + "loss": 1.4792, + "theoretical_loss": 3.880155154656962, + "tokens_seen": 582680576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653881644934805, + "loss": 1.3691, + "theoretical_loss": 3.880109720726157, + "tokens_seen": 582746112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653721163490472, + "loss": 1.4147, + "theoretical_loss": 3.880064293335078, + "tokens_seen": 582811648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653560682046139, + "loss": 1.422, + "theoretical_loss": 3.880018872482049, + "tokens_seen": 582877184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653400200601806, + "loss": 1.4849, + "theoretical_loss": 3.8799734581653933, + "tokens_seen": 582942720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653239719157473, + "loss": 1.5962, + "theoretical_loss": 3.8799280503834366, + "tokens_seen": 583008256 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665307923771314, + "loss": 1.4192, + "theoretical_loss": 3.879882649134503, + "tokens_seen": 583073792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652918756268807, + "loss": 1.4484, + "theoretical_loss": 3.8798372544169197, + "tokens_seen": 583139328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652758274824474, + "loss": 1.4309, + "theoretical_loss": 3.879791866229013, + "tokens_seen": 583204864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 200344, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4507158994674683, + "objective/train/theoretical_loss": 3.879746484569109, + "objective/train/tokens_used": 603730400, + "theoretical_loss": 3.879746484569109, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665259779338014, + "loss": 1.5196, + "theoretical_loss": 3.879746484569109, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652437311935808, + "loss": 1.4805, + "theoretical_loss": 3.879701109435537, + "tokens_seen": 583335936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652276830491474, + "loss": 1.4719, + "theoretical_loss": 3.8796557408266237, + "tokens_seen": 583401472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652116349047142, + "loss": 1.5439, + "theoretical_loss": 3.8796103787406997, + "tokens_seen": 583467008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651955867602808, + "loss": 1.4825, + "theoretical_loss": 3.8795650231760934, + "tokens_seen": 583532544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651795386158476, + "loss": 1.4274, + "theoretical_loss": 3.879519674131136, + "tokens_seen": 583598080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651634904714142, + "loss": 1.3955, + "theoretical_loss": 3.8794743316041584, + "tokens_seen": 583663616 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665147442326981, + "loss": 1.4537, + "theoretical_loss": 3.879428995593491, + "tokens_seen": 583729152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651313941825476, + "loss": 1.468, + "theoretical_loss": 3.8793836660974668, + "tokens_seen": 583794688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651153460381144, + "loss": 1.552, + "theoretical_loss": 3.8793383431144175, + "tokens_seen": 583860224 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665099297893681, + "loss": 1.4288, + "theoretical_loss": 3.879293026642678, + "tokens_seen": 583925760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650832497492478, + "loss": 1.3514, + "theoretical_loss": 3.879247716680581, + "tokens_seen": 583991296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650672016048144, + "loss": 1.4898, + "theoretical_loss": 3.8792024132264613, + "tokens_seen": 584056832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650511534603812, + "loss": 1.5033, + "theoretical_loss": 3.8791571162786544, + "tokens_seen": 584122368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650351053159478, + "loss": 1.3391, + "theoretical_loss": 3.8791118258354955, + "tokens_seen": 584187904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650190571715146, + "loss": 1.3986, + "theoretical_loss": 3.8790665418953214, + "tokens_seen": 584253440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650030090270812, + "loss": 1.4383, + "theoretical_loss": 3.879021264456469, + "tokens_seen": 584318976 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664986960882648, + "loss": 1.5654, + "theoretical_loss": 3.878975993517276, + "tokens_seen": 584384512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006649709127382146, + "loss": 1.5445, + "theoretical_loss": 3.8789307290760804, + "tokens_seen": 584450048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006649548645937814, + "loss": 1.4391, + "theoretical_loss": 3.8788854711312206, + "tokens_seen": 584515584 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664938816449348, + "loss": 1.4836, + "theoretical_loss": 3.878840219681037, + "tokens_seen": 584581120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006649227683049148, + "loss": 1.5231, + "theoretical_loss": 3.8787949747238697, + "tokens_seen": 584646656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006649067201604814, + "loss": 1.5907, + "theoretical_loss": 3.878749736258058, + "tokens_seen": 584712192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648906720160482, + "loss": 1.4663, + "theoretical_loss": 3.8787045042819446, + "tokens_seen": 584777728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648746238716148, + "loss": 1.4545, + "theoretical_loss": 3.8786592787938705, + "tokens_seen": 584843264 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648585757271816, + "loss": 1.4241, + "theoretical_loss": 3.878614059792178, + "tokens_seen": 584908800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648425275827482, + "loss": 1.5496, + "theoretical_loss": 3.878568847275212, + "tokens_seen": 584974336 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664826479438315, + "loss": 1.379, + "theoretical_loss": 3.8785236412413138, + "tokens_seen": 585039872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648104312938816, + "loss": 1.5821, + "theoretical_loss": 3.878478441688829, + "tokens_seen": 585105408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647943831494484, + "loss": 1.5253, + "theoretical_loss": 3.878433248616102, + "tokens_seen": 585170944 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664778335005015, + "loss": 1.4069, + "theoretical_loss": 3.878388062021479, + "tokens_seen": 585236480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647622868605818, + "loss": 1.3906, + "theoretical_loss": 3.8783428819033055, + "tokens_seen": 585302016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647462387161484, + "loss": 1.5373, + "theoretical_loss": 3.8782977082599284, + "tokens_seen": 585367552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647301905717152, + "loss": 1.4043, + "theoretical_loss": 3.8782525410896955, + "tokens_seen": 585433088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647141424272818, + "loss": 1.3978, + "theoretical_loss": 3.8782073803909545, + "tokens_seen": 585498624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646980942828486, + "loss": 1.443, + "theoretical_loss": 3.878162226162053, + "tokens_seen": 585564160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646820461384152, + "loss": 1.4232, + "theoretical_loss": 3.878117078401342, + "tokens_seen": 585629696 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664665997993982, + "loss": 1.4794, + "theoretical_loss": 3.87807193710717, + "tokens_seen": 585695232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646499498495486, + "loss": 1.4249, + "theoretical_loss": 3.8780268022778874, + "tokens_seen": 585760768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646339017051154, + "loss": 1.4592, + "theoretical_loss": 3.877981673911846, + "tokens_seen": 585826304 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664617853560682, + "loss": 1.4521, + "theoretical_loss": 3.8779365520073963, + "tokens_seen": 585891840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646018054162488, + "loss": 1.5234, + "theoretical_loss": 3.8778914365628916, + "tokens_seen": 585957376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645857572718154, + "loss": 1.5314, + "theoretical_loss": 3.877846327576684, + "tokens_seen": 586022912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645697091273822, + "loss": 1.4303, + "theoretical_loss": 3.8778012250471265, + "tokens_seen": 586088448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645536609829488, + "loss": 1.4924, + "theoretical_loss": 3.8777561289725746, + "tokens_seen": 586153984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645376128385156, + "loss": 1.3857, + "theoretical_loss": 3.8777110393513814, + "tokens_seen": 586219520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645215646940822, + "loss": 1.4557, + "theoretical_loss": 3.877665956181903, + "tokens_seen": 586285056 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664505516549649, + "loss": 1.4093, + "theoretical_loss": 3.8776208794624942, + "tokens_seen": 586350592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644894684052156, + "loss": 1.4833, + "theoretical_loss": 3.8775758091915127, + "tokens_seen": 586416128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644734202607824, + "loss": 1.6214, + "theoretical_loss": 3.8775307453673147, + "tokens_seen": 586481664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 201447, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.557558298110962, + "objective/train/theoretical_loss": 3.8774856879882584, + "objective/train/tokens_used": 607007200, + "theoretical_loss": 3.8774856879882584, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664457372116349, + "loss": 1.5159, + "theoretical_loss": 3.8774856879882584, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644413239719158, + "loss": 1.4372, + "theoretical_loss": 3.8774406370527013, + "tokens_seen": 586612736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644252758274824, + "loss": 1.5486, + "theoretical_loss": 3.8773955925590036, + "tokens_seen": 586678272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644092276830492, + "loss": 1.4192, + "theoretical_loss": 3.8773505545055227, + "tokens_seen": 586743808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643931795386158, + "loss": 1.4344, + "theoretical_loss": 3.87730552289062, + "tokens_seen": 586809344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643771313941826, + "loss": 1.5196, + "theoretical_loss": 3.8772604977126557, + "tokens_seen": 586874880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643610832497492, + "loss": 1.3887, + "theoretical_loss": 3.8772154789699913, + "tokens_seen": 586940416 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664345035105316, + "loss": 1.3559, + "theoretical_loss": 3.8771704666609885, + "tokens_seen": 587005952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643289869608826, + "loss": 1.3598, + "theoretical_loss": 3.8771254607840095, + "tokens_seen": 587071488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643129388164494, + "loss": 1.3921, + "theoretical_loss": 3.877080461337418, + "tokens_seen": 587137024 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664296890672016, + "loss": 1.4247, + "theoretical_loss": 3.877035468319577, + "tokens_seen": 587202560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642808425275828, + "loss": 1.4363, + "theoretical_loss": 3.8769904817288507, + "tokens_seen": 587268096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642647943831495, + "loss": 1.5142, + "theoretical_loss": 3.8769455015636045, + "tokens_seen": 587333632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642487462387162, + "loss": 1.4592, + "theoretical_loss": 3.8769005278222033, + "tokens_seen": 587399168 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664232698094283, + "loss": 1.5649, + "theoretical_loss": 3.876855560503013, + "tokens_seen": 587464704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642166499498496, + "loss": 1.435, + "theoretical_loss": 3.8768105996044007, + "tokens_seen": 587530240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642006018054164, + "loss": 1.3991, + "theoretical_loss": 3.876765645124734, + "tokens_seen": 587595776 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664184553660983, + "loss": 1.4028, + "theoretical_loss": 3.876720697062379, + "tokens_seen": 587661312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641685055165498, + "loss": 1.4663, + "theoretical_loss": 3.876675755415706, + "tokens_seen": 587726848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641524573721164, + "loss": 1.4308, + "theoretical_loss": 3.8766308201830837, + "tokens_seen": 587792384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641364092276832, + "loss": 1.5021, + "theoretical_loss": 3.8765858913628803, + "tokens_seen": 587857920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641203610832498, + "loss": 1.5041, + "theoretical_loss": 3.8765409689534676, + "tokens_seen": 587923456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641043129388166, + "loss": 1.4928, + "theoretical_loss": 3.8764960529532155, + "tokens_seen": 587988992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006640882647943832, + "loss": 1.4278, + "theoretical_loss": 3.876451143360496, + "tokens_seen": 588054528 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066407221664995, + "loss": 1.534, + "theoretical_loss": 3.8764062401736803, + "tokens_seen": 588120064 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006640561685055166, + "loss": 1.4744, + "theoretical_loss": 3.8763613433911415, + "tokens_seen": 588185600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006640401203610834, + "loss": 1.4814, + "theoretical_loss": 3.8763164530112535, + "tokens_seen": 588251136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066402407221665, + "loss": 1.4645, + "theoretical_loss": 3.8762715690323883, + "tokens_seen": 588316672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006640080240722168, + "loss": 1.4931, + "theoretical_loss": 3.876226691452922, + "tokens_seen": 588382208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639919759277834, + "loss": 1.3968, + "theoretical_loss": 3.8761818202712286, + "tokens_seen": 588447744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639759277833501, + "loss": 1.5262, + "theoretical_loss": 3.8761369554856837, + "tokens_seen": 588513280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639598796389168, + "loss": 1.3893, + "theoretical_loss": 3.8760920970946637, + "tokens_seen": 588578816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639438314944835, + "loss": 1.3683, + "theoretical_loss": 3.8760472450965455, + "tokens_seen": 588644352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639277833500502, + "loss": 1.4467, + "theoretical_loss": 3.876002399489706, + "tokens_seen": 588709888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639117352056169, + "loss": 1.4814, + "theoretical_loss": 3.8759575602725236, + "tokens_seen": 588775424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638956870611836, + "loss": 1.4286, + "theoretical_loss": 3.8759127274433762, + "tokens_seen": 588840960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638796389167503, + "loss": 1.4914, + "theoretical_loss": 3.8758679010006434, + "tokens_seen": 588906496 + }, + { + "epoch": 0.18, + "learning_rate": 0.000663863590772317, + "loss": 1.38, + "theoretical_loss": 3.875823080942705, + "tokens_seen": 588972032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638475426278837, + "loss": 1.5693, + "theoretical_loss": 3.875778267267941, + "tokens_seen": 589037568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638314944834503, + "loss": 1.4688, + "theoretical_loss": 3.8757334599747324, + "tokens_seen": 589103104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638154463390171, + "loss": 1.3994, + "theoretical_loss": 3.8756886590614608, + "tokens_seen": 589168640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637993981945837, + "loss": 1.4926, + "theoretical_loss": 3.8756438645265074, + "tokens_seen": 589234176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637833500501505, + "loss": 1.4596, + "theoretical_loss": 3.8755990763682564, + "tokens_seen": 589299712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637673019057171, + "loss": 1.4927, + "theoretical_loss": 3.87555429458509, + "tokens_seen": 589365248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637512537612839, + "loss": 1.4579, + "theoretical_loss": 3.8755095191753917, + "tokens_seen": 589430784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637352056168505, + "loss": 1.4657, + "theoretical_loss": 3.8754647501375468, + "tokens_seen": 589496320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637191574724173, + "loss": 1.4161, + "theoretical_loss": 3.8754199874699404, + "tokens_seen": 589561856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637031093279839, + "loss": 1.5285, + "theoretical_loss": 3.875375231170957, + "tokens_seen": 589627392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636870611835507, + "loss": 1.4242, + "theoretical_loss": 3.8753304812389837, + "tokens_seen": 589692928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636710130391173, + "loss": 1.5375, + "theoretical_loss": 3.8752857376724075, + "tokens_seen": 589758464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 202342, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.514643907546997, + "objective/train/theoretical_loss": 3.875241000469615, + "objective/train/tokens_used": 610284000, + "theoretical_loss": 3.875241000469615, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636549648946841, + "loss": 1.5381, + "theoretical_loss": 3.875241000469615, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636389167502507, + "loss": 1.4772, + "theoretical_loss": 3.8751962696289945, + "tokens_seen": 589889536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636228686058175, + "loss": 1.4867, + "theoretical_loss": 3.8751515451489347, + "tokens_seen": 589955072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636068204613841, + "loss": 1.4871, + "theoretical_loss": 3.875106827027824, + "tokens_seen": 590020608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635907723169509, + "loss": 1.4657, + "theoretical_loss": 3.875062115264053, + "tokens_seen": 590086144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635747241725175, + "loss": 1.4551, + "theoretical_loss": 3.875017409856012, + "tokens_seen": 590151680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635586760280843, + "loss": 1.4035, + "theoretical_loss": 3.8749727108020915, + "tokens_seen": 590217216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635426278836509, + "loss": 1.4181, + "theoretical_loss": 3.8749280181006824, + "tokens_seen": 590282752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635265797392177, + "loss": 1.5073, + "theoretical_loss": 3.874883331750178, + "tokens_seen": 590348288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635105315947843, + "loss": 1.537, + "theoretical_loss": 3.874838651748971, + "tokens_seen": 590413824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634944834503511, + "loss": 1.5114, + "theoretical_loss": 3.874793978095453, + "tokens_seen": 590479360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634784353059177, + "loss": 1.3964, + "theoretical_loss": 3.874749310788019, + "tokens_seen": 590544896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634623871614845, + "loss": 1.4324, + "theoretical_loss": 3.8747046498250626, + "tokens_seen": 590610432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634463390170511, + "loss": 1.474, + "theoretical_loss": 3.87465999520498, + "tokens_seen": 590675968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634302908726179, + "loss": 1.5118, + "theoretical_loss": 3.8746153469261664, + "tokens_seen": 590741504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634142427281845, + "loss": 1.4831, + "theoretical_loss": 3.8745707049870175, + "tokens_seen": 590807040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633981945837513, + "loss": 1.4809, + "theoretical_loss": 3.8745260693859302, + "tokens_seen": 590872576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633821464393179, + "loss": 1.319, + "theoretical_loss": 3.8744814401213024, + "tokens_seen": 590938112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633660982948847, + "loss": 1.4283, + "theoretical_loss": 3.8744368171915307, + "tokens_seen": 591003648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633500501504513, + "loss": 1.4634, + "theoretical_loss": 3.8743922005950147, + "tokens_seen": 591069184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633340020060181, + "loss": 1.4687, + "theoretical_loss": 3.8743475903301534, + "tokens_seen": 591134720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633179538615847, + "loss": 1.451, + "theoretical_loss": 3.874302986395346, + "tokens_seen": 591200256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633019057171515, + "loss": 1.5493, + "theoretical_loss": 3.8742583887889928, + "tokens_seen": 591265792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632858575727181, + "loss": 1.5102, + "theoretical_loss": 3.874213797509495, + "tokens_seen": 591331328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632698094282849, + "loss": 1.41, + "theoretical_loss": 3.874169212555253, + "tokens_seen": 591396864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632537612838515, + "loss": 1.3676, + "theoretical_loss": 3.87412463392467, + "tokens_seen": 591462400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632377131394183, + "loss": 1.4752, + "theoretical_loss": 3.8740800616161484, + "tokens_seen": 591527936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632216649949849, + "loss": 1.4781, + "theoretical_loss": 3.8740354956280907, + "tokens_seen": 591593472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632056168505517, + "loss": 1.4841, + "theoretical_loss": 3.873990935958901, + "tokens_seen": 591659008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631895687061183, + "loss": 1.5448, + "theoretical_loss": 3.8739463826069835, + "tokens_seen": 591724544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631735205616851, + "loss": 1.3107, + "theoretical_loss": 3.8739018355707433, + "tokens_seen": 591790080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631574724172517, + "loss": 1.4157, + "theoretical_loss": 3.8738572948485848, + "tokens_seen": 591855616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631414242728185, + "loss": 1.4669, + "theoretical_loss": 3.873812760438916, + "tokens_seen": 591921152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631253761283851, + "loss": 1.4281, + "theoretical_loss": 3.873768232340142, + "tokens_seen": 591986688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631093279839519, + "loss": 1.4568, + "theoretical_loss": 3.87372371055067, + "tokens_seen": 592052224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630932798395185, + "loss": 1.5104, + "theoretical_loss": 3.8736791950689087, + "tokens_seen": 592117760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630772316950853, + "loss": 1.4712, + "theoretical_loss": 3.8736346858932658, + "tokens_seen": 592183296 + }, + { + "epoch": 0.18, + "learning_rate": 0.000663061183550652, + "loss": 1.4616, + "theoretical_loss": 3.8735901830221495, + "tokens_seen": 592248832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630451354062187, + "loss": 1.4809, + "theoretical_loss": 3.873545686453971, + "tokens_seen": 592314368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630290872617855, + "loss": 1.4827, + "theoretical_loss": 3.8735011961871395, + "tokens_seen": 592379904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630130391173521, + "loss": 1.5461, + "theoretical_loss": 3.873456712220066, + "tokens_seen": 592445440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629969909729189, + "loss": 1.4825, + "theoretical_loss": 3.873412234551161, + "tokens_seen": 592510976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629809428284855, + "loss": 1.4295, + "theoretical_loss": 3.8733677631788366, + "tokens_seen": 592576512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629648946840523, + "loss": 1.491, + "theoretical_loss": 3.8733232981015058, + "tokens_seen": 592642048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629488465396189, + "loss": 1.5089, + "theoretical_loss": 3.8732788393175808, + "tokens_seen": 592707584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629327983951857, + "loss": 1.5842, + "theoretical_loss": 3.8732343868254753, + "tokens_seen": 592773120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629167502507523, + "loss": 1.384, + "theoretical_loss": 3.873189940623604, + "tokens_seen": 592838656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629007021063191, + "loss": 1.4859, + "theoretical_loss": 3.8731455007103808, + "tokens_seen": 592904192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628846539618857, + "loss": 1.4207, + "theoretical_loss": 3.873101067084221, + "tokens_seen": 592969728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628686058174525, + "loss": 1.2965, + "theoretical_loss": 3.873056639743541, + "tokens_seen": 593035264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 203552, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4285786151885986, + "objective/train/theoretical_loss": 3.8730122186867573, + "objective/train/tokens_used": 613560800, + "theoretical_loss": 3.8730122186867573, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628525576730191, + "loss": 1.394, + "theoretical_loss": 3.8730122186867573, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628365095285859, + "loss": 1.6074, + "theoretical_loss": 3.872967803912286, + "tokens_seen": 593166336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628204613841525, + "loss": 1.3419, + "theoretical_loss": 3.872923395418546, + "tokens_seen": 593231872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628044132397193, + "loss": 1.4308, + "theoretical_loss": 3.872878993203954, + "tokens_seen": 593297408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627883650952859, + "loss": 1.4721, + "theoretical_loss": 3.8728345972669294, + "tokens_seen": 593362944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627723169508527, + "loss": 1.4272, + "theoretical_loss": 3.872790207605891, + "tokens_seen": 593428480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627562688064193, + "loss": 1.3887, + "theoretical_loss": 3.87274582421926, + "tokens_seen": 593494016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627402206619861, + "loss": 1.4862, + "theoretical_loss": 3.872701447105455, + "tokens_seen": 593559552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627241725175527, + "loss": 1.5783, + "theoretical_loss": 3.8726570762628985, + "tokens_seen": 593625088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627081243731195, + "loss": 1.4866, + "theoretical_loss": 3.872612711690011, + "tokens_seen": 593690624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626920762286861, + "loss": 1.46, + "theoretical_loss": 3.8725683533852155, + "tokens_seen": 593756160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626760280842529, + "loss": 1.4521, + "theoretical_loss": 3.8725240013469344, + "tokens_seen": 593821696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626599799398195, + "loss": 1.5239, + "theoretical_loss": 3.8724796555735903, + "tokens_seen": 593887232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626439317953863, + "loss": 1.5429, + "theoretical_loss": 3.8724353160636085, + "tokens_seen": 593952768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626278836509529, + "loss": 1.4103, + "theoretical_loss": 3.872390982815412, + "tokens_seen": 594018304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626118355065197, + "loss": 1.3815, + "theoretical_loss": 3.8723466558274264, + "tokens_seen": 594083840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625957873620863, + "loss": 1.5167, + "theoretical_loss": 3.872302335098077, + "tokens_seen": 594149376 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662579739217653, + "loss": 1.4964, + "theoretical_loss": 3.872258020625791, + "tokens_seen": 594214912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625636910732197, + "loss": 1.5179, + "theoretical_loss": 3.8722137124089935, + "tokens_seen": 594280448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625476429287864, + "loss": 1.4576, + "theoretical_loss": 3.8721694104461126, + "tokens_seen": 594345984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625315947843531, + "loss": 1.4038, + "theoretical_loss": 3.872125114735576, + "tokens_seen": 594411520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625155466399198, + "loss": 1.4124, + "theoretical_loss": 3.8720808252758125, + "tokens_seen": 594477056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624994984954865, + "loss": 1.5148, + "theoretical_loss": 3.872036542065251, + "tokens_seen": 594542592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624834503510532, + "loss": 1.4547, + "theoretical_loss": 3.87199226510232, + "tokens_seen": 594608128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624674022066199, + "loss": 1.5369, + "theoretical_loss": 3.871947994385451, + "tokens_seen": 594673664 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624513540621866, + "loss": 1.5483, + "theoretical_loss": 3.871903729913074, + "tokens_seen": 594739200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624353059177532, + "loss": 1.4984, + "theoretical_loss": 3.8718594716836203, + "tokens_seen": 594804736 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066241925777332, + "loss": 1.5162, + "theoretical_loss": 3.871815219695522, + "tokens_seen": 594870272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624032096288866, + "loss": 1.3593, + "theoretical_loss": 3.8717709739472106, + "tokens_seen": 594935808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623871614844534, + "loss": 1.4981, + "theoretical_loss": 3.87172673443712, + "tokens_seen": 595001344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066237111334002, + "loss": 1.4377, + "theoretical_loss": 3.8716825011636837, + "tokens_seen": 595066880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623550651955868, + "loss": 1.5133, + "theoretical_loss": 3.871638274125335, + "tokens_seen": 595132416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623390170511534, + "loss": 1.3823, + "theoretical_loss": 3.871594053320509, + "tokens_seen": 595197952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623229689067202, + "loss": 1.4658, + "theoretical_loss": 3.871549838747641, + "tokens_seen": 595263488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623069207622868, + "loss": 1.4668, + "theoretical_loss": 3.871505630405167, + "tokens_seen": 595329024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006622908726178536, + "loss": 1.5038, + "theoretical_loss": 3.8714614282915227, + "tokens_seen": 595394560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006622748244734202, + "loss": 1.5095, + "theoretical_loss": 3.8714172324051455, + "tokens_seen": 595460096 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662258776328987, + "loss": 1.4693, + "theoretical_loss": 3.8713730427444726, + "tokens_seen": 595525632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006622427281845536, + "loss": 1.4081, + "theoretical_loss": 3.8713288593079422, + "tokens_seen": 595591168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006622266800401204, + "loss": 1.3935, + "theoretical_loss": 3.871284682093993, + "tokens_seen": 595656704 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662210631895687, + "loss": 1.4838, + "theoretical_loss": 3.871240511101064, + "tokens_seen": 595722240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621945837512538, + "loss": 1.491, + "theoretical_loss": 3.8711963463275945, + "tokens_seen": 595787776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621785356068204, + "loss": 1.4342, + "theoretical_loss": 3.871152187772026, + "tokens_seen": 595853312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621624874623872, + "loss": 1.4372, + "theoretical_loss": 3.8711080354327976, + "tokens_seen": 595918848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621464393179538, + "loss": 1.4873, + "theoretical_loss": 3.8710638893083518, + "tokens_seen": 595984384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621303911735206, + "loss": 1.3888, + "theoretical_loss": 3.8710197493971306, + "tokens_seen": 596049920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621143430290872, + "loss": 1.5513, + "theoretical_loss": 3.870975615697576, + "tokens_seen": 596115456 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662098294884654, + "loss": 1.3557, + "theoretical_loss": 3.8709314882081323, + "tokens_seen": 596180992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620822467402206, + "loss": 1.4748, + "theoretical_loss": 3.870887366927242, + "tokens_seen": 596246528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620661985957874, + "loss": 1.4486, + "theoretical_loss": 3.8708432518533495, + "tokens_seen": 596312064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 204530, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5145456790924072, + "objective/train/theoretical_loss": 3.8707991429848994, + "objective/train/tokens_used": 616837600, + "theoretical_loss": 3.8707991429848994, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662050150451354, + "loss": 1.4601, + "theoretical_loss": 3.8707991429848994, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620341023069208, + "loss": 1.4813, + "theoretical_loss": 3.8707550403203377, + "tokens_seen": 596443136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620180541624874, + "loss": 1.4676, + "theoretical_loss": 3.8707109438581098, + "tokens_seen": 596508672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620020060180542, + "loss": 1.429, + "theoretical_loss": 3.8706668535966626, + "tokens_seen": 596574208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619859578736208, + "loss": 1.4921, + "theoretical_loss": 3.8706227695344424, + "tokens_seen": 596639744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619699097291876, + "loss": 1.4169, + "theoretical_loss": 3.8705786916698974, + "tokens_seen": 596705280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619538615847542, + "loss": 1.411, + "theoretical_loss": 3.870534620001476, + "tokens_seen": 596770816 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661937813440321, + "loss": 1.4635, + "theoretical_loss": 3.870490554527626, + "tokens_seen": 596836352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619217652958876, + "loss": 1.4659, + "theoretical_loss": 3.8704464952467967, + "tokens_seen": 596901888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619057171514544, + "loss": 1.3666, + "theoretical_loss": 3.870402442157439, + "tokens_seen": 596967424 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661889669007021, + "loss": 1.6077, + "theoretical_loss": 3.8703583952580023, + "tokens_seen": 597032960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006618736208625878, + "loss": 1.4423, + "theoretical_loss": 3.8703143545469376, + "tokens_seen": 597098496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006618575727181545, + "loss": 1.5461, + "theoretical_loss": 3.870270320022697, + "tokens_seen": 597164032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006618415245737212, + "loss": 1.3763, + "theoretical_loss": 3.8702262916837316, + "tokens_seen": 597229568 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661825476429288, + "loss": 1.5186, + "theoretical_loss": 3.8701822695284953, + "tokens_seen": 597295104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006618094282848546, + "loss": 1.4728, + "theoretical_loss": 3.8701382535554396, + "tokens_seen": 597360640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617933801404213, + "loss": 1.4291, + "theoretical_loss": 3.8700942437630195, + "tokens_seen": 597426176 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661777331995988, + "loss": 1.51, + "theoretical_loss": 3.870050240149689, + "tokens_seen": 597491712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617612838515548, + "loss": 1.4125, + "theoretical_loss": 3.870006242713903, + "tokens_seen": 597557248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617452357071214, + "loss": 1.5887, + "theoretical_loss": 3.8699622514541163, + "tokens_seen": 597622784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617291875626882, + "loss": 1.5154, + "theoretical_loss": 3.8699182663687854, + "tokens_seen": 597688320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617131394182548, + "loss": 1.4929, + "theoretical_loss": 3.8698742874563665, + "tokens_seen": 597753856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616970912738216, + "loss": 1.4577, + "theoretical_loss": 3.8698303147153172, + "tokens_seen": 597819392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616810431293882, + "loss": 1.3801, + "theoretical_loss": 3.8697863481440944, + "tokens_seen": 597884928 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661664994984955, + "loss": 1.5343, + "theoretical_loss": 3.8697423877411565, + "tokens_seen": 597950464 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616489468405216, + "loss": 1.4261, + "theoretical_loss": 3.8696984335049622, + "tokens_seen": 598016000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616328986960884, + "loss": 1.3522, + "theoretical_loss": 3.869654485433971, + "tokens_seen": 598081536 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661616850551655, + "loss": 1.4609, + "theoretical_loss": 3.869610543526643, + "tokens_seen": 598147072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616008024072218, + "loss": 1.4074, + "theoretical_loss": 3.8695666077814375, + "tokens_seen": 598212608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615847542627884, + "loss": 1.4669, + "theoretical_loss": 3.869522678196816, + "tokens_seen": 598278144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615687061183552, + "loss": 1.5038, + "theoretical_loss": 3.86947875477124, + "tokens_seen": 598343680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615526579739218, + "loss": 1.4047, + "theoretical_loss": 3.8694348375031717, + "tokens_seen": 598409216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615366098294886, + "loss": 1.3909, + "theoretical_loss": 3.8693909263910737, + "tokens_seen": 598474752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615205616850552, + "loss": 1.4707, + "theoretical_loss": 3.869347021433409, + "tokens_seen": 598540288 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661504513540622, + "loss": 1.4068, + "theoretical_loss": 3.869303122628641, + "tokens_seen": 598605824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614884653961886, + "loss": 1.378, + "theoretical_loss": 3.869259229975235, + "tokens_seen": 598671360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614724172517554, + "loss": 1.4765, + "theoretical_loss": 3.869215343471654, + "tokens_seen": 598736896 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661456369107322, + "loss": 1.5062, + "theoretical_loss": 3.8691714631163645, + "tokens_seen": 598802432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614403209628888, + "loss": 1.4873, + "theoretical_loss": 3.8691275889078325, + "tokens_seen": 598867968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614242728184554, + "loss": 1.5145, + "theoretical_loss": 3.8690837208445243, + "tokens_seen": 598933504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614082246740222, + "loss": 1.5306, + "theoretical_loss": 3.869039858924906, + "tokens_seen": 598999040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613921765295888, + "loss": 1.5118, + "theoretical_loss": 3.868996003147447, + "tokens_seen": 599064576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613761283851556, + "loss": 1.3758, + "theoretical_loss": 3.868952153510614, + "tokens_seen": 599130112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613600802407222, + "loss": 1.4249, + "theoretical_loss": 3.868908310012876, + "tokens_seen": 599195648 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661344032096289, + "loss": 1.481, + "theoretical_loss": 3.868864472652702, + "tokens_seen": 599261184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613279839518556, + "loss": 1.3103, + "theoretical_loss": 3.868820641428562, + "tokens_seen": 599326720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613119358074224, + "loss": 1.5188, + "theoretical_loss": 3.868776816338926, + "tokens_seen": 599392256 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661295887662989, + "loss": 1.5063, + "theoretical_loss": 3.8687329973822653, + "tokens_seen": 599457792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612798395185558, + "loss": 1.4514, + "theoretical_loss": 3.868689184557051, + "tokens_seen": 599523328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612637913741224, + "loss": 1.5114, + "theoretical_loss": 3.8686453778617547, + "tokens_seen": 599588864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 205743, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4836275577545166, + "objective/train/theoretical_loss": 3.868601577294849, + "objective/train/tokens_used": 620114400, + "theoretical_loss": 3.868601577294849, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612477432296892, + "loss": 1.5095, + "theoretical_loss": 3.868601577294849, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612316950852558, + "loss": 1.2991, + "theoretical_loss": 3.8685577828548086, + "tokens_seen": 599719936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612156469408226, + "loss": 1.3201, + "theoretical_loss": 3.8685139945401046, + "tokens_seen": 599785472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611995987963892, + "loss": 1.4232, + "theoretical_loss": 3.8684702123492123, + "tokens_seen": 599851008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611835506519559, + "loss": 1.5699, + "theoretical_loss": 3.8684264362806062, + "tokens_seen": 599916544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611675025075226, + "loss": 1.5524, + "theoretical_loss": 3.868382666332762, + "tokens_seen": 599982080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611514543630893, + "loss": 1.4625, + "theoretical_loss": 3.8683389025041546, + "tokens_seen": 600047616 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661135406218656, + "loss": 1.4177, + "theoretical_loss": 3.8682951447932608, + "tokens_seen": 600113152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611193580742227, + "loss": 1.3461, + "theoretical_loss": 3.8682513931985576, + "tokens_seen": 600178688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611033099297894, + "loss": 1.3989, + "theoretical_loss": 3.8682076477185223, + "tokens_seen": 600244224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610872617853561, + "loss": 1.4005, + "theoretical_loss": 3.868163908351632, + "tokens_seen": 600309760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610712136409228, + "loss": 1.4583, + "theoretical_loss": 3.868120175096367, + "tokens_seen": 600375296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610551654964895, + "loss": 1.5047, + "theoretical_loss": 3.868076447951205, + "tokens_seen": 600440832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610391173520561, + "loss": 1.4884, + "theoretical_loss": 3.8680327269146257, + "tokens_seen": 600506368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610230692076229, + "loss": 1.347, + "theoretical_loss": 3.8679890119851095, + "tokens_seen": 600571904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610070210631895, + "loss": 1.466, + "theoretical_loss": 3.867945303161137, + "tokens_seen": 600637440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609909729187563, + "loss": 1.4438, + "theoretical_loss": 3.8679016004411895, + "tokens_seen": 600702976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609749247743229, + "loss": 1.4553, + "theoretical_loss": 3.867857903823748, + "tokens_seen": 600768512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609588766298897, + "loss": 1.4764, + "theoretical_loss": 3.8678142133072964, + "tokens_seen": 600834048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609428284854563, + "loss": 1.4829, + "theoretical_loss": 3.8677705288903166, + "tokens_seen": 600899584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609267803410231, + "loss": 1.5542, + "theoretical_loss": 3.867726850571291, + "tokens_seen": 600965120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609107321965897, + "loss": 1.3085, + "theoretical_loss": 3.867683178348705, + "tokens_seen": 601030656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608946840521565, + "loss": 1.4878, + "theoretical_loss": 3.867639512221043, + "tokens_seen": 601096192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608786359077231, + "loss": 1.4221, + "theoretical_loss": 3.8675958521867893, + "tokens_seen": 601161728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608625877632899, + "loss": 1.4264, + "theoretical_loss": 3.8675521982444296, + "tokens_seen": 601227264 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608465396188565, + "loss": 1.5051, + "theoretical_loss": 3.8675085503924507, + "tokens_seen": 601292800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608304914744233, + "loss": 1.4914, + "theoretical_loss": 3.867464908629338, + "tokens_seen": 601358336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608144433299899, + "loss": 1.4987, + "theoretical_loss": 3.86742127295358, + "tokens_seen": 601423872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607983951855567, + "loss": 1.3915, + "theoretical_loss": 3.8673776433636635, + "tokens_seen": 601489408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607823470411233, + "loss": 1.5461, + "theoretical_loss": 3.8673340198580766, + "tokens_seen": 601554944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607662988966901, + "loss": 1.3752, + "theoretical_loss": 3.867290402435309, + "tokens_seen": 601620480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607502507522567, + "loss": 1.5197, + "theoretical_loss": 3.8672467910938497, + "tokens_seen": 601686016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607342026078235, + "loss": 1.4824, + "theoretical_loss": 3.8672031858321887, + "tokens_seen": 601751552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607181544633901, + "loss": 1.3691, + "theoretical_loss": 3.8671595866488158, + "tokens_seen": 601817088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607021063189569, + "loss": 1.4666, + "theoretical_loss": 3.867115993542222, + "tokens_seen": 601882624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606860581745235, + "loss": 1.5184, + "theoretical_loss": 3.8670724065109, + "tokens_seen": 601948160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606700100300903, + "loss": 1.4481, + "theoretical_loss": 3.8670288255533403, + "tokens_seen": 602013696 + }, + { + "epoch": 0.18, + "learning_rate": 0.000660653961885657, + "loss": 1.4513, + "theoretical_loss": 3.8669852506680362, + "tokens_seen": 602079232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606379137412237, + "loss": 1.4266, + "theoretical_loss": 3.866941681853481, + "tokens_seen": 602144768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606218655967904, + "loss": 1.4738, + "theoretical_loss": 3.866898119108168, + "tokens_seen": 602210304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606058174523571, + "loss": 1.4137, + "theoretical_loss": 3.866854562430592, + "tokens_seen": 602275840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605897693079238, + "loss": 1.4907, + "theoretical_loss": 3.8668110118192462, + "tokens_seen": 602341376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605737211634905, + "loss": 1.4957, + "theoretical_loss": 3.8667674672726275, + "tokens_seen": 602406912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605576730190572, + "loss": 1.465, + "theoretical_loss": 3.866723928789231, + "tokens_seen": 602472448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605416248746239, + "loss": 1.4293, + "theoretical_loss": 3.8666803963675536, + "tokens_seen": 602537984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605255767301906, + "loss": 1.4752, + "theoretical_loss": 3.866636870006091, + "tokens_seen": 602603520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605095285857573, + "loss": 1.4357, + "theoretical_loss": 3.866593349703342, + "tokens_seen": 602669056 + }, + { + "epoch": 0.18, + "learning_rate": 0.000660493480441324, + "loss": 1.4409, + "theoretical_loss": 3.8665498354578034, + "tokens_seen": 602734592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604774322968907, + "loss": 1.5654, + "theoretical_loss": 3.866506327267974, + "tokens_seen": 602800128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604613841524574, + "loss": 1.4332, + "theoretical_loss": 3.866462825132353, + "tokens_seen": 602865664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 206667, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4438321590423584, + "objective/train/theoretical_loss": 3.86641932904944, + "objective/train/tokens_used": 623391200, + "theoretical_loss": 3.86641932904944, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604453360080241, + "loss": 1.458, + "theoretical_loss": 3.86641932904944, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604292878635908, + "loss": 1.3835, + "theoretical_loss": 3.866375839017735, + "tokens_seen": 602996736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604132397191575, + "loss": 1.3458, + "theoretical_loss": 3.8663323550357385, + "tokens_seen": 603062272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603971915747243, + "loss": 1.3798, + "theoretical_loss": 3.866288877101952, + "tokens_seen": 603127808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603811434302909, + "loss": 1.4673, + "theoretical_loss": 3.866245405214877, + "tokens_seen": 603193344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603650952858577, + "loss": 1.4339, + "theoretical_loss": 3.8662019393730156, + "tokens_seen": 603258880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603490471414243, + "loss": 1.4291, + "theoretical_loss": 3.86615847957487, + "tokens_seen": 603324416 + }, + { + "epoch": 0.18, + "learning_rate": 0.000660332998996991, + "loss": 1.4796, + "theoretical_loss": 3.866115025818945, + "tokens_seen": 603389952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603169508525577, + "loss": 1.2908, + "theoretical_loss": 3.8660715781037434, + "tokens_seen": 603455488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603009027081245, + "loss": 1.6529, + "theoretical_loss": 3.8660281364277695, + "tokens_seen": 603521024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602848545636911, + "loss": 1.4889, + "theoretical_loss": 3.865984700789528, + "tokens_seen": 603586560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602688064192579, + "loss": 1.397, + "theoretical_loss": 3.865941271187525, + "tokens_seen": 603652096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602527582748245, + "loss": 1.415, + "theoretical_loss": 3.865897847620266, + "tokens_seen": 603717632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602367101303913, + "loss": 1.479, + "theoretical_loss": 3.865854430086258, + "tokens_seen": 603783168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602206619859579, + "loss": 1.3538, + "theoretical_loss": 3.865811018584007, + "tokens_seen": 603848704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602046138415247, + "loss": 1.4362, + "theoretical_loss": 3.8657676131120224, + "tokens_seen": 603914240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601885656970913, + "loss": 1.3837, + "theoretical_loss": 3.86572421366881, + "tokens_seen": 603979776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601725175526581, + "loss": 1.4328, + "theoretical_loss": 3.86568082025288, + "tokens_seen": 604045312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601564694082247, + "loss": 1.3513, + "theoretical_loss": 3.8656374328627408, + "tokens_seen": 604110848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601404212637915, + "loss": 1.4515, + "theoretical_loss": 3.8655940514969025, + "tokens_seen": 604176384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601243731193581, + "loss": 1.4263, + "theoretical_loss": 3.865550676153875, + "tokens_seen": 604241920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601083249749249, + "loss": 1.3591, + "theoretical_loss": 3.8655073068321695, + "tokens_seen": 604307456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600922768304915, + "loss": 1.5243, + "theoretical_loss": 3.8654639435302967, + "tokens_seen": 604372992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600762286860583, + "loss": 1.4241, + "theoretical_loss": 3.865420586246768, + "tokens_seen": 604438528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600601805416249, + "loss": 1.4579, + "theoretical_loss": 3.865377234980097, + "tokens_seen": 604504064 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600441323971917, + "loss": 1.3609, + "theoretical_loss": 3.865333889728795, + "tokens_seen": 604569600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600280842527583, + "loss": 1.392, + "theoretical_loss": 3.8652905504913777, + "tokens_seen": 604635136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600120361083251, + "loss": 1.3966, + "theoretical_loss": 3.865247217266356, + "tokens_seen": 604700672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599959879638917, + "loss": 1.4724, + "theoretical_loss": 3.865203890052247, + "tokens_seen": 604766208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599799398194585, + "loss": 1.3838, + "theoretical_loss": 3.865160568847564, + "tokens_seen": 604831744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599638916750251, + "loss": 1.427, + "theoretical_loss": 3.865117253650823, + "tokens_seen": 604897280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599478435305919, + "loss": 1.382, + "theoretical_loss": 3.8650739444605406, + "tokens_seen": 604962816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599317953861585, + "loss": 1.4716, + "theoretical_loss": 3.8650306412752324, + "tokens_seen": 605028352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599157472417253, + "loss": 1.4328, + "theoretical_loss": 3.8649873440934157, + "tokens_seen": 605093888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598996990972919, + "loss": 1.4793, + "theoretical_loss": 3.8649440529136085, + "tokens_seen": 605159424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598836509528587, + "loss": 1.3659, + "theoretical_loss": 3.864900767734329, + "tokens_seen": 605224960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598676028084253, + "loss": 1.4324, + "theoretical_loss": 3.8648574885540947, + "tokens_seen": 605290496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598515546639921, + "loss": 1.3815, + "theoretical_loss": 3.8648142153714264, + "tokens_seen": 605356032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598355065195587, + "loss": 1.4118, + "theoretical_loss": 3.8647709481848427, + "tokens_seen": 605421568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598194583751255, + "loss": 1.4245, + "theoretical_loss": 3.8647276869928637, + "tokens_seen": 605487104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598034102306921, + "loss": 1.4302, + "theoretical_loss": 3.864684431794011, + "tokens_seen": 605552640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597873620862588, + "loss": 1.4232, + "theoretical_loss": 3.864641182586806, + "tokens_seen": 605618176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597713139418255, + "loss": 1.4132, + "theoretical_loss": 3.8645979393697694, + "tokens_seen": 605683712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597552657973922, + "loss": 1.4686, + "theoretical_loss": 3.864554702141424, + "tokens_seen": 605749248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597392176529589, + "loss": 1.3692, + "theoretical_loss": 3.8645114709002932, + "tokens_seen": 605814784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597231695085256, + "loss": 1.4305, + "theoretical_loss": 3.8644682456448995, + "tokens_seen": 605880320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597071213640923, + "loss": 1.3826, + "theoretical_loss": 3.864425026373768, + "tokens_seen": 605945856 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659691073219659, + "loss": 1.464, + "theoretical_loss": 3.864381813085422, + "tokens_seen": 606011392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006596750250752257, + "loss": 1.3892, + "theoretical_loss": 3.8643386057783866, + "tokens_seen": 606076928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006596589769307924, + "loss": 1.4343, + "theoretical_loss": 3.8642954044511875, + "tokens_seen": 606142464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 207838, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.6236754655838013, + "objective/train/theoretical_loss": 3.8642522091023506, + "objective/train/tokens_used": 626668000, + "theoretical_loss": 3.8642522091023506, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659642928786359, + "loss": 1.5245, + "theoretical_loss": 3.8642522091023506, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006596268806419258, + "loss": 1.5009, + "theoretical_loss": 3.864209019730403, + "tokens_seen": 606273536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006596108324974924, + "loss": 1.3572, + "theoretical_loss": 3.8641658363338705, + "tokens_seen": 606339072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595947843530592, + "loss": 1.4147, + "theoretical_loss": 3.8641226589112816, + "tokens_seen": 606404608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595787362086258, + "loss": 1.3962, + "theoretical_loss": 3.8640794874611646, + "tokens_seen": 606470144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595626880641926, + "loss": 1.4121, + "theoretical_loss": 3.8640363219820473, + "tokens_seen": 606535680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595466399197592, + "loss": 1.4212, + "theoretical_loss": 3.8639931624724593, + "tokens_seen": 606601216 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659530591775326, + "loss": 1.5439, + "theoretical_loss": 3.8639500089309307, + "tokens_seen": 606666752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595145436308926, + "loss": 1.4606, + "theoretical_loss": 3.86390686135599, + "tokens_seen": 606732288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594984954864594, + "loss": 1.4124, + "theoretical_loss": 3.86386371974617, + "tokens_seen": 606797824 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659482447342026, + "loss": 1.4786, + "theoretical_loss": 3.863820584100001, + "tokens_seen": 606863360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594663991975928, + "loss": 1.4325, + "theoretical_loss": 3.863777454416014, + "tokens_seen": 606928896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594503510531595, + "loss": 1.4714, + "theoretical_loss": 3.863734330692742, + "tokens_seen": 606994432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594343029087262, + "loss": 1.4178, + "theoretical_loss": 3.8636912129287175, + "tokens_seen": 607059968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594182547642929, + "loss": 1.4254, + "theoretical_loss": 3.8636481011224744, + "tokens_seen": 607125504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594022066198596, + "loss": 1.5808, + "theoretical_loss": 3.8636049952725466, + "tokens_seen": 607191040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593861584754263, + "loss": 1.3405, + "theoretical_loss": 3.863561895377467, + "tokens_seen": 607256576 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659370110330993, + "loss": 1.4353, + "theoretical_loss": 3.8635188014357715, + "tokens_seen": 607322112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593540621865597, + "loss": 1.4427, + "theoretical_loss": 3.8634757134459954, + "tokens_seen": 607387648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593380140421264, + "loss": 1.5434, + "theoretical_loss": 3.8634326314066745, + "tokens_seen": 607453184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593219658976931, + "loss": 1.4577, + "theoretical_loss": 3.8633895553163455, + "tokens_seen": 607518720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593059177532598, + "loss": 1.4115, + "theoretical_loss": 3.8633464851735444, + "tokens_seen": 607584256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592898696088265, + "loss": 1.512, + "theoretical_loss": 3.8633034209768096, + "tokens_seen": 607649792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592738214643932, + "loss": 1.4788, + "theoretical_loss": 3.8632603627246787, + "tokens_seen": 607715328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592577733199599, + "loss": 1.3012, + "theoretical_loss": 3.86321731041569, + "tokens_seen": 607780864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592417251755266, + "loss": 1.341, + "theoretical_loss": 3.863174264048383, + "tokens_seen": 607846400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592256770310933, + "loss": 1.3701, + "theoretical_loss": 3.8631312236212967, + "tokens_seen": 607911936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00065920962888666, + "loss": 1.3869, + "theoretical_loss": 3.863088189132971, + "tokens_seen": 607977472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591935807422267, + "loss": 1.4, + "theoretical_loss": 3.8630451605819474, + "tokens_seen": 608043008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591775325977934, + "loss": 1.4417, + "theoretical_loss": 3.8630021379667654, + "tokens_seen": 608108544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591614844533601, + "loss": 1.3807, + "theoretical_loss": 3.862959121285968, + "tokens_seen": 608174080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591454363089268, + "loss": 1.4167, + "theoretical_loss": 3.8629161105380962, + "tokens_seen": 608239616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591293881644935, + "loss": 1.367, + "theoretical_loss": 3.862873105721694, + "tokens_seen": 608305152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591133400200602, + "loss": 1.3104, + "theoretical_loss": 3.862830106835303, + "tokens_seen": 608370688 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659097291875627, + "loss": 1.4652, + "theoretical_loss": 3.8627871138774674, + "tokens_seen": 608436224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590812437311936, + "loss": 1.5179, + "theoretical_loss": 3.862744126846732, + "tokens_seen": 608501760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590651955867603, + "loss": 1.3854, + "theoretical_loss": 3.8627011457416405, + "tokens_seen": 608567296 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659049147442327, + "loss": 1.5049, + "theoretical_loss": 3.862658170560738, + "tokens_seen": 608632832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590330992978938, + "loss": 1.4641, + "theoretical_loss": 3.862615201302571, + "tokens_seen": 608698368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590170511534604, + "loss": 1.4137, + "theoretical_loss": 3.8625722379656855, + "tokens_seen": 608763904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590010030090272, + "loss": 1.4818, + "theoretical_loss": 3.862529280548628, + "tokens_seen": 608829440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589849548645938, + "loss": 1.403, + "theoretical_loss": 3.8624863290499456, + "tokens_seen": 608894976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589689067201606, + "loss": 1.4422, + "theoretical_loss": 3.862443383468186, + "tokens_seen": 608960512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589528585757272, + "loss": 1.4226, + "theoretical_loss": 3.8624004438018984, + "tokens_seen": 609026048 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658936810431294, + "loss": 1.3911, + "theoretical_loss": 3.8623575100496303, + "tokens_seen": 609091584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589207622868606, + "loss": 1.4543, + "theoretical_loss": 3.8623145822099314, + "tokens_seen": 609157120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589047141424274, + "loss": 1.4233, + "theoretical_loss": 3.8622716602813516, + "tokens_seen": 609222656 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658888665997994, + "loss": 1.5068, + "theoretical_loss": 3.862228744262441, + "tokens_seen": 609288192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588726178535608, + "loss": 1.3251, + "theoretical_loss": 3.8621858341517514, + "tokens_seen": 609353728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588565697091274, + "loss": 1.4483, + "theoretical_loss": 3.8621429299478325, + "tokens_seen": 609419264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 208904, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.2397621870040894, + "objective/train/theoretical_loss": 3.862100031649237, + "objective/train/tokens_used": 629944800, + "theoretical_loss": 3.862100031649237, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588405215646942, + "loss": 1.297, + "theoretical_loss": 3.862100031649237, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588244734202608, + "loss": 1.3689, + "theoretical_loss": 3.8620571392545173, + "tokens_seen": 609550336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588084252758276, + "loss": 1.5308, + "theoretical_loss": 3.8620142527622265, + "tokens_seen": 609615872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006587923771313942, + "loss": 1.3965, + "theoretical_loss": 3.8619713721709172, + "tokens_seen": 609681408 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658776328986961, + "loss": 1.3237, + "theoretical_loss": 3.861928497479144, + "tokens_seen": 609746944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006587602808425276, + "loss": 1.4447, + "theoretical_loss": 3.86188562868546, + "tokens_seen": 609812480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006587442326980944, + "loss": 1.4676, + "theoretical_loss": 3.8618427657884222, + "tokens_seen": 609878016 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658728184553661, + "loss": 1.4838, + "theoretical_loss": 3.861799908786584, + "tokens_seen": 609943552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006587121364092278, + "loss": 1.5459, + "theoretical_loss": 3.8617570576785027, + "tokens_seen": 610009088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586960882647944, + "loss": 1.315, + "theoretical_loss": 3.8617142124627346, + "tokens_seen": 610074624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586800401203612, + "loss": 1.5473, + "theoretical_loss": 3.8616713731378356, + "tokens_seen": 610140160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586639919759278, + "loss": 1.4366, + "theoretical_loss": 3.8616285397023637, + "tokens_seen": 610205696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586479438314946, + "loss": 1.4457, + "theoretical_loss": 3.861585712154877, + "tokens_seen": 610271232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586318956870612, + "loss": 1.4372, + "theoretical_loss": 3.8615428904939337, + "tokens_seen": 610336768 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658615847542628, + "loss": 1.4573, + "theoretical_loss": 3.861500074718093, + "tokens_seen": 610402304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006585997993981946, + "loss": 1.4509, + "theoretical_loss": 3.8614572648259142, + "tokens_seen": 610467840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585837512537614, + "loss": 1.4814, + "theoretical_loss": 3.8614144608159577, + "tokens_seen": 610533376 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658567703109328, + "loss": 1.4871, + "theoretical_loss": 3.8613716626867833, + "tokens_seen": 610598912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585516549648948, + "loss": 1.5432, + "theoretical_loss": 3.861328870436952, + "tokens_seen": 610664448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585356068204614, + "loss": 1.502, + "theoretical_loss": 3.8612860840650267, + "tokens_seen": 610729984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585195586760282, + "loss": 1.391, + "theoretical_loss": 3.8612433035695672, + "tokens_seen": 610795520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585035105315948, + "loss": 1.4072, + "theoretical_loss": 3.861200528949138, + "tokens_seen": 610861056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006584874623871616, + "loss": 1.378, + "theoretical_loss": 3.8611577602023006, + "tokens_seen": 610926592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006584714142427282, + "loss": 1.4051, + "theoretical_loss": 3.8611149973276193, + "tokens_seen": 610992128 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658455366098295, + "loss": 1.3886, + "theoretical_loss": 3.861072240323658, + "tokens_seen": 611057664 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006584393179538616, + "loss": 1.4285, + "theoretical_loss": 3.861029489188981, + "tokens_seen": 611123200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006584232698094284, + "loss": 1.3538, + "theoretical_loss": 3.8609867439221537, + "tokens_seen": 611188736 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658407221664995, + "loss": 1.4264, + "theoretical_loss": 3.860944004521742, + "tokens_seen": 611254272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583911735205618, + "loss": 1.4412, + "theoretical_loss": 3.860901270986311, + "tokens_seen": 611319808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583751253761284, + "loss": 1.4681, + "theoretical_loss": 3.8608585433144276, + "tokens_seen": 611385344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583590772316951, + "loss": 1.4123, + "theoretical_loss": 3.860815821504659, + "tokens_seen": 611450880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583430290872618, + "loss": 1.3016, + "theoretical_loss": 3.860773105555573, + "tokens_seen": 611516416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583269809428285, + "loss": 1.4889, + "theoretical_loss": 3.860730395465737, + "tokens_seen": 611581952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583109327983952, + "loss": 1.3215, + "theoretical_loss": 3.86068769123372, + "tokens_seen": 611647488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582948846539619, + "loss": 1.4208, + "theoretical_loss": 3.860644992858091, + "tokens_seen": 611713024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582788365095286, + "loss": 1.459, + "theoretical_loss": 3.8606023003374195, + "tokens_seen": 611778560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582627883650953, + "loss": 1.3311, + "theoretical_loss": 3.860559613670276, + "tokens_seen": 611844096 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658246740220662, + "loss": 1.4441, + "theoretical_loss": 3.86051693285523, + "tokens_seen": 611909632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582306920762287, + "loss": 1.3947, + "theoretical_loss": 3.8604742578908535, + "tokens_seen": 611975168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582146439317954, + "loss": 1.4688, + "theoretical_loss": 3.860431588775718, + "tokens_seen": 612040704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581985957873621, + "loss": 1.4368, + "theoretical_loss": 3.860388925508395, + "tokens_seen": 612106240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581825476429288, + "loss": 1.397, + "theoretical_loss": 3.8603462680874583, + "tokens_seen": 612171776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581664994984955, + "loss": 1.4094, + "theoretical_loss": 3.8603036165114797, + "tokens_seen": 612237312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581504513540622, + "loss": 1.5308, + "theoretical_loss": 3.8602609707790334, + "tokens_seen": 612302848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581344032096289, + "loss": 1.4203, + "theoretical_loss": 3.860218330888693, + "tokens_seen": 612368384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581183550651956, + "loss": 1.4991, + "theoretical_loss": 3.8601756968390335, + "tokens_seen": 612433920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581023069207623, + "loss": 1.4507, + "theoretical_loss": 3.86013306862863, + "tokens_seen": 612499456 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658086258776329, + "loss": 1.3386, + "theoretical_loss": 3.8600904462560583, + "tokens_seen": 612564992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580702106318957, + "loss": 1.5372, + "theoretical_loss": 3.8600478297198944, + "tokens_seen": 612630528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580541624874624, + "loss": 1.551, + "theoretical_loss": 3.860005219018714, + "tokens_seen": 612696064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 210170, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.420013427734375, + "objective/train/theoretical_loss": 3.8599626141510948, + "objective/train/tokens_used": 633221600, + "theoretical_loss": 3.8599626141510948, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580381143430291, + "loss": 1.4537, + "theoretical_loss": 3.8599626141510948, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580220661985958, + "loss": 1.4975, + "theoretical_loss": 3.859920015115615, + "tokens_seen": 612827136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580060180541625, + "loss": 1.5048, + "theoretical_loss": 3.8598774219108516, + "tokens_seen": 612892672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579899699097292, + "loss": 1.4598, + "theoretical_loss": 3.859834834535384, + "tokens_seen": 612958208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579739217652959, + "loss": 1.5072, + "theoretical_loss": 3.859792252987791, + "tokens_seen": 613023744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579578736208626, + "loss": 1.5103, + "theoretical_loss": 3.859749677266652, + "tokens_seen": 613089280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579418254764293, + "loss": 1.3005, + "theoretical_loss": 3.8597071073705473, + "tokens_seen": 613154816 + }, + { + "epoch": 0.19, + "learning_rate": 0.000657925777331996, + "loss": 1.4161, + "theoretical_loss": 3.8596645432980567, + "tokens_seen": 613220352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579097291875627, + "loss": 1.415, + "theoretical_loss": 3.859621985047763, + "tokens_seen": 613285888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578936810431294, + "loss": 1.4236, + "theoretical_loss": 3.859579432618246, + "tokens_seen": 613351424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578776328986961, + "loss": 1.4362, + "theoretical_loss": 3.8595368860080894, + "tokens_seen": 613416960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578615847542628, + "loss": 1.3682, + "theoretical_loss": 3.8594943452158743, + "tokens_seen": 613482496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578455366098295, + "loss": 1.3953, + "theoretical_loss": 3.8594518102401842, + "tokens_seen": 613548032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578294884653962, + "loss": 1.3524, + "theoretical_loss": 3.8594092810796026, + "tokens_seen": 613613568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578134403209629, + "loss": 1.4545, + "theoretical_loss": 3.859366757732714, + "tokens_seen": 613679104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577973921765296, + "loss": 1.3618, + "theoretical_loss": 3.8593242401981027, + "tokens_seen": 613744640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577813440320963, + "loss": 1.4137, + "theoretical_loss": 3.859281728474354, + "tokens_seen": 613810176 + }, + { + "epoch": 0.19, + "learning_rate": 0.000657765295887663, + "loss": 1.4826, + "theoretical_loss": 3.8592392225600527, + "tokens_seen": 613875712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577492477432297, + "loss": 1.5189, + "theoretical_loss": 3.859196722453785, + "tokens_seen": 613941248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577331995987964, + "loss": 1.4141, + "theoretical_loss": 3.859154228154138, + "tokens_seen": 614006784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577171514543631, + "loss": 1.3756, + "theoretical_loss": 3.8591117396596983, + "tokens_seen": 614072320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577011033099298, + "loss": 1.3612, + "theoretical_loss": 3.859069256969054, + "tokens_seen": 614137856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576850551654965, + "loss": 1.3814, + "theoretical_loss": 3.859026780080792, + "tokens_seen": 614203392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576690070210633, + "loss": 1.4247, + "theoretical_loss": 3.8589843089935014, + "tokens_seen": 614268928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576529588766299, + "loss": 1.4333, + "theoretical_loss": 3.8589418437057716, + "tokens_seen": 614334464 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576369107321967, + "loss": 1.4784, + "theoretical_loss": 3.8588993842161923, + "tokens_seen": 614400000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576208625877633, + "loss": 1.4326, + "theoretical_loss": 3.858856930523352, + "tokens_seen": 614465536 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065760481444333, + "loss": 1.5097, + "theoretical_loss": 3.8588144826258426, + "tokens_seen": 614531072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575887662988967, + "loss": 1.3983, + "theoretical_loss": 3.8587720405222545, + "tokens_seen": 614596608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575727181544635, + "loss": 1.3622, + "theoretical_loss": 3.8587296042111796, + "tokens_seen": 614662144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575566700100301, + "loss": 1.5156, + "theoretical_loss": 3.8586871736912087, + "tokens_seen": 614727680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575406218655969, + "loss": 1.3586, + "theoretical_loss": 3.858644748960936, + "tokens_seen": 614793216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575245737211635, + "loss": 1.5279, + "theoretical_loss": 3.858602330018953, + "tokens_seen": 614858752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575085255767303, + "loss": 1.5332, + "theoretical_loss": 3.858559916863853, + "tokens_seen": 614924288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574924774322969, + "loss": 1.4379, + "theoretical_loss": 3.8585175094942317, + "tokens_seen": 614989824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574764292878637, + "loss": 1.5811, + "theoretical_loss": 3.8584751079086814, + "tokens_seen": 615055360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574603811434303, + "loss": 1.5058, + "theoretical_loss": 3.8584327121057984, + "tokens_seen": 615120896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574443329989971, + "loss": 1.4936, + "theoretical_loss": 3.858390322084178, + "tokens_seen": 615186432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574282848545637, + "loss": 1.4568, + "theoretical_loss": 3.8583479378424155, + "tokens_seen": 615251968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574122367101305, + "loss": 1.4072, + "theoretical_loss": 3.858305559379107, + "tokens_seen": 615317504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573961885656971, + "loss": 1.5034, + "theoretical_loss": 3.8582631866928505, + "tokens_seen": 615383040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573801404212639, + "loss": 1.4569, + "theoretical_loss": 3.858220819782243, + "tokens_seen": 615448576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573640922768305, + "loss": 1.4396, + "theoretical_loss": 3.858178458645882, + "tokens_seen": 615514112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573480441323973, + "loss": 1.4184, + "theoretical_loss": 3.858136103282366, + "tokens_seen": 615579648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573319959879639, + "loss": 1.4304, + "theoretical_loss": 3.8580937536902935, + "tokens_seen": 615645184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573159478435307, + "loss": 1.4899, + "theoretical_loss": 3.8580514098682643, + "tokens_seen": 615710720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572998996990973, + "loss": 1.5058, + "theoretical_loss": 3.8580090718148776, + "tokens_seen": 615776256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572838515546641, + "loss": 1.3675, + "theoretical_loss": 3.8579667395287345, + "tokens_seen": 615841792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572678034102307, + "loss": 1.5073, + "theoretical_loss": 3.8579244130084356, + "tokens_seen": 615907328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572517552657975, + "loss": 1.3504, + "theoretical_loss": 3.8578820922525816, + "tokens_seen": 615972864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 211365, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4504954814910889, + "objective/train/theoretical_loss": 3.857839777259775, + "objective/train/tokens_used": 636498400, + "theoretical_loss": 3.857839777259775, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572357071213641, + "loss": 1.4745, + "theoretical_loss": 3.857839777259775, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572196589769309, + "loss": 1.4646, + "theoretical_loss": 3.857797468028618, + "tokens_seen": 616103936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572036108324975, + "loss": 1.4291, + "theoretical_loss": 3.8577551645577124, + "tokens_seen": 616169472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571875626880643, + "loss": 1.4413, + "theoretical_loss": 3.857712866845662, + "tokens_seen": 616235008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571715145436309, + "loss": 1.417, + "theoretical_loss": 3.857670574891071, + "tokens_seen": 616300544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571554663991977, + "loss": 1.469, + "theoretical_loss": 3.857628288692543, + "tokens_seen": 616366080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571394182547643, + "loss": 1.4186, + "theoretical_loss": 3.857586008248683, + "tokens_seen": 616431616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571233701103311, + "loss": 1.5126, + "theoretical_loss": 3.8575437335580958, + "tokens_seen": 616497152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571073219658977, + "loss": 1.418, + "theoretical_loss": 3.8575014646193875, + "tokens_seen": 616562688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570912738214645, + "loss": 1.4638, + "theoretical_loss": 3.8574592014311637, + "tokens_seen": 616628224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570752256770311, + "loss": 1.4218, + "theoretical_loss": 3.857416943992032, + "tokens_seen": 616693760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570591775325979, + "loss": 1.5737, + "theoretical_loss": 3.8573746923005983, + "tokens_seen": 616759296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570431293881645, + "loss": 1.5094, + "theoretical_loss": 3.857332446355471, + "tokens_seen": 616824832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570270812437313, + "loss": 1.3835, + "theoretical_loss": 3.857290206155258, + "tokens_seen": 616890368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570110330992979, + "loss": 1.3138, + "theoretical_loss": 3.8572479716985675, + "tokens_seen": 616955904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569949849548647, + "loss": 1.416, + "theoretical_loss": 3.857205742984009, + "tokens_seen": 617021440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569789368104313, + "loss": 1.3857, + "theoretical_loss": 3.857163520010192, + "tokens_seen": 617086976 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656962888665998, + "loss": 1.4585, + "theoretical_loss": 3.8571213027757274, + "tokens_seen": 617152512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569468405215647, + "loss": 1.3681, + "theoretical_loss": 3.857079091279224, + "tokens_seen": 617218048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569307923771314, + "loss": 1.4211, + "theoretical_loss": 3.857036885519293, + "tokens_seen": 617283584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569147442326981, + "loss": 1.4748, + "theoretical_loss": 3.8569946854945467, + "tokens_seen": 617349120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568986960882648, + "loss": 1.4468, + "theoretical_loss": 3.8569524912035975, + "tokens_seen": 617414656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568826479438315, + "loss": 1.4331, + "theoretical_loss": 3.8569103026450566, + "tokens_seen": 617480192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568665997993982, + "loss": 1.4306, + "theoretical_loss": 3.8568681198175376, + "tokens_seen": 617545728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568505516549649, + "loss": 1.5051, + "theoretical_loss": 3.856825942719654, + "tokens_seen": 617611264 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568345035105316, + "loss": 1.4273, + "theoretical_loss": 3.856783771350019, + "tokens_seen": 617676800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568184553660983, + "loss": 1.4189, + "theoretical_loss": 3.8567416057072474, + "tokens_seen": 617742336 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656802407221665, + "loss": 1.39, + "theoretical_loss": 3.8566994457899546, + "tokens_seen": 617807872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567863590772317, + "loss": 1.4286, + "theoretical_loss": 3.8566572915967554, + "tokens_seen": 617873408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567703109327984, + "loss": 1.449, + "theoretical_loss": 3.8566151431262656, + "tokens_seen": 617938944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567542627883651, + "loss": 1.4484, + "theoretical_loss": 3.856573000377101, + "tokens_seen": 618004480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567382146439318, + "loss": 1.4595, + "theoretical_loss": 3.8565308633478796, + "tokens_seen": 618070016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567221664994985, + "loss": 1.543, + "theoretical_loss": 3.856488732037218, + "tokens_seen": 618135552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567061183550652, + "loss": 1.4479, + "theoretical_loss": 3.856446606443734, + "tokens_seen": 618201088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566900702106319, + "loss": 1.404, + "theoretical_loss": 3.8564044865660456, + "tokens_seen": 618266624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566740220661986, + "loss": 1.3884, + "theoretical_loss": 3.8563623724027716, + "tokens_seen": 618332160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566579739217653, + "loss": 1.5086, + "theoretical_loss": 3.856320263952531, + "tokens_seen": 618397696 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656641925777332, + "loss": 1.4269, + "theoretical_loss": 3.856278161213944, + "tokens_seen": 618463232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566258776328987, + "loss": 1.4405, + "theoretical_loss": 3.856236064185631, + "tokens_seen": 618528768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566098294884654, + "loss": 1.4679, + "theoretical_loss": 3.856193972866212, + "tokens_seen": 618594304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565937813440321, + "loss": 1.4031, + "theoretical_loss": 3.8561518872543075, + "tokens_seen": 618659840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565777331995988, + "loss": 1.3028, + "theoretical_loss": 3.85610980734854, + "tokens_seen": 618725376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565616850551655, + "loss": 1.4489, + "theoretical_loss": 3.8560677331475315, + "tokens_seen": 618790912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565456369107322, + "loss": 1.5921, + "theoretical_loss": 3.8560256646499047, + "tokens_seen": 618856448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565295887662989, + "loss": 1.418, + "theoretical_loss": 3.855983601854282, + "tokens_seen": 618921984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565135406218656, + "loss": 1.5715, + "theoretical_loss": 3.8559415447592875, + "tokens_seen": 618987520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564974924774323, + "loss": 1.4975, + "theoretical_loss": 3.8558994933635447, + "tokens_seen": 619053056 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656481444332999, + "loss": 1.4568, + "theoretical_loss": 3.855857447665678, + "tokens_seen": 619118592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564653961885657, + "loss": 1.4231, + "theoretical_loss": 3.855815407664313, + "tokens_seen": 619184128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564493480441324, + "loss": 1.4654, + "theoretical_loss": 3.8557733733580744, + "tokens_seen": 619249664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 212287, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3307487964630127, + "objective/train/theoretical_loss": 3.855731344745589, + "objective/train/tokens_used": 639775200, + "theoretical_loss": 3.855731344745589, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564332998996991, + "loss": 1.3697, + "theoretical_loss": 3.855731344745589, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564172517552658, + "loss": 1.3096, + "theoretical_loss": 3.8556893218254817, + "tokens_seen": 619380736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564012036108325, + "loss": 1.4491, + "theoretical_loss": 3.855647304596381, + "tokens_seen": 619446272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006563851554663992, + "loss": 1.5931, + "theoretical_loss": 3.8556052930569127, + "tokens_seen": 619511808 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656369107321966, + "loss": 1.4521, + "theoretical_loss": 3.8555632872057055, + "tokens_seen": 619577344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006563530591775326, + "loss": 1.4756, + "theoretical_loss": 3.8555212870413875, + "tokens_seen": 619642880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006563370110330993, + "loss": 1.3477, + "theoretical_loss": 3.8554792925625874, + "tokens_seen": 619708416 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656320962888666, + "loss": 1.4574, + "theoretical_loss": 3.855437303767934, + "tokens_seen": 619773952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006563049147442327, + "loss": 1.4968, + "theoretical_loss": 3.855395320656058, + "tokens_seen": 619839488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562888665997994, + "loss": 1.4919, + "theoretical_loss": 3.8553533432255884, + "tokens_seen": 619905024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562728184553662, + "loss": 1.4488, + "theoretical_loss": 3.855311371475157, + "tokens_seen": 619970560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562567703109328, + "loss": 1.4551, + "theoretical_loss": 3.8552694054033942, + "tokens_seen": 620036096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562407221664996, + "loss": 1.5415, + "theoretical_loss": 3.855227445008931, + "tokens_seen": 620101632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562246740220662, + "loss": 1.4233, + "theoretical_loss": 3.8551854902904017, + "tokens_seen": 620167168 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656208625877633, + "loss": 1.44, + "theoretical_loss": 3.855143541246436, + "tokens_seen": 620232704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561925777331996, + "loss": 1.3843, + "theoretical_loss": 3.855101597875669, + "tokens_seen": 620298240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561765295887664, + "loss": 1.511, + "theoretical_loss": 3.8550596601767326, + "tokens_seen": 620363776 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656160481444333, + "loss": 1.4461, + "theoretical_loss": 3.8550177281482627, + "tokens_seen": 620429312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561444332998998, + "loss": 1.3921, + "theoretical_loss": 3.8549758017888918, + "tokens_seen": 620494848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561283851554664, + "loss": 1.3798, + "theoretical_loss": 3.8549338810972564, + "tokens_seen": 620560384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561123370110332, + "loss": 1.4377, + "theoretical_loss": 3.8548919660719907, + "tokens_seen": 620625920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560962888665998, + "loss": 1.333, + "theoretical_loss": 3.854850056711731, + "tokens_seen": 620691456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560802407221666, + "loss": 1.5228, + "theoretical_loss": 3.8548081530151137, + "tokens_seen": 620756992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560641925777332, + "loss": 1.5282, + "theoretical_loss": 3.8547662549807757, + "tokens_seen": 620822528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560481444333, + "loss": 1.3841, + "theoretical_loss": 3.854724362607354, + "tokens_seen": 620888064 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560320962888666, + "loss": 1.4249, + "theoretical_loss": 3.854682475893486, + "tokens_seen": 620953600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560160481444334, + "loss": 1.4984, + "theoretical_loss": 3.854640594837811, + "tokens_seen": 621019136 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656, + "loss": 1.5547, + "theoretical_loss": 3.8545987194389664, + "tokens_seen": 621084672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559839518555668, + "loss": 1.4241, + "theoretical_loss": 3.8545568496955926, + "tokens_seen": 621150208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559679037111334, + "loss": 1.6195, + "theoretical_loss": 3.854514985606328, + "tokens_seen": 621215744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559518555667002, + "loss": 1.3812, + "theoretical_loss": 3.854473127169814, + "tokens_seen": 621281280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559358074222668, + "loss": 1.3515, + "theoretical_loss": 3.85443127438469, + "tokens_seen": 621346816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559197592778336, + "loss": 1.4354, + "theoretical_loss": 3.8543894272495978, + "tokens_seen": 621412352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559037111334002, + "loss": 1.4532, + "theoretical_loss": 3.8543475857631786, + "tokens_seen": 621477888 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655887662988967, + "loss": 1.4251, + "theoretical_loss": 3.854305749924075, + "tokens_seen": 621543424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558716148445336, + "loss": 1.5207, + "theoretical_loss": 3.854263919730928, + "tokens_seen": 621608960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558555667001004, + "loss": 1.4856, + "theoretical_loss": 3.8542220951823816, + "tokens_seen": 621674496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558395185556671, + "loss": 1.5301, + "theoretical_loss": 3.854180276277079, + "tokens_seen": 621740032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558234704112338, + "loss": 1.4044, + "theoretical_loss": 3.8541384630136646, + "tokens_seen": 621805568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558074222668005, + "loss": 1.3659, + "theoretical_loss": 3.854096655390781, + "tokens_seen": 621871104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557913741223672, + "loss": 1.4789, + "theoretical_loss": 3.8540548534070753, + "tokens_seen": 621936640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557753259779339, + "loss": 1.3902, + "theoretical_loss": 3.854013057061191, + "tokens_seen": 622002176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557592778335006, + "loss": 1.5189, + "theoretical_loss": 3.853971266351775, + "tokens_seen": 622067712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557432296890673, + "loss": 1.4863, + "theoretical_loss": 3.853929481277472, + "tokens_seen": 622133248 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655727181544634, + "loss": 1.5079, + "theoretical_loss": 3.8538877018369306, + "tokens_seen": 622198784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557111334002006, + "loss": 1.387, + "theoretical_loss": 3.8538459280287967, + "tokens_seen": 622264320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556950852557674, + "loss": 1.4602, + "theoretical_loss": 3.853804159851718, + "tokens_seen": 622329856 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655679037111334, + "loss": 1.5986, + "theoretical_loss": 3.853762397304343, + "tokens_seen": 622395392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556629889669008, + "loss": 1.4307, + "theoretical_loss": 3.853720640385319, + "tokens_seen": 622460928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556469408224674, + "loss": 1.4352, + "theoretical_loss": 3.853678889093297, + "tokens_seen": 622526464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 213453, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.5051805973052979, + "objective/train/theoretical_loss": 3.8536371434269254, + "objective/train/tokens_used": 643052000, + "theoretical_loss": 3.8536371434269254, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556308926780342, + "loss": 1.5377, + "theoretical_loss": 3.8536371434269254, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556148445336008, + "loss": 1.4414, + "theoretical_loss": 3.853595403384854, + "tokens_seen": 622657536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555987963891676, + "loss": 1.5108, + "theoretical_loss": 3.853553668965733, + "tokens_seen": 622723072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555827482447342, + "loss": 1.486, + "theoretical_loss": 3.8535119401682136, + "tokens_seen": 622788608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555667001003009, + "loss": 1.4433, + "theoretical_loss": 3.8534702169909476, + "tokens_seen": 622854144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555506519558676, + "loss": 1.4521, + "theoretical_loss": 3.853428499432586, + "tokens_seen": 622919680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555346038114343, + "loss": 1.4889, + "theoretical_loss": 3.8533867874917815, + "tokens_seen": 622985216 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655518555667001, + "loss": 1.395, + "theoretical_loss": 3.8533450811671877, + "tokens_seen": 623050752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555025075225677, + "loss": 1.3459, + "theoretical_loss": 3.8533033804574557, + "tokens_seen": 623116288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554864593781344, + "loss": 1.3917, + "theoretical_loss": 3.8532616853612405, + "tokens_seen": 623181824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554704112337011, + "loss": 1.5538, + "theoretical_loss": 3.853219995877196, + "tokens_seen": 623247360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554543630892678, + "loss": 1.418, + "theoretical_loss": 3.853178312003977, + "tokens_seen": 623312896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554383149448345, + "loss": 1.491, + "theoretical_loss": 3.853136633740239, + "tokens_seen": 623378432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554222668004012, + "loss": 1.5011, + "theoretical_loss": 3.853094961084636, + "tokens_seen": 623443968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554062186559679, + "loss": 1.503, + "theoretical_loss": 3.853053294035825, + "tokens_seen": 623509504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553901705115346, + "loss": 1.4062, + "theoretical_loss": 3.8530116325924624, + "tokens_seen": 623575040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553741223671013, + "loss": 1.479, + "theoretical_loss": 3.8529699767532053, + "tokens_seen": 623640576 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655358074222668, + "loss": 1.406, + "theoretical_loss": 3.85292832651671, + "tokens_seen": 623706112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553420260782347, + "loss": 1.4656, + "theoretical_loss": 3.8528866818816354, + "tokens_seen": 623771648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553259779338014, + "loss": 1.4556, + "theoretical_loss": 3.8528450428466403, + "tokens_seen": 623837184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553099297893681, + "loss": 1.3951, + "theoretical_loss": 3.852803409410382, + "tokens_seen": 623902720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552938816449348, + "loss": 1.6086, + "theoretical_loss": 3.85276178157152, + "tokens_seen": 623968256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552778335005015, + "loss": 1.5793, + "theoretical_loss": 3.852720159328715, + "tokens_seen": 624033792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552617853560682, + "loss": 1.5246, + "theoretical_loss": 3.852678542680626, + "tokens_seen": 624099328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552457372116349, + "loss": 1.4312, + "theoretical_loss": 3.8526369316259137, + "tokens_seen": 624164864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552296890672016, + "loss": 1.3875, + "theoretical_loss": 3.8525953261632404, + "tokens_seen": 624230400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552136409227683, + "loss": 1.5285, + "theoretical_loss": 3.852553726291266, + "tokens_seen": 624295936 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655197592778335, + "loss": 1.4572, + "theoretical_loss": 3.8525121320086537, + "tokens_seen": 624361472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551815446339017, + "loss": 1.4146, + "theoretical_loss": 3.8524705433140656, + "tokens_seen": 624427008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551654964894684, + "loss": 1.4862, + "theoretical_loss": 3.8524289602061645, + "tokens_seen": 624492544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551494483450351, + "loss": 1.4295, + "theoretical_loss": 3.8523873826836135, + "tokens_seen": 624558080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551334002006018, + "loss": 1.3939, + "theoretical_loss": 3.8523458107450765, + "tokens_seen": 624623616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551173520561685, + "loss": 1.5243, + "theoretical_loss": 3.8523042443892184, + "tokens_seen": 624689152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551013039117352, + "loss": 1.4286, + "theoretical_loss": 3.852262683614703, + "tokens_seen": 624754688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550852557673019, + "loss": 1.5194, + "theoretical_loss": 3.8522211284201964, + "tokens_seen": 624820224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550692076228686, + "loss": 1.4788, + "theoretical_loss": 3.852179578804364, + "tokens_seen": 624885760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550531594784353, + "loss": 1.404, + "theoretical_loss": 3.852138034765872, + "tokens_seen": 624951296 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655037111334002, + "loss": 1.3126, + "theoretical_loss": 3.8520964963033864, + "tokens_seen": 625016832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550210631895687, + "loss": 1.3926, + "theoretical_loss": 3.8520549634155747, + "tokens_seen": 625082368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550050150451354, + "loss": 1.4594, + "theoretical_loss": 3.852013436101104, + "tokens_seen": 625147904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549889669007021, + "loss": 1.5262, + "theoretical_loss": 3.851971914358643, + "tokens_seen": 625213440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549729187562688, + "loss": 1.4859, + "theoretical_loss": 3.85193039818686, + "tokens_seen": 625278976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549568706118355, + "loss": 1.4388, + "theoretical_loss": 3.8518888875844226, + "tokens_seen": 625344512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549408224674022, + "loss": 1.4641, + "theoretical_loss": 3.8518473825500017, + "tokens_seen": 625410048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549247743229689, + "loss": 1.4174, + "theoretical_loss": 3.8518058830822666, + "tokens_seen": 625475584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549087261785357, + "loss": 1.4281, + "theoretical_loss": 3.8517643891798867, + "tokens_seen": 625541120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548926780341023, + "loss": 1.4682, + "theoretical_loss": 3.8517229008415343, + "tokens_seen": 625606656 + }, + { + "epoch": 0.19, + "learning_rate": 0.000654876629889669, + "loss": 1.4707, + "theoretical_loss": 3.8516814180658794, + "tokens_seen": 625672192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548605817452357, + "loss": 1.5598, + "theoretical_loss": 3.8516399408515936, + "tokens_seen": 625737728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548445336008025, + "loss": 1.549, + "theoretical_loss": 3.8515984691973495, + "tokens_seen": 625803264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 214373, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3765876293182373, + "objective/train/theoretical_loss": 3.85155700310182, + "objective/train/tokens_used": 646328800, + "theoretical_loss": 3.85155700310182, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548284854563691, + "loss": 1.4324, + "theoretical_loss": 3.85155700310182, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548124373119359, + "loss": 1.4596, + "theoretical_loss": 3.8515155425636767, + "tokens_seen": 625934336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547963891675025, + "loss": 1.373, + "theoretical_loss": 3.851474087581594, + "tokens_seen": 625999872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547803410230693, + "loss": 1.5331, + "theoretical_loss": 3.8514326381542454, + "tokens_seen": 626065408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547642928786359, + "loss": 1.4168, + "theoretical_loss": 3.851391194280305, + "tokens_seen": 626130944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547482447342027, + "loss": 1.4006, + "theoretical_loss": 3.8513497559584495, + "tokens_seen": 626196480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547321965897693, + "loss": 1.4852, + "theoretical_loss": 3.851308323187352, + "tokens_seen": 626262016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547161484453361, + "loss": 1.4906, + "theoretical_loss": 3.851266895965689, + "tokens_seen": 626327552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547001003009027, + "loss": 1.4673, + "theoretical_loss": 3.8512254742921357, + "tokens_seen": 626393088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546840521564695, + "loss": 1.4863, + "theoretical_loss": 3.8511840581653707, + "tokens_seen": 626458624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546680040120361, + "loss": 1.3973, + "theoretical_loss": 3.85114264758407, + "tokens_seen": 626524160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546519558676029, + "loss": 1.3883, + "theoretical_loss": 3.8511012425469104, + "tokens_seen": 626589696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546359077231696, + "loss": 1.4545, + "theoretical_loss": 3.851059843052571, + "tokens_seen": 626655232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546198595787363, + "loss": 1.3686, + "theoretical_loss": 3.85101844909973, + "tokens_seen": 626720768 + }, + { + "epoch": 0.19, + "learning_rate": 0.000654603811434303, + "loss": 1.4696, + "theoretical_loss": 3.8509770606870664, + "tokens_seen": 626786304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545877632898697, + "loss": 1.4343, + "theoretical_loss": 3.8509356778132586, + "tokens_seen": 626851840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545717151454364, + "loss": 1.442, + "theoretical_loss": 3.8508943004769876, + "tokens_seen": 626917376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545556670010031, + "loss": 1.3759, + "theoretical_loss": 3.8508529286769324, + "tokens_seen": 626982912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545396188565698, + "loss": 1.4723, + "theoretical_loss": 3.850811562411775, + "tokens_seen": 627048448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545235707121365, + "loss": 1.3319, + "theoretical_loss": 3.8507702016801963, + "tokens_seen": 627113984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545075225677032, + "loss": 1.4292, + "theoretical_loss": 3.850728846480877, + "tokens_seen": 627179520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544914744232699, + "loss": 1.4316, + "theoretical_loss": 3.8506874968125, + "tokens_seen": 627245056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544754262788366, + "loss": 1.3812, + "theoretical_loss": 3.8506461526737477, + "tokens_seen": 627310592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544593781344033, + "loss": 1.4015, + "theoretical_loss": 3.8506048140633027, + "tokens_seen": 627376128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065444332998997, + "loss": 1.4431, + "theoretical_loss": 3.850563480979849, + "tokens_seen": 627441664 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544272818455367, + "loss": 1.4909, + "theoretical_loss": 3.8505221534220695, + "tokens_seen": 627507200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544112337011034, + "loss": 1.395, + "theoretical_loss": 3.8504808313886496, + "tokens_seen": 627572736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543951855566701, + "loss": 1.4019, + "theoretical_loss": 3.850439514878273, + "tokens_seen": 627638272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543791374122368, + "loss": 1.5165, + "theoretical_loss": 3.8503982038896263, + "tokens_seen": 627703808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543630892678035, + "loss": 1.3704, + "theoretical_loss": 3.850356898421394, + "tokens_seen": 627769344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543470411233702, + "loss": 1.3597, + "theoretical_loss": 3.850315598472262, + "tokens_seen": 627834880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543309929789369, + "loss": 1.4165, + "theoretical_loss": 3.8502743040409184, + "tokens_seen": 627900416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543149448345035, + "loss": 1.4233, + "theoretical_loss": 3.850233015126048, + "tokens_seen": 627965952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542988966900703, + "loss": 1.3924, + "theoretical_loss": 3.8501917317263405, + "tokens_seen": 628031488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542828485456369, + "loss": 1.4651, + "theoretical_loss": 3.8501504538404827, + "tokens_seen": 628097024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542668004012037, + "loss": 1.4985, + "theoretical_loss": 3.850109181467163, + "tokens_seen": 628162560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542507522567703, + "loss": 1.4034, + "theoretical_loss": 3.85006791460507, + "tokens_seen": 628228096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542347041123371, + "loss": 1.4475, + "theoretical_loss": 3.850026653252894, + "tokens_seen": 628293632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542186559679037, + "loss": 1.4138, + "theoretical_loss": 3.8499853974093234, + "tokens_seen": 628359168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542026078234705, + "loss": 1.3889, + "theoretical_loss": 3.8499441470730487, + "tokens_seen": 628424704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541865596790371, + "loss": 1.3595, + "theoretical_loss": 3.849902902242761, + "tokens_seen": 628490240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541705115346038, + "loss": 1.4252, + "theoretical_loss": 3.849861662917151, + "tokens_seen": 628555776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541544633901705, + "loss": 1.5239, + "theoretical_loss": 3.8498204290949105, + "tokens_seen": 628621312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541384152457372, + "loss": 1.3816, + "theoretical_loss": 3.849779200774731, + "tokens_seen": 628686848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541223671013039, + "loss": 1.5023, + "theoretical_loss": 3.8497379779553054, + "tokens_seen": 628752384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541063189568706, + "loss": 1.518, + "theoretical_loss": 3.849696760635326, + "tokens_seen": 628817920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540902708124373, + "loss": 1.4259, + "theoretical_loss": 3.8496555488134865, + "tokens_seen": 628883456 + }, + { + "epoch": 0.19, + "learning_rate": 0.000654074222668004, + "loss": 1.534, + "theoretical_loss": 3.84961434248848, + "tokens_seen": 628948992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540581745235707, + "loss": 1.4677, + "theoretical_loss": 3.8495731416590018, + "tokens_seen": 629014528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540421263791374, + "loss": 1.4557, + "theoretical_loss": 3.8495319463237454, + "tokens_seen": 629080064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 215608, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4194252490997314, + "objective/train/theoretical_loss": 3.8494907564814067, + "objective/train/tokens_used": 649605600, + "theoretical_loss": 3.8494907564814067, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540260782347041, + "loss": 1.4274, + "theoretical_loss": 3.8494907564814067, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540100300902708, + "loss": 1.487, + "theoretical_loss": 3.8494495721306805, + "tokens_seen": 629211136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539939819458375, + "loss": 1.5867, + "theoretical_loss": 3.849408393270264, + "tokens_seen": 629276672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539779338014042, + "loss": 1.3724, + "theoretical_loss": 3.849367219898852, + "tokens_seen": 629342208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539618856569709, + "loss": 1.4578, + "theoretical_loss": 3.849326052015142, + "tokens_seen": 629407744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539458375125376, + "loss": 1.5947, + "theoretical_loss": 3.8492848896178318, + "tokens_seen": 629473280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539297893681043, + "loss": 1.4659, + "theoretical_loss": 3.8492437327056184, + "tokens_seen": 629538816 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653913741223671, + "loss": 1.4527, + "theoretical_loss": 3.8492025812772006, + "tokens_seen": 629604352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538976930792377, + "loss": 1.4258, + "theoretical_loss": 3.849161435331277, + "tokens_seen": 629669888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538816449348044, + "loss": 1.429, + "theoretical_loss": 3.849120294866546, + "tokens_seen": 629735424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538655967903711, + "loss": 1.4194, + "theoretical_loss": 3.8490791598817076, + "tokens_seen": 629800960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538495486459378, + "loss": 1.5264, + "theoretical_loss": 3.8490380303754623, + "tokens_seen": 629866496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538335005015045, + "loss": 1.3783, + "theoretical_loss": 3.8489969063465095, + "tokens_seen": 629932032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538174523570712, + "loss": 1.4903, + "theoretical_loss": 3.8489557877935505, + "tokens_seen": 629997568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538014042126379, + "loss": 1.5286, + "theoretical_loss": 3.848914674715287, + "tokens_seen": 630063104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537853560682046, + "loss": 1.4062, + "theoretical_loss": 3.84887356711042, + "tokens_seen": 630128640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537693079237713, + "loss": 1.4923, + "theoretical_loss": 3.8488324649776517, + "tokens_seen": 630194176 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653753259779338, + "loss": 1.3488, + "theoretical_loss": 3.848791368315686, + "tokens_seen": 630259712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537372116349047, + "loss": 1.5031, + "theoretical_loss": 3.848750277123224, + "tokens_seen": 630325248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537211634904714, + "loss": 1.4806, + "theoretical_loss": 3.848709191398971, + "tokens_seen": 630390784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537051153460381, + "loss": 1.3267, + "theoretical_loss": 3.8486681111416297, + "tokens_seen": 630456320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536890672016048, + "loss": 1.4047, + "theoretical_loss": 3.848627036349905, + "tokens_seen": 630521856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536730190571715, + "loss": 1.4194, + "theoretical_loss": 3.848585967022502, + "tokens_seen": 630587392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536569709127382, + "loss": 1.408, + "theoretical_loss": 3.8485449031581256, + "tokens_seen": 630652928 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653640922768305, + "loss": 1.3323, + "theoretical_loss": 3.8485038447554816, + "tokens_seen": 630718464 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536248746238716, + "loss": 1.5932, + "theoretical_loss": 3.848462791813276, + "tokens_seen": 630784000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536088264794383, + "loss": 1.4501, + "theoretical_loss": 3.8484217443302153, + "tokens_seen": 630849536 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653592778335005, + "loss": 1.5799, + "theoretical_loss": 3.8483807023050076, + "tokens_seen": 630915072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535767301905717, + "loss": 1.4477, + "theoretical_loss": 3.848339665736359, + "tokens_seen": 630980608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535606820461384, + "loss": 1.3818, + "theoretical_loss": 3.8482986346229784, + "tokens_seen": 631046144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535446339017052, + "loss": 1.4074, + "theoretical_loss": 3.8482576089635736, + "tokens_seen": 631111680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535285857572718, + "loss": 1.5077, + "theoretical_loss": 3.848216588756854, + "tokens_seen": 631177216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535125376128386, + "loss": 1.4928, + "theoretical_loss": 3.8481755740015275, + "tokens_seen": 631242752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534964894684052, + "loss": 1.4117, + "theoretical_loss": 3.848134564696305, + "tokens_seen": 631308288 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653480441323972, + "loss": 1.4229, + "theoretical_loss": 3.8480935608398967, + "tokens_seen": 631373824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534643931795386, + "loss": 1.3704, + "theoretical_loss": 3.8480525624310125, + "tokens_seen": 631439360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534483450351054, + "loss": 1.4856, + "theoretical_loss": 3.8480115694683636, + "tokens_seen": 631504896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534322968906721, + "loss": 1.4555, + "theoretical_loss": 3.847970581950662, + "tokens_seen": 631570432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534162487462388, + "loss": 1.5628, + "theoretical_loss": 3.8479295998766188, + "tokens_seen": 631635968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534002006018055, + "loss": 1.393, + "theoretical_loss": 3.8478886232449465, + "tokens_seen": 631701504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533841524573722, + "loss": 1.4871, + "theoretical_loss": 3.847847652054358, + "tokens_seen": 631767040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533681043129389, + "loss": 1.4766, + "theoretical_loss": 3.8478066863035663, + "tokens_seen": 631832576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533520561685056, + "loss": 1.4443, + "theoretical_loss": 3.8477657259912856, + "tokens_seen": 631898112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533360080240723, + "loss": 1.4973, + "theoretical_loss": 3.847724771116229, + "tokens_seen": 631963648 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653319959879639, + "loss": 1.4456, + "theoretical_loss": 3.847683821677112, + "tokens_seen": 632029184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533039117352057, + "loss": 1.4088, + "theoretical_loss": 3.847642877672649, + "tokens_seen": 632094720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532878635907724, + "loss": 1.4612, + "theoretical_loss": 3.847601939101555, + "tokens_seen": 632160256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532718154463391, + "loss": 1.5592, + "theoretical_loss": 3.8475610059625467, + "tokens_seen": 632225792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532557673019058, + "loss": 1.484, + "theoretical_loss": 3.84752007825434, + "tokens_seen": 632291328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532397191574725, + "loss": 1.384, + "theoretical_loss": 3.8474791559756514, + "tokens_seen": 632356864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 216545, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.624903678894043, + "objective/train/theoretical_loss": 3.8474382391251982, + "objective/train/tokens_used": 652882400, + "theoretical_loss": 3.8474382391251982, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532236710130392, + "loss": 1.5181, + "theoretical_loss": 3.8474382391251982, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532076228686059, + "loss": 1.3753, + "theoretical_loss": 3.8473973277016977, + "tokens_seen": 632487936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531915747241726, + "loss": 1.4691, + "theoretical_loss": 3.847356421703868, + "tokens_seen": 632553472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531755265797393, + "loss": 1.4561, + "theoretical_loss": 3.8473155211304277, + "tokens_seen": 632619008 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653159478435306, + "loss": 1.507, + "theoretical_loss": 3.847274625980096, + "tokens_seen": 632684544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531434302908727, + "loss": 1.4212, + "theoretical_loss": 3.8472337362515914, + "tokens_seen": 632750080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531273821464394, + "loss": 1.4166, + "theoretical_loss": 3.8471928519436345, + "tokens_seen": 632815616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531113340020061, + "loss": 1.5466, + "theoretical_loss": 3.8471519730549444, + "tokens_seen": 632881152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530952858575728, + "loss": 1.5033, + "theoretical_loss": 3.847111099584242, + "tokens_seen": 632946688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530792377131395, + "loss": 1.443, + "theoretical_loss": 3.8470702315302496, + "tokens_seen": 633012224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530631895687062, + "loss": 1.3511, + "theoretical_loss": 3.847029368891687, + "tokens_seen": 633077760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530471414242729, + "loss": 1.4817, + "theoretical_loss": 3.846988511667277, + "tokens_seen": 633143296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530310932798396, + "loss": 1.3648, + "theoretical_loss": 3.8469476598557417, + "tokens_seen": 633208832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530150451354063, + "loss": 1.353, + "theoretical_loss": 3.8469068134558038, + "tokens_seen": 633274368 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652998996990973, + "loss": 1.3605, + "theoretical_loss": 3.846865972466187, + "tokens_seen": 633339904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529829488465397, + "loss": 1.4737, + "theoretical_loss": 3.846825136885615, + "tokens_seen": 633405440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529669007021064, + "loss": 1.4733, + "theoretical_loss": 3.8467843067128102, + "tokens_seen": 633470976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529508525576731, + "loss": 1.4118, + "theoretical_loss": 3.8467434819464996, + "tokens_seen": 633536512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529348044132398, + "loss": 1.3678, + "theoretical_loss": 3.8467026625854066, + "tokens_seen": 633602048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529187562688064, + "loss": 1.4033, + "theoretical_loss": 3.8466618486282567, + "tokens_seen": 633667584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529187562688064, + "loss": 1.4708, + "theoretical_loss": 3.8466210400737757, + "tokens_seen": 633733120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529027081243732, + "loss": 1.4761, + "theoretical_loss": 3.8465802369206905, + "tokens_seen": 633798656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528866599799398, + "loss": 1.4281, + "theoretical_loss": 3.846539439167727, + "tokens_seen": 633864192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528706118355066, + "loss": 1.5256, + "theoretical_loss": 3.846498646813613, + "tokens_seen": 633929728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528545636910732, + "loss": 1.5862, + "theoretical_loss": 3.8464578598570753, + "tokens_seen": 633995264 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065283851554664, + "loss": 1.442, + "theoretical_loss": 3.8464170782968417, + "tokens_seen": 634060800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528224674022066, + "loss": 1.5478, + "theoretical_loss": 3.8463763021316417, + "tokens_seen": 634126336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528064192577734, + "loss": 1.432, + "theoretical_loss": 3.8463355313602037, + "tokens_seen": 634191872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065279037111334, + "loss": 1.4755, + "theoretical_loss": 3.8462947659812565, + "tokens_seen": 634257408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527743229689067, + "loss": 1.6248, + "theoretical_loss": 3.84625400599353, + "tokens_seen": 634322944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527582748244734, + "loss": 1.3956, + "theoretical_loss": 3.846213251395755, + "tokens_seen": 634388480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527422266800401, + "loss": 1.5417, + "theoretical_loss": 3.846172502186661, + "tokens_seen": 634454016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527261785356068, + "loss": 1.459, + "theoretical_loss": 3.8461317583649794, + "tokens_seen": 634519552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527101303911735, + "loss": 1.4051, + "theoretical_loss": 3.8460910199294416, + "tokens_seen": 634585088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526940822467402, + "loss": 1.447, + "theoretical_loss": 3.8460502868787794, + "tokens_seen": 634650624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526780341023069, + "loss": 1.4647, + "theoretical_loss": 3.8460095592117254, + "tokens_seen": 634716160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526619859578736, + "loss": 1.4076, + "theoretical_loss": 3.845968836927012, + "tokens_seen": 634781696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526459378134403, + "loss": 1.4685, + "theoretical_loss": 3.8459281200233724, + "tokens_seen": 634847232 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652629889669007, + "loss": 1.5148, + "theoretical_loss": 3.8458874084995402, + "tokens_seen": 634912768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526138415245737, + "loss": 1.3509, + "theoretical_loss": 3.8458467023542493, + "tokens_seen": 634978304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525977933801404, + "loss": 1.4074, + "theoretical_loss": 3.845806001586234, + "tokens_seen": 635043840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525817452357071, + "loss": 1.3696, + "theoretical_loss": 3.8457653061942296, + "tokens_seen": 635109376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525656970912738, + "loss": 1.5072, + "theoretical_loss": 3.8457246161769705, + "tokens_seen": 635174912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525496489468405, + "loss": 1.5341, + "theoretical_loss": 3.8456839315331934, + "tokens_seen": 635240448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525336008024072, + "loss": 1.4519, + "theoretical_loss": 3.8456432522616337, + "tokens_seen": 635305984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525175526579739, + "loss": 1.4432, + "theoretical_loss": 3.845602578361029, + "tokens_seen": 635371520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525015045135406, + "loss": 1.5209, + "theoretical_loss": 3.8455619098301144, + "tokens_seen": 635437056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524854563691073, + "loss": 1.3726, + "theoretical_loss": 3.8455212466676287, + "tokens_seen": 635502592 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652469408224674, + "loss": 1.4938, + "theoretical_loss": 3.8454805888723094, + "tokens_seen": 635568128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524533600802407, + "loss": 1.4569, + "theoretical_loss": 3.845439936442895, + "tokens_seen": 635633664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 217757, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.47322678565979, + "objective/train/theoretical_loss": 3.8453992893781246, + "objective/train/tokens_used": 656159200, + "theoretical_loss": 3.8453992893781246, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524373119358074, + "loss": 1.4815, + "theoretical_loss": 3.8453992893781246, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524212637913741, + "loss": 1.4739, + "theoretical_loss": 3.845358647676736, + "tokens_seen": 635764736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524052156469408, + "loss": 1.4726, + "theoretical_loss": 3.8453180113374694, + "tokens_seen": 635830272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523891675025075, + "loss": 1.434, + "theoretical_loss": 3.845277380359065, + "tokens_seen": 635895808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523731193580742, + "loss": 1.5007, + "theoretical_loss": 3.845236754740263, + "tokens_seen": 635961344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523570712136409, + "loss": 1.4894, + "theoretical_loss": 3.8451961344798042, + "tokens_seen": 636026880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523410230692076, + "loss": 1.3507, + "theoretical_loss": 3.84515551957643, + "tokens_seen": 636092416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523249749247743, + "loss": 1.4423, + "theoretical_loss": 3.845114910028882, + "tokens_seen": 636157952 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652308926780341, + "loss": 1.5087, + "theoretical_loss": 3.8450743058359023, + "tokens_seen": 636223488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522928786359077, + "loss": 1.4043, + "theoretical_loss": 3.845033706996233, + "tokens_seen": 636289024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522768304914744, + "loss": 1.3376, + "theoretical_loss": 3.8449931135086177, + "tokens_seen": 636354560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522607823470411, + "loss": 1.4942, + "theoretical_loss": 3.8449525253717995, + "tokens_seen": 636420096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522447342026078, + "loss": 1.354, + "theoretical_loss": 3.844911942584522, + "tokens_seen": 636485632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522286860581746, + "loss": 1.4262, + "theoretical_loss": 3.84487136514553, + "tokens_seen": 636551168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522126379137412, + "loss": 1.5158, + "theoretical_loss": 3.8448307930535672, + "tokens_seen": 636616704 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652196589769308, + "loss": 1.5942, + "theoretical_loss": 3.8447902263073797, + "tokens_seen": 636682240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521805416248747, + "loss": 1.4652, + "theoretical_loss": 3.8447496649057125, + "tokens_seen": 636747776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521644934804414, + "loss": 1.5467, + "theoretical_loss": 3.844709108847311, + "tokens_seen": 636813312 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652148445336008, + "loss": 1.5092, + "theoretical_loss": 3.8446685581309223, + "tokens_seen": 636878848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521323971915748, + "loss": 1.3187, + "theoretical_loss": 3.844628012755293, + "tokens_seen": 636944384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521163490471415, + "loss": 1.4752, + "theoretical_loss": 3.8445874727191702, + "tokens_seen": 637009920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521003009027082, + "loss": 1.3897, + "theoretical_loss": 3.8445469380213013, + "tokens_seen": 637075456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520842527582749, + "loss": 1.5081, + "theoretical_loss": 3.844506408660435, + "tokens_seen": 637140992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520682046138416, + "loss": 1.519, + "theoretical_loss": 3.844465884635319, + "tokens_seen": 637206528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520521564694083, + "loss": 1.4463, + "theoretical_loss": 3.8444253659447023, + "tokens_seen": 637272064 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652036108324975, + "loss": 1.4525, + "theoretical_loss": 3.8443848525873343, + "tokens_seen": 637337600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520200601805417, + "loss": 1.4409, + "theoretical_loss": 3.844344344561965, + "tokens_seen": 637403136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520040120361084, + "loss": 1.4772, + "theoretical_loss": 3.844303841867344, + "tokens_seen": 637468672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519879638916751, + "loss": 1.4162, + "theoretical_loss": 3.8442633445022225, + "tokens_seen": 637534208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519719157472418, + "loss": 1.399, + "theoretical_loss": 3.844222852465351, + "tokens_seen": 637599744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519558676028085, + "loss": 1.4368, + "theoretical_loss": 3.844182365755481, + "tokens_seen": 637665280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519398194583752, + "loss": 1.4406, + "theoretical_loss": 3.844141884371364, + "tokens_seen": 637730816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519237713139419, + "loss": 1.5643, + "theoretical_loss": 3.8441014083117535, + "tokens_seen": 637796352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519077231695086, + "loss": 1.5508, + "theoretical_loss": 3.8440609375754007, + "tokens_seen": 637861888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518916750250753, + "loss": 1.3945, + "theoretical_loss": 3.844020472161059, + "tokens_seen": 637927424 + }, + { + "epoch": 0.19, + "learning_rate": 0.000651875626880642, + "loss": 1.3856, + "theoretical_loss": 3.8439800120674827, + "tokens_seen": 637992960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518595787362087, + "loss": 1.4326, + "theoretical_loss": 3.843939557293425, + "tokens_seen": 638058496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518435305917754, + "loss": 1.5456, + "theoretical_loss": 3.8438991078376405, + "tokens_seen": 638124032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518274824473421, + "loss": 1.4321, + "theoretical_loss": 3.843858663698884, + "tokens_seen": 638189568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518114343029088, + "loss": 1.3336, + "theoretical_loss": 3.84381822487591, + "tokens_seen": 638255104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517953861584755, + "loss": 1.3373, + "theoretical_loss": 3.843777791367475, + "tokens_seen": 638320640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517793380140422, + "loss": 1.4302, + "theoretical_loss": 3.843737363172335, + "tokens_seen": 638386176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517632898696089, + "loss": 1.4137, + "theoretical_loss": 3.843696940289246, + "tokens_seen": 638451712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517472417251756, + "loss": 1.396, + "theoretical_loss": 3.843656522716965, + "tokens_seen": 638517248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517311935807423, + "loss": 1.4537, + "theoretical_loss": 3.843616110454249, + "tokens_seen": 638582784 + }, + { + "epoch": 0.19, + "learning_rate": 0.000651715145436309, + "loss": 1.4281, + "theoretical_loss": 3.843575703499856, + "tokens_seen": 638648320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516990972918757, + "loss": 1.3999, + "theoretical_loss": 3.843535301852544, + "tokens_seen": 638713856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516830491474424, + "loss": 1.3964, + "theoretical_loss": 3.843494905511072, + "tokens_seen": 638779392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516670010030091, + "loss": 1.3862, + "theoretical_loss": 3.8434545144741983, + "tokens_seen": 638844928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516509528585758, + "loss": 1.5054, + "theoretical_loss": 3.8434141287406827, + "tokens_seen": 638910464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 218641, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4308626651763916, + "objective/train/theoretical_loss": 3.8433737483092845, + "objective/train/tokens_used": 659436000, + "theoretical_loss": 3.8433737483092845, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516349047141425, + "loss": 1.4943, + "theoretical_loss": 3.8433737483092845, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516188565697092, + "loss": 1.4438, + "theoretical_loss": 3.8433333731787647, + "tokens_seen": 639041536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516028084252759, + "loss": 1.5739, + "theoretical_loss": 3.843293003347883, + "tokens_seen": 639107072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515867602808426, + "loss": 1.3414, + "theoretical_loss": 3.843252638815401, + "tokens_seen": 639172608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515707121364093, + "loss": 1.4464, + "theoretical_loss": 3.8432122795800803, + "tokens_seen": 639238144 + }, + { + "epoch": 0.19, + "learning_rate": 0.000651554663991976, + "loss": 1.51, + "theoretical_loss": 3.843171925640682, + "tokens_seen": 639303680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515386158475427, + "loss": 1.4377, + "theoretical_loss": 3.8431315769959697, + "tokens_seen": 639369216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515225677031094, + "loss": 1.4204, + "theoretical_loss": 3.8430912336447047, + "tokens_seen": 639434752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515065195586761, + "loss": 1.5045, + "theoretical_loss": 3.8430508955856504, + "tokens_seen": 639500288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514904714142427, + "loss": 1.4237, + "theoretical_loss": 3.843010562817571, + "tokens_seen": 639565824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514744232698095, + "loss": 1.4702, + "theoretical_loss": 3.84297023533923, + "tokens_seen": 639631360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514583751253761, + "loss": 1.4988, + "theoretical_loss": 3.842929913149392, + "tokens_seen": 639696896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514423269809429, + "loss": 1.4406, + "theoretical_loss": 3.8428895962468212, + "tokens_seen": 639762432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514262788365095, + "loss": 1.4651, + "theoretical_loss": 3.842849284630284, + "tokens_seen": 639827968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514102306920763, + "loss": 1.4929, + "theoretical_loss": 3.8428089782985446, + "tokens_seen": 639893504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513941825476429, + "loss": 1.4309, + "theoretical_loss": 3.84276867725037, + "tokens_seen": 639959040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513781344032097, + "loss": 1.4411, + "theoretical_loss": 3.842728381484526, + "tokens_seen": 640024576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513620862587763, + "loss": 1.3425, + "theoretical_loss": 3.8426880909997796, + "tokens_seen": 640090112 + }, + { + "epoch": 0.19, + "learning_rate": 0.000651346038114343, + "loss": 1.5662, + "theoretical_loss": 3.8426478057948983, + "tokens_seen": 640155648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513299899699097, + "loss": 1.5147, + "theoretical_loss": 3.8426075258686496, + "tokens_seen": 640221184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513139418254764, + "loss": 1.4709, + "theoretical_loss": 3.842567251219802, + "tokens_seen": 640286720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512978936810431, + "loss": 1.4913, + "theoretical_loss": 3.8425269818471235, + "tokens_seen": 640352256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512818455366098, + "loss": 1.4533, + "theoretical_loss": 3.8424867177493836, + "tokens_seen": 640417792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512657973921765, + "loss": 1.3171, + "theoretical_loss": 3.8424464589253504, + "tokens_seen": 640483328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512497492477432, + "loss": 1.42, + "theoretical_loss": 3.842406205373795, + "tokens_seen": 640548864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512337011033099, + "loss": 1.474, + "theoretical_loss": 3.8423659570934867, + "tokens_seen": 640614400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512176529588766, + "loss": 1.3579, + "theoretical_loss": 3.842325714083197, + "tokens_seen": 640679936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512016048144433, + "loss": 1.4301, + "theoretical_loss": 3.842285476341696, + "tokens_seen": 640745472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065118555667001, + "loss": 1.3874, + "theoretical_loss": 3.842245243867755, + "tokens_seen": 640811008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511695085255767, + "loss": 1.4085, + "theoretical_loss": 3.842205016660147, + "tokens_seen": 640876544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511534603811434, + "loss": 1.4344, + "theoretical_loss": 3.8421647947176423, + "tokens_seen": 640942080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511374122367101, + "loss": 1.438, + "theoretical_loss": 3.842124578039016, + "tokens_seen": 641007616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511213640922768, + "loss": 1.5147, + "theoretical_loss": 3.842084366623039, + "tokens_seen": 641073152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511053159478435, + "loss": 1.4703, + "theoretical_loss": 3.842044160468485, + "tokens_seen": 641138688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510892678034102, + "loss": 1.4338, + "theoretical_loss": 3.8420039595741295, + "tokens_seen": 641204224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510732196589769, + "loss": 1.3835, + "theoretical_loss": 3.841963763938746, + "tokens_seen": 641269760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510571715145436, + "loss": 1.4765, + "theoretical_loss": 3.8419235735611075, + "tokens_seen": 641335296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510411233701103, + "loss": 1.5916, + "theoretical_loss": 3.8418833884399914, + "tokens_seen": 641400832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510250752256771, + "loss": 1.4546, + "theoretical_loss": 3.841843208574172, + "tokens_seen": 641466368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510090270812437, + "loss": 1.5065, + "theoretical_loss": 3.8418030339624254, + "tokens_seen": 641531904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509929789368105, + "loss": 1.4235, + "theoretical_loss": 3.8417628646035284, + "tokens_seen": 641597440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509769307923771, + "loss": 1.413, + "theoretical_loss": 3.8417227004962573, + "tokens_seen": 641662976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509608826479439, + "loss": 1.4879, + "theoretical_loss": 3.8416825416393894, + "tokens_seen": 641728512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509448345035105, + "loss": 1.531, + "theoretical_loss": 3.841642388031702, + "tokens_seen": 641794048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509287863590773, + "loss": 1.4144, + "theoretical_loss": 3.8416022396719733, + "tokens_seen": 641859584 + }, + { + "epoch": 0.19, + "learning_rate": 0.000650912738214644, + "loss": 1.4167, + "theoretical_loss": 3.841562096558982, + "tokens_seen": 641925120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508966900702107, + "loss": 1.363, + "theoretical_loss": 3.841521958691506, + "tokens_seen": 641990656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508806419257773, + "loss": 1.3944, + "theoretical_loss": 3.841481826068325, + "tokens_seen": 642056192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508645937813441, + "loss": 1.3854, + "theoretical_loss": 3.841441698688219, + "tokens_seen": 642121728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508485456369107, + "loss": 1.439, + "theoretical_loss": 3.8414015765499676, + "tokens_seen": 642187264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 219942, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3430824279785156, + "objective/train/theoretical_loss": 3.841361459652351, + "objective/train/tokens_used": 662712800, + "theoretical_loss": 3.841361459652351, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508324974924775, + "loss": 1.3155, + "theoretical_loss": 3.841361459652351, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508164493480441, + "loss": 1.4824, + "theoretical_loss": 3.8413213479941506, + "tokens_seen": 642318336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508004012036109, + "loss": 1.3852, + "theoretical_loss": 3.841281241574147, + "tokens_seen": 642383872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507843530591776, + "loss": 1.5093, + "theoretical_loss": 3.841241140391123, + "tokens_seen": 642449408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507683049147443, + "loss": 1.4804, + "theoretical_loss": 3.8412010444438587, + "tokens_seen": 642514944 + }, + { + "epoch": 0.19, + "learning_rate": 0.000650752256770311, + "loss": 1.4636, + "theoretical_loss": 3.8411609537311384, + "tokens_seen": 642580480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507362086258777, + "loss": 1.3214, + "theoretical_loss": 3.841120868251744, + "tokens_seen": 642646016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507201604814444, + "loss": 1.4039, + "theoretical_loss": 3.8410807880044593, + "tokens_seen": 642711552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507041123370111, + "loss": 1.5107, + "theoretical_loss": 3.8410407129880673, + "tokens_seen": 642777088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506880641925778, + "loss": 1.4625, + "theoretical_loss": 3.841000643201353, + "tokens_seen": 642842624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506720160481445, + "loss": 1.4997, + "theoretical_loss": 3.8409605786431, + "tokens_seen": 642908160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506559679037112, + "loss": 1.2775, + "theoretical_loss": 3.840920519312094, + "tokens_seen": 642973696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506399197592779, + "loss": 1.4313, + "theoretical_loss": 3.8408804652071193, + "tokens_seen": 643039232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506238716148446, + "loss": 1.4374, + "theoretical_loss": 3.8408404163269627, + "tokens_seen": 643104768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506078234704113, + "loss": 1.419, + "theoretical_loss": 3.84080037267041, + "tokens_seen": 643170304 + }, + { + "epoch": 0.19, + "learning_rate": 0.000650591775325978, + "loss": 1.4437, + "theoretical_loss": 3.840760334236247, + "tokens_seen": 643235840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006505757271815447, + "loss": 1.4948, + "theoretical_loss": 3.8407203010232616, + "tokens_seen": 643301376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006505596790371114, + "loss": 1.4523, + "theoretical_loss": 3.840680273030241, + "tokens_seen": 643366912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006505436308926781, + "loss": 1.5176, + "theoretical_loss": 3.840640250255972, + "tokens_seen": 643432448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006505275827482448, + "loss": 1.4363, + "theoretical_loss": 3.840600232699244, + "tokens_seen": 643497984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006505115346038115, + "loss": 1.4636, + "theoretical_loss": 3.8405602203588445, + "tokens_seen": 643563520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504954864593782, + "loss": 1.4343, + "theoretical_loss": 3.8405202132335634, + "tokens_seen": 643629056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504794383149449, + "loss": 1.3896, + "theoretical_loss": 3.8404802113221894, + "tokens_seen": 643694592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504633901705116, + "loss": 1.3487, + "theoretical_loss": 3.8404402146235124, + "tokens_seen": 643760128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504473420260783, + "loss": 1.3639, + "theoretical_loss": 3.8404002231363226, + "tokens_seen": 643825664 + }, + { + "epoch": 0.2, + "learning_rate": 0.000650431293881645, + "loss": 1.398, + "theoretical_loss": 3.8403602368594107, + "tokens_seen": 643891200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504152457372117, + "loss": 1.4373, + "theoretical_loss": 3.8403202557915677, + "tokens_seen": 643956736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503991975927784, + "loss": 1.3552, + "theoretical_loss": 3.8402802799315845, + "tokens_seen": 644022272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503831494483451, + "loss": 1.3018, + "theoretical_loss": 3.8402403092782533, + "tokens_seen": 644087808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503671013039118, + "loss": 1.4913, + "theoretical_loss": 3.8402003438303662, + "tokens_seen": 644153344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503510531594785, + "loss": 1.5308, + "theoretical_loss": 3.840160383586716, + "tokens_seen": 644218880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503350050150452, + "loss": 1.5119, + "theoretical_loss": 3.8401204285460957, + "tokens_seen": 644284416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503189568706119, + "loss": 1.3809, + "theoretical_loss": 3.840080478707298, + "tokens_seen": 644349952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503029087261786, + "loss": 1.4947, + "theoretical_loss": 3.840040534069118, + "tokens_seen": 644415488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502868605817453, + "loss": 1.4467, + "theoretical_loss": 3.8400005946303484, + "tokens_seen": 644481024 + }, + { + "epoch": 0.2, + "learning_rate": 0.000650270812437312, + "loss": 1.4591, + "theoretical_loss": 3.8399606603897847, + "tokens_seen": 644546560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502547642928787, + "loss": 1.4746, + "theoretical_loss": 3.8399207313462216, + "tokens_seen": 644612096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502387161484454, + "loss": 1.4329, + "theoretical_loss": 3.8398808074984547, + "tokens_seen": 644677632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502226680040121, + "loss": 1.3211, + "theoretical_loss": 3.8398408888452797, + "tokens_seen": 644743168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502066198595788, + "loss": 1.4165, + "theoretical_loss": 3.8398009753854927, + "tokens_seen": 644808704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501905717151455, + "loss": 1.3759, + "theoretical_loss": 3.839761067117891, + "tokens_seen": 644874240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501745235707122, + "loss": 1.4048, + "theoretical_loss": 3.8397211640412703, + "tokens_seen": 644939776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501584754262789, + "loss": 1.4058, + "theoretical_loss": 3.839681266154429, + "tokens_seen": 645005312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501424272818456, + "loss": 1.3807, + "theoretical_loss": 3.839641373456165, + "tokens_seen": 645070848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501263791374123, + "loss": 1.535, + "theoretical_loss": 3.8396014859452756, + "tokens_seen": 645136384 + }, + { + "epoch": 0.2, + "learning_rate": 0.000650110330992979, + "loss": 1.3981, + "theoretical_loss": 3.8395616036205604, + "tokens_seen": 645201920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500942828485456, + "loss": 1.52, + "theoretical_loss": 3.839521726480818, + "tokens_seen": 645267456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500782347041124, + "loss": 1.5095, + "theoretical_loss": 3.8394818545248475, + "tokens_seen": 645332992 + }, + { + "epoch": 0.2, + "learning_rate": 0.000650062186559679, + "loss": 1.4019, + "theoretical_loss": 3.8394419877514494, + "tokens_seen": 645398528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500461384152458, + "loss": 1.4695, + "theoretical_loss": 3.8394021261594236, + "tokens_seen": 645464064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 220851, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.384150505065918, + "objective/train/theoretical_loss": 3.8393622697475704, + "objective/train/tokens_used": 665989600, + "theoretical_loss": 3.8393622697475704, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500300902708124, + "loss": 1.34, + "theoretical_loss": 3.8393622697475704, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500140421263792, + "loss": 1.4082, + "theoretical_loss": 3.839322418514691, + "tokens_seen": 645595136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499979939819458, + "loss": 1.4156, + "theoretical_loss": 3.839282572459587, + "tokens_seen": 645660672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499819458375126, + "loss": 1.4836, + "theoretical_loss": 3.83924273158106, + "tokens_seen": 645726208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499658976930792, + "loss": 1.5027, + "theoretical_loss": 3.8392028958779125, + "tokens_seen": 645791744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499498495486459, + "loss": 1.4867, + "theoretical_loss": 3.8391630653489472, + "tokens_seen": 645857280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499338014042126, + "loss": 1.4544, + "theoretical_loss": 3.8391232399929667, + "tokens_seen": 645922816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499177532597793, + "loss": 1.4677, + "theoretical_loss": 3.839083419808774, + "tokens_seen": 645988352 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649901705115346, + "loss": 1.4005, + "theoretical_loss": 3.839043604795174, + "tokens_seen": 646053888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498856569709127, + "loss": 1.3997, + "theoretical_loss": 3.83900379495097, + "tokens_seen": 646119424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498696088264794, + "loss": 1.3995, + "theoretical_loss": 3.8389639902749675, + "tokens_seen": 646184960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498535606820461, + "loss": 1.4303, + "theoretical_loss": 3.83892419076597, + "tokens_seen": 646250496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498375125376128, + "loss": 1.4978, + "theoretical_loss": 3.8388843964227846, + "tokens_seen": 646316032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498214643931796, + "loss": 1.473, + "theoretical_loss": 3.8388446072442157, + "tokens_seen": 646381568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498054162487462, + "loss": 1.5414, + "theoretical_loss": 3.8388048232290704, + "tokens_seen": 646447104 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649789368104313, + "loss": 1.4878, + "theoretical_loss": 3.8387650443761556, + "tokens_seen": 646512640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006497733199598796, + "loss": 1.434, + "theoretical_loss": 3.838725270684277, + "tokens_seen": 646578176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006497572718154464, + "loss": 1.5353, + "theoretical_loss": 3.8386855021522424, + "tokens_seen": 646643712 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649741223671013, + "loss": 1.3599, + "theoretical_loss": 3.83864573877886, + "tokens_seen": 646709248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006497251755265798, + "loss": 1.4096, + "theoretical_loss": 3.8386059805629382, + "tokens_seen": 646774784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006497091273821464, + "loss": 1.4407, + "theoretical_loss": 3.838566227503285, + "tokens_seen": 646840320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496930792377132, + "loss": 1.4622, + "theoretical_loss": 3.838526479598709, + "tokens_seen": 646905856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496770310932798, + "loss": 1.3577, + "theoretical_loss": 3.838486736848021, + "tokens_seen": 646971392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496609829488466, + "loss": 1.4514, + "theoretical_loss": 3.8384469992500296, + "tokens_seen": 647036928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496449348044132, + "loss": 1.4493, + "theoretical_loss": 3.8384072668035447, + "tokens_seen": 647102464 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064962888665998, + "loss": 1.5102, + "theoretical_loss": 3.8383675395073777, + "tokens_seen": 647168000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496128385155466, + "loss": 1.3916, + "theoretical_loss": 3.838327817360339, + "tokens_seen": 647233536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495967903711134, + "loss": 1.3993, + "theoretical_loss": 3.8382881003612406, + "tokens_seen": 647299072 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064958074222668, + "loss": 1.4586, + "theoretical_loss": 3.8382483885088927, + "tokens_seen": 647364608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495646940822468, + "loss": 1.5188, + "theoretical_loss": 3.8382086818021097, + "tokens_seen": 647430144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495486459378134, + "loss": 1.485, + "theoretical_loss": 3.838168980239702, + "tokens_seen": 647495680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495325977933802, + "loss": 1.5177, + "theoretical_loss": 3.838129283820484, + "tokens_seen": 647561216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495165496489468, + "loss": 1.4066, + "theoretical_loss": 3.838089592543268, + "tokens_seen": 647626752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495005015045136, + "loss": 1.3561, + "theoretical_loss": 3.8380499064068685, + "tokens_seen": 647692288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006494844533600802, + "loss": 1.4377, + "theoretical_loss": 3.8380102254100983, + "tokens_seen": 647757824 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649468405215647, + "loss": 1.3284, + "theoretical_loss": 3.8379705495517733, + "tokens_seen": 647823360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006494523570712136, + "loss": 1.4504, + "theoretical_loss": 3.8379308788307083, + "tokens_seen": 647888896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006494363089267804, + "loss": 1.3618, + "theoretical_loss": 3.837891213245718, + "tokens_seen": 647954432 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649420260782347, + "loss": 1.4628, + "theoretical_loss": 3.837851552795618, + "tokens_seen": 648019968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006494042126379138, + "loss": 1.5271, + "theoretical_loss": 3.8378118974792246, + "tokens_seen": 648085504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493881644934805, + "loss": 1.4865, + "theoretical_loss": 3.837772247295354, + "tokens_seen": 648151040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493721163490472, + "loss": 1.5861, + "theoretical_loss": 3.8377326022428235, + "tokens_seen": 648216576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493560682046139, + "loss": 1.4137, + "theoretical_loss": 3.8376929623204505, + "tokens_seen": 648282112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493400200601806, + "loss": 1.4333, + "theoretical_loss": 3.8376533275270517, + "tokens_seen": 648347648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493239719157473, + "loss": 1.453, + "theoretical_loss": 3.837613697861446, + "tokens_seen": 648413184 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649307923771314, + "loss": 1.5266, + "theoretical_loss": 3.8375740733224513, + "tokens_seen": 648478720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492918756268807, + "loss": 1.4287, + "theoretical_loss": 3.8375344539088867, + "tokens_seen": 648544256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492758274824474, + "loss": 1.495, + "theoretical_loss": 3.837494839619571, + "tokens_seen": 648609792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492597793380141, + "loss": 1.3091, + "theoretical_loss": 3.8374552304533243, + "tokens_seen": 648675328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492437311935808, + "loss": 1.4309, + "theoretical_loss": 3.8374156264089665, + "tokens_seen": 648740864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 222090, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.4386017322540283, + "objective/train/theoretical_loss": 3.8373760274853175, + "objective/train/tokens_used": 669266400, + "theoretical_loss": 3.8373760274853175, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492276830491475, + "loss": 1.4186, + "theoretical_loss": 3.8373760274853175, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492116349047142, + "loss": 1.5391, + "theoretical_loss": 3.8373364336811986, + "tokens_seen": 648871936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491955867602809, + "loss": 1.4377, + "theoretical_loss": 3.8372968449954308, + "tokens_seen": 648937472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491795386158476, + "loss": 1.3458, + "theoretical_loss": 3.8372572614268354, + "tokens_seen": 649003008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491634904714143, + "loss": 1.4029, + "theoretical_loss": 3.8372176829742344, + "tokens_seen": 649068544 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649147442326981, + "loss": 1.4743, + "theoretical_loss": 3.837178109636451, + "tokens_seen": 649134080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491313941825477, + "loss": 1.5013, + "theoretical_loss": 3.837138541412306, + "tokens_seen": 649199616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491153460381144, + "loss": 1.3991, + "theoretical_loss": 3.837098978300624, + "tokens_seen": 649265152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490992978936811, + "loss": 1.533, + "theoretical_loss": 3.8370594203002284, + "tokens_seen": 649330688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490832497492478, + "loss": 1.4891, + "theoretical_loss": 3.8370198674099427, + "tokens_seen": 649396224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490672016048145, + "loss": 1.4534, + "theoretical_loss": 3.8369803196285917, + "tokens_seen": 649461760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490511534603812, + "loss": 1.469, + "theoretical_loss": 3.8369407769549992, + "tokens_seen": 649527296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490351053159479, + "loss": 1.3051, + "theoretical_loss": 3.8369012393879904, + "tokens_seen": 649592832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490190571715146, + "loss": 1.443, + "theoretical_loss": 3.836861706926392, + "tokens_seen": 649658368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490030090270813, + "loss": 1.463, + "theoretical_loss": 3.836822179569028, + "tokens_seen": 649723904 + }, + { + "epoch": 0.2, + "learning_rate": 0.000648986960882648, + "loss": 1.3889, + "theoretical_loss": 3.8367826573147257, + "tokens_seen": 649789440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489709127382147, + "loss": 1.5045, + "theoretical_loss": 3.836743140162312, + "tokens_seen": 649854976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489548645937814, + "loss": 1.3668, + "theoretical_loss": 3.8367036281106124, + "tokens_seen": 649920512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489388164493481, + "loss": 1.4691, + "theoretical_loss": 3.836664121158456, + "tokens_seen": 649986048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489227683049148, + "loss": 1.4015, + "theoretical_loss": 3.83662461930467, + "tokens_seen": 650051584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489067201604815, + "loss": 1.473, + "theoretical_loss": 3.836585122548082, + "tokens_seen": 650117120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488906720160482, + "loss": 1.3815, + "theoretical_loss": 3.836545630887521, + "tokens_seen": 650182656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488746238716149, + "loss": 1.4541, + "theoretical_loss": 3.836506144321816, + "tokens_seen": 650248192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488585757271816, + "loss": 1.3895, + "theoretical_loss": 3.836466662849795, + "tokens_seen": 650313728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488425275827483, + "loss": 1.4039, + "theoretical_loss": 3.8364271864702904, + "tokens_seen": 650379264 + }, + { + "epoch": 0.2, + "learning_rate": 0.000648826479438315, + "loss": 1.4246, + "theoretical_loss": 3.8363877151821297, + "tokens_seen": 650444800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488104312938817, + "loss": 1.4037, + "theoretical_loss": 3.8363482489841454, + "tokens_seen": 650510336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487943831494484, + "loss": 1.4236, + "theoretical_loss": 3.8363087878751667, + "tokens_seen": 650575872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487783350050151, + "loss": 1.4738, + "theoretical_loss": 3.8362693318540257, + "tokens_seen": 650641408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487622868605818, + "loss": 1.4377, + "theoretical_loss": 3.8362298809195545, + "tokens_seen": 650706944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487462387161485, + "loss": 1.4915, + "theoretical_loss": 3.8361904350705833, + "tokens_seen": 650772480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487301905717152, + "loss": 1.3687, + "theoretical_loss": 3.836150994305947, + "tokens_seen": 650838016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487141424272819, + "loss": 1.4345, + "theoretical_loss": 3.8361115586244763, + "tokens_seen": 650903552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486980942828485, + "loss": 1.4897, + "theoretical_loss": 3.8360721280250054, + "tokens_seen": 650969088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486820461384153, + "loss": 1.3686, + "theoretical_loss": 3.836032702506367, + "tokens_seen": 651034624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486659979939819, + "loss": 1.4817, + "theoretical_loss": 3.8359932820673968, + "tokens_seen": 651100160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486499498495487, + "loss": 1.5604, + "theoretical_loss": 3.8359538667069275, + "tokens_seen": 651165696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486339017051153, + "loss": 1.4949, + "theoretical_loss": 3.8359144564237946, + "tokens_seen": 651231232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486178535606821, + "loss": 1.4686, + "theoretical_loss": 3.8358750512168323, + "tokens_seen": 651296768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486018054162487, + "loss": 1.4227, + "theoretical_loss": 3.8358356510848775, + "tokens_seen": 651362304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485857572718155, + "loss": 1.4537, + "theoretical_loss": 3.8357962560267644, + "tokens_seen": 651427840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485697091273821, + "loss": 1.4057, + "theoretical_loss": 3.835756866041331, + "tokens_seen": 651493376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485536609829489, + "loss": 1.3665, + "theoretical_loss": 3.8357174811274133, + "tokens_seen": 651558912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485376128385155, + "loss": 1.4087, + "theoretical_loss": 3.8356781012838472, + "tokens_seen": 651624448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485215646940823, + "loss": 1.4461, + "theoretical_loss": 3.8356387265094716, + "tokens_seen": 651689984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485055165496489, + "loss": 1.4452, + "theoretical_loss": 3.835599356803124, + "tokens_seen": 651755520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484894684052157, + "loss": 1.3589, + "theoretical_loss": 3.835559992163642, + "tokens_seen": 651821056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484734202607823, + "loss": 1.4137, + "theoretical_loss": 3.835520632589864, + "tokens_seen": 651886592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484573721163491, + "loss": 1.4278, + "theoretical_loss": 3.83548127808063, + "tokens_seen": 651952128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484413239719157, + "loss": 1.5026, + "theoretical_loss": 3.8354419286347783, + "tokens_seen": 652017664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 222990, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3786110877990723, + "objective/train/theoretical_loss": 3.8354025842511494, + "objective/train/tokens_used": 672543200, + "theoretical_loss": 3.8354025842511494, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484252758274825, + "loss": 1.5055, + "theoretical_loss": 3.8354025842511494, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484092276830491, + "loss": 1.4173, + "theoretical_loss": 3.8353632449285833, + "tokens_seen": 652148736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483931795386159, + "loss": 1.3441, + "theoretical_loss": 3.8353239106659194, + "tokens_seen": 652214272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483771313941825, + "loss": 1.5339, + "theoretical_loss": 3.835284581462, + "tokens_seen": 652279808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483610832497493, + "loss": 1.5046, + "theoretical_loss": 3.8352452573156652, + "tokens_seen": 652345344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483450351053159, + "loss": 1.5097, + "theoretical_loss": 3.835205938225757, + "tokens_seen": 652410880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483289869608827, + "loss": 1.4368, + "theoretical_loss": 3.8351666241911175, + "tokens_seen": 652476416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483129388164493, + "loss": 1.4001, + "theoretical_loss": 3.8351273152105896, + "tokens_seen": 652541952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482968906720161, + "loss": 1.4079, + "theoretical_loss": 3.8350880112830152, + "tokens_seen": 652607488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482808425275827, + "loss": 1.4434, + "theoretical_loss": 3.8350487124072377, + "tokens_seen": 652673024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482647943831495, + "loss": 1.5073, + "theoretical_loss": 3.835009418582101, + "tokens_seen": 652738560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482487462387161, + "loss": 1.3897, + "theoretical_loss": 3.8349701298064485, + "tokens_seen": 652804096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482326980942829, + "loss": 1.4405, + "theoretical_loss": 3.834930846079125, + "tokens_seen": 652869632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482166499498495, + "loss": 1.4859, + "theoretical_loss": 3.8348915673989743, + "tokens_seen": 652935168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482006018054163, + "loss": 1.4749, + "theoretical_loss": 3.8348522937648424, + "tokens_seen": 653000704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481845536609829, + "loss": 1.4313, + "theoretical_loss": 3.834813025175574, + "tokens_seen": 653066240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481685055165497, + "loss": 1.4317, + "theoretical_loss": 3.8347737616300153, + "tokens_seen": 653131776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481524573721163, + "loss": 1.4701, + "theoretical_loss": 3.8347345031270126, + "tokens_seen": 653197312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481364092276831, + "loss": 1.4103, + "theoretical_loss": 3.834695249665412, + "tokens_seen": 653262848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481203610832497, + "loss": 1.5055, + "theoretical_loss": 3.8346560012440607, + "tokens_seen": 653328384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481043129388165, + "loss": 1.3859, + "theoretical_loss": 3.8346167578618062, + "tokens_seen": 653393920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480882647943831, + "loss": 1.4491, + "theoretical_loss": 3.834577519517496, + "tokens_seen": 653459456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480722166499499, + "loss": 1.4329, + "theoretical_loss": 3.8345382862099786, + "tokens_seen": 653524992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480561685055166, + "loss": 1.4701, + "theoretical_loss": 3.8344990579381015, + "tokens_seen": 653590528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480401203610833, + "loss": 1.5133, + "theoretical_loss": 3.8344598347007146, + "tokens_seen": 653656064 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064802407221665, + "loss": 1.4859, + "theoretical_loss": 3.834420616496666, + "tokens_seen": 653721600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480080240722167, + "loss": 1.4683, + "theoretical_loss": 3.8343814033248065, + "tokens_seen": 653787136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479919759277834, + "loss": 1.4837, + "theoretical_loss": 3.8343421951839853, + "tokens_seen": 653852672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479759277833501, + "loss": 1.46, + "theoretical_loss": 3.834302992073053, + "tokens_seen": 653918208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479598796389168, + "loss": 1.3495, + "theoretical_loss": 3.8342637939908606, + "tokens_seen": 653983744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479438314944835, + "loss": 1.4656, + "theoretical_loss": 3.8342246009362584, + "tokens_seen": 654049280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479277833500502, + "loss": 1.4019, + "theoretical_loss": 3.8341854129080986, + "tokens_seen": 654114816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479117352056169, + "loss": 1.3941, + "theoretical_loss": 3.8341462299052327, + "tokens_seen": 654180352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478956870611836, + "loss": 1.4368, + "theoretical_loss": 3.8341070519265132, + "tokens_seen": 654245888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478796389167503, + "loss": 1.434, + "theoretical_loss": 3.8340678789707927, + "tokens_seen": 654311424 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647863590772317, + "loss": 1.2979, + "theoretical_loss": 3.834028711036924, + "tokens_seen": 654376960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478475426278837, + "loss": 1.4776, + "theoretical_loss": 3.8339895481237605, + "tokens_seen": 654442496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478314944834504, + "loss": 1.4851, + "theoretical_loss": 3.8339503902301564, + "tokens_seen": 654508032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478154463390171, + "loss": 1.4354, + "theoretical_loss": 3.833911237354965, + "tokens_seen": 654573568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477993981945838, + "loss": 1.5246, + "theoretical_loss": 3.8338720894970413, + "tokens_seen": 654639104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477833500501505, + "loss": 1.4338, + "theoretical_loss": 3.8338329466552405, + "tokens_seen": 654704640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477673019057172, + "loss": 1.458, + "theoretical_loss": 3.833793808828417, + "tokens_seen": 654770176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477512537612839, + "loss": 1.3989, + "theoretical_loss": 3.8337546760154275, + "tokens_seen": 654835712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477352056168506, + "loss": 1.396, + "theoretical_loss": 3.8337155482151264, + "tokens_seen": 654901248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477191574724173, + "loss": 1.4821, + "theoretical_loss": 3.833676425426372, + "tokens_seen": 654966784 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647703109327984, + "loss": 1.3428, + "theoretical_loss": 3.8336373076480204, + "tokens_seen": 655032320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476870611835507, + "loss": 1.2759, + "theoretical_loss": 3.8335981948789275, + "tokens_seen": 655097856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476710130391174, + "loss": 1.3351, + "theoretical_loss": 3.833559087117952, + "tokens_seen": 655163392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476549648946841, + "loss": 1.4337, + "theoretical_loss": 3.833519984363952, + "tokens_seen": 655228928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476389167502508, + "loss": 1.521, + "theoretical_loss": 3.833480886615785, + "tokens_seen": 655294464 + }, + { + "debugging/Compilability": 0.0, + "debugging/distinct-1-grams": 0.6755372198767522, + "debugging/entropy-1-grams": 4.8915199782364125, + "debugging/length": 606.5, + "debugging/num_segments": 6, + "debugging/score": 0.054560030918185586, + "debugging/score_std": 0.025575581829586438, + "epoch": 0.2, + "objective/train/docs_used": 224121, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/original_loss": 1.3385496139526367, + "objective/train/theoretical_loss": 3.83344179387231, + "objective/train/tokens_used": 675820000, + "theoretical_loss": 3.83344179387231, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476228686058175, + "loss": 1.35, + "theoretical_loss": 3.83344179387231, + "tokens_seen": 655360000 + } + ], + "max_steps": 50354, + "num_train_epochs": 9223372036854775807, + "total_flos": 3.3445380096e+17, + "trial_name": null, + "trial_params": null +}